blob: 00f1ab5a8aeb7801a33ae409a5042f79a55321bf [file] [log] [blame]
Eugeni Dodonov85208be2012-04-16 22:20:34 -03001/*
2 * Copyright © 2012 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eugeni Dodonov <eugeni.dodonov@intel.com>
25 *
26 */
27
Sam Ravnborgd0e93592019-01-26 13:25:24 +010028#include <linux/module.h>
Chris Wilson08ea70a2018-08-12 23:36:31 +010029#include <linux/pm_runtime.h>
Sam Ravnborgd0e93592019-01-26 13:25:24 +010030
31#include <drm/drm_atomic_helper.h>
32#include <drm/drm_fourcc.h>
Kumar, Mahesh9c2f7a92016-05-16 15:52:00 -070033#include <drm/drm_plane_helper.h>
Sam Ravnborgd0e93592019-01-26 13:25:24 +010034
Jani Nikuladf0566a2019-06-13 11:44:16 +030035#include "display/intel_atomic.h"
Ville Syrjälä3df3fe22020-11-06 19:30:42 +020036#include "display/intel_atomic_plane.h"
Stanislav Lisovskiycac91e62020-05-22 16:18:43 +030037#include "display/intel_bw.h"
Jani Nikula1d455f82019-08-06 14:39:33 +030038#include "display/intel_display_types.h"
Jani Nikuladf0566a2019-06-13 11:44:16 +030039#include "display/intel_fbc.h"
40#include "display/intel_sprite.h"
Dave Airlie46d12f92021-02-05 16:48:36 +020041#include "display/skl_universal_plane.h"
Jani Nikuladf0566a2019-06-13 11:44:16 +030042
Andi Shyti0dc3c562019-10-20 19:41:39 +010043#include "gt/intel_llc.h"
44
Eugeni Dodonov85208be2012-04-16 22:20:34 -030045#include "i915_drv.h"
Jani Nikulaa10510a2020-02-27 19:00:47 +020046#include "i915_fixed.h"
Jani Nikula440e2b32019-04-29 15:29:27 +030047#include "i915_irq.h"
Jani Nikulaa09d9a82019-08-06 13:07:28 +030048#include "i915_trace.h"
Jani Nikula696173b2019-04-05 14:00:15 +030049#include "intel_pm.h"
Chris Wilson56c50982019-04-26 09:17:22 +010050#include "intel_sideband.h"
Daniel Vettereb48eb02012-04-26 23:28:12 +020051#include "../../../platform/x86/intel_ips.h"
Eugeni Dodonov85208be2012-04-16 22:20:34 -030052
Jani Nikulaa10510a2020-02-27 19:00:47 +020053/* Stores plane specific WM parameters */
54struct skl_wm_params {
55 bool x_tiled, y_tiled;
56 bool rc_surface;
57 bool is_planar;
58 u32 width;
59 u8 cpp;
60 u32 plane_pixel_rate;
61 u32 y_min_scanlines;
62 u32 plane_bytes_per_line;
63 uint_fixed_16_16_t plane_blocks_per_line;
64 uint_fixed_16_16_t y_tile_minimum;
65 u32 linetime_us;
66 u32 dbuf_block_size;
67};
68
69/* used in computing the new watermarks state */
70struct intel_wm_config {
71 unsigned int num_pipes_active;
72 bool sprites_enabled;
73 bool sprites_scaled;
74};
75
Ville Syrjälä46f16e62016-10-31 22:37:22 +020076static void gen9_init_clock_gating(struct drm_i915_private *dev_priv)
Mika Kuoppalab033bb62016-06-07 17:19:04 +030077{
Ville Syrjälä93564042017-08-24 22:10:51 +030078 if (HAS_LLC(dev_priv)) {
79 /*
80 * WaCompressedResourceDisplayNewHashMode:skl,kbl
Lucas De Marchie0403cb2017-12-05 11:01:17 -080081 * Display WA #0390: skl,kbl
Ville Syrjälä93564042017-08-24 22:10:51 +030082 *
83 * Must match Sampler, Pixel Back End, and Media. See
84 * WaCompressedResourceSamplerPbeMediaNewHashMode.
85 */
Jani Nikula5f461662020-11-30 13:15:58 +020086 intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR1_1,
87 intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR1_1) |
Ville Syrjälä93564042017-08-24 22:10:51 +030088 SKL_DE_COMPRESSED_HASH_MODE);
89 }
90
Rodrigo Vivi82525c12017-06-08 08:50:00 -070091 /* See Bspec note for PSR2_CTL bit 31, Wa#828:skl,bxt,kbl,cfl */
Jani Nikula5f461662020-11-30 13:15:58 +020092 intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR1_1,
93 intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR1_1) | SKL_EDP_PSR_FIX_RDWRAP);
Mika Kuoppalab033bb62016-06-07 17:19:04 +030094
Rodrigo Vivi82525c12017-06-08 08:50:00 -070095 /* WaEnableChickenDCPR:skl,bxt,kbl,glk,cfl */
Jani Nikula5f461662020-11-30 13:15:58 +020096 intel_uncore_write(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1,
97 intel_uncore_read(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM);
Mika Kuoppala0f78dee2016-06-07 17:19:16 +030098
Ville Syrjälä99bcf64e2020-07-08 16:12:23 +030099 /*
100 * WaFbcWakeMemOn:skl,bxt,kbl,glk,cfl
101 * Display WA #0859: skl,bxt,kbl,glk,cfl
102 */
Jani Nikula5f461662020-11-30 13:15:58 +0200103 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
Mika Kuoppala303d4ea2016-06-07 17:19:17 +0300104 DISP_FBC_MEMORY_WAKE);
Mika Kuoppalab033bb62016-06-07 17:19:04 +0300105}
106
Ville Syrjälä46f16e62016-10-31 22:37:22 +0200107static void bxt_init_clock_gating(struct drm_i915_private *dev_priv)
Imre Deaka82abe42015-03-27 14:00:04 +0200108{
Ville Syrjälä46f16e62016-10-31 22:37:22 +0200109 gen9_init_clock_gating(dev_priv);
Daniel Vetterdc00b6a2016-05-19 09:14:20 +0200110
Nick Hoatha7546152015-06-29 14:07:32 +0100111 /* WaDisableSDEUnitClockGating:bxt */
Jani Nikula5f461662020-11-30 13:15:58 +0200112 intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
Nick Hoatha7546152015-06-29 14:07:32 +0100113 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
114
Imre Deak32608ca2015-03-11 11:10:27 +0200115 /*
116 * FIXME:
Ben Widawsky868434c2015-03-11 10:49:32 +0200117 * GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ applies on 3x6 GT SKUs only.
Imre Deak32608ca2015-03-11 11:10:27 +0200118 */
Jani Nikula5f461662020-11-30 13:15:58 +0200119 intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
Ben Widawsky868434c2015-03-11 10:49:32 +0200120 GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
Imre Deakd965e7ac2015-12-01 10:23:52 +0200121
122 /*
123 * Wa: Backlight PWM may stop in the asserted state, causing backlight
124 * to stay fully on.
125 */
Jani Nikula5f461662020-11-30 13:15:58 +0200126 intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_0, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_0) |
Jani Nikula8aeaf642017-02-15 17:21:37 +0200127 PWM1_GATING_DIS | PWM2_GATING_DIS);
Uma Shankar1d85a292018-08-07 21:15:35 +0530128
129 /*
130 * Lower the display internal timeout.
131 * This is needed to avoid any hard hangs when DSI port PLL
132 * is off and a MMIO access is attempted by any privilege
133 * application, using batch buffers or any other means.
134 */
Jani Nikula5f461662020-11-30 13:15:58 +0200135 intel_uncore_write(&dev_priv->uncore, RM_TIMEOUT, MMIO_TIMEOUT_US(950));
Ville Syrjäläc4615b22020-07-08 16:12:21 +0300136
Ville Syrjälä99bcf64e2020-07-08 16:12:23 +0300137 /*
138 * WaFbcTurnOffFbcWatermark:bxt
139 * Display WA #0562: bxt
140 */
Jani Nikula5f461662020-11-30 13:15:58 +0200141 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
Ville Syrjäläc4615b22020-07-08 16:12:21 +0300142 DISP_FBC_WM_DIS);
Ville Syrjäläcd7a8812020-07-08 16:12:22 +0300143
Ville Syrjälä99bcf64e2020-07-08 16:12:23 +0300144 /*
145 * WaFbcHighMemBwCorruptionAvoidance:bxt
146 * Display WA #0883: bxt
147 */
Jani Nikula5f461662020-11-30 13:15:58 +0200148 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN, intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN) |
Ville Syrjäläcd7a8812020-07-08 16:12:22 +0300149 ILK_DPFC_DISABLE_DUMMY0);
Imre Deaka82abe42015-03-27 14:00:04 +0200150}
151
Ander Conselvan de Oliveira9fb50262017-01-26 11:16:58 +0200152static void glk_init_clock_gating(struct drm_i915_private *dev_priv)
153{
154 gen9_init_clock_gating(dev_priv);
155
156 /*
157 * WaDisablePWMClockGating:glk
158 * Backlight PWM may stop in the asserted state, causing backlight
159 * to stay fully on.
160 */
Jani Nikula5f461662020-11-30 13:15:58 +0200161 intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_0, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_0) |
Ander Conselvan de Oliveira9fb50262017-01-26 11:16:58 +0200162 PWM1_GATING_DIS | PWM2_GATING_DIS);
163}
164
Lucas De Marchi1d218222019-12-24 00:40:04 -0800165static void pnv_get_mem_freq(struct drm_i915_private *dev_priv)
Daniel Vetterc921aba2012-04-26 23:28:17 +0200166{
Daniel Vetterc921aba2012-04-26 23:28:17 +0200167 u32 tmp;
168
Jani Nikula5f461662020-11-30 13:15:58 +0200169 tmp = intel_uncore_read(&dev_priv->uncore, CLKCFG);
Daniel Vetterc921aba2012-04-26 23:28:17 +0200170
171 switch (tmp & CLKCFG_FSB_MASK) {
172 case CLKCFG_FSB_533:
173 dev_priv->fsb_freq = 533; /* 133*4 */
174 break;
175 case CLKCFG_FSB_800:
176 dev_priv->fsb_freq = 800; /* 200*4 */
177 break;
178 case CLKCFG_FSB_667:
179 dev_priv->fsb_freq = 667; /* 167*4 */
180 break;
181 case CLKCFG_FSB_400:
182 dev_priv->fsb_freq = 400; /* 100*4 */
183 break;
184 }
185
186 switch (tmp & CLKCFG_MEM_MASK) {
187 case CLKCFG_MEM_533:
188 dev_priv->mem_freq = 533;
189 break;
190 case CLKCFG_MEM_667:
191 dev_priv->mem_freq = 667;
192 break;
193 case CLKCFG_MEM_800:
194 dev_priv->mem_freq = 800;
195 break;
196 }
197
198 /* detect pineview DDR3 setting */
Jani Nikula5f461662020-11-30 13:15:58 +0200199 tmp = intel_uncore_read(&dev_priv->uncore, CSHRDDR3CTL);
Daniel Vetterc921aba2012-04-26 23:28:17 +0200200 dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
201}
202
Lucas De Marchi9eae5e22019-12-24 00:40:09 -0800203static void ilk_get_mem_freq(struct drm_i915_private *dev_priv)
Daniel Vetterc921aba2012-04-26 23:28:17 +0200204{
Daniel Vetterc921aba2012-04-26 23:28:17 +0200205 u16 ddrpll, csipll;
206
Tvrtko Ursulin4f5fd912019-06-11 11:45:48 +0100207 ddrpll = intel_uncore_read16(&dev_priv->uncore, DDRMPLL1);
208 csipll = intel_uncore_read16(&dev_priv->uncore, CSIPLL0);
Daniel Vetterc921aba2012-04-26 23:28:17 +0200209
210 switch (ddrpll & 0xff) {
211 case 0xc:
212 dev_priv->mem_freq = 800;
213 break;
214 case 0x10:
215 dev_priv->mem_freq = 1066;
216 break;
217 case 0x14:
218 dev_priv->mem_freq = 1333;
219 break;
220 case 0x18:
221 dev_priv->mem_freq = 1600;
222 break;
223 default:
Wambui Karugaf8d18d52020-01-07 18:13:30 +0300224 drm_dbg(&dev_priv->drm, "unknown memory frequency 0x%02x\n",
225 ddrpll & 0xff);
Daniel Vetterc921aba2012-04-26 23:28:17 +0200226 dev_priv->mem_freq = 0;
227 break;
228 }
229
Daniel Vetterc921aba2012-04-26 23:28:17 +0200230 switch (csipll & 0x3ff) {
231 case 0x00c:
232 dev_priv->fsb_freq = 3200;
233 break;
234 case 0x00e:
235 dev_priv->fsb_freq = 3733;
236 break;
237 case 0x010:
238 dev_priv->fsb_freq = 4266;
239 break;
240 case 0x012:
241 dev_priv->fsb_freq = 4800;
242 break;
243 case 0x014:
244 dev_priv->fsb_freq = 5333;
245 break;
246 case 0x016:
247 dev_priv->fsb_freq = 5866;
248 break;
249 case 0x018:
250 dev_priv->fsb_freq = 6400;
251 break;
252 default:
Wambui Karugaf8d18d52020-01-07 18:13:30 +0300253 drm_dbg(&dev_priv->drm, "unknown fsb frequency 0x%04x\n",
254 csipll & 0x3ff);
Daniel Vetterc921aba2012-04-26 23:28:17 +0200255 dev_priv->fsb_freq = 0;
256 break;
257 }
Daniel Vetterc921aba2012-04-26 23:28:17 +0200258}
259
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300260static const struct cxsr_latency cxsr_latency_table[] = {
261 {1, 0, 800, 400, 3382, 33382, 3983, 33983}, /* DDR2-400 SC */
262 {1, 0, 800, 667, 3354, 33354, 3807, 33807}, /* DDR2-667 SC */
263 {1, 0, 800, 800, 3347, 33347, 3763, 33763}, /* DDR2-800 SC */
264 {1, 1, 800, 667, 6420, 36420, 6873, 36873}, /* DDR3-667 SC */
265 {1, 1, 800, 800, 5902, 35902, 6318, 36318}, /* DDR3-800 SC */
266
267 {1, 0, 667, 400, 3400, 33400, 4021, 34021}, /* DDR2-400 SC */
268 {1, 0, 667, 667, 3372, 33372, 3845, 33845}, /* DDR2-667 SC */
269 {1, 0, 667, 800, 3386, 33386, 3822, 33822}, /* DDR2-800 SC */
270 {1, 1, 667, 667, 6438, 36438, 6911, 36911}, /* DDR3-667 SC */
271 {1, 1, 667, 800, 5941, 35941, 6377, 36377}, /* DDR3-800 SC */
272
273 {1, 0, 400, 400, 3472, 33472, 4173, 34173}, /* DDR2-400 SC */
274 {1, 0, 400, 667, 3443, 33443, 3996, 33996}, /* DDR2-667 SC */
275 {1, 0, 400, 800, 3430, 33430, 3946, 33946}, /* DDR2-800 SC */
276 {1, 1, 400, 667, 6509, 36509, 7062, 37062}, /* DDR3-667 SC */
277 {1, 1, 400, 800, 5985, 35985, 6501, 36501}, /* DDR3-800 SC */
278
279 {0, 0, 800, 400, 3438, 33438, 4065, 34065}, /* DDR2-400 SC */
280 {0, 0, 800, 667, 3410, 33410, 3889, 33889}, /* DDR2-667 SC */
281 {0, 0, 800, 800, 3403, 33403, 3845, 33845}, /* DDR2-800 SC */
282 {0, 1, 800, 667, 6476, 36476, 6955, 36955}, /* DDR3-667 SC */
283 {0, 1, 800, 800, 5958, 35958, 6400, 36400}, /* DDR3-800 SC */
284
285 {0, 0, 667, 400, 3456, 33456, 4103, 34106}, /* DDR2-400 SC */
286 {0, 0, 667, 667, 3428, 33428, 3927, 33927}, /* DDR2-667 SC */
287 {0, 0, 667, 800, 3443, 33443, 3905, 33905}, /* DDR2-800 SC */
288 {0, 1, 667, 667, 6494, 36494, 6993, 36993}, /* DDR3-667 SC */
289 {0, 1, 667, 800, 5998, 35998, 6460, 36460}, /* DDR3-800 SC */
290
291 {0, 0, 400, 400, 3528, 33528, 4255, 34255}, /* DDR2-400 SC */
292 {0, 0, 400, 667, 3500, 33500, 4079, 34079}, /* DDR2-667 SC */
293 {0, 0, 400, 800, 3487, 33487, 4029, 34029}, /* DDR2-800 SC */
294 {0, 1, 400, 667, 6566, 36566, 7145, 37145}, /* DDR3-667 SC */
295 {0, 1, 400, 800, 6042, 36042, 6584, 36584}, /* DDR3-800 SC */
296};
297
Tvrtko Ursulin44a655c2016-10-13 11:09:23 +0100298static const struct cxsr_latency *intel_get_cxsr_latency(bool is_desktop,
299 bool is_ddr3,
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300300 int fsb,
301 int mem)
302{
303 const struct cxsr_latency *latency;
304 int i;
305
306 if (fsb == 0 || mem == 0)
307 return NULL;
308
309 for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
310 latency = &cxsr_latency_table[i];
311 if (is_desktop == latency->is_desktop &&
312 is_ddr3 == latency->is_ddr3 &&
313 fsb == latency->fsb_freq && mem == latency->mem_freq)
314 return latency;
315 }
316
317 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
318
319 return NULL;
320}
321
Ville Syrjäläfc1ac8d2015-03-05 21:19:52 +0200322static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable)
323{
324 u32 val;
325
Chris Wilson337fa6e2019-04-26 09:17:20 +0100326 vlv_punit_get(dev_priv);
Ville Syrjäläfc1ac8d2015-03-05 21:19:52 +0200327
328 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
329 if (enable)
330 val &= ~FORCE_DDR_HIGH_FREQ;
331 else
332 val |= FORCE_DDR_HIGH_FREQ;
333 val &= ~FORCE_DDR_LOW_FREQ;
334 val |= FORCE_DDR_FREQ_REQ_ACK;
335 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
336
337 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
338 FORCE_DDR_FREQ_REQ_ACK) == 0, 3))
Wambui Karugaf8d18d52020-01-07 18:13:30 +0300339 drm_err(&dev_priv->drm,
340 "timed out waiting for Punit DDR DVFS request\n");
Ville Syrjäläfc1ac8d2015-03-05 21:19:52 +0200341
Chris Wilson337fa6e2019-04-26 09:17:20 +0100342 vlv_punit_put(dev_priv);
Ville Syrjäläfc1ac8d2015-03-05 21:19:52 +0200343}
344
Ville Syrjäläcfb41412015-03-05 21:19:51 +0200345static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable)
346{
347 u32 val;
348
Chris Wilson337fa6e2019-04-26 09:17:20 +0100349 vlv_punit_get(dev_priv);
Ville Syrjäläcfb41412015-03-05 21:19:51 +0200350
Ville Syrjäläc11b8132018-11-29 19:55:03 +0200351 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
Ville Syrjäläcfb41412015-03-05 21:19:51 +0200352 if (enable)
353 val |= DSP_MAXFIFO_PM5_ENABLE;
354 else
355 val &= ~DSP_MAXFIFO_PM5_ENABLE;
Ville Syrjäläc11b8132018-11-29 19:55:03 +0200356 vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
Ville Syrjäläcfb41412015-03-05 21:19:51 +0200357
Chris Wilson337fa6e2019-04-26 09:17:20 +0100358 vlv_punit_put(dev_priv);
Ville Syrjäläcfb41412015-03-05 21:19:51 +0200359}
360
Ville Syrjäläf4998962015-03-10 17:02:21 +0200361#define FW_WM(value, plane) \
362 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK)
363
Ville Syrjälä11a85d62016-11-28 19:37:12 +0200364static bool _intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300365{
Ville Syrjälä11a85d62016-11-28 19:37:12 +0200366 bool was_enabled;
Imre Deak5209b1f2014-07-01 12:36:17 +0300367 u32 val;
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300368
Tvrtko Ursulin920a14b2016-10-14 10:13:44 +0100369 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
Jani Nikula5f461662020-11-30 13:15:58 +0200370 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
371 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
372 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF_VLV);
Jani Nikulac0f86832016-12-07 12:13:04 +0200373 } else if (IS_G4X(dev_priv) || IS_I965GM(dev_priv)) {
Jani Nikula5f461662020-11-30 13:15:58 +0200374 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
375 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
376 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF);
Ville Syrjälä9b1e14f2016-10-31 22:37:15 +0200377 } else if (IS_PINEVIEW(dev_priv)) {
Jani Nikula5f461662020-11-30 13:15:58 +0200378 val = intel_uncore_read(&dev_priv->uncore, DSPFW3);
Ville Syrjälä11a85d62016-11-28 19:37:12 +0200379 was_enabled = val & PINEVIEW_SELF_REFRESH_EN;
380 if (enable)
381 val |= PINEVIEW_SELF_REFRESH_EN;
382 else
383 val &= ~PINEVIEW_SELF_REFRESH_EN;
Jani Nikula5f461662020-11-30 13:15:58 +0200384 intel_uncore_write(&dev_priv->uncore, DSPFW3, val);
385 intel_uncore_posting_read(&dev_priv->uncore, DSPFW3);
Tvrtko Ursulin50a0bc92016-10-13 11:02:58 +0100386 } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv)) {
Jani Nikula5f461662020-11-30 13:15:58 +0200387 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
Imre Deak5209b1f2014-07-01 12:36:17 +0300388 val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
389 _MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
Jani Nikula5f461662020-11-30 13:15:58 +0200390 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, val);
391 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF);
Tvrtko Ursulin50a0bc92016-10-13 11:02:58 +0100392 } else if (IS_I915GM(dev_priv)) {
Ville Syrjäläacb91352016-07-29 17:57:02 +0300393 /*
394 * FIXME can't find a bit like this for 915G, and
395 * and yet it does have the related watermark in
396 * FW_BLC_SELF. What's going on?
397 */
Jani Nikula5f461662020-11-30 13:15:58 +0200398 was_enabled = intel_uncore_read(&dev_priv->uncore, INSTPM) & INSTPM_SELF_EN;
Imre Deak5209b1f2014-07-01 12:36:17 +0300399 val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
400 _MASKED_BIT_DISABLE(INSTPM_SELF_EN);
Jani Nikula5f461662020-11-30 13:15:58 +0200401 intel_uncore_write(&dev_priv->uncore, INSTPM, val);
402 intel_uncore_posting_read(&dev_priv->uncore, INSTPM);
Imre Deak5209b1f2014-07-01 12:36:17 +0300403 } else {
Ville Syrjälä11a85d62016-11-28 19:37:12 +0200404 return false;
Imre Deak5209b1f2014-07-01 12:36:17 +0300405 }
406
Ville Syrjälä1489bba2017-03-02 19:15:07 +0200407 trace_intel_memory_cxsr(dev_priv, was_enabled, enable);
408
Wambui Karugaf8d18d52020-01-07 18:13:30 +0300409 drm_dbg_kms(&dev_priv->drm, "memory self-refresh is %s (was %s)\n",
410 enableddisabled(enable),
411 enableddisabled(was_enabled));
Ville Syrjälä11a85d62016-11-28 19:37:12 +0200412
413 return was_enabled;
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300414}
415
Ville Syrjälä62571fc2017-04-21 21:14:23 +0300416/**
417 * intel_set_memory_cxsr - Configure CxSR state
418 * @dev_priv: i915 device
419 * @enable: Allow vs. disallow CxSR
420 *
421 * Allow or disallow the system to enter a special CxSR
422 * (C-state self refresh) state. What typically happens in CxSR mode
423 * is that several display FIFOs may get combined into a single larger
424 * FIFO for a particular plane (so called max FIFO mode) to allow the
425 * system to defer memory fetches longer, and the memory will enter
426 * self refresh.
427 *
428 * Note that enabling CxSR does not guarantee that the system enter
429 * this special mode, nor does it guarantee that the system stays
430 * in that mode once entered. So this just allows/disallows the system
431 * to autonomously utilize the CxSR mode. Other factors such as core
432 * C-states will affect when/if the system actually enters/exits the
433 * CxSR mode.
434 *
435 * Note that on VLV/CHV this actually only controls the max FIFO mode,
436 * and the system is free to enter/exit memory self refresh at any time
437 * even when the use of CxSR has been disallowed.
438 *
439 * While the system is actually in the CxSR/max FIFO mode, some plane
440 * control registers will not get latched on vblank. Thus in order to
441 * guarantee the system will respond to changes in the plane registers
442 * we must always disallow CxSR prior to making changes to those registers.
443 * Unfortunately the system will re-evaluate the CxSR conditions at
444 * frame start which happens after vblank start (which is when the plane
445 * registers would get latched), so we can't proceed with the plane update
446 * during the same frame where we disallowed CxSR.
447 *
448 * Certain platforms also have a deeper HPLL SR mode. Fortunately the
449 * HPLL SR mode depends on CxSR itself, so we don't have to hand hold
450 * the hardware w.r.t. HPLL SR when writing to plane registers.
451 * Disallowing just CxSR is sufficient.
452 */
Ville Syrjälä11a85d62016-11-28 19:37:12 +0200453bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
Ville Syrjälä3d90e642016-11-28 19:37:11 +0200454{
Ville Syrjälä11a85d62016-11-28 19:37:12 +0200455 bool ret;
456
Ville Syrjälä3d90e642016-11-28 19:37:11 +0200457 mutex_lock(&dev_priv->wm.wm_mutex);
Ville Syrjälä11a85d62016-11-28 19:37:12 +0200458 ret = _intel_set_memory_cxsr(dev_priv, enable);
Ville Syrjälä04548cb2017-04-21 21:14:29 +0300459 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
460 dev_priv->wm.vlv.cxsr = enable;
461 else if (IS_G4X(dev_priv))
462 dev_priv->wm.g4x.cxsr = enable;
Ville Syrjälä3d90e642016-11-28 19:37:11 +0200463 mutex_unlock(&dev_priv->wm.wm_mutex);
Ville Syrjälä11a85d62016-11-28 19:37:12 +0200464
465 return ret;
Ville Syrjälä3d90e642016-11-28 19:37:11 +0200466}
Ville Syrjäläfc1ac8d2015-03-05 21:19:52 +0200467
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300468/*
469 * Latency for FIFO fetches is dependent on several factors:
470 * - memory configuration (speed, channels)
471 * - chipset
472 * - current MCH state
473 * It can be fairly high in some situations, so here we assume a fairly
474 * pessimal value. It's a tradeoff between extra memory fetches (if we
475 * set this value too high, the FIFO will fetch frequently to stay full)
476 * and power consumption (set it too low to save power and we might see
477 * FIFO underruns and display "flicker").
478 *
479 * A value of 5us seems to be a good balance; safe for very low end
480 * platforms but not overly aggressive on lower latency configs.
481 */
Chris Wilson5aef6002014-09-03 11:56:07 +0100482static const int pessimal_latency_ns = 5000;
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300483
Ville Syrjäläb5004722015-03-05 21:19:47 +0200484#define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \
485 ((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8))
486
Ville Syrjälä814e7f02017-03-02 19:14:55 +0200487static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state)
Ville Syrjäläb5004722015-03-05 21:19:47 +0200488{
Maarten Lankhorst2225f3c2019-10-31 12:26:03 +0100489 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
Ville Syrjäläf07d43d2017-03-02 19:14:52 +0200490 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
Ville Syrjälä814e7f02017-03-02 19:14:55 +0200491 struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
Ville Syrjäläf07d43d2017-03-02 19:14:52 +0200492 enum pipe pipe = crtc->pipe;
493 int sprite0_start, sprite1_start;
Kees Cook2713eb42020-02-20 16:05:17 -0800494 u32 dsparb, dsparb2, dsparb3;
Ville Syrjäläb5004722015-03-05 21:19:47 +0200495
Ville Syrjäläf07d43d2017-03-02 19:14:52 +0200496 switch (pipe) {
Ville Syrjäläb5004722015-03-05 21:19:47 +0200497 case PIPE_A:
Jani Nikula5f461662020-11-30 13:15:58 +0200498 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
499 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
Ville Syrjäläb5004722015-03-05 21:19:47 +0200500 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0);
501 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4);
502 break;
503 case PIPE_B:
Jani Nikula5f461662020-11-30 13:15:58 +0200504 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
505 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
Ville Syrjäläb5004722015-03-05 21:19:47 +0200506 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8);
507 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12);
508 break;
509 case PIPE_C:
Jani Nikula5f461662020-11-30 13:15:58 +0200510 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
511 dsparb3 = intel_uncore_read(&dev_priv->uncore, DSPARB3);
Ville Syrjäläb5004722015-03-05 21:19:47 +0200512 sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16);
513 sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20);
514 break;
515 default:
Ville Syrjäläf07d43d2017-03-02 19:14:52 +0200516 MISSING_CASE(pipe);
517 return;
Ville Syrjäläb5004722015-03-05 21:19:47 +0200518 }
519
Ville Syrjäläf07d43d2017-03-02 19:14:52 +0200520 fifo_state->plane[PLANE_PRIMARY] = sprite0_start;
521 fifo_state->plane[PLANE_SPRITE0] = sprite1_start - sprite0_start;
522 fifo_state->plane[PLANE_SPRITE1] = 511 - sprite1_start;
523 fifo_state->plane[PLANE_CURSOR] = 63;
Ville Syrjäläb5004722015-03-05 21:19:47 +0200524}
525
Ville Syrjäläbdaf8432017-11-17 21:19:11 +0200526static int i9xx_get_fifo_size(struct drm_i915_private *dev_priv,
527 enum i9xx_plane_id i9xx_plane)
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300528{
Jani Nikula5f461662020-11-30 13:15:58 +0200529 u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300530 int size;
531
532 size = dsparb & 0x7f;
Ville Syrjäläbdaf8432017-11-17 21:19:11 +0200533 if (i9xx_plane == PLANE_B)
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300534 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
535
Wambui Karugaf8d18d52020-01-07 18:13:30 +0300536 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
537 dsparb, plane_name(i9xx_plane), size);
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300538
539 return size;
540}
541
Ville Syrjäläbdaf8432017-11-17 21:19:11 +0200542static int i830_get_fifo_size(struct drm_i915_private *dev_priv,
543 enum i9xx_plane_id i9xx_plane)
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300544{
Jani Nikula5f461662020-11-30 13:15:58 +0200545 u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300546 int size;
547
548 size = dsparb & 0x1ff;
Ville Syrjäläbdaf8432017-11-17 21:19:11 +0200549 if (i9xx_plane == PLANE_B)
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300550 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
551 size >>= 1; /* Convert to cachelines */
552
Wambui Karugaf8d18d52020-01-07 18:13:30 +0300553 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
554 dsparb, plane_name(i9xx_plane), size);
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300555
556 return size;
557}
558
Ville Syrjäläbdaf8432017-11-17 21:19:11 +0200559static int i845_get_fifo_size(struct drm_i915_private *dev_priv,
560 enum i9xx_plane_id i9xx_plane)
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300561{
Jani Nikula5f461662020-11-30 13:15:58 +0200562 u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300563 int size;
564
565 size = dsparb & 0x7f;
566 size >>= 2; /* Convert to cachelines */
567
Wambui Karugaf8d18d52020-01-07 18:13:30 +0300568 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
569 dsparb, plane_name(i9xx_plane), size);
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300570
571 return size;
572}
573
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300574/* Pineview has different values for various configs */
Lucas De Marchi1d218222019-12-24 00:40:04 -0800575static const struct intel_watermark_params pnv_display_wm = {
Ville Syrjäläe0f02732014-06-05 19:15:50 +0300576 .fifo_size = PINEVIEW_DISPLAY_FIFO,
577 .max_wm = PINEVIEW_MAX_WM,
578 .default_wm = PINEVIEW_DFT_WM,
579 .guard_size = PINEVIEW_GUARD_WM,
580 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300581};
Lucas De Marchi1d218222019-12-24 00:40:04 -0800582
583static const struct intel_watermark_params pnv_display_hplloff_wm = {
Ville Syrjäläe0f02732014-06-05 19:15:50 +0300584 .fifo_size = PINEVIEW_DISPLAY_FIFO,
585 .max_wm = PINEVIEW_MAX_WM,
586 .default_wm = PINEVIEW_DFT_HPLLOFF_WM,
587 .guard_size = PINEVIEW_GUARD_WM,
588 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300589};
Lucas De Marchi1d218222019-12-24 00:40:04 -0800590
591static const struct intel_watermark_params pnv_cursor_wm = {
Ville Syrjäläe0f02732014-06-05 19:15:50 +0300592 .fifo_size = PINEVIEW_CURSOR_FIFO,
593 .max_wm = PINEVIEW_CURSOR_MAX_WM,
594 .default_wm = PINEVIEW_CURSOR_DFT_WM,
595 .guard_size = PINEVIEW_CURSOR_GUARD_WM,
596 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300597};
Lucas De Marchi1d218222019-12-24 00:40:04 -0800598
599static const struct intel_watermark_params pnv_cursor_hplloff_wm = {
Ville Syrjäläe0f02732014-06-05 19:15:50 +0300600 .fifo_size = PINEVIEW_CURSOR_FIFO,
601 .max_wm = PINEVIEW_CURSOR_MAX_WM,
602 .default_wm = PINEVIEW_CURSOR_DFT_WM,
603 .guard_size = PINEVIEW_CURSOR_GUARD_WM,
604 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300605};
Lucas De Marchi1d218222019-12-24 00:40:04 -0800606
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300607static const struct intel_watermark_params i965_cursor_wm_info = {
Ville Syrjäläe0f02732014-06-05 19:15:50 +0300608 .fifo_size = I965_CURSOR_FIFO,
609 .max_wm = I965_CURSOR_MAX_WM,
610 .default_wm = I965_CURSOR_DFT_WM,
611 .guard_size = 2,
612 .cacheline_size = I915_FIFO_LINE_SIZE,
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300613};
Lucas De Marchi1d218222019-12-24 00:40:04 -0800614
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300615static const struct intel_watermark_params i945_wm_info = {
Ville Syrjäläe0f02732014-06-05 19:15:50 +0300616 .fifo_size = I945_FIFO_SIZE,
617 .max_wm = I915_MAX_WM,
618 .default_wm = 1,
619 .guard_size = 2,
620 .cacheline_size = I915_FIFO_LINE_SIZE,
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300621};
Lucas De Marchi1d218222019-12-24 00:40:04 -0800622
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300623static const struct intel_watermark_params i915_wm_info = {
Ville Syrjäläe0f02732014-06-05 19:15:50 +0300624 .fifo_size = I915_FIFO_SIZE,
625 .max_wm = I915_MAX_WM,
626 .default_wm = 1,
627 .guard_size = 2,
628 .cacheline_size = I915_FIFO_LINE_SIZE,
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300629};
Lucas De Marchi1d218222019-12-24 00:40:04 -0800630
Ville Syrjälä9d539102014-08-15 01:21:53 +0300631static const struct intel_watermark_params i830_a_wm_info = {
Ville Syrjäläe0f02732014-06-05 19:15:50 +0300632 .fifo_size = I855GM_FIFO_SIZE,
633 .max_wm = I915_MAX_WM,
634 .default_wm = 1,
635 .guard_size = 2,
636 .cacheline_size = I830_FIFO_LINE_SIZE,
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300637};
Lucas De Marchi1d218222019-12-24 00:40:04 -0800638
Ville Syrjälä9d539102014-08-15 01:21:53 +0300639static const struct intel_watermark_params i830_bc_wm_info = {
640 .fifo_size = I855GM_FIFO_SIZE,
641 .max_wm = I915_MAX_WM/2,
642 .default_wm = 1,
643 .guard_size = 2,
644 .cacheline_size = I830_FIFO_LINE_SIZE,
645};
Lucas De Marchi1d218222019-12-24 00:40:04 -0800646
Daniel Vetterfeb56b92013-12-14 20:38:30 -0200647static const struct intel_watermark_params i845_wm_info = {
Ville Syrjäläe0f02732014-06-05 19:15:50 +0300648 .fifo_size = I830_FIFO_SIZE,
649 .max_wm = I915_MAX_WM,
650 .default_wm = 1,
651 .guard_size = 2,
652 .cacheline_size = I830_FIFO_LINE_SIZE,
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300653};
654
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300655/**
Ville Syrjäläbaf69ca2017-04-21 21:14:27 +0300656 * intel_wm_method1 - Method 1 / "small buffer" watermark formula
657 * @pixel_rate: Pipe pixel rate in kHz
658 * @cpp: Plane bytes per pixel
659 * @latency: Memory wakeup latency in 0.1us units
660 *
661 * Compute the watermark using the method 1 or "small buffer"
662 * formula. The caller may additonally add extra cachelines
663 * to account for TLB misses and clock crossings.
664 *
665 * This method is concerned with the short term drain rate
666 * of the FIFO, ie. it does not account for blanking periods
667 * which would effectively reduce the average drain rate across
668 * a longer period. The name "small" refers to the fact the
669 * FIFO is relatively small compared to the amount of data
670 * fetched.
671 *
672 * The FIFO level vs. time graph might look something like:
673 *
674 * |\ |\
675 * | \ | \
676 * __---__---__ (- plane active, _ blanking)
677 * -> time
678 *
679 * or perhaps like this:
680 *
681 * |\|\ |\|\
682 * __----__----__ (- plane active, _ blanking)
683 * -> time
684 *
685 * Returns:
686 * The watermark in bytes
687 */
688static unsigned int intel_wm_method1(unsigned int pixel_rate,
689 unsigned int cpp,
690 unsigned int latency)
691{
Jani Nikula5ce9a6492019-01-18 14:01:20 +0200692 u64 ret;
Ville Syrjäläbaf69ca2017-04-21 21:14:27 +0300693
Ville Syrjäläd492a292019-04-08 18:27:01 +0300694 ret = mul_u32_u32(pixel_rate, cpp * latency);
Ville Syrjäläbaf69ca2017-04-21 21:14:27 +0300695 ret = DIV_ROUND_UP_ULL(ret, 10000);
696
697 return ret;
698}
699
700/**
701 * intel_wm_method2 - Method 2 / "large buffer" watermark formula
702 * @pixel_rate: Pipe pixel rate in kHz
703 * @htotal: Pipe horizontal total
704 * @width: Plane width in pixels
705 * @cpp: Plane bytes per pixel
706 * @latency: Memory wakeup latency in 0.1us units
707 *
708 * Compute the watermark using the method 2 or "large buffer"
709 * formula. The caller may additonally add extra cachelines
710 * to account for TLB misses and clock crossings.
711 *
712 * This method is concerned with the long term drain rate
713 * of the FIFO, ie. it does account for blanking periods
714 * which effectively reduce the average drain rate across
715 * a longer period. The name "large" refers to the fact the
716 * FIFO is relatively large compared to the amount of data
717 * fetched.
718 *
719 * The FIFO level vs. time graph might look something like:
720 *
721 * |\___ |\___
722 * | \___ | \___
723 * | \ | \
724 * __ --__--__--__--__--__--__ (- plane active, _ blanking)
725 * -> time
726 *
727 * Returns:
728 * The watermark in bytes
729 */
730static unsigned int intel_wm_method2(unsigned int pixel_rate,
731 unsigned int htotal,
732 unsigned int width,
733 unsigned int cpp,
734 unsigned int latency)
735{
736 unsigned int ret;
737
738 /*
739 * FIXME remove once all users are computing
740 * watermarks in the correct place.
741 */
742 if (WARN_ON_ONCE(htotal == 0))
743 htotal = 1;
744
745 ret = (latency * pixel_rate) / (htotal * 10000);
746 ret = (ret + 1) * width * cpp;
747
748 return ret;
749}
750
751/**
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300752 * intel_calculate_wm - calculate watermark level
Ville Syrjäläbaf69ca2017-04-21 21:14:27 +0300753 * @pixel_rate: pixel clock
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300754 * @wm: chip FIFO params
Chris Wilson31383412018-02-14 14:03:03 +0000755 * @fifo_size: size of the FIFO buffer
Ville Syrjäläac484962016-01-20 21:05:26 +0200756 * @cpp: bytes per pixel
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300757 * @latency_ns: memory latency for the platform
758 *
759 * Calculate the watermark level (the level at which the display plane will
760 * start fetching from memory again). Each chip has a different display
761 * FIFO size and allocation, so the caller needs to figure that out and pass
762 * in the correct intel_watermark_params structure.
763 *
764 * As the pixel clock runs, the FIFO will be drained at a rate that depends
765 * on the pixel size. When it reaches the watermark level, it'll start
766 * fetching FIFO line sized based chunks from memory until the FIFO fills
767 * past the watermark point. If the FIFO drains completely, a FIFO underrun
768 * will occur, and a display engine hang could result.
769 */
Ville Syrjäläbaf69ca2017-04-21 21:14:27 +0300770static unsigned int intel_calculate_wm(int pixel_rate,
771 const struct intel_watermark_params *wm,
772 int fifo_size, int cpp,
773 unsigned int latency_ns)
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300774{
Ville Syrjäläbaf69ca2017-04-21 21:14:27 +0300775 int entries, wm_size;
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300776
777 /*
778 * Note: we need to make sure we don't overflow for various clock &
779 * latency values.
780 * clocks go from a few thousand to several hundred thousand.
781 * latency is usually a few thousand
782 */
Ville Syrjäläbaf69ca2017-04-21 21:14:27 +0300783 entries = intel_wm_method1(pixel_rate, cpp,
784 latency_ns / 100);
785 entries = DIV_ROUND_UP(entries, wm->cacheline_size) +
786 wm->guard_size;
787 DRM_DEBUG_KMS("FIFO entries required for mode: %d\n", entries);
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300788
Ville Syrjäläbaf69ca2017-04-21 21:14:27 +0300789 wm_size = fifo_size - entries;
790 DRM_DEBUG_KMS("FIFO watermark level: %d\n", wm_size);
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300791
792 /* Don't promote wm_size to unsigned... */
Ville Syrjäläbaf69ca2017-04-21 21:14:27 +0300793 if (wm_size > wm->max_wm)
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300794 wm_size = wm->max_wm;
795 if (wm_size <= 0)
796 wm_size = wm->default_wm;
Ville Syrjäläd6feb192014-09-05 21:54:13 +0300797
798 /*
799 * Bspec seems to indicate that the value shouldn't be lower than
800 * 'burst size + 1'. Certainly 830 is quite unhappy with low values.
801 * Lets go for 8 which is the burst size since certain platforms
802 * already use a hardcoded 8 (which is what the spec says should be
803 * done).
804 */
805 if (wm_size <= 8)
806 wm_size = 8;
807
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300808 return wm_size;
809}
810
Ville Syrjälä04548cb2017-04-21 21:14:29 +0300811static bool is_disabling(int old, int new, int threshold)
812{
813 return old >= threshold && new < threshold;
814}
815
816static bool is_enabling(int old, int new, int threshold)
817{
818 return old < threshold && new >= threshold;
819}
820
Ville Syrjälä6d5019b2017-04-21 21:14:20 +0300821static int intel_wm_num_levels(struct drm_i915_private *dev_priv)
822{
823 return dev_priv->wm.max_level + 1;
824}
825
Ville Syrjälä24304d812017-03-14 17:10:49 +0200826static bool intel_wm_plane_visible(const struct intel_crtc_state *crtc_state,
827 const struct intel_plane_state *plane_state)
828{
Maarten Lankhorstf90a85e2019-10-31 12:26:08 +0100829 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
Ville Syrjälä24304d812017-03-14 17:10:49 +0200830
831 /* FIXME check the 'enable' instead */
Maarten Lankhorst1326a922019-10-31 12:26:02 +0100832 if (!crtc_state->hw.active)
Ville Syrjälä24304d812017-03-14 17:10:49 +0200833 return false;
834
835 /*
836 * Treat cursor with fb as always visible since cursor updates
837 * can happen faster than the vrefresh rate, and the current
838 * watermark code doesn't handle that correctly. Cursor updates
839 * which set/clear the fb or change the cursor size are going
840 * to get throttled by intel_legacy_cursor_update() to work
841 * around this problem with the watermark code.
842 */
843 if (plane->id == PLANE_CURSOR)
Maarten Lankhorst7b3cb172019-10-31 12:26:07 +0100844 return plane_state->hw.fb != NULL;
Ville Syrjälä24304d812017-03-14 17:10:49 +0200845 else
Maarten Lankhorstf90a85e2019-10-31 12:26:08 +0100846 return plane_state->uapi.visible;
Ville Syrjälä24304d812017-03-14 17:10:49 +0200847}
848
Ville Syrjälä04da7b92019-11-27 22:12:11 +0200849static bool intel_crtc_active(struct intel_crtc *crtc)
850{
851 /* Be paranoid as we can arrive here with only partial
852 * state retrieved from the hardware during setup.
853 *
854 * We can ditch the adjusted_mode.crtc_clock check as soon
855 * as Haswell has gained clock readout/fastboot support.
856 *
857 * We can ditch the crtc->primary->state->fb check as soon as we can
858 * properly reconstruct framebuffers.
859 *
860 * FIXME: The intel_crtc->active here should be switched to
861 * crtc->state->active once we have proper CRTC states wired up
862 * for atomic.
863 */
864 return crtc->active && crtc->base.primary->state->fb &&
865 crtc->config->hw.adjusted_mode.crtc_clock;
866}
867
Ville Syrjäläffc7a762016-10-31 22:37:21 +0200868static struct intel_crtc *single_enabled_crtc(struct drm_i915_private *dev_priv)
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300869{
Ville Syrjäläefc26112016-10-31 22:37:04 +0200870 struct intel_crtc *crtc, *enabled = NULL;
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300871
Ville Syrjäläffc7a762016-10-31 22:37:21 +0200872 for_each_intel_crtc(&dev_priv->drm, crtc) {
Ville Syrjäläefc26112016-10-31 22:37:04 +0200873 if (intel_crtc_active(crtc)) {
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300874 if (enabled)
875 return NULL;
876 enabled = crtc;
877 }
878 }
879
880 return enabled;
881}
882
Lucas De Marchi1d218222019-12-24 00:40:04 -0800883static void pnv_update_wm(struct intel_crtc *unused_crtc)
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300884{
Ville Syrjäläffc7a762016-10-31 22:37:21 +0200885 struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
Ville Syrjäläefc26112016-10-31 22:37:04 +0200886 struct intel_crtc *crtc;
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300887 const struct cxsr_latency *latency;
888 u32 reg;
Ville Syrjäläbaf69ca2017-04-21 21:14:27 +0300889 unsigned int wm;
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300890
Tvrtko Ursulin86d35d42019-03-26 07:40:54 +0000891 latency = intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
Tvrtko Ursulin50a0bc92016-10-13 11:02:58 +0100892 dev_priv->is_ddr3,
893 dev_priv->fsb_freq,
894 dev_priv->mem_freq);
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300895 if (!latency) {
Wambui Karugaf8d18d52020-01-07 18:13:30 +0300896 drm_dbg_kms(&dev_priv->drm,
897 "Unknown FSB/MEM found, disable CxSR\n");
Imre Deak5209b1f2014-07-01 12:36:17 +0300898 intel_set_memory_cxsr(dev_priv, false);
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300899 return;
900 }
901
Ville Syrjäläffc7a762016-10-31 22:37:21 +0200902 crtc = single_enabled_crtc(dev_priv);
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300903 if (crtc) {
Maarten Lankhorstbafcdad2020-11-12 21:17:18 +0200904 const struct drm_display_mode *pipe_mode =
905 &crtc->config->hw.pipe_mode;
Ville Syrjäläefc26112016-10-31 22:37:04 +0200906 const struct drm_framebuffer *fb =
907 crtc->base.primary->state->fb;
Ville Syrjälä353c8592016-12-14 23:30:57 +0200908 int cpp = fb->format->cpp[0];
Maarten Lankhorstbafcdad2020-11-12 21:17:18 +0200909 int clock = pipe_mode->crtc_clock;
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300910
911 /* Display SR */
Lucas De Marchi1d218222019-12-24 00:40:04 -0800912 wm = intel_calculate_wm(clock, &pnv_display_wm,
913 pnv_display_wm.fifo_size,
Ville Syrjäläac484962016-01-20 21:05:26 +0200914 cpp, latency->display_sr);
Jani Nikula5f461662020-11-30 13:15:58 +0200915 reg = intel_uncore_read(&dev_priv->uncore, DSPFW1);
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300916 reg &= ~DSPFW_SR_MASK;
Ville Syrjäläf4998962015-03-10 17:02:21 +0200917 reg |= FW_WM(wm, SR);
Jani Nikula5f461662020-11-30 13:15:58 +0200918 intel_uncore_write(&dev_priv->uncore, DSPFW1, reg);
Wambui Karugaf8d18d52020-01-07 18:13:30 +0300919 drm_dbg_kms(&dev_priv->drm, "DSPFW1 register is %x\n", reg);
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300920
921 /* cursor SR */
Lucas De Marchi1d218222019-12-24 00:40:04 -0800922 wm = intel_calculate_wm(clock, &pnv_cursor_wm,
923 pnv_display_wm.fifo_size,
Ville Syrjälä99834b12017-04-21 21:14:24 +0300924 4, latency->cursor_sr);
Jani Nikula5f461662020-11-30 13:15:58 +0200925 reg = intel_uncore_read(&dev_priv->uncore, DSPFW3);
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300926 reg &= ~DSPFW_CURSOR_SR_MASK;
Ville Syrjäläf4998962015-03-10 17:02:21 +0200927 reg |= FW_WM(wm, CURSOR_SR);
Jani Nikula5f461662020-11-30 13:15:58 +0200928 intel_uncore_write(&dev_priv->uncore, DSPFW3, reg);
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300929
930 /* Display HPLL off SR */
Lucas De Marchi1d218222019-12-24 00:40:04 -0800931 wm = intel_calculate_wm(clock, &pnv_display_hplloff_wm,
932 pnv_display_hplloff_wm.fifo_size,
Ville Syrjäläac484962016-01-20 21:05:26 +0200933 cpp, latency->display_hpll_disable);
Jani Nikula5f461662020-11-30 13:15:58 +0200934 reg = intel_uncore_read(&dev_priv->uncore, DSPFW3);
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300935 reg &= ~DSPFW_HPLL_SR_MASK;
Ville Syrjäläf4998962015-03-10 17:02:21 +0200936 reg |= FW_WM(wm, HPLL_SR);
Jani Nikula5f461662020-11-30 13:15:58 +0200937 intel_uncore_write(&dev_priv->uncore, DSPFW3, reg);
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300938
939 /* cursor HPLL off SR */
Lucas De Marchi1d218222019-12-24 00:40:04 -0800940 wm = intel_calculate_wm(clock, &pnv_cursor_hplloff_wm,
941 pnv_display_hplloff_wm.fifo_size,
Ville Syrjälä99834b12017-04-21 21:14:24 +0300942 4, latency->cursor_hpll_disable);
Jani Nikula5f461662020-11-30 13:15:58 +0200943 reg = intel_uncore_read(&dev_priv->uncore, DSPFW3);
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300944 reg &= ~DSPFW_HPLL_CURSOR_MASK;
Ville Syrjäläf4998962015-03-10 17:02:21 +0200945 reg |= FW_WM(wm, HPLL_CURSOR);
Jani Nikula5f461662020-11-30 13:15:58 +0200946 intel_uncore_write(&dev_priv->uncore, DSPFW3, reg);
Wambui Karugaf8d18d52020-01-07 18:13:30 +0300947 drm_dbg_kms(&dev_priv->drm, "DSPFW3 register is %x\n", reg);
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300948
Imre Deak5209b1f2014-07-01 12:36:17 +0300949 intel_set_memory_cxsr(dev_priv, true);
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300950 } else {
Imre Deak5209b1f2014-07-01 12:36:17 +0300951 intel_set_memory_cxsr(dev_priv, false);
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300952 }
953}
954
Ville Syrjälä0f95ff82017-04-21 21:14:26 +0300955/*
956 * Documentation says:
957 * "If the line size is small, the TLB fetches can get in the way of the
958 * data fetches, causing some lag in the pixel data return which is not
959 * accounted for in the above formulas. The following adjustment only
960 * needs to be applied if eight whole lines fit in the buffer at once.
961 * The WM is adjusted upwards by the difference between the FIFO size
962 * and the size of 8 whole lines. This adjustment is always performed
963 * in the actual pixel depth regardless of whether FBC is enabled or not."
964 */
Chris Wilson1a1f1282017-11-07 14:03:38 +0000965static unsigned int g4x_tlb_miss_wa(int fifo_size, int width, int cpp)
Ville Syrjälä0f95ff82017-04-21 21:14:26 +0300966{
967 int tlb_miss = fifo_size * 64 - width * cpp * 8;
968
969 return max(0, tlb_miss);
970}
971
Ville Syrjälä04548cb2017-04-21 21:14:29 +0300972static void g4x_write_wm_values(struct drm_i915_private *dev_priv,
973 const struct g4x_wm_values *wm)
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300974{
Ville Syrjäläe93329a2017-04-21 21:14:31 +0300975 enum pipe pipe;
976
977 for_each_pipe(dev_priv, pipe)
978 trace_g4x_wm(intel_get_crtc_for_pipe(dev_priv, pipe), wm);
979
Jani Nikula5f461662020-11-30 13:15:58 +0200980 intel_uncore_write(&dev_priv->uncore, DSPFW1,
Ville Syrjälä04548cb2017-04-21 21:14:29 +0300981 FW_WM(wm->sr.plane, SR) |
982 FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
983 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
984 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
Jani Nikula5f461662020-11-30 13:15:58 +0200985 intel_uncore_write(&dev_priv->uncore, DSPFW2,
Ville Syrjälä04548cb2017-04-21 21:14:29 +0300986 (wm->fbc_en ? DSPFW_FBC_SR_EN : 0) |
987 FW_WM(wm->sr.fbc, FBC_SR) |
988 FW_WM(wm->hpll.fbc, FBC_HPLL_SR) |
989 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEB) |
990 FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
991 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
Jani Nikula5f461662020-11-30 13:15:58 +0200992 intel_uncore_write(&dev_priv->uncore, DSPFW3,
Ville Syrjälä04548cb2017-04-21 21:14:29 +0300993 (wm->hpll_en ? DSPFW_HPLL_SR_EN : 0) |
994 FW_WM(wm->sr.cursor, CURSOR_SR) |
995 FW_WM(wm->hpll.cursor, HPLL_CURSOR) |
996 FW_WM(wm->hpll.plane, HPLL_SR));
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300997
Jani Nikula5f461662020-11-30 13:15:58 +0200998 intel_uncore_posting_read(&dev_priv->uncore, DSPFW1);
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -0300999}
1000
Ville Syrjälä15665972015-03-10 16:16:28 +02001001#define FW_WM_VLV(value, plane) \
1002 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV)
1003
Ville Syrjälä50f4cae2016-11-28 19:37:15 +02001004static void vlv_write_wm_values(struct drm_i915_private *dev_priv,
Ville Syrjälä0018fda2015-03-05 21:19:45 +02001005 const struct vlv_wm_values *wm)
1006{
Ville Syrjälä50f4cae2016-11-28 19:37:15 +02001007 enum pipe pipe;
Ville Syrjälä0018fda2015-03-05 21:19:45 +02001008
Ville Syrjälä50f4cae2016-11-28 19:37:15 +02001009 for_each_pipe(dev_priv, pipe) {
Ville Syrjäläc137d662017-03-02 19:15:06 +02001010 trace_vlv_wm(intel_get_crtc_for_pipe(dev_priv, pipe), wm);
1011
Jani Nikula5f461662020-11-30 13:15:58 +02001012 intel_uncore_write(&dev_priv->uncore, VLV_DDL(pipe),
Ville Syrjälä50f4cae2016-11-28 19:37:15 +02001013 (wm->ddl[pipe].plane[PLANE_CURSOR] << DDL_CURSOR_SHIFT) |
1014 (wm->ddl[pipe].plane[PLANE_SPRITE1] << DDL_SPRITE_SHIFT(1)) |
1015 (wm->ddl[pipe].plane[PLANE_SPRITE0] << DDL_SPRITE_SHIFT(0)) |
1016 (wm->ddl[pipe].plane[PLANE_PRIMARY] << DDL_PLANE_SHIFT));
1017 }
Ville Syrjälä0018fda2015-03-05 21:19:45 +02001018
Ville Syrjälä6fe6a7f2016-11-28 19:37:14 +02001019 /*
1020 * Zero the (unused) WM1 watermarks, and also clear all the
1021 * high order bits so that there are no out of bounds values
1022 * present in the registers during the reprogramming.
1023 */
Jani Nikula5f461662020-11-30 13:15:58 +02001024 intel_uncore_write(&dev_priv->uncore, DSPHOWM, 0);
1025 intel_uncore_write(&dev_priv->uncore, DSPHOWM1, 0);
1026 intel_uncore_write(&dev_priv->uncore, DSPFW4, 0);
1027 intel_uncore_write(&dev_priv->uncore, DSPFW5, 0);
1028 intel_uncore_write(&dev_priv->uncore, DSPFW6, 0);
Ville Syrjälä6fe6a7f2016-11-28 19:37:14 +02001029
Jani Nikula5f461662020-11-30 13:15:58 +02001030 intel_uncore_write(&dev_priv->uncore, DSPFW1,
Ville Syrjälä15665972015-03-10 16:16:28 +02001031 FW_WM(wm->sr.plane, SR) |
Ville Syrjälä1b313892016-11-28 19:37:08 +02001032 FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
1033 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
1034 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
Jani Nikula5f461662020-11-30 13:15:58 +02001035 intel_uncore_write(&dev_priv->uncore, DSPFW2,
Ville Syrjälä1b313892016-11-28 19:37:08 +02001036 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE1], SPRITEB) |
1037 FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
1038 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
Jani Nikula5f461662020-11-30 13:15:58 +02001039 intel_uncore_write(&dev_priv->uncore, DSPFW3,
Ville Syrjälä15665972015-03-10 16:16:28 +02001040 FW_WM(wm->sr.cursor, CURSOR_SR));
Ville Syrjäläae801522015-03-05 21:19:49 +02001041
1042 if (IS_CHERRYVIEW(dev_priv)) {
Jani Nikula5f461662020-11-30 13:15:58 +02001043 intel_uncore_write(&dev_priv->uncore, DSPFW7_CHV,
Ville Syrjälä1b313892016-11-28 19:37:08 +02001044 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1045 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
Jani Nikula5f461662020-11-30 13:15:58 +02001046 intel_uncore_write(&dev_priv->uncore, DSPFW8_CHV,
Ville Syrjälä1b313892016-11-28 19:37:08 +02001047 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE1], SPRITEF) |
1048 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE0], SPRITEE));
Jani Nikula5f461662020-11-30 13:15:58 +02001049 intel_uncore_write(&dev_priv->uncore, DSPFW9_CHV,
Ville Syrjälä1b313892016-11-28 19:37:08 +02001050 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_PRIMARY], PLANEC) |
1051 FW_WM(wm->pipe[PIPE_C].plane[PLANE_CURSOR], CURSORC));
Jani Nikula5f461662020-11-30 13:15:58 +02001052 intel_uncore_write(&dev_priv->uncore, DSPHOWM,
Ville Syrjälä15665972015-03-10 16:16:28 +02001053 FW_WM(wm->sr.plane >> 9, SR_HI) |
Ville Syrjälä1b313892016-11-28 19:37:08 +02001054 FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE1] >> 8, SPRITEF_HI) |
1055 FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE0] >> 8, SPRITEE_HI) |
1056 FW_WM(wm->pipe[PIPE_C].plane[PLANE_PRIMARY] >> 8, PLANEC_HI) |
1057 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1058 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1059 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1060 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1061 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1062 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
Ville Syrjäläae801522015-03-05 21:19:49 +02001063 } else {
Jani Nikula5f461662020-11-30 13:15:58 +02001064 intel_uncore_write(&dev_priv->uncore, DSPFW7,
Ville Syrjälä1b313892016-11-28 19:37:08 +02001065 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1066 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
Jani Nikula5f461662020-11-30 13:15:58 +02001067 intel_uncore_write(&dev_priv->uncore, DSPHOWM,
Ville Syrjälä15665972015-03-10 16:16:28 +02001068 FW_WM(wm->sr.plane >> 9, SR_HI) |
Ville Syrjälä1b313892016-11-28 19:37:08 +02001069 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1070 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1071 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1072 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1073 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1074 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
Ville Syrjäläae801522015-03-05 21:19:49 +02001075 }
1076
Jani Nikula5f461662020-11-30 13:15:58 +02001077 intel_uncore_posting_read(&dev_priv->uncore, DSPFW1);
Ville Syrjälä0018fda2015-03-05 21:19:45 +02001078}
1079
Ville Syrjälä15665972015-03-10 16:16:28 +02001080#undef FW_WM_VLV
1081
Ville Syrjälä04548cb2017-04-21 21:14:29 +03001082static void g4x_setup_wm_latency(struct drm_i915_private *dev_priv)
1083{
1084 /* all latencies in usec */
1085 dev_priv->wm.pri_latency[G4X_WM_LEVEL_NORMAL] = 5;
1086 dev_priv->wm.pri_latency[G4X_WM_LEVEL_SR] = 12;
Ville Syrjälä79d94302017-04-21 21:14:30 +03001087 dev_priv->wm.pri_latency[G4X_WM_LEVEL_HPLL] = 35;
Ville Syrjälä04548cb2017-04-21 21:14:29 +03001088
Ville Syrjälä79d94302017-04-21 21:14:30 +03001089 dev_priv->wm.max_level = G4X_WM_LEVEL_HPLL;
Ville Syrjälä04548cb2017-04-21 21:14:29 +03001090}
1091
1092static int g4x_plane_fifo_size(enum plane_id plane_id, int level)
1093{
1094 /*
1095 * DSPCNTR[13] supposedly controls whether the
1096 * primary plane can use the FIFO space otherwise
1097 * reserved for the sprite plane. It's not 100% clear
1098 * what the actual FIFO size is, but it looks like we
1099 * can happily set both primary and sprite watermarks
1100 * up to 127 cachelines. So that would seem to mean
1101 * that either DSPCNTR[13] doesn't do anything, or that
1102 * the total FIFO is >= 256 cachelines in size. Either
1103 * way, we don't seem to have to worry about this
1104 * repartitioning as the maximum watermark value the
1105 * register can hold for each plane is lower than the
1106 * minimum FIFO size.
1107 */
1108 switch (plane_id) {
1109 case PLANE_CURSOR:
1110 return 63;
1111 case PLANE_PRIMARY:
1112 return level == G4X_WM_LEVEL_NORMAL ? 127 : 511;
1113 case PLANE_SPRITE0:
1114 return level == G4X_WM_LEVEL_NORMAL ? 127 : 0;
1115 default:
1116 MISSING_CASE(plane_id);
1117 return 0;
1118 }
1119}
1120
1121static int g4x_fbc_fifo_size(int level)
1122{
1123 switch (level) {
1124 case G4X_WM_LEVEL_SR:
1125 return 7;
1126 case G4X_WM_LEVEL_HPLL:
1127 return 15;
1128 default:
1129 MISSING_CASE(level);
1130 return 0;
1131 }
1132}
1133
Jani Nikula5ce9a6492019-01-18 14:01:20 +02001134static u16 g4x_compute_wm(const struct intel_crtc_state *crtc_state,
1135 const struct intel_plane_state *plane_state,
1136 int level)
Ville Syrjälä04548cb2017-04-21 21:14:29 +03001137{
Maarten Lankhorstf90a85e2019-10-31 12:26:08 +01001138 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
Ville Syrjälä04548cb2017-04-21 21:14:29 +03001139 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
Maarten Lankhorstbafcdad2020-11-12 21:17:18 +02001140 const struct drm_display_mode *pipe_mode =
1141 &crtc_state->hw.pipe_mode;
Chris Wilson1a1f1282017-11-07 14:03:38 +00001142 unsigned int latency = dev_priv->wm.pri_latency[level] * 10;
1143 unsigned int clock, htotal, cpp, width, wm;
Ville Syrjälä04548cb2017-04-21 21:14:29 +03001144
1145 if (latency == 0)
1146 return USHRT_MAX;
1147
1148 if (!intel_wm_plane_visible(crtc_state, plane_state))
1149 return 0;
1150
Maarten Lankhorst7b3cb172019-10-31 12:26:07 +01001151 cpp = plane_state->hw.fb->format->cpp[0];
Ville Syrjäläd56e8232019-07-03 23:08:22 +03001152
Ville Syrjälä04548cb2017-04-21 21:14:29 +03001153 /*
1154 * Not 100% sure which way ELK should go here as the
1155 * spec only says CL/CTG should assume 32bpp and BW
1156 * doesn't need to. But as these things followed the
1157 * mobile vs. desktop lines on gen3 as well, let's
1158 * assume ELK doesn't need this.
1159 *
1160 * The spec also fails to list such a restriction for
1161 * the HPLL watermark, which seems a little strange.
1162 * Let's use 32bpp for the HPLL watermark as well.
1163 */
1164 if (IS_GM45(dev_priv) && plane->id == PLANE_PRIMARY &&
1165 level != G4X_WM_LEVEL_NORMAL)
Ville Syrjäläd56e8232019-07-03 23:08:22 +03001166 cpp = max(cpp, 4u);
Ville Syrjälä04548cb2017-04-21 21:14:29 +03001167
Maarten Lankhorstbafcdad2020-11-12 21:17:18 +02001168 clock = pipe_mode->crtc_clock;
1169 htotal = pipe_mode->crtc_htotal;
Ville Syrjälä04548cb2017-04-21 21:14:29 +03001170
Maarten Lankhorstf90a85e2019-10-31 12:26:08 +01001171 width = drm_rect_width(&plane_state->uapi.dst);
Ville Syrjälä04548cb2017-04-21 21:14:29 +03001172
1173 if (plane->id == PLANE_CURSOR) {
1174 wm = intel_wm_method2(clock, htotal, width, cpp, latency);
1175 } else if (plane->id == PLANE_PRIMARY &&
1176 level == G4X_WM_LEVEL_NORMAL) {
1177 wm = intel_wm_method1(clock, cpp, latency);
1178 } else {
Chris Wilson1a1f1282017-11-07 14:03:38 +00001179 unsigned int small, large;
Ville Syrjälä04548cb2017-04-21 21:14:29 +03001180
1181 small = intel_wm_method1(clock, cpp, latency);
1182 large = intel_wm_method2(clock, htotal, width, cpp, latency);
1183
1184 wm = min(small, large);
1185 }
1186
1187 wm += g4x_tlb_miss_wa(g4x_plane_fifo_size(plane->id, level),
1188 width, cpp);
1189
1190 wm = DIV_ROUND_UP(wm, 64) + 2;
1191
Chris Wilson1a1f1282017-11-07 14:03:38 +00001192 return min_t(unsigned int, wm, USHRT_MAX);
Ville Syrjälä04548cb2017-04-21 21:14:29 +03001193}
1194
1195static bool g4x_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1196 int level, enum plane_id plane_id, u16 value)
1197{
Maarten Lankhorst2225f3c2019-10-31 12:26:03 +01001198 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
Ville Syrjälä04548cb2017-04-21 21:14:29 +03001199 bool dirty = false;
1200
1201 for (; level < intel_wm_num_levels(dev_priv); level++) {
1202 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1203
1204 dirty |= raw->plane[plane_id] != value;
1205 raw->plane[plane_id] = value;
1206 }
1207
1208 return dirty;
1209}
1210
1211static bool g4x_raw_fbc_wm_set(struct intel_crtc_state *crtc_state,
1212 int level, u16 value)
1213{
Maarten Lankhorst2225f3c2019-10-31 12:26:03 +01001214 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
Ville Syrjälä04548cb2017-04-21 21:14:29 +03001215 bool dirty = false;
1216
1217 /* NORMAL level doesn't have an FBC watermark */
1218 level = max(level, G4X_WM_LEVEL_SR);
1219
1220 for (; level < intel_wm_num_levels(dev_priv); level++) {
1221 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1222
1223 dirty |= raw->fbc != value;
1224 raw->fbc = value;
1225 }
1226
1227 return dirty;
1228}
1229
Maarten Lankhorstec193642019-06-28 10:55:17 +02001230static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
1231 const struct intel_plane_state *plane_state,
Jani Nikula5ce9a6492019-01-18 14:01:20 +02001232 u32 pri_val);
Ville Syrjälä04548cb2017-04-21 21:14:29 +03001233
1234static bool g4x_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1235 const struct intel_plane_state *plane_state)
1236{
Maarten Lankhorstf90a85e2019-10-31 12:26:08 +01001237 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
Wambui Karugaf8d18d52020-01-07 18:13:30 +03001238 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
Ville Syrjälä04548cb2017-04-21 21:14:29 +03001239 int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1240 enum plane_id plane_id = plane->id;
1241 bool dirty = false;
1242 int level;
1243
1244 if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1245 dirty |= g4x_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1246 if (plane_id == PLANE_PRIMARY)
1247 dirty |= g4x_raw_fbc_wm_set(crtc_state, 0, 0);
1248 goto out;
1249 }
1250
1251 for (level = 0; level < num_levels; level++) {
1252 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1253 int wm, max_wm;
1254
1255 wm = g4x_compute_wm(crtc_state, plane_state, level);
1256 max_wm = g4x_plane_fifo_size(plane_id, level);
1257
1258 if (wm > max_wm)
1259 break;
1260
1261 dirty |= raw->plane[plane_id] != wm;
1262 raw->plane[plane_id] = wm;
1263
1264 if (plane_id != PLANE_PRIMARY ||
1265 level == G4X_WM_LEVEL_NORMAL)
1266 continue;
1267
1268 wm = ilk_compute_fbc_wm(crtc_state, plane_state,
1269 raw->plane[plane_id]);
1270 max_wm = g4x_fbc_fifo_size(level);
1271
1272 /*
1273 * FBC wm is not mandatory as we
1274 * can always just disable its use.
1275 */
1276 if (wm > max_wm)
1277 wm = USHRT_MAX;
1278
1279 dirty |= raw->fbc != wm;
1280 raw->fbc = wm;
1281 }
1282
1283 /* mark watermarks as invalid */
1284 dirty |= g4x_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1285
1286 if (plane_id == PLANE_PRIMARY)
1287 dirty |= g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
1288
1289 out:
1290 if (dirty) {
Wambui Karugaf8d18d52020-01-07 18:13:30 +03001291 drm_dbg_kms(&dev_priv->drm,
1292 "%s watermarks: normal=%d, SR=%d, HPLL=%d\n",
1293 plane->base.name,
1294 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_NORMAL].plane[plane_id],
1295 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].plane[plane_id],
1296 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].plane[plane_id]);
Ville Syrjälä04548cb2017-04-21 21:14:29 +03001297
1298 if (plane_id == PLANE_PRIMARY)
Wambui Karugaf8d18d52020-01-07 18:13:30 +03001299 drm_dbg_kms(&dev_priv->drm,
1300 "FBC watermarks: SR=%d, HPLL=%d\n",
1301 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].fbc,
1302 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].fbc);
Ville Syrjälä04548cb2017-04-21 21:14:29 +03001303 }
1304
1305 return dirty;
1306}
1307
1308static bool g4x_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1309 enum plane_id plane_id, int level)
1310{
1311 const struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1312
1313 return raw->plane[plane_id] <= g4x_plane_fifo_size(plane_id, level);
1314}
1315
1316static bool g4x_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state,
1317 int level)
1318{
Maarten Lankhorst2225f3c2019-10-31 12:26:03 +01001319 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
Ville Syrjälä04548cb2017-04-21 21:14:29 +03001320
1321 if (level > dev_priv->wm.max_level)
1322 return false;
1323
1324 return g4x_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1325 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1326 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1327}
1328
1329/* mark all levels starting from 'level' as invalid */
1330static void g4x_invalidate_wms(struct intel_crtc *crtc,
1331 struct g4x_wm_state *wm_state, int level)
1332{
1333 if (level <= G4X_WM_LEVEL_NORMAL) {
1334 enum plane_id plane_id;
1335
1336 for_each_plane_id_on_crtc(crtc, plane_id)
1337 wm_state->wm.plane[plane_id] = USHRT_MAX;
1338 }
1339
1340 if (level <= G4X_WM_LEVEL_SR) {
1341 wm_state->cxsr = false;
1342 wm_state->sr.cursor = USHRT_MAX;
1343 wm_state->sr.plane = USHRT_MAX;
1344 wm_state->sr.fbc = USHRT_MAX;
1345 }
1346
1347 if (level <= G4X_WM_LEVEL_HPLL) {
1348 wm_state->hpll_en = false;
1349 wm_state->hpll.cursor = USHRT_MAX;
1350 wm_state->hpll.plane = USHRT_MAX;
1351 wm_state->hpll.fbc = USHRT_MAX;
1352 }
1353}
1354
Ville Syrjäläfd7a9d82020-04-29 13:10:33 +03001355static bool g4x_compute_fbc_en(const struct g4x_wm_state *wm_state,
1356 int level)
1357{
1358 if (level < G4X_WM_LEVEL_SR)
1359 return false;
1360
1361 if (level >= G4X_WM_LEVEL_SR &&
1362 wm_state->sr.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_SR))
1363 return false;
1364
1365 if (level >= G4X_WM_LEVEL_HPLL &&
1366 wm_state->hpll.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_HPLL))
1367 return false;
1368
1369 return true;
1370}
1371
Ville Syrjälä04548cb2017-04-21 21:14:29 +03001372static int g4x_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1373{
Maarten Lankhorst2225f3c2019-10-31 12:26:03 +01001374 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
Ville Syrjälä04548cb2017-04-21 21:14:29 +03001375 struct intel_atomic_state *state =
Maarten Lankhorst2225f3c2019-10-31 12:26:03 +01001376 to_intel_atomic_state(crtc_state->uapi.state);
Ville Syrjälä04548cb2017-04-21 21:14:29 +03001377 struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
Ville Syrjälä0b14d962019-08-21 20:30:33 +03001378 int num_active_planes = hweight8(crtc_state->active_planes &
1379 ~BIT(PLANE_CURSOR));
Ville Syrjälä04548cb2017-04-21 21:14:29 +03001380 const struct g4x_pipe_wm *raw;
Ville Syrjälä7b5104512017-08-23 18:22:22 +03001381 const struct intel_plane_state *old_plane_state;
1382 const struct intel_plane_state *new_plane_state;
Ville Syrjälä04548cb2017-04-21 21:14:29 +03001383 struct intel_plane *plane;
1384 enum plane_id plane_id;
1385 int i, level;
1386 unsigned int dirty = 0;
1387
Ville Syrjälä7b5104512017-08-23 18:22:22 +03001388 for_each_oldnew_intel_plane_in_state(state, plane,
1389 old_plane_state,
1390 new_plane_state, i) {
Maarten Lankhorst7b3cb172019-10-31 12:26:07 +01001391 if (new_plane_state->hw.crtc != &crtc->base &&
1392 old_plane_state->hw.crtc != &crtc->base)
Ville Syrjälä04548cb2017-04-21 21:14:29 +03001393 continue;
1394
Ville Syrjälä7b5104512017-08-23 18:22:22 +03001395 if (g4x_raw_plane_wm_compute(crtc_state, new_plane_state))
Ville Syrjälä04548cb2017-04-21 21:14:29 +03001396 dirty |= BIT(plane->id);
1397 }
1398
1399 if (!dirty)
1400 return 0;
1401
1402 level = G4X_WM_LEVEL_NORMAL;
1403 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1404 goto out;
1405
1406 raw = &crtc_state->wm.g4x.raw[level];
1407 for_each_plane_id_on_crtc(crtc, plane_id)
1408 wm_state->wm.plane[plane_id] = raw->plane[plane_id];
1409
1410 level = G4X_WM_LEVEL_SR;
Ville Syrjälä04548cb2017-04-21 21:14:29 +03001411 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1412 goto out;
1413
1414 raw = &crtc_state->wm.g4x.raw[level];
1415 wm_state->sr.plane = raw->plane[PLANE_PRIMARY];
1416 wm_state->sr.cursor = raw->plane[PLANE_CURSOR];
1417 wm_state->sr.fbc = raw->fbc;
1418
1419 wm_state->cxsr = num_active_planes == BIT(PLANE_PRIMARY);
1420
1421 level = G4X_WM_LEVEL_HPLL;
Ville Syrjälä04548cb2017-04-21 21:14:29 +03001422 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1423 goto out;
1424
1425 raw = &crtc_state->wm.g4x.raw[level];
1426 wm_state->hpll.plane = raw->plane[PLANE_PRIMARY];
1427 wm_state->hpll.cursor = raw->plane[PLANE_CURSOR];
1428 wm_state->hpll.fbc = raw->fbc;
1429
1430 wm_state->hpll_en = wm_state->cxsr;
1431
1432 level++;
1433
1434 out:
1435 if (level == G4X_WM_LEVEL_NORMAL)
1436 return -EINVAL;
1437
1438 /* invalidate the higher levels */
1439 g4x_invalidate_wms(crtc, wm_state, level);
1440
1441 /*
1442 * Determine if the FBC watermark(s) can be used. IF
1443 * this isn't the case we prefer to disable the FBC
Ville Syrjäläfd7a9d82020-04-29 13:10:33 +03001444 * watermark(s) rather than disable the SR/HPLL
1445 * level(s) entirely. 'level-1' is the highest valid
1446 * level here.
Ville Syrjälä04548cb2017-04-21 21:14:29 +03001447 */
Ville Syrjäläfd7a9d82020-04-29 13:10:33 +03001448 wm_state->fbc_en = g4x_compute_fbc_en(wm_state, level - 1);
Ville Syrjälä04548cb2017-04-21 21:14:29 +03001449
1450 return 0;
1451}
1452
Matt Ropercd1d3ee2018-12-10 13:54:14 -08001453static int g4x_compute_intermediate_wm(struct intel_crtc_state *new_crtc_state)
Ville Syrjälä04548cb2017-04-21 21:14:29 +03001454{
Maarten Lankhorst2225f3c2019-10-31 12:26:03 +01001455 struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
Pankaj Bharadiya19edeb382020-05-04 23:45:59 +05301456 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
Maarten Lankhorst248c2432017-11-15 17:31:57 +01001457 struct g4x_wm_state *intermediate = &new_crtc_state->wm.g4x.intermediate;
1458 const struct g4x_wm_state *optimal = &new_crtc_state->wm.g4x.optimal;
1459 struct intel_atomic_state *intel_state =
Maarten Lankhorst2225f3c2019-10-31 12:26:03 +01001460 to_intel_atomic_state(new_crtc_state->uapi.state);
Maarten Lankhorst248c2432017-11-15 17:31:57 +01001461 const struct intel_crtc_state *old_crtc_state =
1462 intel_atomic_get_old_crtc_state(intel_state, crtc);
1463 const struct g4x_wm_state *active = &old_crtc_state->wm.g4x.optimal;
Ville Syrjälä04548cb2017-04-21 21:14:29 +03001464 enum plane_id plane_id;
1465
Maarten Lankhorst2225f3c2019-10-31 12:26:03 +01001466 if (!new_crtc_state->hw.active || drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi)) {
Maarten Lankhorst248c2432017-11-15 17:31:57 +01001467 *intermediate = *optimal;
1468
1469 intermediate->cxsr = false;
1470 intermediate->hpll_en = false;
1471 goto out;
1472 }
1473
Ville Syrjälä04548cb2017-04-21 21:14:29 +03001474 intermediate->cxsr = optimal->cxsr && active->cxsr &&
Maarten Lankhorst248c2432017-11-15 17:31:57 +01001475 !new_crtc_state->disable_cxsr;
Ville Syrjälä04548cb2017-04-21 21:14:29 +03001476 intermediate->hpll_en = optimal->hpll_en && active->hpll_en &&
Maarten Lankhorst248c2432017-11-15 17:31:57 +01001477 !new_crtc_state->disable_cxsr;
Ville Syrjälä04548cb2017-04-21 21:14:29 +03001478 intermediate->fbc_en = optimal->fbc_en && active->fbc_en;
1479
1480 for_each_plane_id_on_crtc(crtc, plane_id) {
1481 intermediate->wm.plane[plane_id] =
1482 max(optimal->wm.plane[plane_id],
1483 active->wm.plane[plane_id]);
1484
Pankaj Bharadiya19edeb382020-05-04 23:45:59 +05301485 drm_WARN_ON(&dev_priv->drm, intermediate->wm.plane[plane_id] >
1486 g4x_plane_fifo_size(plane_id, G4X_WM_LEVEL_NORMAL));
Ville Syrjälä04548cb2017-04-21 21:14:29 +03001487 }
1488
1489 intermediate->sr.plane = max(optimal->sr.plane,
1490 active->sr.plane);
1491 intermediate->sr.cursor = max(optimal->sr.cursor,
1492 active->sr.cursor);
1493 intermediate->sr.fbc = max(optimal->sr.fbc,
1494 active->sr.fbc);
1495
1496 intermediate->hpll.plane = max(optimal->hpll.plane,
1497 active->hpll.plane);
1498 intermediate->hpll.cursor = max(optimal->hpll.cursor,
1499 active->hpll.cursor);
1500 intermediate->hpll.fbc = max(optimal->hpll.fbc,
1501 active->hpll.fbc);
1502
Pankaj Bharadiya19edeb382020-05-04 23:45:59 +05301503 drm_WARN_ON(&dev_priv->drm,
1504 (intermediate->sr.plane >
1505 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_SR) ||
1506 intermediate->sr.cursor >
1507 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_SR)) &&
1508 intermediate->cxsr);
1509 drm_WARN_ON(&dev_priv->drm,
1510 (intermediate->sr.plane >
1511 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_HPLL) ||
1512 intermediate->sr.cursor >
1513 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_HPLL)) &&
1514 intermediate->hpll_en);
Ville Syrjälä04548cb2017-04-21 21:14:29 +03001515
Pankaj Bharadiya19edeb382020-05-04 23:45:59 +05301516 drm_WARN_ON(&dev_priv->drm,
1517 intermediate->sr.fbc > g4x_fbc_fifo_size(1) &&
1518 intermediate->fbc_en && intermediate->cxsr);
1519 drm_WARN_ON(&dev_priv->drm,
1520 intermediate->hpll.fbc > g4x_fbc_fifo_size(2) &&
1521 intermediate->fbc_en && intermediate->hpll_en);
Ville Syrjälä04548cb2017-04-21 21:14:29 +03001522
Maarten Lankhorst248c2432017-11-15 17:31:57 +01001523out:
Ville Syrjälä04548cb2017-04-21 21:14:29 +03001524 /*
1525 * If our intermediate WM are identical to the final WM, then we can
1526 * omit the post-vblank programming; only update if it's different.
1527 */
1528 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
Maarten Lankhorst248c2432017-11-15 17:31:57 +01001529 new_crtc_state->wm.need_postvbl_update = true;
Ville Syrjälä04548cb2017-04-21 21:14:29 +03001530
1531 return 0;
1532}
1533
1534static void g4x_merge_wm(struct drm_i915_private *dev_priv,
1535 struct g4x_wm_values *wm)
1536{
1537 struct intel_crtc *crtc;
Ville Syrjäläc08e9132019-08-21 20:30:32 +03001538 int num_active_pipes = 0;
Ville Syrjälä04548cb2017-04-21 21:14:29 +03001539
1540 wm->cxsr = true;
1541 wm->hpll_en = true;
1542 wm->fbc_en = true;
1543
1544 for_each_intel_crtc(&dev_priv->drm, crtc) {
1545 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1546
1547 if (!crtc->active)
1548 continue;
1549
1550 if (!wm_state->cxsr)
1551 wm->cxsr = false;
1552 if (!wm_state->hpll_en)
1553 wm->hpll_en = false;
1554 if (!wm_state->fbc_en)
1555 wm->fbc_en = false;
1556
Ville Syrjäläc08e9132019-08-21 20:30:32 +03001557 num_active_pipes++;
Ville Syrjälä04548cb2017-04-21 21:14:29 +03001558 }
1559
Ville Syrjäläc08e9132019-08-21 20:30:32 +03001560 if (num_active_pipes != 1) {
Ville Syrjälä04548cb2017-04-21 21:14:29 +03001561 wm->cxsr = false;
1562 wm->hpll_en = false;
1563 wm->fbc_en = false;
1564 }
1565
1566 for_each_intel_crtc(&dev_priv->drm, crtc) {
1567 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1568 enum pipe pipe = crtc->pipe;
1569
1570 wm->pipe[pipe] = wm_state->wm;
1571 if (crtc->active && wm->cxsr)
1572 wm->sr = wm_state->sr;
1573 if (crtc->active && wm->hpll_en)
1574 wm->hpll = wm_state->hpll;
1575 }
1576}
1577
1578static void g4x_program_watermarks(struct drm_i915_private *dev_priv)
1579{
1580 struct g4x_wm_values *old_wm = &dev_priv->wm.g4x;
1581 struct g4x_wm_values new_wm = {};
1582
1583 g4x_merge_wm(dev_priv, &new_wm);
1584
1585 if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
1586 return;
1587
1588 if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
1589 _intel_set_memory_cxsr(dev_priv, false);
1590
1591 g4x_write_wm_values(dev_priv, &new_wm);
1592
1593 if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
1594 _intel_set_memory_cxsr(dev_priv, true);
1595
1596 *old_wm = new_wm;
1597}
1598
1599static void g4x_initial_watermarks(struct intel_atomic_state *state,
Ville Syrjälä7a8fdb1f2019-11-18 18:44:26 +02001600 struct intel_crtc *crtc)
Ville Syrjälä04548cb2017-04-21 21:14:29 +03001601{
Ville Syrjälä7a8fdb1f2019-11-18 18:44:26 +02001602 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1603 const struct intel_crtc_state *crtc_state =
1604 intel_atomic_get_new_crtc_state(state, crtc);
Ville Syrjälä04548cb2017-04-21 21:14:29 +03001605
1606 mutex_lock(&dev_priv->wm.wm_mutex);
1607 crtc->wm.active.g4x = crtc_state->wm.g4x.intermediate;
1608 g4x_program_watermarks(dev_priv);
1609 mutex_unlock(&dev_priv->wm.wm_mutex);
1610}
1611
1612static void g4x_optimize_watermarks(struct intel_atomic_state *state,
Ville Syrjälä7a8fdb1f2019-11-18 18:44:26 +02001613 struct intel_crtc *crtc)
Ville Syrjälä04548cb2017-04-21 21:14:29 +03001614{
Ville Syrjälä7a8fdb1f2019-11-18 18:44:26 +02001615 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1616 const struct intel_crtc_state *crtc_state =
1617 intel_atomic_get_new_crtc_state(state, crtc);
Ville Syrjälä04548cb2017-04-21 21:14:29 +03001618
1619 if (!crtc_state->wm.need_postvbl_update)
1620 return;
1621
1622 mutex_lock(&dev_priv->wm.wm_mutex);
Ville Syrjälä88016a92019-07-01 19:05:45 +03001623 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
Ville Syrjälä04548cb2017-04-21 21:14:29 +03001624 g4x_program_watermarks(dev_priv);
1625 mutex_unlock(&dev_priv->wm.wm_mutex);
1626}
1627
Ville Syrjälä262cd2e2015-06-24 22:00:04 +03001628/* latency must be in 0.1us units. */
1629static unsigned int vlv_wm_method2(unsigned int pixel_rate,
Ville Syrjäläbaf69ca2017-04-21 21:14:27 +03001630 unsigned int htotal,
1631 unsigned int width,
Ville Syrjäläac484962016-01-20 21:05:26 +02001632 unsigned int cpp,
Ville Syrjälä262cd2e2015-06-24 22:00:04 +03001633 unsigned int latency)
1634{
1635 unsigned int ret;
1636
Ville Syrjäläbaf69ca2017-04-21 21:14:27 +03001637 ret = intel_wm_method2(pixel_rate, htotal,
1638 width, cpp, latency);
Ville Syrjälä262cd2e2015-06-24 22:00:04 +03001639 ret = DIV_ROUND_UP(ret, 64);
1640
1641 return ret;
1642}
1643
Ville Syrjäläbb726512016-10-31 22:37:24 +02001644static void vlv_setup_wm_latency(struct drm_i915_private *dev_priv)
Ville Syrjälä262cd2e2015-06-24 22:00:04 +03001645{
Ville Syrjälä262cd2e2015-06-24 22:00:04 +03001646 /* all latencies in usec */
1647 dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;
1648
Ville Syrjälä58590c12015-09-08 21:05:12 +03001649 dev_priv->wm.max_level = VLV_WM_LEVEL_PM2;
1650
Ville Syrjälä262cd2e2015-06-24 22:00:04 +03001651 if (IS_CHERRYVIEW(dev_priv)) {
1652 dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;
1653 dev_priv->wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;
Ville Syrjälä58590c12015-09-08 21:05:12 +03001654
1655 dev_priv->wm.max_level = VLV_WM_LEVEL_DDR_DVFS;
Ville Syrjälä262cd2e2015-06-24 22:00:04 +03001656 }
1657}
1658
Jani Nikula5ce9a6492019-01-18 14:01:20 +02001659static u16 vlv_compute_wm_level(const struct intel_crtc_state *crtc_state,
1660 const struct intel_plane_state *plane_state,
1661 int level)
Ville Syrjälä262cd2e2015-06-24 22:00:04 +03001662{
Maarten Lankhorstf90a85e2019-10-31 12:26:08 +01001663 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
Ville Syrjälä262cd2e2015-06-24 22:00:04 +03001664 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
Maarten Lankhorstbafcdad2020-11-12 21:17:18 +02001665 const struct drm_display_mode *pipe_mode =
1666 &crtc_state->hw.pipe_mode;
Chris Wilson1a1f1282017-11-07 14:03:38 +00001667 unsigned int clock, htotal, cpp, width, wm;
Ville Syrjälä262cd2e2015-06-24 22:00:04 +03001668
1669 if (dev_priv->wm.pri_latency[level] == 0)
1670 return USHRT_MAX;
1671
Ville Syrjäläa07102f2017-03-03 17:19:27 +02001672 if (!intel_wm_plane_visible(crtc_state, plane_state))
Ville Syrjälä262cd2e2015-06-24 22:00:04 +03001673 return 0;
1674
Maarten Lankhorst7b3cb172019-10-31 12:26:07 +01001675 cpp = plane_state->hw.fb->format->cpp[0];
Maarten Lankhorstbafcdad2020-11-12 21:17:18 +02001676 clock = pipe_mode->crtc_clock;
1677 htotal = pipe_mode->crtc_htotal;
Ville Syrjäläe339d672016-11-28 19:37:17 +02001678 width = crtc_state->pipe_src_w;
Ville Syrjälä262cd2e2015-06-24 22:00:04 +03001679
Ville Syrjälä709f3fc2017-03-03 17:19:26 +02001680 if (plane->id == PLANE_CURSOR) {
Ville Syrjälä262cd2e2015-06-24 22:00:04 +03001681 /*
1682 * FIXME the formula gives values that are
1683 * too big for the cursor FIFO, and hence we
1684 * would never be able to use cursors. For
1685 * now just hardcode the watermark.
1686 */
1687 wm = 63;
1688 } else {
Ville Syrjäläac484962016-01-20 21:05:26 +02001689 wm = vlv_wm_method2(clock, htotal, width, cpp,
Ville Syrjälä262cd2e2015-06-24 22:00:04 +03001690 dev_priv->wm.pri_latency[level] * 10);
1691 }
1692
Chris Wilson1a1f1282017-11-07 14:03:38 +00001693 return min_t(unsigned int, wm, USHRT_MAX);
Ville Syrjälä262cd2e2015-06-24 22:00:04 +03001694}
1695
Ville Syrjälä1a10ae62017-03-02 19:15:03 +02001696static bool vlv_need_sprite0_fifo_workaround(unsigned int active_planes)
1697{
1698 return (active_planes & (BIT(PLANE_SPRITE0) |
1699 BIT(PLANE_SPRITE1))) == BIT(PLANE_SPRITE1);
1700}
1701
Ville Syrjälä5012e602017-03-02 19:14:56 +02001702static int vlv_compute_fifo(struct intel_crtc_state *crtc_state)
Ville Syrjälä54f1b6e2015-06-24 22:00:05 +03001703{
Maarten Lankhorst2225f3c2019-10-31 12:26:03 +01001704 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
Pankaj Bharadiya19edeb382020-05-04 23:45:59 +05301705 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
Ville Syrjälä114d7dc2017-04-21 21:14:21 +03001706 const struct g4x_pipe_wm *raw =
Ville Syrjälä5012e602017-03-02 19:14:56 +02001707 &crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2];
Ville Syrjälä814e7f02017-03-02 19:14:55 +02001708 struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
Ville Syrjälä5012e602017-03-02 19:14:56 +02001709 unsigned int active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
Ville Syrjälä0b14d962019-08-21 20:30:33 +03001710 int num_active_planes = hweight8(active_planes);
Ville Syrjälä5012e602017-03-02 19:14:56 +02001711 const int fifo_size = 511;
Ville Syrjälä54f1b6e2015-06-24 22:00:05 +03001712 int fifo_extra, fifo_left = fifo_size;
Ville Syrjälä1a10ae62017-03-02 19:15:03 +02001713 int sprite0_fifo_extra = 0;
Ville Syrjälä5012e602017-03-02 19:14:56 +02001714 unsigned int total_rate;
1715 enum plane_id plane_id;
Ville Syrjälä54f1b6e2015-06-24 22:00:05 +03001716
Ville Syrjälä1a10ae62017-03-02 19:15:03 +02001717 /*
1718 * When enabling sprite0 after sprite1 has already been enabled
1719 * we tend to get an underrun unless sprite0 already has some
1720 * FIFO space allcoated. Hence we always allocate at least one
1721 * cacheline for sprite0 whenever sprite1 is enabled.
1722 *
1723 * All other plane enable sequences appear immune to this problem.
1724 */
1725 if (vlv_need_sprite0_fifo_workaround(active_planes))
1726 sprite0_fifo_extra = 1;
1727
Ville Syrjälä5012e602017-03-02 19:14:56 +02001728 total_rate = raw->plane[PLANE_PRIMARY] +
1729 raw->plane[PLANE_SPRITE0] +
Ville Syrjälä1a10ae62017-03-02 19:15:03 +02001730 raw->plane[PLANE_SPRITE1] +
1731 sprite0_fifo_extra;
Ville Syrjälä54f1b6e2015-06-24 22:00:05 +03001732
Ville Syrjälä5012e602017-03-02 19:14:56 +02001733 if (total_rate > fifo_size)
1734 return -EINVAL;
Ville Syrjälä54f1b6e2015-06-24 22:00:05 +03001735
Ville Syrjälä5012e602017-03-02 19:14:56 +02001736 if (total_rate == 0)
1737 total_rate = 1;
Ville Syrjälä54f1b6e2015-06-24 22:00:05 +03001738
Ville Syrjälä5012e602017-03-02 19:14:56 +02001739 for_each_plane_id_on_crtc(crtc, plane_id) {
Ville Syrjälä54f1b6e2015-06-24 22:00:05 +03001740 unsigned int rate;
1741
Ville Syrjälä5012e602017-03-02 19:14:56 +02001742 if ((active_planes & BIT(plane_id)) == 0) {
1743 fifo_state->plane[plane_id] = 0;
Ville Syrjälä54f1b6e2015-06-24 22:00:05 +03001744 continue;
1745 }
1746
Ville Syrjälä5012e602017-03-02 19:14:56 +02001747 rate = raw->plane[plane_id];
1748 fifo_state->plane[plane_id] = fifo_size * rate / total_rate;
1749 fifo_left -= fifo_state->plane[plane_id];
Ville Syrjälä54f1b6e2015-06-24 22:00:05 +03001750 }
1751
Ville Syrjälä1a10ae62017-03-02 19:15:03 +02001752 fifo_state->plane[PLANE_SPRITE0] += sprite0_fifo_extra;
1753 fifo_left -= sprite0_fifo_extra;
1754
Ville Syrjälä5012e602017-03-02 19:14:56 +02001755 fifo_state->plane[PLANE_CURSOR] = 63;
1756
1757 fifo_extra = DIV_ROUND_UP(fifo_left, num_active_planes ?: 1);
Ville Syrjälä54f1b6e2015-06-24 22:00:05 +03001758
1759 /* spread the remainder evenly */
Ville Syrjälä5012e602017-03-02 19:14:56 +02001760 for_each_plane_id_on_crtc(crtc, plane_id) {
Ville Syrjälä54f1b6e2015-06-24 22:00:05 +03001761 int plane_extra;
1762
1763 if (fifo_left == 0)
1764 break;
1765
Ville Syrjälä5012e602017-03-02 19:14:56 +02001766 if ((active_planes & BIT(plane_id)) == 0)
Ville Syrjälä54f1b6e2015-06-24 22:00:05 +03001767 continue;
1768
1769 plane_extra = min(fifo_extra, fifo_left);
Ville Syrjälä5012e602017-03-02 19:14:56 +02001770 fifo_state->plane[plane_id] += plane_extra;
Ville Syrjälä54f1b6e2015-06-24 22:00:05 +03001771 fifo_left -= plane_extra;
1772 }
1773
Pankaj Bharadiya19edeb382020-05-04 23:45:59 +05301774 drm_WARN_ON(&dev_priv->drm, active_planes != 0 && fifo_left != 0);
Ville Syrjälä5012e602017-03-02 19:14:56 +02001775
1776 /* give it all to the first plane if none are active */
1777 if (active_planes == 0) {
Pankaj Bharadiya19edeb382020-05-04 23:45:59 +05301778 drm_WARN_ON(&dev_priv->drm, fifo_left != fifo_size);
Ville Syrjälä5012e602017-03-02 19:14:56 +02001779 fifo_state->plane[PLANE_PRIMARY] = fifo_left;
1780 }
1781
1782 return 0;
Ville Syrjälä54f1b6e2015-06-24 22:00:05 +03001783}
1784
Ville Syrjäläff32c542017-03-02 19:14:57 +02001785/* mark all levels starting from 'level' as invalid */
1786static void vlv_invalidate_wms(struct intel_crtc *crtc,
1787 struct vlv_wm_state *wm_state, int level)
1788{
1789 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1790
Ville Syrjälä6d5019b2017-04-21 21:14:20 +03001791 for (; level < intel_wm_num_levels(dev_priv); level++) {
Ville Syrjäläff32c542017-03-02 19:14:57 +02001792 enum plane_id plane_id;
1793
1794 for_each_plane_id_on_crtc(crtc, plane_id)
1795 wm_state->wm[level].plane[plane_id] = USHRT_MAX;
1796
1797 wm_state->sr[level].cursor = USHRT_MAX;
1798 wm_state->sr[level].plane = USHRT_MAX;
1799 }
1800}
1801
Ville Syrjälä26cca0e2016-11-28 19:37:09 +02001802static u16 vlv_invert_wm_value(u16 wm, u16 fifo_size)
1803{
1804 if (wm > fifo_size)
1805 return USHRT_MAX;
1806 else
1807 return fifo_size - wm;
1808}
1809
Ville Syrjäläff32c542017-03-02 19:14:57 +02001810/*
1811 * Starting from 'level' set all higher
1812 * levels to 'value' in the "raw" watermarks.
1813 */
Ville Syrjälä236c48e2017-03-02 19:14:58 +02001814static bool vlv_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
Ville Syrjäläff32c542017-03-02 19:14:57 +02001815 int level, enum plane_id plane_id, u16 value)
Ville Syrjälä262cd2e2015-06-24 22:00:04 +03001816{
Maarten Lankhorst2225f3c2019-10-31 12:26:03 +01001817 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
Ville Syrjälä6d5019b2017-04-21 21:14:20 +03001818 int num_levels = intel_wm_num_levels(dev_priv);
Ville Syrjälä236c48e2017-03-02 19:14:58 +02001819 bool dirty = false;
Ville Syrjälä262cd2e2015-06-24 22:00:04 +03001820
Ville Syrjäläff32c542017-03-02 19:14:57 +02001821 for (; level < num_levels; level++) {
Ville Syrjälä114d7dc2017-04-21 21:14:21 +03001822 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
Ville Syrjälä262cd2e2015-06-24 22:00:04 +03001823
Ville Syrjälä236c48e2017-03-02 19:14:58 +02001824 dirty |= raw->plane[plane_id] != value;
Ville Syrjäläff32c542017-03-02 19:14:57 +02001825 raw->plane[plane_id] = value;
Ville Syrjälä262cd2e2015-06-24 22:00:04 +03001826 }
Ville Syrjälä236c48e2017-03-02 19:14:58 +02001827
1828 return dirty;
Ville Syrjälä262cd2e2015-06-24 22:00:04 +03001829}
1830
Ville Syrjälä77d14ee2017-04-21 21:14:18 +03001831static bool vlv_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1832 const struct intel_plane_state *plane_state)
Ville Syrjäläff32c542017-03-02 19:14:57 +02001833{
Maarten Lankhorstf90a85e2019-10-31 12:26:08 +01001834 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
Wambui Karugaf8d18d52020-01-07 18:13:30 +03001835 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
Ville Syrjäläff32c542017-03-02 19:14:57 +02001836 enum plane_id plane_id = plane->id;
Ville Syrjälä6d5019b2017-04-21 21:14:20 +03001837 int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
Ville Syrjäläff32c542017-03-02 19:14:57 +02001838 int level;
Ville Syrjälä236c48e2017-03-02 19:14:58 +02001839 bool dirty = false;
Ville Syrjäläff32c542017-03-02 19:14:57 +02001840
Ville Syrjäläa07102f2017-03-03 17:19:27 +02001841 if (!intel_wm_plane_visible(crtc_state, plane_state)) {
Ville Syrjälä236c48e2017-03-02 19:14:58 +02001842 dirty |= vlv_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1843 goto out;
Ville Syrjäläff32c542017-03-02 19:14:57 +02001844 }
1845
1846 for (level = 0; level < num_levels; level++) {
Ville Syrjälä114d7dc2017-04-21 21:14:21 +03001847 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
Ville Syrjäläff32c542017-03-02 19:14:57 +02001848 int wm = vlv_compute_wm_level(crtc_state, plane_state, level);
1849 int max_wm = plane_id == PLANE_CURSOR ? 63 : 511;
1850
Ville Syrjäläff32c542017-03-02 19:14:57 +02001851 if (wm > max_wm)
1852 break;
1853
Ville Syrjälä236c48e2017-03-02 19:14:58 +02001854 dirty |= raw->plane[plane_id] != wm;
Ville Syrjäläff32c542017-03-02 19:14:57 +02001855 raw->plane[plane_id] = wm;
1856 }
1857
1858 /* mark all higher levels as invalid */
Ville Syrjälä236c48e2017-03-02 19:14:58 +02001859 dirty |= vlv_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
Ville Syrjäläff32c542017-03-02 19:14:57 +02001860
Ville Syrjälä236c48e2017-03-02 19:14:58 +02001861out:
1862 if (dirty)
Wambui Karugaf8d18d52020-01-07 18:13:30 +03001863 drm_dbg_kms(&dev_priv->drm,
1864 "%s watermarks: PM2=%d, PM5=%d, DDR DVFS=%d\n",
1865 plane->base.name,
1866 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2].plane[plane_id],
1867 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM5].plane[plane_id],
1868 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_DDR_DVFS].plane[plane_id]);
Ville Syrjälä236c48e2017-03-02 19:14:58 +02001869
1870 return dirty;
Ville Syrjäläff32c542017-03-02 19:14:57 +02001871}
1872
Ville Syrjälä77d14ee2017-04-21 21:14:18 +03001873static bool vlv_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1874 enum plane_id plane_id, int level)
Ville Syrjäläff32c542017-03-02 19:14:57 +02001875{
Ville Syrjälä114d7dc2017-04-21 21:14:21 +03001876 const struct g4x_pipe_wm *raw =
Ville Syrjäläff32c542017-03-02 19:14:57 +02001877 &crtc_state->wm.vlv.raw[level];
1878 const struct vlv_fifo_state *fifo_state =
1879 &crtc_state->wm.vlv.fifo_state;
1880
1881 return raw->plane[plane_id] <= fifo_state->plane[plane_id];
1882}
1883
Ville Syrjälä77d14ee2017-04-21 21:14:18 +03001884static bool vlv_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, int level)
Ville Syrjäläff32c542017-03-02 19:14:57 +02001885{
Ville Syrjälä77d14ee2017-04-21 21:14:18 +03001886 return vlv_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1887 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1888 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE1, level) &&
1889 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
Ville Syrjäläff32c542017-03-02 19:14:57 +02001890}
1891
1892static int vlv_compute_pipe_wm(struct intel_crtc_state *crtc_state)
Ville Syrjälä262cd2e2015-06-24 22:00:04 +03001893{
Maarten Lankhorst2225f3c2019-10-31 12:26:03 +01001894 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
Ville Syrjälä7c951c02016-11-28 19:37:10 +02001895 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
Ville Syrjäläff32c542017-03-02 19:14:57 +02001896 struct intel_atomic_state *state =
Maarten Lankhorst2225f3c2019-10-31 12:26:03 +01001897 to_intel_atomic_state(crtc_state->uapi.state);
Ville Syrjälä855c79f2017-03-02 19:14:54 +02001898 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
Ville Syrjäläff32c542017-03-02 19:14:57 +02001899 const struct vlv_fifo_state *fifo_state =
1900 &crtc_state->wm.vlv.fifo_state;
Ville Syrjälä0b14d962019-08-21 20:30:33 +03001901 int num_active_planes = hweight8(crtc_state->active_planes &
1902 ~BIT(PLANE_CURSOR));
Maarten Lankhorst2225f3c2019-10-31 12:26:03 +01001903 bool needs_modeset = drm_atomic_crtc_needs_modeset(&crtc_state->uapi);
Ville Syrjälä7b5104512017-08-23 18:22:22 +03001904 const struct intel_plane_state *old_plane_state;
1905 const struct intel_plane_state *new_plane_state;
Ville Syrjälä262cd2e2015-06-24 22:00:04 +03001906 struct intel_plane *plane;
Ville Syrjäläff32c542017-03-02 19:14:57 +02001907 enum plane_id plane_id;
1908 int level, ret, i;
Ville Syrjälä236c48e2017-03-02 19:14:58 +02001909 unsigned int dirty = 0;
Ville Syrjälä262cd2e2015-06-24 22:00:04 +03001910
Ville Syrjälä7b5104512017-08-23 18:22:22 +03001911 for_each_oldnew_intel_plane_in_state(state, plane,
1912 old_plane_state,
1913 new_plane_state, i) {
Maarten Lankhorst7b3cb172019-10-31 12:26:07 +01001914 if (new_plane_state->hw.crtc != &crtc->base &&
1915 old_plane_state->hw.crtc != &crtc->base)
Ville Syrjälä262cd2e2015-06-24 22:00:04 +03001916 continue;
1917
Ville Syrjälä7b5104512017-08-23 18:22:22 +03001918 if (vlv_raw_plane_wm_compute(crtc_state, new_plane_state))
Ville Syrjälä236c48e2017-03-02 19:14:58 +02001919 dirty |= BIT(plane->id);
1920 }
1921
1922 /*
1923 * DSPARB registers may have been reset due to the
1924 * power well being turned off. Make sure we restore
1925 * them to a consistent state even if no primary/sprite
1926 * planes are initially active.
1927 */
1928 if (needs_modeset)
1929 crtc_state->fifo_changed = true;
1930
1931 if (!dirty)
1932 return 0;
1933
1934 /* cursor changes don't warrant a FIFO recompute */
1935 if (dirty & ~BIT(PLANE_CURSOR)) {
1936 const struct intel_crtc_state *old_crtc_state =
Ville Syrjälä7b5104512017-08-23 18:22:22 +03001937 intel_atomic_get_old_crtc_state(state, crtc);
Ville Syrjälä236c48e2017-03-02 19:14:58 +02001938 const struct vlv_fifo_state *old_fifo_state =
1939 &old_crtc_state->wm.vlv.fifo_state;
1940
1941 ret = vlv_compute_fifo(crtc_state);
1942 if (ret)
1943 return ret;
1944
1945 if (needs_modeset ||
1946 memcmp(old_fifo_state, fifo_state,
1947 sizeof(*fifo_state)) != 0)
1948 crtc_state->fifo_changed = true;
Ville Syrjälä5012e602017-03-02 19:14:56 +02001949 }
Ville Syrjälä262cd2e2015-06-24 22:00:04 +03001950
Ville Syrjäläff32c542017-03-02 19:14:57 +02001951 /* initially allow all levels */
Ville Syrjälä6d5019b2017-04-21 21:14:20 +03001952 wm_state->num_levels = intel_wm_num_levels(dev_priv);
Ville Syrjäläff32c542017-03-02 19:14:57 +02001953 /*
1954 * Note that enabling cxsr with no primary/sprite planes
1955 * enabled can wedge the pipe. Hence we only allow cxsr
1956 * with exactly one enabled primary/sprite plane.
1957 */
Ville Syrjälä5eeb7982017-03-02 19:15:00 +02001958 wm_state->cxsr = crtc->pipe != PIPE_C && num_active_planes == 1;
Ville Syrjäläff32c542017-03-02 19:14:57 +02001959
Ville Syrjälä5012e602017-03-02 19:14:56 +02001960 for (level = 0; level < wm_state->num_levels; level++) {
Ville Syrjälä114d7dc2017-04-21 21:14:21 +03001961 const struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
Jani Nikula24977872019-09-11 12:26:08 +03001962 const int sr_fifo_size = INTEL_NUM_PIPES(dev_priv) * 512 - 1;
Ville Syrjälä5012e602017-03-02 19:14:56 +02001963
Ville Syrjälä77d14ee2017-04-21 21:14:18 +03001964 if (!vlv_raw_crtc_wm_is_valid(crtc_state, level))
Ville Syrjäläff32c542017-03-02 19:14:57 +02001965 break;
Ville Syrjälä5012e602017-03-02 19:14:56 +02001966
Ville Syrjäläff32c542017-03-02 19:14:57 +02001967 for_each_plane_id_on_crtc(crtc, plane_id) {
1968 wm_state->wm[level].plane[plane_id] =
1969 vlv_invert_wm_value(raw->plane[plane_id],
1970 fifo_state->plane[plane_id]);
1971 }
1972
1973 wm_state->sr[level].plane =
1974 vlv_invert_wm_value(max3(raw->plane[PLANE_PRIMARY],
Ville Syrjälä5012e602017-03-02 19:14:56 +02001975 raw->plane[PLANE_SPRITE0],
Ville Syrjäläff32c542017-03-02 19:14:57 +02001976 raw->plane[PLANE_SPRITE1]),
1977 sr_fifo_size);
1978
1979 wm_state->sr[level].cursor =
1980 vlv_invert_wm_value(raw->plane[PLANE_CURSOR],
1981 63);
Ville Syrjälä262cd2e2015-06-24 22:00:04 +03001982 }
1983
Ville Syrjäläff32c542017-03-02 19:14:57 +02001984 if (level == 0)
1985 return -EINVAL;
Ville Syrjälä262cd2e2015-06-24 22:00:04 +03001986
Ville Syrjäläff32c542017-03-02 19:14:57 +02001987 /* limit to only levels we can actually handle */
1988 wm_state->num_levels = level;
1989
1990 /* invalidate the higher levels */
1991 vlv_invalidate_wms(crtc, wm_state, level);
1992
1993 return 0;
Ville Syrjälä262cd2e2015-06-24 22:00:04 +03001994}
1995
Ville Syrjälä54f1b6e2015-06-24 22:00:05 +03001996#define VLV_FIFO(plane, value) \
1997 (((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV)
1998
Ville Syrjäläff32c542017-03-02 19:14:57 +02001999static void vlv_atomic_update_fifo(struct intel_atomic_state *state,
Ville Syrjälä7a8fdb1f2019-11-18 18:44:26 +02002000 struct intel_crtc *crtc)
Ville Syrjälä54f1b6e2015-06-24 22:00:05 +03002001{
Ville Syrjäläf07d43d2017-03-02 19:14:52 +02002002 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
Tvrtko Ursuline33a4be2019-06-11 11:45:44 +01002003 struct intel_uncore *uncore = &dev_priv->uncore;
Ville Syrjälä7a8fdb1f2019-11-18 18:44:26 +02002004 const struct intel_crtc_state *crtc_state =
2005 intel_atomic_get_new_crtc_state(state, crtc);
Ville Syrjälä814e7f02017-03-02 19:14:55 +02002006 const struct vlv_fifo_state *fifo_state =
2007 &crtc_state->wm.vlv.fifo_state;
Ville Syrjäläf07d43d2017-03-02 19:14:52 +02002008 int sprite0_start, sprite1_start, fifo_size;
Kees Cook2713eb42020-02-20 16:05:17 -08002009 u32 dsparb, dsparb2, dsparb3;
Ville Syrjälä54f1b6e2015-06-24 22:00:05 +03002010
Ville Syrjälä236c48e2017-03-02 19:14:58 +02002011 if (!crtc_state->fifo_changed)
2012 return;
2013
Ville Syrjäläf07d43d2017-03-02 19:14:52 +02002014 sprite0_start = fifo_state->plane[PLANE_PRIMARY];
2015 sprite1_start = fifo_state->plane[PLANE_SPRITE0] + sprite0_start;
2016 fifo_size = fifo_state->plane[PLANE_SPRITE1] + sprite1_start;
Ville Syrjälä54f1b6e2015-06-24 22:00:05 +03002017
Pankaj Bharadiya48a1b8d2020-01-15 09:14:53 +05302018 drm_WARN_ON(&dev_priv->drm, fifo_state->plane[PLANE_CURSOR] != 63);
2019 drm_WARN_ON(&dev_priv->drm, fifo_size != 511);
Ville Syrjälä54f1b6e2015-06-24 22:00:05 +03002020
Ville Syrjäläc137d662017-03-02 19:15:06 +02002021 trace_vlv_fifo_size(crtc, sprite0_start, sprite1_start, fifo_size);
2022
Ville Syrjälä44e921d2017-03-09 17:44:34 +02002023 /*
2024 * uncore.lock serves a double purpose here. It allows us to
2025 * use the less expensive I915_{READ,WRITE}_FW() functions, and
2026 * it protects the DSPARB registers from getting clobbered by
2027 * parallel updates from multiple pipes.
2028 *
2029 * intel_pipe_update_start() has already disabled interrupts
2030 * for us, so a plain spin_lock() is sufficient here.
2031 */
Tvrtko Ursuline33a4be2019-06-11 11:45:44 +01002032 spin_lock(&uncore->lock);
Ville Syrjälä467a14d2016-12-05 16:13:28 +02002033
Ville Syrjälä54f1b6e2015-06-24 22:00:05 +03002034 switch (crtc->pipe) {
Ville Syrjälä54f1b6e2015-06-24 22:00:05 +03002035 case PIPE_A:
Tvrtko Ursuline33a4be2019-06-11 11:45:44 +01002036 dsparb = intel_uncore_read_fw(uncore, DSPARB);
2037 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
Ville Syrjälä54f1b6e2015-06-24 22:00:05 +03002038
2039 dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) |
2040 VLV_FIFO(SPRITEB, 0xff));
2041 dsparb |= (VLV_FIFO(SPRITEA, sprite0_start) |
2042 VLV_FIFO(SPRITEB, sprite1_start));
2043
2044 dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) |
2045 VLV_FIFO(SPRITEB_HI, 0x1));
2046 dsparb2 |= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) |
2047 VLV_FIFO(SPRITEB_HI, sprite1_start >> 8));
2048
Tvrtko Ursuline33a4be2019-06-11 11:45:44 +01002049 intel_uncore_write_fw(uncore, DSPARB, dsparb);
2050 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
Ville Syrjälä54f1b6e2015-06-24 22:00:05 +03002051 break;
2052 case PIPE_B:
Tvrtko Ursuline33a4be2019-06-11 11:45:44 +01002053 dsparb = intel_uncore_read_fw(uncore, DSPARB);
2054 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
Ville Syrjälä54f1b6e2015-06-24 22:00:05 +03002055
2056 dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) |
2057 VLV_FIFO(SPRITED, 0xff));
2058 dsparb |= (VLV_FIFO(SPRITEC, sprite0_start) |
2059 VLV_FIFO(SPRITED, sprite1_start));
2060
2061 dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) |
2062 VLV_FIFO(SPRITED_HI, 0xff));
2063 dsparb2 |= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) |
2064 VLV_FIFO(SPRITED_HI, sprite1_start >> 8));
2065
Tvrtko Ursuline33a4be2019-06-11 11:45:44 +01002066 intel_uncore_write_fw(uncore, DSPARB, dsparb);
2067 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
Ville Syrjälä54f1b6e2015-06-24 22:00:05 +03002068 break;
2069 case PIPE_C:
Tvrtko Ursuline33a4be2019-06-11 11:45:44 +01002070 dsparb3 = intel_uncore_read_fw(uncore, DSPARB3);
2071 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
Ville Syrjälä54f1b6e2015-06-24 22:00:05 +03002072
2073 dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) |
2074 VLV_FIFO(SPRITEF, 0xff));
2075 dsparb3 |= (VLV_FIFO(SPRITEE, sprite0_start) |
2076 VLV_FIFO(SPRITEF, sprite1_start));
2077
2078 dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) |
2079 VLV_FIFO(SPRITEF_HI, 0xff));
2080 dsparb2 |= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) |
2081 VLV_FIFO(SPRITEF_HI, sprite1_start >> 8));
2082
Tvrtko Ursuline33a4be2019-06-11 11:45:44 +01002083 intel_uncore_write_fw(uncore, DSPARB3, dsparb3);
2084 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
Ville Syrjälä54f1b6e2015-06-24 22:00:05 +03002085 break;
2086 default:
2087 break;
2088 }
Ville Syrjälä467a14d2016-12-05 16:13:28 +02002089
Tvrtko Ursuline33a4be2019-06-11 11:45:44 +01002090 intel_uncore_posting_read_fw(uncore, DSPARB);
Ville Syrjälä467a14d2016-12-05 16:13:28 +02002091
Tvrtko Ursuline33a4be2019-06-11 11:45:44 +01002092 spin_unlock(&uncore->lock);
Ville Syrjälä54f1b6e2015-06-24 22:00:05 +03002093}
2094
2095#undef VLV_FIFO
2096
Matt Ropercd1d3ee2018-12-10 13:54:14 -08002097static int vlv_compute_intermediate_wm(struct intel_crtc_state *new_crtc_state)
Ville Syrjälä4841da52017-03-02 19:14:59 +02002098{
Maarten Lankhorst2225f3c2019-10-31 12:26:03 +01002099 struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
Maarten Lankhorst5b9489c2017-11-15 17:31:56 +01002100 struct vlv_wm_state *intermediate = &new_crtc_state->wm.vlv.intermediate;
2101 const struct vlv_wm_state *optimal = &new_crtc_state->wm.vlv.optimal;
2102 struct intel_atomic_state *intel_state =
Maarten Lankhorst2225f3c2019-10-31 12:26:03 +01002103 to_intel_atomic_state(new_crtc_state->uapi.state);
Maarten Lankhorst5b9489c2017-11-15 17:31:56 +01002104 const struct intel_crtc_state *old_crtc_state =
2105 intel_atomic_get_old_crtc_state(intel_state, crtc);
2106 const struct vlv_wm_state *active = &old_crtc_state->wm.vlv.optimal;
Ville Syrjälä4841da52017-03-02 19:14:59 +02002107 int level;
2108
Maarten Lankhorst2225f3c2019-10-31 12:26:03 +01002109 if (!new_crtc_state->hw.active || drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi)) {
Maarten Lankhorst5b9489c2017-11-15 17:31:56 +01002110 *intermediate = *optimal;
2111
2112 intermediate->cxsr = false;
2113 goto out;
2114 }
2115
Ville Syrjälä4841da52017-03-02 19:14:59 +02002116 intermediate->num_levels = min(optimal->num_levels, active->num_levels);
Ville Syrjälä5eeb7982017-03-02 19:15:00 +02002117 intermediate->cxsr = optimal->cxsr && active->cxsr &&
Maarten Lankhorst5b9489c2017-11-15 17:31:56 +01002118 !new_crtc_state->disable_cxsr;
Ville Syrjälä4841da52017-03-02 19:14:59 +02002119
2120 for (level = 0; level < intermediate->num_levels; level++) {
2121 enum plane_id plane_id;
2122
2123 for_each_plane_id_on_crtc(crtc, plane_id) {
2124 intermediate->wm[level].plane[plane_id] =
2125 min(optimal->wm[level].plane[plane_id],
2126 active->wm[level].plane[plane_id]);
2127 }
2128
2129 intermediate->sr[level].plane = min(optimal->sr[level].plane,
2130 active->sr[level].plane);
2131 intermediate->sr[level].cursor = min(optimal->sr[level].cursor,
2132 active->sr[level].cursor);
2133 }
2134
2135 vlv_invalidate_wms(crtc, intermediate, level);
2136
Maarten Lankhorst5b9489c2017-11-15 17:31:56 +01002137out:
Ville Syrjälä4841da52017-03-02 19:14:59 +02002138 /*
2139 * If our intermediate WM are identical to the final WM, then we can
2140 * omit the post-vblank programming; only update if it's different.
2141 */
Ville Syrjälä5eeb7982017-03-02 19:15:00 +02002142 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
Maarten Lankhorst5b9489c2017-11-15 17:31:56 +01002143 new_crtc_state->wm.need_postvbl_update = true;
Ville Syrjälä4841da52017-03-02 19:14:59 +02002144
2145 return 0;
2146}
2147
Ville Syrjälä7c951c02016-11-28 19:37:10 +02002148static void vlv_merge_wm(struct drm_i915_private *dev_priv,
Ville Syrjälä262cd2e2015-06-24 22:00:04 +03002149 struct vlv_wm_values *wm)
2150{
2151 struct intel_crtc *crtc;
Ville Syrjäläc08e9132019-08-21 20:30:32 +03002152 int num_active_pipes = 0;
Ville Syrjälä262cd2e2015-06-24 22:00:04 +03002153
Ville Syrjälä7c951c02016-11-28 19:37:10 +02002154 wm->level = dev_priv->wm.max_level;
Ville Syrjälä262cd2e2015-06-24 22:00:04 +03002155 wm->cxsr = true;
2156
Ville Syrjälä7c951c02016-11-28 19:37:10 +02002157 for_each_intel_crtc(&dev_priv->drm, crtc) {
Ville Syrjälä7eb49412017-03-02 19:14:53 +02002158 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
Ville Syrjälä262cd2e2015-06-24 22:00:04 +03002159
2160 if (!crtc->active)
2161 continue;
2162
2163 if (!wm_state->cxsr)
2164 wm->cxsr = false;
2165
Ville Syrjäläc08e9132019-08-21 20:30:32 +03002166 num_active_pipes++;
Ville Syrjälä262cd2e2015-06-24 22:00:04 +03002167 wm->level = min_t(int, wm->level, wm_state->num_levels - 1);
2168 }
2169
Ville Syrjäläc08e9132019-08-21 20:30:32 +03002170 if (num_active_pipes != 1)
Ville Syrjälä262cd2e2015-06-24 22:00:04 +03002171 wm->cxsr = false;
2172
Ville Syrjäläc08e9132019-08-21 20:30:32 +03002173 if (num_active_pipes > 1)
Ville Syrjälä6f9c7842015-06-24 22:00:08 +03002174 wm->level = VLV_WM_LEVEL_PM2;
2175
Ville Syrjälä7c951c02016-11-28 19:37:10 +02002176 for_each_intel_crtc(&dev_priv->drm, crtc) {
Ville Syrjälä7eb49412017-03-02 19:14:53 +02002177 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
Ville Syrjälä262cd2e2015-06-24 22:00:04 +03002178 enum pipe pipe = crtc->pipe;
2179
Ville Syrjälä262cd2e2015-06-24 22:00:04 +03002180 wm->pipe[pipe] = wm_state->wm[wm->level];
Ville Syrjäläff32c542017-03-02 19:14:57 +02002181 if (crtc->active && wm->cxsr)
Ville Syrjälä262cd2e2015-06-24 22:00:04 +03002182 wm->sr = wm_state->sr[wm->level];
2183
Ville Syrjälä1b313892016-11-28 19:37:08 +02002184 wm->ddl[pipe].plane[PLANE_PRIMARY] = DDL_PRECISION_HIGH | 2;
2185 wm->ddl[pipe].plane[PLANE_SPRITE0] = DDL_PRECISION_HIGH | 2;
2186 wm->ddl[pipe].plane[PLANE_SPRITE1] = DDL_PRECISION_HIGH | 2;
2187 wm->ddl[pipe].plane[PLANE_CURSOR] = DDL_PRECISION_HIGH | 2;
Ville Syrjälä262cd2e2015-06-24 22:00:04 +03002188 }
2189}
2190
Ville Syrjäläff32c542017-03-02 19:14:57 +02002191static void vlv_program_watermarks(struct drm_i915_private *dev_priv)
Ville Syrjälä262cd2e2015-06-24 22:00:04 +03002192{
Ville Syrjäläfa292a42016-11-28 19:37:16 +02002193 struct vlv_wm_values *old_wm = &dev_priv->wm.vlv;
2194 struct vlv_wm_values new_wm = {};
Ville Syrjälä262cd2e2015-06-24 22:00:04 +03002195
Ville Syrjäläfa292a42016-11-28 19:37:16 +02002196 vlv_merge_wm(dev_priv, &new_wm);
Ville Syrjälä262cd2e2015-06-24 22:00:04 +03002197
Ville Syrjäläff32c542017-03-02 19:14:57 +02002198 if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
Ville Syrjälä262cd2e2015-06-24 22:00:04 +03002199 return;
2200
Ville Syrjäläfa292a42016-11-28 19:37:16 +02002201 if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
Ville Syrjälä262cd2e2015-06-24 22:00:04 +03002202 chv_set_memory_dvfs(dev_priv, false);
2203
Ville Syrjäläfa292a42016-11-28 19:37:16 +02002204 if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
Ville Syrjälä262cd2e2015-06-24 22:00:04 +03002205 chv_set_memory_pm5(dev_priv, false);
2206
Ville Syrjäläfa292a42016-11-28 19:37:16 +02002207 if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
Ville Syrjälä3d90e642016-11-28 19:37:11 +02002208 _intel_set_memory_cxsr(dev_priv, false);
Ville Syrjälä262cd2e2015-06-24 22:00:04 +03002209
Ville Syrjäläfa292a42016-11-28 19:37:16 +02002210 vlv_write_wm_values(dev_priv, &new_wm);
Ville Syrjälä262cd2e2015-06-24 22:00:04 +03002211
Ville Syrjäläfa292a42016-11-28 19:37:16 +02002212 if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
Ville Syrjälä3d90e642016-11-28 19:37:11 +02002213 _intel_set_memory_cxsr(dev_priv, true);
Ville Syrjälä262cd2e2015-06-24 22:00:04 +03002214
Ville Syrjäläfa292a42016-11-28 19:37:16 +02002215 if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
Ville Syrjälä262cd2e2015-06-24 22:00:04 +03002216 chv_set_memory_pm5(dev_priv, true);
2217
Ville Syrjäläfa292a42016-11-28 19:37:16 +02002218 if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
Ville Syrjälä262cd2e2015-06-24 22:00:04 +03002219 chv_set_memory_dvfs(dev_priv, true);
2220
Ville Syrjäläfa292a42016-11-28 19:37:16 +02002221 *old_wm = new_wm;
Ville Syrjälä3c2777f2014-06-26 17:03:06 +03002222}
2223
Ville Syrjäläff32c542017-03-02 19:14:57 +02002224static void vlv_initial_watermarks(struct intel_atomic_state *state,
Ville Syrjälä7a8fdb1f2019-11-18 18:44:26 +02002225 struct intel_crtc *crtc)
Ville Syrjäläff32c542017-03-02 19:14:57 +02002226{
Ville Syrjälä7a8fdb1f2019-11-18 18:44:26 +02002227 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2228 const struct intel_crtc_state *crtc_state =
2229 intel_atomic_get_new_crtc_state(state, crtc);
Ville Syrjäläff32c542017-03-02 19:14:57 +02002230
2231 mutex_lock(&dev_priv->wm.wm_mutex);
Ville Syrjälä4841da52017-03-02 19:14:59 +02002232 crtc->wm.active.vlv = crtc_state->wm.vlv.intermediate;
2233 vlv_program_watermarks(dev_priv);
2234 mutex_unlock(&dev_priv->wm.wm_mutex);
2235}
2236
2237static void vlv_optimize_watermarks(struct intel_atomic_state *state,
Ville Syrjälä7a8fdb1f2019-11-18 18:44:26 +02002238 struct intel_crtc *crtc)
Ville Syrjälä4841da52017-03-02 19:14:59 +02002239{
Ville Syrjälä7a8fdb1f2019-11-18 18:44:26 +02002240 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2241 const struct intel_crtc_state *crtc_state =
2242 intel_atomic_get_new_crtc_state(state, crtc);
Ville Syrjälä4841da52017-03-02 19:14:59 +02002243
2244 if (!crtc_state->wm.need_postvbl_update)
2245 return;
2246
2247 mutex_lock(&dev_priv->wm.wm_mutex);
Ville Syrjälä88016a92019-07-01 19:05:45 +03002248 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
Ville Syrjäläff32c542017-03-02 19:14:57 +02002249 vlv_program_watermarks(dev_priv);
2250 mutex_unlock(&dev_priv->wm.wm_mutex);
2251}
2252
Ville Syrjälä432081b2016-10-31 22:37:03 +02002253static void i965_update_wm(struct intel_crtc *unused_crtc)
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -03002254{
Ville Syrjäläffc7a762016-10-31 22:37:21 +02002255 struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
Ville Syrjäläefc26112016-10-31 22:37:04 +02002256 struct intel_crtc *crtc;
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -03002257 int srwm = 1;
2258 int cursor_sr = 16;
Imre Deak98584252014-06-13 14:54:20 +03002259 bool cxsr_enabled;
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -03002260
2261 /* Calc sr entries for one plane configs */
Ville Syrjäläffc7a762016-10-31 22:37:21 +02002262 crtc = single_enabled_crtc(dev_priv);
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -03002263 if (crtc) {
2264 /* self-refresh has much higher latency */
2265 static const int sr_latency_ns = 12000;
Maarten Lankhorstbafcdad2020-11-12 21:17:18 +02002266 const struct drm_display_mode *pipe_mode =
2267 &crtc->config->hw.pipe_mode;
Ville Syrjäläefc26112016-10-31 22:37:04 +02002268 const struct drm_framebuffer *fb =
2269 crtc->base.primary->state->fb;
Maarten Lankhorstbafcdad2020-11-12 21:17:18 +02002270 int clock = pipe_mode->crtc_clock;
2271 int htotal = pipe_mode->crtc_htotal;
Ville Syrjäläefc26112016-10-31 22:37:04 +02002272 int hdisplay = crtc->config->pipe_src_w;
Ville Syrjälä353c8592016-12-14 23:30:57 +02002273 int cpp = fb->format->cpp[0];
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -03002274 int entries;
2275
Ville Syrjäläbaf69ca2017-04-21 21:14:27 +03002276 entries = intel_wm_method2(clock, htotal,
2277 hdisplay, cpp, sr_latency_ns / 100);
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -03002278 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
2279 srwm = I965_FIFO_SIZE - entries;
2280 if (srwm < 0)
2281 srwm = 1;
2282 srwm &= 0x1ff;
Wambui Karugaf8d18d52020-01-07 18:13:30 +03002283 drm_dbg_kms(&dev_priv->drm,
2284 "self-refresh entries: %d, wm: %d\n",
2285 entries, srwm);
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -03002286
Ville Syrjäläbaf69ca2017-04-21 21:14:27 +03002287 entries = intel_wm_method2(clock, htotal,
2288 crtc->base.cursor->state->crtc_w, 4,
2289 sr_latency_ns / 100);
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -03002290 entries = DIV_ROUND_UP(entries,
Ville Syrjäläbaf69ca2017-04-21 21:14:27 +03002291 i965_cursor_wm_info.cacheline_size) +
2292 i965_cursor_wm_info.guard_size;
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -03002293
Ville Syrjäläbaf69ca2017-04-21 21:14:27 +03002294 cursor_sr = i965_cursor_wm_info.fifo_size - entries;
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -03002295 if (cursor_sr > i965_cursor_wm_info.max_wm)
2296 cursor_sr = i965_cursor_wm_info.max_wm;
2297
Wambui Karugaf8d18d52020-01-07 18:13:30 +03002298 drm_dbg_kms(&dev_priv->drm,
2299 "self-refresh watermark: display plane %d "
2300 "cursor %d\n", srwm, cursor_sr);
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -03002301
Imre Deak98584252014-06-13 14:54:20 +03002302 cxsr_enabled = true;
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -03002303 } else {
Imre Deak98584252014-06-13 14:54:20 +03002304 cxsr_enabled = false;
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -03002305 /* Turn off self refresh if both pipes are enabled */
Imre Deak5209b1f2014-07-01 12:36:17 +03002306 intel_set_memory_cxsr(dev_priv, false);
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -03002307 }
2308
Wambui Karugaf8d18d52020-01-07 18:13:30 +03002309 drm_dbg_kms(&dev_priv->drm,
2310 "Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
2311 srwm);
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -03002312
2313 /* 965 has limitations... */
Jani Nikula5f461662020-11-30 13:15:58 +02002314 intel_uncore_write(&dev_priv->uncore, DSPFW1, FW_WM(srwm, SR) |
Ville Syrjäläf4998962015-03-10 17:02:21 +02002315 FW_WM(8, CURSORB) |
2316 FW_WM(8, PLANEB) |
2317 FW_WM(8, PLANEA));
Jani Nikula5f461662020-11-30 13:15:58 +02002318 intel_uncore_write(&dev_priv->uncore, DSPFW2, FW_WM(8, CURSORA) |
Ville Syrjäläf4998962015-03-10 17:02:21 +02002319 FW_WM(8, PLANEC_OLD));
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -03002320 /* update cursor SR watermark */
Jani Nikula5f461662020-11-30 13:15:58 +02002321 intel_uncore_write(&dev_priv->uncore, DSPFW3, FW_WM(cursor_sr, CURSOR_SR));
Imre Deak98584252014-06-13 14:54:20 +03002322
2323 if (cxsr_enabled)
2324 intel_set_memory_cxsr(dev_priv, true);
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -03002325}
2326
Ville Syrjäläf4998962015-03-10 17:02:21 +02002327#undef FW_WM
2328
Ville Syrjälä432081b2016-10-31 22:37:03 +02002329static void i9xx_update_wm(struct intel_crtc *unused_crtc)
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -03002330{
Ville Syrjäläffc7a762016-10-31 22:37:21 +02002331 struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -03002332 const struct intel_watermark_params *wm_info;
Jani Nikula5ce9a6492019-01-18 14:01:20 +02002333 u32 fwater_lo;
2334 u32 fwater_hi;
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -03002335 int cwm, srwm = 1;
2336 int fifo_size;
2337 int planea_wm, planeb_wm;
Ville Syrjäläefc26112016-10-31 22:37:04 +02002338 struct intel_crtc *crtc, *enabled = NULL;
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -03002339
Ville Syrjäläa9097be2016-10-31 22:37:20 +02002340 if (IS_I945GM(dev_priv))
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -03002341 wm_info = &i945_wm_info;
Lucas De Marchicf819ef2018-12-12 10:10:43 -08002342 else if (!IS_GEN(dev_priv, 2))
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -03002343 wm_info = &i915_wm_info;
2344 else
Ville Syrjälä9d539102014-08-15 01:21:53 +03002345 wm_info = &i830_a_wm_info;
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -03002346
Ville Syrjäläbdaf8432017-11-17 21:19:11 +02002347 fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_A);
2348 crtc = intel_get_crtc_for_plane(dev_priv, PLANE_A);
Ville Syrjäläefc26112016-10-31 22:37:04 +02002349 if (intel_crtc_active(crtc)) {
Maarten Lankhorstbafcdad2020-11-12 21:17:18 +02002350 const struct drm_display_mode *pipe_mode =
2351 &crtc->config->hw.pipe_mode;
Ville Syrjäläefc26112016-10-31 22:37:04 +02002352 const struct drm_framebuffer *fb =
2353 crtc->base.primary->state->fb;
2354 int cpp;
2355
Lucas De Marchicf819ef2018-12-12 10:10:43 -08002356 if (IS_GEN(dev_priv, 2))
Chris Wilsonb9e0bda2012-10-22 12:32:15 +01002357 cpp = 4;
Ville Syrjäläefc26112016-10-31 22:37:04 +02002358 else
Ville Syrjälä353c8592016-12-14 23:30:57 +02002359 cpp = fb->format->cpp[0];
Chris Wilsonb9e0bda2012-10-22 12:32:15 +01002360
Maarten Lankhorstbafcdad2020-11-12 21:17:18 +02002361 planea_wm = intel_calculate_wm(pipe_mode->crtc_clock,
Chris Wilsonb9e0bda2012-10-22 12:32:15 +01002362 wm_info, fifo_size, cpp,
Chris Wilson5aef6002014-09-03 11:56:07 +01002363 pessimal_latency_ns);
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -03002364 enabled = crtc;
Ville Syrjälä9d539102014-08-15 01:21:53 +03002365 } else {
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -03002366 planea_wm = fifo_size - wm_info->guard_size;
Ville Syrjälä9d539102014-08-15 01:21:53 +03002367 if (planea_wm > (long)wm_info->max_wm)
2368 planea_wm = wm_info->max_wm;
2369 }
2370
Lucas De Marchicf819ef2018-12-12 10:10:43 -08002371 if (IS_GEN(dev_priv, 2))
Ville Syrjälä9d539102014-08-15 01:21:53 +03002372 wm_info = &i830_bc_wm_info;
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -03002373
Ville Syrjäläbdaf8432017-11-17 21:19:11 +02002374 fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_B);
2375 crtc = intel_get_crtc_for_plane(dev_priv, PLANE_B);
Ville Syrjäläefc26112016-10-31 22:37:04 +02002376 if (intel_crtc_active(crtc)) {
Maarten Lankhorstbafcdad2020-11-12 21:17:18 +02002377 const struct drm_display_mode *pipe_mode =
2378 &crtc->config->hw.pipe_mode;
Ville Syrjäläefc26112016-10-31 22:37:04 +02002379 const struct drm_framebuffer *fb =
2380 crtc->base.primary->state->fb;
2381 int cpp;
2382
Lucas De Marchicf819ef2018-12-12 10:10:43 -08002383 if (IS_GEN(dev_priv, 2))
Chris Wilsonb9e0bda2012-10-22 12:32:15 +01002384 cpp = 4;
Ville Syrjäläefc26112016-10-31 22:37:04 +02002385 else
Ville Syrjälä353c8592016-12-14 23:30:57 +02002386 cpp = fb->format->cpp[0];
Chris Wilsonb9e0bda2012-10-22 12:32:15 +01002387
Maarten Lankhorstbafcdad2020-11-12 21:17:18 +02002388 planeb_wm = intel_calculate_wm(pipe_mode->crtc_clock,
Chris Wilsonb9e0bda2012-10-22 12:32:15 +01002389 wm_info, fifo_size, cpp,
Chris Wilson5aef6002014-09-03 11:56:07 +01002390 pessimal_latency_ns);
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -03002391 if (enabled == NULL)
2392 enabled = crtc;
2393 else
2394 enabled = NULL;
Ville Syrjälä9d539102014-08-15 01:21:53 +03002395 } else {
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -03002396 planeb_wm = fifo_size - wm_info->guard_size;
Ville Syrjälä9d539102014-08-15 01:21:53 +03002397 if (planeb_wm > (long)wm_info->max_wm)
2398 planeb_wm = wm_info->max_wm;
2399 }
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -03002400
Wambui Karugaf8d18d52020-01-07 18:13:30 +03002401 drm_dbg_kms(&dev_priv->drm,
2402 "FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -03002403
Tvrtko Ursulin50a0bc92016-10-13 11:02:58 +01002404 if (IS_I915GM(dev_priv) && enabled) {
Matt Roper2ff8fde2014-07-08 07:50:07 -07002405 struct drm_i915_gem_object *obj;
Daniel Vetter2ab1bc92014-04-07 08:54:21 +02002406
Ville Syrjäläefc26112016-10-31 22:37:04 +02002407 obj = intel_fb_obj(enabled->base.primary->state->fb);
Daniel Vetter2ab1bc92014-04-07 08:54:21 +02002408
2409 /* self-refresh seems busted with untiled */
Chris Wilson3e510a82016-08-05 10:14:23 +01002410 if (!i915_gem_object_is_tiled(obj))
Daniel Vetter2ab1bc92014-04-07 08:54:21 +02002411 enabled = NULL;
2412 }
2413
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -03002414 /*
2415 * Overlay gets an aggressive default since video jitter is bad.
2416 */
2417 cwm = 2;
2418
2419 /* Play safe and disable self-refresh before adjusting watermarks. */
Imre Deak5209b1f2014-07-01 12:36:17 +03002420 intel_set_memory_cxsr(dev_priv, false);
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -03002421
2422 /* Calc sr entries for one plane configs */
Ville Syrjälä03427fc2016-10-31 22:37:18 +02002423 if (HAS_FW_BLC(dev_priv) && enabled) {
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -03002424 /* self-refresh has much higher latency */
2425 static const int sr_latency_ns = 6000;
Maarten Lankhorstbafcdad2020-11-12 21:17:18 +02002426 const struct drm_display_mode *pipe_mode =
2427 &enabled->config->hw.pipe_mode;
Ville Syrjäläefc26112016-10-31 22:37:04 +02002428 const struct drm_framebuffer *fb =
2429 enabled->base.primary->state->fb;
Maarten Lankhorstbafcdad2020-11-12 21:17:18 +02002430 int clock = pipe_mode->crtc_clock;
2431 int htotal = pipe_mode->crtc_htotal;
Ville Syrjäläefc26112016-10-31 22:37:04 +02002432 int hdisplay = enabled->config->pipe_src_w;
2433 int cpp;
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -03002434 int entries;
2435
Tvrtko Ursulin50a0bc92016-10-13 11:02:58 +01002436 if (IS_I915GM(dev_priv) || IS_I945GM(dev_priv))
Ville Syrjälä2d1b5052016-07-29 17:57:01 +03002437 cpp = 4;
Ville Syrjäläefc26112016-10-31 22:37:04 +02002438 else
Ville Syrjälä353c8592016-12-14 23:30:57 +02002439 cpp = fb->format->cpp[0];
Ville Syrjälä2d1b5052016-07-29 17:57:01 +03002440
Ville Syrjäläbaf69ca2017-04-21 21:14:27 +03002441 entries = intel_wm_method2(clock, htotal, hdisplay, cpp,
2442 sr_latency_ns / 100);
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -03002443 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
Wambui Karugaf8d18d52020-01-07 18:13:30 +03002444 drm_dbg_kms(&dev_priv->drm,
2445 "self-refresh entries: %d\n", entries);
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -03002446 srwm = wm_info->fifo_size - entries;
2447 if (srwm < 0)
2448 srwm = 1;
2449
Tvrtko Ursulin50a0bc92016-10-13 11:02:58 +01002450 if (IS_I945G(dev_priv) || IS_I945GM(dev_priv))
Jani Nikula5f461662020-11-30 13:15:58 +02002451 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF,
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -03002452 FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
Ville Syrjäläacb91352016-07-29 17:57:02 +03002453 else
Jani Nikula5f461662020-11-30 13:15:58 +02002454 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, srwm & 0x3f);
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -03002455 }
2456
Wambui Karugaf8d18d52020-01-07 18:13:30 +03002457 drm_dbg_kms(&dev_priv->drm,
2458 "Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
2459 planea_wm, planeb_wm, cwm, srwm);
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -03002460
2461 fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
2462 fwater_hi = (cwm & 0x1f);
2463
2464 /* Set request length to 8 cachelines per fetch */
2465 fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
2466 fwater_hi = fwater_hi | (1 << 8);
2467
Jani Nikula5f461662020-11-30 13:15:58 +02002468 intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo);
2469 intel_uncore_write(&dev_priv->uncore, FW_BLC2, fwater_hi);
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -03002470
Imre Deak5209b1f2014-07-01 12:36:17 +03002471 if (enabled)
2472 intel_set_memory_cxsr(dev_priv, true);
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -03002473}
2474
Ville Syrjälä432081b2016-10-31 22:37:03 +02002475static void i845_update_wm(struct intel_crtc *unused_crtc)
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -03002476{
Ville Syrjäläffc7a762016-10-31 22:37:21 +02002477 struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev);
Ville Syrjäläefc26112016-10-31 22:37:04 +02002478 struct intel_crtc *crtc;
Maarten Lankhorstbafcdad2020-11-12 21:17:18 +02002479 const struct drm_display_mode *pipe_mode;
Jani Nikula5ce9a6492019-01-18 14:01:20 +02002480 u32 fwater_lo;
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -03002481 int planea_wm;
2482
Ville Syrjäläffc7a762016-10-31 22:37:21 +02002483 crtc = single_enabled_crtc(dev_priv);
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -03002484 if (crtc == NULL)
2485 return;
2486
Maarten Lankhorstbafcdad2020-11-12 21:17:18 +02002487 pipe_mode = &crtc->config->hw.pipe_mode;
2488 planea_wm = intel_calculate_wm(pipe_mode->crtc_clock,
Daniel Vetterfeb56b92013-12-14 20:38:30 -02002489 &i845_wm_info,
Ville Syrjäläbdaf8432017-11-17 21:19:11 +02002490 dev_priv->display.get_fifo_size(dev_priv, PLANE_A),
Chris Wilson5aef6002014-09-03 11:56:07 +01002491 4, pessimal_latency_ns);
Jani Nikula5f461662020-11-30 13:15:58 +02002492 fwater_lo = intel_uncore_read(&dev_priv->uncore, FW_BLC) & ~0xfff;
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -03002493 fwater_lo |= (3<<8) | planea_wm;
2494
Wambui Karugaf8d18d52020-01-07 18:13:30 +03002495 drm_dbg_kms(&dev_priv->drm,
2496 "Setting FIFO watermarks - A: %d\n", planea_wm);
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -03002497
Jani Nikula5f461662020-11-30 13:15:58 +02002498 intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo);
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -03002499}
2500
Ville Syrjälä37126462013-08-01 16:18:55 +03002501/* latency must be in 0.1us units. */
Ville Syrjäläbaf69ca2017-04-21 21:14:27 +03002502static unsigned int ilk_wm_method1(unsigned int pixel_rate,
2503 unsigned int cpp,
2504 unsigned int latency)
Paulo Zanoni801bcff2013-05-31 10:08:35 -03002505{
Ville Syrjäläbaf69ca2017-04-21 21:14:27 +03002506 unsigned int ret;
Paulo Zanoni801bcff2013-05-31 10:08:35 -03002507
Ville Syrjäläbaf69ca2017-04-21 21:14:27 +03002508 ret = intel_wm_method1(pixel_rate, cpp, latency);
2509 ret = DIV_ROUND_UP(ret, 64) + 2;
Paulo Zanoni801bcff2013-05-31 10:08:35 -03002510
2511 return ret;
2512}
2513
Ville Syrjälä37126462013-08-01 16:18:55 +03002514/* latency must be in 0.1us units. */
Ville Syrjäläbaf69ca2017-04-21 21:14:27 +03002515static unsigned int ilk_wm_method2(unsigned int pixel_rate,
2516 unsigned int htotal,
2517 unsigned int width,
2518 unsigned int cpp,
2519 unsigned int latency)
Paulo Zanoni801bcff2013-05-31 10:08:35 -03002520{
Ville Syrjäläbaf69ca2017-04-21 21:14:27 +03002521 unsigned int ret;
Paulo Zanoni801bcff2013-05-31 10:08:35 -03002522
Ville Syrjäläbaf69ca2017-04-21 21:14:27 +03002523 ret = intel_wm_method2(pixel_rate, htotal,
2524 width, cpp, latency);
Paulo Zanoni801bcff2013-05-31 10:08:35 -03002525 ret = DIV_ROUND_UP(ret, 64) + 2;
Ville Syrjäläbaf69ca2017-04-21 21:14:27 +03002526
Paulo Zanoni801bcff2013-05-31 10:08:35 -03002527 return ret;
2528}
2529
Jani Nikula5ce9a6492019-01-18 14:01:20 +02002530static u32 ilk_wm_fbc(u32 pri_val, u32 horiz_pixels, u8 cpp)
Paulo Zanonicca32e92013-05-31 11:45:06 -03002531{
Matt Roper15126882015-12-03 11:37:40 -08002532 /*
2533 * Neither of these should be possible since this function shouldn't be
2534 * called if the CRTC is off or the plane is invisible. But let's be
2535 * extra paranoid to avoid a potential divide-by-zero if we screw up
2536 * elsewhere in the driver.
2537 */
Ville Syrjäläac484962016-01-20 21:05:26 +02002538 if (WARN_ON(!cpp))
Matt Roper15126882015-12-03 11:37:40 -08002539 return 0;
2540 if (WARN_ON(!horiz_pixels))
2541 return 0;
2542
Ville Syrjäläac484962016-01-20 21:05:26 +02002543 return DIV_ROUND_UP(pri_val * 64, horiz_pixels * cpp) + 2;
Paulo Zanonicca32e92013-05-31 11:45:06 -03002544}
2545
Imre Deak820c1982013-12-17 14:46:36 +02002546struct ilk_wm_maximums {
Jani Nikula5ce9a6492019-01-18 14:01:20 +02002547 u16 pri;
2548 u16 spr;
2549 u16 cur;
2550 u16 fbc;
Paulo Zanonicca32e92013-05-31 11:45:06 -03002551};
2552
Ville Syrjälä37126462013-08-01 16:18:55 +03002553/*
2554 * For both WM_PIPE and WM_LP.
2555 * mem_value must be in 0.1us units.
2556 */
Maarten Lankhorstec193642019-06-28 10:55:17 +02002557static u32 ilk_compute_pri_wm(const struct intel_crtc_state *crtc_state,
2558 const struct intel_plane_state *plane_state,
Jani Nikula5ce9a6492019-01-18 14:01:20 +02002559 u32 mem_value, bool is_lp)
Paulo Zanoni801bcff2013-05-31 10:08:35 -03002560{
Jani Nikula5ce9a6492019-01-18 14:01:20 +02002561 u32 method1, method2;
Ville Syrjälä83054942016-11-18 21:53:00 +02002562 int cpp;
Paulo Zanonicca32e92013-05-31 11:45:06 -03002563
Ville Syrjälä03981c62018-11-14 19:34:40 +02002564 if (mem_value == 0)
2565 return U32_MAX;
2566
Maarten Lankhorstec193642019-06-28 10:55:17 +02002567 if (!intel_wm_plane_visible(crtc_state, plane_state))
Paulo Zanoni801bcff2013-05-31 10:08:35 -03002568 return 0;
2569
Maarten Lankhorst7b3cb172019-10-31 12:26:07 +01002570 cpp = plane_state->hw.fb->format->cpp[0];
Ville Syrjälä83054942016-11-18 21:53:00 +02002571
Maarten Lankhorstec193642019-06-28 10:55:17 +02002572 method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
Paulo Zanonicca32e92013-05-31 11:45:06 -03002573
2574 if (!is_lp)
2575 return method1;
2576
Maarten Lankhorstec193642019-06-28 10:55:17 +02002577 method2 = ilk_wm_method2(crtc_state->pixel_rate,
Maarten Lankhorstbafcdad2020-11-12 21:17:18 +02002578 crtc_state->hw.pipe_mode.crtc_htotal,
Maarten Lankhorstf90a85e2019-10-31 12:26:08 +01002579 drm_rect_width(&plane_state->uapi.dst),
Ville Syrjäläac484962016-01-20 21:05:26 +02002580 cpp, mem_value);
Paulo Zanonicca32e92013-05-31 11:45:06 -03002581
2582 return min(method1, method2);
Paulo Zanoni801bcff2013-05-31 10:08:35 -03002583}
2584
Ville Syrjälä37126462013-08-01 16:18:55 +03002585/*
2586 * For both WM_PIPE and WM_LP.
2587 * mem_value must be in 0.1us units.
2588 */
Maarten Lankhorstec193642019-06-28 10:55:17 +02002589static u32 ilk_compute_spr_wm(const struct intel_crtc_state *crtc_state,
2590 const struct intel_plane_state *plane_state,
Jani Nikula5ce9a6492019-01-18 14:01:20 +02002591 u32 mem_value)
Paulo Zanoni801bcff2013-05-31 10:08:35 -03002592{
Jani Nikula5ce9a6492019-01-18 14:01:20 +02002593 u32 method1, method2;
Ville Syrjälä83054942016-11-18 21:53:00 +02002594 int cpp;
Paulo Zanoni801bcff2013-05-31 10:08:35 -03002595
Ville Syrjälä03981c62018-11-14 19:34:40 +02002596 if (mem_value == 0)
2597 return U32_MAX;
2598
Maarten Lankhorstec193642019-06-28 10:55:17 +02002599 if (!intel_wm_plane_visible(crtc_state, plane_state))
Paulo Zanoni801bcff2013-05-31 10:08:35 -03002600 return 0;
2601
Maarten Lankhorst7b3cb172019-10-31 12:26:07 +01002602 cpp = plane_state->hw.fb->format->cpp[0];
Ville Syrjälä83054942016-11-18 21:53:00 +02002603
Maarten Lankhorstec193642019-06-28 10:55:17 +02002604 method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2605 method2 = ilk_wm_method2(crtc_state->pixel_rate,
Maarten Lankhorstbafcdad2020-11-12 21:17:18 +02002606 crtc_state->hw.pipe_mode.crtc_htotal,
Maarten Lankhorstf90a85e2019-10-31 12:26:08 +01002607 drm_rect_width(&plane_state->uapi.dst),
Ville Syrjäläac484962016-01-20 21:05:26 +02002608 cpp, mem_value);
Paulo Zanoni801bcff2013-05-31 10:08:35 -03002609 return min(method1, method2);
2610}
2611
Ville Syrjälä37126462013-08-01 16:18:55 +03002612/*
2613 * For both WM_PIPE and WM_LP.
2614 * mem_value must be in 0.1us units.
2615 */
Maarten Lankhorstec193642019-06-28 10:55:17 +02002616static u32 ilk_compute_cur_wm(const struct intel_crtc_state *crtc_state,
2617 const struct intel_plane_state *plane_state,
Jani Nikula5ce9a6492019-01-18 14:01:20 +02002618 u32 mem_value)
Paulo Zanoni801bcff2013-05-31 10:08:35 -03002619{
Ville Syrjäläa5509ab2017-02-17 17:01:59 +02002620 int cpp;
Matt Roper43d59ed2015-09-24 15:53:07 -07002621
Ville Syrjälä03981c62018-11-14 19:34:40 +02002622 if (mem_value == 0)
2623 return U32_MAX;
2624
Maarten Lankhorstec193642019-06-28 10:55:17 +02002625 if (!intel_wm_plane_visible(crtc_state, plane_state))
Paulo Zanoni801bcff2013-05-31 10:08:35 -03002626 return 0;
2627
Maarten Lankhorst7b3cb172019-10-31 12:26:07 +01002628 cpp = plane_state->hw.fb->format->cpp[0];
Ville Syrjäläa5509ab2017-02-17 17:01:59 +02002629
Maarten Lankhorstec193642019-06-28 10:55:17 +02002630 return ilk_wm_method2(crtc_state->pixel_rate,
Maarten Lankhorstbafcdad2020-11-12 21:17:18 +02002631 crtc_state->hw.pipe_mode.crtc_htotal,
Maarten Lankhorstf90a85e2019-10-31 12:26:08 +01002632 drm_rect_width(&plane_state->uapi.dst),
Maarten Lankhorst3a612762019-10-04 13:34:54 +02002633 cpp, mem_value);
Paulo Zanoni801bcff2013-05-31 10:08:35 -03002634}
2635
Paulo Zanonicca32e92013-05-31 11:45:06 -03002636/* Only for WM_LP. */
Maarten Lankhorstec193642019-06-28 10:55:17 +02002637static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
2638 const struct intel_plane_state *plane_state,
Jani Nikula5ce9a6492019-01-18 14:01:20 +02002639 u32 pri_val)
Paulo Zanonicca32e92013-05-31 11:45:06 -03002640{
Ville Syrjälä83054942016-11-18 21:53:00 +02002641 int cpp;
Matt Roper43d59ed2015-09-24 15:53:07 -07002642
Maarten Lankhorstec193642019-06-28 10:55:17 +02002643 if (!intel_wm_plane_visible(crtc_state, plane_state))
Paulo Zanonicca32e92013-05-31 11:45:06 -03002644 return 0;
2645
Maarten Lankhorst7b3cb172019-10-31 12:26:07 +01002646 cpp = plane_state->hw.fb->format->cpp[0];
Ville Syrjälä83054942016-11-18 21:53:00 +02002647
Maarten Lankhorstf90a85e2019-10-31 12:26:08 +01002648 return ilk_wm_fbc(pri_val, drm_rect_width(&plane_state->uapi.dst),
2649 cpp);
Paulo Zanonicca32e92013-05-31 11:45:06 -03002650}
2651
Tvrtko Ursulin175fded2016-11-16 08:55:42 +00002652static unsigned int
2653ilk_display_fifo_size(const struct drm_i915_private *dev_priv)
Ville Syrjälä158ae642013-08-07 13:28:19 +03002654{
Tvrtko Ursulin175fded2016-11-16 08:55:42 +00002655 if (INTEL_GEN(dev_priv) >= 8)
Ville Syrjälä416f4722013-11-02 21:07:46 -07002656 return 3072;
Tvrtko Ursulin175fded2016-11-16 08:55:42 +00002657 else if (INTEL_GEN(dev_priv) >= 7)
Ville Syrjälä158ae642013-08-07 13:28:19 +03002658 return 768;
2659 else
2660 return 512;
2661}
2662
Tvrtko Ursulin175fded2016-11-16 08:55:42 +00002663static unsigned int
2664ilk_plane_wm_reg_max(const struct drm_i915_private *dev_priv,
2665 int level, bool is_sprite)
Ville Syrjälä4e975082014-03-07 18:32:11 +02002666{
Tvrtko Ursulin175fded2016-11-16 08:55:42 +00002667 if (INTEL_GEN(dev_priv) >= 8)
Ville Syrjälä4e975082014-03-07 18:32:11 +02002668 /* BDW primary/sprite plane watermarks */
2669 return level == 0 ? 255 : 2047;
Tvrtko Ursulin175fded2016-11-16 08:55:42 +00002670 else if (INTEL_GEN(dev_priv) >= 7)
Ville Syrjälä4e975082014-03-07 18:32:11 +02002671 /* IVB/HSW primary/sprite plane watermarks */
2672 return level == 0 ? 127 : 1023;
2673 else if (!is_sprite)
2674 /* ILK/SNB primary plane watermarks */
2675 return level == 0 ? 127 : 511;
2676 else
2677 /* ILK/SNB sprite plane watermarks */
2678 return level == 0 ? 63 : 255;
2679}
2680
Tvrtko Ursulin175fded2016-11-16 08:55:42 +00002681static unsigned int
2682ilk_cursor_wm_reg_max(const struct drm_i915_private *dev_priv, int level)
Ville Syrjälä4e975082014-03-07 18:32:11 +02002683{
Tvrtko Ursulin175fded2016-11-16 08:55:42 +00002684 if (INTEL_GEN(dev_priv) >= 7)
Ville Syrjälä4e975082014-03-07 18:32:11 +02002685 return level == 0 ? 63 : 255;
2686 else
2687 return level == 0 ? 31 : 63;
2688}
2689
Tvrtko Ursulin175fded2016-11-16 08:55:42 +00002690static unsigned int ilk_fbc_wm_reg_max(const struct drm_i915_private *dev_priv)
Ville Syrjälä4e975082014-03-07 18:32:11 +02002691{
Tvrtko Ursulin175fded2016-11-16 08:55:42 +00002692 if (INTEL_GEN(dev_priv) >= 8)
Ville Syrjälä4e975082014-03-07 18:32:11 +02002693 return 31;
2694 else
2695 return 15;
2696}
2697
Ville Syrjälä158ae642013-08-07 13:28:19 +03002698/* Calculate the maximum primary/sprite plane watermark */
Matt Ropercd1d3ee2018-12-10 13:54:14 -08002699static unsigned int ilk_plane_wm_max(const struct drm_i915_private *dev_priv,
Ville Syrjälä158ae642013-08-07 13:28:19 +03002700 int level,
Ville Syrjälä240264f2013-08-07 13:29:12 +03002701 const struct intel_wm_config *config,
Ville Syrjälä158ae642013-08-07 13:28:19 +03002702 enum intel_ddb_partitioning ddb_partitioning,
2703 bool is_sprite)
2704{
Tvrtko Ursulin175fded2016-11-16 08:55:42 +00002705 unsigned int fifo_size = ilk_display_fifo_size(dev_priv);
Ville Syrjälä158ae642013-08-07 13:28:19 +03002706
2707 /* if sprites aren't enabled, sprites get nothing */
Ville Syrjälä240264f2013-08-07 13:29:12 +03002708 if (is_sprite && !config->sprites_enabled)
Ville Syrjälä158ae642013-08-07 13:28:19 +03002709 return 0;
2710
2711 /* HSW allows LP1+ watermarks even with multiple pipes */
Ville Syrjälä240264f2013-08-07 13:29:12 +03002712 if (level == 0 || config->num_pipes_active > 1) {
Jani Nikula24977872019-09-11 12:26:08 +03002713 fifo_size /= INTEL_NUM_PIPES(dev_priv);
Ville Syrjälä158ae642013-08-07 13:28:19 +03002714
2715 /*
2716 * For some reason the non self refresh
2717 * FIFO size is only half of the self
2718 * refresh FIFO size on ILK/SNB.
2719 */
Tvrtko Ursulin175fded2016-11-16 08:55:42 +00002720 if (INTEL_GEN(dev_priv) <= 6)
Ville Syrjälä158ae642013-08-07 13:28:19 +03002721 fifo_size /= 2;
2722 }
2723
Ville Syrjälä240264f2013-08-07 13:29:12 +03002724 if (config->sprites_enabled) {
Ville Syrjälä158ae642013-08-07 13:28:19 +03002725 /* level 0 is always calculated with 1:1 split */
2726 if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
2727 if (is_sprite)
2728 fifo_size *= 5;
2729 fifo_size /= 6;
2730 } else {
2731 fifo_size /= 2;
2732 }
2733 }
2734
2735 /* clamp to max that the registers can hold */
Tvrtko Ursulin175fded2016-11-16 08:55:42 +00002736 return min(fifo_size, ilk_plane_wm_reg_max(dev_priv, level, is_sprite));
Ville Syrjälä158ae642013-08-07 13:28:19 +03002737}
2738
2739/* Calculate the maximum cursor plane watermark */
Matt Ropercd1d3ee2018-12-10 13:54:14 -08002740static unsigned int ilk_cursor_wm_max(const struct drm_i915_private *dev_priv,
Ville Syrjälä240264f2013-08-07 13:29:12 +03002741 int level,
2742 const struct intel_wm_config *config)
Ville Syrjälä158ae642013-08-07 13:28:19 +03002743{
2744 /* HSW LP1+ watermarks w/ multiple pipes */
Ville Syrjälä240264f2013-08-07 13:29:12 +03002745 if (level > 0 && config->num_pipes_active > 1)
Ville Syrjälä158ae642013-08-07 13:28:19 +03002746 return 64;
2747
2748 /* otherwise just report max that registers can hold */
Matt Ropercd1d3ee2018-12-10 13:54:14 -08002749 return ilk_cursor_wm_reg_max(dev_priv, level);
Ville Syrjälä158ae642013-08-07 13:28:19 +03002750}
2751
Matt Ropercd1d3ee2018-12-10 13:54:14 -08002752static void ilk_compute_wm_maximums(const struct drm_i915_private *dev_priv,
Ville Syrjälä34982fe2013-10-09 19:18:09 +03002753 int level,
2754 const struct intel_wm_config *config,
2755 enum intel_ddb_partitioning ddb_partitioning,
Imre Deak820c1982013-12-17 14:46:36 +02002756 struct ilk_wm_maximums *max)
Ville Syrjälä158ae642013-08-07 13:28:19 +03002757{
Matt Ropercd1d3ee2018-12-10 13:54:14 -08002758 max->pri = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, false);
2759 max->spr = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, true);
2760 max->cur = ilk_cursor_wm_max(dev_priv, level, config);
2761 max->fbc = ilk_fbc_wm_reg_max(dev_priv);
Ville Syrjälä158ae642013-08-07 13:28:19 +03002762}
2763
Tvrtko Ursulin175fded2016-11-16 08:55:42 +00002764static void ilk_compute_wm_reg_maximums(const struct drm_i915_private *dev_priv,
Ville Syrjäläa3cb4042014-04-28 15:44:56 +03002765 int level,
2766 struct ilk_wm_maximums *max)
2767{
Tvrtko Ursulin175fded2016-11-16 08:55:42 +00002768 max->pri = ilk_plane_wm_reg_max(dev_priv, level, false);
2769 max->spr = ilk_plane_wm_reg_max(dev_priv, level, true);
2770 max->cur = ilk_cursor_wm_reg_max(dev_priv, level);
2771 max->fbc = ilk_fbc_wm_reg_max(dev_priv);
Ville Syrjäläa3cb4042014-04-28 15:44:56 +03002772}
2773
Ville Syrjäläd9395652013-10-09 19:18:10 +03002774static bool ilk_validate_wm_level(int level,
Imre Deak820c1982013-12-17 14:46:36 +02002775 const struct ilk_wm_maximums *max,
Ville Syrjäläd9395652013-10-09 19:18:10 +03002776 struct intel_wm_level *result)
Ville Syrjäläa9786a12013-08-07 13:24:47 +03002777{
2778 bool ret;
2779
2780 /* already determined to be invalid? */
2781 if (!result->enable)
2782 return false;
2783
2784 result->enable = result->pri_val <= max->pri &&
2785 result->spr_val <= max->spr &&
2786 result->cur_val <= max->cur;
2787
2788 ret = result->enable;
2789
2790 /*
2791 * HACK until we can pre-compute everything,
2792 * and thus fail gracefully if LP0 watermarks
2793 * are exceeded...
2794 */
2795 if (level == 0 && !result->enable) {
2796 if (result->pri_val > max->pri)
2797 DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
2798 level, result->pri_val, max->pri);
2799 if (result->spr_val > max->spr)
2800 DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
2801 level, result->spr_val, max->spr);
2802 if (result->cur_val > max->cur)
2803 DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
2804 level, result->cur_val, max->cur);
2805
Jani Nikula5ce9a6492019-01-18 14:01:20 +02002806 result->pri_val = min_t(u32, result->pri_val, max->pri);
2807 result->spr_val = min_t(u32, result->spr_val, max->spr);
2808 result->cur_val = min_t(u32, result->cur_val, max->cur);
Ville Syrjäläa9786a12013-08-07 13:24:47 +03002809 result->enable = true;
2810 }
2811
Ville Syrjäläa9786a12013-08-07 13:24:47 +03002812 return ret;
2813}
2814
Damien Lespiaud34ff9c2014-01-06 19:17:23 +00002815static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
Ville Syrjälä2a67054b2020-02-25 19:11:06 +02002816 const struct intel_crtc *crtc,
Ville Syrjälä6f5ddd12013-08-06 22:24:02 +03002817 int level,
Maarten Lankhorstec193642019-06-28 10:55:17 +02002818 struct intel_crtc_state *crtc_state,
Maarten Lankhorst28283f42017-10-19 17:13:40 +02002819 const struct intel_plane_state *pristate,
2820 const struct intel_plane_state *sprstate,
2821 const struct intel_plane_state *curstate,
Ville Syrjälä1fd527c2013-08-06 22:24:05 +03002822 struct intel_wm_level *result)
Ville Syrjälä6f5ddd12013-08-06 22:24:02 +03002823{
Jani Nikula5ce9a6492019-01-18 14:01:20 +02002824 u16 pri_latency = dev_priv->wm.pri_latency[level];
2825 u16 spr_latency = dev_priv->wm.spr_latency[level];
2826 u16 cur_latency = dev_priv->wm.cur_latency[level];
Ville Syrjälä6f5ddd12013-08-06 22:24:02 +03002827
2828 /* WM1+ latency values stored in 0.5us units */
2829 if (level > 0) {
2830 pri_latency *= 5;
2831 spr_latency *= 5;
2832 cur_latency *= 5;
2833 }
2834
Maarten Lankhorste3bddde2016-03-01 11:07:22 +01002835 if (pristate) {
Maarten Lankhorstec193642019-06-28 10:55:17 +02002836 result->pri_val = ilk_compute_pri_wm(crtc_state, pristate,
Maarten Lankhorste3bddde2016-03-01 11:07:22 +01002837 pri_latency, level);
Maarten Lankhorstec193642019-06-28 10:55:17 +02002838 result->fbc_val = ilk_compute_fbc_wm(crtc_state, pristate, result->pri_val);
Maarten Lankhorste3bddde2016-03-01 11:07:22 +01002839 }
2840
2841 if (sprstate)
Maarten Lankhorstec193642019-06-28 10:55:17 +02002842 result->spr_val = ilk_compute_spr_wm(crtc_state, sprstate, spr_latency);
Maarten Lankhorste3bddde2016-03-01 11:07:22 +01002843
2844 if (curstate)
Maarten Lankhorstec193642019-06-28 10:55:17 +02002845 result->cur_val = ilk_compute_cur_wm(crtc_state, curstate, cur_latency);
Maarten Lankhorste3bddde2016-03-01 11:07:22 +01002846
Ville Syrjälä6f5ddd12013-08-06 22:24:02 +03002847 result->enable = true;
2848}
2849
Ville Syrjäläbb726512016-10-31 22:37:24 +02002850static void intel_read_wm_latency(struct drm_i915_private *dev_priv,
Jani Nikula5ce9a6492019-01-18 14:01:20 +02002851 u16 wm[8])
Ville Syrjälä12b134d2013-07-05 11:57:21 +03002852{
Tvrtko Ursulin1cea02d2019-06-10 13:06:07 +01002853 struct intel_uncore *uncore = &dev_priv->uncore;
2854
Paulo Zanoni50682ee2017-08-09 13:52:43 -07002855 if (INTEL_GEN(dev_priv) >= 9) {
Jani Nikula5ce9a6492019-01-18 14:01:20 +02002856 u32 val;
Vandana Kannan4f947382014-11-04 17:06:47 +00002857 int ret, i;
Tvrtko Ursulin5db94012016-10-13 11:03:10 +01002858 int level, max_level = ilk_wm_max_level(dev_priv);
Pradeep Bhat2af30a52014-11-04 17:06:38 +00002859
2860 /* read the first set of memory latencies[0:3] */
2861 val = 0; /* data0 to be programmed to 0 for first set */
Pradeep Bhat2af30a52014-11-04 17:06:38 +00002862 ret = sandybridge_pcode_read(dev_priv,
2863 GEN9_PCODE_READ_MEM_LATENCY,
Ville Syrjäläd284d512019-05-21 19:40:24 +03002864 &val, NULL);
Pradeep Bhat2af30a52014-11-04 17:06:38 +00002865
2866 if (ret) {
Wambui Karugaf8d18d52020-01-07 18:13:30 +03002867 drm_err(&dev_priv->drm,
2868 "SKL Mailbox read error = %d\n", ret);
Pradeep Bhat2af30a52014-11-04 17:06:38 +00002869 return;
2870 }
2871
2872 wm[0] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
2873 wm[1] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2874 GEN9_MEM_LATENCY_LEVEL_MASK;
2875 wm[2] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2876 GEN9_MEM_LATENCY_LEVEL_MASK;
2877 wm[3] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2878 GEN9_MEM_LATENCY_LEVEL_MASK;
2879
2880 /* read the second set of memory latencies[4:7] */
2881 val = 1; /* data0 to be programmed to 1 for second set */
Pradeep Bhat2af30a52014-11-04 17:06:38 +00002882 ret = sandybridge_pcode_read(dev_priv,
2883 GEN9_PCODE_READ_MEM_LATENCY,
Ville Syrjäläd284d512019-05-21 19:40:24 +03002884 &val, NULL);
Pradeep Bhat2af30a52014-11-04 17:06:38 +00002885 if (ret) {
Wambui Karugaf8d18d52020-01-07 18:13:30 +03002886 drm_err(&dev_priv->drm,
2887 "SKL Mailbox read error = %d\n", ret);
Pradeep Bhat2af30a52014-11-04 17:06:38 +00002888 return;
2889 }
2890
2891 wm[4] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
2892 wm[5] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2893 GEN9_MEM_LATENCY_LEVEL_MASK;
2894 wm[6] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2895 GEN9_MEM_LATENCY_LEVEL_MASK;
2896 wm[7] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2897 GEN9_MEM_LATENCY_LEVEL_MASK;
2898
Vandana Kannan367294b2014-11-04 17:06:46 +00002899 /*
Paulo Zanoni0727e402016-09-22 18:00:30 -03002900 * If a level n (n > 1) has a 0us latency, all levels m (m >= n)
2901 * need to be disabled. We make sure to sanitize the values out
2902 * of the punit to satisfy this requirement.
2903 */
2904 for (level = 1; level <= max_level; level++) {
2905 if (wm[level] == 0) {
2906 for (i = level + 1; i <= max_level; i++)
2907 wm[i] = 0;
2908 break;
2909 }
2910 }
2911
2912 /*
Paulo Zanoni50682ee2017-08-09 13:52:43 -07002913 * WaWmMemoryReadLatency:skl+,glk
Damien Lespiau6f972352015-02-09 19:33:07 +00002914 *
Vandana Kannan367294b2014-11-04 17:06:46 +00002915 * punit doesn't take into account the read latency so we need
Paulo Zanoni0727e402016-09-22 18:00:30 -03002916 * to add 2us to the various latency levels we retrieve from the
2917 * punit when level 0 response data us 0us.
Vandana Kannan367294b2014-11-04 17:06:46 +00002918 */
Paulo Zanoni0727e402016-09-22 18:00:30 -03002919 if (wm[0] == 0) {
2920 wm[0] += 2;
2921 for (level = 1; level <= max_level; level++) {
2922 if (wm[level] == 0)
2923 break;
Vandana Kannan367294b2014-11-04 17:06:46 +00002924 wm[level] += 2;
Vandana Kannan4f947382014-11-04 17:06:47 +00002925 }
Paulo Zanoni0727e402016-09-22 18:00:30 -03002926 }
2927
Mahesh Kumar86b59282018-08-31 16:39:42 +05302928 /*
2929 * WA Level-0 adjustment for 16GB DIMMs: SKL+
2930 * If we could not get dimm info enable this WA to prevent from
2931 * any underrun. If not able to get Dimm info assume 16GB dimm
2932 * to avoid any underrun.
2933 */
José Roberto de Souza66a24502021-01-28 08:43:12 -08002934 if (dev_priv->dram_info.wm_lv_0_adjust_needed)
Mahesh Kumar86b59282018-08-31 16:39:42 +05302935 wm[0] += 1;
2936
Tvrtko Ursulin86527442016-10-13 11:03:00 +01002937 } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
Tvrtko Ursulin1cea02d2019-06-10 13:06:07 +01002938 u64 sskpd = intel_uncore_read64(uncore, MCH_SSKPD);
Ville Syrjälä12b134d2013-07-05 11:57:21 +03002939
2940 wm[0] = (sskpd >> 56) & 0xFF;
2941 if (wm[0] == 0)
2942 wm[0] = sskpd & 0xF;
Ville Syrjäläe5d50192013-07-05 11:57:22 +03002943 wm[1] = (sskpd >> 4) & 0xFF;
2944 wm[2] = (sskpd >> 12) & 0xFF;
2945 wm[3] = (sskpd >> 20) & 0x1FF;
2946 wm[4] = (sskpd >> 32) & 0x1FF;
Ville Syrjäläbb726512016-10-31 22:37:24 +02002947 } else if (INTEL_GEN(dev_priv) >= 6) {
Tvrtko Ursulin1cea02d2019-06-10 13:06:07 +01002948 u32 sskpd = intel_uncore_read(uncore, MCH_SSKPD);
Ville Syrjälä63cf9a12013-07-05 11:57:23 +03002949
2950 wm[0] = (sskpd >> SSKPD_WM0_SHIFT) & SSKPD_WM_MASK;
2951 wm[1] = (sskpd >> SSKPD_WM1_SHIFT) & SSKPD_WM_MASK;
2952 wm[2] = (sskpd >> SSKPD_WM2_SHIFT) & SSKPD_WM_MASK;
2953 wm[3] = (sskpd >> SSKPD_WM3_SHIFT) & SSKPD_WM_MASK;
Ville Syrjäläbb726512016-10-31 22:37:24 +02002954 } else if (INTEL_GEN(dev_priv) >= 5) {
Tvrtko Ursulin1cea02d2019-06-10 13:06:07 +01002955 u32 mltr = intel_uncore_read(uncore, MLTR_ILK);
Ville Syrjälä3a88d0a2013-08-01 16:18:49 +03002956
2957 /* ILK primary LP0 latency is 700 ns */
2958 wm[0] = 7;
2959 wm[1] = (mltr >> MLTR_WM1_SHIFT) & ILK_SRLT_MASK;
2960 wm[2] = (mltr >> MLTR_WM2_SHIFT) & ILK_SRLT_MASK;
Paulo Zanoni50682ee2017-08-09 13:52:43 -07002961 } else {
2962 MISSING_CASE(INTEL_DEVID(dev_priv));
Ville Syrjälä12b134d2013-07-05 11:57:21 +03002963 }
2964}
2965
Tvrtko Ursulin5db94012016-10-13 11:03:10 +01002966static void intel_fixup_spr_wm_latency(struct drm_i915_private *dev_priv,
Jani Nikula5ce9a6492019-01-18 14:01:20 +02002967 u16 wm[5])
Ville Syrjälä53615a52013-08-01 16:18:50 +03002968{
2969 /* ILK sprite LP0 latency is 1300 ns */
Lucas De Marchicf819ef2018-12-12 10:10:43 -08002970 if (IS_GEN(dev_priv, 5))
Ville Syrjälä53615a52013-08-01 16:18:50 +03002971 wm[0] = 13;
2972}
2973
Tvrtko Ursulinfd6b8f42016-10-14 10:13:06 +01002974static void intel_fixup_cur_wm_latency(struct drm_i915_private *dev_priv,
Jani Nikula5ce9a6492019-01-18 14:01:20 +02002975 u16 wm[5])
Ville Syrjälä53615a52013-08-01 16:18:50 +03002976{
2977 /* ILK cursor LP0 latency is 1300 ns */
Lucas De Marchicf819ef2018-12-12 10:10:43 -08002978 if (IS_GEN(dev_priv, 5))
Ville Syrjälä53615a52013-08-01 16:18:50 +03002979 wm[0] = 13;
Ville Syrjälä53615a52013-08-01 16:18:50 +03002980}
2981
Tvrtko Ursulin5db94012016-10-13 11:03:10 +01002982int ilk_wm_max_level(const struct drm_i915_private *dev_priv)
Ville Syrjäläad0d6dc2013-08-30 14:30:25 +03002983{
2984 /* how many WM levels are we expecting */
Tvrtko Ursulin86527442016-10-13 11:03:00 +01002985 if (INTEL_GEN(dev_priv) >= 9)
Pradeep Bhat2af30a52014-11-04 17:06:38 +00002986 return 7;
Tvrtko Ursulin86527442016-10-13 11:03:00 +01002987 else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
Ville Syrjäläad0d6dc2013-08-30 14:30:25 +03002988 return 4;
Tvrtko Ursulin86527442016-10-13 11:03:00 +01002989 else if (INTEL_GEN(dev_priv) >= 6)
Ville Syrjäläad0d6dc2013-08-30 14:30:25 +03002990 return 3;
2991 else
2992 return 2;
2993}
Daniel Vetter7526ed72014-09-29 15:07:19 +02002994
Tvrtko Ursulin5db94012016-10-13 11:03:10 +01002995static void intel_print_wm_latency(struct drm_i915_private *dev_priv,
Ville Syrjälä26ec9712013-08-01 16:18:52 +03002996 const char *name,
Jani Nikula5ce9a6492019-01-18 14:01:20 +02002997 const u16 wm[8])
Ville Syrjälä26ec9712013-08-01 16:18:52 +03002998{
Tvrtko Ursulin5db94012016-10-13 11:03:10 +01002999 int level, max_level = ilk_wm_max_level(dev_priv);
Ville Syrjälä26ec9712013-08-01 16:18:52 +03003000
3001 for (level = 0; level <= max_level; level++) {
3002 unsigned int latency = wm[level];
3003
3004 if (latency == 0) {
Wambui Karugaf8d18d52020-01-07 18:13:30 +03003005 drm_dbg_kms(&dev_priv->drm,
3006 "%s WM%d latency not provided\n",
3007 name, level);
Ville Syrjälä26ec9712013-08-01 16:18:52 +03003008 continue;
3009 }
3010
Pradeep Bhat2af30a52014-11-04 17:06:38 +00003011 /*
3012 * - latencies are in us on gen9.
3013 * - before then, WM1+ latency values are in 0.5us units
3014 */
Paulo Zanonidfc267a2017-08-09 13:52:46 -07003015 if (INTEL_GEN(dev_priv) >= 9)
Pradeep Bhat2af30a52014-11-04 17:06:38 +00003016 latency *= 10;
3017 else if (level > 0)
Ville Syrjälä26ec9712013-08-01 16:18:52 +03003018 latency *= 5;
3019
Wambui Karugaf8d18d52020-01-07 18:13:30 +03003020 drm_dbg_kms(&dev_priv->drm,
3021 "%s WM%d latency %u (%u.%u usec)\n", name, level,
3022 wm[level], latency / 10, latency % 10);
Ville Syrjälä26ec9712013-08-01 16:18:52 +03003023 }
3024}
3025
Ville Syrjäläe95a2f72014-05-08 15:09:19 +03003026static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
Jani Nikula5ce9a6492019-01-18 14:01:20 +02003027 u16 wm[5], u16 min)
Ville Syrjäläe95a2f72014-05-08 15:09:19 +03003028{
Tvrtko Ursulin5db94012016-10-13 11:03:10 +01003029 int level, max_level = ilk_wm_max_level(dev_priv);
Ville Syrjäläe95a2f72014-05-08 15:09:19 +03003030
3031 if (wm[0] >= min)
3032 return false;
3033
3034 wm[0] = max(wm[0], min);
3035 for (level = 1; level <= max_level; level++)
Jani Nikula5ce9a6492019-01-18 14:01:20 +02003036 wm[level] = max_t(u16, wm[level], DIV_ROUND_UP(min, 5));
Ville Syrjäläe95a2f72014-05-08 15:09:19 +03003037
3038 return true;
3039}
3040
Ville Syrjäläbb726512016-10-31 22:37:24 +02003041static void snb_wm_latency_quirk(struct drm_i915_private *dev_priv)
Ville Syrjäläe95a2f72014-05-08 15:09:19 +03003042{
Ville Syrjäläe95a2f72014-05-08 15:09:19 +03003043 bool changed;
3044
3045 /*
3046 * The BIOS provided WM memory latency values are often
3047 * inadequate for high resolution displays. Adjust them.
3048 */
3049 changed = ilk_increase_wm_latency(dev_priv, dev_priv->wm.pri_latency, 12) |
3050 ilk_increase_wm_latency(dev_priv, dev_priv->wm.spr_latency, 12) |
3051 ilk_increase_wm_latency(dev_priv, dev_priv->wm.cur_latency, 12);
3052
3053 if (!changed)
3054 return;
3055
Wambui Karugaf8d18d52020-01-07 18:13:30 +03003056 drm_dbg_kms(&dev_priv->drm,
3057 "WM latency values increased to avoid potential underruns\n");
Tvrtko Ursulin5db94012016-10-13 11:03:10 +01003058 intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3059 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3060 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
Ville Syrjäläe95a2f72014-05-08 15:09:19 +03003061}
3062
Ville Syrjälä03981c62018-11-14 19:34:40 +02003063static void snb_wm_lp3_irq_quirk(struct drm_i915_private *dev_priv)
3064{
3065 /*
3066 * On some SNB machines (Thinkpad X220 Tablet at least)
3067 * LP3 usage can cause vblank interrupts to be lost.
3068 * The DEIIR bit will go high but it looks like the CPU
3069 * never gets interrupted.
3070 *
3071 * It's not clear whether other interrupt source could
3072 * be affected or if this is somehow limited to vblank
3073 * interrupts only. To play it safe we disable LP3
3074 * watermarks entirely.
3075 */
3076 if (dev_priv->wm.pri_latency[3] == 0 &&
3077 dev_priv->wm.spr_latency[3] == 0 &&
3078 dev_priv->wm.cur_latency[3] == 0)
3079 return;
3080
3081 dev_priv->wm.pri_latency[3] = 0;
3082 dev_priv->wm.spr_latency[3] = 0;
3083 dev_priv->wm.cur_latency[3] = 0;
3084
Wambui Karugaf8d18d52020-01-07 18:13:30 +03003085 drm_dbg_kms(&dev_priv->drm,
3086 "LP3 watermarks disabled due to potential for lost interrupts\n");
Ville Syrjälä03981c62018-11-14 19:34:40 +02003087 intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3088 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3089 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3090}
3091
Ville Syrjäläbb726512016-10-31 22:37:24 +02003092static void ilk_setup_wm_latency(struct drm_i915_private *dev_priv)
Ville Syrjälä53615a52013-08-01 16:18:50 +03003093{
Ville Syrjäläbb726512016-10-31 22:37:24 +02003094 intel_read_wm_latency(dev_priv, dev_priv->wm.pri_latency);
Ville Syrjälä53615a52013-08-01 16:18:50 +03003095
3096 memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency,
3097 sizeof(dev_priv->wm.pri_latency));
3098 memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency,
3099 sizeof(dev_priv->wm.pri_latency));
3100
Tvrtko Ursulin5db94012016-10-13 11:03:10 +01003101 intel_fixup_spr_wm_latency(dev_priv, dev_priv->wm.spr_latency);
Tvrtko Ursulinfd6b8f42016-10-14 10:13:06 +01003102 intel_fixup_cur_wm_latency(dev_priv, dev_priv->wm.cur_latency);
Ville Syrjälä26ec9712013-08-01 16:18:52 +03003103
Tvrtko Ursulin5db94012016-10-13 11:03:10 +01003104 intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3105 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3106 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
Ville Syrjäläe95a2f72014-05-08 15:09:19 +03003107
Lucas De Marchicf819ef2018-12-12 10:10:43 -08003108 if (IS_GEN(dev_priv, 6)) {
Ville Syrjäläbb726512016-10-31 22:37:24 +02003109 snb_wm_latency_quirk(dev_priv);
Ville Syrjälä03981c62018-11-14 19:34:40 +02003110 snb_wm_lp3_irq_quirk(dev_priv);
3111 }
Ville Syrjälä53615a52013-08-01 16:18:50 +03003112}
3113
Ville Syrjäläbb726512016-10-31 22:37:24 +02003114static void skl_setup_wm_latency(struct drm_i915_private *dev_priv)
Pradeep Bhat2af30a52014-11-04 17:06:38 +00003115{
Ville Syrjäläbb726512016-10-31 22:37:24 +02003116 intel_read_wm_latency(dev_priv, dev_priv->wm.skl_latency);
Tvrtko Ursulin5db94012016-10-13 11:03:10 +01003117 intel_print_wm_latency(dev_priv, "Gen9 Plane", dev_priv->wm.skl_latency);
Pradeep Bhat2af30a52014-11-04 17:06:38 +00003118}
3119
Matt Ropercd1d3ee2018-12-10 13:54:14 -08003120static bool ilk_validate_pipe_wm(const struct drm_i915_private *dev_priv,
Matt Ropered4a6a72016-02-23 17:20:13 -08003121 struct intel_pipe_wm *pipe_wm)
3122{
3123 /* LP0 watermark maximums depend on this pipe alone */
3124 const struct intel_wm_config config = {
3125 .num_pipes_active = 1,
3126 .sprites_enabled = pipe_wm->sprites_enabled,
3127 .sprites_scaled = pipe_wm->sprites_scaled,
3128 };
3129 struct ilk_wm_maximums max;
3130
3131 /* LP0 watermarks always use 1/2 DDB partitioning */
Matt Ropercd1d3ee2018-12-10 13:54:14 -08003132 ilk_compute_wm_maximums(dev_priv, 0, &config, INTEL_DDB_PART_1_2, &max);
Matt Ropered4a6a72016-02-23 17:20:13 -08003133
3134 /* At least LP0 must be valid */
3135 if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0])) {
Wambui Karugaf8d18d52020-01-07 18:13:30 +03003136 drm_dbg_kms(&dev_priv->drm, "LP0 watermark invalid\n");
Matt Ropered4a6a72016-02-23 17:20:13 -08003137 return false;
3138 }
3139
3140 return true;
3141}
3142
Matt Roper261a27d2015-10-08 15:28:25 -07003143/* Compute new watermarks for the pipe */
Maarten Lankhorstec193642019-06-28 10:55:17 +02003144static int ilk_compute_pipe_wm(struct intel_crtc_state *crtc_state)
Matt Roper261a27d2015-10-08 15:28:25 -07003145{
Maarten Lankhorst2225f3c2019-10-31 12:26:03 +01003146 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
Ville Syrjälä2a67054b2020-02-25 19:11:06 +02003147 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
Matt Roper86c8bbb2015-09-24 15:53:16 -07003148 struct intel_pipe_wm *pipe_wm;
Maarten Lankhorstaf9fbfa2019-10-04 13:34:53 +02003149 struct intel_plane *plane;
3150 const struct intel_plane_state *plane_state;
Maarten Lankhorst28283f42017-10-19 17:13:40 +02003151 const struct intel_plane_state *pristate = NULL;
3152 const struct intel_plane_state *sprstate = NULL;
3153 const struct intel_plane_state *curstate = NULL;
Tvrtko Ursulin5db94012016-10-13 11:03:10 +01003154 int level, max_level = ilk_wm_max_level(dev_priv), usable_level;
Imre Deak820c1982013-12-17 14:46:36 +02003155 struct ilk_wm_maximums max;
Ville Syrjälä0b2ae6d2013-10-09 19:17:55 +03003156
Maarten Lankhorstec193642019-06-28 10:55:17 +02003157 pipe_wm = &crtc_state->wm.ilk.optimal;
Matt Roper86c8bbb2015-09-24 15:53:16 -07003158
Maarten Lankhorstaf9fbfa2019-10-04 13:34:53 +02003159 intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
3160 if (plane->base.type == DRM_PLANE_TYPE_PRIMARY)
3161 pristate = plane_state;
3162 else if (plane->base.type == DRM_PLANE_TYPE_OVERLAY)
3163 sprstate = plane_state;
3164 else if (plane->base.type == DRM_PLANE_TYPE_CURSOR)
3165 curstate = plane_state;
Matt Roper43d59ed2015-09-24 15:53:07 -07003166 }
3167
Maarten Lankhorst1326a922019-10-31 12:26:02 +01003168 pipe_wm->pipe_enabled = crtc_state->hw.active;
Maarten Lankhorste3bddde2016-03-01 11:07:22 +01003169 if (sprstate) {
Maarten Lankhorstf90a85e2019-10-31 12:26:08 +01003170 pipe_wm->sprites_enabled = sprstate->uapi.visible;
3171 pipe_wm->sprites_scaled = sprstate->uapi.visible &&
3172 (drm_rect_width(&sprstate->uapi.dst) != drm_rect_width(&sprstate->uapi.src) >> 16 ||
3173 drm_rect_height(&sprstate->uapi.dst) != drm_rect_height(&sprstate->uapi.src) >> 16);
Maarten Lankhorste3bddde2016-03-01 11:07:22 +01003174 }
3175
Maarten Lankhorstd81f04c2016-03-02 12:38:06 +01003176 usable_level = max_level;
3177
Ville Syrjälä7b39a0b2013-12-05 15:51:30 +02003178 /* ILK/SNB: LP2+ watermarks only w/o sprites */
Tvrtko Ursulin175fded2016-11-16 08:55:42 +00003179 if (INTEL_GEN(dev_priv) <= 6 && pipe_wm->sprites_enabled)
Maarten Lankhorstd81f04c2016-03-02 12:38:06 +01003180 usable_level = 1;
Ville Syrjälä7b39a0b2013-12-05 15:51:30 +02003181
3182 /* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
Matt Ropered4a6a72016-02-23 17:20:13 -08003183 if (pipe_wm->sprites_scaled)
Maarten Lankhorstd81f04c2016-03-02 12:38:06 +01003184 usable_level = 0;
Ville Syrjälä7b39a0b2013-12-05 15:51:30 +02003185
Maarten Lankhorst71f0a622016-03-08 10:57:16 +01003186 memset(&pipe_wm->wm, 0, sizeof(pipe_wm->wm));
Ville Syrjälä2a67054b2020-02-25 19:11:06 +02003187 ilk_compute_wm_level(dev_priv, crtc, 0, crtc_state,
Maarten Lankhorst28283f42017-10-19 17:13:40 +02003188 pristate, sprstate, curstate, &pipe_wm->wm[0]);
Ville Syrjälä0b2ae6d2013-10-09 19:17:55 +03003189
Matt Ropercd1d3ee2018-12-10 13:54:14 -08003190 if (!ilk_validate_pipe_wm(dev_priv, pipe_wm))
Maarten Lankhorst1a426d62016-03-02 12:36:03 +01003191 return -EINVAL;
Ville Syrjäläa3cb4042014-04-28 15:44:56 +03003192
Tvrtko Ursulin175fded2016-11-16 08:55:42 +00003193 ilk_compute_wm_reg_maximums(dev_priv, 1, &max);
Ville Syrjäläa3cb4042014-04-28 15:44:56 +03003194
Maarten Lankhorst28283f42017-10-19 17:13:40 +02003195 for (level = 1; level <= usable_level; level++) {
3196 struct intel_wm_level *wm = &pipe_wm->wm[level];
Ville Syrjäläa3cb4042014-04-28 15:44:56 +03003197
Ville Syrjälä2a67054b2020-02-25 19:11:06 +02003198 ilk_compute_wm_level(dev_priv, crtc, level, crtc_state,
Maarten Lankhorstd81f04c2016-03-02 12:38:06 +01003199 pristate, sprstate, curstate, wm);
Ville Syrjäläa3cb4042014-04-28 15:44:56 +03003200
3201 /*
3202 * Disable any watermark level that exceeds the
3203 * register maximums since such watermarks are
3204 * always invalid.
3205 */
Maarten Lankhorst28283f42017-10-19 17:13:40 +02003206 if (!ilk_validate_wm_level(level, &max, wm)) {
3207 memset(wm, 0, sizeof(*wm));
3208 break;
3209 }
Ville Syrjäläa3cb4042014-04-28 15:44:56 +03003210 }
3211
Matt Roper86c8bbb2015-09-24 15:53:16 -07003212 return 0;
Ville Syrjälä0b2ae6d2013-10-09 19:17:55 +03003213}
3214
3215/*
Matt Ropered4a6a72016-02-23 17:20:13 -08003216 * Build a set of 'intermediate' watermark values that satisfy both the old
3217 * state and the new state. These can be programmed to the hardware
3218 * immediately.
3219 */
Matt Ropercd1d3ee2018-12-10 13:54:14 -08003220static int ilk_compute_intermediate_wm(struct intel_crtc_state *newstate)
Matt Ropered4a6a72016-02-23 17:20:13 -08003221{
Maarten Lankhorst2225f3c2019-10-31 12:26:03 +01003222 struct intel_crtc *intel_crtc = to_intel_crtc(newstate->uapi.crtc);
Matt Ropercd1d3ee2018-12-10 13:54:14 -08003223 struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
Matt Ropere8f1f022016-05-12 07:05:55 -07003224 struct intel_pipe_wm *a = &newstate->wm.ilk.intermediate;
Maarten Lankhorstb6b178a2017-10-19 17:13:41 +02003225 struct intel_atomic_state *intel_state =
Maarten Lankhorst2225f3c2019-10-31 12:26:03 +01003226 to_intel_atomic_state(newstate->uapi.state);
Maarten Lankhorstb6b178a2017-10-19 17:13:41 +02003227 const struct intel_crtc_state *oldstate =
3228 intel_atomic_get_old_crtc_state(intel_state, intel_crtc);
3229 const struct intel_pipe_wm *b = &oldstate->wm.ilk.optimal;
Matt Ropercd1d3ee2018-12-10 13:54:14 -08003230 int level, max_level = ilk_wm_max_level(dev_priv);
Matt Ropered4a6a72016-02-23 17:20:13 -08003231
3232 /*
3233 * Start with the final, target watermarks, then combine with the
3234 * currently active watermarks to get values that are safe both before
3235 * and after the vblank.
3236 */
Matt Ropere8f1f022016-05-12 07:05:55 -07003237 *a = newstate->wm.ilk.optimal;
Maarten Lankhorst2225f3c2019-10-31 12:26:03 +01003238 if (!newstate->hw.active || drm_atomic_crtc_needs_modeset(&newstate->uapi) ||
Ville Syrjäläf255c622018-11-08 17:10:13 +02003239 intel_state->skip_intermediate_wm)
Maarten Lankhorstb6b178a2017-10-19 17:13:41 +02003240 return 0;
3241
Matt Ropered4a6a72016-02-23 17:20:13 -08003242 a->pipe_enabled |= b->pipe_enabled;
3243 a->sprites_enabled |= b->sprites_enabled;
3244 a->sprites_scaled |= b->sprites_scaled;
3245
3246 for (level = 0; level <= max_level; level++) {
3247 struct intel_wm_level *a_wm = &a->wm[level];
3248 const struct intel_wm_level *b_wm = &b->wm[level];
3249
3250 a_wm->enable &= b_wm->enable;
3251 a_wm->pri_val = max(a_wm->pri_val, b_wm->pri_val);
3252 a_wm->spr_val = max(a_wm->spr_val, b_wm->spr_val);
3253 a_wm->cur_val = max(a_wm->cur_val, b_wm->cur_val);
3254 a_wm->fbc_val = max(a_wm->fbc_val, b_wm->fbc_val);
3255 }
3256
3257 /*
3258 * We need to make sure that these merged watermark values are
3259 * actually a valid configuration themselves. If they're not,
3260 * there's no safe way to transition from the old state to
3261 * the new state, so we need to fail the atomic transaction.
3262 */
Matt Ropercd1d3ee2018-12-10 13:54:14 -08003263 if (!ilk_validate_pipe_wm(dev_priv, a))
Matt Ropered4a6a72016-02-23 17:20:13 -08003264 return -EINVAL;
3265
3266 /*
3267 * If our intermediate WM are identical to the final WM, then we can
3268 * omit the post-vblank programming; only update if it's different.
3269 */
Ville Syrjälä5eeb7982017-03-02 19:15:00 +02003270 if (memcmp(a, &newstate->wm.ilk.optimal, sizeof(*a)) != 0)
3271 newstate->wm.need_postvbl_update = true;
Matt Ropered4a6a72016-02-23 17:20:13 -08003272
3273 return 0;
3274}
3275
3276/*
Ville Syrjälä0b2ae6d2013-10-09 19:17:55 +03003277 * Merge the watermarks from all active pipes for a specific level.
3278 */
Matt Ropercd1d3ee2018-12-10 13:54:14 -08003279static void ilk_merge_wm_level(struct drm_i915_private *dev_priv,
Ville Syrjälä0b2ae6d2013-10-09 19:17:55 +03003280 int level,
3281 struct intel_wm_level *ret_wm)
3282{
3283 const struct intel_crtc *intel_crtc;
3284
Ville Syrjäläd52fea52014-04-28 15:44:57 +03003285 ret_wm->enable = true;
3286
Matt Ropercd1d3ee2018-12-10 13:54:14 -08003287 for_each_intel_crtc(&dev_priv->drm, intel_crtc) {
Matt Ropered4a6a72016-02-23 17:20:13 -08003288 const struct intel_pipe_wm *active = &intel_crtc->wm.active.ilk;
Ville Syrjäläfe392ef2014-03-07 18:32:10 +02003289 const struct intel_wm_level *wm = &active->wm[level];
3290
3291 if (!active->pipe_enabled)
3292 continue;
Ville Syrjälä0b2ae6d2013-10-09 19:17:55 +03003293
Ville Syrjäläd52fea52014-04-28 15:44:57 +03003294 /*
3295 * The watermark values may have been used in the past,
3296 * so we must maintain them in the registers for some
3297 * time even if the level is now disabled.
3298 */
Ville Syrjälä0b2ae6d2013-10-09 19:17:55 +03003299 if (!wm->enable)
Ville Syrjäläd52fea52014-04-28 15:44:57 +03003300 ret_wm->enable = false;
Ville Syrjälä0b2ae6d2013-10-09 19:17:55 +03003301
3302 ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
3303 ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
3304 ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
3305 ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
3306 }
Ville Syrjälä0b2ae6d2013-10-09 19:17:55 +03003307}
3308
3309/*
3310 * Merge all low power watermarks for all active pipes.
3311 */
Matt Ropercd1d3ee2018-12-10 13:54:14 -08003312static void ilk_wm_merge(struct drm_i915_private *dev_priv,
Ville Syrjälä0ba22e22013-12-05 15:51:34 +02003313 const struct intel_wm_config *config,
Imre Deak820c1982013-12-17 14:46:36 +02003314 const struct ilk_wm_maximums *max,
Ville Syrjälä0b2ae6d2013-10-09 19:17:55 +03003315 struct intel_pipe_wm *merged)
3316{
Tvrtko Ursulin5db94012016-10-13 11:03:10 +01003317 int level, max_level = ilk_wm_max_level(dev_priv);
Ville Syrjäläd52fea52014-04-28 15:44:57 +03003318 int last_enabled_level = max_level;
Ville Syrjälä0b2ae6d2013-10-09 19:17:55 +03003319
Ville Syrjälä0ba22e22013-12-05 15:51:34 +02003320 /* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
Tvrtko Ursulinfd6b8f42016-10-14 10:13:06 +01003321 if ((INTEL_GEN(dev_priv) <= 6 || IS_IVYBRIDGE(dev_priv)) &&
Ville Syrjälä0ba22e22013-12-05 15:51:34 +02003322 config->num_pipes_active > 1)
Ville Syrjälä1204d5b2016-04-01 21:53:18 +03003323 last_enabled_level = 0;
Ville Syrjälä0ba22e22013-12-05 15:51:34 +02003324
Ville Syrjälä6c8b6c22013-12-05 15:51:35 +02003325 /* ILK: FBC WM must be disabled always */
Tvrtko Ursulin175fded2016-11-16 08:55:42 +00003326 merged->fbc_wm_enabled = INTEL_GEN(dev_priv) >= 6;
Ville Syrjälä0b2ae6d2013-10-09 19:17:55 +03003327
3328 /* merge each WM1+ level */
3329 for (level = 1; level <= max_level; level++) {
3330 struct intel_wm_level *wm = &merged->wm[level];
3331
Matt Ropercd1d3ee2018-12-10 13:54:14 -08003332 ilk_merge_wm_level(dev_priv, level, wm);
Ville Syrjälä0b2ae6d2013-10-09 19:17:55 +03003333
Ville Syrjäläd52fea52014-04-28 15:44:57 +03003334 if (level > last_enabled_level)
3335 wm->enable = false;
3336 else if (!ilk_validate_wm_level(level, max, wm))
3337 /* make sure all following levels get disabled */
3338 last_enabled_level = level - 1;
Ville Syrjälä0b2ae6d2013-10-09 19:17:55 +03003339
3340 /*
3341 * The spec says it is preferred to disable
3342 * FBC WMs instead of disabling a WM level.
3343 */
3344 if (wm->fbc_val > max->fbc) {
Ville Syrjäläd52fea52014-04-28 15:44:57 +03003345 if (wm->enable)
3346 merged->fbc_wm_enabled = false;
Ville Syrjälä0b2ae6d2013-10-09 19:17:55 +03003347 wm->fbc_val = 0;
3348 }
3349 }
Ville Syrjälä6c8b6c22013-12-05 15:51:35 +02003350
3351 /* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
3352 /*
3353 * FIXME this is racy. FBC might get enabled later.
3354 * What we should check here is whether FBC can be
3355 * enabled sometime later.
3356 */
Lucas De Marchicf819ef2018-12-12 10:10:43 -08003357 if (IS_GEN(dev_priv, 5) && !merged->fbc_wm_enabled &&
Paulo Zanoni0e631ad2015-10-14 17:45:36 -03003358 intel_fbc_is_active(dev_priv)) {
Ville Syrjälä6c8b6c22013-12-05 15:51:35 +02003359 for (level = 2; level <= max_level; level++) {
3360 struct intel_wm_level *wm = &merged->wm[level];
3361
3362 wm->enable = false;
3363 }
3364 }
Ville Syrjälä0b2ae6d2013-10-09 19:17:55 +03003365}
3366
Ville Syrjäläb380ca32013-10-09 19:18:01 +03003367static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
3368{
3369 /* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
3370 return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
3371}
3372
Ville Syrjäläa68d68e2013-12-05 15:51:29 +02003373/* The value we need to program into the WM_LPx latency field */
Matt Ropercd1d3ee2018-12-10 13:54:14 -08003374static unsigned int ilk_wm_lp_latency(struct drm_i915_private *dev_priv,
3375 int level)
Ville Syrjäläa68d68e2013-12-05 15:51:29 +02003376{
Tvrtko Ursulin86527442016-10-13 11:03:00 +01003377 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
Ville Syrjäläa68d68e2013-12-05 15:51:29 +02003378 return 2 * level;
3379 else
3380 return dev_priv->wm.pri_latency[level];
3381}
3382
Matt Ropercd1d3ee2018-12-10 13:54:14 -08003383static void ilk_compute_wm_results(struct drm_i915_private *dev_priv,
Ville Syrjälä0362c782013-10-09 19:17:57 +03003384 const struct intel_pipe_wm *merged,
Ville Syrjälä609cede2013-10-09 19:18:03 +03003385 enum intel_ddb_partitioning partitioning,
Imre Deak820c1982013-12-17 14:46:36 +02003386 struct ilk_wm_values *results)
Paulo Zanoni1011d8c2013-05-09 16:55:50 -03003387{
Ville Syrjälä0b2ae6d2013-10-09 19:17:55 +03003388 struct intel_crtc *intel_crtc;
3389 int level, wm_lp;
Paulo Zanonicca32e92013-05-31 11:45:06 -03003390
Ville Syrjälä0362c782013-10-09 19:17:57 +03003391 results->enable_fbc_wm = merged->fbc_wm_enabled;
Ville Syrjälä609cede2013-10-09 19:18:03 +03003392 results->partitioning = partitioning;
Paulo Zanonicca32e92013-05-31 11:45:06 -03003393
Ville Syrjälä0b2ae6d2013-10-09 19:17:55 +03003394 /* LP1+ register values */
Paulo Zanonicca32e92013-05-31 11:45:06 -03003395 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
Ville Syrjälä1fd527c2013-08-06 22:24:05 +03003396 const struct intel_wm_level *r;
Paulo Zanonicca32e92013-05-31 11:45:06 -03003397
Ville Syrjäläb380ca32013-10-09 19:18:01 +03003398 level = ilk_wm_lp_to_level(wm_lp, merged);
Ville Syrjälä0b2ae6d2013-10-09 19:17:55 +03003399
Ville Syrjälä0362c782013-10-09 19:17:57 +03003400 r = &merged->wm[level];
Paulo Zanonicca32e92013-05-31 11:45:06 -03003401
Ville Syrjäläd52fea52014-04-28 15:44:57 +03003402 /*
3403 * Maintain the watermark values even if the level is
3404 * disabled. Doing otherwise could cause underruns.
3405 */
3406 results->wm_lp[wm_lp - 1] =
Matt Ropercd1d3ee2018-12-10 13:54:14 -08003407 (ilk_wm_lp_latency(dev_priv, level) << WM1_LP_LATENCY_SHIFT) |
Ville Syrjälä416f4722013-11-02 21:07:46 -07003408 (r->pri_val << WM1_LP_SR_SHIFT) |
3409 r->cur_val;
3410
Ville Syrjäläd52fea52014-04-28 15:44:57 +03003411 if (r->enable)
3412 results->wm_lp[wm_lp - 1] |= WM1_LP_SR_EN;
3413
Tvrtko Ursulin175fded2016-11-16 08:55:42 +00003414 if (INTEL_GEN(dev_priv) >= 8)
Ville Syrjälä416f4722013-11-02 21:07:46 -07003415 results->wm_lp[wm_lp - 1] |=
3416 r->fbc_val << WM1_LP_FBC_SHIFT_BDW;
3417 else
3418 results->wm_lp[wm_lp - 1] |=
3419 r->fbc_val << WM1_LP_FBC_SHIFT;
3420
Ville Syrjäläd52fea52014-04-28 15:44:57 +03003421 /*
3422 * Always set WM1S_LP_EN when spr_val != 0, even if the
3423 * level is disabled. Doing otherwise could cause underruns.
3424 */
Tvrtko Ursulin175fded2016-11-16 08:55:42 +00003425 if (INTEL_GEN(dev_priv) <= 6 && r->spr_val) {
Pankaj Bharadiya48a1b8d2020-01-15 09:14:53 +05303426 drm_WARN_ON(&dev_priv->drm, wm_lp != 1);
Ville Syrjälä6cef2b8a2013-12-05 15:51:32 +02003427 results->wm_lp_spr[wm_lp - 1] = WM1S_LP_EN | r->spr_val;
3428 } else
3429 results->wm_lp_spr[wm_lp - 1] = r->spr_val;
Paulo Zanonicca32e92013-05-31 11:45:06 -03003430 }
Paulo Zanoni801bcff2013-05-31 10:08:35 -03003431
Ville Syrjälä0b2ae6d2013-10-09 19:17:55 +03003432 /* LP0 register values */
Matt Ropercd1d3ee2018-12-10 13:54:14 -08003433 for_each_intel_crtc(&dev_priv->drm, intel_crtc) {
Ville Syrjälä0b2ae6d2013-10-09 19:17:55 +03003434 enum pipe pipe = intel_crtc->pipe;
Ville Syrjälä0560b0c2020-01-20 19:47:11 +02003435 const struct intel_pipe_wm *pipe_wm = &intel_crtc->wm.active.ilk;
3436 const struct intel_wm_level *r = &pipe_wm->wm[0];
Paulo Zanoni1011d8c2013-05-09 16:55:50 -03003437
Pankaj Bharadiya48a1b8d2020-01-15 09:14:53 +05303438 if (drm_WARN_ON(&dev_priv->drm, !r->enable))
Ville Syrjälä0b2ae6d2013-10-09 19:17:55 +03003439 continue;
Ville Syrjälä0b2ae6d2013-10-09 19:17:55 +03003440
3441 results->wm_pipe[pipe] =
3442 (r->pri_val << WM0_PIPE_PLANE_SHIFT) |
3443 (r->spr_val << WM0_PIPE_SPRITE_SHIFT) |
3444 r->cur_val;
Paulo Zanoni801bcff2013-05-31 10:08:35 -03003445 }
3446}
3447
Paulo Zanoni861f3382013-05-31 10:19:21 -03003448/* Find the result with the highest level enabled. Check for enable_fbc_wm in
3449 * case both are at the same level. Prefer r1 in case they're the same. */
Matt Ropercd1d3ee2018-12-10 13:54:14 -08003450static struct intel_pipe_wm *
3451ilk_find_best_result(struct drm_i915_private *dev_priv,
3452 struct intel_pipe_wm *r1,
3453 struct intel_pipe_wm *r2)
Paulo Zanoni861f3382013-05-31 10:19:21 -03003454{
Matt Ropercd1d3ee2018-12-10 13:54:14 -08003455 int level, max_level = ilk_wm_max_level(dev_priv);
Ville Syrjälä198a1e92013-10-09 19:17:58 +03003456 int level1 = 0, level2 = 0;
Paulo Zanoni861f3382013-05-31 10:19:21 -03003457
Ville Syrjälä198a1e92013-10-09 19:17:58 +03003458 for (level = 1; level <= max_level; level++) {
3459 if (r1->wm[level].enable)
3460 level1 = level;
3461 if (r2->wm[level].enable)
3462 level2 = level;
Paulo Zanoni861f3382013-05-31 10:19:21 -03003463 }
3464
Ville Syrjälä198a1e92013-10-09 19:17:58 +03003465 if (level1 == level2) {
3466 if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
Paulo Zanoni861f3382013-05-31 10:19:21 -03003467 return r2;
3468 else
3469 return r1;
Ville Syrjälä198a1e92013-10-09 19:17:58 +03003470 } else if (level1 > level2) {
Paulo Zanoni861f3382013-05-31 10:19:21 -03003471 return r1;
3472 } else {
3473 return r2;
3474 }
3475}
3476
Ville Syrjälä49a687c2013-10-11 19:39:52 +03003477/* dirty bits used to track which watermarks need changes */
3478#define WM_DIRTY_PIPE(pipe) (1 << (pipe))
Ville Syrjälä49a687c2013-10-11 19:39:52 +03003479#define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
3480#define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))
3481#define WM_DIRTY_FBC (1 << 24)
3482#define WM_DIRTY_DDB (1 << 25)
3483
Damien Lespiau055e3932014-08-18 13:49:10 +01003484static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
Imre Deak820c1982013-12-17 14:46:36 +02003485 const struct ilk_wm_values *old,
3486 const struct ilk_wm_values *new)
Ville Syrjälä49a687c2013-10-11 19:39:52 +03003487{
3488 unsigned int dirty = 0;
3489 enum pipe pipe;
3490 int wm_lp;
3491
Damien Lespiau055e3932014-08-18 13:49:10 +01003492 for_each_pipe(dev_priv, pipe) {
Ville Syrjälä49a687c2013-10-11 19:39:52 +03003493 if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
3494 dirty |= WM_DIRTY_PIPE(pipe);
3495 /* Must disable LP1+ watermarks too */
3496 dirty |= WM_DIRTY_LP_ALL;
3497 }
3498 }
3499
3500 if (old->enable_fbc_wm != new->enable_fbc_wm) {
3501 dirty |= WM_DIRTY_FBC;
3502 /* Must disable LP1+ watermarks too */
3503 dirty |= WM_DIRTY_LP_ALL;
3504 }
3505
3506 if (old->partitioning != new->partitioning) {
3507 dirty |= WM_DIRTY_DDB;
3508 /* Must disable LP1+ watermarks too */
3509 dirty |= WM_DIRTY_LP_ALL;
3510 }
3511
3512 /* LP1+ watermarks already deemed dirty, no need to continue */
3513 if (dirty & WM_DIRTY_LP_ALL)
3514 return dirty;
3515
3516 /* Find the lowest numbered LP1+ watermark in need of an update... */
3517 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3518 if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||
3519 old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
3520 break;
3521 }
3522
3523 /* ...and mark it and all higher numbered LP1+ watermarks as dirty */
3524 for (; wm_lp <= 3; wm_lp++)
3525 dirty |= WM_DIRTY_LP(wm_lp);
3526
3527 return dirty;
3528}
3529
Ville Syrjälä8553c182013-12-05 15:51:39 +02003530static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
3531 unsigned int dirty)
3532{
Imre Deak820c1982013-12-17 14:46:36 +02003533 struct ilk_wm_values *previous = &dev_priv->wm.hw;
Ville Syrjälä8553c182013-12-05 15:51:39 +02003534 bool changed = false;
3535
3536 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM1_LP_SR_EN) {
3537 previous->wm_lp[2] &= ~WM1_LP_SR_EN;
Jani Nikula5f461662020-11-30 13:15:58 +02003538 intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, previous->wm_lp[2]);
Ville Syrjälä8553c182013-12-05 15:51:39 +02003539 changed = true;
3540 }
3541 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM1_LP_SR_EN) {
3542 previous->wm_lp[1] &= ~WM1_LP_SR_EN;
Jani Nikula5f461662020-11-30 13:15:58 +02003543 intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, previous->wm_lp[1]);
Ville Syrjälä8553c182013-12-05 15:51:39 +02003544 changed = true;
3545 }
3546 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM1_LP_SR_EN) {
3547 previous->wm_lp[0] &= ~WM1_LP_SR_EN;
Jani Nikula5f461662020-11-30 13:15:58 +02003548 intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, previous->wm_lp[0]);
Ville Syrjälä8553c182013-12-05 15:51:39 +02003549 changed = true;
3550 }
3551
3552 /*
3553 * Don't touch WM1S_LP_EN here.
3554 * Doing so could cause underruns.
3555 */
3556
3557 return changed;
3558}
3559
Paulo Zanoni801bcff2013-05-31 10:08:35 -03003560/*
3561 * The spec says we shouldn't write when we don't need, because every write
3562 * causes WMs to be re-evaluated, expending some power.
3563 */
Imre Deak820c1982013-12-17 14:46:36 +02003564static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
3565 struct ilk_wm_values *results)
Paulo Zanoni801bcff2013-05-31 10:08:35 -03003566{
Imre Deak820c1982013-12-17 14:46:36 +02003567 struct ilk_wm_values *previous = &dev_priv->wm.hw;
Ville Syrjälä49a687c2013-10-11 19:39:52 +03003568 unsigned int dirty;
Jani Nikula5ce9a6492019-01-18 14:01:20 +02003569 u32 val;
Paulo Zanoni801bcff2013-05-31 10:08:35 -03003570
Damien Lespiau055e3932014-08-18 13:49:10 +01003571 dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
Ville Syrjälä49a687c2013-10-11 19:39:52 +03003572 if (!dirty)
Paulo Zanoni801bcff2013-05-31 10:08:35 -03003573 return;
3574
Ville Syrjälä8553c182013-12-05 15:51:39 +02003575 _ilk_disable_lp_wm(dev_priv, dirty);
Ville Syrjälä6cef2b8a2013-12-05 15:51:32 +02003576
Ville Syrjälä49a687c2013-10-11 19:39:52 +03003577 if (dirty & WM_DIRTY_PIPE(PIPE_A))
Jani Nikula5f461662020-11-30 13:15:58 +02003578 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_A), results->wm_pipe[0]);
Ville Syrjälä49a687c2013-10-11 19:39:52 +03003579 if (dirty & WM_DIRTY_PIPE(PIPE_B))
Jani Nikula5f461662020-11-30 13:15:58 +02003580 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_B), results->wm_pipe[1]);
Ville Syrjälä49a687c2013-10-11 19:39:52 +03003581 if (dirty & WM_DIRTY_PIPE(PIPE_C))
Jani Nikula5f461662020-11-30 13:15:58 +02003582 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_C), results->wm_pipe[2]);
Paulo Zanoni801bcff2013-05-31 10:08:35 -03003583
Ville Syrjälä49a687c2013-10-11 19:39:52 +03003584 if (dirty & WM_DIRTY_DDB) {
Tvrtko Ursulin86527442016-10-13 11:03:00 +01003585 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
Jani Nikula5f461662020-11-30 13:15:58 +02003586 val = intel_uncore_read(&dev_priv->uncore, WM_MISC);
Ville Syrjäläac9545f2013-12-05 15:51:28 +02003587 if (results->partitioning == INTEL_DDB_PART_1_2)
3588 val &= ~WM_MISC_DATA_PARTITION_5_6;
3589 else
3590 val |= WM_MISC_DATA_PARTITION_5_6;
Jani Nikula5f461662020-11-30 13:15:58 +02003591 intel_uncore_write(&dev_priv->uncore, WM_MISC, val);
Ville Syrjäläac9545f2013-12-05 15:51:28 +02003592 } else {
Jani Nikula5f461662020-11-30 13:15:58 +02003593 val = intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2);
Ville Syrjäläac9545f2013-12-05 15:51:28 +02003594 if (results->partitioning == INTEL_DDB_PART_1_2)
3595 val &= ~DISP_DATA_PARTITION_5_6;
3596 else
3597 val |= DISP_DATA_PARTITION_5_6;
Jani Nikula5f461662020-11-30 13:15:58 +02003598 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL2, val);
Ville Syrjäläac9545f2013-12-05 15:51:28 +02003599 }
Paulo Zanoni1011d8c2013-05-09 16:55:50 -03003600 }
3601
Ville Syrjälä49a687c2013-10-11 19:39:52 +03003602 if (dirty & WM_DIRTY_FBC) {
Jani Nikula5f461662020-11-30 13:15:58 +02003603 val = intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL);
Paulo Zanonicca32e92013-05-31 11:45:06 -03003604 if (results->enable_fbc_wm)
3605 val &= ~DISP_FBC_WM_DIS;
3606 else
3607 val |= DISP_FBC_WM_DIS;
Jani Nikula5f461662020-11-30 13:15:58 +02003608 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, val);
Paulo Zanonicca32e92013-05-31 11:45:06 -03003609 }
3610
Imre Deak954911e2013-12-17 14:46:34 +02003611 if (dirty & WM_DIRTY_LP(1) &&
3612 previous->wm_lp_spr[0] != results->wm_lp_spr[0])
Jani Nikula5f461662020-11-30 13:15:58 +02003613 intel_uncore_write(&dev_priv->uncore, WM1S_LP_ILK, results->wm_lp_spr[0]);
Imre Deak954911e2013-12-17 14:46:34 +02003614
Tvrtko Ursulin175fded2016-11-16 08:55:42 +00003615 if (INTEL_GEN(dev_priv) >= 7) {
Ville Syrjälä6cef2b8a2013-12-05 15:51:32 +02003616 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
Jani Nikula5f461662020-11-30 13:15:58 +02003617 intel_uncore_write(&dev_priv->uncore, WM2S_LP_IVB, results->wm_lp_spr[1]);
Ville Syrjälä6cef2b8a2013-12-05 15:51:32 +02003618 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
Jani Nikula5f461662020-11-30 13:15:58 +02003619 intel_uncore_write(&dev_priv->uncore, WM3S_LP_IVB, results->wm_lp_spr[2]);
Ville Syrjälä6cef2b8a2013-12-05 15:51:32 +02003620 }
Paulo Zanoni801bcff2013-05-31 10:08:35 -03003621
Ville Syrjäläfacd619b2013-12-05 15:51:33 +02003622 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
Jani Nikula5f461662020-11-30 13:15:58 +02003623 intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, results->wm_lp[0]);
Ville Syrjäläfacd619b2013-12-05 15:51:33 +02003624 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
Jani Nikula5f461662020-11-30 13:15:58 +02003625 intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, results->wm_lp[1]);
Ville Syrjäläfacd619b2013-12-05 15:51:33 +02003626 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
Jani Nikula5f461662020-11-30 13:15:58 +02003627 intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, results->wm_lp[2]);
Ville Syrjälä609cede2013-10-09 19:18:03 +03003628
3629 dev_priv->wm.hw = *results;
Paulo Zanoni801bcff2013-05-31 10:08:35 -03003630}
3631
Ville Syrjälä60aca572019-11-27 21:05:51 +02003632bool ilk_disable_lp_wm(struct drm_i915_private *dev_priv)
Ville Syrjälä8553c182013-12-05 15:51:39 +02003633{
Ville Syrjälä8553c182013-12-05 15:51:39 +02003634 return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
3635}
3636
Stanislav Lisovskiy0f0f9ae2020-02-03 01:06:29 +02003637u8 intel_enabled_dbuf_slices_mask(struct drm_i915_private *dev_priv)
Mahesh Kumar74bd8002018-04-26 19:55:15 +05303638{
Stanislav Lisovskiy0f0f9ae2020-02-03 01:06:29 +02003639 int i;
3640 int max_slices = INTEL_INFO(dev_priv)->num_supported_dbuf_slices;
3641 u8 enabled_slices_mask = 0;
Mahesh Kumar74bd8002018-04-26 19:55:15 +05303642
Stanislav Lisovskiy0f0f9ae2020-02-03 01:06:29 +02003643 for (i = 0; i < max_slices; i++) {
Jani Nikula5f461662020-11-30 13:15:58 +02003644 if (intel_uncore_read(&dev_priv->uncore, DBUF_CTL_S(i)) & DBUF_POWER_STATE)
Stanislav Lisovskiy0f0f9ae2020-02-03 01:06:29 +02003645 enabled_slices_mask |= BIT(i);
3646 }
Mahesh Kumar74bd8002018-04-26 19:55:15 +05303647
Stanislav Lisovskiy0f0f9ae2020-02-03 01:06:29 +02003648 return enabled_slices_mask;
Mahesh Kumar74bd8002018-04-26 19:55:15 +05303649}
3650
Matt Roper024c9042015-09-24 15:53:11 -07003651/*
Paulo Zanoniee3d5322016-10-11 15:25:38 -03003652 * FIXME: We still don't have the proper code detect if we need to apply the WA,
3653 * so assume we'll always need it in order to avoid underruns.
3654 */
Ville Syrjälä60e983f2018-12-21 19:14:33 +02003655static bool skl_needs_memory_bw_wa(struct drm_i915_private *dev_priv)
Paulo Zanoniee3d5322016-10-11 15:25:38 -03003656{
Ville Syrjälä60e983f2018-12-21 19:14:33 +02003657 return IS_GEN9_BC(dev_priv) || IS_BROXTON(dev_priv);
Paulo Zanoniee3d5322016-10-11 15:25:38 -03003658}
3659
Paulo Zanoni56feca92016-09-22 18:00:28 -03003660static bool
3661intel_has_sagv(struct drm_i915_private *dev_priv)
3662{
Rodrigo Vivi1ca2b062018-10-26 13:03:17 -07003663 return (IS_GEN9_BC(dev_priv) || INTEL_GEN(dev_priv) >= 10) &&
3664 dev_priv->sagv_status != I915_SAGV_NOT_CONTROLLED;
Paulo Zanoni56feca92016-09-22 18:00:28 -03003665}
3666
James Ausmusb068a862019-10-09 10:23:14 -07003667static void
3668skl_setup_sagv_block_time(struct drm_i915_private *dev_priv)
3669{
James Ausmusda80f042019-10-09 10:23:15 -07003670 if (INTEL_GEN(dev_priv) >= 12) {
3671 u32 val = 0;
3672 int ret;
3673
3674 ret = sandybridge_pcode_read(dev_priv,
3675 GEN12_PCODE_READ_SAGV_BLOCK_TIME_US,
3676 &val, NULL);
3677 if (!ret) {
3678 dev_priv->sagv_block_time_us = val;
3679 return;
3680 }
3681
Wambui Karugaf8d18d52020-01-07 18:13:30 +03003682 drm_dbg(&dev_priv->drm, "Couldn't read SAGV block time!\n");
James Ausmusda80f042019-10-09 10:23:15 -07003683 } else if (IS_GEN(dev_priv, 11)) {
James Ausmusb068a862019-10-09 10:23:14 -07003684 dev_priv->sagv_block_time_us = 10;
3685 return;
3686 } else if (IS_GEN(dev_priv, 10)) {
3687 dev_priv->sagv_block_time_us = 20;
3688 return;
3689 } else if (IS_GEN(dev_priv, 9)) {
3690 dev_priv->sagv_block_time_us = 30;
3691 return;
3692 } else {
3693 MISSING_CASE(INTEL_GEN(dev_priv));
3694 }
3695
3696 /* Default to an unusable block time */
3697 dev_priv->sagv_block_time_us = -1;
3698}
3699
Lyude656d1b82016-08-17 15:55:54 -04003700/*
3701 * SAGV dynamically adjusts the system agent voltage and clock frequencies
3702 * depending on power and performance requirements. The display engine access
3703 * to system memory is blocked during the adjustment time. Because of the
3704 * blocking time, having this enabled can cause full system hangs and/or pipe
3705 * underruns if we don't meet all of the following requirements:
3706 *
3707 * - <= 1 pipe enabled
3708 * - All planes can enable watermarks for latencies >= SAGV engine block time
3709 * - We're not using an interlaced display configuration
3710 */
Ville Syrjälä71024042020-09-25 15:17:48 +03003711static int
Paulo Zanoni16dcdc42016-09-22 18:00:27 -03003712intel_enable_sagv(struct drm_i915_private *dev_priv)
Lyude656d1b82016-08-17 15:55:54 -04003713{
3714 int ret;
3715
Paulo Zanoni56feca92016-09-22 18:00:28 -03003716 if (!intel_has_sagv(dev_priv))
3717 return 0;
3718
3719 if (dev_priv->sagv_status == I915_SAGV_ENABLED)
Lyude656d1b82016-08-17 15:55:54 -04003720 return 0;
3721
Wambui Karugaf8d18d52020-01-07 18:13:30 +03003722 drm_dbg_kms(&dev_priv->drm, "Enabling SAGV\n");
Lyude656d1b82016-08-17 15:55:54 -04003723 ret = sandybridge_pcode_write(dev_priv, GEN9_PCODE_SAGV_CONTROL,
3724 GEN9_SAGV_ENABLE);
3725
Ville Syrjäläff61a972018-12-21 19:14:34 +02003726 /* We don't need to wait for SAGV when enabling */
Lyude656d1b82016-08-17 15:55:54 -04003727
3728 /*
3729 * Some skl systems, pre-release machines in particular,
Ville Syrjäläff61a972018-12-21 19:14:34 +02003730 * don't actually have SAGV.
Lyude656d1b82016-08-17 15:55:54 -04003731 */
Paulo Zanoni6e3100e2016-09-22 18:00:29 -03003732 if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
Wambui Karugaf8d18d52020-01-07 18:13:30 +03003733 drm_dbg(&dev_priv->drm, "No SAGV found on system, ignoring\n");
Paulo Zanoni16dcdc42016-09-22 18:00:27 -03003734 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
Lyude656d1b82016-08-17 15:55:54 -04003735 return 0;
3736 } else if (ret < 0) {
Wambui Karugaf8d18d52020-01-07 18:13:30 +03003737 drm_err(&dev_priv->drm, "Failed to enable SAGV\n");
Lyude656d1b82016-08-17 15:55:54 -04003738 return ret;
3739 }
3740
Paulo Zanoni16dcdc42016-09-22 18:00:27 -03003741 dev_priv->sagv_status = I915_SAGV_ENABLED;
Lyude656d1b82016-08-17 15:55:54 -04003742 return 0;
3743}
3744
Ville Syrjälä71024042020-09-25 15:17:48 +03003745static int
Paulo Zanoni16dcdc42016-09-22 18:00:27 -03003746intel_disable_sagv(struct drm_i915_private *dev_priv)
Lyude656d1b82016-08-17 15:55:54 -04003747{
Imre Deakb3b8e992016-12-05 18:27:38 +02003748 int ret;
Lyude656d1b82016-08-17 15:55:54 -04003749
Paulo Zanoni56feca92016-09-22 18:00:28 -03003750 if (!intel_has_sagv(dev_priv))
3751 return 0;
3752
3753 if (dev_priv->sagv_status == I915_SAGV_DISABLED)
Lyude656d1b82016-08-17 15:55:54 -04003754 return 0;
3755
Wambui Karugaf8d18d52020-01-07 18:13:30 +03003756 drm_dbg_kms(&dev_priv->drm, "Disabling SAGV\n");
Lyude656d1b82016-08-17 15:55:54 -04003757 /* bspec says to keep retrying for at least 1 ms */
Imre Deakb3b8e992016-12-05 18:27:38 +02003758 ret = skl_pcode_request(dev_priv, GEN9_PCODE_SAGV_CONTROL,
3759 GEN9_SAGV_DISABLE,
3760 GEN9_SAGV_IS_DISABLED, GEN9_SAGV_IS_DISABLED,
3761 1);
Lyude656d1b82016-08-17 15:55:54 -04003762 /*
3763 * Some skl systems, pre-release machines in particular,
Ville Syrjäläff61a972018-12-21 19:14:34 +02003764 * don't actually have SAGV.
Lyude656d1b82016-08-17 15:55:54 -04003765 */
Imre Deakb3b8e992016-12-05 18:27:38 +02003766 if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
Wambui Karugaf8d18d52020-01-07 18:13:30 +03003767 drm_dbg(&dev_priv->drm, "No SAGV found on system, ignoring\n");
Paulo Zanoni16dcdc42016-09-22 18:00:27 -03003768 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
Lyude656d1b82016-08-17 15:55:54 -04003769 return 0;
Imre Deakb3b8e992016-12-05 18:27:38 +02003770 } else if (ret < 0) {
Wambui Karugaf8d18d52020-01-07 18:13:30 +03003771 drm_err(&dev_priv->drm, "Failed to disable SAGV (%d)\n", ret);
Imre Deakb3b8e992016-12-05 18:27:38 +02003772 return ret;
Lyude656d1b82016-08-17 15:55:54 -04003773 }
3774
Paulo Zanoni16dcdc42016-09-22 18:00:27 -03003775 dev_priv->sagv_status = I915_SAGV_DISABLED;
Lyude656d1b82016-08-17 15:55:54 -04003776 return 0;
3777}
3778
Stanislav Lisovskiy680e1af2020-04-15 17:39:04 +03003779void intel_sagv_pre_plane_update(struct intel_atomic_state *state)
3780{
3781 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
Stanislav Lisovskiy97288892020-04-30 22:17:57 +03003782 const struct intel_bw_state *new_bw_state;
Stanislav Lisovskiy20f505f2020-05-14 10:48:52 +03003783 const struct intel_bw_state *old_bw_state;
3784 u32 new_mask = 0;
Stanislav Lisovskiy680e1af2020-04-15 17:39:04 +03003785
Stanislav Lisovskiy97288892020-04-30 22:17:57 +03003786 /*
3787 * Just return if we can't control SAGV or don't have it.
3788 * This is different from situation when we have SAGV but just can't
3789 * afford it due to DBuf limitation - in case if SAGV is completely
3790 * disabled in a BIOS, we are not even allowed to send a PCode request,
3791 * as it will throw an error. So have to check it here.
3792 */
3793 if (!intel_has_sagv(dev_priv))
3794 return;
3795
3796 new_bw_state = intel_atomic_get_new_bw_state(state);
3797 if (!new_bw_state)
3798 return;
3799
Stanislav Lisovskiy20f505f2020-05-14 10:48:52 +03003800 if (INTEL_GEN(dev_priv) < 11 && !intel_can_enable_sagv(dev_priv, new_bw_state)) {
Stanislav Lisovskiy680e1af2020-04-15 17:39:04 +03003801 intel_disable_sagv(dev_priv);
Stanislav Lisovskiy20f505f2020-05-14 10:48:52 +03003802 return;
3803 }
3804
3805 old_bw_state = intel_atomic_get_old_bw_state(state);
3806 /*
3807 * Nothing to mask
3808 */
3809 if (new_bw_state->qgv_points_mask == old_bw_state->qgv_points_mask)
3810 return;
3811
3812 new_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask;
3813
3814 /*
3815 * If new mask is zero - means there is nothing to mask,
3816 * we can only unmask, which should be done in unmask.
3817 */
3818 if (!new_mask)
3819 return;
3820
3821 /*
3822 * Restrict required qgv points before updating the configuration.
3823 * According to BSpec we can't mask and unmask qgv points at the same
3824 * time. Also masking should be done before updating the configuration
3825 * and unmasking afterwards.
3826 */
3827 icl_pcode_restrict_qgv_points(dev_priv, new_mask);
Stanislav Lisovskiy680e1af2020-04-15 17:39:04 +03003828}
3829
3830void intel_sagv_post_plane_update(struct intel_atomic_state *state)
3831{
3832 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
Stanislav Lisovskiy97288892020-04-30 22:17:57 +03003833 const struct intel_bw_state *new_bw_state;
Stanislav Lisovskiy20f505f2020-05-14 10:48:52 +03003834 const struct intel_bw_state *old_bw_state;
3835 u32 new_mask = 0;
Stanislav Lisovskiy680e1af2020-04-15 17:39:04 +03003836
Stanislav Lisovskiy97288892020-04-30 22:17:57 +03003837 /*
3838 * Just return if we can't control SAGV or don't have it.
3839 * This is different from situation when we have SAGV but just can't
3840 * afford it due to DBuf limitation - in case if SAGV is completely
3841 * disabled in a BIOS, we are not even allowed to send a PCode request,
3842 * as it will throw an error. So have to check it here.
3843 */
3844 if (!intel_has_sagv(dev_priv))
3845 return;
3846
3847 new_bw_state = intel_atomic_get_new_bw_state(state);
3848 if (!new_bw_state)
3849 return;
3850
Stanislav Lisovskiy20f505f2020-05-14 10:48:52 +03003851 if (INTEL_GEN(dev_priv) < 11 && intel_can_enable_sagv(dev_priv, new_bw_state)) {
Stanislav Lisovskiy680e1af2020-04-15 17:39:04 +03003852 intel_enable_sagv(dev_priv);
Stanislav Lisovskiy20f505f2020-05-14 10:48:52 +03003853 return;
3854 }
3855
3856 old_bw_state = intel_atomic_get_old_bw_state(state);
3857 /*
3858 * Nothing to unmask
3859 */
3860 if (new_bw_state->qgv_points_mask == old_bw_state->qgv_points_mask)
3861 return;
3862
3863 new_mask = new_bw_state->qgv_points_mask;
3864
3865 /*
3866 * Allow required qgv points after updating the configuration.
3867 * According to BSpec we can't mask and unmask qgv points at the same
3868 * time. Also masking should be done before updating the configuration
3869 * and unmasking afterwards.
3870 */
3871 icl_pcode_restrict_qgv_points(dev_priv, new_mask);
Stanislav Lisovskiy680e1af2020-04-15 17:39:04 +03003872}
3873
Stanislav Lisovskiy1d0a6c82020-05-13 12:38:12 +03003874static bool skl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
Lyude656d1b82016-08-17 15:55:54 -04003875{
Stanislav Lisovskiya389c492020-04-15 17:57:40 +03003876 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
Stanislav Lisovskiy97288892020-04-30 22:17:57 +03003877 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
Ville Syrjälä9c312122020-11-06 19:30:40 +02003878 enum plane_id plane_id;
Lyude656d1b82016-08-17 15:55:54 -04003879
Stanislav Lisovskiy97288892020-04-30 22:17:57 +03003880 if (!intel_has_sagv(dev_priv))
3881 return false;
3882
Stanislav Lisovskiya389c492020-04-15 17:57:40 +03003883 if (!crtc_state->hw.active)
Lyude656d1b82016-08-17 15:55:54 -04003884 return true;
Lucas De Marchida172232019-04-04 16:04:26 -07003885
Maarten Lankhorstbafcdad2020-11-12 21:17:18 +02003886 if (crtc_state->hw.pipe_mode.flags & DRM_MODE_FLAG_INTERLACE)
Lyude656d1b82016-08-17 15:55:54 -04003887 return false;
3888
Ville Syrjälä9c312122020-11-06 19:30:40 +02003889 for_each_plane_id_on_crtc(crtc, plane_id) {
Stanislav Lisovskiya389c492020-04-15 17:57:40 +03003890 const struct skl_plane_wm *wm =
Ville Syrjälä9c312122020-11-06 19:30:40 +02003891 &crtc_state->wm.skl.optimal.planes[plane_id];
3892 int level;
Paulo Zanoniee3d5322016-10-11 15:25:38 -03003893
Lyude656d1b82016-08-17 15:55:54 -04003894 /* Skip this plane if it's not enabled */
cpaul@redhat.comd8c0faf2016-10-18 16:09:49 -02003895 if (!wm->wm[0].plane_en)
Lyude656d1b82016-08-17 15:55:54 -04003896 continue;
3897
3898 /* Find the highest enabled wm level for this plane */
Tvrtko Ursulin5db94012016-10-13 11:03:10 +01003899 for (level = ilk_wm_max_level(dev_priv);
cpaul@redhat.comd8c0faf2016-10-18 16:09:49 -02003900 !wm->wm[level].plane_en; --level)
Lyude656d1b82016-08-17 15:55:54 -04003901 { }
3902
3903 /*
Paulo Zanonifdd11c22017-08-09 13:52:45 -07003904 * If any of the planes on this pipe don't enable wm levels that
3905 * incur memory latencies higher than sagv_block_time_us we
Ville Syrjäläff61a972018-12-21 19:14:34 +02003906 * can't enable SAGV.
Lyude656d1b82016-08-17 15:55:54 -04003907 */
Ville Syrjälä9c312122020-11-06 19:30:40 +02003908 if (!wm->wm[level].can_sagv)
Lyude656d1b82016-08-17 15:55:54 -04003909 return false;
3910 }
3911
3912 return true;
3913}
3914
Stanislav Lisovskiy7241c572020-05-14 10:48:51 +03003915static bool tgl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
3916{
3917 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3918 enum plane_id plane_id;
3919
3920 if (!crtc_state->hw.active)
3921 return true;
3922
3923 for_each_plane_id_on_crtc(crtc, plane_id) {
Stanislav Lisovskiy7241c572020-05-14 10:48:51 +03003924 const struct skl_plane_wm *wm =
3925 &crtc_state->wm.skl.optimal.planes[plane_id];
3926
Ville Syrjäläa68aa482021-02-26 17:32:01 +02003927 if (wm->wm[0].plane_en && !wm->sagv.wm0.plane_en)
Stanislav Lisovskiy7241c572020-05-14 10:48:51 +03003928 return false;
3929 }
3930
3931 return true;
3932}
3933
Stanislav Lisovskiy1d0a6c82020-05-13 12:38:12 +03003934static bool intel_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
3935{
Stanislav Lisovskiy7241c572020-05-14 10:48:51 +03003936 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3937 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3938
3939 if (INTEL_GEN(dev_priv) >= 12)
3940 return tgl_crtc_can_enable_sagv(crtc_state);
3941 else
3942 return skl_crtc_can_enable_sagv(crtc_state);
Stanislav Lisovskiy1d0a6c82020-05-13 12:38:12 +03003943}
3944
Stanislav Lisovskiyd8d5afe2020-05-13 12:38:13 +03003945bool intel_can_enable_sagv(struct drm_i915_private *dev_priv,
3946 const struct intel_bw_state *bw_state)
Stanislav Lisovskiya389c492020-04-15 17:57:40 +03003947{
Stanislav Lisovskiyd8d5afe2020-05-13 12:38:13 +03003948 if (INTEL_GEN(dev_priv) < 11 &&
3949 bw_state->active_pipes && !is_power_of_2(bw_state->active_pipes))
Stanislav Lisovskiyecab0f32020-04-30 22:56:34 +03003950 return false;
3951
Stanislav Lisovskiy97288892020-04-30 22:17:57 +03003952 return bw_state->pipe_sagv_reject == 0;
3953}
3954
3955static int intel_compute_sagv_mask(struct intel_atomic_state *state)
3956{
Stanislav Lisovskiyd8d5afe2020-05-13 12:38:13 +03003957 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
Stanislav Lisovskiy97288892020-04-30 22:17:57 +03003958 int ret;
Stanislav Lisovskiya389c492020-04-15 17:57:40 +03003959 struct intel_crtc *crtc;
Stanislav Lisovskiy7241c572020-05-14 10:48:51 +03003960 struct intel_crtc_state *new_crtc_state;
Stanislav Lisovskiy97288892020-04-30 22:17:57 +03003961 struct intel_bw_state *new_bw_state = NULL;
3962 const struct intel_bw_state *old_bw_state = NULL;
3963 int i;
Stanislav Lisovskiya389c492020-04-15 17:57:40 +03003964
Stanislav Lisovskiy97288892020-04-30 22:17:57 +03003965 for_each_new_intel_crtc_in_state(state, crtc,
3966 new_crtc_state, i) {
3967 new_bw_state = intel_atomic_get_bw_state(state);
3968 if (IS_ERR(new_bw_state))
3969 return PTR_ERR(new_bw_state);
Stanislav Lisovskiya389c492020-04-15 17:57:40 +03003970
Stanislav Lisovskiy97288892020-04-30 22:17:57 +03003971 old_bw_state = intel_atomic_get_old_bw_state(state);
Stanislav Lisovskiya389c492020-04-15 17:57:40 +03003972
Stanislav Lisovskiy97288892020-04-30 22:17:57 +03003973 if (intel_crtc_can_enable_sagv(new_crtc_state))
3974 new_bw_state->pipe_sagv_reject &= ~BIT(crtc->pipe);
3975 else
3976 new_bw_state->pipe_sagv_reject |= BIT(crtc->pipe);
3977 }
Stanislav Lisovskiya389c492020-04-15 17:57:40 +03003978
Stanislav Lisovskiy97288892020-04-30 22:17:57 +03003979 if (!new_bw_state)
3980 return 0;
Stanislav Lisovskiya389c492020-04-15 17:57:40 +03003981
Stanislav Lisovskiyecab0f32020-04-30 22:56:34 +03003982 new_bw_state->active_pipes =
3983 intel_calc_active_pipes(state, old_bw_state->active_pipes);
Stanislav Lisovskiy1d0a6c82020-05-13 12:38:12 +03003984
Stanislav Lisovskiyecab0f32020-04-30 22:56:34 +03003985 if (new_bw_state->active_pipes != old_bw_state->active_pipes) {
3986 ret = intel_atomic_lock_global_state(&new_bw_state->base);
3987 if (ret)
3988 return ret;
3989 }
3990
Stanislav Lisovskiy7241c572020-05-14 10:48:51 +03003991 for_each_new_intel_crtc_in_state(state, crtc,
3992 new_crtc_state, i) {
3993 struct skl_pipe_wm *pipe_wm = &new_crtc_state->wm.skl.optimal;
3994
3995 /*
3996 * We store use_sagv_wm in the crtc state rather than relying on
3997 * that bw state since we have no convenient way to get at the
3998 * latter from the plane commit hooks (especially in the legacy
3999 * cursor case)
4000 */
4001 pipe_wm->use_sagv_wm = INTEL_GEN(dev_priv) >= 12 &&
4002 intel_can_enable_sagv(dev_priv, new_bw_state);
4003 }
4004
Stanislav Lisovskiyd8d5afe2020-05-13 12:38:13 +03004005 if (intel_can_enable_sagv(dev_priv, new_bw_state) !=
4006 intel_can_enable_sagv(dev_priv, old_bw_state)) {
Stanislav Lisovskiy97288892020-04-30 22:17:57 +03004007 ret = intel_atomic_serialize_global_state(&new_bw_state->base);
4008 if (ret)
4009 return ret;
4010 } else if (new_bw_state->pipe_sagv_reject != old_bw_state->pipe_sagv_reject) {
4011 ret = intel_atomic_lock_global_state(&new_bw_state->base);
4012 if (ret)
4013 return ret;
4014 }
4015
4016 return 0;
Stanislav Lisovskiya389c492020-04-15 17:57:40 +03004017}
4018
Ville Syrjälä944a5e32021-01-22 22:56:28 +02004019static int intel_dbuf_size(struct drm_i915_private *dev_priv)
4020{
4021 int ddb_size = INTEL_INFO(dev_priv)->ddb_size;
4022
4023 drm_WARN_ON(&dev_priv->drm, ddb_size == 0);
4024
4025 if (INTEL_GEN(dev_priv) < 11)
4026 return ddb_size - 4; /* 4 blocks for bypass path allocation */
4027
4028 return ddb_size;
4029}
4030
4031static int intel_dbuf_slice_size(struct drm_i915_private *dev_priv)
4032{
4033 return intel_dbuf_size(dev_priv) /
4034 INTEL_INFO(dev_priv)->num_supported_dbuf_slices;
4035}
4036
Ville Syrjälä96dc6ed2021-01-22 22:56:29 +02004037static void
4038skl_ddb_entry_for_slices(struct drm_i915_private *dev_priv, u8 slice_mask,
4039 struct skl_ddb_entry *ddb)
Mahesh Kumaraa9664f2018-04-26 19:55:16 +05304040{
Ville Syrjälä96dc6ed2021-01-22 22:56:29 +02004041 int slice_size = intel_dbuf_slice_size(dev_priv);
Stanislav Lisovskiyff2cd862020-02-03 01:06:30 +02004042
Ville Syrjälä96dc6ed2021-01-22 22:56:29 +02004043 if (!slice_mask) {
4044 ddb->start = 0;
4045 ddb->end = 0;
4046 return;
4047 }
Stanislav Lisovskiyff2cd862020-02-03 01:06:30 +02004048
Ville Syrjälä96dc6ed2021-01-22 22:56:29 +02004049 ddb->start = (ffs(slice_mask) - 1) * slice_size;
4050 ddb->end = fls(slice_mask) * slice_size;
Stanislav Lisovskiyff2cd862020-02-03 01:06:30 +02004051
Ville Syrjälä96dc6ed2021-01-22 22:56:29 +02004052 WARN_ON(ddb->start >= ddb->end);
4053 WARN_ON(ddb->end > intel_dbuf_size(dev_priv));
Stanislav Lisovskiyff2cd862020-02-03 01:06:30 +02004054}
4055
Stanislav Lisovskiycd191542020-05-20 18:00:58 +03004056u32 skl_ddb_dbuf_slice_mask(struct drm_i915_private *dev_priv,
4057 const struct skl_ddb_entry *entry)
4058{
4059 u32 slice_mask = 0;
Ville Syrjälä944a5e32021-01-22 22:56:28 +02004060 u16 ddb_size = intel_dbuf_size(dev_priv);
Stanislav Lisovskiycd191542020-05-20 18:00:58 +03004061 u16 num_supported_slices = INTEL_INFO(dev_priv)->num_supported_dbuf_slices;
4062 u16 slice_size = ddb_size / num_supported_slices;
4063 u16 start_slice;
4064 u16 end_slice;
4065
4066 if (!skl_ddb_entry_size(entry))
4067 return 0;
4068
4069 start_slice = entry->start / slice_size;
4070 end_slice = (entry->end - 1) / slice_size;
4071
4072 /*
4073 * Per plane DDB entry can in a really worst case be on multiple slices
4074 * but single entry is anyway contigious.
4075 */
4076 while (start_slice <= end_slice) {
4077 slice_mask |= BIT(start_slice);
4078 start_slice++;
4079 }
4080
4081 return slice_mask;
4082}
4083
Ville Syrjälä2791a402021-01-22 22:56:26 +02004084static unsigned int intel_crtc_ddb_weight(const struct intel_crtc_state *crtc_state)
4085{
4086 const struct drm_display_mode *pipe_mode = &crtc_state->hw.pipe_mode;
4087 int hdisplay, vdisplay;
4088
4089 if (!crtc_state->hw.active)
4090 return 0;
4091
4092 /*
4093 * Watermark/ddb requirement highly depends upon width of the
4094 * framebuffer, So instead of allocating DDB equally among pipes
4095 * distribute DDB based on resolution/width of the display.
4096 */
4097 drm_mode_get_hv_timing(pipe_mode, &hdisplay, &vdisplay);
4098
4099 return hdisplay;
4100}
4101
Ville Syrjäläef79d622021-01-22 22:56:32 +02004102static void intel_crtc_dbuf_weights(const struct intel_dbuf_state *dbuf_state,
4103 enum pipe for_pipe,
4104 unsigned int *weight_start,
4105 unsigned int *weight_end,
4106 unsigned int *weight_total)
Damien Lespiaub9cec072014-11-04 17:06:43 +00004107{
Ville Syrjäläef79d622021-01-22 22:56:32 +02004108 struct drm_i915_private *dev_priv =
4109 to_i915(dbuf_state->base.state->base.dev);
4110 enum pipe pipe;
Ville Syrjälä53630962021-01-22 22:56:31 +02004111
4112 *weight_start = 0;
4113 *weight_end = 0;
4114 *weight_total = 0;
4115
Ville Syrjäläef79d622021-01-22 22:56:32 +02004116 for_each_pipe(dev_priv, pipe) {
4117 int weight = dbuf_state->weight[pipe];
Stanislav Lisovskiyff2cd862020-02-03 01:06:30 +02004118
4119 /*
4120 * Do not account pipes using other slice sets
4121 * luckily as of current BSpec slice sets do not partially
4122 * intersect(pipes share either same one slice or same slice set
4123 * i.e no partial intersection), so it is enough to check for
4124 * equality for now.
4125 */
Ville Syrjäläef79d622021-01-22 22:56:32 +02004126 if (dbuf_state->slices[pipe] != dbuf_state->slices[for_pipe])
Mahesh Kumarcf1f6972018-08-01 20:41:13 +05304127 continue;
4128
Ville Syrjälä53630962021-01-22 22:56:31 +02004129 *weight_total += weight;
Ville Syrjälä53630962021-01-22 22:56:31 +02004130 if (pipe < for_pipe) {
4131 *weight_start += weight;
4132 *weight_end += weight;
4133 } else if (pipe == for_pipe) {
4134 *weight_end += weight;
4135 }
Mahesh Kumarcf1f6972018-08-01 20:41:13 +05304136 }
Ville Syrjälä53630962021-01-22 22:56:31 +02004137}
4138
4139static int
Ville Syrjäläef79d622021-01-22 22:56:32 +02004140skl_crtc_allocate_ddb(struct intel_atomic_state *state, struct intel_crtc *crtc)
Ville Syrjälä53630962021-01-22 22:56:31 +02004141{
Ville Syrjäläef79d622021-01-22 22:56:32 +02004142 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4143 unsigned int weight_total, weight_start, weight_end;
Ville Syrjälä53630962021-01-22 22:56:31 +02004144 const struct intel_dbuf_state *old_dbuf_state =
4145 intel_atomic_get_old_dbuf_state(state);
4146 struct intel_dbuf_state *new_dbuf_state =
4147 intel_atomic_get_new_dbuf_state(state);
Ville Syrjäläef79d622021-01-22 22:56:32 +02004148 struct intel_crtc_state *crtc_state;
Ville Syrjälä53630962021-01-22 22:56:31 +02004149 struct skl_ddb_entry ddb_slices;
Ville Syrjäläef79d622021-01-22 22:56:32 +02004150 enum pipe pipe = crtc->pipe;
Ville Syrjälä53630962021-01-22 22:56:31 +02004151 u32 ddb_range_size;
4152 u32 dbuf_slice_mask;
4153 u32 start, end;
4154 int ret;
4155
Ville Syrjäläef79d622021-01-22 22:56:32 +02004156 if (new_dbuf_state->weight[pipe] == 0) {
4157 new_dbuf_state->ddb[pipe].start = 0;
4158 new_dbuf_state->ddb[pipe].end = 0;
4159 goto out;
Ville Syrjälä53630962021-01-22 22:56:31 +02004160 }
4161
Ville Syrjäläef79d622021-01-22 22:56:32 +02004162 dbuf_slice_mask = new_dbuf_state->slices[pipe];
Ville Syrjälä53630962021-01-22 22:56:31 +02004163
4164 skl_ddb_entry_for_slices(dev_priv, dbuf_slice_mask, &ddb_slices);
4165 ddb_range_size = skl_ddb_entry_size(&ddb_slices);
4166
Ville Syrjäläef79d622021-01-22 22:56:32 +02004167 intel_crtc_dbuf_weights(new_dbuf_state, pipe,
4168 &weight_start, &weight_end, &weight_total);
Ville Syrjälä53630962021-01-22 22:56:31 +02004169
4170 start = ddb_range_size * weight_start / weight_total;
4171 end = ddb_range_size * weight_end / weight_total;
Stanislav Lisovskiyff2cd862020-02-03 01:06:30 +02004172
Ville Syrjäläef79d622021-01-22 22:56:32 +02004173 new_dbuf_state->ddb[pipe].start = ddb_slices.start + start;
4174 new_dbuf_state->ddb[pipe].end = ddb_slices.start + end;
4175
4176out:
4177 if (skl_ddb_entry_equal(&old_dbuf_state->ddb[pipe],
4178 &new_dbuf_state->ddb[pipe]))
4179 return 0;
4180
4181 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
4182 if (ret)
4183 return ret;
4184
4185 crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
4186 if (IS_ERR(crtc_state))
4187 return PTR_ERR(crtc_state);
4188
4189 crtc_state->wm.skl.ddb = new_dbuf_state->ddb[pipe];
Stanislav Lisovskiyff2cd862020-02-03 01:06:30 +02004190
Ville Syrjälä70b1a262020-02-25 19:11:16 +02004191 drm_dbg_kms(&dev_priv->drm,
Ville Syrjäläef79d622021-01-22 22:56:32 +02004192 "[CRTC:%d:%s] dbuf slices 0x%x -> 0x%x, ddb (%d - %d) -> (%d - %d), active pipes 0x%x -> 0x%x\n",
Ville Syrjälä53630962021-01-22 22:56:31 +02004193 crtc->base.base.id, crtc->base.name,
Ville Syrjäläef79d622021-01-22 22:56:32 +02004194 old_dbuf_state->slices[pipe], new_dbuf_state->slices[pipe],
4195 old_dbuf_state->ddb[pipe].start, old_dbuf_state->ddb[pipe].end,
4196 new_dbuf_state->ddb[pipe].start, new_dbuf_state->ddb[pipe].end,
4197 old_dbuf_state->active_pipes, new_dbuf_state->active_pipes);
Ville Syrjälä3cf43cd2020-02-25 19:11:13 +02004198
4199 return 0;
Damien Lespiaub9cec072014-11-04 17:06:43 +00004200}
4201
Ville Syrjälädf331de2019-03-19 18:03:11 +02004202static int skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
4203 int width, const struct drm_format_info *format,
4204 u64 modifier, unsigned int rotation,
4205 u32 plane_pixel_rate, struct skl_wm_params *wp,
4206 int color_plane);
Maarten Lankhorstec193642019-06-28 10:55:17 +02004207static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
Ville Syrjälädf331de2019-03-19 18:03:11 +02004208 int level,
Stanislav Lisovskiy7b994752020-04-09 18:47:18 +03004209 unsigned int latency,
Ville Syrjälädf331de2019-03-19 18:03:11 +02004210 const struct skl_wm_params *wp,
4211 const struct skl_wm_level *result_prev,
4212 struct skl_wm_level *result /* out */);
Damien Lespiaub9cec072014-11-04 17:06:43 +00004213
Ville Syrjälädf331de2019-03-19 18:03:11 +02004214static unsigned int
4215skl_cursor_allocation(const struct intel_crtc_state *crtc_state,
4216 int num_active)
4217{
Maarten Lankhorst2225f3c2019-10-31 12:26:03 +01004218 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
Ville Syrjälädf331de2019-03-19 18:03:11 +02004219 int level, max_level = ilk_wm_max_level(dev_priv);
4220 struct skl_wm_level wm = {};
4221 int ret, min_ddb_alloc = 0;
4222 struct skl_wm_params wp;
4223
4224 ret = skl_compute_wm_params(crtc_state, 256,
4225 drm_format_info(DRM_FORMAT_ARGB8888),
4226 DRM_FORMAT_MOD_LINEAR,
4227 DRM_MODE_ROTATE_0,
4228 crtc_state->pixel_rate, &wp, 0);
Pankaj Bharadiya48a1b8d2020-01-15 09:14:53 +05304229 drm_WARN_ON(&dev_priv->drm, ret);
Ville Syrjälädf331de2019-03-19 18:03:11 +02004230
4231 for (level = 0; level <= max_level; level++) {
Stanislav Lisovskiy7b994752020-04-09 18:47:18 +03004232 unsigned int latency = dev_priv->wm.skl_latency[level];
4233
4234 skl_compute_plane_wm(crtc_state, level, latency, &wp, &wm, &wm);
Ville Syrjälädf331de2019-03-19 18:03:11 +02004235 if (wm.min_ddb_alloc == U16_MAX)
4236 break;
4237
4238 min_ddb_alloc = wm.min_ddb_alloc;
4239 }
4240
4241 return max(num_active == 1 ? 32 : 8, min_ddb_alloc);
Damien Lespiaub9cec072014-11-04 17:06:43 +00004242}
4243
Mahesh Kumar37cde112018-04-26 19:55:17 +05304244static void skl_ddb_entry_init_from_hw(struct drm_i915_private *dev_priv,
4245 struct skl_ddb_entry *entry, u32 reg)
Damien Lespiaua269c582014-11-04 17:06:49 +00004246{
Mahesh Kumar37cde112018-04-26 19:55:17 +05304247
Ville Syrjäläd7e449a2019-02-05 22:50:56 +02004248 entry->start = reg & DDB_ENTRY_MASK;
4249 entry->end = (reg >> DDB_ENTRY_END_SHIFT) & DDB_ENTRY_MASK;
Mahesh Kumar37cde112018-04-26 19:55:17 +05304250
Damien Lespiau16160e32014-11-04 17:06:53 +00004251 if (entry->end)
4252 entry->end += 1;
Damien Lespiaua269c582014-11-04 17:06:49 +00004253}
4254
Mahesh Kumarddf34312018-04-09 09:11:03 +05304255static void
4256skl_ddb_get_hw_plane_state(struct drm_i915_private *dev_priv,
4257 const enum pipe pipe,
4258 const enum plane_id plane_id,
Ville Syrjäläff43bc32018-11-27 18:59:00 +02004259 struct skl_ddb_entry *ddb_y,
4260 struct skl_ddb_entry *ddb_uv)
Mahesh Kumarddf34312018-04-09 09:11:03 +05304261{
Ville Syrjäläff43bc32018-11-27 18:59:00 +02004262 u32 val, val2;
4263 u32 fourcc = 0;
Mahesh Kumarddf34312018-04-09 09:11:03 +05304264
4265 /* Cursor doesn't support NV12/planar, so no extra calculation needed */
4266 if (plane_id == PLANE_CURSOR) {
Jani Nikula5f461662020-11-30 13:15:58 +02004267 val = intel_uncore_read(&dev_priv->uncore, CUR_BUF_CFG(pipe));
Ville Syrjäläff43bc32018-11-27 18:59:00 +02004268 skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
Mahesh Kumarddf34312018-04-09 09:11:03 +05304269 return;
4270 }
4271
Jani Nikula5f461662020-11-30 13:15:58 +02004272 val = intel_uncore_read(&dev_priv->uncore, PLANE_CTL(pipe, plane_id));
Mahesh Kumarddf34312018-04-09 09:11:03 +05304273
4274 /* No DDB allocated for disabled planes */
Ville Syrjäläff43bc32018-11-27 18:59:00 +02004275 if (val & PLANE_CTL_ENABLE)
4276 fourcc = skl_format_to_fourcc(val & PLANE_CTL_FORMAT_MASK,
4277 val & PLANE_CTL_ORDER_RGBX,
4278 val & PLANE_CTL_ALPHA_MASK);
Mahesh Kumarddf34312018-04-09 09:11:03 +05304279
Ville Syrjäläff43bc32018-11-27 18:59:00 +02004280 if (INTEL_GEN(dev_priv) >= 11) {
Jani Nikula5f461662020-11-30 13:15:58 +02004281 val = intel_uncore_read(&dev_priv->uncore, PLANE_BUF_CFG(pipe, plane_id));
Ville Syrjäläff43bc32018-11-27 18:59:00 +02004282 skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
4283 } else {
Jani Nikula5f461662020-11-30 13:15:58 +02004284 val = intel_uncore_read(&dev_priv->uncore, PLANE_BUF_CFG(pipe, plane_id));
4285 val2 = intel_uncore_read(&dev_priv->uncore, PLANE_NV12_BUF_CFG(pipe, plane_id));
Mahesh Kumarddf34312018-04-09 09:11:03 +05304286
Ville Syrjäläd1d23d72019-09-13 22:31:54 +03004287 if (fourcc &&
4288 drm_format_info_is_yuv_semiplanar(drm_format_info(fourcc)))
Ville Syrjäläff43bc32018-11-27 18:59:00 +02004289 swap(val, val2);
4290
4291 skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
4292 skl_ddb_entry_init_from_hw(dev_priv, ddb_uv, val2);
Mahesh Kumarddf34312018-04-09 09:11:03 +05304293 }
4294}
4295
Ville Syrjäläff43bc32018-11-27 18:59:00 +02004296void skl_pipe_ddb_get_hw_state(struct intel_crtc *crtc,
4297 struct skl_ddb_entry *ddb_y,
4298 struct skl_ddb_entry *ddb_uv)
4299{
4300 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4301 enum intel_display_power_domain power_domain;
4302 enum pipe pipe = crtc->pipe;
Chris Wilson0e6e0be2019-01-14 14:21:24 +00004303 intel_wakeref_t wakeref;
Ville Syrjäläff43bc32018-11-27 18:59:00 +02004304 enum plane_id plane_id;
4305
4306 power_domain = POWER_DOMAIN_PIPE(pipe);
Chris Wilson0e6e0be2019-01-14 14:21:24 +00004307 wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
4308 if (!wakeref)
Ville Syrjäläff43bc32018-11-27 18:59:00 +02004309 return;
4310
4311 for_each_plane_id_on_crtc(crtc, plane_id)
4312 skl_ddb_get_hw_plane_state(dev_priv, pipe,
4313 plane_id,
4314 &ddb_y[plane_id],
4315 &ddb_uv[plane_id]);
4316
Chris Wilson0e6e0be2019-01-14 14:21:24 +00004317 intel_display_power_put(dev_priv, power_domain, wakeref);
Ville Syrjäläff43bc32018-11-27 18:59:00 +02004318}
4319
Kumar, Mahesh9c2f7a92016-05-16 15:52:00 -07004320/*
4321 * Determines the downscale amount of a plane for the purposes of watermark calculations.
4322 * The bspec defines downscale amount as:
4323 *
4324 * """
4325 * Horizontal down scale amount = maximum[1, Horizontal source size /
4326 * Horizontal destination size]
4327 * Vertical down scale amount = maximum[1, Vertical source size /
4328 * Vertical destination size]
4329 * Total down scale amount = Horizontal down scale amount *
4330 * Vertical down scale amount
4331 * """
4332 *
4333 * Return value is provided in 16.16 fixed point form to retain fractional part.
4334 * Caller should take care of dividing & rounding off the value.
4335 */
Kumar, Mahesh7084b502017-05-17 17:28:23 +05304336static uint_fixed_16_16_t
Maarten Lankhorstec193642019-06-28 10:55:17 +02004337skl_plane_downscale_amount(const struct intel_crtc_state *crtc_state,
4338 const struct intel_plane_state *plane_state)
Kumar, Mahesh9c2f7a92016-05-16 15:52:00 -07004339{
Pankaj Bharadiya19edeb382020-05-04 23:45:59 +05304340 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
Jani Nikula5ce9a6492019-01-18 14:01:20 +02004341 u32 src_w, src_h, dst_w, dst_h;
Kumar, Mahesh7084b502017-05-17 17:28:23 +05304342 uint_fixed_16_16_t fp_w_ratio, fp_h_ratio;
4343 uint_fixed_16_16_t downscale_h, downscale_w;
Kumar, Mahesh9c2f7a92016-05-16 15:52:00 -07004344
Pankaj Bharadiya19edeb382020-05-04 23:45:59 +05304345 if (drm_WARN_ON(&dev_priv->drm,
4346 !intel_wm_plane_visible(crtc_state, plane_state)))
Kumar, Mahesheac2cb82017-07-05 20:01:46 +05304347 return u32_to_fixed16(0);
Kumar, Mahesh9c2f7a92016-05-16 15:52:00 -07004348
Maarten Lankhorst3a612762019-10-04 13:34:54 +02004349 /*
4350 * Src coordinates are already rotated by 270 degrees for
4351 * the 90/270 degree plane rotation cases (to match the
4352 * GTT mapping), hence no need to account for rotation here.
4353 *
4354 * n.b., src is 16.16 fixed point, dst is whole integer.
4355 */
Maarten Lankhorstf90a85e2019-10-31 12:26:08 +01004356 src_w = drm_rect_width(&plane_state->uapi.src) >> 16;
4357 src_h = drm_rect_height(&plane_state->uapi.src) >> 16;
4358 dst_w = drm_rect_width(&plane_state->uapi.dst);
4359 dst_h = drm_rect_height(&plane_state->uapi.dst);
Ville Syrjälä93aa2a12017-03-14 17:10:50 +02004360
Kumar, Mahesheac2cb82017-07-05 20:01:46 +05304361 fp_w_ratio = div_fixed16(src_w, dst_w);
4362 fp_h_ratio = div_fixed16(src_h, dst_h);
4363 downscale_w = max_fixed16(fp_w_ratio, u32_to_fixed16(1));
4364 downscale_h = max_fixed16(fp_h_ratio, u32_to_fixed16(1));
Kumar, Mahesh9c2f7a92016-05-16 15:52:00 -07004365
Kumar, Mahesh7084b502017-05-17 17:28:23 +05304366 return mul_fixed16(downscale_w, downscale_h);
Kumar, Mahesh9c2f7a92016-05-16 15:52:00 -07004367}
4368
Stanislav Lisovskiyff2cd862020-02-03 01:06:30 +02004369struct dbuf_slice_conf_entry {
4370 u8 active_pipes;
4371 u8 dbuf_mask[I915_MAX_PIPES];
4372};
4373
4374/*
4375 * Table taken from Bspec 12716
4376 * Pipes do have some preferred DBuf slice affinity,
4377 * plus there are some hardcoded requirements on how
4378 * those should be distributed for multipipe scenarios.
4379 * For more DBuf slices algorithm can get even more messy
4380 * and less readable, so decided to use a table almost
4381 * as is from BSpec itself - that way it is at least easier
4382 * to compare, change and check.
4383 */
Jani Nikulaf8226d02020-02-19 17:45:42 +02004384static const struct dbuf_slice_conf_entry icl_allowed_dbufs[] =
Stanislav Lisovskiyff2cd862020-02-03 01:06:30 +02004385/* Autogenerated with igt/tools/intel_dbuf_map tool: */
4386{
4387 {
4388 .active_pipes = BIT(PIPE_A),
4389 .dbuf_mask = {
Ville Syrjälä06812bd2020-02-25 19:11:08 +02004390 [PIPE_A] = BIT(DBUF_S1),
4391 },
Stanislav Lisovskiyff2cd862020-02-03 01:06:30 +02004392 },
4393 {
4394 .active_pipes = BIT(PIPE_B),
4395 .dbuf_mask = {
Ville Syrjälä06812bd2020-02-25 19:11:08 +02004396 [PIPE_B] = BIT(DBUF_S1),
4397 },
Stanislav Lisovskiyff2cd862020-02-03 01:06:30 +02004398 },
4399 {
4400 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
4401 .dbuf_mask = {
4402 [PIPE_A] = BIT(DBUF_S1),
Ville Syrjälä06812bd2020-02-25 19:11:08 +02004403 [PIPE_B] = BIT(DBUF_S2),
4404 },
Stanislav Lisovskiyff2cd862020-02-03 01:06:30 +02004405 },
4406 {
4407 .active_pipes = BIT(PIPE_C),
4408 .dbuf_mask = {
Ville Syrjälä06812bd2020-02-25 19:11:08 +02004409 [PIPE_C] = BIT(DBUF_S2),
4410 },
Stanislav Lisovskiyff2cd862020-02-03 01:06:30 +02004411 },
4412 {
4413 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
4414 .dbuf_mask = {
4415 [PIPE_A] = BIT(DBUF_S1),
Ville Syrjälä06812bd2020-02-25 19:11:08 +02004416 [PIPE_C] = BIT(DBUF_S2),
4417 },
Stanislav Lisovskiyff2cd862020-02-03 01:06:30 +02004418 },
4419 {
4420 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
4421 .dbuf_mask = {
4422 [PIPE_B] = BIT(DBUF_S1),
Ville Syrjälä06812bd2020-02-25 19:11:08 +02004423 [PIPE_C] = BIT(DBUF_S2),
4424 },
Stanislav Lisovskiyff2cd862020-02-03 01:06:30 +02004425 },
4426 {
4427 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
4428 .dbuf_mask = {
4429 [PIPE_A] = BIT(DBUF_S1),
4430 [PIPE_B] = BIT(DBUF_S1),
Ville Syrjälä06812bd2020-02-25 19:11:08 +02004431 [PIPE_C] = BIT(DBUF_S2),
4432 },
Stanislav Lisovskiyff2cd862020-02-03 01:06:30 +02004433 },
Ville Syrjälä05e81552020-02-25 19:11:09 +02004434 {}
Stanislav Lisovskiyff2cd862020-02-03 01:06:30 +02004435};
4436
4437/*
4438 * Table taken from Bspec 49255
4439 * Pipes do have some preferred DBuf slice affinity,
4440 * plus there are some hardcoded requirements on how
4441 * those should be distributed for multipipe scenarios.
4442 * For more DBuf slices algorithm can get even more messy
4443 * and less readable, so decided to use a table almost
4444 * as is from BSpec itself - that way it is at least easier
4445 * to compare, change and check.
4446 */
Jani Nikulaf8226d02020-02-19 17:45:42 +02004447static const struct dbuf_slice_conf_entry tgl_allowed_dbufs[] =
Stanislav Lisovskiyff2cd862020-02-03 01:06:30 +02004448/* Autogenerated with igt/tools/intel_dbuf_map tool: */
4449{
4450 {
4451 .active_pipes = BIT(PIPE_A),
4452 .dbuf_mask = {
Ville Syrjälä06812bd2020-02-25 19:11:08 +02004453 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4454 },
Stanislav Lisovskiyff2cd862020-02-03 01:06:30 +02004455 },
4456 {
4457 .active_pipes = BIT(PIPE_B),
4458 .dbuf_mask = {
Ville Syrjälä06812bd2020-02-25 19:11:08 +02004459 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
4460 },
Stanislav Lisovskiyff2cd862020-02-03 01:06:30 +02004461 },
4462 {
4463 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
4464 .dbuf_mask = {
4465 [PIPE_A] = BIT(DBUF_S2),
Ville Syrjälä06812bd2020-02-25 19:11:08 +02004466 [PIPE_B] = BIT(DBUF_S1),
4467 },
Stanislav Lisovskiyff2cd862020-02-03 01:06:30 +02004468 },
4469 {
4470 .active_pipes = BIT(PIPE_C),
4471 .dbuf_mask = {
Ville Syrjälä06812bd2020-02-25 19:11:08 +02004472 [PIPE_C] = BIT(DBUF_S2) | BIT(DBUF_S1),
4473 },
Stanislav Lisovskiyff2cd862020-02-03 01:06:30 +02004474 },
4475 {
4476 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
4477 .dbuf_mask = {
4478 [PIPE_A] = BIT(DBUF_S1),
Ville Syrjälä06812bd2020-02-25 19:11:08 +02004479 [PIPE_C] = BIT(DBUF_S2),
4480 },
Stanislav Lisovskiyff2cd862020-02-03 01:06:30 +02004481 },
4482 {
4483 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
4484 .dbuf_mask = {
4485 [PIPE_B] = BIT(DBUF_S1),
Ville Syrjälä06812bd2020-02-25 19:11:08 +02004486 [PIPE_C] = BIT(DBUF_S2),
4487 },
Stanislav Lisovskiyff2cd862020-02-03 01:06:30 +02004488 },
4489 {
4490 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
4491 .dbuf_mask = {
4492 [PIPE_A] = BIT(DBUF_S1),
4493 [PIPE_B] = BIT(DBUF_S1),
Ville Syrjälä06812bd2020-02-25 19:11:08 +02004494 [PIPE_C] = BIT(DBUF_S2),
4495 },
Stanislav Lisovskiyff2cd862020-02-03 01:06:30 +02004496 },
4497 {
4498 .active_pipes = BIT(PIPE_D),
4499 .dbuf_mask = {
Ville Syrjälä06812bd2020-02-25 19:11:08 +02004500 [PIPE_D] = BIT(DBUF_S2) | BIT(DBUF_S1),
4501 },
Stanislav Lisovskiyff2cd862020-02-03 01:06:30 +02004502 },
4503 {
4504 .active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
4505 .dbuf_mask = {
4506 [PIPE_A] = BIT(DBUF_S1),
Ville Syrjälä06812bd2020-02-25 19:11:08 +02004507 [PIPE_D] = BIT(DBUF_S2),
4508 },
Stanislav Lisovskiyff2cd862020-02-03 01:06:30 +02004509 },
4510 {
4511 .active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
4512 .dbuf_mask = {
4513 [PIPE_B] = BIT(DBUF_S1),
Ville Syrjälä06812bd2020-02-25 19:11:08 +02004514 [PIPE_D] = BIT(DBUF_S2),
4515 },
Stanislav Lisovskiyff2cd862020-02-03 01:06:30 +02004516 },
4517 {
4518 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
4519 .dbuf_mask = {
4520 [PIPE_A] = BIT(DBUF_S1),
4521 [PIPE_B] = BIT(DBUF_S1),
Ville Syrjälä06812bd2020-02-25 19:11:08 +02004522 [PIPE_D] = BIT(DBUF_S2),
4523 },
Stanislav Lisovskiyff2cd862020-02-03 01:06:30 +02004524 },
4525 {
4526 .active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
4527 .dbuf_mask = {
4528 [PIPE_C] = BIT(DBUF_S1),
Ville Syrjälä06812bd2020-02-25 19:11:08 +02004529 [PIPE_D] = BIT(DBUF_S2),
4530 },
Stanislav Lisovskiyff2cd862020-02-03 01:06:30 +02004531 },
4532 {
4533 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
4534 .dbuf_mask = {
4535 [PIPE_A] = BIT(DBUF_S1),
4536 [PIPE_C] = BIT(DBUF_S2),
Ville Syrjälä06812bd2020-02-25 19:11:08 +02004537 [PIPE_D] = BIT(DBUF_S2),
4538 },
Stanislav Lisovskiyff2cd862020-02-03 01:06:30 +02004539 },
4540 {
4541 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4542 .dbuf_mask = {
4543 [PIPE_B] = BIT(DBUF_S1),
4544 [PIPE_C] = BIT(DBUF_S2),
Ville Syrjälä06812bd2020-02-25 19:11:08 +02004545 [PIPE_D] = BIT(DBUF_S2),
4546 },
Stanislav Lisovskiyff2cd862020-02-03 01:06:30 +02004547 },
4548 {
4549 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4550 .dbuf_mask = {
4551 [PIPE_A] = BIT(DBUF_S1),
4552 [PIPE_B] = BIT(DBUF_S1),
4553 [PIPE_C] = BIT(DBUF_S2),
Ville Syrjälä06812bd2020-02-25 19:11:08 +02004554 [PIPE_D] = BIT(DBUF_S2),
4555 },
Stanislav Lisovskiyff2cd862020-02-03 01:06:30 +02004556 },
Ville Syrjälä05e81552020-02-25 19:11:09 +02004557 {}
Stanislav Lisovskiyff2cd862020-02-03 01:06:30 +02004558};
4559
Ville Syrjälä05e81552020-02-25 19:11:09 +02004560static u8 compute_dbuf_slices(enum pipe pipe, u8 active_pipes,
4561 const struct dbuf_slice_conf_entry *dbuf_slices)
Stanislav Lisovskiyff2cd862020-02-03 01:06:30 +02004562{
4563 int i;
4564
Ville Syrjälä05e81552020-02-25 19:11:09 +02004565 for (i = 0; i < dbuf_slices[i].active_pipes; i++) {
Stanislav Lisovskiyff2cd862020-02-03 01:06:30 +02004566 if (dbuf_slices[i].active_pipes == active_pipes)
4567 return dbuf_slices[i].dbuf_mask[pipe];
4568 }
4569 return 0;
4570}
4571
4572/*
4573 * This function finds an entry with same enabled pipe configuration and
4574 * returns correspondent DBuf slice mask as stated in BSpec for particular
4575 * platform.
4576 */
Ville Syrjälä05e81552020-02-25 19:11:09 +02004577static u8 icl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes)
Stanislav Lisovskiyff2cd862020-02-03 01:06:30 +02004578{
4579 /*
4580 * FIXME: For ICL this is still a bit unclear as prev BSpec revision
4581 * required calculating "pipe ratio" in order to determine
4582 * if one or two slices can be used for single pipe configurations
4583 * as additional constraint to the existing table.
4584 * However based on recent info, it should be not "pipe ratio"
4585 * but rather ratio between pixel_rate and cdclk with additional
4586 * constants, so for now we are using only table until this is
4587 * clarified. Also this is the reason why crtc_state param is
4588 * still here - we will need it once those additional constraints
4589 * pop up.
4590 */
Ville Syrjälä05e81552020-02-25 19:11:09 +02004591 return compute_dbuf_slices(pipe, active_pipes, icl_allowed_dbufs);
Stanislav Lisovskiyff2cd862020-02-03 01:06:30 +02004592}
4593
Ville Syrjälä05e81552020-02-25 19:11:09 +02004594static u8 tgl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes)
Stanislav Lisovskiyff2cd862020-02-03 01:06:30 +02004595{
Ville Syrjälä05e81552020-02-25 19:11:09 +02004596 return compute_dbuf_slices(pipe, active_pipes, tgl_allowed_dbufs);
Stanislav Lisovskiyff2cd862020-02-03 01:06:30 +02004597}
4598
Ville Syrjälä2d42f322021-01-22 22:56:27 +02004599static u8 skl_compute_dbuf_slices(struct intel_crtc *crtc, u8 active_pipes)
Stanislav Lisovskiyff2cd862020-02-03 01:06:30 +02004600{
Stanislav Lisovskiyff2cd862020-02-03 01:06:30 +02004601 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4602 enum pipe pipe = crtc->pipe;
4603
4604 if (IS_GEN(dev_priv, 12))
Ville Syrjälä05e81552020-02-25 19:11:09 +02004605 return tgl_compute_dbuf_slices(pipe, active_pipes);
Stanislav Lisovskiyff2cd862020-02-03 01:06:30 +02004606 else if (IS_GEN(dev_priv, 11))
Ville Syrjälä05e81552020-02-25 19:11:09 +02004607 return icl_compute_dbuf_slices(pipe, active_pipes);
Stanislav Lisovskiyff2cd862020-02-03 01:06:30 +02004608 /*
4609 * For anything else just return one slice yet.
4610 * Should be extended for other platforms.
4611 */
Ville Syrjälä2f9078c2020-02-25 19:11:10 +02004612 return active_pipes & BIT(pipe) ? BIT(DBUF_S1) : 0;
Stanislav Lisovskiyff2cd862020-02-03 01:06:30 +02004613}
4614
Maarten Lankhorst24719e92018-10-22 12:20:00 +02004615static u64
Maarten Lankhorstec193642019-06-28 10:55:17 +02004616skl_plane_relative_data_rate(const struct intel_crtc_state *crtc_state,
4617 const struct intel_plane_state *plane_state,
Ville Syrjäläd1d23d72019-09-13 22:31:54 +03004618 int color_plane)
Damien Lespiaub9cec072014-11-04 17:06:43 +00004619{
Maarten Lankhorstf90a85e2019-10-31 12:26:08 +01004620 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
Maarten Lankhorst7b3cb172019-10-31 12:26:07 +01004621 const struct drm_framebuffer *fb = plane_state->hw.fb;
Jani Nikula5ce9a6492019-01-18 14:01:20 +02004622 u32 data_rate;
4623 u32 width = 0, height = 0;
Kumar, Mahesh7084b502017-05-17 17:28:23 +05304624 uint_fixed_16_16_t down_scale_amount;
Maarten Lankhorst24719e92018-10-22 12:20:00 +02004625 u64 rate;
Matt Ropera1de91e2016-05-12 07:05:57 -07004626
Maarten Lankhorstf90a85e2019-10-31 12:26:08 +01004627 if (!plane_state->uapi.visible)
Matt Ropera1de91e2016-05-12 07:05:57 -07004628 return 0;
Ville Syrjälä83054942016-11-18 21:53:00 +02004629
Ville Syrjäläd1d23d72019-09-13 22:31:54 +03004630 if (plane->id == PLANE_CURSOR)
Matt Ropera1de91e2016-05-12 07:05:57 -07004631 return 0;
Ville Syrjäläd1d23d72019-09-13 22:31:54 +03004632
4633 if (color_plane == 1 &&
Imre Deak4941f352019-12-21 14:05:43 +02004634 !intel_format_info_is_yuv_semiplanar(fb->format, fb->modifier))
Matt Ropera1de91e2016-05-12 07:05:57 -07004635 return 0;
Kumar, Mahesha280f7d2016-04-06 08:26:39 -07004636
Ville Syrjäläfce5adf2017-03-31 21:00:55 +03004637 /*
4638 * Src coordinates are already rotated by 270 degrees for
4639 * the 90/270 degree plane rotation cases (to match the
4640 * GTT mapping), hence no need to account for rotation here.
4641 */
Maarten Lankhorstf90a85e2019-10-31 12:26:08 +01004642 width = drm_rect_width(&plane_state->uapi.src) >> 16;
4643 height = drm_rect_height(&plane_state->uapi.src) >> 16;
Kumar, Mahesha280f7d2016-04-06 08:26:39 -07004644
Mahesh Kumarb879d582018-04-09 09:11:01 +05304645 /* UV plane does 1/2 pixel sub-sampling */
Ville Syrjäläd1d23d72019-09-13 22:31:54 +03004646 if (color_plane == 1) {
Mahesh Kumarb879d582018-04-09 09:11:01 +05304647 width /= 2;
4648 height /= 2;
Chandra Konduru2cd601c2015-04-27 15:47:37 -07004649 }
4650
Maarten Lankhorst24719e92018-10-22 12:20:00 +02004651 data_rate = width * height;
Mahesh Kumarb879d582018-04-09 09:11:01 +05304652
Maarten Lankhorstec193642019-06-28 10:55:17 +02004653 down_scale_amount = skl_plane_downscale_amount(crtc_state, plane_state);
Kumar, Mahesh8d19d7d2016-05-19 15:03:01 -07004654
Maarten Lankhorst24719e92018-10-22 12:20:00 +02004655 rate = mul_round_up_u32_fixed16(data_rate, down_scale_amount);
4656
Ville Syrjäläd1d23d72019-09-13 22:31:54 +03004657 rate *= fb->format->cpp[color_plane];
Maarten Lankhorst24719e92018-10-22 12:20:00 +02004658 return rate;
Damien Lespiaub9cec072014-11-04 17:06:43 +00004659}
4660
Maarten Lankhorst24719e92018-10-22 12:20:00 +02004661static u64
Ville Syrjäläab016302020-11-06 19:30:41 +02004662skl_get_total_relative_data_rate(struct intel_atomic_state *state,
4663 struct intel_crtc *crtc)
Damien Lespiaub9cec072014-11-04 17:06:43 +00004664{
Ville Syrjäläab016302020-11-06 19:30:41 +02004665 struct intel_crtc_state *crtc_state =
4666 intel_atomic_get_new_crtc_state(state, crtc);
Maarten Lankhorstaf9fbfa2019-10-04 13:34:53 +02004667 const struct intel_plane_state *plane_state;
Ville Syrjäläab016302020-11-06 19:30:41 +02004668 struct intel_plane *plane;
Maarten Lankhorst24719e92018-10-22 12:20:00 +02004669 u64 total_data_rate = 0;
Ville Syrjäläab016302020-11-06 19:30:41 +02004670 enum plane_id plane_id;
4671 int i;
Matt Ropera6d3460e2016-05-12 07:06:04 -07004672
Matt Ropera1de91e2016-05-12 07:05:57 -07004673 /* Calculate and cache data rate for each plane */
Ville Syrjäläab016302020-11-06 19:30:41 +02004674 for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
4675 if (plane->pipe != crtc->pipe)
4676 continue;
4677
4678 plane_id = plane->id;
Matt Roper024c9042015-09-24 15:53:11 -07004679
Mahesh Kumarb879d582018-04-09 09:11:01 +05304680 /* packed/y */
Ville Syrjäläab016302020-11-06 19:30:41 +02004681 crtc_state->plane_data_rate[plane_id] =
4682 skl_plane_relative_data_rate(crtc_state, plane_state, 0);
Matt Roper9c74d822016-05-12 07:05:58 -07004683
Mahesh Kumarb879d582018-04-09 09:11:01 +05304684 /* uv-plane */
Ville Syrjäläab016302020-11-06 19:30:41 +02004685 crtc_state->uv_plane_data_rate[plane_id] =
4686 skl_plane_relative_data_rate(crtc_state, plane_state, 1);
4687 }
4688
4689 for_each_plane_id_on_crtc(crtc, plane_id) {
4690 total_data_rate += crtc_state->plane_data_rate[plane_id];
4691 total_data_rate += crtc_state->uv_plane_data_rate[plane_id];
Damien Lespiaub9cec072014-11-04 17:06:43 +00004692 }
4693
4694 return total_data_rate;
4695}
4696
Maarten Lankhorstb048a002018-10-18 13:51:30 +02004697static u64
Ville Syrjäläab016302020-11-06 19:30:41 +02004698icl_get_total_relative_data_rate(struct intel_atomic_state *state,
4699 struct intel_crtc *crtc)
Maarten Lankhorstb048a002018-10-18 13:51:30 +02004700{
Ville Syrjäläab016302020-11-06 19:30:41 +02004701 struct intel_crtc_state *crtc_state =
4702 intel_atomic_get_new_crtc_state(state, crtc);
Maarten Lankhorstaf9fbfa2019-10-04 13:34:53 +02004703 const struct intel_plane_state *plane_state;
Ville Syrjäläab016302020-11-06 19:30:41 +02004704 struct intel_plane *plane;
Maarten Lankhorstb048a002018-10-18 13:51:30 +02004705 u64 total_data_rate = 0;
Ville Syrjäläab016302020-11-06 19:30:41 +02004706 enum plane_id plane_id;
4707 int i;
Maarten Lankhorstb048a002018-10-18 13:51:30 +02004708
Maarten Lankhorstb048a002018-10-18 13:51:30 +02004709 /* Calculate and cache data rate for each plane */
Ville Syrjäläab016302020-11-06 19:30:41 +02004710 for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
4711 if (plane->pipe != crtc->pipe)
4712 continue;
4713
4714 plane_id = plane->id;
Maarten Lankhorstb048a002018-10-18 13:51:30 +02004715
Maarten Lankhorstc47b7dd2019-09-20 13:42:20 +02004716 if (!plane_state->planar_linked_plane) {
Ville Syrjäläab016302020-11-06 19:30:41 +02004717 crtc_state->plane_data_rate[plane_id] =
4718 skl_plane_relative_data_rate(crtc_state, plane_state, 0);
Maarten Lankhorstb048a002018-10-18 13:51:30 +02004719 } else {
4720 enum plane_id y_plane_id;
4721
4722 /*
4723 * The slave plane might not iterate in
Maarten Lankhorstaf9fbfa2019-10-04 13:34:53 +02004724 * intel_atomic_crtc_state_for_each_plane_state(),
Maarten Lankhorstb048a002018-10-18 13:51:30 +02004725 * and needs the master plane state which may be
4726 * NULL if we try get_new_plane_state(), so we
4727 * always calculate from the master.
4728 */
Maarten Lankhorstc47b7dd2019-09-20 13:42:20 +02004729 if (plane_state->planar_slave)
Maarten Lankhorstb048a002018-10-18 13:51:30 +02004730 continue;
4731
4732 /* Y plane rate is calculated on the slave */
Maarten Lankhorstc47b7dd2019-09-20 13:42:20 +02004733 y_plane_id = plane_state->planar_linked_plane->id;
Ville Syrjäläab016302020-11-06 19:30:41 +02004734 crtc_state->plane_data_rate[y_plane_id] =
4735 skl_plane_relative_data_rate(crtc_state, plane_state, 0);
Maarten Lankhorstb048a002018-10-18 13:51:30 +02004736
Ville Syrjäläab016302020-11-06 19:30:41 +02004737 crtc_state->plane_data_rate[plane_id] =
4738 skl_plane_relative_data_rate(crtc_state, plane_state, 1);
Maarten Lankhorstb048a002018-10-18 13:51:30 +02004739 }
4740 }
4741
Ville Syrjäläab016302020-11-06 19:30:41 +02004742 for_each_plane_id_on_crtc(crtc, plane_id)
4743 total_data_rate += crtc_state->plane_data_rate[plane_id];
4744
Maarten Lankhorstb048a002018-10-18 13:51:30 +02004745 return total_data_rate;
4746}
4747
Ville Syrjälä5516e892021-02-26 17:32:03 +02004748const struct skl_wm_level *
Ville Syrjälä2871b2f2021-02-26 17:31:58 +02004749skl_plane_wm_level(const struct skl_pipe_wm *pipe_wm,
Stanislav Lisovskiyd9162342020-05-13 12:38:11 +03004750 enum plane_id plane_id,
4751 int level)
4752{
Stanislav Lisovskiy7241c572020-05-14 10:48:51 +03004753 const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
4754
4755 if (level == 0 && pipe_wm->use_sagv_wm)
Ville Syrjäläa68aa482021-02-26 17:32:01 +02004756 return &wm->sagv.wm0;
Stanislav Lisovskiyd9162342020-05-13 12:38:11 +03004757
4758 return &wm->wm[level];
4759}
4760
Ville Syrjälä5516e892021-02-26 17:32:03 +02004761const struct skl_wm_level *
Ville Syrjälä5f25e6a2021-02-26 17:32:02 +02004762skl_plane_trans_wm(const struct skl_pipe_wm *pipe_wm,
4763 enum plane_id plane_id)
4764{
4765 const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
4766
4767 if (pipe_wm->use_sagv_wm)
4768 return &wm->sagv.trans_wm;
4769
4770 return &wm->trans_wm;
4771}
4772
Matt Roperc107acf2016-05-12 07:06:01 -07004773static int
Ville Syrjäläef79d622021-01-22 22:56:32 +02004774skl_allocate_plane_ddb(struct intel_atomic_state *state,
4775 struct intel_crtc *crtc)
Damien Lespiaub9cec072014-11-04 17:06:43 +00004776{
Ville Syrjäläef79d622021-01-22 22:56:32 +02004777 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
Ville Syrjäläffc90032020-11-06 19:30:37 +02004778 struct intel_crtc_state *crtc_state =
4779 intel_atomic_get_new_crtc_state(state, crtc);
Ville Syrjäläef79d622021-01-22 22:56:32 +02004780 const struct intel_dbuf_state *dbuf_state =
Ville Syrjälä47a14952021-01-22 22:56:30 +02004781 intel_atomic_get_new_dbuf_state(state);
Ville Syrjäläef79d622021-01-22 22:56:32 +02004782 const struct skl_ddb_entry *alloc = &dbuf_state->ddb[crtc->pipe];
4783 int num_active = hweight8(dbuf_state->active_pipes);
Jani Nikula5ce9a6492019-01-18 14:01:20 +02004784 u16 alloc_size, start = 0;
4785 u16 total[I915_MAX_PLANES] = {};
4786 u16 uv_total[I915_MAX_PLANES] = {};
Maarten Lankhorst24719e92018-10-22 12:20:00 +02004787 u64 total_data_rate;
Ville Syrjäläd5cdfdf52016-11-22 18:01:58 +02004788 enum plane_id plane_id;
Ville Syrjälä0aded172019-02-05 17:50:53 +02004789 u32 blocks;
Matt Roperd8e87492018-12-11 09:31:07 -08004790 int level;
Damien Lespiaub9cec072014-11-04 17:06:43 +00004791
Paulo Zanoni5a920b82016-10-04 14:37:32 -03004792 /* Clear the partitioning for disabled planes. */
Maarten Lankhorstec193642019-06-28 10:55:17 +02004793 memset(crtc_state->wm.skl.plane_ddb_y, 0, sizeof(crtc_state->wm.skl.plane_ddb_y));
4794 memset(crtc_state->wm.skl.plane_ddb_uv, 0, sizeof(crtc_state->wm.skl.plane_ddb_uv));
Paulo Zanoni5a920b82016-10-04 14:37:32 -03004795
Ville Syrjäläef79d622021-01-22 22:56:32 +02004796 if (!crtc_state->hw.active)
Matt Roperc107acf2016-05-12 07:06:01 -07004797 return 0;
Matt Roperc107acf2016-05-12 07:06:01 -07004798
Lucas De Marchi323b0a82019-04-04 16:04:25 -07004799 if (INTEL_GEN(dev_priv) >= 11)
4800 total_data_rate =
Ville Syrjäläab016302020-11-06 19:30:41 +02004801 icl_get_total_relative_data_rate(state, crtc);
Lucas De Marchi323b0a82019-04-04 16:04:25 -07004802 else
Maarten Lankhorstb048a002018-10-18 13:51:30 +02004803 total_data_rate =
Ville Syrjäläab016302020-11-06 19:30:41 +02004804 skl_get_total_relative_data_rate(state, crtc);
Lucas De Marchi323b0a82019-04-04 16:04:25 -07004805
Damien Lespiau34bb56a2014-11-04 17:07:01 +00004806 alloc_size = skl_ddb_entry_size(alloc);
Kumar, Mahesh336031e2017-05-17 17:28:25 +05304807 if (alloc_size == 0)
Matt Roperc107acf2016-05-12 07:06:01 -07004808 return 0;
Damien Lespiaub9cec072014-11-04 17:06:43 +00004809
Matt Roperd8e87492018-12-11 09:31:07 -08004810 /* Allocate fixed number of blocks for cursor. */
Maarten Lankhorstec193642019-06-28 10:55:17 +02004811 total[PLANE_CURSOR] = skl_cursor_allocation(crtc_state, num_active);
Matt Roperd8e87492018-12-11 09:31:07 -08004812 alloc_size -= total[PLANE_CURSOR];
Maarten Lankhorstec193642019-06-28 10:55:17 +02004813 crtc_state->wm.skl.plane_ddb_y[PLANE_CURSOR].start =
Matt Roperd8e87492018-12-11 09:31:07 -08004814 alloc->end - total[PLANE_CURSOR];
Maarten Lankhorstec193642019-06-28 10:55:17 +02004815 crtc_state->wm.skl.plane_ddb_y[PLANE_CURSOR].end = alloc->end;
Maarten Lankhorst49845a72016-10-26 15:41:34 +02004816
Matt Ropera1de91e2016-05-12 07:05:57 -07004817 if (total_data_rate == 0)
Matt Roperc107acf2016-05-12 07:06:01 -07004818 return 0;
Damien Lespiaub9cec072014-11-04 17:06:43 +00004819
Matt Roperd8e87492018-12-11 09:31:07 -08004820 /*
4821 * Find the highest watermark level for which we can satisfy the block
4822 * requirement of active planes.
4823 */
4824 for (level = ilk_wm_max_level(dev_priv); level >= 0; level--) {
Matt Roper25db2ea2018-12-12 11:17:20 -08004825 blocks = 0;
Ville Syrjälä2a67054b2020-02-25 19:11:06 +02004826 for_each_plane_id_on_crtc(crtc, plane_id) {
Ville Syrjälä5e6037c2019-03-12 22:58:42 +02004827 const struct skl_plane_wm *wm =
Maarten Lankhorstec193642019-06-28 10:55:17 +02004828 &crtc_state->wm.skl.optimal.planes[plane_id];
Ville Syrjälä10a7e072019-03-12 22:58:40 +02004829
4830 if (plane_id == PLANE_CURSOR) {
Vandita Kulkarni4ba48702019-12-16 13:36:19 +05304831 if (wm->wm[level].min_ddb_alloc > total[PLANE_CURSOR]) {
Pankaj Bharadiya48a1b8d2020-01-15 09:14:53 +05304832 drm_WARN_ON(&dev_priv->drm,
4833 wm->wm[level].min_ddb_alloc != U16_MAX);
Ville Syrjälä10a7e072019-03-12 22:58:40 +02004834 blocks = U32_MAX;
4835 break;
4836 }
4837 continue;
4838 }
4839
Ville Syrjälä961d95e2018-12-21 19:14:32 +02004840 blocks += wm->wm[level].min_ddb_alloc;
4841 blocks += wm->uv_wm[level].min_ddb_alloc;
Matt Roperd8e87492018-12-11 09:31:07 -08004842 }
4843
Ville Syrjälä3cf963c2019-03-12 22:58:36 +02004844 if (blocks <= alloc_size) {
Matt Roperd8e87492018-12-11 09:31:07 -08004845 alloc_size -= blocks;
4846 break;
4847 }
4848 }
4849
4850 if (level < 0) {
Wambui Karugaf8d18d52020-01-07 18:13:30 +03004851 drm_dbg_kms(&dev_priv->drm,
4852 "Requested display configuration exceeds system DDB limitations");
4853 drm_dbg_kms(&dev_priv->drm, "minimum required %d/%d\n",
4854 blocks, alloc_size);
Matt Roperd8e87492018-12-11 09:31:07 -08004855 return -EINVAL;
4856 }
4857
4858 /*
4859 * Grant each plane the blocks it requires at the highest achievable
4860 * watermark level, plus an extra share of the leftover blocks
4861 * proportional to its relative data rate.
4862 */
Ville Syrjälä2a67054b2020-02-25 19:11:06 +02004863 for_each_plane_id_on_crtc(crtc, plane_id) {
Ville Syrjälä5e6037c2019-03-12 22:58:42 +02004864 const struct skl_plane_wm *wm =
Maarten Lankhorstec193642019-06-28 10:55:17 +02004865 &crtc_state->wm.skl.optimal.planes[plane_id];
Matt Roperd8e87492018-12-11 09:31:07 -08004866 u64 rate;
4867 u16 extra;
Damien Lespiaub9cec072014-11-04 17:06:43 +00004868
Ville Syrjäläd5cdfdf52016-11-22 18:01:58 +02004869 if (plane_id == PLANE_CURSOR)
Maarten Lankhorst49845a72016-10-26 15:41:34 +02004870 continue;
4871
Damien Lespiaub9cec072014-11-04 17:06:43 +00004872 /*
Matt Roperd8e87492018-12-11 09:31:07 -08004873 * We've accounted for all active planes; remaining planes are
4874 * all disabled.
Damien Lespiaub9cec072014-11-04 17:06:43 +00004875 */
Matt Roperd8e87492018-12-11 09:31:07 -08004876 if (total_data_rate == 0)
4877 break;
Damien Lespiaub9cec072014-11-04 17:06:43 +00004878
Ville Syrjäläab016302020-11-06 19:30:41 +02004879 rate = crtc_state->plane_data_rate[plane_id];
Matt Roperd8e87492018-12-11 09:31:07 -08004880 extra = min_t(u16, alloc_size,
4881 DIV64_U64_ROUND_UP(alloc_size * rate,
4882 total_data_rate));
Ville Syrjälä961d95e2018-12-21 19:14:32 +02004883 total[plane_id] = wm->wm[level].min_ddb_alloc + extra;
Matt Roperd8e87492018-12-11 09:31:07 -08004884 alloc_size -= extra;
4885 total_data_rate -= rate;
Rodrigo Vivi9a30a262017-06-13 10:52:30 -07004886
Matt Roperd8e87492018-12-11 09:31:07 -08004887 if (total_data_rate == 0)
4888 break;
Chandra Konduru2cd601c2015-04-27 15:47:37 -07004889
Ville Syrjäläab016302020-11-06 19:30:41 +02004890 rate = crtc_state->uv_plane_data_rate[plane_id];
Matt Roperd8e87492018-12-11 09:31:07 -08004891 extra = min_t(u16, alloc_size,
4892 DIV64_U64_ROUND_UP(alloc_size * rate,
4893 total_data_rate));
Ville Syrjälä961d95e2018-12-21 19:14:32 +02004894 uv_total[plane_id] = wm->uv_wm[level].min_ddb_alloc + extra;
Matt Roperd8e87492018-12-11 09:31:07 -08004895 alloc_size -= extra;
4896 total_data_rate -= rate;
4897 }
Pankaj Bharadiya48a1b8d2020-01-15 09:14:53 +05304898 drm_WARN_ON(&dev_priv->drm, alloc_size != 0 || total_data_rate != 0);
Matt Roperd8e87492018-12-11 09:31:07 -08004899
4900 /* Set the actual DDB start/end points for each plane */
4901 start = alloc->start;
Ville Syrjälä2a67054b2020-02-25 19:11:06 +02004902 for_each_plane_id_on_crtc(crtc, plane_id) {
Ville Syrjälä5e6037c2019-03-12 22:58:42 +02004903 struct skl_ddb_entry *plane_alloc =
Maarten Lankhorstec193642019-06-28 10:55:17 +02004904 &crtc_state->wm.skl.plane_ddb_y[plane_id];
Ville Syrjälä5e6037c2019-03-12 22:58:42 +02004905 struct skl_ddb_entry *uv_plane_alloc =
Maarten Lankhorstec193642019-06-28 10:55:17 +02004906 &crtc_state->wm.skl.plane_ddb_uv[plane_id];
Matt Roperd8e87492018-12-11 09:31:07 -08004907
4908 if (plane_id == PLANE_CURSOR)
4909 continue;
4910
Maarten Lankhorstb048a002018-10-18 13:51:30 +02004911 /* Gen11+ uses a separate plane for UV watermarks */
Pankaj Bharadiya48a1b8d2020-01-15 09:14:53 +05304912 drm_WARN_ON(&dev_priv->drm,
4913 INTEL_GEN(dev_priv) >= 11 && uv_total[plane_id]);
Maarten Lankhorstb048a002018-10-18 13:51:30 +02004914
Matt Roperd8e87492018-12-11 09:31:07 -08004915 /* Leave disabled planes at (0,0) */
4916 if (total[plane_id]) {
4917 plane_alloc->start = start;
4918 start += total[plane_id];
4919 plane_alloc->end = start;
Matt Roperc107acf2016-05-12 07:06:01 -07004920 }
Rodrigo Vivi9a30a262017-06-13 10:52:30 -07004921
Matt Roperd8e87492018-12-11 09:31:07 -08004922 if (uv_total[plane_id]) {
4923 uv_plane_alloc->start = start;
4924 start += uv_total[plane_id];
4925 uv_plane_alloc->end = start;
4926 }
4927 }
4928
4929 /*
4930 * When we calculated watermark values we didn't know how high
4931 * of a level we'd actually be able to hit, so we just marked
4932 * all levels as "enabled." Go back now and disable the ones
4933 * that aren't actually possible.
4934 */
4935 for (level++; level <= ilk_wm_max_level(dev_priv); level++) {
Ville Syrjälä2a67054b2020-02-25 19:11:06 +02004936 for_each_plane_id_on_crtc(crtc, plane_id) {
Ville Syrjälä5e6037c2019-03-12 22:58:42 +02004937 struct skl_plane_wm *wm =
Maarten Lankhorstec193642019-06-28 10:55:17 +02004938 &crtc_state->wm.skl.optimal.planes[plane_id];
Ville Syrjäläa301cb02019-03-12 22:58:41 +02004939
4940 /*
4941 * We only disable the watermarks for each plane if
4942 * they exceed the ddb allocation of said plane. This
4943 * is done so that we don't end up touching cursor
4944 * watermarks needlessly when some other plane reduces
4945 * our max possible watermark level.
4946 *
4947 * Bspec has this to say about the PLANE_WM enable bit:
4948 * "All the watermarks at this level for all enabled
4949 * planes must be enabled before the level will be used."
4950 * So this is actually safe to do.
4951 */
4952 if (wm->wm[level].min_ddb_alloc > total[plane_id] ||
4953 wm->uv_wm[level].min_ddb_alloc > uv_total[plane_id])
4954 memset(&wm->wm[level], 0, sizeof(wm->wm[level]));
Ville Syrjälä290248c2019-02-13 18:54:24 +02004955
Ville Syrjäläc384afe2019-02-28 19:36:39 +02004956 /*
Bob Paauwe39564ae2019-04-12 11:09:20 -07004957 * Wa_1408961008:icl, ehl
Ville Syrjäläc384afe2019-02-28 19:36:39 +02004958 * Underruns with WM1+ disabled
4959 */
Bob Paauwe39564ae2019-04-12 11:09:20 -07004960 if (IS_GEN(dev_priv, 11) &&
Ville Syrjälä290248c2019-02-13 18:54:24 +02004961 level == 1 && wm->wm[0].plane_en) {
4962 wm->wm[level].plane_res_b = wm->wm[0].plane_res_b;
Ville Syrjäläc384afe2019-02-28 19:36:39 +02004963 wm->wm[level].plane_res_l = wm->wm[0].plane_res_l;
4964 wm->wm[level].ignore_lines = wm->wm[0].ignore_lines;
Ville Syrjälä290248c2019-02-13 18:54:24 +02004965 }
Matt Roperd8e87492018-12-11 09:31:07 -08004966 }
4967 }
4968
4969 /*
Ville Syrjälädf4a50a2021-02-26 17:31:59 +02004970 * Go back and disable the transition and SAGV watermarks
4971 * if it turns out we don't have enough DDB blocks for them.
Matt Roperd8e87492018-12-11 09:31:07 -08004972 */
Ville Syrjälä2a67054b2020-02-25 19:11:06 +02004973 for_each_plane_id_on_crtc(crtc, plane_id) {
Ville Syrjälä5e6037c2019-03-12 22:58:42 +02004974 struct skl_plane_wm *wm =
Maarten Lankhorstec193642019-06-28 10:55:17 +02004975 &crtc_state->wm.skl.optimal.planes[plane_id];
Ville Syrjälä5e6037c2019-03-12 22:58:42 +02004976
Ville Syrjäläb19c9bc2018-12-21 19:14:31 +02004977 if (wm->trans_wm.plane_res_b >= total[plane_id])
Matt Roperd8e87492018-12-11 09:31:07 -08004978 memset(&wm->trans_wm, 0, sizeof(wm->trans_wm));
Ville Syrjälädf4a50a2021-02-26 17:31:59 +02004979
Ville Syrjäläa68aa482021-02-26 17:32:01 +02004980 if (wm->sagv.wm0.plane_res_b >= total[plane_id])
4981 memset(&wm->sagv.wm0, 0, sizeof(wm->sagv.wm0));
Ville Syrjälä5f25e6a2021-02-26 17:32:02 +02004982
4983 if (wm->sagv.trans_wm.plane_res_b >= total[plane_id])
4984 memset(&wm->sagv.trans_wm, 0, sizeof(wm->sagv.trans_wm));
Damien Lespiaub9cec072014-11-04 17:06:43 +00004985 }
4986
Matt Roperc107acf2016-05-12 07:06:01 -07004987 return 0;
Damien Lespiaub9cec072014-11-04 17:06:43 +00004988}
4989
Pradeep Bhat2d41c0b2014-11-04 17:06:42 +00004990/*
4991 * The max latency should be 257 (max the punit can code is 255 and we add 2us
Ville Syrjäläac484962016-01-20 21:05:26 +02004992 * for the read latency) and cpp should always be <= 8, so that
Pradeep Bhat2d41c0b2014-11-04 17:06:42 +00004993 * should allow pixel_rate up to ~2 GHz which seems sufficient since max
4994 * 2xcdclk is 1350 MHz and the pixel rate should never exceed that.
4995*/
Paulo Zanoni6c64dd32017-08-11 16:38:25 -07004996static uint_fixed_16_16_t
Jani Nikula5ce9a6492019-01-18 14:01:20 +02004997skl_wm_method1(const struct drm_i915_private *dev_priv, u32 pixel_rate,
4998 u8 cpp, u32 latency, u32 dbuf_block_size)
Pradeep Bhat2d41c0b2014-11-04 17:06:42 +00004999{
Jani Nikula5ce9a6492019-01-18 14:01:20 +02005000 u32 wm_intermediate_val;
Mahesh Kumarb95320b2016-12-01 21:19:37 +05305001 uint_fixed_16_16_t ret;
Pradeep Bhat2d41c0b2014-11-04 17:06:42 +00005002
5003 if (latency == 0)
Mahesh Kumarb95320b2016-12-01 21:19:37 +05305004 return FP_16_16_MAX;
Pradeep Bhat2d41c0b2014-11-04 17:06:42 +00005005
Mahesh Kumarb95320b2016-12-01 21:19:37 +05305006 wm_intermediate_val = latency * pixel_rate * cpp;
Mahesh Kumardf8ee192018-01-30 11:49:11 -02005007 ret = div_fixed16(wm_intermediate_val, 1000 * dbuf_block_size);
Paulo Zanoni6c64dd32017-08-11 16:38:25 -07005008
Ville Syrjälä260a6c1b2020-04-30 15:58:21 +03005009 if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
Paulo Zanoni6c64dd32017-08-11 16:38:25 -07005010 ret = add_fixed16_u32(ret, 1);
5011
Pradeep Bhat2d41c0b2014-11-04 17:06:42 +00005012 return ret;
5013}
5014
Jani Nikula5ce9a6492019-01-18 14:01:20 +02005015static uint_fixed_16_16_t
5016skl_wm_method2(u32 pixel_rate, u32 pipe_htotal, u32 latency,
5017 uint_fixed_16_16_t plane_blocks_per_line)
Pradeep Bhat2d41c0b2014-11-04 17:06:42 +00005018{
Jani Nikula5ce9a6492019-01-18 14:01:20 +02005019 u32 wm_intermediate_val;
Mahesh Kumarb95320b2016-12-01 21:19:37 +05305020 uint_fixed_16_16_t ret;
Pradeep Bhat2d41c0b2014-11-04 17:06:42 +00005021
5022 if (latency == 0)
Mahesh Kumarb95320b2016-12-01 21:19:37 +05305023 return FP_16_16_MAX;
Pradeep Bhat2d41c0b2014-11-04 17:06:42 +00005024
Pradeep Bhat2d41c0b2014-11-04 17:06:42 +00005025 wm_intermediate_val = latency * pixel_rate;
Mahesh Kumarb95320b2016-12-01 21:19:37 +05305026 wm_intermediate_val = DIV_ROUND_UP(wm_intermediate_val,
5027 pipe_htotal * 1000);
Kumar, Mahesheac2cb82017-07-05 20:01:46 +05305028 ret = mul_u32_fixed16(wm_intermediate_val, plane_blocks_per_line);
Pradeep Bhat2d41c0b2014-11-04 17:06:42 +00005029 return ret;
5030}
5031
Kumar, Maheshd555cb52017-05-17 17:28:29 +05305032static uint_fixed_16_16_t
Maarten Lankhorstec193642019-06-28 10:55:17 +02005033intel_get_linetime_us(const struct intel_crtc_state *crtc_state)
Kumar, Maheshd555cb52017-05-17 17:28:29 +05305034{
Pankaj Bharadiya19edeb382020-05-04 23:45:59 +05305035 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
Jani Nikula5ce9a6492019-01-18 14:01:20 +02005036 u32 pixel_rate;
5037 u32 crtc_htotal;
Kumar, Maheshd555cb52017-05-17 17:28:29 +05305038 uint_fixed_16_16_t linetime_us;
5039
Maarten Lankhorst1326a922019-10-31 12:26:02 +01005040 if (!crtc_state->hw.active)
Kumar, Mahesheac2cb82017-07-05 20:01:46 +05305041 return u32_to_fixed16(0);
Kumar, Maheshd555cb52017-05-17 17:28:29 +05305042
Maarten Lankhorstec193642019-06-28 10:55:17 +02005043 pixel_rate = crtc_state->pixel_rate;
Kumar, Maheshd555cb52017-05-17 17:28:29 +05305044
Pankaj Bharadiya19edeb382020-05-04 23:45:59 +05305045 if (drm_WARN_ON(&dev_priv->drm, pixel_rate == 0))
Kumar, Mahesheac2cb82017-07-05 20:01:46 +05305046 return u32_to_fixed16(0);
Kumar, Maheshd555cb52017-05-17 17:28:29 +05305047
Maarten Lankhorstbafcdad2020-11-12 21:17:18 +02005048 crtc_htotal = crtc_state->hw.pipe_mode.crtc_htotal;
Kumar, Mahesheac2cb82017-07-05 20:01:46 +05305049 linetime_us = div_fixed16(crtc_htotal * 1000, pixel_rate);
Kumar, Maheshd555cb52017-05-17 17:28:29 +05305050
5051 return linetime_us;
5052}
5053
Kumar, Mahesh7e452fd2017-08-17 19:15:23 +05305054static int
Ville Syrjäläc92558a2019-03-12 22:58:38 +02005055skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
5056 int width, const struct drm_format_info *format,
5057 u64 modifier, unsigned int rotation,
5058 u32 plane_pixel_rate, struct skl_wm_params *wp,
5059 int color_plane)
Kumar, Mahesh7e452fd2017-08-17 19:15:23 +05305060{
Maarten Lankhorst2225f3c2019-10-31 12:26:03 +01005061 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
Ville Syrjäläc92558a2019-03-12 22:58:38 +02005062 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
Jani Nikula5ce9a6492019-01-18 14:01:20 +02005063 u32 interm_pbpl;
Kumar, Mahesh7e452fd2017-08-17 19:15:23 +05305064
Juha-Pekka Heikkiladf7d4152019-03-04 17:26:31 +05305065 /* only planar format has two planes */
Imre Deak4941f352019-12-21 14:05:43 +02005066 if (color_plane == 1 &&
5067 !intel_format_info_is_yuv_semiplanar(format, modifier)) {
Wambui Karugaf8d18d52020-01-07 18:13:30 +03005068 drm_dbg_kms(&dev_priv->drm,
5069 "Non planar format have single plane\n");
Mahesh Kumar942aa2d2018-04-09 09:11:04 +05305070 return -EINVAL;
5071 }
5072
Ville Syrjäläc92558a2019-03-12 22:58:38 +02005073 wp->y_tiled = modifier == I915_FORMAT_MOD_Y_TILED ||
5074 modifier == I915_FORMAT_MOD_Yf_TILED ||
5075 modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
5076 modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
5077 wp->x_tiled = modifier == I915_FORMAT_MOD_X_TILED;
5078 wp->rc_surface = modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
5079 modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
Imre Deak4941f352019-12-21 14:05:43 +02005080 wp->is_planar = intel_format_info_is_yuv_semiplanar(format, modifier);
Kumar, Mahesh7e452fd2017-08-17 19:15:23 +05305081
Ville Syrjäläc92558a2019-03-12 22:58:38 +02005082 wp->width = width;
Ville Syrjälä45bee432018-11-14 23:07:28 +02005083 if (color_plane == 1 && wp->is_planar)
Mahesh Kumar942aa2d2018-04-09 09:11:04 +05305084 wp->width /= 2;
5085
Ville Syrjäläc92558a2019-03-12 22:58:38 +02005086 wp->cpp = format->cpp[color_plane];
5087 wp->plane_pixel_rate = plane_pixel_rate;
Kumar, Mahesh7e452fd2017-08-17 19:15:23 +05305088
Mahesh Kumardf8ee192018-01-30 11:49:11 -02005089 if (INTEL_GEN(dev_priv) >= 11 &&
Ville Syrjäläc92558a2019-03-12 22:58:38 +02005090 modifier == I915_FORMAT_MOD_Yf_TILED && wp->cpp == 1)
Mahesh Kumardf8ee192018-01-30 11:49:11 -02005091 wp->dbuf_block_size = 256;
5092 else
5093 wp->dbuf_block_size = 512;
5094
Ville Syrjäläc92558a2019-03-12 22:58:38 +02005095 if (drm_rotation_90_or_270(rotation)) {
Kumar, Mahesh7e452fd2017-08-17 19:15:23 +05305096 switch (wp->cpp) {
5097 case 1:
5098 wp->y_min_scanlines = 16;
5099 break;
5100 case 2:
5101 wp->y_min_scanlines = 8;
5102 break;
5103 case 4:
5104 wp->y_min_scanlines = 4;
5105 break;
5106 default:
5107 MISSING_CASE(wp->cpp);
5108 return -EINVAL;
5109 }
5110 } else {
5111 wp->y_min_scanlines = 4;
5112 }
5113
Ville Syrjälä60e983f2018-12-21 19:14:33 +02005114 if (skl_needs_memory_bw_wa(dev_priv))
Kumar, Mahesh7e452fd2017-08-17 19:15:23 +05305115 wp->y_min_scanlines *= 2;
5116
5117 wp->plane_bytes_per_line = wp->width * wp->cpp;
5118 if (wp->y_tiled) {
5119 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line *
Mahesh Kumardf8ee192018-01-30 11:49:11 -02005120 wp->y_min_scanlines,
5121 wp->dbuf_block_size);
Kumar, Mahesh7e452fd2017-08-17 19:15:23 +05305122
Ville Syrjälä260a6c1b2020-04-30 15:58:21 +03005123 if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
Kumar, Mahesh7e452fd2017-08-17 19:15:23 +05305124 interm_pbpl++;
5125
5126 wp->plane_blocks_per_line = div_fixed16(interm_pbpl,
5127 wp->y_min_scanlines);
Kumar, Mahesh7e452fd2017-08-17 19:15:23 +05305128 } else {
Mahesh Kumardf8ee192018-01-30 11:49:11 -02005129 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line,
Ville Syrjälä260a6c1b2020-04-30 15:58:21 +03005130 wp->dbuf_block_size);
5131
5132 if (!wp->x_tiled ||
5133 INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
5134 interm_pbpl++;
5135
Kumar, Mahesh7e452fd2017-08-17 19:15:23 +05305136 wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl);
5137 }
5138
5139 wp->y_tile_minimum = mul_u32_fixed16(wp->y_min_scanlines,
5140 wp->plane_blocks_per_line);
Ville Syrjäläc92558a2019-03-12 22:58:38 +02005141
Kumar, Mahesh7e452fd2017-08-17 19:15:23 +05305142 wp->linetime_us = fixed16_to_u32_round_up(
Ville Syrjäläc92558a2019-03-12 22:58:38 +02005143 intel_get_linetime_us(crtc_state));
Kumar, Mahesh7e452fd2017-08-17 19:15:23 +05305144
5145 return 0;
5146}
5147
Ville Syrjäläc92558a2019-03-12 22:58:38 +02005148static int
5149skl_compute_plane_wm_params(const struct intel_crtc_state *crtc_state,
5150 const struct intel_plane_state *plane_state,
5151 struct skl_wm_params *wp, int color_plane)
5152{
Maarten Lankhorst7b3cb172019-10-31 12:26:07 +01005153 const struct drm_framebuffer *fb = plane_state->hw.fb;
Ville Syrjäläc92558a2019-03-12 22:58:38 +02005154 int width;
5155
Maarten Lankhorst3a612762019-10-04 13:34:54 +02005156 /*
5157 * Src coordinates are already rotated by 270 degrees for
5158 * the 90/270 degree plane rotation cases (to match the
5159 * GTT mapping), hence no need to account for rotation here.
5160 */
Maarten Lankhorstf90a85e2019-10-31 12:26:08 +01005161 width = drm_rect_width(&plane_state->uapi.src) >> 16;
Ville Syrjäläc92558a2019-03-12 22:58:38 +02005162
5163 return skl_compute_wm_params(crtc_state, width,
5164 fb->format, fb->modifier,
Maarten Lankhorst7b3cb172019-10-31 12:26:07 +01005165 plane_state->hw.rotation,
Ville Syrjälä3df3fe22020-11-06 19:30:42 +02005166 intel_plane_pixel_rate(crtc_state, plane_state),
Ville Syrjäläc92558a2019-03-12 22:58:38 +02005167 wp, color_plane);
5168}
5169
Ville Syrjäläb52c2732018-12-21 19:14:28 +02005170static bool skl_wm_has_lines(struct drm_i915_private *dev_priv, int level)
5171{
5172 if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
5173 return true;
5174
5175 /* The number of lines are ignored for the level 0 watermark. */
5176 return level > 0;
5177}
5178
Maarten Lankhorstec193642019-06-28 10:55:17 +02005179static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
Matt Roperd8e87492018-12-11 09:31:07 -08005180 int level,
Stanislav Lisovskiy7b994752020-04-09 18:47:18 +03005181 unsigned int latency,
Matt Roperd8e87492018-12-11 09:31:07 -08005182 const struct skl_wm_params *wp,
5183 const struct skl_wm_level *result_prev,
5184 struct skl_wm_level *result /* out */)
Pradeep Bhat2d41c0b2014-11-04 17:06:42 +00005185{
Maarten Lankhorst2225f3c2019-10-31 12:26:03 +01005186 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
Mahesh Kumarb95320b2016-12-01 21:19:37 +05305187 uint_fixed_16_16_t method1, method2;
Mahesh Kumarb95320b2016-12-01 21:19:37 +05305188 uint_fixed_16_16_t selected_result;
Ville Syrjälä961d95e2018-12-21 19:14:32 +02005189 u32 res_blocks, res_lines, min_ddb_alloc = 0;
Ville Syrjäläce110ec2018-11-14 23:07:21 +02005190
Ville Syrjälä0aded172019-02-05 17:50:53 +02005191 if (latency == 0) {
5192 /* reject it */
5193 result->min_ddb_alloc = U16_MAX;
Ville Syrjälä692927f2018-12-21 19:14:29 +02005194 return;
Ville Syrjälä0aded172019-02-05 17:50:53 +02005195 }
Ville Syrjälä692927f2018-12-21 19:14:29 +02005196
Ville Syrjälä25312ef2019-05-03 20:38:05 +03005197 /*
5198 * WaIncreaseLatencyIPCEnabled: kbl,cfl
5199 * Display WA #1141: kbl,cfl
5200 */
Chris Wilson5f4ae272020-06-02 15:05:40 +01005201 if ((IS_KABYLAKE(dev_priv) ||
5202 IS_COFFEELAKE(dev_priv) ||
5203 IS_COMETLAKE(dev_priv)) &&
Rodrigo Vivi82525c12017-06-08 08:50:00 -07005204 dev_priv->ipc_enabled)
Mahesh Kumar4b7b2332016-12-01 21:19:35 +05305205 latency += 4;
5206
Ville Syrjälä60e983f2018-12-21 19:14:33 +02005207 if (skl_needs_memory_bw_wa(dev_priv) && wp->x_tiled)
Paulo Zanoniee3d5322016-10-11 15:25:38 -03005208 latency += 15;
5209
Kumar, Mahesh7e452fd2017-08-17 19:15:23 +05305210 method1 = skl_wm_method1(dev_priv, wp->plane_pixel_rate,
Mahesh Kumardf8ee192018-01-30 11:49:11 -02005211 wp->cpp, latency, wp->dbuf_block_size);
Kumar, Mahesh7e452fd2017-08-17 19:15:23 +05305212 method2 = skl_wm_method2(wp->plane_pixel_rate,
Maarten Lankhorstbafcdad2020-11-12 21:17:18 +02005213 crtc_state->hw.pipe_mode.crtc_htotal,
Paulo Zanoni1186fa82016-09-22 18:00:31 -03005214 latency,
Kumar, Mahesh7e452fd2017-08-17 19:15:23 +05305215 wp->plane_blocks_per_line);
Pradeep Bhat2d41c0b2014-11-04 17:06:42 +00005216
Kumar, Mahesh7e452fd2017-08-17 19:15:23 +05305217 if (wp->y_tiled) {
5218 selected_result = max_fixed16(method2, wp->y_tile_minimum);
Tvrtko Ursulin0fda6562015-02-27 15:12:35 +00005219 } else {
Maarten Lankhorstbafcdad2020-11-12 21:17:18 +02005220 if ((wp->cpp * crtc_state->hw.pipe_mode.crtc_htotal /
Mahesh Kumardf8ee192018-01-30 11:49:11 -02005221 wp->dbuf_block_size < 1) &&
Paulo Zanoni077b5822018-10-04 16:15:57 -07005222 (wp->plane_bytes_per_line / wp->dbuf_block_size < 1)) {
Paulo Zanonif1db3ea2016-09-22 18:00:34 -03005223 selected_result = method2;
Paulo Zanoni077b5822018-10-04 16:15:57 -07005224 } else if (latency >= wp->linetime_us) {
Lucas De Marchicf819ef2018-12-12 10:10:43 -08005225 if (IS_GEN(dev_priv, 9) &&
Paulo Zanoni077b5822018-10-04 16:15:57 -07005226 !IS_GEMINILAKE(dev_priv))
5227 selected_result = min_fixed16(method1, method2);
5228 else
5229 selected_result = method2;
5230 } else {
Tvrtko Ursulin0fda6562015-02-27 15:12:35 +00005231 selected_result = method1;
Paulo Zanoni077b5822018-10-04 16:15:57 -07005232 }
Tvrtko Ursulin0fda6562015-02-27 15:12:35 +00005233 }
Pradeep Bhat2d41c0b2014-11-04 17:06:42 +00005234
Kumar, Mahesheac2cb82017-07-05 20:01:46 +05305235 res_blocks = fixed16_to_u32_round_up(selected_result) + 1;
Kumar, Maheshd273ecc2017-05-17 17:28:22 +05305236 res_lines = div_round_up_fixed16(selected_result,
Kumar, Mahesh7e452fd2017-08-17 19:15:23 +05305237 wp->plane_blocks_per_line);
Damien Lespiaue6d66172014-11-04 17:06:55 +00005238
Paulo Zanonia5b79d32018-11-13 17:24:32 -08005239 if (IS_GEN9_BC(dev_priv) || IS_BROXTON(dev_priv)) {
5240 /* Display WA #1125: skl,bxt,kbl */
5241 if (level == 0 && wp->rc_surface)
5242 res_blocks +=
5243 fixed16_to_u32_round_up(wp->y_tile_minimum);
Ville Syrjälä2e2adb02017-08-01 09:58:13 -07005244
Paulo Zanonia5b79d32018-11-13 17:24:32 -08005245 /* Display WA #1126: skl,bxt,kbl */
5246 if (level >= 1 && level <= 7) {
5247 if (wp->y_tiled) {
5248 res_blocks +=
5249 fixed16_to_u32_round_up(wp->y_tile_minimum);
5250 res_lines += wp->y_min_scanlines;
5251 } else {
5252 res_blocks++;
5253 }
5254
5255 /*
5256 * Make sure result blocks for higher latency levels are
5257 * atleast as high as level below the current level.
5258 * Assumption in DDB algorithm optimization for special
5259 * cases. Also covers Display WA #1125 for RC.
5260 */
5261 if (result_prev->plane_res_b > res_blocks)
5262 res_blocks = result_prev->plane_res_b;
Paulo Zanoni75676ed2016-09-22 18:00:33 -03005263 }
Tvrtko Ursulin0fda6562015-02-27 15:12:35 +00005264 }
Tvrtko Ursulind4c2aa62015-02-27 11:15:22 +00005265
Ville Syrjälä961d95e2018-12-21 19:14:32 +02005266 if (INTEL_GEN(dev_priv) >= 11) {
5267 if (wp->y_tiled) {
5268 int extra_lines;
5269
5270 if (res_lines % wp->y_min_scanlines == 0)
5271 extra_lines = wp->y_min_scanlines;
5272 else
5273 extra_lines = wp->y_min_scanlines * 2 -
5274 res_lines % wp->y_min_scanlines;
5275
5276 min_ddb_alloc = mul_round_up_u32_fixed16(res_lines + extra_lines,
5277 wp->plane_blocks_per_line);
5278 } else {
5279 min_ddb_alloc = res_blocks +
5280 DIV_ROUND_UP(res_blocks, 10);
5281 }
5282 }
5283
Ville Syrjäläb52c2732018-12-21 19:14:28 +02005284 if (!skl_wm_has_lines(dev_priv, level))
5285 res_lines = 0;
5286
Ville Syrjälä0aded172019-02-05 17:50:53 +02005287 if (res_lines > 31) {
5288 /* reject it */
5289 result->min_ddb_alloc = U16_MAX;
Matt Roperd8e87492018-12-11 09:31:07 -08005290 return;
Ville Syrjälä0aded172019-02-05 17:50:53 +02005291 }
Matt Roperd8e87492018-12-11 09:31:07 -08005292
5293 /*
5294 * If res_lines is valid, assume we can use this watermark level
5295 * for now. We'll come back and disable it after we calculate the
5296 * DDB allocation if it turns out we don't actually have enough
5297 * blocks to satisfy it.
5298 */
Mahesh Kumar62027b72018-04-09 09:11:05 +05305299 result->plane_res_b = res_blocks;
5300 result->plane_res_l = res_lines;
Ville Syrjälä961d95e2018-12-21 19:14:32 +02005301 /* Bspec says: value >= plane ddb allocation -> invalid, hence the +1 here */
5302 result->min_ddb_alloc = max(min_ddb_alloc, res_blocks) + 1;
Mahesh Kumar62027b72018-04-09 09:11:05 +05305303 result->plane_en = true;
Ville Syrjälä9c312122020-11-06 19:30:40 +02005304
5305 if (INTEL_GEN(dev_priv) < 12)
5306 result->can_sagv = latency >= dev_priv->sagv_block_time_us;
Pradeep Bhat2d41c0b2014-11-04 17:06:42 +00005307}
5308
Matt Roperd8e87492018-12-11 09:31:07 -08005309static void
Maarten Lankhorstec193642019-06-28 10:55:17 +02005310skl_compute_wm_levels(const struct intel_crtc_state *crtc_state,
Kumar, Mahesh7e452fd2017-08-17 19:15:23 +05305311 const struct skl_wm_params *wm_params,
Maarten Lankhorstb048a002018-10-18 13:51:30 +02005312 struct skl_wm_level *levels)
Pradeep Bhat2d41c0b2014-11-04 17:06:42 +00005313{
Maarten Lankhorst2225f3c2019-10-31 12:26:03 +01005314 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
Kumar, Maheshd2f5e362017-05-17 17:28:28 +05305315 int level, max_level = ilk_wm_max_level(dev_priv);
Maarten Lankhorstb048a002018-10-18 13:51:30 +02005316 struct skl_wm_level *result_prev = &levels[0];
Lyudea62163e2016-10-04 14:28:20 -04005317
Kumar, Maheshd2f5e362017-05-17 17:28:28 +05305318 for (level = 0; level <= max_level; level++) {
Maarten Lankhorstb048a002018-10-18 13:51:30 +02005319 struct skl_wm_level *result = &levels[level];
Stanislav Lisovskiy7b994752020-04-09 18:47:18 +03005320 unsigned int latency = dev_priv->wm.skl_latency[level];
Kumar, Maheshd2f5e362017-05-17 17:28:28 +05305321
Stanislav Lisovskiy7b994752020-04-09 18:47:18 +03005322 skl_compute_plane_wm(crtc_state, level, latency,
5323 wm_params, result_prev, result);
Maarten Lankhorstb048a002018-10-18 13:51:30 +02005324
5325 result_prev = result;
Kumar, Maheshd2f5e362017-05-17 17:28:28 +05305326 }
Pradeep Bhat2d41c0b2014-11-04 17:06:42 +00005327}
5328
Stanislav Lisovskiy7241c572020-05-14 10:48:51 +03005329static void tgl_compute_sagv_wm(const struct intel_crtc_state *crtc_state,
5330 const struct skl_wm_params *wm_params,
5331 struct skl_plane_wm *plane_wm)
5332{
5333 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
Ville Syrjäläa68aa482021-02-26 17:32:01 +02005334 struct skl_wm_level *sagv_wm = &plane_wm->sagv.wm0;
Stanislav Lisovskiy7241c572020-05-14 10:48:51 +03005335 struct skl_wm_level *levels = plane_wm->wm;
5336 unsigned int latency = dev_priv->wm.skl_latency[0] + dev_priv->sagv_block_time_us;
5337
5338 skl_compute_plane_wm(crtc_state, 0, latency,
5339 wm_params, &levels[0],
5340 sagv_wm);
5341}
5342
Ville Syrjälä5f25e6a2021-02-26 17:32:02 +02005343static void skl_compute_transition_wm(struct drm_i915_private *dev_priv,
5344 struct skl_wm_level *trans_wm,
5345 const struct skl_wm_level *wm0,
5346 const struct skl_wm_params *wp)
Damien Lespiau407b50f2014-11-04 17:06:57 +00005347{
Ville Syrjäläc834d032020-02-28 22:35:52 +02005348 u16 trans_min, trans_amount, trans_y_tile_min;
Jani Nikula5ce9a6492019-01-18 14:01:20 +02005349 u16 wm0_sel_res_b, trans_offset_b, res_blocks;
Damien Lespiau9414f562014-11-04 17:06:58 +00005350
Kumar, Maheshca476672017-08-17 19:15:24 +05305351 /* Transition WM don't make any sense if ipc is disabled */
5352 if (!dev_priv->ipc_enabled)
Ville Syrjälä14a43062018-11-14 23:07:22 +02005353 return;
Kumar, Maheshca476672017-08-17 19:15:24 +05305354
Ville Syrjäläa7f1e8e2020-02-28 22:35:51 +02005355 /*
5356 * WaDisableTWM:skl,kbl,cfl,bxt
5357 * Transition WM are not recommended by HW team for GEN9
5358 */
5359 if (IS_GEN9_BC(dev_priv) || IS_BROXTON(dev_priv))
5360 return;
5361
Paulo Zanoni91961a82018-10-04 16:15:56 -07005362 if (INTEL_GEN(dev_priv) >= 11)
Kumar, Maheshca476672017-08-17 19:15:24 +05305363 trans_min = 4;
Ville Syrjäläc834d032020-02-28 22:35:52 +02005364 else
5365 trans_min = 14;
5366
5367 /* Display WA #1140: glk,cnl */
5368 if (IS_CANNONLAKE(dev_priv) || IS_GEMINILAKE(dev_priv))
5369 trans_amount = 0;
5370 else
5371 trans_amount = 10; /* This is configurable amount */
Kumar, Maheshca476672017-08-17 19:15:24 +05305372
5373 trans_offset_b = trans_min + trans_amount;
5374
Paulo Zanonicbacc792018-10-04 16:15:58 -07005375 /*
5376 * The spec asks for Selected Result Blocks for wm0 (the real value),
5377 * not Result Blocks (the integer value). Pay attention to the capital
5378 * letters. The value wm_l0->plane_res_b is actually Result Blocks, but
5379 * since Result Blocks is the ceiling of Selected Result Blocks plus 1,
5380 * and since we later will have to get the ceiling of the sum in the
5381 * transition watermarks calculation, we can just pretend Selected
5382 * Result Blocks is Result Blocks minus 1 and it should work for the
5383 * current platforms.
5384 */
Ville Syrjälä5f25e6a2021-02-26 17:32:02 +02005385 wm0_sel_res_b = wm0->plane_res_b - 1;
Paulo Zanonicbacc792018-10-04 16:15:58 -07005386
Kumar, Maheshca476672017-08-17 19:15:24 +05305387 if (wp->y_tiled) {
Jani Nikula5ce9a6492019-01-18 14:01:20 +02005388 trans_y_tile_min =
5389 (u16)mul_round_up_u32_fixed16(2, wp->y_tile_minimum);
Paulo Zanonicbacc792018-10-04 16:15:58 -07005390 res_blocks = max(wm0_sel_res_b, trans_y_tile_min) +
Kumar, Maheshca476672017-08-17 19:15:24 +05305391 trans_offset_b;
5392 } else {
Paulo Zanonicbacc792018-10-04 16:15:58 -07005393 res_blocks = wm0_sel_res_b + trans_offset_b;
Kumar, Maheshca476672017-08-17 19:15:24 +05305394 }
5395
Matt Roperd8e87492018-12-11 09:31:07 -08005396 /*
5397 * Just assume we can enable the transition watermark. After
5398 * computing the DDB we'll come back and disable it if that
5399 * assumption turns out to be false.
5400 */
Ville Syrjälä5f25e6a2021-02-26 17:32:02 +02005401 trans_wm->plane_res_b = res_blocks + 1;
5402 trans_wm->plane_en = true;
Damien Lespiau407b50f2014-11-04 17:06:57 +00005403}
5404
Ville Syrjäläff43bc32018-11-27 18:59:00 +02005405static int skl_build_plane_wm_single(struct intel_crtc_state *crtc_state,
Ville Syrjälä83158472018-11-27 18:57:26 +02005406 const struct intel_plane_state *plane_state,
5407 enum plane_id plane_id, int color_plane)
Maarten Lankhorstb048a002018-10-18 13:51:30 +02005408{
Stanislav Lisovskiy7241c572020-05-14 10:48:51 +03005409 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5410 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
Ville Syrjälädbf71382020-11-06 19:30:38 +02005411 struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
Maarten Lankhorstb048a002018-10-18 13:51:30 +02005412 struct skl_wm_params wm_params;
Maarten Lankhorstb048a002018-10-18 13:51:30 +02005413 int ret;
5414
Ville Syrjälä51de9c62018-11-14 23:07:25 +02005415 ret = skl_compute_plane_wm_params(crtc_state, plane_state,
Ville Syrjälä83158472018-11-27 18:57:26 +02005416 &wm_params, color_plane);
Maarten Lankhorstb048a002018-10-18 13:51:30 +02005417 if (ret)
5418 return ret;
5419
Ville Syrjälä67155a62019-03-12 22:58:37 +02005420 skl_compute_wm_levels(crtc_state, &wm_params, wm->wm);
Stanislav Lisovskiy7241c572020-05-14 10:48:51 +03005421
Ville Syrjälä5f25e6a2021-02-26 17:32:02 +02005422 skl_compute_transition_wm(dev_priv, &wm->trans_wm,
5423 &wm->wm[0], &wm_params);
5424
5425 if (INTEL_GEN(dev_priv) >= 12) {
Stanislav Lisovskiy7241c572020-05-14 10:48:51 +03005426 tgl_compute_sagv_wm(crtc_state, &wm_params, wm);
5427
Ville Syrjälä5f25e6a2021-02-26 17:32:02 +02005428 skl_compute_transition_wm(dev_priv, &wm->sagv.trans_wm,
5429 &wm->sagv.wm0, &wm_params);
5430 }
Ville Syrjälä83158472018-11-27 18:57:26 +02005431
5432 return 0;
Maarten Lankhorstb048a002018-10-18 13:51:30 +02005433}
5434
Ville Syrjäläff43bc32018-11-27 18:59:00 +02005435static int skl_build_plane_wm_uv(struct intel_crtc_state *crtc_state,
Ville Syrjälä83158472018-11-27 18:57:26 +02005436 const struct intel_plane_state *plane_state,
5437 enum plane_id plane_id)
Maarten Lankhorstb048a002018-10-18 13:51:30 +02005438{
Ville Syrjälädbf71382020-11-06 19:30:38 +02005439 struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
Ville Syrjälä83158472018-11-27 18:57:26 +02005440 struct skl_wm_params wm_params;
Maarten Lankhorstb048a002018-10-18 13:51:30 +02005441 int ret;
Maarten Lankhorstb048a002018-10-18 13:51:30 +02005442
Ville Syrjälä83158472018-11-27 18:57:26 +02005443 wm->is_planar = true;
5444
5445 /* uv plane watermarks must also be validated for NV12/Planar */
Ville Syrjälä51de9c62018-11-14 23:07:25 +02005446 ret = skl_compute_plane_wm_params(crtc_state, plane_state,
Ville Syrjälä83158472018-11-27 18:57:26 +02005447 &wm_params, 1);
Maarten Lankhorstb048a002018-10-18 13:51:30 +02005448 if (ret)
5449 return ret;
5450
Ville Syrjälä67155a62019-03-12 22:58:37 +02005451 skl_compute_wm_levels(crtc_state, &wm_params, wm->uv_wm);
Ville Syrjälä83158472018-11-27 18:57:26 +02005452
5453 return 0;
5454}
5455
Ville Syrjälä96cb7cd2019-03-12 22:58:43 +02005456static int skl_build_plane_wm(struct intel_crtc_state *crtc_state,
Ville Syrjälä83158472018-11-27 18:57:26 +02005457 const struct intel_plane_state *plane_state)
5458{
Maarten Lankhorstf90a85e2019-10-31 12:26:08 +01005459 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
Ville Syrjälä83158472018-11-27 18:57:26 +02005460 enum plane_id plane_id = plane->id;
Ville Syrjälädbf71382020-11-06 19:30:38 +02005461 struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
5462 const struct drm_framebuffer *fb = plane_state->hw.fb;
Ville Syrjälä83158472018-11-27 18:57:26 +02005463 int ret;
5464
Ville Syrjälädbf71382020-11-06 19:30:38 +02005465 memset(wm, 0, sizeof(*wm));
5466
Ville Syrjälä83158472018-11-27 18:57:26 +02005467 if (!intel_wm_plane_visible(crtc_state, plane_state))
5468 return 0;
5469
Ville Syrjäläff43bc32018-11-27 18:59:00 +02005470 ret = skl_build_plane_wm_single(crtc_state, plane_state,
Ville Syrjälä83158472018-11-27 18:57:26 +02005471 plane_id, 0);
5472 if (ret)
5473 return ret;
5474
5475 if (fb->format->is_yuv && fb->format->num_planes > 1) {
Ville Syrjäläff43bc32018-11-27 18:59:00 +02005476 ret = skl_build_plane_wm_uv(crtc_state, plane_state,
Ville Syrjälä83158472018-11-27 18:57:26 +02005477 plane_id);
5478 if (ret)
5479 return ret;
5480 }
5481
5482 return 0;
5483}
5484
Ville Syrjälä96cb7cd2019-03-12 22:58:43 +02005485static int icl_build_plane_wm(struct intel_crtc_state *crtc_state,
Ville Syrjälä83158472018-11-27 18:57:26 +02005486 const struct intel_plane_state *plane_state)
5487{
Ville Syrjälädbf71382020-11-06 19:30:38 +02005488 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
5489 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
5490 enum plane_id plane_id = plane->id;
5491 struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
Ville Syrjälä83158472018-11-27 18:57:26 +02005492 int ret;
5493
Ville Syrjälädbf71382020-11-06 19:30:38 +02005494 memset(wm, 0, sizeof(*wm));
5495
Ville Syrjälä83158472018-11-27 18:57:26 +02005496 /* Watermarks calculated in master */
Maarten Lankhorstc47b7dd2019-09-20 13:42:20 +02005497 if (plane_state->planar_slave)
Ville Syrjälä83158472018-11-27 18:57:26 +02005498 return 0;
5499
Maarten Lankhorstc47b7dd2019-09-20 13:42:20 +02005500 if (plane_state->planar_linked_plane) {
Maarten Lankhorst7b3cb172019-10-31 12:26:07 +01005501 const struct drm_framebuffer *fb = plane_state->hw.fb;
Maarten Lankhorstc47b7dd2019-09-20 13:42:20 +02005502 enum plane_id y_plane_id = plane_state->planar_linked_plane->id;
Ville Syrjälä83158472018-11-27 18:57:26 +02005503
Pankaj Bharadiya19edeb382020-05-04 23:45:59 +05305504 drm_WARN_ON(&dev_priv->drm,
5505 !intel_wm_plane_visible(crtc_state, plane_state));
5506 drm_WARN_ON(&dev_priv->drm, !fb->format->is_yuv ||
5507 fb->format->num_planes == 1);
Ville Syrjälä83158472018-11-27 18:57:26 +02005508
Ville Syrjäläff43bc32018-11-27 18:59:00 +02005509 ret = skl_build_plane_wm_single(crtc_state, plane_state,
Ville Syrjälä83158472018-11-27 18:57:26 +02005510 y_plane_id, 0);
5511 if (ret)
5512 return ret;
5513
Ville Syrjäläff43bc32018-11-27 18:59:00 +02005514 ret = skl_build_plane_wm_single(crtc_state, plane_state,
Ville Syrjälä83158472018-11-27 18:57:26 +02005515 plane_id, 1);
5516 if (ret)
5517 return ret;
5518 } else if (intel_wm_plane_visible(crtc_state, plane_state)) {
Ville Syrjäläff43bc32018-11-27 18:59:00 +02005519 ret = skl_build_plane_wm_single(crtc_state, plane_state,
Ville Syrjälä83158472018-11-27 18:57:26 +02005520 plane_id, 0);
5521 if (ret)
5522 return ret;
5523 }
5524
5525 return 0;
Maarten Lankhorstb048a002018-10-18 13:51:30 +02005526}
5527
Ville Syrjäläffc90032020-11-06 19:30:37 +02005528static int skl_build_pipe_wm(struct intel_atomic_state *state,
5529 struct intel_crtc *crtc)
Pradeep Bhat2d41c0b2014-11-04 17:06:42 +00005530{
Ville Syrjäläffc90032020-11-06 19:30:37 +02005531 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5532 struct intel_crtc_state *crtc_state =
5533 intel_atomic_get_new_crtc_state(state, crtc);
Maarten Lankhorstaf9fbfa2019-10-04 13:34:53 +02005534 const struct intel_plane_state *plane_state;
Ville Syrjälädbf71382020-11-06 19:30:38 +02005535 struct intel_plane *plane;
5536 int ret, i;
Pradeep Bhat2d41c0b2014-11-04 17:06:42 +00005537
Ville Syrjälädbf71382020-11-06 19:30:38 +02005538 for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
5539 /*
5540 * FIXME should perhaps check {old,new}_plane_crtc->hw.crtc
5541 * instead but we don't populate that correctly for NV12 Y
5542 * planes so for now hack this.
5543 */
5544 if (plane->pipe != crtc->pipe)
5545 continue;
Kumar, Mahesheb2fdcd2017-05-17 17:28:27 +05305546
Ville Syrjälä83158472018-11-27 18:57:26 +02005547 if (INTEL_GEN(dev_priv) >= 11)
Maarten Lankhorstec193642019-06-28 10:55:17 +02005548 ret = icl_build_plane_wm(crtc_state, plane_state);
Maarten Lankhorstb048a002018-10-18 13:51:30 +02005549 else
Maarten Lankhorstec193642019-06-28 10:55:17 +02005550 ret = skl_build_plane_wm(crtc_state, plane_state);
Kumar, Mahesh7e452fd2017-08-17 19:15:23 +05305551 if (ret)
5552 return ret;
Pradeep Bhat2d41c0b2014-11-04 17:06:42 +00005553 }
Mahesh Kumar942aa2d2018-04-09 09:11:04 +05305554
Ville Syrjälädbf71382020-11-06 19:30:38 +02005555 crtc_state->wm.skl.optimal = crtc_state->wm.skl.raw;
5556
Matt Roper55994c22016-05-12 07:06:08 -07005557 return 0;
Pradeep Bhat2d41c0b2014-11-04 17:06:42 +00005558}
5559
Ville Syrjäläf0f59a02015-11-18 15:33:26 +02005560static void skl_ddb_entry_write(struct drm_i915_private *dev_priv,
5561 i915_reg_t reg,
Damien Lespiau16160e32014-11-04 17:06:53 +00005562 const struct skl_ddb_entry *entry)
5563{
5564 if (entry->end)
Jani Nikula9b6320a2020-01-23 16:00:04 +02005565 intel_de_write_fw(dev_priv, reg,
5566 (entry->end - 1) << 16 | entry->start);
Damien Lespiau16160e32014-11-04 17:06:53 +00005567 else
Jani Nikula9b6320a2020-01-23 16:00:04 +02005568 intel_de_write_fw(dev_priv, reg, 0);
Damien Lespiau16160e32014-11-04 17:06:53 +00005569}
5570
cpaul@redhat.comd8c0faf2016-10-18 16:09:49 -02005571static void skl_write_wm_level(struct drm_i915_private *dev_priv,
5572 i915_reg_t reg,
5573 const struct skl_wm_level *level)
5574{
Jani Nikula5ce9a6492019-01-18 14:01:20 +02005575 u32 val = 0;
cpaul@redhat.comd8c0faf2016-10-18 16:09:49 -02005576
Ville Syrjälä2ed8e1f2019-02-13 18:54:23 +02005577 if (level->plane_en)
cpaul@redhat.comd8c0faf2016-10-18 16:09:49 -02005578 val |= PLANE_WM_EN;
Ville Syrjälä2ed8e1f2019-02-13 18:54:23 +02005579 if (level->ignore_lines)
5580 val |= PLANE_WM_IGNORE_LINES;
5581 val |= level->plane_res_b;
5582 val |= level->plane_res_l << PLANE_WM_LINES_SHIFT;
cpaul@redhat.comd8c0faf2016-10-18 16:09:49 -02005583
Jani Nikula9b6320a2020-01-23 16:00:04 +02005584 intel_de_write_fw(dev_priv, reg, val);
cpaul@redhat.comd8c0faf2016-10-18 16:09:49 -02005585}
5586
Ville Syrjäläff43bc32018-11-27 18:59:00 +02005587void skl_write_plane_wm(struct intel_plane *plane,
5588 const struct intel_crtc_state *crtc_state)
Lyude62e0fb82016-08-22 12:50:08 -04005589{
Ville Syrjäläff43bc32018-11-27 18:59:00 +02005590 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
Tvrtko Ursulin5db94012016-10-13 11:03:10 +01005591 int level, max_level = ilk_wm_max_level(dev_priv);
Ville Syrjäläff43bc32018-11-27 18:59:00 +02005592 enum plane_id plane_id = plane->id;
5593 enum pipe pipe = plane->pipe;
Ville Syrjälä2871b2f2021-02-26 17:31:58 +02005594 const struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
5595 const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
Ville Syrjäläff43bc32018-11-27 18:59:00 +02005596 const struct skl_ddb_entry *ddb_y =
5597 &crtc_state->wm.skl.plane_ddb_y[plane_id];
5598 const struct skl_ddb_entry *ddb_uv =
5599 &crtc_state->wm.skl.plane_ddb_uv[plane_id];
Lyude62e0fb82016-08-22 12:50:08 -04005600
Ville Syrjälä2871b2f2021-02-26 17:31:58 +02005601 for (level = 0; level <= max_level; level++)
Ville Syrjäläd5cdfdf52016-11-22 18:01:58 +02005602 skl_write_wm_level(dev_priv, PLANE_WM(pipe, plane_id, level),
Ville Syrjälä2871b2f2021-02-26 17:31:58 +02005603 skl_plane_wm_level(pipe_wm, plane_id, level));
5604
Ville Syrjäläd5cdfdf52016-11-22 18:01:58 +02005605 skl_write_wm_level(dev_priv, PLANE_WM_TRANS(pipe, plane_id),
Ville Syrjälä5f25e6a2021-02-26 17:32:02 +02005606 skl_plane_trans_wm(pipe_wm, plane_id));
Lyude27082492016-08-24 07:48:10 +02005607
Ville Syrjäläff43bc32018-11-27 18:59:00 +02005608 if (INTEL_GEN(dev_priv) >= 11) {
Mahesh Kumar234059d2018-01-30 11:49:13 -02005609 skl_ddb_entry_write(dev_priv,
Ville Syrjäläff43bc32018-11-27 18:59:00 +02005610 PLANE_BUF_CFG(pipe, plane_id), ddb_y);
5611 return;
Mahesh Kumarb879d582018-04-09 09:11:01 +05305612 }
Ville Syrjäläff43bc32018-11-27 18:59:00 +02005613
5614 if (wm->is_planar)
5615 swap(ddb_y, ddb_uv);
5616
5617 skl_ddb_entry_write(dev_priv,
5618 PLANE_BUF_CFG(pipe, plane_id), ddb_y);
5619 skl_ddb_entry_write(dev_priv,
5620 PLANE_NV12_BUF_CFG(pipe, plane_id), ddb_uv);
Lyude62e0fb82016-08-22 12:50:08 -04005621}
5622
Ville Syrjäläff43bc32018-11-27 18:59:00 +02005623void skl_write_cursor_wm(struct intel_plane *plane,
5624 const struct intel_crtc_state *crtc_state)
Lyude62e0fb82016-08-22 12:50:08 -04005625{
Ville Syrjäläff43bc32018-11-27 18:59:00 +02005626 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
Tvrtko Ursulin5db94012016-10-13 11:03:10 +01005627 int level, max_level = ilk_wm_max_level(dev_priv);
Ville Syrjäläff43bc32018-11-27 18:59:00 +02005628 enum plane_id plane_id = plane->id;
5629 enum pipe pipe = plane->pipe;
Ville Syrjälä2871b2f2021-02-26 17:31:58 +02005630 const struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
Ville Syrjäläff43bc32018-11-27 18:59:00 +02005631 const struct skl_ddb_entry *ddb =
5632 &crtc_state->wm.skl.plane_ddb_y[plane_id];
Lyude62e0fb82016-08-22 12:50:08 -04005633
Ville Syrjälä2871b2f2021-02-26 17:31:58 +02005634 for (level = 0; level <= max_level; level++)
cpaul@redhat.comd8c0faf2016-10-18 16:09:49 -02005635 skl_write_wm_level(dev_priv, CUR_WM(pipe, level),
Ville Syrjälä2871b2f2021-02-26 17:31:58 +02005636 skl_plane_wm_level(pipe_wm, plane_id, level));
5637
Ville Syrjälä5f25e6a2021-02-26 17:32:02 +02005638 skl_write_wm_level(dev_priv, CUR_WM_TRANS(pipe),
5639 skl_plane_trans_wm(pipe_wm, plane_id));
Lyude27082492016-08-24 07:48:10 +02005640
Ville Syrjäläff43bc32018-11-27 18:59:00 +02005641 skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe), ddb);
Lyude62e0fb82016-08-22 12:50:08 -04005642}
5643
cpaul@redhat.com45ece232016-10-14 17:31:56 -04005644bool skl_wm_level_equals(const struct skl_wm_level *l1,
5645 const struct skl_wm_level *l2)
5646{
Ville Syrjäläff43bc32018-11-27 18:59:00 +02005647 return l1->plane_en == l2->plane_en &&
Ville Syrjälä2ed8e1f2019-02-13 18:54:23 +02005648 l1->ignore_lines == l2->ignore_lines &&
Ville Syrjäläff43bc32018-11-27 18:59:00 +02005649 l1->plane_res_l == l2->plane_res_l &&
5650 l1->plane_res_b == l2->plane_res_b;
5651}
cpaul@redhat.com45ece232016-10-14 17:31:56 -04005652
Ville Syrjäläff43bc32018-11-27 18:59:00 +02005653static bool skl_plane_wm_equals(struct drm_i915_private *dev_priv,
5654 const struct skl_plane_wm *wm1,
5655 const struct skl_plane_wm *wm2)
5656{
5657 int level, max_level = ilk_wm_max_level(dev_priv);
cpaul@redhat.com45ece232016-10-14 17:31:56 -04005658
Ville Syrjäläff43bc32018-11-27 18:59:00 +02005659 for (level = 0; level <= max_level; level++) {
Ville Syrjäläe7f54e62020-02-28 22:35:49 +02005660 /*
5661 * We don't check uv_wm as the hardware doesn't actually
5662 * use it. It only gets used for calculating the required
5663 * ddb allocation.
5664 */
5665 if (!skl_wm_level_equals(&wm1->wm[level], &wm2->wm[level]))
Ville Syrjäläff43bc32018-11-27 18:59:00 +02005666 return false;
5667 }
5668
Ville Syrjäläf11449d2021-02-26 17:32:00 +02005669 return skl_wm_level_equals(&wm1->trans_wm, &wm2->trans_wm) &&
Ville Syrjälä5f25e6a2021-02-26 17:32:02 +02005670 skl_wm_level_equals(&wm1->sagv.wm0, &wm2->sagv.wm0) &&
5671 skl_wm_level_equals(&wm1->sagv.trans_wm, &wm2->sagv.trans_wm);
cpaul@redhat.com45ece232016-10-14 17:31:56 -04005672}
5673
Jani Nikula81b55ef2020-04-20 17:04:38 +03005674static bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a,
5675 const struct skl_ddb_entry *b)
Damien Lespiau0e8fb7b2014-11-04 17:07:02 +00005676{
Lyude27082492016-08-24 07:48:10 +02005677 return a->start < b->end && b->start < a->end;
Damien Lespiau0e8fb7b2014-11-04 17:07:02 +00005678}
5679
Ville Syrjälä33c9c502021-01-22 22:56:33 +02005680static void skl_ddb_entry_union(struct skl_ddb_entry *a,
5681 const struct skl_ddb_entry *b)
5682{
5683 if (a->end && b->end) {
5684 a->start = min(a->start, b->start);
5685 a->end = max(a->end, b->end);
5686 } else if (b->end) {
5687 a->start = b->start;
5688 a->end = b->end;
5689 }
5690}
5691
Ville Syrjälä53cc68802018-11-01 17:05:59 +02005692bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry *ddb,
Jani Nikula696173b2019-04-05 14:00:15 +03005693 const struct skl_ddb_entry *entries,
Ville Syrjälä53cc68802018-11-01 17:05:59 +02005694 int num_entries, int ignore_idx)
Damien Lespiau0e8fb7b2014-11-04 17:07:02 +00005695{
Ville Syrjälä53cc68802018-11-01 17:05:59 +02005696 int i;
Damien Lespiau0e8fb7b2014-11-04 17:07:02 +00005697
Ville Syrjälä53cc68802018-11-01 17:05:59 +02005698 for (i = 0; i < num_entries; i++) {
5699 if (i != ignore_idx &&
5700 skl_ddb_entries_overlap(ddb, &entries[i]))
Lyude27082492016-08-24 07:48:10 +02005701 return true;
Mika Kahola2b685042017-10-10 13:17:03 +03005702 }
Damien Lespiau0e8fb7b2014-11-04 17:07:02 +00005703
Lyude27082492016-08-24 07:48:10 +02005704 return false;
Damien Lespiau0e8fb7b2014-11-04 17:07:02 +00005705}
5706
Jani Nikulabb7791b2016-10-04 12:29:17 +03005707static int
Ville Syrjäläff43bc32018-11-27 18:59:00 +02005708skl_ddb_add_affected_planes(const struct intel_crtc_state *old_crtc_state,
5709 struct intel_crtc_state *new_crtc_state)
Rodrigo Vivi9a30a262017-06-13 10:52:30 -07005710{
Maarten Lankhorst2225f3c2019-10-31 12:26:03 +01005711 struct intel_atomic_state *state = to_intel_atomic_state(new_crtc_state->uapi.state);
5712 struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
Ville Syrjäläff43bc32018-11-27 18:59:00 +02005713 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5714 struct intel_plane *plane;
Rodrigo Vivi9a30a262017-06-13 10:52:30 -07005715
Ville Syrjäläff43bc32018-11-27 18:59:00 +02005716 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
5717 struct intel_plane_state *plane_state;
5718 enum plane_id plane_id = plane->id;
Rodrigo Vivi9a30a262017-06-13 10:52:30 -07005719
Ville Syrjäläff43bc32018-11-27 18:59:00 +02005720 if (skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_y[plane_id],
5721 &new_crtc_state->wm.skl.plane_ddb_y[plane_id]) &&
5722 skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_uv[plane_id],
5723 &new_crtc_state->wm.skl.plane_ddb_uv[plane_id]))
Rodrigo Vivi9a30a262017-06-13 10:52:30 -07005724 continue;
5725
Ville Syrjäläff43bc32018-11-27 18:59:00 +02005726 plane_state = intel_atomic_get_plane_state(state, plane);
Rodrigo Vivi9a30a262017-06-13 10:52:30 -07005727 if (IS_ERR(plane_state))
5728 return PTR_ERR(plane_state);
Maarten Lankhorst1ab554b2018-10-22 15:51:52 +02005729
Ville Syrjäläff43bc32018-11-27 18:59:00 +02005730 new_crtc_state->update_planes |= BIT(plane_id);
Rodrigo Vivi9a30a262017-06-13 10:52:30 -07005731 }
5732
5733 return 0;
5734}
5735
Ville Syrjäläef79d622021-01-22 22:56:32 +02005736static u8 intel_dbuf_enabled_slices(const struct intel_dbuf_state *dbuf_state)
5737{
5738 struct drm_i915_private *dev_priv = to_i915(dbuf_state->base.state->base.dev);
5739 u8 enabled_slices;
5740 enum pipe pipe;
5741
5742 /*
5743 * FIXME: For now we always enable slice S1 as per
5744 * the Bspec display initialization sequence.
5745 */
5746 enabled_slices = BIT(DBUF_S1);
5747
5748 for_each_pipe(dev_priv, pipe)
5749 enabled_slices |= dbuf_state->slices[pipe];
5750
5751 return enabled_slices;
5752}
5753
Rodrigo Vivi9a30a262017-06-13 10:52:30 -07005754static int
Matt Ropercd1d3ee2018-12-10 13:54:14 -08005755skl_compute_ddb(struct intel_atomic_state *state)
Matt Roper98d39492016-05-12 07:06:03 -07005756{
Ville Syrjälä70b1a262020-02-25 19:11:16 +02005757 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5758 const struct intel_dbuf_state *old_dbuf_state;
Ville Syrjäläef79d622021-01-22 22:56:32 +02005759 struct intel_dbuf_state *new_dbuf_state = NULL;
Ville Syrjälä70b1a262020-02-25 19:11:16 +02005760 const struct intel_crtc_state *old_crtc_state;
Ville Syrjäläff43bc32018-11-27 18:59:00 +02005761 struct intel_crtc_state *new_crtc_state;
Mahesh Kumare1f96a62018-04-09 09:11:08 +05305762 struct intel_crtc *crtc;
Mahesh Kumare1f96a62018-04-09 09:11:08 +05305763 int ret, i;
Matt Roper98d39492016-05-12 07:06:03 -07005764
Ville Syrjäläef79d622021-01-22 22:56:32 +02005765 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
5766 new_dbuf_state = intel_atomic_get_dbuf_state(state);
5767 if (IS_ERR(new_dbuf_state))
5768 return PTR_ERR(new_dbuf_state);
5769
5770 old_dbuf_state = intel_atomic_get_old_dbuf_state(state);
5771 break;
5772 }
5773
5774 if (!new_dbuf_state)
5775 return 0;
5776
5777 new_dbuf_state->active_pipes =
5778 intel_calc_active_pipes(state, old_dbuf_state->active_pipes);
5779
5780 if (old_dbuf_state->active_pipes != new_dbuf_state->active_pipes) {
5781 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
5782 if (ret)
5783 return ret;
5784 }
5785
5786 for_each_intel_crtc(&dev_priv->drm, crtc) {
5787 enum pipe pipe = crtc->pipe;
5788
5789 new_dbuf_state->slices[pipe] =
5790 skl_compute_dbuf_slices(crtc, new_dbuf_state->active_pipes);
5791
5792 if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe])
5793 continue;
5794
5795 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
5796 if (ret)
5797 return ret;
5798 }
5799
5800 new_dbuf_state->enabled_slices = intel_dbuf_enabled_slices(new_dbuf_state);
5801
5802 if (old_dbuf_state->enabled_slices != new_dbuf_state->enabled_slices) {
5803 ret = intel_atomic_serialize_global_state(&new_dbuf_state->base);
5804 if (ret)
5805 return ret;
5806
5807 drm_dbg_kms(&dev_priv->drm,
5808 "Enabled dbuf slices 0x%x -> 0x%x (out of %d dbuf slices)\n",
5809 old_dbuf_state->enabled_slices,
5810 new_dbuf_state->enabled_slices,
5811 INTEL_INFO(dev_priv)->num_supported_dbuf_slices);
5812 }
5813
5814 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
5815 enum pipe pipe = crtc->pipe;
5816
5817 new_dbuf_state->weight[pipe] = intel_crtc_ddb_weight(new_crtc_state);
5818
5819 if (old_dbuf_state->weight[pipe] == new_dbuf_state->weight[pipe])
5820 continue;
5821
5822 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
5823 if (ret)
5824 return ret;
5825 }
5826
5827 for_each_intel_crtc(&dev_priv->drm, crtc) {
5828 ret = skl_crtc_allocate_ddb(state, crtc);
5829 if (ret)
5830 return ret;
5831 }
5832
Matt Ropercd1d3ee2018-12-10 13:54:14 -08005833 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
Ville Syrjäläff43bc32018-11-27 18:59:00 +02005834 new_crtc_state, i) {
Ville Syrjäläef79d622021-01-22 22:56:32 +02005835 ret = skl_allocate_plane_ddb(state, crtc);
Rodrigo Vivi9a30a262017-06-13 10:52:30 -07005836 if (ret)
5837 return ret;
5838
Ville Syrjäläff43bc32018-11-27 18:59:00 +02005839 ret = skl_ddb_add_affected_planes(old_crtc_state,
5840 new_crtc_state);
Rodrigo Vivi9a30a262017-06-13 10:52:30 -07005841 if (ret)
5842 return ret;
Matt Roper98d39492016-05-12 07:06:03 -07005843 }
5844
5845 return 0;
5846}
5847
Ville Syrjäläab98e942019-02-08 22:05:27 +02005848static char enast(bool enable)
5849{
5850 return enable ? '*' : ' ';
5851}
5852
Matt Roper2722efb2016-08-17 15:55:55 -04005853static void
Ville Syrjäläff43bc32018-11-27 18:59:00 +02005854skl_print_wm_changes(struct intel_atomic_state *state)
cpaul@redhat.com413fc532016-10-14 17:31:54 -04005855{
Ville Syrjäläff43bc32018-11-27 18:59:00 +02005856 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5857 const struct intel_crtc_state *old_crtc_state;
5858 const struct intel_crtc_state *new_crtc_state;
5859 struct intel_plane *plane;
5860 struct intel_crtc *crtc;
Maarten Lankhorst75704982016-11-01 12:04:10 +01005861 int i;
cpaul@redhat.com413fc532016-10-14 17:31:54 -04005862
Jani Nikulabdbf43d2019-10-28 12:38:15 +02005863 if (!drm_debug_enabled(DRM_UT_KMS))
Ville Syrjäläab98e942019-02-08 22:05:27 +02005864 return;
5865
Ville Syrjäläff43bc32018-11-27 18:59:00 +02005866 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
5867 new_crtc_state, i) {
Ville Syrjäläab98e942019-02-08 22:05:27 +02005868 const struct skl_pipe_wm *old_pipe_wm, *new_pipe_wm;
5869
5870 old_pipe_wm = &old_crtc_state->wm.skl.optimal;
5871 new_pipe_wm = &new_crtc_state->wm.skl.optimal;
5872
Ville Syrjäläff43bc32018-11-27 18:59:00 +02005873 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
5874 enum plane_id plane_id = plane->id;
cpaul@redhat.com413fc532016-10-14 17:31:54 -04005875 const struct skl_ddb_entry *old, *new;
5876
Ville Syrjäläff43bc32018-11-27 18:59:00 +02005877 old = &old_crtc_state->wm.skl.plane_ddb_y[plane_id];
5878 new = &new_crtc_state->wm.skl.plane_ddb_y[plane_id];
cpaul@redhat.com413fc532016-10-14 17:31:54 -04005879
cpaul@redhat.com413fc532016-10-14 17:31:54 -04005880 if (skl_ddb_entry_equal(old, new))
5881 continue;
5882
Wambui Karugaf8d18d52020-01-07 18:13:30 +03005883 drm_dbg_kms(&dev_priv->drm,
5884 "[PLANE:%d:%s] ddb (%4d - %4d) -> (%4d - %4d), size %4d -> %4d\n",
5885 plane->base.base.id, plane->base.name,
5886 old->start, old->end, new->start, new->end,
5887 skl_ddb_entry_size(old), skl_ddb_entry_size(new));
Ville Syrjäläab98e942019-02-08 22:05:27 +02005888 }
5889
5890 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
5891 enum plane_id plane_id = plane->id;
5892 const struct skl_plane_wm *old_wm, *new_wm;
5893
5894 old_wm = &old_pipe_wm->planes[plane_id];
5895 new_wm = &new_pipe_wm->planes[plane_id];
5896
5897 if (skl_plane_wm_equals(dev_priv, old_wm, new_wm))
5898 continue;
5899
Wambui Karugaf8d18d52020-01-07 18:13:30 +03005900 drm_dbg_kms(&dev_priv->drm,
Ville Syrjälä5f25e6a2021-02-26 17:32:02 +02005901 "[PLANE:%d:%s] level %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm"
5902 " -> %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm\n",
Wambui Karugaf8d18d52020-01-07 18:13:30 +03005903 plane->base.base.id, plane->base.name,
5904 enast(old_wm->wm[0].plane_en), enast(old_wm->wm[1].plane_en),
5905 enast(old_wm->wm[2].plane_en), enast(old_wm->wm[3].plane_en),
5906 enast(old_wm->wm[4].plane_en), enast(old_wm->wm[5].plane_en),
5907 enast(old_wm->wm[6].plane_en), enast(old_wm->wm[7].plane_en),
5908 enast(old_wm->trans_wm.plane_en),
Ville Syrjäläa68aa482021-02-26 17:32:01 +02005909 enast(old_wm->sagv.wm0.plane_en),
Ville Syrjälä5f25e6a2021-02-26 17:32:02 +02005910 enast(old_wm->sagv.trans_wm.plane_en),
Wambui Karugaf8d18d52020-01-07 18:13:30 +03005911 enast(new_wm->wm[0].plane_en), enast(new_wm->wm[1].plane_en),
5912 enast(new_wm->wm[2].plane_en), enast(new_wm->wm[3].plane_en),
5913 enast(new_wm->wm[4].plane_en), enast(new_wm->wm[5].plane_en),
5914 enast(new_wm->wm[6].plane_en), enast(new_wm->wm[7].plane_en),
Stanislav Lisovskiy7241c572020-05-14 10:48:51 +03005915 enast(new_wm->trans_wm.plane_en),
Ville Syrjälä5f25e6a2021-02-26 17:32:02 +02005916 enast(new_wm->sagv.wm0.plane_en),
5917 enast(new_wm->sagv.trans_wm.plane_en));
Ville Syrjäläab98e942019-02-08 22:05:27 +02005918
Wambui Karugaf8d18d52020-01-07 18:13:30 +03005919 drm_dbg_kms(&dev_priv->drm,
Ville Syrjälä5f25e6a2021-02-26 17:32:02 +02005920 "[PLANE:%d:%s] lines %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d"
5921 " -> %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d\n",
Wambui Karugaf8d18d52020-01-07 18:13:30 +03005922 plane->base.base.id, plane->base.name,
5923 enast(old_wm->wm[0].ignore_lines), old_wm->wm[0].plane_res_l,
5924 enast(old_wm->wm[1].ignore_lines), old_wm->wm[1].plane_res_l,
5925 enast(old_wm->wm[2].ignore_lines), old_wm->wm[2].plane_res_l,
5926 enast(old_wm->wm[3].ignore_lines), old_wm->wm[3].plane_res_l,
5927 enast(old_wm->wm[4].ignore_lines), old_wm->wm[4].plane_res_l,
5928 enast(old_wm->wm[5].ignore_lines), old_wm->wm[5].plane_res_l,
5929 enast(old_wm->wm[6].ignore_lines), old_wm->wm[6].plane_res_l,
5930 enast(old_wm->wm[7].ignore_lines), old_wm->wm[7].plane_res_l,
5931 enast(old_wm->trans_wm.ignore_lines), old_wm->trans_wm.plane_res_l,
Ville Syrjäläa68aa482021-02-26 17:32:01 +02005932 enast(old_wm->sagv.wm0.ignore_lines), old_wm->sagv.wm0.plane_res_l,
Ville Syrjälä5f25e6a2021-02-26 17:32:02 +02005933 enast(old_wm->sagv.trans_wm.ignore_lines), old_wm->sagv.trans_wm.plane_res_l,
Wambui Karugaf8d18d52020-01-07 18:13:30 +03005934 enast(new_wm->wm[0].ignore_lines), new_wm->wm[0].plane_res_l,
5935 enast(new_wm->wm[1].ignore_lines), new_wm->wm[1].plane_res_l,
5936 enast(new_wm->wm[2].ignore_lines), new_wm->wm[2].plane_res_l,
5937 enast(new_wm->wm[3].ignore_lines), new_wm->wm[3].plane_res_l,
5938 enast(new_wm->wm[4].ignore_lines), new_wm->wm[4].plane_res_l,
5939 enast(new_wm->wm[5].ignore_lines), new_wm->wm[5].plane_res_l,
5940 enast(new_wm->wm[6].ignore_lines), new_wm->wm[6].plane_res_l,
5941 enast(new_wm->wm[7].ignore_lines), new_wm->wm[7].plane_res_l,
Stanislav Lisovskiy7241c572020-05-14 10:48:51 +03005942 enast(new_wm->trans_wm.ignore_lines), new_wm->trans_wm.plane_res_l,
Ville Syrjälä5f25e6a2021-02-26 17:32:02 +02005943 enast(new_wm->sagv.wm0.ignore_lines), new_wm->sagv.wm0.plane_res_l,
5944 enast(new_wm->sagv.trans_wm.ignore_lines), new_wm->sagv.trans_wm.plane_res_l);
Ville Syrjäläab98e942019-02-08 22:05:27 +02005945
Wambui Karugaf8d18d52020-01-07 18:13:30 +03005946 drm_dbg_kms(&dev_priv->drm,
Ville Syrjälä5f25e6a2021-02-26 17:32:02 +02005947 "[PLANE:%d:%s] blocks %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d"
5948 " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n",
Wambui Karugaf8d18d52020-01-07 18:13:30 +03005949 plane->base.base.id, plane->base.name,
5950 old_wm->wm[0].plane_res_b, old_wm->wm[1].plane_res_b,
5951 old_wm->wm[2].plane_res_b, old_wm->wm[3].plane_res_b,
5952 old_wm->wm[4].plane_res_b, old_wm->wm[5].plane_res_b,
5953 old_wm->wm[6].plane_res_b, old_wm->wm[7].plane_res_b,
5954 old_wm->trans_wm.plane_res_b,
Ville Syrjäläa68aa482021-02-26 17:32:01 +02005955 old_wm->sagv.wm0.plane_res_b,
Ville Syrjälä5f25e6a2021-02-26 17:32:02 +02005956 old_wm->sagv.trans_wm.plane_res_b,
Wambui Karugaf8d18d52020-01-07 18:13:30 +03005957 new_wm->wm[0].plane_res_b, new_wm->wm[1].plane_res_b,
5958 new_wm->wm[2].plane_res_b, new_wm->wm[3].plane_res_b,
5959 new_wm->wm[4].plane_res_b, new_wm->wm[5].plane_res_b,
5960 new_wm->wm[6].plane_res_b, new_wm->wm[7].plane_res_b,
Stanislav Lisovskiy7241c572020-05-14 10:48:51 +03005961 new_wm->trans_wm.plane_res_b,
Ville Syrjälä5f25e6a2021-02-26 17:32:02 +02005962 new_wm->sagv.wm0.plane_res_b,
5963 new_wm->sagv.trans_wm.plane_res_b);
Ville Syrjäläab98e942019-02-08 22:05:27 +02005964
Wambui Karugaf8d18d52020-01-07 18:13:30 +03005965 drm_dbg_kms(&dev_priv->drm,
Ville Syrjälä5f25e6a2021-02-26 17:32:02 +02005966 "[PLANE:%d:%s] min_ddb %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d"
5967 " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n",
Wambui Karugaf8d18d52020-01-07 18:13:30 +03005968 plane->base.base.id, plane->base.name,
5969 old_wm->wm[0].min_ddb_alloc, old_wm->wm[1].min_ddb_alloc,
5970 old_wm->wm[2].min_ddb_alloc, old_wm->wm[3].min_ddb_alloc,
5971 old_wm->wm[4].min_ddb_alloc, old_wm->wm[5].min_ddb_alloc,
5972 old_wm->wm[6].min_ddb_alloc, old_wm->wm[7].min_ddb_alloc,
5973 old_wm->trans_wm.min_ddb_alloc,
Ville Syrjäläa68aa482021-02-26 17:32:01 +02005974 old_wm->sagv.wm0.min_ddb_alloc,
Ville Syrjälä5f25e6a2021-02-26 17:32:02 +02005975 old_wm->sagv.trans_wm.min_ddb_alloc,
Wambui Karugaf8d18d52020-01-07 18:13:30 +03005976 new_wm->wm[0].min_ddb_alloc, new_wm->wm[1].min_ddb_alloc,
5977 new_wm->wm[2].min_ddb_alloc, new_wm->wm[3].min_ddb_alloc,
5978 new_wm->wm[4].min_ddb_alloc, new_wm->wm[5].min_ddb_alloc,
5979 new_wm->wm[6].min_ddb_alloc, new_wm->wm[7].min_ddb_alloc,
Stanislav Lisovskiy7241c572020-05-14 10:48:51 +03005980 new_wm->trans_wm.min_ddb_alloc,
Ville Syrjälä5f25e6a2021-02-26 17:32:02 +02005981 new_wm->sagv.wm0.min_ddb_alloc,
5982 new_wm->sagv.trans_wm.min_ddb_alloc);
cpaul@redhat.com413fc532016-10-14 17:31:54 -04005983 }
5984 }
5985}
5986
Ville Syrjälä2871b2f2021-02-26 17:31:58 +02005987static bool skl_plane_selected_wm_equals(struct intel_plane *plane,
5988 const struct skl_pipe_wm *old_pipe_wm,
5989 const struct skl_pipe_wm *new_pipe_wm)
5990{
Ville Syrjälä2871b2f2021-02-26 17:31:58 +02005991 struct drm_i915_private *i915 = to_i915(plane->base.dev);
5992 int level, max_level = ilk_wm_max_level(i915);
5993
5994 for (level = 0; level <= max_level; level++) {
5995 /*
5996 * We don't check uv_wm as the hardware doesn't actually
5997 * use it. It only gets used for calculating the required
5998 * ddb allocation.
5999 */
6000 if (!skl_wm_level_equals(skl_plane_wm_level(old_pipe_wm, level, plane->id),
6001 skl_plane_wm_level(new_pipe_wm, level, plane->id)))
6002 return false;
6003 }
6004
Ville Syrjälä5f25e6a2021-02-26 17:32:02 +02006005 return skl_wm_level_equals(skl_plane_trans_wm(old_pipe_wm, plane->id),
6006 skl_plane_trans_wm(new_pipe_wm, plane->id));
Ville Syrjälä2871b2f2021-02-26 17:31:58 +02006007}
6008
Ville Syrjäläff43bc32018-11-27 18:59:00 +02006009/*
6010 * To make sure the cursor watermark registers are always consistent
6011 * with our computed state the following scenario needs special
6012 * treatment:
6013 *
6014 * 1. enable cursor
6015 * 2. move cursor entirely offscreen
6016 * 3. disable cursor
6017 *
6018 * Step 2. does call .disable_plane() but does not zero the watermarks
6019 * (since we consider an offscreen cursor still active for the purposes
6020 * of watermarks). Step 3. would not normally call .disable_plane()
6021 * because the actual plane visibility isn't changing, and we don't
6022 * deallocate the cursor ddb until the pipe gets disabled. So we must
6023 * force step 3. to call .disable_plane() to update the watermark
6024 * registers properly.
6025 *
6026 * Other planes do not suffer from this issues as their watermarks are
6027 * calculated based on the actual plane visibility. The only time this
6028 * can trigger for the other planes is during the initial readout as the
6029 * default value of the watermarks registers is not zero.
6030 */
6031static int skl_wm_add_affected_planes(struct intel_atomic_state *state,
6032 struct intel_crtc *crtc)
6033{
6034 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6035 const struct intel_crtc_state *old_crtc_state =
6036 intel_atomic_get_old_crtc_state(state, crtc);
6037 struct intel_crtc_state *new_crtc_state =
6038 intel_atomic_get_new_crtc_state(state, crtc);
6039 struct intel_plane *plane;
6040
6041 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
6042 struct intel_plane_state *plane_state;
6043 enum plane_id plane_id = plane->id;
6044
6045 /*
6046 * Force a full wm update for every plane on modeset.
6047 * Required because the reset value of the wm registers
6048 * is non-zero, whereas we want all disabled planes to
6049 * have zero watermarks. So if we turn off the relevant
6050 * power well the hardware state will go out of sync
6051 * with the software state.
6052 */
Maarten Lankhorst2225f3c2019-10-31 12:26:03 +01006053 if (!drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi) &&
Ville Syrjälä2871b2f2021-02-26 17:31:58 +02006054 skl_plane_selected_wm_equals(plane,
6055 &old_crtc_state->wm.skl.optimal,
6056 &new_crtc_state->wm.skl.optimal))
Ville Syrjäläff43bc32018-11-27 18:59:00 +02006057 continue;
6058
6059 plane_state = intel_atomic_get_plane_state(state, plane);
6060 if (IS_ERR(plane_state))
6061 return PTR_ERR(plane_state);
6062
6063 new_crtc_state->update_planes |= BIT(plane_id);
6064 }
6065
6066 return 0;
6067}
6068
Mahesh Kumare1f96a62018-04-09 09:11:08 +05306069static int
Matt Ropercd1d3ee2018-12-10 13:54:14 -08006070skl_compute_wm(struct intel_atomic_state *state)
Mahesh Kumare1f96a62018-04-09 09:11:08 +05306071{
Matt Ropercd1d3ee2018-12-10 13:54:14 -08006072 struct intel_crtc *crtc;
Ville Syrjälä8cac9fd2019-03-12 22:58:44 +02006073 struct intel_crtc_state *new_crtc_state;
Mahesh Kumare1f96a62018-04-09 09:11:08 +05306074 int ret, i;
6075
Ville Syrjäläffc90032020-11-06 19:30:37 +02006076 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
6077 ret = skl_build_pipe_wm(state, crtc);
Ville Syrjäläff43bc32018-11-27 18:59:00 +02006078 if (ret)
6079 return ret;
Matt Roper734fa012016-05-12 15:11:40 -07006080 }
6081
Matt Roperd8e87492018-12-11 09:31:07 -08006082 ret = skl_compute_ddb(state);
6083 if (ret)
6084 return ret;
6085
Stanislav Lisovskiyecab0f32020-04-30 22:56:34 +03006086 ret = intel_compute_sagv_mask(state);
6087 if (ret)
6088 return ret;
Stanislav Lisovskiy97288892020-04-30 22:17:57 +03006089
Ville Syrjälä23baedd2020-02-28 22:35:50 +02006090 /*
6091 * skl_compute_ddb() will have adjusted the final watermarks
6092 * based on how much ddb is available. Now we can actually
6093 * check if the final watermarks changed.
6094 */
Ville Syrjäläffc90032020-11-06 19:30:37 +02006095 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
Ville Syrjälä23baedd2020-02-28 22:35:50 +02006096 ret = skl_wm_add_affected_planes(state, crtc);
6097 if (ret)
6098 return ret;
6099 }
6100
Matt Ropercd1d3ee2018-12-10 13:54:14 -08006101 skl_print_wm_changes(state);
cpaul@redhat.com413fc532016-10-14 17:31:54 -04006102
Matt Roper98d39492016-05-12 07:06:03 -07006103 return 0;
6104}
6105
Matt Ropercd1d3ee2018-12-10 13:54:14 -08006106static void ilk_compute_wm_config(struct drm_i915_private *dev_priv,
Ville Syrjäläd8905652016-01-14 14:53:35 +02006107 struct intel_wm_config *config)
6108{
6109 struct intel_crtc *crtc;
6110
6111 /* Compute the currently _active_ config */
Matt Ropercd1d3ee2018-12-10 13:54:14 -08006112 for_each_intel_crtc(&dev_priv->drm, crtc) {
Ville Syrjäläd8905652016-01-14 14:53:35 +02006113 const struct intel_pipe_wm *wm = &crtc->wm.active.ilk;
6114
6115 if (!wm->pipe_enabled)
6116 continue;
6117
6118 config->sprites_enabled |= wm->sprites_enabled;
6119 config->sprites_scaled |= wm->sprites_scaled;
6120 config->num_pipes_active++;
6121 }
6122}
6123
Matt Ropered4a6a72016-02-23 17:20:13 -08006124static void ilk_program_watermarks(struct drm_i915_private *dev_priv)
Paulo Zanoni801bcff2013-05-31 10:08:35 -03006125{
Ville Syrjäläb9d5c832015-09-24 15:53:14 -07006126 struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
Imre Deak820c1982013-12-17 14:46:36 +02006127 struct ilk_wm_maximums max;
Ville Syrjäläd8905652016-01-14 14:53:35 +02006128 struct intel_wm_config config = {};
Imre Deak820c1982013-12-17 14:46:36 +02006129 struct ilk_wm_values results = {};
Ville Syrjälä77c122b2013-08-06 22:24:04 +03006130 enum intel_ddb_partitioning partitioning;
Matt Roper261a27d2015-10-08 15:28:25 -07006131
Matt Ropercd1d3ee2018-12-10 13:54:14 -08006132 ilk_compute_wm_config(dev_priv, &config);
Ville Syrjäläd8905652016-01-14 14:53:35 +02006133
Matt Ropercd1d3ee2018-12-10 13:54:14 -08006134 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_1_2, &max);
6135 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_1_2);
Ville Syrjälä0362c782013-10-09 19:17:57 +03006136
Ville Syrjäläa485bfb2013-10-09 19:17:59 +03006137 /* 5/6 split only in single pipe config on IVB+ */
Tvrtko Ursulin175fded2016-11-16 08:55:42 +00006138 if (INTEL_GEN(dev_priv) >= 7 &&
Ville Syrjäläd8905652016-01-14 14:53:35 +02006139 config.num_pipes_active == 1 && config.sprites_enabled) {
Matt Ropercd1d3ee2018-12-10 13:54:14 -08006140 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_5_6, &max);
6141 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_5_6);
Ville Syrjäläa485bfb2013-10-09 19:17:59 +03006142
Matt Ropercd1d3ee2018-12-10 13:54:14 -08006143 best_lp_wm = ilk_find_best_result(dev_priv, &lp_wm_1_2, &lp_wm_5_6);
Paulo Zanoni861f3382013-05-31 10:19:21 -03006144 } else {
Ville Syrjälä198a1e92013-10-09 19:17:58 +03006145 best_lp_wm = &lp_wm_1_2;
Paulo Zanoni861f3382013-05-31 10:19:21 -03006146 }
6147
Ville Syrjälä198a1e92013-10-09 19:17:58 +03006148 partitioning = (best_lp_wm == &lp_wm_1_2) ?
Ville Syrjälä77c122b2013-08-06 22:24:04 +03006149 INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
Paulo Zanoni861f3382013-05-31 10:19:21 -03006150
Matt Ropercd1d3ee2018-12-10 13:54:14 -08006151 ilk_compute_wm_results(dev_priv, best_lp_wm, partitioning, &results);
Ville Syrjälä609cede2013-10-09 19:18:03 +03006152
Imre Deak820c1982013-12-17 14:46:36 +02006153 ilk_write_wm_values(dev_priv, &results);
Paulo Zanoni1011d8c2013-05-09 16:55:50 -03006154}
6155
Maarten Lankhorstccf010f2016-11-08 13:55:32 +01006156static void ilk_initial_watermarks(struct intel_atomic_state *state,
Ville Syrjälä7a8fdb1f2019-11-18 18:44:26 +02006157 struct intel_crtc *crtc)
Ville Syrjäläb9d5c832015-09-24 15:53:14 -07006158{
Ville Syrjälä7a8fdb1f2019-11-18 18:44:26 +02006159 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6160 const struct intel_crtc_state *crtc_state =
6161 intel_atomic_get_new_crtc_state(state, crtc);
Ville Syrjäläb9d5c832015-09-24 15:53:14 -07006162
Matt Ropered4a6a72016-02-23 17:20:13 -08006163 mutex_lock(&dev_priv->wm.wm_mutex);
Ville Syrjälä88016a92019-07-01 19:05:45 +03006164 crtc->wm.active.ilk = crtc_state->wm.ilk.intermediate;
Matt Ropered4a6a72016-02-23 17:20:13 -08006165 ilk_program_watermarks(dev_priv);
6166 mutex_unlock(&dev_priv->wm.wm_mutex);
6167}
Ville Syrjäläb9d5c832015-09-24 15:53:14 -07006168
Maarten Lankhorstccf010f2016-11-08 13:55:32 +01006169static void ilk_optimize_watermarks(struct intel_atomic_state *state,
Ville Syrjälä7a8fdb1f2019-11-18 18:44:26 +02006170 struct intel_crtc *crtc)
Matt Ropered4a6a72016-02-23 17:20:13 -08006171{
Ville Syrjälä7a8fdb1f2019-11-18 18:44:26 +02006172 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6173 const struct intel_crtc_state *crtc_state =
6174 intel_atomic_get_new_crtc_state(state, crtc);
Ville Syrjälä88016a92019-07-01 19:05:45 +03006175
6176 if (!crtc_state->wm.need_postvbl_update)
6177 return;
Matt Ropered4a6a72016-02-23 17:20:13 -08006178
6179 mutex_lock(&dev_priv->wm.wm_mutex);
Ville Syrjälä88016a92019-07-01 19:05:45 +03006180 crtc->wm.active.ilk = crtc_state->wm.ilk.optimal;
6181 ilk_program_watermarks(dev_priv);
Matt Ropered4a6a72016-02-23 17:20:13 -08006182 mutex_unlock(&dev_priv->wm.wm_mutex);
Ville Syrjäläb9d5c832015-09-24 15:53:14 -07006183}
6184
Jani Nikula81b55ef2020-04-20 17:04:38 +03006185static void skl_wm_level_from_reg_val(u32 val, struct skl_wm_level *level)
Pradeep Bhat30789992014-11-04 17:06:45 +00006186{
cpaul@redhat.comd8c0faf2016-10-18 16:09:49 -02006187 level->plane_en = val & PLANE_WM_EN;
Ville Syrjälä2ed8e1f2019-02-13 18:54:23 +02006188 level->ignore_lines = val & PLANE_WM_IGNORE_LINES;
cpaul@redhat.comd8c0faf2016-10-18 16:09:49 -02006189 level->plane_res_b = val & PLANE_WM_BLOCKS_MASK;
6190 level->plane_res_l = (val >> PLANE_WM_LINES_SHIFT) &
6191 PLANE_WM_LINES_MASK;
Pradeep Bhat30789992014-11-04 17:06:45 +00006192}
6193
Matt Ropercd1d3ee2018-12-10 13:54:14 -08006194void skl_pipe_wm_get_hw_state(struct intel_crtc *crtc,
cpaul@redhat.combf9d99a2016-10-14 17:31:55 -04006195 struct skl_pipe_wm *out)
Pradeep Bhat30789992014-11-04 17:06:45 +00006196{
Matt Ropercd1d3ee2018-12-10 13:54:14 -08006197 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6198 enum pipe pipe = crtc->pipe;
Ville Syrjäläd5cdfdf52016-11-22 18:01:58 +02006199 int level, max_level;
6200 enum plane_id plane_id;
Jani Nikula5ce9a6492019-01-18 14:01:20 +02006201 u32 val;
Pradeep Bhat30789992014-11-04 17:06:45 +00006202
Tvrtko Ursulin5db94012016-10-13 11:03:10 +01006203 max_level = ilk_wm_max_level(dev_priv);
Pradeep Bhat30789992014-11-04 17:06:45 +00006204
Matt Ropercd1d3ee2018-12-10 13:54:14 -08006205 for_each_plane_id_on_crtc(crtc, plane_id) {
Ville Syrjäläd5cdfdf52016-11-22 18:01:58 +02006206 struct skl_plane_wm *wm = &out->planes[plane_id];
Pradeep Bhat30789992014-11-04 17:06:45 +00006207
cpaul@redhat.comd8c0faf2016-10-18 16:09:49 -02006208 for (level = 0; level <= max_level; level++) {
Ville Syrjäläd5cdfdf52016-11-22 18:01:58 +02006209 if (plane_id != PLANE_CURSOR)
Jani Nikula5f461662020-11-30 13:15:58 +02006210 val = intel_uncore_read(&dev_priv->uncore, PLANE_WM(pipe, plane_id, level));
cpaul@redhat.comd8c0faf2016-10-18 16:09:49 -02006211 else
Jani Nikula5f461662020-11-30 13:15:58 +02006212 val = intel_uncore_read(&dev_priv->uncore, CUR_WM(pipe, level));
cpaul@redhat.comd8c0faf2016-10-18 16:09:49 -02006213
6214 skl_wm_level_from_reg_val(val, &wm->wm[level]);
6215 }
6216
Ville Syrjäläd5cdfdf52016-11-22 18:01:58 +02006217 if (plane_id != PLANE_CURSOR)
Jani Nikula5f461662020-11-30 13:15:58 +02006218 val = intel_uncore_read(&dev_priv->uncore, PLANE_WM_TRANS(pipe, plane_id));
cpaul@redhat.comd8c0faf2016-10-18 16:09:49 -02006219 else
Jani Nikula5f461662020-11-30 13:15:58 +02006220 val = intel_uncore_read(&dev_priv->uncore, CUR_WM_TRANS(pipe));
cpaul@redhat.comd8c0faf2016-10-18 16:09:49 -02006221
6222 skl_wm_level_from_reg_val(val, &wm->trans_wm);
Ville Syrjälä5f25e6a2021-02-26 17:32:02 +02006223
6224 if (INTEL_GEN(dev_priv) >= 12) {
6225 wm->sagv.wm0 = wm->wm[0];
6226 wm->sagv.trans_wm = wm->trans_wm;
6227 }
cpaul@redhat.comd8c0faf2016-10-18 16:09:49 -02006228 }
Pradeep Bhat30789992014-11-04 17:06:45 +00006229}
6230
Matt Ropercd1d3ee2018-12-10 13:54:14 -08006231void skl_wm_get_hw_state(struct drm_i915_private *dev_priv)
Pradeep Bhat30789992014-11-04 17:06:45 +00006232{
Ville Syrjälä33c9c502021-01-22 22:56:33 +02006233 struct intel_dbuf_state *dbuf_state =
6234 to_intel_dbuf_state(dev_priv->dbuf.obj.state);
Matt Ropercd1d3ee2018-12-10 13:54:14 -08006235 struct intel_crtc *crtc;
Pradeep Bhat30789992014-11-04 17:06:45 +00006236
Matt Ropercd1d3ee2018-12-10 13:54:14 -08006237 for_each_intel_crtc(&dev_priv->drm, crtc) {
Ville Syrjälä33c9c502021-01-22 22:56:33 +02006238 struct intel_crtc_state *crtc_state =
6239 to_intel_crtc_state(crtc->base.state);
6240 enum pipe pipe = crtc->pipe;
6241 enum plane_id plane_id;
cpaul@redhat.combf9d99a2016-10-14 17:31:55 -04006242
Maarten Lankhorstec193642019-06-28 10:55:17 +02006243 skl_pipe_wm_get_hw_state(crtc, &crtc_state->wm.skl.optimal);
Ville Syrjälädbf71382020-11-06 19:30:38 +02006244 crtc_state->wm.skl.raw = crtc_state->wm.skl.optimal;
Ville Syrjälä33c9c502021-01-22 22:56:33 +02006245
6246 memset(&dbuf_state->ddb[pipe], 0, sizeof(dbuf_state->ddb[pipe]));
6247
6248 for_each_plane_id_on_crtc(crtc, plane_id) {
6249 struct skl_ddb_entry *ddb_y =
6250 &crtc_state->wm.skl.plane_ddb_y[plane_id];
6251 struct skl_ddb_entry *ddb_uv =
6252 &crtc_state->wm.skl.plane_ddb_uv[plane_id];
6253
6254 skl_ddb_get_hw_plane_state(dev_priv, crtc->pipe,
6255 plane_id, ddb_y, ddb_uv);
6256
6257 skl_ddb_entry_union(&dbuf_state->ddb[pipe], ddb_y);
6258 skl_ddb_entry_union(&dbuf_state->ddb[pipe], ddb_uv);
6259 }
6260
6261 dbuf_state->slices[pipe] =
6262 skl_compute_dbuf_slices(crtc, dbuf_state->active_pipes);
6263
6264 dbuf_state->weight[pipe] = intel_crtc_ddb_weight(crtc_state);
6265
6266 crtc_state->wm.skl.ddb = dbuf_state->ddb[pipe];
6267
6268 drm_dbg_kms(&dev_priv->drm,
6269 "[CRTC:%d:%s] dbuf slices 0x%x, ddb (%d - %d), active pipes 0x%x\n",
6270 crtc->base.base.id, crtc->base.name,
6271 dbuf_state->slices[pipe], dbuf_state->ddb[pipe].start,
6272 dbuf_state->ddb[pipe].end, dbuf_state->active_pipes);
cpaul@redhat.combf9d99a2016-10-14 17:31:55 -04006273 }
Ville Syrjälä33c9c502021-01-22 22:56:33 +02006274
6275 dbuf_state->enabled_slices = dev_priv->dbuf.enabled_slices;
Pradeep Bhat30789992014-11-04 17:06:45 +00006276}
6277
Matt Ropercd1d3ee2018-12-10 13:54:14 -08006278static void ilk_pipe_wm_get_hw_state(struct intel_crtc *crtc)
Ville Syrjälä243e6a42013-10-14 14:55:24 +03006279{
Matt Ropercd1d3ee2018-12-10 13:54:14 -08006280 struct drm_device *dev = crtc->base.dev;
Chris Wilsonfac5e232016-07-04 11:34:36 +01006281 struct drm_i915_private *dev_priv = to_i915(dev);
Imre Deak820c1982013-12-17 14:46:36 +02006282 struct ilk_wm_values *hw = &dev_priv->wm.hw;
Maarten Lankhorstec193642019-06-28 10:55:17 +02006283 struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
6284 struct intel_pipe_wm *active = &crtc_state->wm.ilk.optimal;
Matt Ropercd1d3ee2018-12-10 13:54:14 -08006285 enum pipe pipe = crtc->pipe;
Ville Syrjälä243e6a42013-10-14 14:55:24 +03006286
Jani Nikula5f461662020-11-30 13:15:58 +02006287 hw->wm_pipe[pipe] = intel_uncore_read(&dev_priv->uncore, WM0_PIPE_ILK(pipe));
Ville Syrjälä243e6a42013-10-14 14:55:24 +03006288
Ville Syrjälä15606532016-05-13 17:55:17 +03006289 memset(active, 0, sizeof(*active));
6290
Matt Ropercd1d3ee2018-12-10 13:54:14 -08006291 active->pipe_enabled = crtc->active;
Ville Syrjälä2a44b762014-03-07 18:32:09 +02006292
6293 if (active->pipe_enabled) {
Ville Syrjälä243e6a42013-10-14 14:55:24 +03006294 u32 tmp = hw->wm_pipe[pipe];
6295
6296 /*
6297 * For active pipes LP0 watermark is marked as
6298 * enabled, and LP1+ watermaks as disabled since
6299 * we can't really reverse compute them in case
6300 * multiple pipes are active.
6301 */
6302 active->wm[0].enable = true;
6303 active->wm[0].pri_val = (tmp & WM0_PIPE_PLANE_MASK) >> WM0_PIPE_PLANE_SHIFT;
6304 active->wm[0].spr_val = (tmp & WM0_PIPE_SPRITE_MASK) >> WM0_PIPE_SPRITE_SHIFT;
6305 active->wm[0].cur_val = tmp & WM0_PIPE_CURSOR_MASK;
Ville Syrjälä243e6a42013-10-14 14:55:24 +03006306 } else {
Tvrtko Ursulin5db94012016-10-13 11:03:10 +01006307 int level, max_level = ilk_wm_max_level(dev_priv);
Ville Syrjälä243e6a42013-10-14 14:55:24 +03006308
6309 /*
6310 * For inactive pipes, all watermark levels
6311 * should be marked as enabled but zeroed,
6312 * which is what we'd compute them to.
6313 */
6314 for (level = 0; level <= max_level; level++)
6315 active->wm[level].enable = true;
6316 }
Matt Roper4e0963c2015-09-24 15:53:15 -07006317
Matt Ropercd1d3ee2018-12-10 13:54:14 -08006318 crtc->wm.active.ilk = *active;
Ville Syrjälä243e6a42013-10-14 14:55:24 +03006319}
6320
Ville Syrjälä6eb1a682015-06-24 22:00:03 +03006321#define _FW_WM(value, plane) \
6322 (((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT)
6323#define _FW_WM_VLV(value, plane) \
6324 (((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT)
6325
Ville Syrjälä04548cb2017-04-21 21:14:29 +03006326static void g4x_read_wm_values(struct drm_i915_private *dev_priv,
6327 struct g4x_wm_values *wm)
6328{
Jani Nikula5ce9a6492019-01-18 14:01:20 +02006329 u32 tmp;
Ville Syrjälä04548cb2017-04-21 21:14:29 +03006330
Jani Nikula5f461662020-11-30 13:15:58 +02006331 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1);
Ville Syrjälä04548cb2017-04-21 21:14:29 +03006332 wm->sr.plane = _FW_WM(tmp, SR);
6333 wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
6334 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEB);
6335 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEA);
6336
Jani Nikula5f461662020-11-30 13:15:58 +02006337 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2);
Ville Syrjälä04548cb2017-04-21 21:14:29 +03006338 wm->fbc_en = tmp & DSPFW_FBC_SR_EN;
6339 wm->sr.fbc = _FW_WM(tmp, FBC_SR);
6340 wm->hpll.fbc = _FW_WM(tmp, FBC_HPLL_SR);
6341 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEB);
6342 wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
6343 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEA);
6344
Jani Nikula5f461662020-11-30 13:15:58 +02006345 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3);
Ville Syrjälä04548cb2017-04-21 21:14:29 +03006346 wm->hpll_en = tmp & DSPFW_HPLL_SR_EN;
6347 wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
6348 wm->hpll.cursor = _FW_WM(tmp, HPLL_CURSOR);
6349 wm->hpll.plane = _FW_WM(tmp, HPLL_SR);
6350}
6351
Ville Syrjälä6eb1a682015-06-24 22:00:03 +03006352static void vlv_read_wm_values(struct drm_i915_private *dev_priv,
6353 struct vlv_wm_values *wm)
6354{
6355 enum pipe pipe;
Jani Nikula5ce9a6492019-01-18 14:01:20 +02006356 u32 tmp;
Ville Syrjälä6eb1a682015-06-24 22:00:03 +03006357
6358 for_each_pipe(dev_priv, pipe) {
Jani Nikula5f461662020-11-30 13:15:58 +02006359 tmp = intel_uncore_read(&dev_priv->uncore, VLV_DDL(pipe));
Ville Syrjälä6eb1a682015-06-24 22:00:03 +03006360
Ville Syrjälä1b313892016-11-28 19:37:08 +02006361 wm->ddl[pipe].plane[PLANE_PRIMARY] =
Ville Syrjälä6eb1a682015-06-24 22:00:03 +03006362 (tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
Ville Syrjälä1b313892016-11-28 19:37:08 +02006363 wm->ddl[pipe].plane[PLANE_CURSOR] =
Ville Syrjälä6eb1a682015-06-24 22:00:03 +03006364 (tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
Ville Syrjälä1b313892016-11-28 19:37:08 +02006365 wm->ddl[pipe].plane[PLANE_SPRITE0] =
Ville Syrjälä6eb1a682015-06-24 22:00:03 +03006366 (tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
Ville Syrjälä1b313892016-11-28 19:37:08 +02006367 wm->ddl[pipe].plane[PLANE_SPRITE1] =
Ville Syrjälä6eb1a682015-06-24 22:00:03 +03006368 (tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6369 }
6370
Jani Nikula5f461662020-11-30 13:15:58 +02006371 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1);
Ville Syrjälä6eb1a682015-06-24 22:00:03 +03006372 wm->sr.plane = _FW_WM(tmp, SR);
Ville Syrjälä1b313892016-11-28 19:37:08 +02006373 wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
6374 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEB);
6375 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEA);
Ville Syrjälä6eb1a682015-06-24 22:00:03 +03006376
Jani Nikula5f461662020-11-30 13:15:58 +02006377 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2);
Ville Syrjälä1b313892016-11-28 19:37:08 +02006378 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEB);
6379 wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
6380 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEA);
Ville Syrjälä6eb1a682015-06-24 22:00:03 +03006381
Jani Nikula5f461662020-11-30 13:15:58 +02006382 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3);
Ville Syrjälä6eb1a682015-06-24 22:00:03 +03006383 wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
6384
6385 if (IS_CHERRYVIEW(dev_priv)) {
Jani Nikula5f461662020-11-30 13:15:58 +02006386 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7_CHV);
Ville Syrjälä1b313892016-11-28 19:37:08 +02006387 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
6388 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
Ville Syrjälä6eb1a682015-06-24 22:00:03 +03006389
Jani Nikula5f461662020-11-30 13:15:58 +02006390 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW8_CHV);
Ville Syrjälä1b313892016-11-28 19:37:08 +02006391 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEF);
6392 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEE);
Ville Syrjälä6eb1a682015-06-24 22:00:03 +03006393
Jani Nikula5f461662020-11-30 13:15:58 +02006394 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW9_CHV);
Ville Syrjälä1b313892016-11-28 19:37:08 +02006395 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEC);
6396 wm->pipe[PIPE_C].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORC);
Ville Syrjälä6eb1a682015-06-24 22:00:03 +03006397
Jani Nikula5f461662020-11-30 13:15:58 +02006398 tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM);
Ville Syrjälä6eb1a682015-06-24 22:00:03 +03006399 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
Ville Syrjälä1b313892016-11-28 19:37:08 +02006400 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEF_HI) << 8;
6401 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEE_HI) << 8;
6402 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEC_HI) << 8;
6403 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
6404 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
6405 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
6406 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
6407 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
6408 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
Ville Syrjälä6eb1a682015-06-24 22:00:03 +03006409 } else {
Jani Nikula5f461662020-11-30 13:15:58 +02006410 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7);
Ville Syrjälä1b313892016-11-28 19:37:08 +02006411 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
6412 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
Ville Syrjälä6eb1a682015-06-24 22:00:03 +03006413
Jani Nikula5f461662020-11-30 13:15:58 +02006414 tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM);
Ville Syrjälä6eb1a682015-06-24 22:00:03 +03006415 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
Ville Syrjälä1b313892016-11-28 19:37:08 +02006416 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
6417 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
6418 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
6419 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
6420 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
6421 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
Ville Syrjälä6eb1a682015-06-24 22:00:03 +03006422 }
6423}
6424
6425#undef _FW_WM
6426#undef _FW_WM_VLV
6427
Matt Ropercd1d3ee2018-12-10 13:54:14 -08006428void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv)
Ville Syrjälä04548cb2017-04-21 21:14:29 +03006429{
Ville Syrjälä04548cb2017-04-21 21:14:29 +03006430 struct g4x_wm_values *wm = &dev_priv->wm.g4x;
6431 struct intel_crtc *crtc;
6432
6433 g4x_read_wm_values(dev_priv, wm);
6434
Jani Nikula5f461662020-11-30 13:15:58 +02006435 wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
Ville Syrjälä04548cb2017-04-21 21:14:29 +03006436
Matt Ropercd1d3ee2018-12-10 13:54:14 -08006437 for_each_intel_crtc(&dev_priv->drm, crtc) {
Ville Syrjälä04548cb2017-04-21 21:14:29 +03006438 struct intel_crtc_state *crtc_state =
6439 to_intel_crtc_state(crtc->base.state);
6440 struct g4x_wm_state *active = &crtc->wm.active.g4x;
6441 struct g4x_pipe_wm *raw;
6442 enum pipe pipe = crtc->pipe;
6443 enum plane_id plane_id;
6444 int level, max_level;
6445
6446 active->cxsr = wm->cxsr;
6447 active->hpll_en = wm->hpll_en;
6448 active->fbc_en = wm->fbc_en;
6449
6450 active->sr = wm->sr;
6451 active->hpll = wm->hpll;
6452
6453 for_each_plane_id_on_crtc(crtc, plane_id) {
6454 active->wm.plane[plane_id] =
6455 wm->pipe[pipe].plane[plane_id];
6456 }
6457
6458 if (wm->cxsr && wm->hpll_en)
6459 max_level = G4X_WM_LEVEL_HPLL;
6460 else if (wm->cxsr)
6461 max_level = G4X_WM_LEVEL_SR;
6462 else
6463 max_level = G4X_WM_LEVEL_NORMAL;
6464
6465 level = G4X_WM_LEVEL_NORMAL;
6466 raw = &crtc_state->wm.g4x.raw[level];
6467 for_each_plane_id_on_crtc(crtc, plane_id)
6468 raw->plane[plane_id] = active->wm.plane[plane_id];
6469
6470 if (++level > max_level)
6471 goto out;
6472
6473 raw = &crtc_state->wm.g4x.raw[level];
6474 raw->plane[PLANE_PRIMARY] = active->sr.plane;
6475 raw->plane[PLANE_CURSOR] = active->sr.cursor;
6476 raw->plane[PLANE_SPRITE0] = 0;
6477 raw->fbc = active->sr.fbc;
6478
6479 if (++level > max_level)
6480 goto out;
6481
6482 raw = &crtc_state->wm.g4x.raw[level];
6483 raw->plane[PLANE_PRIMARY] = active->hpll.plane;
6484 raw->plane[PLANE_CURSOR] = active->hpll.cursor;
6485 raw->plane[PLANE_SPRITE0] = 0;
6486 raw->fbc = active->hpll.fbc;
6487
6488 out:
6489 for_each_plane_id_on_crtc(crtc, plane_id)
6490 g4x_raw_plane_wm_set(crtc_state, level,
6491 plane_id, USHRT_MAX);
6492 g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
6493
6494 crtc_state->wm.g4x.optimal = *active;
6495 crtc_state->wm.g4x.intermediate = *active;
6496
Wambui Karugaf8d18d52020-01-07 18:13:30 +03006497 drm_dbg_kms(&dev_priv->drm,
6498 "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite=%d\n",
6499 pipe_name(pipe),
6500 wm->pipe[pipe].plane[PLANE_PRIMARY],
6501 wm->pipe[pipe].plane[PLANE_CURSOR],
6502 wm->pipe[pipe].plane[PLANE_SPRITE0]);
Ville Syrjälä04548cb2017-04-21 21:14:29 +03006503 }
6504
Wambui Karugaf8d18d52020-01-07 18:13:30 +03006505 drm_dbg_kms(&dev_priv->drm,
6506 "Initial SR watermarks: plane=%d, cursor=%d fbc=%d\n",
6507 wm->sr.plane, wm->sr.cursor, wm->sr.fbc);
6508 drm_dbg_kms(&dev_priv->drm,
6509 "Initial HPLL watermarks: plane=%d, SR cursor=%d fbc=%d\n",
6510 wm->hpll.plane, wm->hpll.cursor, wm->hpll.fbc);
6511 drm_dbg_kms(&dev_priv->drm, "Initial SR=%s HPLL=%s FBC=%s\n",
6512 yesno(wm->cxsr), yesno(wm->hpll_en), yesno(wm->fbc_en));
Ville Syrjälä04548cb2017-04-21 21:14:29 +03006513}
6514
6515void g4x_wm_sanitize(struct drm_i915_private *dev_priv)
6516{
6517 struct intel_plane *plane;
6518 struct intel_crtc *crtc;
6519
6520 mutex_lock(&dev_priv->wm.wm_mutex);
6521
6522 for_each_intel_plane(&dev_priv->drm, plane) {
6523 struct intel_crtc *crtc =
6524 intel_get_crtc_for_pipe(dev_priv, plane->pipe);
6525 struct intel_crtc_state *crtc_state =
6526 to_intel_crtc_state(crtc->base.state);
6527 struct intel_plane_state *plane_state =
6528 to_intel_plane_state(plane->base.state);
6529 struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
6530 enum plane_id plane_id = plane->id;
6531 int level;
6532
Maarten Lankhorstf90a85e2019-10-31 12:26:08 +01006533 if (plane_state->uapi.visible)
Ville Syrjälä04548cb2017-04-21 21:14:29 +03006534 continue;
6535
6536 for (level = 0; level < 3; level++) {
6537 struct g4x_pipe_wm *raw =
6538 &crtc_state->wm.g4x.raw[level];
6539
6540 raw->plane[plane_id] = 0;
6541 wm_state->wm.plane[plane_id] = 0;
6542 }
6543
6544 if (plane_id == PLANE_PRIMARY) {
6545 for (level = 0; level < 3; level++) {
6546 struct g4x_pipe_wm *raw =
6547 &crtc_state->wm.g4x.raw[level];
6548 raw->fbc = 0;
6549 }
6550
6551 wm_state->sr.fbc = 0;
6552 wm_state->hpll.fbc = 0;
6553 wm_state->fbc_en = false;
6554 }
6555 }
6556
6557 for_each_intel_crtc(&dev_priv->drm, crtc) {
6558 struct intel_crtc_state *crtc_state =
6559 to_intel_crtc_state(crtc->base.state);
6560
6561 crtc_state->wm.g4x.intermediate =
6562 crtc_state->wm.g4x.optimal;
6563 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
6564 }
6565
6566 g4x_program_watermarks(dev_priv);
6567
6568 mutex_unlock(&dev_priv->wm.wm_mutex);
6569}
6570
Matt Ropercd1d3ee2018-12-10 13:54:14 -08006571void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv)
Ville Syrjälä6eb1a682015-06-24 22:00:03 +03006572{
Ville Syrjälä6eb1a682015-06-24 22:00:03 +03006573 struct vlv_wm_values *wm = &dev_priv->wm.vlv;
Ville Syrjäläf07d43d2017-03-02 19:14:52 +02006574 struct intel_crtc *crtc;
Ville Syrjälä6eb1a682015-06-24 22:00:03 +03006575 u32 val;
6576
6577 vlv_read_wm_values(dev_priv, wm);
6578
Jani Nikula5f461662020-11-30 13:15:58 +02006579 wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
Ville Syrjälä6eb1a682015-06-24 22:00:03 +03006580 wm->level = VLV_WM_LEVEL_PM2;
6581
6582 if (IS_CHERRYVIEW(dev_priv)) {
Chris Wilson337fa6e2019-04-26 09:17:20 +01006583 vlv_punit_get(dev_priv);
Ville Syrjälä6eb1a682015-06-24 22:00:03 +03006584
Ville Syrjäläc11b8132018-11-29 19:55:03 +02006585 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
Ville Syrjälä6eb1a682015-06-24 22:00:03 +03006586 if (val & DSP_MAXFIFO_PM5_ENABLE)
6587 wm->level = VLV_WM_LEVEL_PM5;
6588
Ville Syrjälä58590c12015-09-08 21:05:12 +03006589 /*
6590 * If DDR DVFS is disabled in the BIOS, Punit
6591 * will never ack the request. So if that happens
6592 * assume we don't have to enable/disable DDR DVFS
6593 * dynamically. To test that just set the REQ_ACK
6594 * bit to poke the Punit, but don't change the
6595 * HIGH/LOW bits so that we don't actually change
6596 * the current state.
6597 */
Ville Syrjälä6eb1a682015-06-24 22:00:03 +03006598 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
Ville Syrjälä58590c12015-09-08 21:05:12 +03006599 val |= FORCE_DDR_FREQ_REQ_ACK;
6600 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
6601
6602 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
6603 FORCE_DDR_FREQ_REQ_ACK) == 0, 3)) {
Wambui Karugaf8d18d52020-01-07 18:13:30 +03006604 drm_dbg_kms(&dev_priv->drm,
6605 "Punit not acking DDR DVFS request, "
6606 "assuming DDR DVFS is disabled\n");
Ville Syrjälä58590c12015-09-08 21:05:12 +03006607 dev_priv->wm.max_level = VLV_WM_LEVEL_PM5;
6608 } else {
6609 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
6610 if ((val & FORCE_DDR_HIGH_FREQ) == 0)
6611 wm->level = VLV_WM_LEVEL_DDR_DVFS;
6612 }
Ville Syrjälä6eb1a682015-06-24 22:00:03 +03006613
Chris Wilson337fa6e2019-04-26 09:17:20 +01006614 vlv_punit_put(dev_priv);
Ville Syrjälä6eb1a682015-06-24 22:00:03 +03006615 }
6616
Matt Ropercd1d3ee2018-12-10 13:54:14 -08006617 for_each_intel_crtc(&dev_priv->drm, crtc) {
Ville Syrjäläff32c542017-03-02 19:14:57 +02006618 struct intel_crtc_state *crtc_state =
6619 to_intel_crtc_state(crtc->base.state);
6620 struct vlv_wm_state *active = &crtc->wm.active.vlv;
6621 const struct vlv_fifo_state *fifo_state =
6622 &crtc_state->wm.vlv.fifo_state;
6623 enum pipe pipe = crtc->pipe;
6624 enum plane_id plane_id;
6625 int level;
6626
6627 vlv_get_fifo_size(crtc_state);
6628
6629 active->num_levels = wm->level + 1;
6630 active->cxsr = wm->cxsr;
6631
Ville Syrjäläff32c542017-03-02 19:14:57 +02006632 for (level = 0; level < active->num_levels; level++) {
Ville Syrjälä114d7dc2017-04-21 21:14:21 +03006633 struct g4x_pipe_wm *raw =
Ville Syrjäläff32c542017-03-02 19:14:57 +02006634 &crtc_state->wm.vlv.raw[level];
6635
6636 active->sr[level].plane = wm->sr.plane;
6637 active->sr[level].cursor = wm->sr.cursor;
6638
6639 for_each_plane_id_on_crtc(crtc, plane_id) {
6640 active->wm[level].plane[plane_id] =
6641 wm->pipe[pipe].plane[plane_id];
6642
6643 raw->plane[plane_id] =
6644 vlv_invert_wm_value(active->wm[level].plane[plane_id],
6645 fifo_state->plane[plane_id]);
6646 }
6647 }
6648
6649 for_each_plane_id_on_crtc(crtc, plane_id)
6650 vlv_raw_plane_wm_set(crtc_state, level,
6651 plane_id, USHRT_MAX);
6652 vlv_invalidate_wms(crtc, active, level);
6653
6654 crtc_state->wm.vlv.optimal = *active;
Ville Syrjälä4841da52017-03-02 19:14:59 +02006655 crtc_state->wm.vlv.intermediate = *active;
Ville Syrjäläff32c542017-03-02 19:14:57 +02006656
Wambui Karugaf8d18d52020-01-07 18:13:30 +03006657 drm_dbg_kms(&dev_priv->drm,
6658 "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n",
6659 pipe_name(pipe),
6660 wm->pipe[pipe].plane[PLANE_PRIMARY],
6661 wm->pipe[pipe].plane[PLANE_CURSOR],
6662 wm->pipe[pipe].plane[PLANE_SPRITE0],
6663 wm->pipe[pipe].plane[PLANE_SPRITE1]);
Ville Syrjäläff32c542017-03-02 19:14:57 +02006664 }
Ville Syrjälä6eb1a682015-06-24 22:00:03 +03006665
Wambui Karugaf8d18d52020-01-07 18:13:30 +03006666 drm_dbg_kms(&dev_priv->drm,
6667 "Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n",
6668 wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr);
Ville Syrjälä6eb1a682015-06-24 22:00:03 +03006669}
6670
Ville Syrjälä602ae832017-03-02 19:15:02 +02006671void vlv_wm_sanitize(struct drm_i915_private *dev_priv)
6672{
6673 struct intel_plane *plane;
6674 struct intel_crtc *crtc;
6675
6676 mutex_lock(&dev_priv->wm.wm_mutex);
6677
6678 for_each_intel_plane(&dev_priv->drm, plane) {
6679 struct intel_crtc *crtc =
6680 intel_get_crtc_for_pipe(dev_priv, plane->pipe);
6681 struct intel_crtc_state *crtc_state =
6682 to_intel_crtc_state(crtc->base.state);
6683 struct intel_plane_state *plane_state =
6684 to_intel_plane_state(plane->base.state);
6685 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
6686 const struct vlv_fifo_state *fifo_state =
6687 &crtc_state->wm.vlv.fifo_state;
6688 enum plane_id plane_id = plane->id;
6689 int level;
6690
Maarten Lankhorstf90a85e2019-10-31 12:26:08 +01006691 if (plane_state->uapi.visible)
Ville Syrjälä602ae832017-03-02 19:15:02 +02006692 continue;
6693
6694 for (level = 0; level < wm_state->num_levels; level++) {
Ville Syrjälä114d7dc2017-04-21 21:14:21 +03006695 struct g4x_pipe_wm *raw =
Ville Syrjälä602ae832017-03-02 19:15:02 +02006696 &crtc_state->wm.vlv.raw[level];
6697
6698 raw->plane[plane_id] = 0;
6699
6700 wm_state->wm[level].plane[plane_id] =
6701 vlv_invert_wm_value(raw->plane[plane_id],
6702 fifo_state->plane[plane_id]);
6703 }
6704 }
6705
6706 for_each_intel_crtc(&dev_priv->drm, crtc) {
6707 struct intel_crtc_state *crtc_state =
6708 to_intel_crtc_state(crtc->base.state);
6709
6710 crtc_state->wm.vlv.intermediate =
6711 crtc_state->wm.vlv.optimal;
6712 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
6713 }
6714
6715 vlv_program_watermarks(dev_priv);
6716
6717 mutex_unlock(&dev_priv->wm.wm_mutex);
6718}
6719
Ville Syrjäläf72b84c2017-11-08 15:35:55 +02006720/*
6721 * FIXME should probably kill this and improve
6722 * the real watermark readout/sanitation instead
6723 */
6724static void ilk_init_lp_watermarks(struct drm_i915_private *dev_priv)
6725{
Jani Nikula5f461662020-11-30 13:15:58 +02006726 intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, intel_uncore_read(&dev_priv->uncore, WM3_LP_ILK) & ~WM1_LP_SR_EN);
6727 intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, intel_uncore_read(&dev_priv->uncore, WM2_LP_ILK) & ~WM1_LP_SR_EN);
6728 intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, intel_uncore_read(&dev_priv->uncore, WM1_LP_ILK) & ~WM1_LP_SR_EN);
Ville Syrjäläf72b84c2017-11-08 15:35:55 +02006729
6730 /*
6731 * Don't touch WM1S_LP_EN here.
6732 * Doing so could cause underruns.
6733 */
6734}
6735
Matt Ropercd1d3ee2018-12-10 13:54:14 -08006736void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv)
Ville Syrjälä243e6a42013-10-14 14:55:24 +03006737{
Imre Deak820c1982013-12-17 14:46:36 +02006738 struct ilk_wm_values *hw = &dev_priv->wm.hw;
Matt Ropercd1d3ee2018-12-10 13:54:14 -08006739 struct intel_crtc *crtc;
Ville Syrjälä243e6a42013-10-14 14:55:24 +03006740
Ville Syrjäläf72b84c2017-11-08 15:35:55 +02006741 ilk_init_lp_watermarks(dev_priv);
6742
Matt Ropercd1d3ee2018-12-10 13:54:14 -08006743 for_each_intel_crtc(&dev_priv->drm, crtc)
Ville Syrjälä243e6a42013-10-14 14:55:24 +03006744 ilk_pipe_wm_get_hw_state(crtc);
6745
Jani Nikula5f461662020-11-30 13:15:58 +02006746 hw->wm_lp[0] = intel_uncore_read(&dev_priv->uncore, WM1_LP_ILK);
6747 hw->wm_lp[1] = intel_uncore_read(&dev_priv->uncore, WM2_LP_ILK);
6748 hw->wm_lp[2] = intel_uncore_read(&dev_priv->uncore, WM3_LP_ILK);
Ville Syrjälä243e6a42013-10-14 14:55:24 +03006749
Jani Nikula5f461662020-11-30 13:15:58 +02006750 hw->wm_lp_spr[0] = intel_uncore_read(&dev_priv->uncore, WM1S_LP_ILK);
Tvrtko Ursulin175fded2016-11-16 08:55:42 +00006751 if (INTEL_GEN(dev_priv) >= 7) {
Jani Nikula5f461662020-11-30 13:15:58 +02006752 hw->wm_lp_spr[1] = intel_uncore_read(&dev_priv->uncore, WM2S_LP_IVB);
6753 hw->wm_lp_spr[2] = intel_uncore_read(&dev_priv->uncore, WM3S_LP_IVB);
Ville Syrjäläcfa76982014-03-07 18:32:08 +02006754 }
Ville Syrjälä243e6a42013-10-14 14:55:24 +03006755
Tvrtko Ursulin86527442016-10-13 11:03:00 +01006756 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
Jani Nikula5f461662020-11-30 13:15:58 +02006757 hw->partitioning = (intel_uncore_read(&dev_priv->uncore, WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
Ville Syrjäläac9545f2013-12-05 15:51:28 +02006758 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
Tvrtko Ursulinfd6b8f42016-10-14 10:13:06 +01006759 else if (IS_IVYBRIDGE(dev_priv))
Jani Nikula5f461662020-11-30 13:15:58 +02006760 hw->partitioning = (intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
Ville Syrjäläac9545f2013-12-05 15:51:28 +02006761 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
Ville Syrjälä243e6a42013-10-14 14:55:24 +03006762
6763 hw->enable_fbc_wm =
Jani Nikula5f461662020-11-30 13:15:58 +02006764 !(intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) & DISP_FBC_WM_DIS);
Ville Syrjälä243e6a42013-10-14 14:55:24 +03006765}
6766
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -03006767/**
6768 * intel_update_watermarks - update FIFO watermark values based on current modes
Chris Wilson31383412018-02-14 14:03:03 +00006769 * @crtc: the #intel_crtc on which to compute the WM
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -03006770 *
6771 * Calculate watermark values for the various WM regs based on current mode
6772 * and plane configuration.
6773 *
6774 * There are several cases to deal with here:
6775 * - normal (i.e. non-self-refresh)
6776 * - self-refresh (SR) mode
6777 * - lines are large relative to FIFO size (buffer can hold up to 2)
6778 * - lines are small relative to FIFO size (buffer can hold more than 2
6779 * lines), so need to account for TLB latency
6780 *
6781 * The normal calculation is:
6782 * watermark = dotclock * bytes per pixel * latency
6783 * where latency is platform & configuration dependent (we assume pessimal
6784 * values here).
6785 *
6786 * The SR calculation is:
6787 * watermark = (trunc(latency/line time)+1) * surface width *
6788 * bytes per pixel
6789 * where
6790 * line time = htotal / dotclock
6791 * surface width = hdisplay for normal plane and 64 for cursor
6792 * and latency is assumed to be high, as above.
6793 *
6794 * The final value programmed to the register should always be rounded up,
6795 * and include an extra 2 entries to account for clock crossings.
6796 *
6797 * We don't use the sprite, so we can ignore that. And on Crestline we have
6798 * to set the non-SR watermarks to 8.
6799 */
Ville Syrjälä432081b2016-10-31 22:37:03 +02006800void intel_update_watermarks(struct intel_crtc *crtc)
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -03006801{
Ville Syrjälä432081b2016-10-31 22:37:03 +02006802 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -03006803
6804 if (dev_priv->display.update_wm)
Ville Syrjälä46ba6142013-09-10 11:40:40 +03006805 dev_priv->display.update_wm(crtc);
Eugeni Dodonovb445e3b2012-04-16 22:20:35 -03006806}
6807
Kumar, Mahesh2503a0f2017-08-17 19:15:28 +05306808void intel_enable_ipc(struct drm_i915_private *dev_priv)
6809{
6810 u32 val;
6811
José Roberto de Souzafd847b82018-09-18 13:47:11 -07006812 if (!HAS_IPC(dev_priv))
6813 return;
6814
Jani Nikula5f461662020-11-30 13:15:58 +02006815 val = intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2);
Kumar, Mahesh2503a0f2017-08-17 19:15:28 +05306816
6817 if (dev_priv->ipc_enabled)
6818 val |= DISP_IPC_ENABLE;
6819 else
6820 val &= ~DISP_IPC_ENABLE;
6821
Jani Nikula5f461662020-11-30 13:15:58 +02006822 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL2, val);
Kumar, Mahesh2503a0f2017-08-17 19:15:28 +05306823}
6824
Ville Syrjäläc91a45f2019-05-03 20:38:07 +03006825static bool intel_can_enable_ipc(struct drm_i915_private *dev_priv)
6826{
6827 /* Display WA #0477 WaDisableIPC: skl */
6828 if (IS_SKYLAKE(dev_priv))
6829 return false;
6830
6831 /* Display WA #1141: SKL:all KBL:all CFL */
Chris Wilson5f4ae272020-06-02 15:05:40 +01006832 if (IS_KABYLAKE(dev_priv) ||
6833 IS_COFFEELAKE(dev_priv) ||
6834 IS_COMETLAKE(dev_priv))
Ville Syrjäläc91a45f2019-05-03 20:38:07 +03006835 return dev_priv->dram_info.symmetric_memory;
6836
6837 return true;
6838}
6839
Kumar, Mahesh2503a0f2017-08-17 19:15:28 +05306840void intel_init_ipc(struct drm_i915_private *dev_priv)
6841{
Kumar, Mahesh2503a0f2017-08-17 19:15:28 +05306842 if (!HAS_IPC(dev_priv))
6843 return;
6844
Ville Syrjäläc91a45f2019-05-03 20:38:07 +03006845 dev_priv->ipc_enabled = intel_can_enable_ipc(dev_priv);
José Roberto de Souzac9b818d2018-09-18 13:47:13 -07006846
Kumar, Mahesh2503a0f2017-08-17 19:15:28 +05306847 intel_enable_ipc(dev_priv);
6848}
6849
Ville Syrjälä46f16e62016-10-31 22:37:22 +02006850static void ibx_init_clock_gating(struct drm_i915_private *dev_priv)
Daniel Vetter3107bd42012-10-31 22:52:31 +01006851{
Daniel Vetter3107bd42012-10-31 22:52:31 +01006852 /*
6853 * On Ibex Peak and Cougar Point, we need to disable clock
6854 * gating for the panel power sequencer or it will fail to
6855 * start up when no ports are active.
6856 */
Jani Nikula5f461662020-11-30 13:15:58 +02006857 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
Daniel Vetter3107bd42012-10-31 22:52:31 +01006858}
6859
Ville Syrjälä46f16e62016-10-31 22:37:22 +02006860static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv)
Ville Syrjälä0e088b82013-06-07 10:47:04 +03006861{
Ville Syrjäläb12ce1d2015-05-26 20:27:23 +03006862 enum pipe pipe;
Ville Syrjälä0e088b82013-06-07 10:47:04 +03006863
Damien Lespiau055e3932014-08-18 13:49:10 +01006864 for_each_pipe(dev_priv, pipe) {
Jani Nikula5f461662020-11-30 13:15:58 +02006865 intel_uncore_write(&dev_priv->uncore, DSPCNTR(pipe),
6866 intel_uncore_read(&dev_priv->uncore, DSPCNTR(pipe)) |
Ville Syrjälä0e088b82013-06-07 10:47:04 +03006867 DISPPLANE_TRICKLE_FEED_DISABLE);
Ville Syrjäläb12ce1d2015-05-26 20:27:23 +03006868
Jani Nikula5f461662020-11-30 13:15:58 +02006869 intel_uncore_write(&dev_priv->uncore, DSPSURF(pipe), intel_uncore_read(&dev_priv->uncore, DSPSURF(pipe)));
6870 intel_uncore_posting_read(&dev_priv->uncore, DSPSURF(pipe));
Ville Syrjälä0e088b82013-06-07 10:47:04 +03006871 }
6872}
6873
Rodrigo Vivi91200c02017-08-28 22:20:26 -07006874static void ilk_init_clock_gating(struct drm_i915_private *dev_priv)
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03006875{
Jani Nikula5ce9a6492019-01-18 14:01:20 +02006876 u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03006877
Damien Lespiauf1e8fa52013-06-07 17:41:09 +01006878 /*
6879 * Required for FBC
6880 * WaFbcDisableDpfcClockGating:ilk
6881 */
Damien Lespiau4d47e4f2012-10-19 17:55:42 +01006882 dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
6883 ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
6884 ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03006885
Jani Nikula5f461662020-11-30 13:15:58 +02006886 intel_uncore_write(&dev_priv->uncore, PCH_3DCGDIS0,
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03006887 MARIUNIT_CLOCK_GATE_DISABLE |
6888 SVSMUNIT_CLOCK_GATE_DISABLE);
Jani Nikula5f461662020-11-30 13:15:58 +02006889 intel_uncore_write(&dev_priv->uncore, PCH_3DCGDIS1,
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03006890 VFMUNIT_CLOCK_GATE_DISABLE);
6891
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03006892 /*
6893 * According to the spec the following bits should be set in
6894 * order to enable memory self-refresh
6895 * The bit 22/21 of 0x42004
6896 * The bit 5 of 0x42020
6897 * The bit 15 of 0x45000
6898 */
Jani Nikula5f461662020-11-30 13:15:58 +02006899 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
6900 (intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03006901 ILK_DPARB_GATE | ILK_VSDPFD_FULL));
Damien Lespiau4d47e4f2012-10-19 17:55:42 +01006902 dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
Jani Nikula5f461662020-11-30 13:15:58 +02006903 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL,
6904 (intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03006905 DISP_FBC_WM_DIS));
Ville Syrjälä017636c2013-12-05 15:51:37 +02006906
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03006907 /*
6908 * Based on the document from hardware guys the following bits
6909 * should be set unconditionally in order to enable FBC.
6910 * The bit 22 of 0x42000
6911 * The bit 22 of 0x42004
6912 * The bit 7,8,9 of 0x42020.
6913 */
Tvrtko Ursulin50a0bc92016-10-13 11:02:58 +01006914 if (IS_IRONLAKE_M(dev_priv)) {
Damien Lespiau4bb35332013-06-14 15:23:24 +01006915 /* WaFbcAsynchFlipDisableFbcQueue:ilk */
Jani Nikula5f461662020-11-30 13:15:58 +02006916 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1,
6917 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1) |
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03006918 ILK_FBCQ_DIS);
Jani Nikula5f461662020-11-30 13:15:58 +02006919 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
6920 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03006921 ILK_DPARB_GATE);
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03006922 }
6923
Jani Nikula5f461662020-11-30 13:15:58 +02006924 intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
Damien Lespiau4d47e4f2012-10-19 17:55:42 +01006925
Jani Nikula5f461662020-11-30 13:15:58 +02006926 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
6927 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03006928 ILK_ELPIN_409_SELECT);
Akash Goel4e046322014-04-04 17:14:38 +05306929
Ville Syrjälä46f16e62016-10-31 22:37:22 +02006930 g4x_disable_trickle_feed(dev_priv);
Ville Syrjäläbdad2b22013-06-07 10:47:03 +03006931
Ville Syrjälä46f16e62016-10-31 22:37:22 +02006932 ibx_init_clock_gating(dev_priv);
Daniel Vetter3107bd42012-10-31 22:52:31 +01006933}
6934
Ville Syrjälä46f16e62016-10-31 22:37:22 +02006935static void cpt_init_clock_gating(struct drm_i915_private *dev_priv)
Daniel Vetter3107bd42012-10-31 22:52:31 +01006936{
Ville Syrjäläd048a262019-08-21 20:30:31 +03006937 enum pipe pipe;
Jani Nikula5ce9a6492019-01-18 14:01:20 +02006938 u32 val;
Daniel Vetter3107bd42012-10-31 22:52:31 +01006939
6940 /*
6941 * On Ibex Peak and Cougar Point, we need to disable clock
6942 * gating for the panel power sequencer or it will fail to
6943 * start up when no ports are active.
6944 */
Jani Nikula5f461662020-11-30 13:15:58 +02006945 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
Jesse Barnescd664072013-10-02 10:34:19 -07006946 PCH_DPLUNIT_CLOCK_GATE_DISABLE |
6947 PCH_CPUNIT_CLOCK_GATE_DISABLE);
Jani Nikula5f461662020-11-30 13:15:58 +02006948 intel_uncore_write(&dev_priv->uncore, SOUTH_CHICKEN2, intel_uncore_read(&dev_priv->uncore, SOUTH_CHICKEN2) |
Daniel Vetter3107bd42012-10-31 22:52:31 +01006949 DPLS_EDP_PPS_FIX_DIS);
Takashi Iwai335c07b2012-12-11 11:46:29 +01006950 /* The below fixes the weird display corruption, a few pixels shifted
6951 * downward, on (only) LVDS of some HP laptops with IVY.
6952 */
Damien Lespiau055e3932014-08-18 13:49:10 +01006953 for_each_pipe(dev_priv, pipe) {
Jani Nikula5f461662020-11-30 13:15:58 +02006954 val = intel_uncore_read(&dev_priv->uncore, TRANS_CHICKEN2(pipe));
Paulo Zanonidc4bd2d2013-04-08 15:48:08 -03006955 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
6956 val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
Rodrigo Vivi41aa3442013-05-09 20:03:18 -03006957 if (dev_priv->vbt.fdi_rx_polarity_inverted)
Paulo Zanoni3f704fa2013-04-08 15:48:07 -03006958 val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
Paulo Zanonidc4bd2d2013-04-08 15:48:08 -03006959 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
6960 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
Jani Nikula5f461662020-11-30 13:15:58 +02006961 intel_uncore_write(&dev_priv->uncore, TRANS_CHICKEN2(pipe), val);
Paulo Zanoni3f704fa2013-04-08 15:48:07 -03006962 }
Daniel Vetter3107bd42012-10-31 22:52:31 +01006963 /* WADP0ClockGatingDisable */
Damien Lespiau055e3932014-08-18 13:49:10 +01006964 for_each_pipe(dev_priv, pipe) {
Jani Nikula5f461662020-11-30 13:15:58 +02006965 intel_uncore_write(&dev_priv->uncore, TRANS_CHICKEN1(pipe),
Daniel Vetter3107bd42012-10-31 22:52:31 +01006966 TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
6967 }
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03006968}
6969
Ville Syrjälä46f16e62016-10-31 22:37:22 +02006970static void gen6_check_mch_setup(struct drm_i915_private *dev_priv)
Daniel Vetter1d7aaa02013-02-09 21:03:42 +01006971{
Jani Nikula5ce9a6492019-01-18 14:01:20 +02006972 u32 tmp;
Daniel Vetter1d7aaa02013-02-09 21:03:42 +01006973
Jani Nikula5f461662020-11-30 13:15:58 +02006974 tmp = intel_uncore_read(&dev_priv->uncore, MCH_SSKPD);
Daniel Vetterdf662a22014-08-04 11:17:25 +02006975 if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL)
Wambui Karugaf8d18d52020-01-07 18:13:30 +03006976 drm_dbg_kms(&dev_priv->drm,
6977 "Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
6978 tmp);
Daniel Vetter1d7aaa02013-02-09 21:03:42 +01006979}
6980
Ville Syrjälä46f16e62016-10-31 22:37:22 +02006981static void gen6_init_clock_gating(struct drm_i915_private *dev_priv)
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03006982{
Jani Nikula5ce9a6492019-01-18 14:01:20 +02006983 u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03006984
Jani Nikula5f461662020-11-30 13:15:58 +02006985 intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03006986
Jani Nikula5f461662020-11-30 13:15:58 +02006987 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
6988 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03006989 ILK_ELPIN_409_SELECT);
6990
Jani Nikula5f461662020-11-30 13:15:58 +02006991 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1,
6992 intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) |
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03006993 GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
6994 GEN6_CSUNIT_CLOCK_GATE_DISABLE);
6995
6996 /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
6997 * gating disable must be set. Failure to set it results in
6998 * flickering pixels due to Z write ordering failures after
6999 * some amount of runtime in the Mesa "fire" demo, and Unigine
7000 * Sanctuary and Tropics, and apparently anything else with
7001 * alpha test or pixel discard.
7002 *
7003 * According to the spec, bit 11 (RCCUNIT) must also be set,
7004 * but we didn't debug actual testcases to find it out.
Jesse Barnes0f846f82012-06-14 11:04:47 -07007005 *
Ville Syrjäläef593182014-01-22 21:32:47 +02007006 * WaDisableRCCUnitClockGating:snb
7007 * WaDisableRCPBUnitClockGating:snb
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03007008 */
Jani Nikula5f461662020-11-30 13:15:58 +02007009 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03007010 GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
7011 GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
7012
Ville Syrjäläe927ecd2014-02-04 21:59:18 +02007013 /*
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03007014 * According to the spec the following bits should be
7015 * set in order to enable memory self-refresh and fbc:
7016 * The bit21 and bit22 of 0x42000
7017 * The bit21 and bit22 of 0x42004
7018 * The bit5 and bit7 of 0x42020
7019 * The bit14 of 0x70180
7020 * The bit14 of 0x71180
Damien Lespiau4bb35332013-06-14 15:23:24 +01007021 *
7022 * WaFbcAsynchFlipDisableFbcQueue:snb
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03007023 */
Jani Nikula5f461662020-11-30 13:15:58 +02007024 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1,
7025 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1) |
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03007026 ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
Jani Nikula5f461662020-11-30 13:15:58 +02007027 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
7028 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03007029 ILK_DPARB_GATE | ILK_VSDPFD_FULL);
Jani Nikula5f461662020-11-30 13:15:58 +02007030 intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D,
7031 intel_uncore_read(&dev_priv->uncore, ILK_DSPCLK_GATE_D) |
Damien Lespiau231e54f2012-10-19 17:55:41 +01007032 ILK_DPARBUNIT_CLOCK_GATE_ENABLE |
7033 ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03007034
Ville Syrjälä46f16e62016-10-31 22:37:22 +02007035 g4x_disable_trickle_feed(dev_priv);
Ben Widawskyf8f2ac92012-10-03 19:34:24 -07007036
Ville Syrjälä46f16e62016-10-31 22:37:22 +02007037 cpt_init_clock_gating(dev_priv);
Daniel Vetter1d7aaa02013-02-09 21:03:42 +01007038
Ville Syrjälä46f16e62016-10-31 22:37:22 +02007039 gen6_check_mch_setup(dev_priv);
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03007040}
7041
Ville Syrjälä46f16e62016-10-31 22:37:22 +02007042static void lpt_init_clock_gating(struct drm_i915_private *dev_priv)
Paulo Zanoni17a303e2012-11-20 15:12:07 -02007043{
Paulo Zanoni17a303e2012-11-20 15:12:07 -02007044 /*
7045 * TODO: this bit should only be enabled when really needed, then
7046 * disabled when not needed anymore in order to save power.
7047 */
Tvrtko Ursulin4f8036a2016-10-13 11:02:52 +01007048 if (HAS_PCH_LPT_LP(dev_priv))
Jani Nikula5f461662020-11-30 13:15:58 +02007049 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D,
7050 intel_uncore_read(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D) |
Paulo Zanoni17a303e2012-11-20 15:12:07 -02007051 PCH_LP_PARTITION_LEVEL_DISABLE);
Paulo Zanoni0a790cd2013-04-17 18:15:49 -03007052
7053 /* WADPOClockGatingDisable:hsw */
Jani Nikula5f461662020-11-30 13:15:58 +02007054 intel_uncore_write(&dev_priv->uncore, TRANS_CHICKEN1(PIPE_A),
7055 intel_uncore_read(&dev_priv->uncore, TRANS_CHICKEN1(PIPE_A)) |
Paulo Zanoni0a790cd2013-04-17 18:15:49 -03007056 TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
Paulo Zanoni17a303e2012-11-20 15:12:07 -02007057}
7058
Ville Syrjälä712bf362016-10-31 22:37:23 +02007059static void lpt_suspend_hw(struct drm_i915_private *dev_priv)
Imre Deak7d708ee2013-04-17 14:04:50 +03007060{
Tvrtko Ursulin4f8036a2016-10-13 11:02:52 +01007061 if (HAS_PCH_LPT_LP(dev_priv)) {
Jani Nikula5f461662020-11-30 13:15:58 +02007062 u32 val = intel_uncore_read(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D);
Imre Deak7d708ee2013-04-17 14:04:50 +03007063
7064 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
Jani Nikula5f461662020-11-30 13:15:58 +02007065 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, val);
Imre Deak7d708ee2013-04-17 14:04:50 +03007066 }
7067}
7068
Imre Deak450174f2016-05-03 15:54:21 +03007069static void gen8_set_l3sqc_credits(struct drm_i915_private *dev_priv,
7070 int general_prio_credits,
7071 int high_prio_credits)
7072{
7073 u32 misccpctl;
Oscar Mateo930a7842017-10-17 13:25:45 -07007074 u32 val;
Imre Deak450174f2016-05-03 15:54:21 +03007075
7076 /* WaTempDisableDOPClkGating:bdw */
Jani Nikula5f461662020-11-30 13:15:58 +02007077 misccpctl = intel_uncore_read(&dev_priv->uncore, GEN7_MISCCPCTL);
7078 intel_uncore_write(&dev_priv->uncore, GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
Imre Deak450174f2016-05-03 15:54:21 +03007079
Jani Nikula5f461662020-11-30 13:15:58 +02007080 val = intel_uncore_read(&dev_priv->uncore, GEN8_L3SQCREG1);
Oscar Mateo930a7842017-10-17 13:25:45 -07007081 val &= ~L3_PRIO_CREDITS_MASK;
7082 val |= L3_GENERAL_PRIO_CREDITS(general_prio_credits);
7083 val |= L3_HIGH_PRIO_CREDITS(high_prio_credits);
Jani Nikula5f461662020-11-30 13:15:58 +02007084 intel_uncore_write(&dev_priv->uncore, GEN8_L3SQCREG1, val);
Imre Deak450174f2016-05-03 15:54:21 +03007085
7086 /*
7087 * Wait at least 100 clocks before re-enabling clock gating.
7088 * See the definition of L3SQCREG1 in BSpec.
7089 */
Jani Nikula5f461662020-11-30 13:15:58 +02007090 intel_uncore_posting_read(&dev_priv->uncore, GEN8_L3SQCREG1);
Imre Deak450174f2016-05-03 15:54:21 +03007091 udelay(1);
Jani Nikula5f461662020-11-30 13:15:58 +02007092 intel_uncore_write(&dev_priv->uncore, GEN7_MISCCPCTL, misccpctl);
Imre Deak450174f2016-05-03 15:54:21 +03007093}
7094
Oscar Mateod65dc3e2018-05-08 14:29:24 -07007095static void icl_init_clock_gating(struct drm_i915_private *dev_priv)
7096{
Ville Syrjälä885f1822020-07-08 16:12:20 +03007097 /* Wa_1409120013:icl,ehl */
Jani Nikula5f461662020-11-30 13:15:58 +02007098 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN,
Ville Syrjälä885f1822020-07-08 16:12:20 +03007099 ILK_DPFC_CHICKEN_COMP_DUMMY_PIXEL);
7100
Oscar Mateod65dc3e2018-05-08 14:29:24 -07007101 /* This is not an Wa. Enable to reduce Sampler power */
Jani Nikula5f461662020-11-30 13:15:58 +02007102 intel_uncore_write(&dev_priv->uncore, GEN10_DFR_RATIO_EN_AND_CHICKEN,
7103 intel_uncore_read(&dev_priv->uncore, GEN10_DFR_RATIO_EN_AND_CHICKEN) & ~DFR_DISABLE);
Radhakrishna Sripada622b3f62018-10-30 01:45:01 -07007104
Matt Atwood6f4194c2020-01-13 23:11:28 -05007105 /*Wa_14010594013:icl, ehl */
7106 intel_uncore_rmw(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1,
7107 0, CNL_DELAY_PMRSP);
Oscar Mateod65dc3e2018-05-08 14:29:24 -07007108}
7109
José Roberto de Souza35f08372021-01-13 05:37:59 -08007110static void gen12lp_init_clock_gating(struct drm_i915_private *dev_priv)
Michel Thierry5d869232019-08-23 01:20:34 -07007111{
José Roberto de Souza35f08372021-01-13 05:37:59 -08007112 /* Wa_1409120013:tgl,rkl,adl_s,dg1 */
Jani Nikula5f461662020-11-30 13:15:58 +02007113 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN,
José Roberto de Souza35f08372021-01-13 05:37:59 -08007114 ILK_DPFC_CHICKEN_COMP_DUMMY_PIXEL);
Ville Syrjälä885f1822020-07-08 16:12:20 +03007115
Radhakrishna Sripadaf78d5da2020-01-09 14:37:27 -08007116 /* Wa_1409825376:tgl (pre-prod)*/
Aditya Swarup7e6c0642021-01-19 11:29:30 -08007117 if (IS_TGL_DISP_STEPPING(dev_priv, STEP_A0, STEP_B1))
Jani Nikula5f461662020-11-30 13:15:58 +02007118 intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_3, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_3) |
Radhakrishna Sripadaf78d5da2020-01-09 14:37:27 -08007119 TGL_VRH_GATING_DIS);
Matt Atwoodf9d77422020-04-15 15:35:35 -04007120
José Roberto de Souza35f08372021-01-13 05:37:59 -08007121 /* Wa_14011059788:tgl,rkl,adl_s,dg1 */
Matt Atwoodf9d77422020-04-15 15:35:35 -04007122 intel_uncore_rmw(&dev_priv->uncore, GEN10_DFR_RATIO_EN_AND_CHICKEN,
7123 0, DFR_DISABLE);
Michel Thierry5d869232019-08-23 01:20:34 -07007124}
7125
Stuart Summersda9427502020-10-14 12:19:34 -07007126static void dg1_init_clock_gating(struct drm_i915_private *dev_priv)
7127{
José Roberto de Souza35f08372021-01-13 05:37:59 -08007128 gen12lp_init_clock_gating(dev_priv);
7129
Stuart Summersda9427502020-10-14 12:19:34 -07007130 /* Wa_1409836686:dg1[a0] */
7131 if (IS_DG1_REVID(dev_priv, DG1_REVID_A0, DG1_REVID_A0))
Jani Nikula5f461662020-11-30 13:15:58 +02007132 intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_3, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_3) |
Stuart Summersda9427502020-10-14 12:19:34 -07007133 DPT_GATING_DIS);
7134}
7135
Rodrigo Vivi0a46ddd2017-08-30 21:52:23 -07007136static void cnp_init_clock_gating(struct drm_i915_private *dev_priv)
7137{
7138 if (!HAS_PCH_CNP(dev_priv))
7139 return;
7140
Rodrigo Vivi470e7c62018-03-05 17:28:12 -08007141 /* Display WA #1181 WaSouthDisplayDisablePWMCGEGating: cnp */
Jani Nikula5f461662020-11-30 13:15:58 +02007142 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, intel_uncore_read(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D) |
Rodrigo Vivi4cc6feb2017-09-08 16:45:33 -07007143 CNP_PWM_CGE_GATING_DISABLE);
Rodrigo Vivi0a46ddd2017-08-30 21:52:23 -07007144}
7145
Rodrigo Vivi91200c02017-08-28 22:20:26 -07007146static void cnl_init_clock_gating(struct drm_i915_private *dev_priv)
Rodrigo Vivi90007bc2017-08-15 16:16:48 -07007147{
Rodrigo Vivi8f067832017-09-05 12:30:13 -07007148 u32 val;
Rodrigo Vivi0a46ddd2017-08-30 21:52:23 -07007149 cnp_init_clock_gating(dev_priv);
7150
Rodrigo Vivi1a25db62017-08-15 16:16:51 -07007151 /* This is not an Wa. Enable for better image quality */
Jani Nikula5f461662020-11-30 13:15:58 +02007152 intel_uncore_write(&dev_priv->uncore, _3D_CHICKEN3,
Rodrigo Vivi1a25db62017-08-15 16:16:51 -07007153 _MASKED_BIT_ENABLE(_3D_CHICKEN3_AA_LINE_QUALITY_FIX_ENABLE));
7154
Rodrigo Vivi90007bc2017-08-15 16:16:48 -07007155 /* WaEnableChickenDCPR:cnl */
Jani Nikula5f461662020-11-30 13:15:58 +02007156 intel_uncore_write(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1,
7157 intel_uncore_read(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM);
Rodrigo Vivi90007bc2017-08-15 16:16:48 -07007158
Ville Syrjälä99bcf64e2020-07-08 16:12:23 +03007159 /*
7160 * WaFbcWakeMemOn:cnl
7161 * Display WA #0859: cnl
7162 */
Jani Nikula5f461662020-11-30 13:15:58 +02007163 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
Rodrigo Vivi90007bc2017-08-15 16:16:48 -07007164 DISP_FBC_MEMORY_WAKE);
7165
Jani Nikula5f461662020-11-30 13:15:58 +02007166 val = intel_uncore_read(&dev_priv->uncore, SLICE_UNIT_LEVEL_CLKGATE);
Chris Wilson34991bd2017-11-11 10:03:36 +00007167 /* ReadHitWriteOnlyDisable:cnl */
7168 val |= RCCUNIT_CLKGATE_DIS;
Jani Nikula5f461662020-11-30 13:15:58 +02007169 intel_uncore_write(&dev_priv->uncore, SLICE_UNIT_LEVEL_CLKGATE, val);
Rafael Antognolli01ab0f92017-12-15 16:11:16 -08007170
Rodrigo Vivia4713c52018-03-07 14:09:12 -08007171 /* Wa_2201832410:cnl */
Jani Nikula5f461662020-11-30 13:15:58 +02007172 val = intel_uncore_read(&dev_priv->uncore, SUBSLICE_UNIT_LEVEL_CLKGATE);
Rodrigo Vivia4713c52018-03-07 14:09:12 -08007173 val |= GWUNIT_CLKGATE_DIS;
Jani Nikula5f461662020-11-30 13:15:58 +02007174 intel_uncore_write(&dev_priv->uncore, SUBSLICE_UNIT_LEVEL_CLKGATE, val);
Rodrigo Vivia4713c52018-03-07 14:09:12 -08007175
Rafael Antognolli01ab0f92017-12-15 16:11:16 -08007176 /* WaDisableVFclkgate:cnl */
Rodrigo Vivi14941b62018-03-05 17:20:00 -08007177 /* WaVFUnitClockGatingDisable:cnl */
Jani Nikula5f461662020-11-30 13:15:58 +02007178 val = intel_uncore_read(&dev_priv->uncore, UNSLICE_UNIT_LEVEL_CLKGATE);
Rafael Antognolli01ab0f92017-12-15 16:11:16 -08007179 val |= VFUNIT_CLKGATE_DIS;
Jani Nikula5f461662020-11-30 13:15:58 +02007180 intel_uncore_write(&dev_priv->uncore, UNSLICE_UNIT_LEVEL_CLKGATE, val);
Rodrigo Vivi90007bc2017-08-15 16:16:48 -07007181}
7182
Rodrigo Vivi0a46ddd2017-08-30 21:52:23 -07007183static void cfl_init_clock_gating(struct drm_i915_private *dev_priv)
7184{
7185 cnp_init_clock_gating(dev_priv);
7186 gen9_init_clock_gating(dev_priv);
7187
Ville Syrjälä4d6bde582020-07-16 22:04:26 +03007188 /* WAC6entrylatency:cfl */
Jani Nikula5f461662020-11-30 13:15:58 +02007189 intel_uncore_write(&dev_priv->uncore, FBC_LLC_READ_CTRL, intel_uncore_read(&dev_priv->uncore, FBC_LLC_READ_CTRL) |
Ville Syrjälä4d6bde582020-07-16 22:04:26 +03007190 FBC_LLC_FULLY_OPEN);
7191
Ville Syrjälä99bcf64e2020-07-08 16:12:23 +03007192 /*
7193 * WaFbcTurnOffFbcWatermark:cfl
7194 * Display WA #0562: cfl
7195 */
Jani Nikula5f461662020-11-30 13:15:58 +02007196 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
Ville Syrjäläc4615b22020-07-08 16:12:21 +03007197 DISP_FBC_WM_DIS);
7198
Ville Syrjälä99bcf64e2020-07-08 16:12:23 +03007199 /*
7200 * WaFbcNukeOnHostModify:cfl
7201 * Display WA #0873: cfl
7202 */
Jani Nikula5f461662020-11-30 13:15:58 +02007203 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN, intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN) |
Rodrigo Vivi0a46ddd2017-08-30 21:52:23 -07007204 ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
7205}
7206
Rodrigo Vivi91200c02017-08-28 22:20:26 -07007207static void kbl_init_clock_gating(struct drm_i915_private *dev_priv)
Mika Kuoppala9498dba2016-06-07 17:19:01 +03007208{
Ville Syrjälä46f16e62016-10-31 22:37:22 +02007209 gen9_init_clock_gating(dev_priv);
Mika Kuoppala9498dba2016-06-07 17:19:01 +03007210
Ville Syrjälä4d6bde582020-07-16 22:04:26 +03007211 /* WAC6entrylatency:kbl */
Jani Nikula5f461662020-11-30 13:15:58 +02007212 intel_uncore_write(&dev_priv->uncore, FBC_LLC_READ_CTRL, intel_uncore_read(&dev_priv->uncore, FBC_LLC_READ_CTRL) |
Ville Syrjälä4d6bde582020-07-16 22:04:26 +03007213 FBC_LLC_FULLY_OPEN);
7214
Mika Kuoppala9498dba2016-06-07 17:19:01 +03007215 /* WaDisableSDEUnitClockGating:kbl */
Matt Roper96c5a152020-08-10 20:21:05 -07007216 if (IS_KBL_GT_REVID(dev_priv, 0, KBL_REVID_B0))
Jani Nikula5f461662020-11-30 13:15:58 +02007217 intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
Mika Kuoppala9498dba2016-06-07 17:19:01 +03007218 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
Mika Kuoppala8aeb7f62016-06-07 17:19:05 +03007219
7220 /* WaDisableGamClockGating:kbl */
Matt Roper96c5a152020-08-10 20:21:05 -07007221 if (IS_KBL_GT_REVID(dev_priv, 0, KBL_REVID_B0))
Jani Nikula5f461662020-11-30 13:15:58 +02007222 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1, intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) |
Mika Kuoppala8aeb7f62016-06-07 17:19:05 +03007223 GEN6_GAMUNIT_CLOCK_GATE_DISABLE);
Mika Kuoppala031cd8c2016-06-07 17:19:18 +03007224
Ville Syrjälä99bcf64e2020-07-08 16:12:23 +03007225 /*
7226 * WaFbcTurnOffFbcWatermark:kbl
7227 * Display WA #0562: kbl
7228 */
Jani Nikula5f461662020-11-30 13:15:58 +02007229 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
Ville Syrjäläc4615b22020-07-08 16:12:21 +03007230 DISP_FBC_WM_DIS);
7231
Ville Syrjälä99bcf64e2020-07-08 16:12:23 +03007232 /*
7233 * WaFbcNukeOnHostModify:kbl
7234 * Display WA #0873: kbl
7235 */
Jani Nikula5f461662020-11-30 13:15:58 +02007236 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN, intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN) |
Mika Kuoppala031cd8c2016-06-07 17:19:18 +03007237 ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
Mika Kuoppala9498dba2016-06-07 17:19:01 +03007238}
7239
Rodrigo Vivi91200c02017-08-28 22:20:26 -07007240static void skl_init_clock_gating(struct drm_i915_private *dev_priv)
Daniel Vetterdc00b6a2016-05-19 09:14:20 +02007241{
Ville Syrjälä46f16e62016-10-31 22:37:22 +02007242 gen9_init_clock_gating(dev_priv);
Mika Kuoppala44fff992016-06-07 17:19:09 +03007243
Ville Syrjäläf1421192020-07-16 22:04:25 +03007244 /* WaDisableDopClockGating:skl */
Jani Nikula5f461662020-11-30 13:15:58 +02007245 intel_uncore_write(&dev_priv->uncore, GEN7_MISCCPCTL, intel_uncore_read(&dev_priv->uncore, GEN7_MISCCPCTL) &
Ville Syrjäläf1421192020-07-16 22:04:25 +03007246 ~GEN7_DOP_CLOCK_GATE_ENABLE);
7247
Mika Kuoppala44fff992016-06-07 17:19:09 +03007248 /* WAC6entrylatency:skl */
Jani Nikula5f461662020-11-30 13:15:58 +02007249 intel_uncore_write(&dev_priv->uncore, FBC_LLC_READ_CTRL, intel_uncore_read(&dev_priv->uncore, FBC_LLC_READ_CTRL) |
Mika Kuoppala44fff992016-06-07 17:19:09 +03007250 FBC_LLC_FULLY_OPEN);
Mika Kuoppala031cd8c2016-06-07 17:19:18 +03007251
Ville Syrjälä99bcf64e2020-07-08 16:12:23 +03007252 /*
7253 * WaFbcTurnOffFbcWatermark:skl
7254 * Display WA #0562: skl
7255 */
Jani Nikula5f461662020-11-30 13:15:58 +02007256 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
Ville Syrjäläc4615b22020-07-08 16:12:21 +03007257 DISP_FBC_WM_DIS);
7258
Ville Syrjälä99bcf64e2020-07-08 16:12:23 +03007259 /*
7260 * WaFbcNukeOnHostModify:skl
7261 * Display WA #0873: skl
7262 */
Jani Nikula5f461662020-11-30 13:15:58 +02007263 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN, intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN) |
Mika Kuoppala031cd8c2016-06-07 17:19:18 +03007264 ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
Ville Syrjäläcd7a8812020-07-08 16:12:22 +03007265
Ville Syrjälä99bcf64e2020-07-08 16:12:23 +03007266 /*
7267 * WaFbcHighMemBwCorruptionAvoidance:skl
7268 * Display WA #0883: skl
7269 */
Jani Nikula5f461662020-11-30 13:15:58 +02007270 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN, intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN) |
Ville Syrjäläcd7a8812020-07-08 16:12:22 +03007271 ILK_DPFC_DISABLE_DUMMY0);
Daniel Vetterdc00b6a2016-05-19 09:14:20 +02007272}
7273
Rodrigo Vivi91200c02017-08-28 22:20:26 -07007274static void bdw_init_clock_gating(struct drm_i915_private *dev_priv)
Ben Widawsky1020a5c2013-11-02 21:07:06 -07007275{
Damien Lespiau07d27e22014-03-03 17:31:46 +00007276 enum pipe pipe;
Ben Widawsky1020a5c2013-11-02 21:07:06 -07007277
Ville Syrjälä885f1822020-07-08 16:12:20 +03007278 /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
Jani Nikula5f461662020-11-30 13:15:58 +02007279 intel_uncore_write(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A),
7280 intel_uncore_read(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A)) |
Ville Syrjälä885f1822020-07-08 16:12:20 +03007281 HSW_FBCQ_DIS);
7282
Ben Widawskyab57fff2013-12-12 15:28:04 -08007283 /* WaSwitchSolVfFArbitrationPriority:bdw */
Jani Nikula5f461662020-11-30 13:15:58 +02007284 intel_uncore_write(&dev_priv->uncore, GAM_ECOCHK, intel_uncore_read(&dev_priv->uncore, GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
Ben Widawskyfe4ab3c2013-11-02 21:07:54 -07007285
Ben Widawskyab57fff2013-12-12 15:28:04 -08007286 /* WaPsrDPAMaskVBlankInSRD:bdw */
Jani Nikula5f461662020-11-30 13:15:58 +02007287 intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR1_1,
7288 intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR1_1) | DPA_MASK_VBLANK_SRD);
Ben Widawskyfe4ab3c2013-11-02 21:07:54 -07007289
Damien Lespiau055e3932014-08-18 13:49:10 +01007290 for_each_pipe(dev_priv, pipe) {
Ville Syrjäläb7a70532021-02-20 12:33:03 +02007291 /* WaPsrDPRSUnmaskVBlankInSRD:bdw */
Jani Nikula5f461662020-11-30 13:15:58 +02007292 intel_uncore_write(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe),
7293 intel_uncore_read(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe)) |
Ville Syrjälä8f670bb2014-03-05 13:05:47 +02007294 BDW_DPRS_MASK_VBLANK_SRD);
Ville Syrjäläb7a70532021-02-20 12:33:03 +02007295
7296 /* Undocumented but fixes async flip + VT-d corruption */
7297 if (intel_vtd_active())
7298 intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe),
7299 HSW_PRI_STRETCH_MAX_MASK, HSW_PRI_STRETCH_MAX_X1);
Ben Widawskyfe4ab3c2013-11-02 21:07:54 -07007300 }
Ben Widawsky63801f22013-12-12 17:26:03 -08007301
Ben Widawskyab57fff2013-12-12 15:28:04 -08007302 /* WaVSRefCountFullforceMissDisable:bdw */
7303 /* WaDSRefCountFullforceMissDisable:bdw */
Jani Nikula5f461662020-11-30 13:15:58 +02007304 intel_uncore_write(&dev_priv->uncore, GEN7_FF_THREAD_MODE,
7305 intel_uncore_read(&dev_priv->uncore, GEN7_FF_THREAD_MODE) &
Ben Widawskyab57fff2013-12-12 15:28:04 -08007306 ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
Ville Syrjälä36075a42014-02-04 21:59:21 +02007307
Jani Nikula5f461662020-11-30 13:15:58 +02007308 intel_uncore_write(&dev_priv->uncore, GEN6_RC_SLEEP_PSMI_CONTROL,
Ville Syrjälä295e8bb2014-02-27 21:59:01 +02007309 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
Ville Syrjälä4f1ca9e2014-02-27 21:59:02 +02007310
7311 /* WaDisableSDEUnitClockGating:bdw */
Jani Nikula5f461662020-11-30 13:15:58 +02007312 intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
Ville Syrjälä4f1ca9e2014-02-27 21:59:02 +02007313 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
Damien Lespiau5d708682014-03-26 18:41:51 +00007314
Imre Deak450174f2016-05-03 15:54:21 +03007315 /* WaProgramL3SqcReg1Default:bdw */
7316 gen8_set_l3sqc_credits(dev_priv, 30, 2);
Ville Syrjälä4d487cf2015-05-19 20:32:56 +03007317
Mika Kuoppala17e0adf2016-06-07 17:19:02 +03007318 /* WaKVMNotificationOnConfigChange:bdw */
Jani Nikula5f461662020-11-30 13:15:58 +02007319 intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR2_1, intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR2_1)
Mika Kuoppala17e0adf2016-06-07 17:19:02 +03007320 | KVM_CONFIG_CHANGE_NOTIFICATION_SELECT);
7321
Ville Syrjälä46f16e62016-10-31 22:37:22 +02007322 lpt_init_clock_gating(dev_priv);
Robert Bragg9cc19732017-02-12 13:32:52 +00007323
7324 /* WaDisableDopClockGating:bdw
7325 *
7326 * Also see the CHICKEN2 write in bdw_init_workarounds() to disable DOP
7327 * clock gating.
7328 */
Jani Nikula5f461662020-11-30 13:15:58 +02007329 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1,
7330 intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) | GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE);
Ben Widawsky1020a5c2013-11-02 21:07:06 -07007331}
7332
Rodrigo Vivi91200c02017-08-28 22:20:26 -07007333static void hsw_init_clock_gating(struct drm_i915_private *dev_priv)
Eugeni Dodonovcad2a2d2012-07-02 11:51:09 -03007334{
Ville Syrjäläb7a70532021-02-20 12:33:03 +02007335 enum pipe pipe;
7336
Ville Syrjälä885f1822020-07-08 16:12:20 +03007337 /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
Jani Nikula5f461662020-11-30 13:15:58 +02007338 intel_uncore_write(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A),
7339 intel_uncore_read(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A)) |
Ville Syrjälä885f1822020-07-08 16:12:20 +03007340 HSW_FBCQ_DIS);
7341
Ville Syrjäläb7a70532021-02-20 12:33:03 +02007342 for_each_pipe(dev_priv, pipe) {
7343 /* Undocumented but fixes async flip + VT-d corruption */
7344 if (intel_vtd_active())
7345 intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe),
7346 HSW_PRI_STRETCH_MAX_MASK, HSW_PRI_STRETCH_MAX_X1);
7347 }
7348
Damien Lespiauecdb4eb72013-05-03 18:48:10 +01007349 /* This is required by WaCatErrorRejectionIssue:hsw */
Jani Nikula5f461662020-11-30 13:15:58 +02007350 intel_uncore_write(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7351 intel_uncore_read(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
Chris Wilsonf93ec5f2020-06-11 10:30:15 +01007352 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
Kenneth Graunke94411592014-12-31 16:23:00 -08007353
Damien Lespiauecdb4eb72013-05-03 18:48:10 +01007354 /* WaSwitchSolVfFArbitrationPriority:hsw */
Jani Nikula5f461662020-11-30 13:15:58 +02007355 intel_uncore_write(&dev_priv->uncore, GAM_ECOCHK, intel_uncore_read(&dev_priv->uncore, GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
Ben Widawskye3dff582013-03-20 14:49:14 -07007356
Ville Syrjälä46f16e62016-10-31 22:37:22 +02007357 lpt_init_clock_gating(dev_priv);
Eugeni Dodonovcad2a2d2012-07-02 11:51:09 -03007358}
7359
Rodrigo Vivi91200c02017-08-28 22:20:26 -07007360static void ivb_init_clock_gating(struct drm_i915_private *dev_priv)
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03007361{
Jani Nikula5ce9a6492019-01-18 14:01:20 +02007362 u32 snpcr;
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03007363
Jani Nikula5f461662020-11-30 13:15:58 +02007364 intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03007365
Ville Syrjälä885f1822020-07-08 16:12:20 +03007366 /* WaFbcAsynchFlipDisableFbcQueue:ivb */
Jani Nikula5f461662020-11-30 13:15:58 +02007367 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1,
7368 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1) |
Ville Syrjälä885f1822020-07-08 16:12:20 +03007369 ILK_FBCQ_DIS);
7370
Damien Lespiauecdb4eb72013-05-03 18:48:10 +01007371 /* WaDisableBackToBackFlipFix:ivb */
Jani Nikula5f461662020-11-30 13:15:58 +02007372 intel_uncore_write(&dev_priv->uncore, IVB_CHICKEN3,
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03007373 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
7374 CHICKEN3_DGMG_DONE_FIX_DISABLE);
7375
Tvrtko Ursulin50a0bc92016-10-13 11:02:58 +01007376 if (IS_IVB_GT1(dev_priv))
Jani Nikula5f461662020-11-30 13:15:58 +02007377 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
Jesse Barnes8ab43972012-10-25 12:15:42 -07007378 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
Ville Syrjälä412236c2014-01-22 21:32:44 +02007379 else {
7380 /* must write both registers */
Jani Nikula5f461662020-11-30 13:15:58 +02007381 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
Ville Syrjälä412236c2014-01-22 21:32:44 +02007382 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
Jani Nikula5f461662020-11-30 13:15:58 +02007383 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2_GT2,
Jesse Barnes8ab43972012-10-25 12:15:42 -07007384 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
Ville Syrjälä412236c2014-01-22 21:32:44 +02007385 }
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03007386
Ville Syrjälä1b80a19a2014-01-22 21:32:53 +02007387 /*
Jesse Barnes0f846f82012-06-14 11:04:47 -07007388 * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
Damien Lespiauecdb4eb72013-05-03 18:48:10 +01007389 * This implements the WaDisableRCZUnitClockGating:ivb workaround.
Jesse Barnes0f846f82012-06-14 11:04:47 -07007390 */
Jani Nikula5f461662020-11-30 13:15:58 +02007391 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
Ville Syrjälä28acf3b2014-01-22 21:32:48 +02007392 GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
Jesse Barnes0f846f82012-06-14 11:04:47 -07007393
Damien Lespiauecdb4eb72013-05-03 18:48:10 +01007394 /* This is required by WaCatErrorRejectionIssue:ivb */
Jani Nikula5f461662020-11-30 13:15:58 +02007395 intel_uncore_write(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7396 intel_uncore_read(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03007397 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7398
Ville Syrjälä46f16e62016-10-31 22:37:22 +02007399 g4x_disable_trickle_feed(dev_priv);
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03007400
Jani Nikula5f461662020-11-30 13:15:58 +02007401 snpcr = intel_uncore_read(&dev_priv->uncore, GEN6_MBCUNIT_SNPCR);
Ben Widawsky20848222012-05-04 18:58:59 -07007402 snpcr &= ~GEN6_MBC_SNPCR_MASK;
7403 snpcr |= GEN6_MBC_SNPCR_MED;
Jani Nikula5f461662020-11-30 13:15:58 +02007404 intel_uncore_write(&dev_priv->uncore, GEN6_MBCUNIT_SNPCR, snpcr);
Daniel Vetter3107bd42012-10-31 22:52:31 +01007405
Tvrtko Ursulin6e266952016-10-13 11:02:53 +01007406 if (!HAS_PCH_NOP(dev_priv))
Ville Syrjälä46f16e62016-10-31 22:37:22 +02007407 cpt_init_clock_gating(dev_priv);
Daniel Vetter1d7aaa02013-02-09 21:03:42 +01007408
Ville Syrjälä46f16e62016-10-31 22:37:22 +02007409 gen6_check_mch_setup(dev_priv);
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03007410}
7411
Rodrigo Vivi91200c02017-08-28 22:20:26 -07007412static void vlv_init_clock_gating(struct drm_i915_private *dev_priv)
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03007413{
Damien Lespiauecdb4eb72013-05-03 18:48:10 +01007414 /* WaDisableBackToBackFlipFix:vlv */
Jani Nikula5f461662020-11-30 13:15:58 +02007415 intel_uncore_write(&dev_priv->uncore, IVB_CHICKEN3,
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03007416 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
7417 CHICKEN3_DGMG_DONE_FIX_DISABLE);
7418
Damien Lespiauecdb4eb72013-05-03 18:48:10 +01007419 /* WaDisableDopClockGating:vlv */
Jani Nikula5f461662020-11-30 13:15:58 +02007420 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
Jesse Barnes8ab43972012-10-25 12:15:42 -07007421 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7422
Damien Lespiauecdb4eb72013-05-03 18:48:10 +01007423 /* This is required by WaCatErrorRejectionIssue:vlv */
Jani Nikula5f461662020-11-30 13:15:58 +02007424 intel_uncore_write(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7425 intel_uncore_read(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03007426 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7427
Ville Syrjälä3c0edae2014-01-22 21:32:56 +02007428 /*
Jesse Barnes0f846f82012-06-14 11:04:47 -07007429 * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
Damien Lespiauecdb4eb72013-05-03 18:48:10 +01007430 * This implements the WaDisableRCZUnitClockGating:vlv workaround.
Jesse Barnes0f846f82012-06-14 11:04:47 -07007431 */
Jani Nikula5f461662020-11-30 13:15:58 +02007432 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
Ville Syrjälä3c0edae2014-01-22 21:32:56 +02007433 GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
Jesse Barnes0f846f82012-06-14 11:04:47 -07007434
Akash Goelc98f5062014-03-24 23:00:07 +05307435 /* WaDisableL3Bank2xClockGate:vlv
7436 * Disabling L3 clock gating- MMIO 940c[25] = 1
7437 * Set bit 25, to disable L3_BANK_2x_CLK_GATING */
Jani Nikula5f461662020-11-30 13:15:58 +02007438 intel_uncore_write(&dev_priv->uncore, GEN7_UCGCTL4,
7439 intel_uncore_read(&dev_priv->uncore, GEN7_UCGCTL4) | GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
Jesse Barnese3f33d42012-06-14 11:04:50 -07007440
Ville Syrjäläafd58e72014-01-22 21:33:03 +02007441 /*
Damien Lespiauecdb4eb72013-05-03 18:48:10 +01007442 * WaDisableVLVClockGating_VBIIssue:vlv
Jesse Barnes2d809572012-10-25 12:15:44 -07007443 * Disable clock gating on th GCFG unit to prevent a delay
7444 * in the reporting of vblank events.
7445 */
Jani Nikula5f461662020-11-30 13:15:58 +02007446 intel_uncore_write(&dev_priv->uncore, VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03007447}
7448
Rodrigo Vivi91200c02017-08-28 22:20:26 -07007449static void chv_init_clock_gating(struct drm_i915_private *dev_priv)
Ville Syrjäläa4565da2014-04-09 13:28:10 +03007450{
Ville Syrjälä232ce332014-04-09 13:28:35 +03007451 /* WaVSRefCountFullforceMissDisable:chv */
7452 /* WaDSRefCountFullforceMissDisable:chv */
Jani Nikula5f461662020-11-30 13:15:58 +02007453 intel_uncore_write(&dev_priv->uncore, GEN7_FF_THREAD_MODE,
7454 intel_uncore_read(&dev_priv->uncore, GEN7_FF_THREAD_MODE) &
Ville Syrjälä232ce332014-04-09 13:28:35 +03007455 ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
Ville Syrjäläacea6f92014-04-09 13:28:36 +03007456
7457 /* WaDisableSemaphoreAndSyncFlipWait:chv */
Jani Nikula5f461662020-11-30 13:15:58 +02007458 intel_uncore_write(&dev_priv->uncore, GEN6_RC_SLEEP_PSMI_CONTROL,
Ville Syrjäläacea6f92014-04-09 13:28:36 +03007459 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
Ville Syrjälä08466972014-04-09 13:28:37 +03007460
7461 /* WaDisableCSUnitClockGating:chv */
Jani Nikula5f461662020-11-30 13:15:58 +02007462 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1, intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) |
Ville Syrjälä08466972014-04-09 13:28:37 +03007463 GEN6_CSUNIT_CLOCK_GATE_DISABLE);
Ville Syrjäläc6317802014-04-09 13:28:38 +03007464
7465 /* WaDisableSDEUnitClockGating:chv */
Jani Nikula5f461662020-11-30 13:15:58 +02007466 intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
Ville Syrjäläc6317802014-04-09 13:28:38 +03007467 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
Ville Syrjälä6d50b062015-05-19 20:32:57 +03007468
7469 /*
Imre Deak450174f2016-05-03 15:54:21 +03007470 * WaProgramL3SqcReg1Default:chv
7471 * See gfxspecs/Related Documents/Performance Guide/
7472 * LSQC Setting Recommendations.
7473 */
7474 gen8_set_l3sqc_credits(dev_priv, 38, 2);
Ville Syrjäläa4565da2014-04-09 13:28:10 +03007475}
7476
Ville Syrjälä46f16e62016-10-31 22:37:22 +02007477static void g4x_init_clock_gating(struct drm_i915_private *dev_priv)
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03007478{
Jani Nikula5ce9a6492019-01-18 14:01:20 +02007479 u32 dspclk_gate;
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03007480
Jani Nikula5f461662020-11-30 13:15:58 +02007481 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, 0);
7482 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03007483 GS_UNIT_CLOCK_GATE_DISABLE |
7484 CL_UNIT_CLOCK_GATE_DISABLE);
Jani Nikula5f461662020-11-30 13:15:58 +02007485 intel_uncore_write(&dev_priv->uncore, RAMCLK_GATE_D, 0);
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03007486 dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
7487 OVRUNIT_CLOCK_GATE_DISABLE |
7488 OVCUNIT_CLOCK_GATE_DISABLE;
Tvrtko Ursulin50a0bc92016-10-13 11:02:58 +01007489 if (IS_GM45(dev_priv))
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03007490 dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
Jani Nikula5f461662020-11-30 13:15:58 +02007491 intel_uncore_write(&dev_priv->uncore, DSPCLK_GATE_D, dspclk_gate);
Daniel Vetter4358a372012-10-18 11:49:51 +02007492
Ville Syrjälä46f16e62016-10-31 22:37:22 +02007493 g4x_disable_trickle_feed(dev_priv);
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03007494}
7495
Rodrigo Vivi91200c02017-08-28 22:20:26 -07007496static void i965gm_init_clock_gating(struct drm_i915_private *dev_priv)
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03007497{
Tvrtko Ursulin4f5fd912019-06-11 11:45:48 +01007498 struct intel_uncore *uncore = &dev_priv->uncore;
7499
7500 intel_uncore_write(uncore, RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
7501 intel_uncore_write(uncore, RENCLK_GATE_D2, 0);
7502 intel_uncore_write(uncore, DSPCLK_GATE_D, 0);
7503 intel_uncore_write(uncore, RAMCLK_GATE_D, 0);
7504 intel_uncore_write16(uncore, DEUC, 0);
7505 intel_uncore_write(uncore,
7506 MI_ARB_STATE,
7507 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03007508}
7509
Rodrigo Vivi91200c02017-08-28 22:20:26 -07007510static void i965g_init_clock_gating(struct drm_i915_private *dev_priv)
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03007511{
Jani Nikula5f461662020-11-30 13:15:58 +02007512 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03007513 I965_RCC_CLOCK_GATE_DISABLE |
7514 I965_RCPB_CLOCK_GATE_DISABLE |
7515 I965_ISC_CLOCK_GATE_DISABLE |
7516 I965_FBC_CLOCK_GATE_DISABLE);
Jani Nikula5f461662020-11-30 13:15:58 +02007517 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D2, 0);
7518 intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE,
Ville Syrjälä20f94962013-06-07 10:47:02 +03007519 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03007520}
7521
Ville Syrjälä46f16e62016-10-31 22:37:22 +02007522static void gen3_init_clock_gating(struct drm_i915_private *dev_priv)
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03007523{
Jani Nikula5f461662020-11-30 13:15:58 +02007524 u32 dstate = intel_uncore_read(&dev_priv->uncore, D_STATE);
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03007525
7526 dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
7527 DSTATE_DOT_CLOCK_GATING;
Jani Nikula5f461662020-11-30 13:15:58 +02007528 intel_uncore_write(&dev_priv->uncore, D_STATE, dstate);
Chris Wilson13a86b82012-04-24 14:51:43 +01007529
Ville Syrjälä9b1e14f2016-10-31 22:37:15 +02007530 if (IS_PINEVIEW(dev_priv))
Jani Nikula5f461662020-11-30 13:15:58 +02007531 intel_uncore_write(&dev_priv->uncore, ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
Daniel Vetter974a3b02012-09-09 11:54:16 +02007532
7533 /* IIR "flip pending" means done if this bit is set */
Jani Nikula5f461662020-11-30 13:15:58 +02007534 intel_uncore_write(&dev_priv->uncore, ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
Ville Syrjälä12fabbcb92014-02-25 15:13:38 +02007535
7536 /* interrupts should cause a wake up from C3 */
Jani Nikula5f461662020-11-30 13:15:58 +02007537 intel_uncore_write(&dev_priv->uncore, INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
Ville Syrjälädbb42742014-02-25 15:13:41 +02007538
7539 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
Jani Nikula5f461662020-11-30 13:15:58 +02007540 intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
Ville Syrjälä10383922014-08-15 01:21:54 +03007541
Jani Nikula5f461662020-11-30 13:15:58 +02007542 intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE,
Ville Syrjälä10383922014-08-15 01:21:54 +03007543 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03007544}
7545
Ville Syrjälä46f16e62016-10-31 22:37:22 +02007546static void i85x_init_clock_gating(struct drm_i915_private *dev_priv)
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03007547{
Jani Nikula5f461662020-11-30 13:15:58 +02007548 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
Ville Syrjälä54e472a2014-02-25 15:13:40 +02007549
7550 /* interrupts should cause a wake up from C3 */
Jani Nikula5f461662020-11-30 13:15:58 +02007551 intel_uncore_write(&dev_priv->uncore, MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) |
Ville Syrjälä54e472a2014-02-25 15:13:40 +02007552 _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
Ville Syrjälä10383922014-08-15 01:21:54 +03007553
Jani Nikula5f461662020-11-30 13:15:58 +02007554 intel_uncore_write(&dev_priv->uncore, MEM_MODE,
Ville Syrjälä10383922014-08-15 01:21:54 +03007555 _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
Ville Syrjälä5cecf502020-07-02 18:37:23 +03007556
7557 /*
7558 * Have FBC ignore 3D activity since we use software
7559 * render tracking, and otherwise a pure 3D workload
7560 * (even if it just renders a single frame and then does
7561 * abosultely nothing) would not allow FBC to recompress
7562 * until a 2D blit occurs.
7563 */
Jani Nikula5f461662020-11-30 13:15:58 +02007564 intel_uncore_write(&dev_priv->uncore, SCPD0,
Ville Syrjälä5cecf502020-07-02 18:37:23 +03007565 _MASKED_BIT_ENABLE(SCPD_FBC_IGNORE_3D));
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03007566}
7567
Ville Syrjälä46f16e62016-10-31 22:37:22 +02007568static void i830_init_clock_gating(struct drm_i915_private *dev_priv)
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03007569{
Jani Nikula5f461662020-11-30 13:15:58 +02007570 intel_uncore_write(&dev_priv->uncore, MEM_MODE,
Ville Syrjälä10383922014-08-15 01:21:54 +03007571 _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) |
7572 _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03007573}
7574
Ville Syrjälä46f16e62016-10-31 22:37:22 +02007575void intel_init_clock_gating(struct drm_i915_private *dev_priv)
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03007576{
Ville Syrjälä46f16e62016-10-31 22:37:22 +02007577 dev_priv->display.init_clock_gating(dev_priv);
Eugeni Dodonov6f1d69b2012-04-18 15:29:25 -03007578}
7579
Ville Syrjälä712bf362016-10-31 22:37:23 +02007580void intel_suspend_hw(struct drm_i915_private *dev_priv)
Imre Deak7d708ee2013-04-17 14:04:50 +03007581{
Ville Syrjälä712bf362016-10-31 22:37:23 +02007582 if (HAS_PCH_LPT(dev_priv))
7583 lpt_suspend_hw(dev_priv);
Imre Deak7d708ee2013-04-17 14:04:50 +03007584}
7585
Ville Syrjälä46f16e62016-10-31 22:37:22 +02007586static void nop_init_clock_gating(struct drm_i915_private *dev_priv)
Imre Deakbb400da2016-03-16 13:38:54 +02007587{
Wambui Karugaf8d18d52020-01-07 18:13:30 +03007588 drm_dbg_kms(&dev_priv->drm,
7589 "No clock gating settings or workarounds applied.\n");
Imre Deakbb400da2016-03-16 13:38:54 +02007590}
7591
7592/**
7593 * intel_init_clock_gating_hooks - setup the clock gating hooks
7594 * @dev_priv: device private
7595 *
7596 * Setup the hooks that configure which clocks of a given platform can be
7597 * gated and also apply various GT and display specific workarounds for these
7598 * platforms. Note that some GT specific workarounds are applied separately
7599 * when GPU contexts or batchbuffers start their execution.
7600 */
7601void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv)
7602{
Stuart Summersda9427502020-10-14 12:19:34 -07007603 if (IS_DG1(dev_priv))
7604 dev_priv->display.init_clock_gating = dg1_init_clock_gating;
7605 else if (IS_GEN(dev_priv, 12))
José Roberto de Souza35f08372021-01-13 05:37:59 -08007606 dev_priv->display.init_clock_gating = gen12lp_init_clock_gating;
Lucas De Marchi13e53c52019-08-17 02:38:42 -07007607 else if (IS_GEN(dev_priv, 11))
Oscar Mateod65dc3e2018-05-08 14:29:24 -07007608 dev_priv->display.init_clock_gating = icl_init_clock_gating;
Oscar Mateocc38cae2018-05-08 14:29:23 -07007609 else if (IS_CANNONLAKE(dev_priv))
Rodrigo Vivi91200c02017-08-28 22:20:26 -07007610 dev_priv->display.init_clock_gating = cnl_init_clock_gating;
Chris Wilson5f4ae272020-06-02 15:05:40 +01007611 else if (IS_COFFEELAKE(dev_priv) || IS_COMETLAKE(dev_priv))
Rodrigo Vivi0a46ddd2017-08-30 21:52:23 -07007612 dev_priv->display.init_clock_gating = cfl_init_clock_gating;
Rodrigo Vivi90007bc2017-08-15 16:16:48 -07007613 else if (IS_SKYLAKE(dev_priv))
Rodrigo Vivi91200c02017-08-28 22:20:26 -07007614 dev_priv->display.init_clock_gating = skl_init_clock_gating;
Rodrigo Vivi0a46ddd2017-08-30 21:52:23 -07007615 else if (IS_KABYLAKE(dev_priv))
Rodrigo Vivi91200c02017-08-28 22:20:26 -07007616 dev_priv->display.init_clock_gating = kbl_init_clock_gating;
Ander Conselvan de Oliveira9fb50262017-01-26 11:16:58 +02007617 else if (IS_BROXTON(dev_priv))
Imre Deakbb400da2016-03-16 13:38:54 +02007618 dev_priv->display.init_clock_gating = bxt_init_clock_gating;
Ander Conselvan de Oliveira9fb50262017-01-26 11:16:58 +02007619 else if (IS_GEMINILAKE(dev_priv))
7620 dev_priv->display.init_clock_gating = glk_init_clock_gating;
Imre Deakbb400da2016-03-16 13:38:54 +02007621 else if (IS_BROADWELL(dev_priv))
Rodrigo Vivi91200c02017-08-28 22:20:26 -07007622 dev_priv->display.init_clock_gating = bdw_init_clock_gating;
Imre Deakbb400da2016-03-16 13:38:54 +02007623 else if (IS_CHERRYVIEW(dev_priv))
Rodrigo Vivi91200c02017-08-28 22:20:26 -07007624 dev_priv->display.init_clock_gating = chv_init_clock_gating;
Imre Deakbb400da2016-03-16 13:38:54 +02007625 else if (IS_HASWELL(dev_priv))
Rodrigo Vivi91200c02017-08-28 22:20:26 -07007626 dev_priv->display.init_clock_gating = hsw_init_clock_gating;
Imre Deakbb400da2016-03-16 13:38:54 +02007627 else if (IS_IVYBRIDGE(dev_priv))
Rodrigo Vivi91200c02017-08-28 22:20:26 -07007628 dev_priv->display.init_clock_gating = ivb_init_clock_gating;
Imre Deakbb400da2016-03-16 13:38:54 +02007629 else if (IS_VALLEYVIEW(dev_priv))
Rodrigo Vivi91200c02017-08-28 22:20:26 -07007630 dev_priv->display.init_clock_gating = vlv_init_clock_gating;
Lucas De Marchicf819ef2018-12-12 10:10:43 -08007631 else if (IS_GEN(dev_priv, 6))
Imre Deakbb400da2016-03-16 13:38:54 +02007632 dev_priv->display.init_clock_gating = gen6_init_clock_gating;
Lucas De Marchicf819ef2018-12-12 10:10:43 -08007633 else if (IS_GEN(dev_priv, 5))
Rodrigo Vivi91200c02017-08-28 22:20:26 -07007634 dev_priv->display.init_clock_gating = ilk_init_clock_gating;
Imre Deakbb400da2016-03-16 13:38:54 +02007635 else if (IS_G4X(dev_priv))
7636 dev_priv->display.init_clock_gating = g4x_init_clock_gating;
Jani Nikulac0f86832016-12-07 12:13:04 +02007637 else if (IS_I965GM(dev_priv))
Rodrigo Vivi91200c02017-08-28 22:20:26 -07007638 dev_priv->display.init_clock_gating = i965gm_init_clock_gating;
Jani Nikulac0f86832016-12-07 12:13:04 +02007639 else if (IS_I965G(dev_priv))
Rodrigo Vivi91200c02017-08-28 22:20:26 -07007640 dev_priv->display.init_clock_gating = i965g_init_clock_gating;
Lucas De Marchicf819ef2018-12-12 10:10:43 -08007641 else if (IS_GEN(dev_priv, 3))
Imre Deakbb400da2016-03-16 13:38:54 +02007642 dev_priv->display.init_clock_gating = gen3_init_clock_gating;
7643 else if (IS_I85X(dev_priv) || IS_I865G(dev_priv))
7644 dev_priv->display.init_clock_gating = i85x_init_clock_gating;
Lucas De Marchicf819ef2018-12-12 10:10:43 -08007645 else if (IS_GEN(dev_priv, 2))
Imre Deakbb400da2016-03-16 13:38:54 +02007646 dev_priv->display.init_clock_gating = i830_init_clock_gating;
7647 else {
7648 MISSING_CASE(INTEL_DEVID(dev_priv));
7649 dev_priv->display.init_clock_gating = nop_init_clock_gating;
7650 }
7651}
7652
Eugeni Dodonov1fa61102012-04-18 15:29:26 -03007653/* Set up chip specific power management-related functions */
Ville Syrjälä62d75df2016-10-31 22:37:25 +02007654void intel_init_pm(struct drm_i915_private *dev_priv)
Eugeni Dodonov1fa61102012-04-18 15:29:26 -03007655{
Daniel Vetterc921aba2012-04-26 23:28:17 +02007656 /* For cxsr */
Ville Syrjälä9b1e14f2016-10-31 22:37:15 +02007657 if (IS_PINEVIEW(dev_priv))
Lucas De Marchi1d218222019-12-24 00:40:04 -08007658 pnv_get_mem_freq(dev_priv);
Lucas De Marchicf819ef2018-12-12 10:10:43 -08007659 else if (IS_GEN(dev_priv, 5))
Lucas De Marchi9eae5e22019-12-24 00:40:09 -08007660 ilk_get_mem_freq(dev_priv);
Daniel Vetterc921aba2012-04-26 23:28:17 +02007661
James Ausmusb068a862019-10-09 10:23:14 -07007662 if (intel_has_sagv(dev_priv))
7663 skl_setup_sagv_block_time(dev_priv);
7664
Eugeni Dodonov1fa61102012-04-18 15:29:26 -03007665 /* For FIFO watermark updates */
Ville Syrjälä62d75df2016-10-31 22:37:25 +02007666 if (INTEL_GEN(dev_priv) >= 9) {
Ville Syrjäläbb726512016-10-31 22:37:24 +02007667 skl_setup_wm_latency(dev_priv);
Matt Roper98d39492016-05-12 07:06:03 -07007668 dev_priv->display.compute_global_watermarks = skl_compute_wm;
Tvrtko Ursulin6e266952016-10-13 11:02:53 +01007669 } else if (HAS_PCH_SPLIT(dev_priv)) {
Ville Syrjäläbb726512016-10-31 22:37:24 +02007670 ilk_setup_wm_latency(dev_priv);
Ville Syrjälä53615a52013-08-01 16:18:50 +03007671
Lucas De Marchicf819ef2018-12-12 10:10:43 -08007672 if ((IS_GEN(dev_priv, 5) && dev_priv->wm.pri_latency[1] &&
Ville Syrjäläbd602542014-01-07 16:14:10 +02007673 dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) ||
Lucas De Marchicf819ef2018-12-12 10:10:43 -08007674 (!IS_GEN(dev_priv, 5) && dev_priv->wm.pri_latency[0] &&
Ville Syrjäläbd602542014-01-07 16:14:10 +02007675 dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) {
Matt Roper86c8bbb2015-09-24 15:53:16 -07007676 dev_priv->display.compute_pipe_wm = ilk_compute_pipe_wm;
Matt Ropered4a6a72016-02-23 17:20:13 -08007677 dev_priv->display.compute_intermediate_wm =
7678 ilk_compute_intermediate_wm;
7679 dev_priv->display.initial_watermarks =
7680 ilk_initial_watermarks;
7681 dev_priv->display.optimize_watermarks =
7682 ilk_optimize_watermarks;
Ville Syrjäläbd602542014-01-07 16:14:10 +02007683 } else {
Wambui Karugaf8d18d52020-01-07 18:13:30 +03007684 drm_dbg_kms(&dev_priv->drm,
7685 "Failed to read display plane latency. "
7686 "Disable CxSR\n");
Ville Syrjäläbd602542014-01-07 16:14:10 +02007687 }
Ville Syrjälä6b6b3ee2016-11-28 19:37:07 +02007688 } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
Ville Syrjäläbb726512016-10-31 22:37:24 +02007689 vlv_setup_wm_latency(dev_priv);
Ville Syrjäläff32c542017-03-02 19:14:57 +02007690 dev_priv->display.compute_pipe_wm = vlv_compute_pipe_wm;
Ville Syrjälä4841da52017-03-02 19:14:59 +02007691 dev_priv->display.compute_intermediate_wm = vlv_compute_intermediate_wm;
Ville Syrjäläff32c542017-03-02 19:14:57 +02007692 dev_priv->display.initial_watermarks = vlv_initial_watermarks;
Ville Syrjälä4841da52017-03-02 19:14:59 +02007693 dev_priv->display.optimize_watermarks = vlv_optimize_watermarks;
Ville Syrjäläff32c542017-03-02 19:14:57 +02007694 dev_priv->display.atomic_update_watermarks = vlv_atomic_update_fifo;
Ville Syrjälä04548cb2017-04-21 21:14:29 +03007695 } else if (IS_G4X(dev_priv)) {
7696 g4x_setup_wm_latency(dev_priv);
7697 dev_priv->display.compute_pipe_wm = g4x_compute_pipe_wm;
7698 dev_priv->display.compute_intermediate_wm = g4x_compute_intermediate_wm;
7699 dev_priv->display.initial_watermarks = g4x_initial_watermarks;
7700 dev_priv->display.optimize_watermarks = g4x_optimize_watermarks;
Ville Syrjälä9b1e14f2016-10-31 22:37:15 +02007701 } else if (IS_PINEVIEW(dev_priv)) {
Tvrtko Ursulin86d35d42019-03-26 07:40:54 +00007702 if (!intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
Eugeni Dodonov1fa61102012-04-18 15:29:26 -03007703 dev_priv->is_ddr3,
7704 dev_priv->fsb_freq,
7705 dev_priv->mem_freq)) {
Wambui Karugaf8d18d52020-01-07 18:13:30 +03007706 drm_info(&dev_priv->drm,
7707 "failed to find known CxSR latency "
Eugeni Dodonov1fa61102012-04-18 15:29:26 -03007708 "(found ddr%s fsb freq %d, mem freq %d), "
7709 "disabling CxSR\n",
7710 (dev_priv->is_ddr3 == 1) ? "3" : "2",
7711 dev_priv->fsb_freq, dev_priv->mem_freq);
7712 /* Disable CxSR and never update its watermark again */
Imre Deak5209b1f2014-07-01 12:36:17 +03007713 intel_set_memory_cxsr(dev_priv, false);
Eugeni Dodonov1fa61102012-04-18 15:29:26 -03007714 dev_priv->display.update_wm = NULL;
7715 } else
Lucas De Marchi1d218222019-12-24 00:40:04 -08007716 dev_priv->display.update_wm = pnv_update_wm;
Lucas De Marchicf819ef2018-12-12 10:10:43 -08007717 } else if (IS_GEN(dev_priv, 4)) {
Eugeni Dodonov1fa61102012-04-18 15:29:26 -03007718 dev_priv->display.update_wm = i965_update_wm;
Lucas De Marchicf819ef2018-12-12 10:10:43 -08007719 } else if (IS_GEN(dev_priv, 3)) {
Eugeni Dodonov1fa61102012-04-18 15:29:26 -03007720 dev_priv->display.update_wm = i9xx_update_wm;
7721 dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
Lucas De Marchicf819ef2018-12-12 10:10:43 -08007722 } else if (IS_GEN(dev_priv, 2)) {
Jani Nikula24977872019-09-11 12:26:08 +03007723 if (INTEL_NUM_PIPES(dev_priv) == 1) {
Daniel Vetterfeb56b92013-12-14 20:38:30 -02007724 dev_priv->display.update_wm = i845_update_wm;
Eugeni Dodonov1fa61102012-04-18 15:29:26 -03007725 dev_priv->display.get_fifo_size = i845_get_fifo_size;
Daniel Vetterfeb56b92013-12-14 20:38:30 -02007726 } else {
7727 dev_priv->display.update_wm = i9xx_update_wm;
Eugeni Dodonov1fa61102012-04-18 15:29:26 -03007728 dev_priv->display.get_fifo_size = i830_get_fifo_size;
Daniel Vetterfeb56b92013-12-14 20:38:30 -02007729 }
Daniel Vetterfeb56b92013-12-14 20:38:30 -02007730 } else {
Wambui Karugaf8d18d52020-01-07 18:13:30 +03007731 drm_err(&dev_priv->drm,
7732 "unexpected fall-through in %s\n", __func__);
Eugeni Dodonov1fa61102012-04-18 15:29:26 -03007733 }
7734}
7735
Tvrtko Ursulin192aa182016-12-01 14:16:45 +00007736void intel_pm_setup(struct drm_i915_private *dev_priv)
Chris Wilson907b28c2013-07-19 20:36:52 +01007737{
Sagar Arun Kamblead1443f2017-10-10 22:30:04 +01007738 dev_priv->runtime_pm.suspended = false;
7739 atomic_set(&dev_priv->runtime_pm.wakeref_count, 0);
Chris Wilson907b28c2013-07-19 20:36:52 +01007740}
Ville Syrjälä3cf43cd2020-02-25 19:11:13 +02007741
7742static struct intel_global_state *intel_dbuf_duplicate_state(struct intel_global_obj *obj)
7743{
7744 struct intel_dbuf_state *dbuf_state;
7745
7746 dbuf_state = kmemdup(obj->state, sizeof(*dbuf_state), GFP_KERNEL);
7747 if (!dbuf_state)
7748 return NULL;
7749
7750 return &dbuf_state->base;
7751}
7752
7753static void intel_dbuf_destroy_state(struct intel_global_obj *obj,
7754 struct intel_global_state *state)
7755{
7756 kfree(state);
7757}
7758
7759static const struct intel_global_state_funcs intel_dbuf_funcs = {
7760 .atomic_duplicate_state = intel_dbuf_duplicate_state,
7761 .atomic_destroy_state = intel_dbuf_destroy_state,
7762};
7763
7764struct intel_dbuf_state *
7765intel_atomic_get_dbuf_state(struct intel_atomic_state *state)
7766{
7767 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
7768 struct intel_global_state *dbuf_state;
7769
7770 dbuf_state = intel_atomic_get_global_obj_state(state, &dev_priv->dbuf.obj);
7771 if (IS_ERR(dbuf_state))
7772 return ERR_CAST(dbuf_state);
7773
7774 return to_intel_dbuf_state(dbuf_state);
7775}
7776
7777int intel_dbuf_init(struct drm_i915_private *dev_priv)
7778{
7779 struct intel_dbuf_state *dbuf_state;
7780
7781 dbuf_state = kzalloc(sizeof(*dbuf_state), GFP_KERNEL);
7782 if (!dbuf_state)
7783 return -ENOMEM;
7784
7785 intel_atomic_global_obj_init(dev_priv, &dev_priv->dbuf.obj,
7786 &dbuf_state->base, &intel_dbuf_funcs);
7787
7788 return 0;
7789}
Ville Syrjäläc7c0e7e2020-02-25 19:11:15 +02007790
7791void intel_dbuf_pre_plane_update(struct intel_atomic_state *state)
7792{
7793 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
7794 const struct intel_dbuf_state *new_dbuf_state =
7795 intel_atomic_get_new_dbuf_state(state);
7796 const struct intel_dbuf_state *old_dbuf_state =
7797 intel_atomic_get_old_dbuf_state(state);
7798
7799 if (!new_dbuf_state ||
7800 new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices)
7801 return;
7802
7803 WARN_ON(!new_dbuf_state->base.changed);
7804
7805 gen9_dbuf_slices_update(dev_priv,
7806 old_dbuf_state->enabled_slices |
7807 new_dbuf_state->enabled_slices);
7808}
7809
7810void intel_dbuf_post_plane_update(struct intel_atomic_state *state)
7811{
7812 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
7813 const struct intel_dbuf_state *new_dbuf_state =
7814 intel_atomic_get_new_dbuf_state(state);
7815 const struct intel_dbuf_state *old_dbuf_state =
7816 intel_atomic_get_old_dbuf_state(state);
7817
7818 if (!new_dbuf_state ||
7819 new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices)
7820 return;
7821
7822 WARN_ON(!new_dbuf_state->base.changed);
7823
7824 gen9_dbuf_slices_update(dev_priv,
7825 new_dbuf_state->enabled_slices);
7826}