Gitiles
Code Review Sign In
review.shift-gmbh.com / SHIFTPHONES / kernel / shift / mainline / 8efff33742f5e2ed6184c2dacace5a1fe695909c / . / drivers / gpu / drm / radeon / cik.c
blob: 6ca1f87af87a0b893b6b60030c131c40d58d6033 [file] [log] [blame]
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]1/*
2 * Copyright 2012 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24#include <linux/firmware.h>
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]25#include <linux/slab.h>
26#include <linux/module.h>
27#include "drmP.h"
28#include "radeon.h"
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]29#include "radeon_asic.h"
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]30#include "cikd.h"
31#include "atom.h"
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]32#include "cik_blit_shaders.h"
Alex Deucher8c68e392013-06-21 15:38:37 -0400[diff] [blame]33#include "radeon_ucode.h"
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]34#include "clearstate_ci.h"
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]35
36MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
37MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
38MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
39MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
40MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
41MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]42MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]43MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]44MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
45MODULE_FIRMWARE("radeon/KAVERI_me.bin");
46MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
47MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
48MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]49MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]50MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
51MODULE_FIRMWARE("radeon/KABINI_me.bin");
52MODULE_FIRMWARE("radeon/KABINI_ce.bin");
53MODULE_FIRMWARE("radeon/KABINI_mec.bin");
54MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]55MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]56
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]57extern int r600_ih_ring_alloc(struct radeon_device *rdev);
58extern void r600_ih_ring_fini(struct radeon_device *rdev);
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]59extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
60extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]61extern bool evergreen_is_display_hung(struct radeon_device *rdev);
Alex Deucher1fd11772013-04-17 17:53:50 -0400[diff] [blame]62extern void sumo_rlc_fini(struct radeon_device *rdev);
63extern int sumo_rlc_init(struct radeon_device *rdev);
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]64extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]65extern void si_rlc_reset(struct radeon_device *rdev);
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]66extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
Christian König2483b4e2013-08-13 11:56:54 +0200[diff] [blame]67extern int cik_sdma_resume(struct radeon_device *rdev);
68extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
69extern void cik_sdma_fini(struct radeon_device *rdev);
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]70static void cik_rlc_stop(struct radeon_device *rdev);
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]71static void cik_pcie_gen3_enable(struct radeon_device *rdev);
Alex Deucher7235711a42013-04-04 13:58:09 -0400[diff] [blame]72static void cik_program_aspm(struct radeon_device *rdev);
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]73static void cik_init_pg(struct radeon_device *rdev);
74static void cik_init_cg(struct radeon_device *rdev);
Alex Deucherfb2c7f42013-10-02 14:54:44 -0400[diff] [blame]75static void cik_fini_pg(struct radeon_device *rdev);
76static void cik_fini_cg(struct radeon_device *rdev);
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]77static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
78 bool enable);
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]79
Alex Deucher286d9cc2013-06-21 15:50:47 -0400[diff] [blame]80/* get temperature in millidegrees */
81int ci_get_temp(struct radeon_device *rdev)
82{
83 u32 temp;
84 int actual_temp = 0;
85
86 temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
87 CTF_TEMP_SHIFT;
88
89 if (temp & 0x200)
90 actual_temp = 255;
91 else
92 actual_temp = temp & 0x1ff;
93
94 actual_temp = actual_temp * 1000;
95
96 return actual_temp;
97}
98
99/* get temperature in millidegrees */
100int kv_get_temp(struct radeon_device *rdev)
101{
102 u32 temp;
103 int actual_temp = 0;
104
105 temp = RREG32_SMC(0xC0300E0C);
106
107 if (temp)
108 actual_temp = (temp / 8) - 49;
109 else
110 actual_temp = 0;
111
112 actual_temp = actual_temp * 1000;
113
114 return actual_temp;
115}
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]116
Alex Deucher6e2c3c02013-04-03 19:28:32 -0400[diff] [blame]117/*
118 * Indirect registers accessor
119 */
120u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
121{
Alex Deucher0a5b7b02013-09-03 19:00:09 -0400[diff] [blame]122 unsigned long flags;
Alex Deucher6e2c3c02013-04-03 19:28:32 -0400[diff] [blame]123 u32 r;
124
Alex Deucher0a5b7b02013-09-03 19:00:09 -0400[diff] [blame]125 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
Alex Deucher6e2c3c02013-04-03 19:28:32 -0400[diff] [blame]126 WREG32(PCIE_INDEX, reg);
127 (void)RREG32(PCIE_INDEX);
128 r = RREG32(PCIE_DATA);
Alex Deucher0a5b7b02013-09-03 19:00:09 -0400[diff] [blame]129 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
Alex Deucher6e2c3c02013-04-03 19:28:32 -0400[diff] [blame]130 return r;
131}
132
133void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
134{
Alex Deucher0a5b7b02013-09-03 19:00:09 -0400[diff] [blame]135 unsigned long flags;
136
137 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
Alex Deucher6e2c3c02013-04-03 19:28:32 -0400[diff] [blame]138 WREG32(PCIE_INDEX, reg);
139 (void)RREG32(PCIE_INDEX);
140 WREG32(PCIE_DATA, v);
141 (void)RREG32(PCIE_DATA);
Alex Deucher0a5b7b02013-09-03 19:00:09 -0400[diff] [blame]142 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
Alex Deucher6e2c3c02013-04-03 19:28:32 -0400[diff] [blame]143}
144
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]145static const u32 spectre_rlc_save_restore_register_list[] =
146{
147 (0x0e00 << 16) | (0xc12c >> 2),
148 0x00000000,
149 (0x0e00 << 16) | (0xc140 >> 2),
150 0x00000000,
151 (0x0e00 << 16) | (0xc150 >> 2),
152 0x00000000,
153 (0x0e00 << 16) | (0xc15c >> 2),
154 0x00000000,
155 (0x0e00 << 16) | (0xc168 >> 2),
156 0x00000000,
157 (0x0e00 << 16) | (0xc170 >> 2),
158 0x00000000,
159 (0x0e00 << 16) | (0xc178 >> 2),
160 0x00000000,
161 (0x0e00 << 16) | (0xc204 >> 2),
162 0x00000000,
163 (0x0e00 << 16) | (0xc2b4 >> 2),
164 0x00000000,
165 (0x0e00 << 16) | (0xc2b8 >> 2),
166 0x00000000,
167 (0x0e00 << 16) | (0xc2bc >> 2),
168 0x00000000,
169 (0x0e00 << 16) | (0xc2c0 >> 2),
170 0x00000000,
171 (0x0e00 << 16) | (0x8228 >> 2),
172 0x00000000,
173 (0x0e00 << 16) | (0x829c >> 2),
174 0x00000000,
175 (0x0e00 << 16) | (0x869c >> 2),
176 0x00000000,
177 (0x0600 << 16) | (0x98f4 >> 2),
178 0x00000000,
179 (0x0e00 << 16) | (0x98f8 >> 2),
180 0x00000000,
181 (0x0e00 << 16) | (0x9900 >> 2),
182 0x00000000,
183 (0x0e00 << 16) | (0xc260 >> 2),
184 0x00000000,
185 (0x0e00 << 16) | (0x90e8 >> 2),
186 0x00000000,
187 (0x0e00 << 16) | (0x3c000 >> 2),
188 0x00000000,
189 (0x0e00 << 16) | (0x3c00c >> 2),
190 0x00000000,
191 (0x0e00 << 16) | (0x8c1c >> 2),
192 0x00000000,
193 (0x0e00 << 16) | (0x9700 >> 2),
194 0x00000000,
195 (0x0e00 << 16) | (0xcd20 >> 2),
196 0x00000000,
197 (0x4e00 << 16) | (0xcd20 >> 2),
198 0x00000000,
199 (0x5e00 << 16) | (0xcd20 >> 2),
200 0x00000000,
201 (0x6e00 << 16) | (0xcd20 >> 2),
202 0x00000000,
203 (0x7e00 << 16) | (0xcd20 >> 2),
204 0x00000000,
205 (0x8e00 << 16) | (0xcd20 >> 2),
206 0x00000000,
207 (0x9e00 << 16) | (0xcd20 >> 2),
208 0x00000000,
209 (0xae00 << 16) | (0xcd20 >> 2),
210 0x00000000,
211 (0xbe00 << 16) | (0xcd20 >> 2),
212 0x00000000,
213 (0x0e00 << 16) | (0x89bc >> 2),
214 0x00000000,
215 (0x0e00 << 16) | (0x8900 >> 2),
216 0x00000000,
217 0x3,
218 (0x0e00 << 16) | (0xc130 >> 2),
219 0x00000000,
220 (0x0e00 << 16) | (0xc134 >> 2),
221 0x00000000,
222 (0x0e00 << 16) | (0xc1fc >> 2),
223 0x00000000,
224 (0x0e00 << 16) | (0xc208 >> 2),
225 0x00000000,
226 (0x0e00 << 16) | (0xc264 >> 2),
227 0x00000000,
228 (0x0e00 << 16) | (0xc268 >> 2),
229 0x00000000,
230 (0x0e00 << 16) | (0xc26c >> 2),
231 0x00000000,
232 (0x0e00 << 16) | (0xc270 >> 2),
233 0x00000000,
234 (0x0e00 << 16) | (0xc274 >> 2),
235 0x00000000,
236 (0x0e00 << 16) | (0xc278 >> 2),
237 0x00000000,
238 (0x0e00 << 16) | (0xc27c >> 2),
239 0x00000000,
240 (0x0e00 << 16) | (0xc280 >> 2),
241 0x00000000,
242 (0x0e00 << 16) | (0xc284 >> 2),
243 0x00000000,
244 (0x0e00 << 16) | (0xc288 >> 2),
245 0x00000000,
246 (0x0e00 << 16) | (0xc28c >> 2),
247 0x00000000,
248 (0x0e00 << 16) | (0xc290 >> 2),
249 0x00000000,
250 (0x0e00 << 16) | (0xc294 >> 2),
251 0x00000000,
252 (0x0e00 << 16) | (0xc298 >> 2),
253 0x00000000,
254 (0x0e00 << 16) | (0xc29c >> 2),
255 0x00000000,
256 (0x0e00 << 16) | (0xc2a0 >> 2),
257 0x00000000,
258 (0x0e00 << 16) | (0xc2a4 >> 2),
259 0x00000000,
260 (0x0e00 << 16) | (0xc2a8 >> 2),
261 0x00000000,
262 (0x0e00 << 16) | (0xc2ac >> 2),
263 0x00000000,
264 (0x0e00 << 16) | (0xc2b0 >> 2),
265 0x00000000,
266 (0x0e00 << 16) | (0x301d0 >> 2),
267 0x00000000,
268 (0x0e00 << 16) | (0x30238 >> 2),
269 0x00000000,
270 (0x0e00 << 16) | (0x30250 >> 2),
271 0x00000000,
272 (0x0e00 << 16) | (0x30254 >> 2),
273 0x00000000,
274 (0x0e00 << 16) | (0x30258 >> 2),
275 0x00000000,
276 (0x0e00 << 16) | (0x3025c >> 2),
277 0x00000000,
278 (0x4e00 << 16) | (0xc900 >> 2),
279 0x00000000,
280 (0x5e00 << 16) | (0xc900 >> 2),
281 0x00000000,
282 (0x6e00 << 16) | (0xc900 >> 2),
283 0x00000000,
284 (0x7e00 << 16) | (0xc900 >> 2),
285 0x00000000,
286 (0x8e00 << 16) | (0xc900 >> 2),
287 0x00000000,
288 (0x9e00 << 16) | (0xc900 >> 2),
289 0x00000000,
290 (0xae00 << 16) | (0xc900 >> 2),
291 0x00000000,
292 (0xbe00 << 16) | (0xc900 >> 2),
293 0x00000000,
294 (0x4e00 << 16) | (0xc904 >> 2),
295 0x00000000,
296 (0x5e00 << 16) | (0xc904 >> 2),
297 0x00000000,
298 (0x6e00 << 16) | (0xc904 >> 2),
299 0x00000000,
300 (0x7e00 << 16) | (0xc904 >> 2),
301 0x00000000,
302 (0x8e00 << 16) | (0xc904 >> 2),
303 0x00000000,
304 (0x9e00 << 16) | (0xc904 >> 2),
305 0x00000000,
306 (0xae00 << 16) | (0xc904 >> 2),
307 0x00000000,
308 (0xbe00 << 16) | (0xc904 >> 2),
309 0x00000000,
310 (0x4e00 << 16) | (0xc908 >> 2),
311 0x00000000,
312 (0x5e00 << 16) | (0xc908 >> 2),
313 0x00000000,
314 (0x6e00 << 16) | (0xc908 >> 2),
315 0x00000000,
316 (0x7e00 << 16) | (0xc908 >> 2),
317 0x00000000,
318 (0x8e00 << 16) | (0xc908 >> 2),
319 0x00000000,
320 (0x9e00 << 16) | (0xc908 >> 2),
321 0x00000000,
322 (0xae00 << 16) | (0xc908 >> 2),
323 0x00000000,
324 (0xbe00 << 16) | (0xc908 >> 2),
325 0x00000000,
326 (0x4e00 << 16) | (0xc90c >> 2),
327 0x00000000,
328 (0x5e00 << 16) | (0xc90c >> 2),
329 0x00000000,
330 (0x6e00 << 16) | (0xc90c >> 2),
331 0x00000000,
332 (0x7e00 << 16) | (0xc90c >> 2),
333 0x00000000,
334 (0x8e00 << 16) | (0xc90c >> 2),
335 0x00000000,
336 (0x9e00 << 16) | (0xc90c >> 2),
337 0x00000000,
338 (0xae00 << 16) | (0xc90c >> 2),
339 0x00000000,
340 (0xbe00 << 16) | (0xc90c >> 2),
341 0x00000000,
342 (0x4e00 << 16) | (0xc910 >> 2),
343 0x00000000,
344 (0x5e00 << 16) | (0xc910 >> 2),
345 0x00000000,
346 (0x6e00 << 16) | (0xc910 >> 2),
347 0x00000000,
348 (0x7e00 << 16) | (0xc910 >> 2),
349 0x00000000,
350 (0x8e00 << 16) | (0xc910 >> 2),
351 0x00000000,
352 (0x9e00 << 16) | (0xc910 >> 2),
353 0x00000000,
354 (0xae00 << 16) | (0xc910 >> 2),
355 0x00000000,
356 (0xbe00 << 16) | (0xc910 >> 2),
357 0x00000000,
358 (0x0e00 << 16) | (0xc99c >> 2),
359 0x00000000,
360 (0x0e00 << 16) | (0x9834 >> 2),
361 0x00000000,
362 (0x0000 << 16) | (0x30f00 >> 2),
363 0x00000000,
364 (0x0001 << 16) | (0x30f00 >> 2),
365 0x00000000,
366 (0x0000 << 16) | (0x30f04 >> 2),
367 0x00000000,
368 (0x0001 << 16) | (0x30f04 >> 2),
369 0x00000000,
370 (0x0000 << 16) | (0x30f08 >> 2),
371 0x00000000,
372 (0x0001 << 16) | (0x30f08 >> 2),
373 0x00000000,
374 (0x0000 << 16) | (0x30f0c >> 2),
375 0x00000000,
376 (0x0001 << 16) | (0x30f0c >> 2),
377 0x00000000,
378 (0x0600 << 16) | (0x9b7c >> 2),
379 0x00000000,
380 (0x0e00 << 16) | (0x8a14 >> 2),
381 0x00000000,
382 (0x0e00 << 16) | (0x8a18 >> 2),
383 0x00000000,
384 (0x0600 << 16) | (0x30a00 >> 2),
385 0x00000000,
386 (0x0e00 << 16) | (0x8bf0 >> 2),
387 0x00000000,
388 (0x0e00 << 16) | (0x8bcc >> 2),
389 0x00000000,
390 (0x0e00 << 16) | (0x8b24 >> 2),
391 0x00000000,
392 (0x0e00 << 16) | (0x30a04 >> 2),
393 0x00000000,
394 (0x0600 << 16) | (0x30a10 >> 2),
395 0x00000000,
396 (0x0600 << 16) | (0x30a14 >> 2),
397 0x00000000,
398 (0x0600 << 16) | (0x30a18 >> 2),
399 0x00000000,
400 (0x0600 << 16) | (0x30a2c >> 2),
401 0x00000000,
402 (0x0e00 << 16) | (0xc700 >> 2),
403 0x00000000,
404 (0x0e00 << 16) | (0xc704 >> 2),
405 0x00000000,
406 (0x0e00 << 16) | (0xc708 >> 2),
407 0x00000000,
408 (0x0e00 << 16) | (0xc768 >> 2),
409 0x00000000,
410 (0x0400 << 16) | (0xc770 >> 2),
411 0x00000000,
412 (0x0400 << 16) | (0xc774 >> 2),
413 0x00000000,
414 (0x0400 << 16) | (0xc778 >> 2),
415 0x00000000,
416 (0x0400 << 16) | (0xc77c >> 2),
417 0x00000000,
418 (0x0400 << 16) | (0xc780 >> 2),
419 0x00000000,
420 (0x0400 << 16) | (0xc784 >> 2),
421 0x00000000,
422 (0x0400 << 16) | (0xc788 >> 2),
423 0x00000000,
424 (0x0400 << 16) | (0xc78c >> 2),
425 0x00000000,
426 (0x0400 << 16) | (0xc798 >> 2),
427 0x00000000,
428 (0x0400 << 16) | (0xc79c >> 2),
429 0x00000000,
430 (0x0400 << 16) | (0xc7a0 >> 2),
431 0x00000000,
432 (0x0400 << 16) | (0xc7a4 >> 2),
433 0x00000000,
434 (0x0400 << 16) | (0xc7a8 >> 2),
435 0x00000000,
436 (0x0400 << 16) | (0xc7ac >> 2),
437 0x00000000,
438 (0x0400 << 16) | (0xc7b0 >> 2),
439 0x00000000,
440 (0x0400 << 16) | (0xc7b4 >> 2),
441 0x00000000,
442 (0x0e00 << 16) | (0x9100 >> 2),
443 0x00000000,
444 (0x0e00 << 16) | (0x3c010 >> 2),
445 0x00000000,
446 (0x0e00 << 16) | (0x92a8 >> 2),
447 0x00000000,
448 (0x0e00 << 16) | (0x92ac >> 2),
449 0x00000000,
450 (0x0e00 << 16) | (0x92b4 >> 2),
451 0x00000000,
452 (0x0e00 << 16) | (0x92b8 >> 2),
453 0x00000000,
454 (0x0e00 << 16) | (0x92bc >> 2),
455 0x00000000,
456 (0x0e00 << 16) | (0x92c0 >> 2),
457 0x00000000,
458 (0x0e00 << 16) | (0x92c4 >> 2),
459 0x00000000,
460 (0x0e00 << 16) | (0x92c8 >> 2),
461 0x00000000,
462 (0x0e00 << 16) | (0x92cc >> 2),
463 0x00000000,
464 (0x0e00 << 16) | (0x92d0 >> 2),
465 0x00000000,
466 (0x0e00 << 16) | (0x8c00 >> 2),
467 0x00000000,
468 (0x0e00 << 16) | (0x8c04 >> 2),
469 0x00000000,
470 (0x0e00 << 16) | (0x8c20 >> 2),
471 0x00000000,
472 (0x0e00 << 16) | (0x8c38 >> 2),
473 0x00000000,
474 (0x0e00 << 16) | (0x8c3c >> 2),
475 0x00000000,
476 (0x0e00 << 16) | (0xae00 >> 2),
477 0x00000000,
478 (0x0e00 << 16) | (0x9604 >> 2),
479 0x00000000,
480 (0x0e00 << 16) | (0xac08 >> 2),
481 0x00000000,
482 (0x0e00 << 16) | (0xac0c >> 2),
483 0x00000000,
484 (0x0e00 << 16) | (0xac10 >> 2),
485 0x00000000,
486 (0x0e00 << 16) | (0xac14 >> 2),
487 0x00000000,
488 (0x0e00 << 16) | (0xac58 >> 2),
489 0x00000000,
490 (0x0e00 << 16) | (0xac68 >> 2),
491 0x00000000,
492 (0x0e00 << 16) | (0xac6c >> 2),
493 0x00000000,
494 (0x0e00 << 16) | (0xac70 >> 2),
495 0x00000000,
496 (0x0e00 << 16) | (0xac74 >> 2),
497 0x00000000,
498 (0x0e00 << 16) | (0xac78 >> 2),
499 0x00000000,
500 (0x0e00 << 16) | (0xac7c >> 2),
501 0x00000000,
502 (0x0e00 << 16) | (0xac80 >> 2),
503 0x00000000,
504 (0x0e00 << 16) | (0xac84 >> 2),
505 0x00000000,
506 (0x0e00 << 16) | (0xac88 >> 2),
507 0x00000000,
508 (0x0e00 << 16) | (0xac8c >> 2),
509 0x00000000,
510 (0x0e00 << 16) | (0x970c >> 2),
511 0x00000000,
512 (0x0e00 << 16) | (0x9714 >> 2),
513 0x00000000,
514 (0x0e00 << 16) | (0x9718 >> 2),
515 0x00000000,
516 (0x0e00 << 16) | (0x971c >> 2),
517 0x00000000,
518 (0x0e00 << 16) | (0x31068 >> 2),
519 0x00000000,
520 (0x4e00 << 16) | (0x31068 >> 2),
521 0x00000000,
522 (0x5e00 << 16) | (0x31068 >> 2),
523 0x00000000,
524 (0x6e00 << 16) | (0x31068 >> 2),
525 0x00000000,
526 (0x7e00 << 16) | (0x31068 >> 2),
527 0x00000000,
528 (0x8e00 << 16) | (0x31068 >> 2),
529 0x00000000,
530 (0x9e00 << 16) | (0x31068 >> 2),
531 0x00000000,
532 (0xae00 << 16) | (0x31068 >> 2),
533 0x00000000,
534 (0xbe00 << 16) | (0x31068 >> 2),
535 0x00000000,
536 (0x0e00 << 16) | (0xcd10 >> 2),
537 0x00000000,
538 (0x0e00 << 16) | (0xcd14 >> 2),
539 0x00000000,
540 (0x0e00 << 16) | (0x88b0 >> 2),
541 0x00000000,
542 (0x0e00 << 16) | (0x88b4 >> 2),
543 0x00000000,
544 (0x0e00 << 16) | (0x88b8 >> 2),
545 0x00000000,
546 (0x0e00 << 16) | (0x88bc >> 2),
547 0x00000000,
548 (0x0400 << 16) | (0x89c0 >> 2),
549 0x00000000,
550 (0x0e00 << 16) | (0x88c4 >> 2),
551 0x00000000,
552 (0x0e00 << 16) | (0x88c8 >> 2),
553 0x00000000,
554 (0x0e00 << 16) | (0x88d0 >> 2),
555 0x00000000,
556 (0x0e00 << 16) | (0x88d4 >> 2),
557 0x00000000,
558 (0x0e00 << 16) | (0x88d8 >> 2),
559 0x00000000,
560 (0x0e00 << 16) | (0x8980 >> 2),
561 0x00000000,
562 (0x0e00 << 16) | (0x30938 >> 2),
563 0x00000000,
564 (0x0e00 << 16) | (0x3093c >> 2),
565 0x00000000,
566 (0x0e00 << 16) | (0x30940 >> 2),
567 0x00000000,
568 (0x0e00 << 16) | (0x89a0 >> 2),
569 0x00000000,
570 (0x0e00 << 16) | (0x30900 >> 2),
571 0x00000000,
572 (0x0e00 << 16) | (0x30904 >> 2),
573 0x00000000,
574 (0x0e00 << 16) | (0x89b4 >> 2),
575 0x00000000,
576 (0x0e00 << 16) | (0x3c210 >> 2),
577 0x00000000,
578 (0x0e00 << 16) | (0x3c214 >> 2),
579 0x00000000,
580 (0x0e00 << 16) | (0x3c218 >> 2),
581 0x00000000,
582 (0x0e00 << 16) | (0x8904 >> 2),
583 0x00000000,
584 0x5,
585 (0x0e00 << 16) | (0x8c28 >> 2),
586 (0x0e00 << 16) | (0x8c2c >> 2),
587 (0x0e00 << 16) | (0x8c30 >> 2),
588 (0x0e00 << 16) | (0x8c34 >> 2),
589 (0x0e00 << 16) | (0x9600 >> 2),
590};
591
592static const u32 kalindi_rlc_save_restore_register_list[] =
593{
594 (0x0e00 << 16) | (0xc12c >> 2),
595 0x00000000,
596 (0x0e00 << 16) | (0xc140 >> 2),
597 0x00000000,
598 (0x0e00 << 16) | (0xc150 >> 2),
599 0x00000000,
600 (0x0e00 << 16) | (0xc15c >> 2),
601 0x00000000,
602 (0x0e00 << 16) | (0xc168 >> 2),
603 0x00000000,
604 (0x0e00 << 16) | (0xc170 >> 2),
605 0x00000000,
606 (0x0e00 << 16) | (0xc204 >> 2),
607 0x00000000,
608 (0x0e00 << 16) | (0xc2b4 >> 2),
609 0x00000000,
610 (0x0e00 << 16) | (0xc2b8 >> 2),
611 0x00000000,
612 (0x0e00 << 16) | (0xc2bc >> 2),
613 0x00000000,
614 (0x0e00 << 16) | (0xc2c0 >> 2),
615 0x00000000,
616 (0x0e00 << 16) | (0x8228 >> 2),
617 0x00000000,
618 (0x0e00 << 16) | (0x829c >> 2),
619 0x00000000,
620 (0x0e00 << 16) | (0x869c >> 2),
621 0x00000000,
622 (0x0600 << 16) | (0x98f4 >> 2),
623 0x00000000,
624 (0x0e00 << 16) | (0x98f8 >> 2),
625 0x00000000,
626 (0x0e00 << 16) | (0x9900 >> 2),
627 0x00000000,
628 (0x0e00 << 16) | (0xc260 >> 2),
629 0x00000000,
630 (0x0e00 << 16) | (0x90e8 >> 2),
631 0x00000000,
632 (0x0e00 << 16) | (0x3c000 >> 2),
633 0x00000000,
634 (0x0e00 << 16) | (0x3c00c >> 2),
635 0x00000000,
636 (0x0e00 << 16) | (0x8c1c >> 2),
637 0x00000000,
638 (0x0e00 << 16) | (0x9700 >> 2),
639 0x00000000,
640 (0x0e00 << 16) | (0xcd20 >> 2),
641 0x00000000,
642 (0x4e00 << 16) | (0xcd20 >> 2),
643 0x00000000,
644 (0x5e00 << 16) | (0xcd20 >> 2),
645 0x00000000,
646 (0x6e00 << 16) | (0xcd20 >> 2),
647 0x00000000,
648 (0x7e00 << 16) | (0xcd20 >> 2),
649 0x00000000,
650 (0x0e00 << 16) | (0x89bc >> 2),
651 0x00000000,
652 (0x0e00 << 16) | (0x8900 >> 2),
653 0x00000000,
654 0x3,
655 (0x0e00 << 16) | (0xc130 >> 2),
656 0x00000000,
657 (0x0e00 << 16) | (0xc134 >> 2),
658 0x00000000,
659 (0x0e00 << 16) | (0xc1fc >> 2),
660 0x00000000,
661 (0x0e00 << 16) | (0xc208 >> 2),
662 0x00000000,
663 (0x0e00 << 16) | (0xc264 >> 2),
664 0x00000000,
665 (0x0e00 << 16) | (0xc268 >> 2),
666 0x00000000,
667 (0x0e00 << 16) | (0xc26c >> 2),
668 0x00000000,
669 (0x0e00 << 16) | (0xc270 >> 2),
670 0x00000000,
671 (0x0e00 << 16) | (0xc274 >> 2),
672 0x00000000,
673 (0x0e00 << 16) | (0xc28c >> 2),
674 0x00000000,
675 (0x0e00 << 16) | (0xc290 >> 2),
676 0x00000000,
677 (0x0e00 << 16) | (0xc294 >> 2),
678 0x00000000,
679 (0x0e00 << 16) | (0xc298 >> 2),
680 0x00000000,
681 (0x0e00 << 16) | (0xc2a0 >> 2),
682 0x00000000,
683 (0x0e00 << 16) | (0xc2a4 >> 2),
684 0x00000000,
685 (0x0e00 << 16) | (0xc2a8 >> 2),
686 0x00000000,
687 (0x0e00 << 16) | (0xc2ac >> 2),
688 0x00000000,
689 (0x0e00 << 16) | (0x301d0 >> 2),
690 0x00000000,
691 (0x0e00 << 16) | (0x30238 >> 2),
692 0x00000000,
693 (0x0e00 << 16) | (0x30250 >> 2),
694 0x00000000,
695 (0x0e00 << 16) | (0x30254 >> 2),
696 0x00000000,
697 (0x0e00 << 16) | (0x30258 >> 2),
698 0x00000000,
699 (0x0e00 << 16) | (0x3025c >> 2),
700 0x00000000,
701 (0x4e00 << 16) | (0xc900 >> 2),
702 0x00000000,
703 (0x5e00 << 16) | (0xc900 >> 2),
704 0x00000000,
705 (0x6e00 << 16) | (0xc900 >> 2),
706 0x00000000,
707 (0x7e00 << 16) | (0xc900 >> 2),
708 0x00000000,
709 (0x4e00 << 16) | (0xc904 >> 2),
710 0x00000000,
711 (0x5e00 << 16) | (0xc904 >> 2),
712 0x00000000,
713 (0x6e00 << 16) | (0xc904 >> 2),
714 0x00000000,
715 (0x7e00 << 16) | (0xc904 >> 2),
716 0x00000000,
717 (0x4e00 << 16) | (0xc908 >> 2),
718 0x00000000,
719 (0x5e00 << 16) | (0xc908 >> 2),
720 0x00000000,
721 (0x6e00 << 16) | (0xc908 >> 2),
722 0x00000000,
723 (0x7e00 << 16) | (0xc908 >> 2),
724 0x00000000,
725 (0x4e00 << 16) | (0xc90c >> 2),
726 0x00000000,
727 (0x5e00 << 16) | (0xc90c >> 2),
728 0x00000000,
729 (0x6e00 << 16) | (0xc90c >> 2),
730 0x00000000,
731 (0x7e00 << 16) | (0xc90c >> 2),
732 0x00000000,
733 (0x4e00 << 16) | (0xc910 >> 2),
734 0x00000000,
735 (0x5e00 << 16) | (0xc910 >> 2),
736 0x00000000,
737 (0x6e00 << 16) | (0xc910 >> 2),
738 0x00000000,
739 (0x7e00 << 16) | (0xc910 >> 2),
740 0x00000000,
741 (0x0e00 << 16) | (0xc99c >> 2),
742 0x00000000,
743 (0x0e00 << 16) | (0x9834 >> 2),
744 0x00000000,
745 (0x0000 << 16) | (0x30f00 >> 2),
746 0x00000000,
747 (0x0000 << 16) | (0x30f04 >> 2),
748 0x00000000,
749 (0x0000 << 16) | (0x30f08 >> 2),
750 0x00000000,
751 (0x0000 << 16) | (0x30f0c >> 2),
752 0x00000000,
753 (0x0600 << 16) | (0x9b7c >> 2),
754 0x00000000,
755 (0x0e00 << 16) | (0x8a14 >> 2),
756 0x00000000,
757 (0x0e00 << 16) | (0x8a18 >> 2),
758 0x00000000,
759 (0x0600 << 16) | (0x30a00 >> 2),
760 0x00000000,
761 (0x0e00 << 16) | (0x8bf0 >> 2),
762 0x00000000,
763 (0x0e00 << 16) | (0x8bcc >> 2),
764 0x00000000,
765 (0x0e00 << 16) | (0x8b24 >> 2),
766 0x00000000,
767 (0x0e00 << 16) | (0x30a04 >> 2),
768 0x00000000,
769 (0x0600 << 16) | (0x30a10 >> 2),
770 0x00000000,
771 (0x0600 << 16) | (0x30a14 >> 2),
772 0x00000000,
773 (0x0600 << 16) | (0x30a18 >> 2),
774 0x00000000,
775 (0x0600 << 16) | (0x30a2c >> 2),
776 0x00000000,
777 (0x0e00 << 16) | (0xc700 >> 2),
778 0x00000000,
779 (0x0e00 << 16) | (0xc704 >> 2),
780 0x00000000,
781 (0x0e00 << 16) | (0xc708 >> 2),
782 0x00000000,
783 (0x0e00 << 16) | (0xc768 >> 2),
784 0x00000000,
785 (0x0400 << 16) | (0xc770 >> 2),
786 0x00000000,
787 (0x0400 << 16) | (0xc774 >> 2),
788 0x00000000,
789 (0x0400 << 16) | (0xc798 >> 2),
790 0x00000000,
791 (0x0400 << 16) | (0xc79c >> 2),
792 0x00000000,
793 (0x0e00 << 16) | (0x9100 >> 2),
794 0x00000000,
795 (0x0e00 << 16) | (0x3c010 >> 2),
796 0x00000000,
797 (0x0e00 << 16) | (0x8c00 >> 2),
798 0x00000000,
799 (0x0e00 << 16) | (0x8c04 >> 2),
800 0x00000000,
801 (0x0e00 << 16) | (0x8c20 >> 2),
802 0x00000000,
803 (0x0e00 << 16) | (0x8c38 >> 2),
804 0x00000000,
805 (0x0e00 << 16) | (0x8c3c >> 2),
806 0x00000000,
807 (0x0e00 << 16) | (0xae00 >> 2),
808 0x00000000,
809 (0x0e00 << 16) | (0x9604 >> 2),
810 0x00000000,
811 (0x0e00 << 16) | (0xac08 >> 2),
812 0x00000000,
813 (0x0e00 << 16) | (0xac0c >> 2),
814 0x00000000,
815 (0x0e00 << 16) | (0xac10 >> 2),
816 0x00000000,
817 (0x0e00 << 16) | (0xac14 >> 2),
818 0x00000000,
819 (0x0e00 << 16) | (0xac58 >> 2),
820 0x00000000,
821 (0x0e00 << 16) | (0xac68 >> 2),
822 0x00000000,
823 (0x0e00 << 16) | (0xac6c >> 2),
824 0x00000000,
825 (0x0e00 << 16) | (0xac70 >> 2),
826 0x00000000,
827 (0x0e00 << 16) | (0xac74 >> 2),
828 0x00000000,
829 (0x0e00 << 16) | (0xac78 >> 2),
830 0x00000000,
831 (0x0e00 << 16) | (0xac7c >> 2),
832 0x00000000,
833 (0x0e00 << 16) | (0xac80 >> 2),
834 0x00000000,
835 (0x0e00 << 16) | (0xac84 >> 2),
836 0x00000000,
837 (0x0e00 << 16) | (0xac88 >> 2),
838 0x00000000,
839 (0x0e00 << 16) | (0xac8c >> 2),
840 0x00000000,
841 (0x0e00 << 16) | (0x970c >> 2),
842 0x00000000,
843 (0x0e00 << 16) | (0x9714 >> 2),
844 0x00000000,
845 (0x0e00 << 16) | (0x9718 >> 2),
846 0x00000000,
847 (0x0e00 << 16) | (0x971c >> 2),
848 0x00000000,
849 (0x0e00 << 16) | (0x31068 >> 2),
850 0x00000000,
851 (0x4e00 << 16) | (0x31068 >> 2),
852 0x00000000,
853 (0x5e00 << 16) | (0x31068 >> 2),
854 0x00000000,
855 (0x6e00 << 16) | (0x31068 >> 2),
856 0x00000000,
857 (0x7e00 << 16) | (0x31068 >> 2),
858 0x00000000,
859 (0x0e00 << 16) | (0xcd10 >> 2),
860 0x00000000,
861 (0x0e00 << 16) | (0xcd14 >> 2),
862 0x00000000,
863 (0x0e00 << 16) | (0x88b0 >> 2),
864 0x00000000,
865 (0x0e00 << 16) | (0x88b4 >> 2),
866 0x00000000,
867 (0x0e00 << 16) | (0x88b8 >> 2),
868 0x00000000,
869 (0x0e00 << 16) | (0x88bc >> 2),
870 0x00000000,
871 (0x0400 << 16) | (0x89c0 >> 2),
872 0x00000000,
873 (0x0e00 << 16) | (0x88c4 >> 2),
874 0x00000000,
875 (0x0e00 << 16) | (0x88c8 >> 2),
876 0x00000000,
877 (0x0e00 << 16) | (0x88d0 >> 2),
878 0x00000000,
879 (0x0e00 << 16) | (0x88d4 >> 2),
880 0x00000000,
881 (0x0e00 << 16) | (0x88d8 >> 2),
882 0x00000000,
883 (0x0e00 << 16) | (0x8980 >> 2),
884 0x00000000,
885 (0x0e00 << 16) | (0x30938 >> 2),
886 0x00000000,
887 (0x0e00 << 16) | (0x3093c >> 2),
888 0x00000000,
889 (0x0e00 << 16) | (0x30940 >> 2),
890 0x00000000,
891 (0x0e00 << 16) | (0x89a0 >> 2),
892 0x00000000,
893 (0x0e00 << 16) | (0x30900 >> 2),
894 0x00000000,
895 (0x0e00 << 16) | (0x30904 >> 2),
896 0x00000000,
897 (0x0e00 << 16) | (0x89b4 >> 2),
898 0x00000000,
899 (0x0e00 << 16) | (0x3e1fc >> 2),
900 0x00000000,
901 (0x0e00 << 16) | (0x3c210 >> 2),
902 0x00000000,
903 (0x0e00 << 16) | (0x3c214 >> 2),
904 0x00000000,
905 (0x0e00 << 16) | (0x3c218 >> 2),
906 0x00000000,
907 (0x0e00 << 16) | (0x8904 >> 2),
908 0x00000000,
909 0x5,
910 (0x0e00 << 16) | (0x8c28 >> 2),
911 (0x0e00 << 16) | (0x8c2c >> 2),
912 (0x0e00 << 16) | (0x8c30 >> 2),
913 (0x0e00 << 16) | (0x8c34 >> 2),
914 (0x0e00 << 16) | (0x9600 >> 2),
915};
916
Alex Deucher0aafd312013-04-09 14:43:30 -0400[diff] [blame]917static const u32 bonaire_golden_spm_registers[] =
918{
919 0x30800, 0xe0ffffff, 0xe0000000
920};
921
922static const u32 bonaire_golden_common_registers[] =
923{
924 0xc770, 0xffffffff, 0x00000800,
925 0xc774, 0xffffffff, 0x00000800,
926 0xc798, 0xffffffff, 0x00007fbf,
927 0xc79c, 0xffffffff, 0x00007faf
928};
929
930static const u32 bonaire_golden_registers[] =
931{
932 0x3354, 0x00000333, 0x00000333,
933 0x3350, 0x000c0fc0, 0x00040200,
934 0x9a10, 0x00010000, 0x00058208,
935 0x3c000, 0xffff1fff, 0x00140000,
936 0x3c200, 0xfdfc0fff, 0x00000100,
937 0x3c234, 0x40000000, 0x40000200,
938 0x9830, 0xffffffff, 0x00000000,
939 0x9834, 0xf00fffff, 0x00000400,
940 0x9838, 0x0002021c, 0x00020200,
941 0xc78, 0x00000080, 0x00000000,
942 0x5bb0, 0x000000f0, 0x00000070,
943 0x5bc0, 0xf0311fff, 0x80300000,
944 0x98f8, 0x73773777, 0x12010001,
945 0x350c, 0x00810000, 0x408af000,
946 0x7030, 0x31000111, 0x00000011,
947 0x2f48, 0x73773777, 0x12010001,
948 0x220c, 0x00007fb6, 0x0021a1b1,
949 0x2210, 0x00007fb6, 0x002021b1,
950 0x2180, 0x00007fb6, 0x00002191,
951 0x2218, 0x00007fb6, 0x002121b1,
952 0x221c, 0x00007fb6, 0x002021b1,
953 0x21dc, 0x00007fb6, 0x00002191,
954 0x21e0, 0x00007fb6, 0x00002191,
955 0x3628, 0x0000003f, 0x0000000a,
956 0x362c, 0x0000003f, 0x0000000a,
957 0x2ae4, 0x00073ffe, 0x000022a2,
958 0x240c, 0x000007ff, 0x00000000,
959 0x8a14, 0xf000003f, 0x00000007,
960 0x8bf0, 0x00002001, 0x00000001,
961 0x8b24, 0xffffffff, 0x00ffffff,
962 0x30a04, 0x0000ff0f, 0x00000000,
963 0x28a4c, 0x07ffffff, 0x06000000,
964 0x4d8, 0x00000fff, 0x00000100,
965 0x3e78, 0x00000001, 0x00000002,
966 0x9100, 0x03000000, 0x0362c688,
967 0x8c00, 0x000000ff, 0x00000001,
968 0xe40, 0x00001fff, 0x00001fff,
969 0x9060, 0x0000007f, 0x00000020,
970 0x9508, 0x00010000, 0x00010000,
971 0xac14, 0x000003ff, 0x000000f3,
972 0xac0c, 0xffffffff, 0x00001032
973};
974
975static const u32 bonaire_mgcg_cgcg_init[] =
976{
977 0xc420, 0xffffffff, 0xfffffffc,
978 0x30800, 0xffffffff, 0xe0000000,
979 0x3c2a0, 0xffffffff, 0x00000100,
980 0x3c208, 0xffffffff, 0x00000100,
981 0x3c2c0, 0xffffffff, 0xc0000100,
982 0x3c2c8, 0xffffffff, 0xc0000100,
983 0x3c2c4, 0xffffffff, 0xc0000100,
984 0x55e4, 0xffffffff, 0x00600100,
985 0x3c280, 0xffffffff, 0x00000100,
986 0x3c214, 0xffffffff, 0x06000100,
987 0x3c220, 0xffffffff, 0x00000100,
988 0x3c218, 0xffffffff, 0x06000100,
989 0x3c204, 0xffffffff, 0x00000100,
990 0x3c2e0, 0xffffffff, 0x00000100,
991 0x3c224, 0xffffffff, 0x00000100,
992 0x3c200, 0xffffffff, 0x00000100,
993 0x3c230, 0xffffffff, 0x00000100,
994 0x3c234, 0xffffffff, 0x00000100,
995 0x3c250, 0xffffffff, 0x00000100,
996 0x3c254, 0xffffffff, 0x00000100,
997 0x3c258, 0xffffffff, 0x00000100,
998 0x3c25c, 0xffffffff, 0x00000100,
999 0x3c260, 0xffffffff, 0x00000100,
1000 0x3c27c, 0xffffffff, 0x00000100,
1001 0x3c278, 0xffffffff, 0x00000100,
1002 0x3c210, 0xffffffff, 0x06000100,
1003 0x3c290, 0xffffffff, 0x00000100,
1004 0x3c274, 0xffffffff, 0x00000100,
1005 0x3c2b4, 0xffffffff, 0x00000100,
1006 0x3c2b0, 0xffffffff, 0x00000100,
1007 0x3c270, 0xffffffff, 0x00000100,
1008 0x30800, 0xffffffff, 0xe0000000,
1009 0x3c020, 0xffffffff, 0x00010000,
1010 0x3c024, 0xffffffff, 0x00030002,
1011 0x3c028, 0xffffffff, 0x00040007,
1012 0x3c02c, 0xffffffff, 0x00060005,
1013 0x3c030, 0xffffffff, 0x00090008,
1014 0x3c034, 0xffffffff, 0x00010000,
1015 0x3c038, 0xffffffff, 0x00030002,
1016 0x3c03c, 0xffffffff, 0x00040007,
1017 0x3c040, 0xffffffff, 0x00060005,
1018 0x3c044, 0xffffffff, 0x00090008,
1019 0x3c048, 0xffffffff, 0x00010000,
1020 0x3c04c, 0xffffffff, 0x00030002,
1021 0x3c050, 0xffffffff, 0x00040007,
1022 0x3c054, 0xffffffff, 0x00060005,
1023 0x3c058, 0xffffffff, 0x00090008,
1024 0x3c05c, 0xffffffff, 0x00010000,
1025 0x3c060, 0xffffffff, 0x00030002,
1026 0x3c064, 0xffffffff, 0x00040007,
1027 0x3c068, 0xffffffff, 0x00060005,
1028 0x3c06c, 0xffffffff, 0x00090008,
1029 0x3c070, 0xffffffff, 0x00010000,
1030 0x3c074, 0xffffffff, 0x00030002,
1031 0x3c078, 0xffffffff, 0x00040007,
1032 0x3c07c, 0xffffffff, 0x00060005,
1033 0x3c080, 0xffffffff, 0x00090008,
1034 0x3c084, 0xffffffff, 0x00010000,
1035 0x3c088, 0xffffffff, 0x00030002,
1036 0x3c08c, 0xffffffff, 0x00040007,
1037 0x3c090, 0xffffffff, 0x00060005,
1038 0x3c094, 0xffffffff, 0x00090008,
1039 0x3c098, 0xffffffff, 0x00010000,
1040 0x3c09c, 0xffffffff, 0x00030002,
1041 0x3c0a0, 0xffffffff, 0x00040007,
1042 0x3c0a4, 0xffffffff, 0x00060005,
1043 0x3c0a8, 0xffffffff, 0x00090008,
1044 0x3c000, 0xffffffff, 0x96e00200,
1045 0x8708, 0xffffffff, 0x00900100,
1046 0xc424, 0xffffffff, 0x0020003f,
1047 0x38, 0xffffffff, 0x0140001c,
1048 0x3c, 0x000f0000, 0x000f0000,
1049 0x220, 0xffffffff, 0xC060000C,
1050 0x224, 0xc0000fff, 0x00000100,
1051 0xf90, 0xffffffff, 0x00000100,
1052 0xf98, 0x00000101, 0x00000000,
1053 0x20a8, 0xffffffff, 0x00000104,
1054 0x55e4, 0xff000fff, 0x00000100,
1055 0x30cc, 0xc0000fff, 0x00000104,
1056 0xc1e4, 0x00000001, 0x00000001,
1057 0xd00c, 0xff000ff0, 0x00000100,
1058 0xd80c, 0xff000ff0, 0x00000100
1059};
1060
1061static const u32 spectre_golden_spm_registers[] =
1062{
1063 0x30800, 0xe0ffffff, 0xe0000000
1064};
1065
1066static const u32 spectre_golden_common_registers[] =
1067{
1068 0xc770, 0xffffffff, 0x00000800,
1069 0xc774, 0xffffffff, 0x00000800,
1070 0xc798, 0xffffffff, 0x00007fbf,
1071 0xc79c, 0xffffffff, 0x00007faf
1072};
1073
1074static const u32 spectre_golden_registers[] =
1075{
1076 0x3c000, 0xffff1fff, 0x96940200,
1077 0x3c00c, 0xffff0001, 0xff000000,
1078 0x3c200, 0xfffc0fff, 0x00000100,
1079 0x6ed8, 0x00010101, 0x00010000,
1080 0x9834, 0xf00fffff, 0x00000400,
1081 0x9838, 0xfffffffc, 0x00020200,
1082 0x5bb0, 0x000000f0, 0x00000070,
1083 0x5bc0, 0xf0311fff, 0x80300000,
1084 0x98f8, 0x73773777, 0x12010001,
1085 0x9b7c, 0x00ff0000, 0x00fc0000,
1086 0x2f48, 0x73773777, 0x12010001,
1087 0x8a14, 0xf000003f, 0x00000007,
1088 0x8b24, 0xffffffff, 0x00ffffff,
1089 0x28350, 0x3f3f3fff, 0x00000082,
1090 0x28355, 0x0000003f, 0x00000000,
1091 0x3e78, 0x00000001, 0x00000002,
1092 0x913c, 0xffff03df, 0x00000004,
1093 0xc768, 0x00000008, 0x00000008,
1094 0x8c00, 0x000008ff, 0x00000800,
1095 0x9508, 0x00010000, 0x00010000,
1096 0xac0c, 0xffffffff, 0x54763210,
1097 0x214f8, 0x01ff01ff, 0x00000002,
1098 0x21498, 0x007ff800, 0x00200000,
1099 0x2015c, 0xffffffff, 0x00000f40,
1100 0x30934, 0xffffffff, 0x00000001
1101};
1102
1103static const u32 spectre_mgcg_cgcg_init[] =
1104{
1105 0xc420, 0xffffffff, 0xfffffffc,
1106 0x30800, 0xffffffff, 0xe0000000,
1107 0x3c2a0, 0xffffffff, 0x00000100,
1108 0x3c208, 0xffffffff, 0x00000100,
1109 0x3c2c0, 0xffffffff, 0x00000100,
1110 0x3c2c8, 0xffffffff, 0x00000100,
1111 0x3c2c4, 0xffffffff, 0x00000100,
1112 0x55e4, 0xffffffff, 0x00600100,
1113 0x3c280, 0xffffffff, 0x00000100,
1114 0x3c214, 0xffffffff, 0x06000100,
1115 0x3c220, 0xffffffff, 0x00000100,
1116 0x3c218, 0xffffffff, 0x06000100,
1117 0x3c204, 0xffffffff, 0x00000100,
1118 0x3c2e0, 0xffffffff, 0x00000100,
1119 0x3c224, 0xffffffff, 0x00000100,
1120 0x3c200, 0xffffffff, 0x00000100,
1121 0x3c230, 0xffffffff, 0x00000100,
1122 0x3c234, 0xffffffff, 0x00000100,
1123 0x3c250, 0xffffffff, 0x00000100,
1124 0x3c254, 0xffffffff, 0x00000100,
1125 0x3c258, 0xffffffff, 0x00000100,
1126 0x3c25c, 0xffffffff, 0x00000100,
1127 0x3c260, 0xffffffff, 0x00000100,
1128 0x3c27c, 0xffffffff, 0x00000100,
1129 0x3c278, 0xffffffff, 0x00000100,
1130 0x3c210, 0xffffffff, 0x06000100,
1131 0x3c290, 0xffffffff, 0x00000100,
1132 0x3c274, 0xffffffff, 0x00000100,
1133 0x3c2b4, 0xffffffff, 0x00000100,
1134 0x3c2b0, 0xffffffff, 0x00000100,
1135 0x3c270, 0xffffffff, 0x00000100,
1136 0x30800, 0xffffffff, 0xe0000000,
1137 0x3c020, 0xffffffff, 0x00010000,
1138 0x3c024, 0xffffffff, 0x00030002,
1139 0x3c028, 0xffffffff, 0x00040007,
1140 0x3c02c, 0xffffffff, 0x00060005,
1141 0x3c030, 0xffffffff, 0x00090008,
1142 0x3c034, 0xffffffff, 0x00010000,
1143 0x3c038, 0xffffffff, 0x00030002,
1144 0x3c03c, 0xffffffff, 0x00040007,
1145 0x3c040, 0xffffffff, 0x00060005,
1146 0x3c044, 0xffffffff, 0x00090008,
1147 0x3c048, 0xffffffff, 0x00010000,
1148 0x3c04c, 0xffffffff, 0x00030002,
1149 0x3c050, 0xffffffff, 0x00040007,
1150 0x3c054, 0xffffffff, 0x00060005,
1151 0x3c058, 0xffffffff, 0x00090008,
1152 0x3c05c, 0xffffffff, 0x00010000,
1153 0x3c060, 0xffffffff, 0x00030002,
1154 0x3c064, 0xffffffff, 0x00040007,
1155 0x3c068, 0xffffffff, 0x00060005,
1156 0x3c06c, 0xffffffff, 0x00090008,
1157 0x3c070, 0xffffffff, 0x00010000,
1158 0x3c074, 0xffffffff, 0x00030002,
1159 0x3c078, 0xffffffff, 0x00040007,
1160 0x3c07c, 0xffffffff, 0x00060005,
1161 0x3c080, 0xffffffff, 0x00090008,
1162 0x3c084, 0xffffffff, 0x00010000,
1163 0x3c088, 0xffffffff, 0x00030002,
1164 0x3c08c, 0xffffffff, 0x00040007,
1165 0x3c090, 0xffffffff, 0x00060005,
1166 0x3c094, 0xffffffff, 0x00090008,
1167 0x3c098, 0xffffffff, 0x00010000,
1168 0x3c09c, 0xffffffff, 0x00030002,
1169 0x3c0a0, 0xffffffff, 0x00040007,
1170 0x3c0a4, 0xffffffff, 0x00060005,
1171 0x3c0a8, 0xffffffff, 0x00090008,
1172 0x3c0ac, 0xffffffff, 0x00010000,
1173 0x3c0b0, 0xffffffff, 0x00030002,
1174 0x3c0b4, 0xffffffff, 0x00040007,
1175 0x3c0b8, 0xffffffff, 0x00060005,
1176 0x3c0bc, 0xffffffff, 0x00090008,
1177 0x3c000, 0xffffffff, 0x96e00200,
1178 0x8708, 0xffffffff, 0x00900100,
1179 0xc424, 0xffffffff, 0x0020003f,
1180 0x38, 0xffffffff, 0x0140001c,
1181 0x3c, 0x000f0000, 0x000f0000,
1182 0x220, 0xffffffff, 0xC060000C,
1183 0x224, 0xc0000fff, 0x00000100,
1184 0xf90, 0xffffffff, 0x00000100,
1185 0xf98, 0x00000101, 0x00000000,
1186 0x20a8, 0xffffffff, 0x00000104,
1187 0x55e4, 0xff000fff, 0x00000100,
1188 0x30cc, 0xc0000fff, 0x00000104,
1189 0xc1e4, 0x00000001, 0x00000001,
1190 0xd00c, 0xff000ff0, 0x00000100,
1191 0xd80c, 0xff000ff0, 0x00000100
1192};
1193
1194static const u32 kalindi_golden_spm_registers[] =
1195{
1196 0x30800, 0xe0ffffff, 0xe0000000
1197};
1198
1199static const u32 kalindi_golden_common_registers[] =
1200{
1201 0xc770, 0xffffffff, 0x00000800,
1202 0xc774, 0xffffffff, 0x00000800,
1203 0xc798, 0xffffffff, 0x00007fbf,
1204 0xc79c, 0xffffffff, 0x00007faf
1205};
1206
1207static const u32 kalindi_golden_registers[] =
1208{
1209 0x3c000, 0xffffdfff, 0x6e944040,
1210 0x55e4, 0xff607fff, 0xfc000100,
1211 0x3c220, 0xff000fff, 0x00000100,
1212 0x3c224, 0xff000fff, 0x00000100,
1213 0x3c200, 0xfffc0fff, 0x00000100,
1214 0x6ed8, 0x00010101, 0x00010000,
1215 0x9830, 0xffffffff, 0x00000000,
1216 0x9834, 0xf00fffff, 0x00000400,
1217 0x5bb0, 0x000000f0, 0x00000070,
1218 0x5bc0, 0xf0311fff, 0x80300000,
1219 0x98f8, 0x73773777, 0x12010001,
1220 0x98fc, 0xffffffff, 0x00000010,
1221 0x9b7c, 0x00ff0000, 0x00fc0000,
1222 0x8030, 0x00001f0f, 0x0000100a,
1223 0x2f48, 0x73773777, 0x12010001,
1224 0x2408, 0x000fffff, 0x000c007f,
1225 0x8a14, 0xf000003f, 0x00000007,
1226 0x8b24, 0x3fff3fff, 0x00ffcfff,
1227 0x30a04, 0x0000ff0f, 0x00000000,
1228 0x28a4c, 0x07ffffff, 0x06000000,
1229 0x4d8, 0x00000fff, 0x00000100,
1230 0x3e78, 0x00000001, 0x00000002,
1231 0xc768, 0x00000008, 0x00000008,
1232 0x8c00, 0x000000ff, 0x00000003,
1233 0x214f8, 0x01ff01ff, 0x00000002,
1234 0x21498, 0x007ff800, 0x00200000,
1235 0x2015c, 0xffffffff, 0x00000f40,
1236 0x88c4, 0x001f3ae3, 0x00000082,
1237 0x88d4, 0x0000001f, 0x00000010,
1238 0x30934, 0xffffffff, 0x00000000
1239};
1240
1241static const u32 kalindi_mgcg_cgcg_init[] =
1242{
1243 0xc420, 0xffffffff, 0xfffffffc,
1244 0x30800, 0xffffffff, 0xe0000000,
1245 0x3c2a0, 0xffffffff, 0x00000100,
1246 0x3c208, 0xffffffff, 0x00000100,
1247 0x3c2c0, 0xffffffff, 0x00000100,
1248 0x3c2c8, 0xffffffff, 0x00000100,
1249 0x3c2c4, 0xffffffff, 0x00000100,
1250 0x55e4, 0xffffffff, 0x00600100,
1251 0x3c280, 0xffffffff, 0x00000100,
1252 0x3c214, 0xffffffff, 0x06000100,
1253 0x3c220, 0xffffffff, 0x00000100,
1254 0x3c218, 0xffffffff, 0x06000100,
1255 0x3c204, 0xffffffff, 0x00000100,
1256 0x3c2e0, 0xffffffff, 0x00000100,
1257 0x3c224, 0xffffffff, 0x00000100,
1258 0x3c200, 0xffffffff, 0x00000100,
1259 0x3c230, 0xffffffff, 0x00000100,
1260 0x3c234, 0xffffffff, 0x00000100,
1261 0x3c250, 0xffffffff, 0x00000100,
1262 0x3c254, 0xffffffff, 0x00000100,
1263 0x3c258, 0xffffffff, 0x00000100,
1264 0x3c25c, 0xffffffff, 0x00000100,
1265 0x3c260, 0xffffffff, 0x00000100,
1266 0x3c27c, 0xffffffff, 0x00000100,
1267 0x3c278, 0xffffffff, 0x00000100,
1268 0x3c210, 0xffffffff, 0x06000100,
1269 0x3c290, 0xffffffff, 0x00000100,
1270 0x3c274, 0xffffffff, 0x00000100,
1271 0x3c2b4, 0xffffffff, 0x00000100,
1272 0x3c2b0, 0xffffffff, 0x00000100,
1273 0x3c270, 0xffffffff, 0x00000100,
1274 0x30800, 0xffffffff, 0xe0000000,
1275 0x3c020, 0xffffffff, 0x00010000,
1276 0x3c024, 0xffffffff, 0x00030002,
1277 0x3c028, 0xffffffff, 0x00040007,
1278 0x3c02c, 0xffffffff, 0x00060005,
1279 0x3c030, 0xffffffff, 0x00090008,
1280 0x3c034, 0xffffffff, 0x00010000,
1281 0x3c038, 0xffffffff, 0x00030002,
1282 0x3c03c, 0xffffffff, 0x00040007,
1283 0x3c040, 0xffffffff, 0x00060005,
1284 0x3c044, 0xffffffff, 0x00090008,
1285 0x3c000, 0xffffffff, 0x96e00200,
1286 0x8708, 0xffffffff, 0x00900100,
1287 0xc424, 0xffffffff, 0x0020003f,
1288 0x38, 0xffffffff, 0x0140001c,
1289 0x3c, 0x000f0000, 0x000f0000,
1290 0x220, 0xffffffff, 0xC060000C,
1291 0x224, 0xc0000fff, 0x00000100,
1292 0x20a8, 0xffffffff, 0x00000104,
1293 0x55e4, 0xff000fff, 0x00000100,
1294 0x30cc, 0xc0000fff, 0x00000104,
1295 0xc1e4, 0x00000001, 0x00000001,
1296 0xd00c, 0xff000ff0, 0x00000100,
1297 0xd80c, 0xff000ff0, 0x00000100
1298};
1299
Alex Deucher8efff332013-08-07 19:20:14 -0400[diff] [blame^]1300static const u32 hawaii_golden_spm_registers[] =
1301{
1302 0x30800, 0xe0ffffff, 0xe0000000
1303};
1304
1305static const u32 hawaii_golden_common_registers[] =
1306{
1307 0x30800, 0xffffffff, 0xe0000000,
1308 0x28350, 0xffffffff, 0x3a00161a,
1309 0x28354, 0xffffffff, 0x0000002e,
1310 0x9a10, 0xffffffff, 0x00018208,
1311 0x98f8, 0xffffffff, 0x12011003
1312};
1313
1314static const u32 hawaii_golden_registers[] =
1315{
1316 0x3354, 0x00000333, 0x00000333,
1317 0x9a10, 0x00010000, 0x00058208,
1318 0x9830, 0xffffffff, 0x00000000,
1319 0x9834, 0xf00fffff, 0x00000400,
1320 0x9838, 0x0002021c, 0x00020200,
1321 0xc78, 0x00000080, 0x00000000,
1322 0x5bb0, 0x000000f0, 0x00000070,
1323 0x5bc0, 0xf0311fff, 0x80300000,
1324 0x350c, 0x00810000, 0x408af000,
1325 0x7030, 0x31000111, 0x00000011,
1326 0x2f48, 0x73773777, 0x12010001,
1327 0x2120, 0x0000007f, 0x0000001b,
1328 0x21dc, 0x00007fb6, 0x00002191,
1329 0x3628, 0x0000003f, 0x0000000a,
1330 0x362c, 0x0000003f, 0x0000000a,
1331 0x2ae4, 0x00073ffe, 0x000022a2,
1332 0x240c, 0x000007ff, 0x00000000,
1333 0x8bf0, 0x00002001, 0x00000001,
1334 0x8b24, 0xffffffff, 0x00ffffff,
1335 0x30a04, 0x0000ff0f, 0x00000000,
1336 0x28a4c, 0x07ffffff, 0x06000000,
1337 0x3e78, 0x00000001, 0x00000002,
1338 0xc768, 0x00000008, 0x00000008,
1339 0xc770, 0x00000f00, 0x00000800,
1340 0xc774, 0x00000f00, 0x00000800,
1341 0xc798, 0x00ffffff, 0x00ff7fbf,
1342 0xc79c, 0x00ffffff, 0x00ff7faf,
1343 0x8c00, 0x000000ff, 0x00000800,
1344 0xe40, 0x00001fff, 0x00001fff,
1345 0x9060, 0x0000007f, 0x00000020,
1346 0x9508, 0x00010000, 0x00010000,
1347 0xae00, 0x00100000, 0x000ff07c,
1348 0xac14, 0x000003ff, 0x0000000f,
1349 0xac10, 0xffffffff, 0x7564fdec,
1350 0xac0c, 0xffffffff, 0x3120b9a8,
1351 0xac08, 0x20000000, 0x0f9c0000
1352};
1353
1354static const u32 hawaii_mgcg_cgcg_init[] =
1355{
1356 0xc420, 0xffffffff, 0xfffffffd,
1357 0x30800, 0xffffffff, 0xe0000000,
1358 0x3c2a0, 0xffffffff, 0x00000100,
1359 0x3c208, 0xffffffff, 0x00000100,
1360 0x3c2c0, 0xffffffff, 0x00000100,
1361 0x3c2c8, 0xffffffff, 0x00000100,
1362 0x3c2c4, 0xffffffff, 0x00000100,
1363 0x55e4, 0xffffffff, 0x00200100,
1364 0x3c280, 0xffffffff, 0x00000100,
1365 0x3c214, 0xffffffff, 0x06000100,
1366 0x3c220, 0xffffffff, 0x00000100,
1367 0x3c218, 0xffffffff, 0x06000100,
1368 0x3c204, 0xffffffff, 0x00000100,
1369 0x3c2e0, 0xffffffff, 0x00000100,
1370 0x3c224, 0xffffffff, 0x00000100,
1371 0x3c200, 0xffffffff, 0x00000100,
1372 0x3c230, 0xffffffff, 0x00000100,
1373 0x3c234, 0xffffffff, 0x00000100,
1374 0x3c250, 0xffffffff, 0x00000100,
1375 0x3c254, 0xffffffff, 0x00000100,
1376 0x3c258, 0xffffffff, 0x00000100,
1377 0x3c25c, 0xffffffff, 0x00000100,
1378 0x3c260, 0xffffffff, 0x00000100,
1379 0x3c27c, 0xffffffff, 0x00000100,
1380 0x3c278, 0xffffffff, 0x00000100,
1381 0x3c210, 0xffffffff, 0x06000100,
1382 0x3c290, 0xffffffff, 0x00000100,
1383 0x3c274, 0xffffffff, 0x00000100,
1384 0x3c2b4, 0xffffffff, 0x00000100,
1385 0x3c2b0, 0xffffffff, 0x00000100,
1386 0x3c270, 0xffffffff, 0x00000100,
1387 0x30800, 0xffffffff, 0xe0000000,
1388 0x3c020, 0xffffffff, 0x00010000,
1389 0x3c024, 0xffffffff, 0x00030002,
1390 0x3c028, 0xffffffff, 0x00040007,
1391 0x3c02c, 0xffffffff, 0x00060005,
1392 0x3c030, 0xffffffff, 0x00090008,
1393 0x3c034, 0xffffffff, 0x00010000,
1394 0x3c038, 0xffffffff, 0x00030002,
1395 0x3c03c, 0xffffffff, 0x00040007,
1396 0x3c040, 0xffffffff, 0x00060005,
1397 0x3c044, 0xffffffff, 0x00090008,
1398 0x3c048, 0xffffffff, 0x00010000,
1399 0x3c04c, 0xffffffff, 0x00030002,
1400 0x3c050, 0xffffffff, 0x00040007,
1401 0x3c054, 0xffffffff, 0x00060005,
1402 0x3c058, 0xffffffff, 0x00090008,
1403 0x3c05c, 0xffffffff, 0x00010000,
1404 0x3c060, 0xffffffff, 0x00030002,
1405 0x3c064, 0xffffffff, 0x00040007,
1406 0x3c068, 0xffffffff, 0x00060005,
1407 0x3c06c, 0xffffffff, 0x00090008,
1408 0x3c070, 0xffffffff, 0x00010000,
1409 0x3c074, 0xffffffff, 0x00030002,
1410 0x3c078, 0xffffffff, 0x00040007,
1411 0x3c07c, 0xffffffff, 0x00060005,
1412 0x3c080, 0xffffffff, 0x00090008,
1413 0x3c084, 0xffffffff, 0x00010000,
1414 0x3c088, 0xffffffff, 0x00030002,
1415 0x3c08c, 0xffffffff, 0x00040007,
1416 0x3c090, 0xffffffff, 0x00060005,
1417 0x3c094, 0xffffffff, 0x00090008,
1418 0x3c098, 0xffffffff, 0x00010000,
1419 0x3c09c, 0xffffffff, 0x00030002,
1420 0x3c0a0, 0xffffffff, 0x00040007,
1421 0x3c0a4, 0xffffffff, 0x00060005,
1422 0x3c0a8, 0xffffffff, 0x00090008,
1423 0x3c0ac, 0xffffffff, 0x00010000,
1424 0x3c0b0, 0xffffffff, 0x00030002,
1425 0x3c0b4, 0xffffffff, 0x00040007,
1426 0x3c0b8, 0xffffffff, 0x00060005,
1427 0x3c0bc, 0xffffffff, 0x00090008,
1428 0x3c0c0, 0xffffffff, 0x00010000,
1429 0x3c0c4, 0xffffffff, 0x00030002,
1430 0x3c0c8, 0xffffffff, 0x00040007,
1431 0x3c0cc, 0xffffffff, 0x00060005,
1432 0x3c0d0, 0xffffffff, 0x00090008,
1433 0x3c0d4, 0xffffffff, 0x00010000,
1434 0x3c0d8, 0xffffffff, 0x00030002,
1435 0x3c0dc, 0xffffffff, 0x00040007,
1436 0x3c0e0, 0xffffffff, 0x00060005,
1437 0x3c0e4, 0xffffffff, 0x00090008,
1438 0x3c0e8, 0xffffffff, 0x00010000,
1439 0x3c0ec, 0xffffffff, 0x00030002,
1440 0x3c0f0, 0xffffffff, 0x00040007,
1441 0x3c0f4, 0xffffffff, 0x00060005,
1442 0x3c0f8, 0xffffffff, 0x00090008,
1443 0xc318, 0xffffffff, 0x00020200,
1444 0x3350, 0xffffffff, 0x00000200,
1445 0x15c0, 0xffffffff, 0x00000400,
1446 0x55e8, 0xffffffff, 0x00000000,
1447 0x2f50, 0xffffffff, 0x00000902,
1448 0x3c000, 0xffffffff, 0x96940200,
1449 0x8708, 0xffffffff, 0x00900100,
1450 0xc424, 0xffffffff, 0x0020003f,
1451 0x38, 0xffffffff, 0x0140001c,
1452 0x3c, 0x000f0000, 0x000f0000,
1453 0x220, 0xffffffff, 0xc060000c,
1454 0x224, 0xc0000fff, 0x00000100,
1455 0xf90, 0xffffffff, 0x00000100,
1456 0xf98, 0x00000101, 0x00000000,
1457 0x20a8, 0xffffffff, 0x00000104,
1458 0x55e4, 0xff000fff, 0x00000100,
1459 0x30cc, 0xc0000fff, 0x00000104,
1460 0xc1e4, 0x00000001, 0x00000001,
1461 0xd00c, 0xff000ff0, 0x00000100,
1462 0xd80c, 0xff000ff0, 0x00000100
1463};
1464
Alex Deucher0aafd312013-04-09 14:43:30 -0400[diff] [blame]1465static void cik_init_golden_registers(struct radeon_device *rdev)
1466{
1467 switch (rdev->family) {
1468 case CHIP_BONAIRE:
1469 radeon_program_register_sequence(rdev,
1470 bonaire_mgcg_cgcg_init,
1471 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1472 radeon_program_register_sequence(rdev,
1473 bonaire_golden_registers,
1474 (const u32)ARRAY_SIZE(bonaire_golden_registers));
1475 radeon_program_register_sequence(rdev,
1476 bonaire_golden_common_registers,
1477 (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1478 radeon_program_register_sequence(rdev,
1479 bonaire_golden_spm_registers,
1480 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1481 break;
1482 case CHIP_KABINI:
1483 radeon_program_register_sequence(rdev,
1484 kalindi_mgcg_cgcg_init,
1485 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1486 radeon_program_register_sequence(rdev,
1487 kalindi_golden_registers,
1488 (const u32)ARRAY_SIZE(kalindi_golden_registers));
1489 radeon_program_register_sequence(rdev,
1490 kalindi_golden_common_registers,
1491 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1492 radeon_program_register_sequence(rdev,
1493 kalindi_golden_spm_registers,
1494 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1495 break;
1496 case CHIP_KAVERI:
1497 radeon_program_register_sequence(rdev,
1498 spectre_mgcg_cgcg_init,
1499 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1500 radeon_program_register_sequence(rdev,
1501 spectre_golden_registers,
1502 (const u32)ARRAY_SIZE(spectre_golden_registers));
1503 radeon_program_register_sequence(rdev,
1504 spectre_golden_common_registers,
1505 (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1506 radeon_program_register_sequence(rdev,
1507 spectre_golden_spm_registers,
1508 (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1509 break;
Alex Deucher8efff332013-08-07 19:20:14 -0400[diff] [blame^]1510 case CHIP_HAWAII:
1511 radeon_program_register_sequence(rdev,
1512 hawaii_mgcg_cgcg_init,
1513 (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
1514 radeon_program_register_sequence(rdev,
1515 hawaii_golden_registers,
1516 (const u32)ARRAY_SIZE(hawaii_golden_registers));
1517 radeon_program_register_sequence(rdev,
1518 hawaii_golden_common_registers,
1519 (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
1520 radeon_program_register_sequence(rdev,
1521 hawaii_golden_spm_registers,
1522 (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
1523 break;
Alex Deucher0aafd312013-04-09 14:43:30 -0400[diff] [blame]1524 default:
1525 break;
1526 }
1527}
1528
Alex Deucher2c679122013-04-09 13:32:18 -0400[diff] [blame]1529/**
1530 * cik_get_xclk - get the xclk
1531 *
1532 * @rdev: radeon_device pointer
1533 *
1534 * Returns the reference clock used by the gfx engine
1535 * (CIK).
1536 */
1537u32 cik_get_xclk(struct radeon_device *rdev)
1538{
1539 u32 reference_clock = rdev->clock.spll.reference_freq;
1540
1541 if (rdev->flags & RADEON_IS_IGP) {
1542 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1543 return reference_clock / 2;
1544 } else {
1545 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1546 return reference_clock / 4;
1547 }
1548 return reference_clock;
1549}
1550
Alex Deucher75efdee2013-03-04 12:47:46 -0500[diff] [blame]1551/**
1552 * cik_mm_rdoorbell - read a doorbell dword
1553 *
1554 * @rdev: radeon_device pointer
1555 * @offset: byte offset into the aperture
1556 *
1557 * Returns the value in the doorbell aperture at the
1558 * requested offset (CIK).
1559 */
1560u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 offset)
1561{
1562 if (offset < rdev->doorbell.size) {
1563 return readl(((void __iomem *)rdev->doorbell.ptr) + offset);
1564 } else {
1565 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", offset);
1566 return 0;
1567 }
1568}
1569
1570/**
1571 * cik_mm_wdoorbell - write a doorbell dword
1572 *
1573 * @rdev: radeon_device pointer
1574 * @offset: byte offset into the aperture
1575 * @v: value to write
1576 *
1577 * Writes @v to the doorbell aperture at the
1578 * requested offset (CIK).
1579 */
1580void cik_mm_wdoorbell(struct radeon_device *rdev, u32 offset, u32 v)
1581{
1582 if (offset < rdev->doorbell.size) {
1583 writel(v, ((void __iomem *)rdev->doorbell.ptr) + offset);
1584 } else {
1585 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", offset);
1586 }
1587}
1588
Alex Deucherbc8273f2012-06-29 19:44:04 -0400[diff] [blame]1589#define BONAIRE_IO_MC_REGS_SIZE 36
1590
1591static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1592{
1593 {0x00000070, 0x04400000},
1594 {0x00000071, 0x80c01803},
1595 {0x00000072, 0x00004004},
1596 {0x00000073, 0x00000100},
1597 {0x00000074, 0x00ff0000},
1598 {0x00000075, 0x34000000},
1599 {0x00000076, 0x08000014},
1600 {0x00000077, 0x00cc08ec},
1601 {0x00000078, 0x00000400},
1602 {0x00000079, 0x00000000},
1603 {0x0000007a, 0x04090000},
1604 {0x0000007c, 0x00000000},
1605 {0x0000007e, 0x4408a8e8},
1606 {0x0000007f, 0x00000304},
1607 {0x00000080, 0x00000000},
1608 {0x00000082, 0x00000001},
1609 {0x00000083, 0x00000002},
1610 {0x00000084, 0xf3e4f400},
1611 {0x00000085, 0x052024e3},
1612 {0x00000087, 0x00000000},
1613 {0x00000088, 0x01000000},
1614 {0x0000008a, 0x1c0a0000},
1615 {0x0000008b, 0xff010000},
1616 {0x0000008d, 0xffffefff},
1617 {0x0000008e, 0xfff3efff},
1618 {0x0000008f, 0xfff3efbf},
1619 {0x00000092, 0xf7ffffff},
1620 {0x00000093, 0xffffff7f},
1621 {0x00000095, 0x00101101},
1622 {0x00000096, 0x00000fff},
1623 {0x00000097, 0x00116fff},
1624 {0x00000098, 0x60010000},
1625 {0x00000099, 0x10010000},
1626 {0x0000009a, 0x00006000},
1627 {0x0000009b, 0x00001000},
1628 {0x0000009f, 0x00b48000}
1629};
1630
Alex Deucherb556b122013-01-29 10:44:22 -0500[diff] [blame]1631/**
1632 * cik_srbm_select - select specific register instances
1633 *
1634 * @rdev: radeon_device pointer
1635 * @me: selected ME (micro engine)
1636 * @pipe: pipe
1637 * @queue: queue
1638 * @vmid: VMID
1639 *
1640 * Switches the currently active registers instances. Some
1641 * registers are instanced per VMID, others are instanced per
1642 * me/pipe/queue combination.
1643 */
1644static void cik_srbm_select(struct radeon_device *rdev,
1645 u32 me, u32 pipe, u32 queue, u32 vmid)
1646{
1647 u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1648 MEID(me & 0x3) |
1649 VMID(vmid & 0xf) |
1650 QUEUEID(queue & 0x7));
1651 WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1652}
1653
Alex Deucherbc8273f2012-06-29 19:44:04 -0400[diff] [blame]1654/* ucode loading */
1655/**
1656 * ci_mc_load_microcode - load MC ucode into the hw
1657 *
1658 * @rdev: radeon_device pointer
1659 *
1660 * Load the GDDR MC ucode into the hw (CIK).
1661 * Returns 0 on success, error on failure.
1662 */
1663static int ci_mc_load_microcode(struct radeon_device *rdev)
1664{
1665 const __be32 *fw_data;
1666 u32 running, blackout = 0;
1667 u32 *io_mc_regs;
1668 int i, ucode_size, regs_size;
1669
1670 if (!rdev->mc_fw)
1671 return -EINVAL;
1672
1673 switch (rdev->family) {
1674 case CHIP_BONAIRE:
1675 default:
1676 io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1677 ucode_size = CIK_MC_UCODE_SIZE;
1678 regs_size = BONAIRE_IO_MC_REGS_SIZE;
1679 break;
1680 }
1681
1682 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1683
1684 if (running == 0) {
1685 if (running) {
1686 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1687 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1688 }
1689
1690 /* reset the engine and set to writable */
1691 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1692 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1693
1694 /* load mc io regs */
1695 for (i = 0; i < regs_size; i++) {
1696 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1697 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1698 }
1699 /* load the MC ucode */
1700 fw_data = (const __be32 *)rdev->mc_fw->data;
1701 for (i = 0; i < ucode_size; i++)
1702 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1703
1704 /* put the engine back into the active state */
1705 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1706 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1707 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1708
1709 /* wait for training to complete */
1710 for (i = 0; i < rdev->usec_timeout; i++) {
1711 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1712 break;
1713 udelay(1);
1714 }
1715 for (i = 0; i < rdev->usec_timeout; i++) {
1716 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1717 break;
1718 udelay(1);
1719 }
1720
1721 if (running)
1722 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1723 }
1724
1725 return 0;
1726}
1727
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1728/**
1729 * cik_init_microcode - load ucode images from disk
1730 *
1731 * @rdev: radeon_device pointer
1732 *
1733 * Use the firmware interface to load the ucode images into
1734 * the driver (not loaded into hw).
1735 * Returns 0 on success, error on failure.
1736 */
1737static int cik_init_microcode(struct radeon_device *rdev)
1738{
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1739 const char *chip_name;
1740 size_t pfp_req_size, me_req_size, ce_req_size,
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]1741 mec_req_size, rlc_req_size, mc_req_size,
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]1742 sdma_req_size, smc_req_size;
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1743 char fw_name[30];
1744 int err;
1745
1746 DRM_DEBUG("\n");
1747
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1748 switch (rdev->family) {
1749 case CHIP_BONAIRE:
1750 chip_name = "BONAIRE";
1751 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1752 me_req_size = CIK_ME_UCODE_SIZE * 4;
1753 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1754 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1755 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1756 mc_req_size = CIK_MC_UCODE_SIZE * 4;
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]1757 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]1758 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1759 break;
1760 case CHIP_KAVERI:
1761 chip_name = "KAVERI";
1762 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1763 me_req_size = CIK_ME_UCODE_SIZE * 4;
1764 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1765 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1766 rlc_req_size = KV_RLC_UCODE_SIZE * 4;
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]1767 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1768 break;
1769 case CHIP_KABINI:
1770 chip_name = "KABINI";
1771 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1772 me_req_size = CIK_ME_UCODE_SIZE * 4;
1773 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1774 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1775 rlc_req_size = KB_RLC_UCODE_SIZE * 4;
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]1776 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1777 break;
1778 default: BUG();
1779 }
1780
1781 DRM_INFO("Loading %s Microcode\n", chip_name);
1782
1783 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -0400[diff] [blame]1784 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1785 if (err)
1786 goto out;
1787 if (rdev->pfp_fw->size != pfp_req_size) {
1788 printk(KERN_ERR
1789 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1790 rdev->pfp_fw->size, fw_name);
1791 err = -EINVAL;
1792 goto out;
1793 }
1794
1795 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -0400[diff] [blame]1796 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1797 if (err)
1798 goto out;
1799 if (rdev->me_fw->size != me_req_size) {
1800 printk(KERN_ERR
1801 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1802 rdev->me_fw->size, fw_name);
1803 err = -EINVAL;
1804 }
1805
1806 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -0400[diff] [blame]1807 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1808 if (err)
1809 goto out;
1810 if (rdev->ce_fw->size != ce_req_size) {
1811 printk(KERN_ERR
1812 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1813 rdev->ce_fw->size, fw_name);
1814 err = -EINVAL;
1815 }
1816
1817 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -0400[diff] [blame]1818 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1819 if (err)
1820 goto out;
1821 if (rdev->mec_fw->size != mec_req_size) {
1822 printk(KERN_ERR
1823 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1824 rdev->mec_fw->size, fw_name);
1825 err = -EINVAL;
1826 }
1827
1828 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -0400[diff] [blame]1829 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1830 if (err)
1831 goto out;
1832 if (rdev->rlc_fw->size != rlc_req_size) {
1833 printk(KERN_ERR
1834 "cik_rlc: Bogus length %zu in firmware \"%s\"\n",
1835 rdev->rlc_fw->size, fw_name);
1836 err = -EINVAL;
1837 }
1838
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]1839 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -0400[diff] [blame]1840 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]1841 if (err)
1842 goto out;
1843 if (rdev->sdma_fw->size != sdma_req_size) {
1844 printk(KERN_ERR
1845 "cik_sdma: Bogus length %zu in firmware \"%s\"\n",
1846 rdev->sdma_fw->size, fw_name);
1847 err = -EINVAL;
1848 }
1849
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]1850 /* No SMC, MC ucode on APUs */
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1851 if (!(rdev->flags & RADEON_IS_IGP)) {
1852 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -0400[diff] [blame]1853 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1854 if (err)
1855 goto out;
1856 if (rdev->mc_fw->size != mc_req_size) {
1857 printk(KERN_ERR
1858 "cik_mc: Bogus length %zu in firmware \"%s\"\n",
1859 rdev->mc_fw->size, fw_name);
1860 err = -EINVAL;
1861 }
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]1862
1863 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
1864 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1865 if (err) {
1866 printk(KERN_ERR
1867 "smc: error loading firmware \"%s\"\n",
1868 fw_name);
1869 release_firmware(rdev->smc_fw);
1870 rdev->smc_fw = NULL;
Alex Deucherd30d71e2013-10-16 11:40:48 -0400[diff] [blame]1871 err = 0;
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]1872 } else if (rdev->smc_fw->size != smc_req_size) {
1873 printk(KERN_ERR
1874 "cik_smc: Bogus length %zu in firmware \"%s\"\n",
1875 rdev->smc_fw->size, fw_name);
1876 err = -EINVAL;
1877 }
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1878 }
1879
1880out:
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1881 if (err) {
1882 if (err != -EINVAL)
1883 printk(KERN_ERR
1884 "cik_cp: Failed to load firmware \"%s\"\n",
1885 fw_name);
1886 release_firmware(rdev->pfp_fw);
1887 rdev->pfp_fw = NULL;
1888 release_firmware(rdev->me_fw);
1889 rdev->me_fw = NULL;
1890 release_firmware(rdev->ce_fw);
1891 rdev->ce_fw = NULL;
1892 release_firmware(rdev->rlc_fw);
1893 rdev->rlc_fw = NULL;
1894 release_firmware(rdev->mc_fw);
1895 rdev->mc_fw = NULL;
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]1896 release_firmware(rdev->smc_fw);
1897 rdev->smc_fw = NULL;
Alex Deucher02c81322012-12-18 21:43:07 -0500[diff] [blame]1898 }
1899 return err;
1900}
1901
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]1902/*
1903 * Core functions
1904 */
1905/**
1906 * cik_tiling_mode_table_init - init the hw tiling table
1907 *
1908 * @rdev: radeon_device pointer
1909 *
1910 * Starting with SI, the tiling setup is done globally in a
1911 * set of 32 tiling modes. Rather than selecting each set of
1912 * parameters per surface as on older asics, we just select
1913 * which index in the tiling table we want to use, and the
1914 * surface uses those parameters (CIK).
1915 */
1916static void cik_tiling_mode_table_init(struct radeon_device *rdev)
1917{
1918 const u32 num_tile_mode_states = 32;
1919 const u32 num_secondary_tile_mode_states = 16;
1920 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
1921 u32 num_pipe_configs;
1922 u32 num_rbs = rdev->config.cik.max_backends_per_se *
1923 rdev->config.cik.max_shader_engines;
1924
1925 switch (rdev->config.cik.mem_row_size_in_kb) {
1926 case 1:
1927 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
1928 break;
1929 case 2:
1930 default:
1931 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
1932 break;
1933 case 4:
1934 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
1935 break;
1936 }
1937
1938 num_pipe_configs = rdev->config.cik.max_tile_pipes;
1939 if (num_pipe_configs > 8)
Alex Deucher21e438a2013-08-06 16:58:53 -0400[diff] [blame]1940 num_pipe_configs = 16;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]1941
Alex Deucher21e438a2013-08-06 16:58:53 -0400[diff] [blame]1942 if (num_pipe_configs == 16) {
1943 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
1944 switch (reg_offset) {
1945 case 0:
1946 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1947 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1948 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1949 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
1950 break;
1951 case 1:
1952 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1953 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1954 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1955 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
1956 break;
1957 case 2:
1958 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1959 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1960 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1961 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
1962 break;
1963 case 3:
1964 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1965 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1966 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1967 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
1968 break;
1969 case 4:
1970 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1971 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1972 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1973 TILE_SPLIT(split_equal_to_row_size));
1974 break;
1975 case 5:
1976 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1977 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
1978 break;
1979 case 6:
1980 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1981 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1982 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1983 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
1984 break;
1985 case 7:
1986 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1987 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1988 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1989 TILE_SPLIT(split_equal_to_row_size));
1990 break;
1991 case 8:
1992 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
1993 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
1994 break;
1995 case 9:
1996 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1997 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
1998 break;
1999 case 10:
2000 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2001 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2002 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2003 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2004 break;
2005 case 11:
2006 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2007 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2008 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2009 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2010 break;
2011 case 12:
2012 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2013 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2014 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2015 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2016 break;
2017 case 13:
2018 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2019 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2020 break;
2021 case 14:
2022 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2023 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2024 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2025 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2026 break;
2027 case 16:
2028 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2029 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2030 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2031 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2032 break;
2033 case 17:
2034 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2035 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2036 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2037 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2038 break;
2039 case 27:
2040 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2041 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2042 break;
2043 case 28:
2044 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2045 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2046 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2047 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2048 break;
2049 case 29:
2050 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2051 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2052 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2053 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2054 break;
2055 case 30:
2056 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2057 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2058 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2059 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2060 break;
2061 default:
2062 gb_tile_moden = 0;
2063 break;
2064 }
2065 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2066 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2067 }
2068 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2069 switch (reg_offset) {
2070 case 0:
2071 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2072 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2073 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2074 NUM_BANKS(ADDR_SURF_16_BANK));
2075 break;
2076 case 1:
2077 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2078 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2079 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2080 NUM_BANKS(ADDR_SURF_16_BANK));
2081 break;
2082 case 2:
2083 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2084 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2085 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2086 NUM_BANKS(ADDR_SURF_16_BANK));
2087 break;
2088 case 3:
2089 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2090 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2091 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2092 NUM_BANKS(ADDR_SURF_16_BANK));
2093 break;
2094 case 4:
2095 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2096 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2097 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2098 NUM_BANKS(ADDR_SURF_8_BANK));
2099 break;
2100 case 5:
2101 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2102 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2103 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2104 NUM_BANKS(ADDR_SURF_4_BANK));
2105 break;
2106 case 6:
2107 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2108 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2109 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2110 NUM_BANKS(ADDR_SURF_2_BANK));
2111 break;
2112 case 8:
2113 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2114 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2115 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2116 NUM_BANKS(ADDR_SURF_16_BANK));
2117 break;
2118 case 9:
2119 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2120 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2121 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2122 NUM_BANKS(ADDR_SURF_16_BANK));
2123 break;
2124 case 10:
2125 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2126 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2127 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2128 NUM_BANKS(ADDR_SURF_16_BANK));
2129 break;
2130 case 11:
2131 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2132 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2133 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2134 NUM_BANKS(ADDR_SURF_8_BANK));
2135 break;
2136 case 12:
2137 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2138 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2139 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2140 NUM_BANKS(ADDR_SURF_4_BANK));
2141 break;
2142 case 13:
2143 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2144 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2145 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2146 NUM_BANKS(ADDR_SURF_2_BANK));
2147 break;
2148 case 14:
2149 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2150 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2151 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2152 NUM_BANKS(ADDR_SURF_2_BANK));
2153 break;
2154 default:
2155 gb_tile_moden = 0;
2156 break;
2157 }
2158 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2159 }
2160 } else if (num_pipe_configs == 8) {
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]2161 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2162 switch (reg_offset) {
2163 case 0:
2164 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2165 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2166 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2167 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2168 break;
2169 case 1:
2170 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2171 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2172 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2173 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2174 break;
2175 case 2:
2176 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2177 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2178 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2179 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2180 break;
2181 case 3:
2182 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2183 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2184 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2185 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2186 break;
2187 case 4:
2188 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2189 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2190 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2191 TILE_SPLIT(split_equal_to_row_size));
2192 break;
2193 case 5:
2194 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2195 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2196 break;
2197 case 6:
2198 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2199 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2200 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2201 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2202 break;
2203 case 7:
2204 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2205 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2206 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2207 TILE_SPLIT(split_equal_to_row_size));
2208 break;
2209 case 8:
2210 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2211 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2212 break;
2213 case 9:
2214 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2215 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2216 break;
2217 case 10:
2218 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2219 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2220 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2221 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2222 break;
2223 case 11:
2224 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2225 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2226 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2227 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2228 break;
2229 case 12:
2230 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2231 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2232 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2233 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2234 break;
2235 case 13:
2236 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2237 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2238 break;
2239 case 14:
2240 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2241 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2242 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2243 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2244 break;
2245 case 16:
2246 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2247 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2248 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2249 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2250 break;
2251 case 17:
2252 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2253 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2254 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2255 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2256 break;
2257 case 27:
2258 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2259 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2260 break;
2261 case 28:
2262 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2263 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2264 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2265 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2266 break;
2267 case 29:
2268 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2269 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2270 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2271 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2272 break;
2273 case 30:
2274 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2275 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2276 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2277 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2278 break;
2279 default:
2280 gb_tile_moden = 0;
2281 break;
2282 }
Alex Deucher39aee492013-04-10 13:41:25 -0400[diff] [blame]2283 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]2284 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2285 }
2286 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2287 switch (reg_offset) {
2288 case 0:
2289 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2290 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2291 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2292 NUM_BANKS(ADDR_SURF_16_BANK));
2293 break;
2294 case 1:
2295 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2296 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2297 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2298 NUM_BANKS(ADDR_SURF_16_BANK));
2299 break;
2300 case 2:
2301 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2302 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2303 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2304 NUM_BANKS(ADDR_SURF_16_BANK));
2305 break;
2306 case 3:
2307 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2308 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2309 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2310 NUM_BANKS(ADDR_SURF_16_BANK));
2311 break;
2312 case 4:
2313 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2314 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2315 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2316 NUM_BANKS(ADDR_SURF_8_BANK));
2317 break;
2318 case 5:
2319 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2320 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2321 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2322 NUM_BANKS(ADDR_SURF_4_BANK));
2323 break;
2324 case 6:
2325 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2326 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2327 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2328 NUM_BANKS(ADDR_SURF_2_BANK));
2329 break;
2330 case 8:
2331 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2332 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2333 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2334 NUM_BANKS(ADDR_SURF_16_BANK));
2335 break;
2336 case 9:
2337 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2338 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2339 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2340 NUM_BANKS(ADDR_SURF_16_BANK));
2341 break;
2342 case 10:
2343 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2344 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2345 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2346 NUM_BANKS(ADDR_SURF_16_BANK));
2347 break;
2348 case 11:
2349 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2350 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2351 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2352 NUM_BANKS(ADDR_SURF_16_BANK));
2353 break;
2354 case 12:
2355 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2356 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2357 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2358 NUM_BANKS(ADDR_SURF_8_BANK));
2359 break;
2360 case 13:
2361 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2362 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2363 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2364 NUM_BANKS(ADDR_SURF_4_BANK));
2365 break;
2366 case 14:
2367 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2368 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2369 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2370 NUM_BANKS(ADDR_SURF_2_BANK));
2371 break;
2372 default:
2373 gb_tile_moden = 0;
2374 break;
2375 }
2376 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2377 }
2378 } else if (num_pipe_configs == 4) {
2379 if (num_rbs == 4) {
2380 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2381 switch (reg_offset) {
2382 case 0:
2383 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2384 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2385 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2386 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2387 break;
2388 case 1:
2389 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2390 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2391 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2392 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2393 break;
2394 case 2:
2395 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2396 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2397 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2398 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2399 break;
2400 case 3:
2401 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2402 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2403 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2404 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2405 break;
2406 case 4:
2407 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2408 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2409 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2410 TILE_SPLIT(split_equal_to_row_size));
2411 break;
2412 case 5:
2413 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2414 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2415 break;
2416 case 6:
2417 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2418 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2419 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2420 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2421 break;
2422 case 7:
2423 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2424 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2425 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2426 TILE_SPLIT(split_equal_to_row_size));
2427 break;
2428 case 8:
2429 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2430 PIPE_CONFIG(ADDR_SURF_P4_16x16));
2431 break;
2432 case 9:
2433 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2434 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2435 break;
2436 case 10:
2437 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2438 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2439 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2440 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2441 break;
2442 case 11:
2443 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2444 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2445 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2446 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2447 break;
2448 case 12:
2449 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2450 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2451 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2452 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2453 break;
2454 case 13:
2455 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2456 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2457 break;
2458 case 14:
2459 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2460 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2461 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2462 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2463 break;
2464 case 16:
2465 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2466 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2467 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2468 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2469 break;
2470 case 17:
2471 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2472 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2473 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2474 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2475 break;
2476 case 27:
2477 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2478 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2479 break;
2480 case 28:
2481 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2482 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2483 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2484 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2485 break;
2486 case 29:
2487 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2488 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2489 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2490 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2491 break;
2492 case 30:
2493 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2494 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2495 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2496 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2497 break;
2498 default:
2499 gb_tile_moden = 0;
2500 break;
2501 }
Alex Deucher39aee492013-04-10 13:41:25 -0400[diff] [blame]2502 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]2503 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2504 }
2505 } else if (num_rbs < 4) {
2506 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2507 switch (reg_offset) {
2508 case 0:
2509 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2510 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2511 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2512 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2513 break;
2514 case 1:
2515 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2516 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2517 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2518 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2519 break;
2520 case 2:
2521 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2522 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2523 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2524 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2525 break;
2526 case 3:
2527 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2528 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2529 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2530 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2531 break;
2532 case 4:
2533 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2534 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2535 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2536 TILE_SPLIT(split_equal_to_row_size));
2537 break;
2538 case 5:
2539 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2540 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2541 break;
2542 case 6:
2543 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2544 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2545 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2546 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2547 break;
2548 case 7:
2549 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2550 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2551 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2552 TILE_SPLIT(split_equal_to_row_size));
2553 break;
2554 case 8:
2555 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2556 PIPE_CONFIG(ADDR_SURF_P4_8x16));
2557 break;
2558 case 9:
2559 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2560 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2561 break;
2562 case 10:
2563 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2564 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2565 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2566 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2567 break;
2568 case 11:
2569 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2570 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2571 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2572 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2573 break;
2574 case 12:
2575 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2576 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2577 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2578 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2579 break;
2580 case 13:
2581 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2582 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2583 break;
2584 case 14:
2585 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2586 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2587 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2588 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2589 break;
2590 case 16:
2591 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2592 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2593 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2594 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2595 break;
2596 case 17:
2597 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2598 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2599 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2600 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2601 break;
2602 case 27:
2603 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2604 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2605 break;
2606 case 28:
2607 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2608 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2609 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2610 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2611 break;
2612 case 29:
2613 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2614 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2615 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2616 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2617 break;
2618 case 30:
2619 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2620 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2621 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2622 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2623 break;
2624 default:
2625 gb_tile_moden = 0;
2626 break;
2627 }
Alex Deucher39aee492013-04-10 13:41:25 -0400[diff] [blame]2628 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]2629 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2630 }
2631 }
2632 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2633 switch (reg_offset) {
2634 case 0:
2635 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2636 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2637 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2638 NUM_BANKS(ADDR_SURF_16_BANK));
2639 break;
2640 case 1:
2641 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2642 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2643 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2644 NUM_BANKS(ADDR_SURF_16_BANK));
2645 break;
2646 case 2:
2647 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2648 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2649 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2650 NUM_BANKS(ADDR_SURF_16_BANK));
2651 break;
2652 case 3:
2653 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2654 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2655 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2656 NUM_BANKS(ADDR_SURF_16_BANK));
2657 break;
2658 case 4:
2659 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2660 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2661 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2662 NUM_BANKS(ADDR_SURF_16_BANK));
2663 break;
2664 case 5:
2665 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2666 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2667 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2668 NUM_BANKS(ADDR_SURF_8_BANK));
2669 break;
2670 case 6:
2671 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2672 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2673 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2674 NUM_BANKS(ADDR_SURF_4_BANK));
2675 break;
2676 case 8:
2677 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2678 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2679 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2680 NUM_BANKS(ADDR_SURF_16_BANK));
2681 break;
2682 case 9:
2683 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2684 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2685 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2686 NUM_BANKS(ADDR_SURF_16_BANK));
2687 break;
2688 case 10:
2689 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2690 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2691 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2692 NUM_BANKS(ADDR_SURF_16_BANK));
2693 break;
2694 case 11:
2695 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2696 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2697 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2698 NUM_BANKS(ADDR_SURF_16_BANK));
2699 break;
2700 case 12:
2701 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2702 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2703 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2704 NUM_BANKS(ADDR_SURF_16_BANK));
2705 break;
2706 case 13:
2707 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2708 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2709 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2710 NUM_BANKS(ADDR_SURF_8_BANK));
2711 break;
2712 case 14:
2713 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2714 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2715 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2716 NUM_BANKS(ADDR_SURF_4_BANK));
2717 break;
2718 default:
2719 gb_tile_moden = 0;
2720 break;
2721 }
2722 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2723 }
2724 } else if (num_pipe_configs == 2) {
2725 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2726 switch (reg_offset) {
2727 case 0:
2728 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2729 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2730 PIPE_CONFIG(ADDR_SURF_P2) |
2731 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2732 break;
2733 case 1:
2734 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2735 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2736 PIPE_CONFIG(ADDR_SURF_P2) |
2737 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2738 break;
2739 case 2:
2740 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2741 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2742 PIPE_CONFIG(ADDR_SURF_P2) |
2743 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2744 break;
2745 case 3:
2746 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2747 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2748 PIPE_CONFIG(ADDR_SURF_P2) |
2749 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2750 break;
2751 case 4:
2752 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2753 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2754 PIPE_CONFIG(ADDR_SURF_P2) |
2755 TILE_SPLIT(split_equal_to_row_size));
2756 break;
2757 case 5:
2758 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2759 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2760 break;
2761 case 6:
2762 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2763 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2764 PIPE_CONFIG(ADDR_SURF_P2) |
2765 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2766 break;
2767 case 7:
2768 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2769 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2770 PIPE_CONFIG(ADDR_SURF_P2) |
2771 TILE_SPLIT(split_equal_to_row_size));
2772 break;
2773 case 8:
2774 gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED);
2775 break;
2776 case 9:
2777 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2778 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2779 break;
2780 case 10:
2781 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2782 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2783 PIPE_CONFIG(ADDR_SURF_P2) |
2784 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2785 break;
2786 case 11:
2787 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2788 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2789 PIPE_CONFIG(ADDR_SURF_P2) |
2790 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2791 break;
2792 case 12:
2793 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2794 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2795 PIPE_CONFIG(ADDR_SURF_P2) |
2796 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2797 break;
2798 case 13:
2799 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2800 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2801 break;
2802 case 14:
2803 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2804 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2805 PIPE_CONFIG(ADDR_SURF_P2) |
2806 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2807 break;
2808 case 16:
2809 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2810 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2811 PIPE_CONFIG(ADDR_SURF_P2) |
2812 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2813 break;
2814 case 17:
2815 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2816 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2817 PIPE_CONFIG(ADDR_SURF_P2) |
2818 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2819 break;
2820 case 27:
2821 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2822 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2823 break;
2824 case 28:
2825 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2826 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2827 PIPE_CONFIG(ADDR_SURF_P2) |
2828 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2829 break;
2830 case 29:
2831 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2832 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2833 PIPE_CONFIG(ADDR_SURF_P2) |
2834 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2835 break;
2836 case 30:
2837 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2838 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2839 PIPE_CONFIG(ADDR_SURF_P2) |
2840 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2841 break;
2842 default:
2843 gb_tile_moden = 0;
2844 break;
2845 }
Alex Deucher39aee492013-04-10 13:41:25 -0400[diff] [blame]2846 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]2847 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2848 }
2849 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2850 switch (reg_offset) {
2851 case 0:
2852 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2853 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2854 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2855 NUM_BANKS(ADDR_SURF_16_BANK));
2856 break;
2857 case 1:
2858 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2859 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2860 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2861 NUM_BANKS(ADDR_SURF_16_BANK));
2862 break;
2863 case 2:
2864 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2865 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2866 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2867 NUM_BANKS(ADDR_SURF_16_BANK));
2868 break;
2869 case 3:
2870 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2871 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2872 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2873 NUM_BANKS(ADDR_SURF_16_BANK));
2874 break;
2875 case 4:
2876 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2877 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2878 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2879 NUM_BANKS(ADDR_SURF_16_BANK));
2880 break;
2881 case 5:
2882 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2883 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2884 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2885 NUM_BANKS(ADDR_SURF_16_BANK));
2886 break;
2887 case 6:
2888 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2889 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2890 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2891 NUM_BANKS(ADDR_SURF_8_BANK));
2892 break;
2893 case 8:
2894 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2895 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2896 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2897 NUM_BANKS(ADDR_SURF_16_BANK));
2898 break;
2899 case 9:
2900 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2901 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2902 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2903 NUM_BANKS(ADDR_SURF_16_BANK));
2904 break;
2905 case 10:
2906 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2907 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2908 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2909 NUM_BANKS(ADDR_SURF_16_BANK));
2910 break;
2911 case 11:
2912 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2913 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2914 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2915 NUM_BANKS(ADDR_SURF_16_BANK));
2916 break;
2917 case 12:
2918 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2919 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2920 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2921 NUM_BANKS(ADDR_SURF_16_BANK));
2922 break;
2923 case 13:
2924 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2925 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2926 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2927 NUM_BANKS(ADDR_SURF_16_BANK));
2928 break;
2929 case 14:
2930 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2931 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2932 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2933 NUM_BANKS(ADDR_SURF_8_BANK));
2934 break;
2935 default:
2936 gb_tile_moden = 0;
2937 break;
2938 }
2939 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2940 }
2941 } else
2942 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
2943}
2944
2945/**
2946 * cik_select_se_sh - select which SE, SH to address
2947 *
2948 * @rdev: radeon_device pointer
2949 * @se_num: shader engine to address
2950 * @sh_num: sh block to address
2951 *
2952 * Select which SE, SH combinations to address. Certain
2953 * registers are instanced per SE or SH. 0xffffffff means
2954 * broadcast to all SEs or SHs (CIK).
2955 */
2956static void cik_select_se_sh(struct radeon_device *rdev,
2957 u32 se_num, u32 sh_num)
2958{
2959 u32 data = INSTANCE_BROADCAST_WRITES;
2960
2961 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
Alex Deucherb0fe3d32013-04-18 16:25:47 -0400[diff] [blame]2962 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]2963 else if (se_num == 0xffffffff)
2964 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
2965 else if (sh_num == 0xffffffff)
2966 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
2967 else
2968 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
2969 WREG32(GRBM_GFX_INDEX, data);
2970}
2971
2972/**
2973 * cik_create_bitmask - create a bitmask
2974 *
2975 * @bit_width: length of the mask
2976 *
2977 * create a variable length bit mask (CIK).
2978 * Returns the bitmask.
2979 */
2980static u32 cik_create_bitmask(u32 bit_width)
2981{
2982 u32 i, mask = 0;
2983
2984 for (i = 0; i < bit_width; i++) {
2985 mask <<= 1;
2986 mask |= 1;
2987 }
2988 return mask;
2989}
2990
2991/**
2992 * cik_select_se_sh - select which SE, SH to address
2993 *
2994 * @rdev: radeon_device pointer
2995 * @max_rb_num: max RBs (render backends) for the asic
2996 * @se_num: number of SEs (shader engines) for the asic
2997 * @sh_per_se: number of SH blocks per SE for the asic
2998 *
2999 * Calculates the bitmask of disabled RBs (CIK).
3000 * Returns the disabled RB bitmask.
3001 */
3002static u32 cik_get_rb_disabled(struct radeon_device *rdev,
3003 u32 max_rb_num, u32 se_num,
3004 u32 sh_per_se)
3005{
3006 u32 data, mask;
3007
3008 data = RREG32(CC_RB_BACKEND_DISABLE);
3009 if (data & 1)
3010 data &= BACKEND_DISABLE_MASK;
3011 else
3012 data = 0;
3013 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3014
3015 data >>= BACKEND_DISABLE_SHIFT;
3016
3017 mask = cik_create_bitmask(max_rb_num / se_num / sh_per_se);
3018
3019 return data & mask;
3020}
3021
3022/**
3023 * cik_setup_rb - setup the RBs on the asic
3024 *
3025 * @rdev: radeon_device pointer
3026 * @se_num: number of SEs (shader engines) for the asic
3027 * @sh_per_se: number of SH blocks per SE for the asic
3028 * @max_rb_num: max RBs (render backends) for the asic
3029 *
3030 * Configures per-SE/SH RB registers (CIK).
3031 */
3032static void cik_setup_rb(struct radeon_device *rdev,
3033 u32 se_num, u32 sh_per_se,
3034 u32 max_rb_num)
3035{
3036 int i, j;
3037 u32 data, mask;
3038 u32 disabled_rbs = 0;
3039 u32 enabled_rbs = 0;
3040
3041 for (i = 0; i < se_num; i++) {
3042 for (j = 0; j < sh_per_se; j++) {
3043 cik_select_se_sh(rdev, i, j);
3044 data = cik_get_rb_disabled(rdev, max_rb_num, se_num, sh_per_se);
3045 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
3046 }
3047 }
3048 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3049
3050 mask = 1;
3051 for (i = 0; i < max_rb_num; i++) {
3052 if (!(disabled_rbs & mask))
3053 enabled_rbs |= mask;
3054 mask <<= 1;
3055 }
3056
3057 for (i = 0; i < se_num; i++) {
3058 cik_select_se_sh(rdev, i, 0xffffffff);
3059 data = 0;
3060 for (j = 0; j < sh_per_se; j++) {
3061 switch (enabled_rbs & 3) {
3062 case 1:
3063 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3064 break;
3065 case 2:
3066 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3067 break;
3068 case 3:
3069 default:
3070 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3071 break;
3072 }
3073 enabled_rbs >>= 2;
3074 }
3075 WREG32(PA_SC_RASTER_CONFIG, data);
3076 }
3077 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3078}
3079
3080/**
3081 * cik_gpu_init - setup the 3D engine
3082 *
3083 * @rdev: radeon_device pointer
3084 *
3085 * Configures the 3D engine and tiling configuration
3086 * registers so that the 3D engine is usable.
3087 */
3088static void cik_gpu_init(struct radeon_device *rdev)
3089{
3090 u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
3091 u32 mc_shared_chmap, mc_arb_ramcfg;
3092 u32 hdp_host_path_cntl;
3093 u32 tmp;
3094 int i, j;
3095
3096 switch (rdev->family) {
3097 case CHIP_BONAIRE:
3098 rdev->config.cik.max_shader_engines = 2;
3099 rdev->config.cik.max_tile_pipes = 4;
3100 rdev->config.cik.max_cu_per_sh = 7;
3101 rdev->config.cik.max_sh_per_se = 1;
3102 rdev->config.cik.max_backends_per_se = 2;
3103 rdev->config.cik.max_texture_channel_caches = 4;
3104 rdev->config.cik.max_gprs = 256;
3105 rdev->config.cik.max_gs_threads = 32;
3106 rdev->config.cik.max_hw_contexts = 8;
3107
3108 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3109 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3110 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3111 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3112 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3113 break;
Alex Deucherb4960382013-08-06 15:42:49 -0400[diff] [blame]3114 case CHIP_HAWAII:
3115 rdev->config.cik.max_shader_engines = 4;
3116 rdev->config.cik.max_tile_pipes = 16;
3117 rdev->config.cik.max_cu_per_sh = 11;
3118 rdev->config.cik.max_sh_per_se = 1;
3119 rdev->config.cik.max_backends_per_se = 4;
3120 rdev->config.cik.max_texture_channel_caches = 16;
3121 rdev->config.cik.max_gprs = 256;
3122 rdev->config.cik.max_gs_threads = 32;
3123 rdev->config.cik.max_hw_contexts = 8;
3124
3125 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3126 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3127 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3128 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3129 gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN;
3130 break;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3131 case CHIP_KAVERI:
Alex Deucherb2e4c702013-06-10 15:18:26 -0400[diff] [blame]3132 rdev->config.cik.max_shader_engines = 1;
3133 rdev->config.cik.max_tile_pipes = 4;
3134 if ((rdev->pdev->device == 0x1304) ||
3135 (rdev->pdev->device == 0x1305) ||
3136 (rdev->pdev->device == 0x130C) ||
3137 (rdev->pdev->device == 0x130F) ||
3138 (rdev->pdev->device == 0x1310) ||
3139 (rdev->pdev->device == 0x1311) ||
3140 (rdev->pdev->device == 0x131C)) {
3141 rdev->config.cik.max_cu_per_sh = 8;
3142 rdev->config.cik.max_backends_per_se = 2;
3143 } else if ((rdev->pdev->device == 0x1309) ||
3144 (rdev->pdev->device == 0x130A) ||
3145 (rdev->pdev->device == 0x130D) ||
Alex Deucher7c4622d2013-09-04 16:46:07 -0400[diff] [blame]3146 (rdev->pdev->device == 0x1313) ||
3147 (rdev->pdev->device == 0x131D)) {
Alex Deucherb2e4c702013-06-10 15:18:26 -0400[diff] [blame]3148 rdev->config.cik.max_cu_per_sh = 6;
3149 rdev->config.cik.max_backends_per_se = 2;
3150 } else if ((rdev->pdev->device == 0x1306) ||
3151 (rdev->pdev->device == 0x1307) ||
3152 (rdev->pdev->device == 0x130B) ||
3153 (rdev->pdev->device == 0x130E) ||
3154 (rdev->pdev->device == 0x1315) ||
3155 (rdev->pdev->device == 0x131B)) {
3156 rdev->config.cik.max_cu_per_sh = 4;
3157 rdev->config.cik.max_backends_per_se = 1;
3158 } else {
3159 rdev->config.cik.max_cu_per_sh = 3;
3160 rdev->config.cik.max_backends_per_se = 1;
3161 }
3162 rdev->config.cik.max_sh_per_se = 1;
3163 rdev->config.cik.max_texture_channel_caches = 4;
3164 rdev->config.cik.max_gprs = 256;
3165 rdev->config.cik.max_gs_threads = 16;
3166 rdev->config.cik.max_hw_contexts = 8;
3167
3168 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3169 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3170 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3171 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3172 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3173 break;
3174 case CHIP_KABINI:
3175 default:
3176 rdev->config.cik.max_shader_engines = 1;
3177 rdev->config.cik.max_tile_pipes = 2;
3178 rdev->config.cik.max_cu_per_sh = 2;
3179 rdev->config.cik.max_sh_per_se = 1;
3180 rdev->config.cik.max_backends_per_se = 1;
3181 rdev->config.cik.max_texture_channel_caches = 2;
3182 rdev->config.cik.max_gprs = 256;
3183 rdev->config.cik.max_gs_threads = 16;
3184 rdev->config.cik.max_hw_contexts = 8;
3185
3186 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3187 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3188 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3189 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3190 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3191 break;
3192 }
3193
3194 /* Initialize HDP */
3195 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3196 WREG32((0x2c14 + j), 0x00000000);
3197 WREG32((0x2c18 + j), 0x00000000);
3198 WREG32((0x2c1c + j), 0x00000000);
3199 WREG32((0x2c20 + j), 0x00000000);
3200 WREG32((0x2c24 + j), 0x00000000);
3201 }
3202
3203 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3204
3205 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3206
3207 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3208 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3209
3210 rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
3211 rdev->config.cik.mem_max_burst_length_bytes = 256;
3212 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3213 rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3214 if (rdev->config.cik.mem_row_size_in_kb > 4)
3215 rdev->config.cik.mem_row_size_in_kb = 4;
3216 /* XXX use MC settings? */
3217 rdev->config.cik.shader_engine_tile_size = 32;
3218 rdev->config.cik.num_gpus = 1;
3219 rdev->config.cik.multi_gpu_tile_size = 64;
3220
3221 /* fix up row size */
3222 gb_addr_config &= ~ROW_SIZE_MASK;
3223 switch (rdev->config.cik.mem_row_size_in_kb) {
3224 case 1:
3225 default:
3226 gb_addr_config |= ROW_SIZE(0);
3227 break;
3228 case 2:
3229 gb_addr_config |= ROW_SIZE(1);
3230 break;
3231 case 4:
3232 gb_addr_config |= ROW_SIZE(2);
3233 break;
3234 }
3235
3236 /* setup tiling info dword. gb_addr_config is not adequate since it does
3237 * not have bank info, so create a custom tiling dword.
3238 * bits 3:0 num_pipes
3239 * bits 7:4 num_banks
3240 * bits 11:8 group_size
3241 * bits 15:12 row_size
3242 */
3243 rdev->config.cik.tile_config = 0;
3244 switch (rdev->config.cik.num_tile_pipes) {
3245 case 1:
3246 rdev->config.cik.tile_config |= (0 << 0);
3247 break;
3248 case 2:
3249 rdev->config.cik.tile_config |= (1 << 0);
3250 break;
3251 case 4:
3252 rdev->config.cik.tile_config |= (2 << 0);
3253 break;
3254 case 8:
3255 default:
3256 /* XXX what about 12? */
3257 rdev->config.cik.tile_config |= (3 << 0);
3258 break;
3259 }
Michel Dänzera5373142013-09-18 15:39:41 +0200[diff] [blame]3260 rdev->config.cik.tile_config |=
3261 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3262 rdev->config.cik.tile_config |=
3263 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3264 rdev->config.cik.tile_config |=
3265 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3266
3267 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3268 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3269 WREG32(DMIF_ADDR_CALC, gb_addr_config);
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]3270 WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
3271 WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]3272 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3273 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3274 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
Alex Deucher8cc1a532013-04-09 12:41:24 -0400[diff] [blame]3275
3276 cik_tiling_mode_table_init(rdev);
3277
3278 cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
3279 rdev->config.cik.max_sh_per_se,
3280 rdev->config.cik.max_backends_per_se);
3281
3282 /* set HW defaults for 3D engine */
3283 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3284
3285 WREG32(SX_DEBUG_1, 0x20);
3286
3287 WREG32(TA_CNTL_AUX, 0x00010000);
3288
3289 tmp = RREG32(SPI_CONFIG_CNTL);
3290 tmp |= 0x03000000;
3291 WREG32(SPI_CONFIG_CNTL, tmp);
3292
3293 WREG32(SQ_CONFIG, 1);
3294
3295 WREG32(DB_DEBUG, 0);
3296
3297 tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
3298 tmp |= 0x00000400;
3299 WREG32(DB_DEBUG2, tmp);
3300
3301 tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
3302 tmp |= 0x00020200;
3303 WREG32(DB_DEBUG3, tmp);
3304
3305 tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
3306 tmp |= 0x00018208;
3307 WREG32(CB_HW_CONTROL, tmp);
3308
3309 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3310
3311 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
3312 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
3313 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
3314 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
3315
3316 WREG32(VGT_NUM_INSTANCES, 1);
3317
3318 WREG32(CP_PERFMON_CNTL, 0);
3319
3320 WREG32(SQ_CONFIG, 0);
3321
3322 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3323 FORCE_EOV_MAX_REZ_CNT(255)));
3324
3325 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3326 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3327
3328 WREG32(VGT_GS_VERTEX_REUSE, 16);
3329 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3330
3331 tmp = RREG32(HDP_MISC_CNTL);
3332 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3333 WREG32(HDP_MISC_CNTL, tmp);
3334
3335 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3336 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3337
3338 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3339 WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
3340
3341 udelay(50);
3342}
3343
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]3344/*
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3345 * GPU scratch registers helpers function.
3346 */
3347/**
3348 * cik_scratch_init - setup driver info for CP scratch regs
3349 *
3350 * @rdev: radeon_device pointer
3351 *
3352 * Set up the number and offset of the CP scratch registers.
3353 * NOTE: use of CP scratch registers is a legacy inferface and
3354 * is not used by default on newer asics (r6xx+). On newer asics,
3355 * memory buffers are used for fences rather than scratch regs.
3356 */
3357static void cik_scratch_init(struct radeon_device *rdev)
3358{
3359 int i;
3360
3361 rdev->scratch.num_reg = 7;
3362 rdev->scratch.reg_base = SCRATCH_REG0;
3363 for (i = 0; i < rdev->scratch.num_reg; i++) {
3364 rdev->scratch.free[i] = true;
3365 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3366 }
3367}
3368
3369/**
Alex Deucherfbc832c2012-07-20 14:41:35 -0400[diff] [blame]3370 * cik_ring_test - basic gfx ring test
3371 *
3372 * @rdev: radeon_device pointer
3373 * @ring: radeon_ring structure holding ring information
3374 *
3375 * Allocate a scratch register and write to it using the gfx ring (CIK).
3376 * Provides a basic gfx ring test to verify that the ring is working.
3377 * Used by cik_cp_gfx_resume();
3378 * Returns 0 on success, error on failure.
3379 */
3380int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
3381{
3382 uint32_t scratch;
3383 uint32_t tmp = 0;
3384 unsigned i;
3385 int r;
3386
3387 r = radeon_scratch_get(rdev, &scratch);
3388 if (r) {
3389 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
3390 return r;
3391 }
3392 WREG32(scratch, 0xCAFEDEAD);
3393 r = radeon_ring_lock(rdev, ring, 3);
3394 if (r) {
3395 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
3396 radeon_scratch_free(rdev, scratch);
3397 return r;
3398 }
3399 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3400 radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
3401 radeon_ring_write(ring, 0xDEADBEEF);
3402 radeon_ring_unlock_commit(rdev, ring);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]3403
Alex Deucherfbc832c2012-07-20 14:41:35 -0400[diff] [blame]3404 for (i = 0; i < rdev->usec_timeout; i++) {
3405 tmp = RREG32(scratch);
3406 if (tmp == 0xDEADBEEF)
3407 break;
3408 DRM_UDELAY(1);
3409 }
3410 if (i < rdev->usec_timeout) {
3411 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3412 } else {
3413 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
3414 ring->idx, scratch, tmp);
3415 r = -EINVAL;
3416 }
3417 radeon_scratch_free(rdev, scratch);
3418 return r;
3419}
3420
3421/**
Alex Deucherb07fdd32013-04-11 09:36:17 -0400[diff] [blame]3422 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3423 *
3424 * @rdev: radeon_device pointer
3425 * @fence: radeon fence object
3426 *
3427 * Emits a fence sequnce number on the gfx ring and flushes
3428 * GPU caches.
3429 */
Alex Deucherb07fdd32013-04-11 09:36:17 -0400[diff] [blame]3430void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3431 struct radeon_fence *fence)
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3432{
3433 struct radeon_ring *ring = &rdev->ring[fence->ring];
3434 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3435
3436 /* EVENT_WRITE_EOP - flush caches, send int */
3437 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3438 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3439 EOP_TC_ACTION_EN |
3440 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3441 EVENT_INDEX(5)));
3442 radeon_ring_write(ring, addr & 0xfffffffc);
3443 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3444 radeon_ring_write(ring, fence->seq);
3445 radeon_ring_write(ring, 0);
3446 /* HDP flush */
3447 /* We should be using the new WAIT_REG_MEM special op packet here
3448 * but it causes the CP to hang
3449 */
3450 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3451 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
3452 WRITE_DATA_DST_SEL(0)));
3453 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
3454 radeon_ring_write(ring, 0);
3455 radeon_ring_write(ring, 0);
3456}
3457
Alex Deucherb07fdd32013-04-11 09:36:17 -0400[diff] [blame]3458/**
3459 * cik_fence_compute_ring_emit - emit a fence on the compute ring
3460 *
3461 * @rdev: radeon_device pointer
3462 * @fence: radeon fence object
3463 *
3464 * Emits a fence sequnce number on the compute ring and flushes
3465 * GPU caches.
3466 */
3467void cik_fence_compute_ring_emit(struct radeon_device *rdev,
3468 struct radeon_fence *fence)
3469{
3470 struct radeon_ring *ring = &rdev->ring[fence->ring];
3471 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3472
3473 /* RELEASE_MEM - flush caches, send int */
3474 radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
3475 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3476 EOP_TC_ACTION_EN |
3477 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3478 EVENT_INDEX(5)));
3479 radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
3480 radeon_ring_write(ring, addr & 0xfffffffc);
3481 radeon_ring_write(ring, upper_32_bits(addr));
3482 radeon_ring_write(ring, fence->seq);
3483 radeon_ring_write(ring, 0);
3484 /* HDP flush */
3485 /* We should be using the new WAIT_REG_MEM special op packet here
3486 * but it causes the CP to hang
3487 */
3488 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3489 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
3490 WRITE_DATA_DST_SEL(0)));
3491 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
3492 radeon_ring_write(ring, 0);
3493 radeon_ring_write(ring, 0);
3494}
3495
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3496void cik_semaphore_ring_emit(struct radeon_device *rdev,
3497 struct radeon_ring *ring,
3498 struct radeon_semaphore *semaphore,
3499 bool emit_wait)
3500{
3501 uint64_t addr = semaphore->gpu_addr;
3502 unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3503
3504 radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
3505 radeon_ring_write(ring, addr & 0xffffffff);
3506 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
3507}
3508
Alex Deucherc9dbd702013-10-01 16:36:51 -0400[diff] [blame]3509/**
3510 * cik_copy_cpdma - copy pages using the CP DMA engine
3511 *
3512 * @rdev: radeon_device pointer
3513 * @src_offset: src GPU address
3514 * @dst_offset: dst GPU address
3515 * @num_gpu_pages: number of GPU pages to xfer
3516 * @fence: radeon fence object
3517 *
3518 * Copy GPU paging using the CP DMA engine (CIK+).
3519 * Used by the radeon ttm implementation to move pages if
3520 * registered as the asic copy callback.
3521 */
3522int cik_copy_cpdma(struct radeon_device *rdev,
3523 uint64_t src_offset, uint64_t dst_offset,
3524 unsigned num_gpu_pages,
3525 struct radeon_fence **fence)
3526{
3527 struct radeon_semaphore *sem = NULL;
3528 int ring_index = rdev->asic->copy.blit_ring_index;
3529 struct radeon_ring *ring = &rdev->ring[ring_index];
3530 u32 size_in_bytes, cur_size_in_bytes, control;
3531 int i, num_loops;
3532 int r = 0;
3533
3534 r = radeon_semaphore_create(rdev, &sem);
3535 if (r) {
3536 DRM_ERROR("radeon: moving bo (%d).\n", r);
3537 return r;
3538 }
3539
3540 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
3541 num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
3542 r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18);
3543 if (r) {
3544 DRM_ERROR("radeon: moving bo (%d).\n", r);
3545 radeon_semaphore_free(rdev, &sem, NULL);
3546 return r;
3547 }
3548
3549 if (radeon_fence_need_sync(*fence, ring->idx)) {
3550 radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring,
3551 ring->idx);
3552 radeon_fence_note_sync(*fence, ring->idx);
3553 } else {
3554 radeon_semaphore_free(rdev, &sem, NULL);
3555 }
3556
3557 for (i = 0; i < num_loops; i++) {
3558 cur_size_in_bytes = size_in_bytes;
3559 if (cur_size_in_bytes > 0x1fffff)
3560 cur_size_in_bytes = 0x1fffff;
3561 size_in_bytes -= cur_size_in_bytes;
3562 control = 0;
3563 if (size_in_bytes == 0)
3564 control |= PACKET3_DMA_DATA_CP_SYNC;
3565 radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
3566 radeon_ring_write(ring, control);
3567 radeon_ring_write(ring, lower_32_bits(src_offset));
3568 radeon_ring_write(ring, upper_32_bits(src_offset));
3569 radeon_ring_write(ring, lower_32_bits(dst_offset));
3570 radeon_ring_write(ring, upper_32_bits(dst_offset));
3571 radeon_ring_write(ring, cur_size_in_bytes);
3572 src_offset += cur_size_in_bytes;
3573 dst_offset += cur_size_in_bytes;
3574 }
3575
3576 r = radeon_fence_emit(rdev, fence, ring->idx);
3577 if (r) {
3578 radeon_ring_unlock_undo(rdev, ring);
3579 return r;
3580 }
3581
3582 radeon_ring_unlock_commit(rdev, ring);
3583 radeon_semaphore_free(rdev, &sem, *fence);
3584
3585 return r;
3586}
3587
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3588/*
3589 * IB stuff
3590 */
3591/**
3592 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
3593 *
3594 * @rdev: radeon_device pointer
3595 * @ib: radeon indirect buffer object
3596 *
3597 * Emits an DE (drawing engine) or CE (constant engine) IB
3598 * on the gfx ring. IBs are usually generated by userspace
3599 * acceleration drivers and submitted to the kernel for
3600 * sheduling on the ring. This function schedules the IB
3601 * on the gfx ring for execution by the GPU.
3602 */
3603void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3604{
3605 struct radeon_ring *ring = &rdev->ring[ib->ring];
3606 u32 header, control = INDIRECT_BUFFER_VALID;
3607
3608 if (ib->is_const_ib) {
3609 /* set switch buffer packet before const IB */
3610 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3611 radeon_ring_write(ring, 0);
3612
3613 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3614 } else {
3615 u32 next_rptr;
3616 if (ring->rptr_save_reg) {
3617 next_rptr = ring->wptr + 3 + 4;
3618 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3619 radeon_ring_write(ring, ((ring->rptr_save_reg -
3620 PACKET3_SET_UCONFIG_REG_START) >> 2));
3621 radeon_ring_write(ring, next_rptr);
3622 } else if (rdev->wb.enabled) {
3623 next_rptr = ring->wptr + 5 + 4;
3624 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3625 radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
3626 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3627 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
3628 radeon_ring_write(ring, next_rptr);
3629 }
3630
3631 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3632 }
3633
3634 control |= ib->length_dw |
3635 (ib->vm ? (ib->vm->id << 24) : 0);
3636
3637 radeon_ring_write(ring, header);
3638 radeon_ring_write(ring,
3639#ifdef __BIG_ENDIAN
3640 (2 << 0) |
3641#endif
3642 (ib->gpu_addr & 0xFFFFFFFC));
3643 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3644 radeon_ring_write(ring, control);
3645}
3646
Alex Deucherfbc832c2012-07-20 14:41:35 -0400[diff] [blame]3647/**
3648 * cik_ib_test - basic gfx ring IB test
3649 *
3650 * @rdev: radeon_device pointer
3651 * @ring: radeon_ring structure holding ring information
3652 *
3653 * Allocate an IB and execute it on the gfx ring (CIK).
3654 * Provides a basic gfx ring test to verify that IBs are working.
3655 * Returns 0 on success, error on failure.
3656 */
3657int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3658{
3659 struct radeon_ib ib;
3660 uint32_t scratch;
3661 uint32_t tmp = 0;
3662 unsigned i;
3663 int r;
3664
3665 r = radeon_scratch_get(rdev, &scratch);
3666 if (r) {
3667 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
3668 return r;
3669 }
3670 WREG32(scratch, 0xCAFEDEAD);
3671 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3672 if (r) {
3673 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
Christian König5510f122013-10-14 11:32:28 +0200[diff] [blame]3674 radeon_scratch_free(rdev, scratch);
Alex Deucherfbc832c2012-07-20 14:41:35 -0400[diff] [blame]3675 return r;
3676 }
3677 ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
3678 ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
3679 ib.ptr[2] = 0xDEADBEEF;
3680 ib.length_dw = 3;
3681 r = radeon_ib_schedule(rdev, &ib, NULL);
3682 if (r) {
3683 radeon_scratch_free(rdev, scratch);
3684 radeon_ib_free(rdev, &ib);
3685 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
3686 return r;
3687 }
3688 r = radeon_fence_wait(ib.fence, false);
3689 if (r) {
3690 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
Christian König5510f122013-10-14 11:32:28 +0200[diff] [blame]3691 radeon_scratch_free(rdev, scratch);
3692 radeon_ib_free(rdev, &ib);
Alex Deucherfbc832c2012-07-20 14:41:35 -0400[diff] [blame]3693 return r;
3694 }
3695 for (i = 0; i < rdev->usec_timeout; i++) {
3696 tmp = RREG32(scratch);
3697 if (tmp == 0xDEADBEEF)
3698 break;
3699 DRM_UDELAY(1);
3700 }
3701 if (i < rdev->usec_timeout) {
3702 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3703 } else {
3704 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
3705 scratch, tmp);
3706 r = -EINVAL;
3707 }
3708 radeon_scratch_free(rdev, scratch);
3709 radeon_ib_free(rdev, &ib);
3710 return r;
3711}
3712
Alex Deucher2cae3bc2012-07-05 11:45:40 -0400[diff] [blame]3713/*
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]3714 * CP.
3715 * On CIK, gfx and compute now have independant command processors.
3716 *
3717 * GFX
3718 * Gfx consists of a single ring and can process both gfx jobs and
3719 * compute jobs. The gfx CP consists of three microengines (ME):
3720 * PFP - Pre-Fetch Parser
3721 * ME - Micro Engine
3722 * CE - Constant Engine
3723 * The PFP and ME make up what is considered the Drawing Engine (DE).
3724 * The CE is an asynchronous engine used for updating buffer desciptors
3725 * used by the DE so that they can be loaded into cache in parallel
3726 * while the DE is processing state update packets.
3727 *
3728 * Compute
3729 * The compute CP consists of two microengines (ME):
3730 * MEC1 - Compute MicroEngine 1
3731 * MEC2 - Compute MicroEngine 2
3732 * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
3733 * The queues are exposed to userspace and are programmed directly
3734 * by the compute runtime.
3735 */
3736/**
3737 * cik_cp_gfx_enable - enable/disable the gfx CP MEs
3738 *
3739 * @rdev: radeon_device pointer
3740 * @enable: enable or disable the MEs
3741 *
3742 * Halts or unhalts the gfx MEs.
3743 */
3744static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
3745{
3746 if (enable)
3747 WREG32(CP_ME_CNTL, 0);
3748 else {
3749 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3750 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3751 }
3752 udelay(50);
3753}
3754
3755/**
3756 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
3757 *
3758 * @rdev: radeon_device pointer
3759 *
3760 * Loads the gfx PFP, ME, and CE ucode.
3761 * Returns 0 for success, -EINVAL if the ucode is not available.
3762 */
3763static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
3764{
3765 const __be32 *fw_data;
3766 int i;
3767
3768 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
3769 return -EINVAL;
3770
3771 cik_cp_gfx_enable(rdev, false);
3772
3773 /* PFP */
3774 fw_data = (const __be32 *)rdev->pfp_fw->data;
3775 WREG32(CP_PFP_UCODE_ADDR, 0);
3776 for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
3777 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3778 WREG32(CP_PFP_UCODE_ADDR, 0);
3779
3780 /* CE */
3781 fw_data = (const __be32 *)rdev->ce_fw->data;
3782 WREG32(CP_CE_UCODE_ADDR, 0);
3783 for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
3784 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3785 WREG32(CP_CE_UCODE_ADDR, 0);
3786
3787 /* ME */
3788 fw_data = (const __be32 *)rdev->me_fw->data;
3789 WREG32(CP_ME_RAM_WADDR, 0);
3790 for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
3791 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3792 WREG32(CP_ME_RAM_WADDR, 0);
3793
3794 WREG32(CP_PFP_UCODE_ADDR, 0);
3795 WREG32(CP_CE_UCODE_ADDR, 0);
3796 WREG32(CP_ME_RAM_WADDR, 0);
3797 WREG32(CP_ME_RAM_RADDR, 0);
3798 return 0;
3799}
3800
3801/**
3802 * cik_cp_gfx_start - start the gfx ring
3803 *
3804 * @rdev: radeon_device pointer
3805 *
3806 * Enables the ring and loads the clear state context and other
3807 * packets required to init the ring.
3808 * Returns 0 for success, error for failure.
3809 */
3810static int cik_cp_gfx_start(struct radeon_device *rdev)
3811{
3812 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3813 int r, i;
3814
3815 /* init the CP */
3816 WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
3817 WREG32(CP_ENDIAN_SWAP, 0);
3818 WREG32(CP_DEVICE_ID, 1);
3819
3820 cik_cp_gfx_enable(rdev, true);
3821
3822 r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
3823 if (r) {
3824 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3825 return r;
3826 }
3827
3828 /* init the CE partitions. CE only used for gfx on CIK */
3829 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3830 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3831 radeon_ring_write(ring, 0xc000);
3832 radeon_ring_write(ring, 0xc000);
3833
3834 /* setup clear context state */
3835 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3836 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3837
3838 radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
3839 radeon_ring_write(ring, 0x80000000);
3840 radeon_ring_write(ring, 0x80000000);
3841
3842 for (i = 0; i < cik_default_size; i++)
3843 radeon_ring_write(ring, cik_default_state[i]);
3844
3845 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3846 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3847
3848 /* set clear context state */
3849 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3850 radeon_ring_write(ring, 0);
3851
3852 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3853 radeon_ring_write(ring, 0x00000316);
3854 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3855 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3856
3857 radeon_ring_unlock_commit(rdev, ring);
3858
3859 return 0;
3860}
3861
3862/**
3863 * cik_cp_gfx_fini - stop the gfx ring
3864 *
3865 * @rdev: radeon_device pointer
3866 *
3867 * Stop the gfx ring and tear down the driver ring
3868 * info.
3869 */
3870static void cik_cp_gfx_fini(struct radeon_device *rdev)
3871{
3872 cik_cp_gfx_enable(rdev, false);
3873 radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3874}
3875
3876/**
3877 * cik_cp_gfx_resume - setup the gfx ring buffer registers
3878 *
3879 * @rdev: radeon_device pointer
3880 *
3881 * Program the location and size of the gfx ring buffer
3882 * and test it to make sure it's working.
3883 * Returns 0 for success, error for failure.
3884 */
3885static int cik_cp_gfx_resume(struct radeon_device *rdev)
3886{
3887 struct radeon_ring *ring;
3888 u32 tmp;
3889 u32 rb_bufsz;
3890 u64 rb_addr;
3891 int r;
3892
3893 WREG32(CP_SEM_WAIT_TIMER, 0x0);
Alex Deucher939c0d32013-09-30 18:03:06 -0400[diff] [blame]3894 if (rdev->family != CHIP_HAWAII)
3895 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]3896
3897 /* Set the write pointer delay */
3898 WREG32(CP_RB_WPTR_DELAY, 0);
3899
3900 /* set the RB to use vmid 0 */
3901 WREG32(CP_RB_VMID, 0);
3902
3903 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3904
3905 /* ring 0 - compute and gfx */
3906 /* Set ring buffer size */
3907 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
Daniel Vetterb72a8922013-07-10 14:11:59 +0200[diff] [blame]3908 rb_bufsz = order_base_2(ring->ring_size / 8);
3909 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]3910#ifdef __BIG_ENDIAN
3911 tmp |= BUF_SWAP_32BIT;
3912#endif
3913 WREG32(CP_RB0_CNTL, tmp);
3914
3915 /* Initialize the ring buffer's read and write pointers */
3916 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3917 ring->wptr = 0;
3918 WREG32(CP_RB0_WPTR, ring->wptr);
3919
3920 /* set the wb address wether it's enabled or not */
3921 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3922 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3923
3924 /* scratch register shadowing is no longer supported */
3925 WREG32(SCRATCH_UMSK, 0);
3926
3927 if (!rdev->wb.enabled)
3928 tmp |= RB_NO_UPDATE;
3929
3930 mdelay(1);
3931 WREG32(CP_RB0_CNTL, tmp);
3932
3933 rb_addr = ring->gpu_addr >> 8;
3934 WREG32(CP_RB0_BASE, rb_addr);
3935 WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
3936
3937 ring->rptr = RREG32(CP_RB0_RPTR);
3938
3939 /* start the ring */
3940 cik_cp_gfx_start(rdev);
3941 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
3942 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3943 if (r) {
3944 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3945 return r;
3946 }
3947 return 0;
3948}
3949
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]3950u32 cik_compute_ring_get_rptr(struct radeon_device *rdev,
3951 struct radeon_ring *ring)
3952{
3953 u32 rptr;
3954
3955
3956
3957 if (rdev->wb.enabled) {
3958 rptr = le32_to_cpu(rdev->wb.wb[ring->rptr_offs/4]);
3959 } else {
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]3960 mutex_lock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]3961 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
3962 rptr = RREG32(CP_HQD_PQ_RPTR);
3963 cik_srbm_select(rdev, 0, 0, 0, 0);
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]3964 mutex_unlock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]3965 }
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]3966
3967 return rptr;
3968}
3969
3970u32 cik_compute_ring_get_wptr(struct radeon_device *rdev,
3971 struct radeon_ring *ring)
3972{
3973 u32 wptr;
3974
3975 if (rdev->wb.enabled) {
3976 wptr = le32_to_cpu(rdev->wb.wb[ring->wptr_offs/4]);
3977 } else {
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]3978 mutex_lock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]3979 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
3980 wptr = RREG32(CP_HQD_PQ_WPTR);
3981 cik_srbm_select(rdev, 0, 0, 0, 0);
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]3982 mutex_unlock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]3983 }
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]3984
3985 return wptr;
3986}
3987
3988void cik_compute_ring_set_wptr(struct radeon_device *rdev,
3989 struct radeon_ring *ring)
3990{
Christian König2e1e6da2013-08-13 11:56:52 +0200[diff] [blame]3991 rdev->wb.wb[ring->wptr_offs/4] = cpu_to_le32(ring->wptr);
3992 WDOORBELL32(ring->doorbell_offset, ring->wptr);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]3993}
3994
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]3995/**
3996 * cik_cp_compute_enable - enable/disable the compute CP MEs
3997 *
3998 * @rdev: radeon_device pointer
3999 * @enable: enable or disable the MEs
4000 *
4001 * Halts or unhalts the compute MEs.
4002 */
4003static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
4004{
4005 if (enable)
4006 WREG32(CP_MEC_CNTL, 0);
4007 else
4008 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
4009 udelay(50);
4010}
4011
4012/**
4013 * cik_cp_compute_load_microcode - load the compute CP ME ucode
4014 *
4015 * @rdev: radeon_device pointer
4016 *
4017 * Loads the compute MEC1&2 ucode.
4018 * Returns 0 for success, -EINVAL if the ucode is not available.
4019 */
4020static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
4021{
4022 const __be32 *fw_data;
4023 int i;
4024
4025 if (!rdev->mec_fw)
4026 return -EINVAL;
4027
4028 cik_cp_compute_enable(rdev, false);
4029
4030 /* MEC1 */
4031 fw_data = (const __be32 *)rdev->mec_fw->data;
4032 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4033 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4034 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
4035 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4036
4037 if (rdev->family == CHIP_KAVERI) {
4038 /* MEC2 */
4039 fw_data = (const __be32 *)rdev->mec_fw->data;
4040 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4041 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4042 WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
4043 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4044 }
4045
4046 return 0;
4047}
4048
4049/**
4050 * cik_cp_compute_start - start the compute queues
4051 *
4052 * @rdev: radeon_device pointer
4053 *
4054 * Enable the compute queues.
4055 * Returns 0 for success, error for failure.
4056 */
4057static int cik_cp_compute_start(struct radeon_device *rdev)
4058{
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4059 cik_cp_compute_enable(rdev, true);
4060
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4061 return 0;
4062}
4063
4064/**
4065 * cik_cp_compute_fini - stop the compute queues
4066 *
4067 * @rdev: radeon_device pointer
4068 *
4069 * Stop the compute queues and tear down the driver queue
4070 * info.
4071 */
4072static void cik_cp_compute_fini(struct radeon_device *rdev)
4073{
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4074 int i, idx, r;
4075
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4076 cik_cp_compute_enable(rdev, false);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4077
4078 for (i = 0; i < 2; i++) {
4079 if (i == 0)
4080 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4081 else
4082 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4083
4084 if (rdev->ring[idx].mqd_obj) {
4085 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4086 if (unlikely(r != 0))
4087 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
4088
4089 radeon_bo_unpin(rdev->ring[idx].mqd_obj);
4090 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4091
4092 radeon_bo_unref(&rdev->ring[idx].mqd_obj);
4093 rdev->ring[idx].mqd_obj = NULL;
4094 }
4095 }
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4096}
4097
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4098static void cik_mec_fini(struct radeon_device *rdev)
4099{
4100 int r;
4101
4102 if (rdev->mec.hpd_eop_obj) {
4103 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4104 if (unlikely(r != 0))
4105 dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
4106 radeon_bo_unpin(rdev->mec.hpd_eop_obj);
4107 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4108
4109 radeon_bo_unref(&rdev->mec.hpd_eop_obj);
4110 rdev->mec.hpd_eop_obj = NULL;
4111 }
4112}
4113
4114#define MEC_HPD_SIZE 2048
4115
4116static int cik_mec_init(struct radeon_device *rdev)
4117{
4118 int r;
4119 u32 *hpd;
4120
4121 /*
4122 * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
4123 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
4124 */
4125 if (rdev->family == CHIP_KAVERI)
4126 rdev->mec.num_mec = 2;
4127 else
4128 rdev->mec.num_mec = 1;
4129 rdev->mec.num_pipe = 4;
4130 rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
4131
4132 if (rdev->mec.hpd_eop_obj == NULL) {
4133 r = radeon_bo_create(rdev,
4134 rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
4135 PAGE_SIZE, true,
4136 RADEON_GEM_DOMAIN_GTT, NULL,
4137 &rdev->mec.hpd_eop_obj);
4138 if (r) {
4139 dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
4140 return r;
4141 }
4142 }
4143
4144 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4145 if (unlikely(r != 0)) {
4146 cik_mec_fini(rdev);
4147 return r;
4148 }
4149 r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
4150 &rdev->mec.hpd_eop_gpu_addr);
4151 if (r) {
4152 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
4153 cik_mec_fini(rdev);
4154 return r;
4155 }
4156 r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
4157 if (r) {
4158 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
4159 cik_mec_fini(rdev);
4160 return r;
4161 }
4162
4163 /* clear memory. Not sure if this is required or not */
4164 memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
4165
4166 radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
4167 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4168
4169 return 0;
4170}
4171
4172struct hqd_registers
4173{
4174 u32 cp_mqd_base_addr;
4175 u32 cp_mqd_base_addr_hi;
4176 u32 cp_hqd_active;
4177 u32 cp_hqd_vmid;
4178 u32 cp_hqd_persistent_state;
4179 u32 cp_hqd_pipe_priority;
4180 u32 cp_hqd_queue_priority;
4181 u32 cp_hqd_quantum;
4182 u32 cp_hqd_pq_base;
4183 u32 cp_hqd_pq_base_hi;
4184 u32 cp_hqd_pq_rptr;
4185 u32 cp_hqd_pq_rptr_report_addr;
4186 u32 cp_hqd_pq_rptr_report_addr_hi;
4187 u32 cp_hqd_pq_wptr_poll_addr;
4188 u32 cp_hqd_pq_wptr_poll_addr_hi;
4189 u32 cp_hqd_pq_doorbell_control;
4190 u32 cp_hqd_pq_wptr;
4191 u32 cp_hqd_pq_control;
4192 u32 cp_hqd_ib_base_addr;
4193 u32 cp_hqd_ib_base_addr_hi;
4194 u32 cp_hqd_ib_rptr;
4195 u32 cp_hqd_ib_control;
4196 u32 cp_hqd_iq_timer;
4197 u32 cp_hqd_iq_rptr;
4198 u32 cp_hqd_dequeue_request;
4199 u32 cp_hqd_dma_offload;
4200 u32 cp_hqd_sema_cmd;
4201 u32 cp_hqd_msg_type;
4202 u32 cp_hqd_atomic0_preop_lo;
4203 u32 cp_hqd_atomic0_preop_hi;
4204 u32 cp_hqd_atomic1_preop_lo;
4205 u32 cp_hqd_atomic1_preop_hi;
4206 u32 cp_hqd_hq_scheduler0;
4207 u32 cp_hqd_hq_scheduler1;
4208 u32 cp_mqd_control;
4209};
4210
4211struct bonaire_mqd
4212{
4213 u32 header;
4214 u32 dispatch_initiator;
4215 u32 dimensions[3];
4216 u32 start_idx[3];
4217 u32 num_threads[3];
4218 u32 pipeline_stat_enable;
4219 u32 perf_counter_enable;
4220 u32 pgm[2];
4221 u32 tba[2];
4222 u32 tma[2];
4223 u32 pgm_rsrc[2];
4224 u32 vmid;
4225 u32 resource_limits;
4226 u32 static_thread_mgmt01[2];
4227 u32 tmp_ring_size;
4228 u32 static_thread_mgmt23[2];
4229 u32 restart[3];
4230 u32 thread_trace_enable;
4231 u32 reserved1;
4232 u32 user_data[16];
4233 u32 vgtcs_invoke_count[2];
4234 struct hqd_registers queue_state;
4235 u32 dequeue_cntr;
4236 u32 interrupt_queue[64];
4237};
4238
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4239/**
4240 * cik_cp_compute_resume - setup the compute queue registers
4241 *
4242 * @rdev: radeon_device pointer
4243 *
4244 * Program the compute queues and test them to make sure they
4245 * are working.
4246 * Returns 0 for success, error for failure.
4247 */
4248static int cik_cp_compute_resume(struct radeon_device *rdev)
4249{
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4250 int r, i, idx;
4251 u32 tmp;
4252 bool use_doorbell = true;
4253 u64 hqd_gpu_addr;
4254 u64 mqd_gpu_addr;
4255 u64 eop_gpu_addr;
4256 u64 wb_gpu_addr;
4257 u32 *buf;
4258 struct bonaire_mqd *mqd;
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4259
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4260 r = cik_cp_compute_start(rdev);
4261 if (r)
4262 return r;
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4263
4264 /* fix up chicken bits */
4265 tmp = RREG32(CP_CPF_DEBUG);
4266 tmp |= (1 << 23);
4267 WREG32(CP_CPF_DEBUG, tmp);
4268
4269 /* init the pipes */
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]4270 mutex_lock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4271 for (i = 0; i < (rdev->mec.num_pipe * rdev->mec.num_mec); i++) {
4272 int me = (i < 4) ? 1 : 2;
4273 int pipe = (i < 4) ? i : (i - 4);
4274
4275 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2);
4276
4277 cik_srbm_select(rdev, me, pipe, 0, 0);
4278
4279 /* write the EOP addr */
4280 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
4281 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
4282
4283 /* set the VMID assigned */
4284 WREG32(CP_HPD_EOP_VMID, 0);
4285
4286 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4287 tmp = RREG32(CP_HPD_EOP_CONTROL);
4288 tmp &= ~EOP_SIZE_MASK;
Daniel Vetterb72a8922013-07-10 14:11:59 +0200[diff] [blame]4289 tmp |= order_base_2(MEC_HPD_SIZE / 8);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4290 WREG32(CP_HPD_EOP_CONTROL, tmp);
4291 }
4292 cik_srbm_select(rdev, 0, 0, 0, 0);
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]4293 mutex_unlock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4294
4295 /* init the queues. Just two for now. */
4296 for (i = 0; i < 2; i++) {
4297 if (i == 0)
4298 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4299 else
4300 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4301
4302 if (rdev->ring[idx].mqd_obj == NULL) {
4303 r = radeon_bo_create(rdev,
4304 sizeof(struct bonaire_mqd),
4305 PAGE_SIZE, true,
4306 RADEON_GEM_DOMAIN_GTT, NULL,
4307 &rdev->ring[idx].mqd_obj);
4308 if (r) {
4309 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
4310 return r;
4311 }
4312 }
4313
4314 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4315 if (unlikely(r != 0)) {
4316 cik_cp_compute_fini(rdev);
4317 return r;
4318 }
4319 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
4320 &mqd_gpu_addr);
4321 if (r) {
4322 dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
4323 cik_cp_compute_fini(rdev);
4324 return r;
4325 }
4326 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
4327 if (r) {
4328 dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
4329 cik_cp_compute_fini(rdev);
4330 return r;
4331 }
4332
4333 /* doorbell offset */
4334 rdev->ring[idx].doorbell_offset =
4335 (rdev->ring[idx].doorbell_page_num * PAGE_SIZE) + 0;
4336
4337 /* init the mqd struct */
4338 memset(buf, 0, sizeof(struct bonaire_mqd));
4339
4340 mqd = (struct bonaire_mqd *)buf;
4341 mqd->header = 0xC0310800;
4342 mqd->static_thread_mgmt01[0] = 0xffffffff;
4343 mqd->static_thread_mgmt01[1] = 0xffffffff;
4344 mqd->static_thread_mgmt23[0] = 0xffffffff;
4345 mqd->static_thread_mgmt23[1] = 0xffffffff;
4346
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]4347 mutex_lock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4348 cik_srbm_select(rdev, rdev->ring[idx].me,
4349 rdev->ring[idx].pipe,
4350 rdev->ring[idx].queue, 0);
4351
4352 /* disable wptr polling */
4353 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4354 tmp &= ~WPTR_POLL_EN;
4355 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4356
4357 /* enable doorbell? */
4358 mqd->queue_state.cp_hqd_pq_doorbell_control =
4359 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4360 if (use_doorbell)
4361 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4362 else
4363 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
4364 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4365 mqd->queue_state.cp_hqd_pq_doorbell_control);
4366
4367 /* disable the queue if it's active */
4368 mqd->queue_state.cp_hqd_dequeue_request = 0;
4369 mqd->queue_state.cp_hqd_pq_rptr = 0;
4370 mqd->queue_state.cp_hqd_pq_wptr= 0;
4371 if (RREG32(CP_HQD_ACTIVE) & 1) {
4372 WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
4373 for (i = 0; i < rdev->usec_timeout; i++) {
4374 if (!(RREG32(CP_HQD_ACTIVE) & 1))
4375 break;
4376 udelay(1);
4377 }
4378 WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
4379 WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
4380 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4381 }
4382
4383 /* set the pointer to the MQD */
4384 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
4385 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
4386 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
4387 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
4388 /* set MQD vmid to 0 */
4389 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
4390 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
4391 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
4392
4393 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4394 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
4395 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
4396 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4397 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
4398 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
4399
4400 /* set up the HQD, this is similar to CP_RB0_CNTL */
4401 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
4402 mqd->queue_state.cp_hqd_pq_control &=
4403 ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
4404
4405 mqd->queue_state.cp_hqd_pq_control |=
Daniel Vetterb72a8922013-07-10 14:11:59 +0200[diff] [blame]4406 order_base_2(rdev->ring[idx].ring_size / 8);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4407 mqd->queue_state.cp_hqd_pq_control |=
Daniel Vetterb72a8922013-07-10 14:11:59 +0200[diff] [blame]4408 (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4409#ifdef __BIG_ENDIAN
4410 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
4411#endif
4412 mqd->queue_state.cp_hqd_pq_control &=
4413 ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
4414 mqd->queue_state.cp_hqd_pq_control |=
4415 PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
4416 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
4417
4418 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
4419 if (i == 0)
4420 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
4421 else
4422 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
4423 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
4424 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4425 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
4426 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
4427 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
4428
4429 /* set the wb address wether it's enabled or not */
4430 if (i == 0)
4431 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
4432 else
4433 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
4434 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
4435 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
4436 upper_32_bits(wb_gpu_addr) & 0xffff;
4437 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
4438 mqd->queue_state.cp_hqd_pq_rptr_report_addr);
4439 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
4440 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
4441
4442 /* enable the doorbell if requested */
4443 if (use_doorbell) {
4444 mqd->queue_state.cp_hqd_pq_doorbell_control =
4445 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4446 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
4447 mqd->queue_state.cp_hqd_pq_doorbell_control |=
4448 DOORBELL_OFFSET(rdev->ring[idx].doorbell_offset / 4);
4449 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4450 mqd->queue_state.cp_hqd_pq_doorbell_control &=
4451 ~(DOORBELL_SOURCE | DOORBELL_HIT);
4452
4453 } else {
4454 mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
4455 }
4456 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4457 mqd->queue_state.cp_hqd_pq_doorbell_control);
4458
4459 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4460 rdev->ring[idx].wptr = 0;
4461 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
4462 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4463 rdev->ring[idx].rptr = RREG32(CP_HQD_PQ_RPTR);
4464 mqd->queue_state.cp_hqd_pq_rptr = rdev->ring[idx].rptr;
4465
4466 /* set the vmid for the queue */
4467 mqd->queue_state.cp_hqd_vmid = 0;
4468 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
4469
4470 /* activate the queue */
4471 mqd->queue_state.cp_hqd_active = 1;
4472 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
4473
4474 cik_srbm_select(rdev, 0, 0, 0, 0);
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]4475 mutex_unlock(&rdev->srbm_mutex);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4476
4477 radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
4478 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4479
4480 rdev->ring[idx].ready = true;
4481 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
4482 if (r)
4483 rdev->ring[idx].ready = false;
4484 }
4485
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4486 return 0;
4487}
4488
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4489static void cik_cp_enable(struct radeon_device *rdev, bool enable)
4490{
4491 cik_cp_gfx_enable(rdev, enable);
4492 cik_cp_compute_enable(rdev, enable);
4493}
4494
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4495static int cik_cp_load_microcode(struct radeon_device *rdev)
4496{
4497 int r;
4498
4499 r = cik_cp_gfx_load_microcode(rdev);
4500 if (r)
4501 return r;
4502 r = cik_cp_compute_load_microcode(rdev);
4503 if (r)
4504 return r;
4505
4506 return 0;
4507}
4508
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4509static void cik_cp_fini(struct radeon_device *rdev)
4510{
4511 cik_cp_gfx_fini(rdev);
4512 cik_cp_compute_fini(rdev);
4513}
4514
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4515static int cik_cp_resume(struct radeon_device *rdev)
4516{
4517 int r;
4518
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]4519 cik_enable_gui_idle_interrupt(rdev, false);
4520
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4521 r = cik_cp_load_microcode(rdev);
4522 if (r)
4523 return r;
4524
4525 r = cik_cp_gfx_resume(rdev);
4526 if (r)
4527 return r;
4528 r = cik_cp_compute_resume(rdev);
4529 if (r)
4530 return r;
4531
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]4532 cik_enable_gui_idle_interrupt(rdev, true);
4533
Alex Deucher841cf442012-12-18 21:47:44 -0500[diff] [blame]4534 return 0;
4535}
4536
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]4537static void cik_print_gpu_status_regs(struct radeon_device *rdev)
4538{
4539 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
4540 RREG32(GRBM_STATUS));
4541 dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n",
4542 RREG32(GRBM_STATUS2));
4543 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
4544 RREG32(GRBM_STATUS_SE0));
4545 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
4546 RREG32(GRBM_STATUS_SE1));
4547 dev_info(rdev->dev, " GRBM_STATUS_SE2=0x%08X\n",
4548 RREG32(GRBM_STATUS_SE2));
4549 dev_info(rdev->dev, " GRBM_STATUS_SE3=0x%08X\n",
4550 RREG32(GRBM_STATUS_SE3));
4551 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
4552 RREG32(SRBM_STATUS));
4553 dev_info(rdev->dev, " SRBM_STATUS2=0x%08X\n",
4554 RREG32(SRBM_STATUS2));
4555 dev_info(rdev->dev, " SDMA0_STATUS_REG = 0x%08X\n",
4556 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
4557 dev_info(rdev->dev, " SDMA1_STATUS_REG = 0x%08X\n",
4558 RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]4559 dev_info(rdev->dev, " CP_STAT = 0x%08x\n", RREG32(CP_STAT));
4560 dev_info(rdev->dev, " CP_STALLED_STAT1 = 0x%08x\n",
4561 RREG32(CP_STALLED_STAT1));
4562 dev_info(rdev->dev, " CP_STALLED_STAT2 = 0x%08x\n",
4563 RREG32(CP_STALLED_STAT2));
4564 dev_info(rdev->dev, " CP_STALLED_STAT3 = 0x%08x\n",
4565 RREG32(CP_STALLED_STAT3));
4566 dev_info(rdev->dev, " CP_CPF_BUSY_STAT = 0x%08x\n",
4567 RREG32(CP_CPF_BUSY_STAT));
4568 dev_info(rdev->dev, " CP_CPF_STALLED_STAT1 = 0x%08x\n",
4569 RREG32(CP_CPF_STALLED_STAT1));
4570 dev_info(rdev->dev, " CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
4571 dev_info(rdev->dev, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
4572 dev_info(rdev->dev, " CP_CPC_STALLED_STAT1 = 0x%08x\n",
4573 RREG32(CP_CPC_STALLED_STAT1));
4574 dev_info(rdev->dev, " CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]4575}
4576
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]4577/**
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]4578 * cik_gpu_check_soft_reset - check which blocks are busy
4579 *
4580 * @rdev: radeon_device pointer
4581 *
4582 * Check which blocks are busy and return the relevant reset
4583 * mask to be used by cik_gpu_soft_reset().
4584 * Returns a mask of the blocks to be reset.
4585 */
Christian König2483b4e2013-08-13 11:56:54 +0200[diff] [blame]4586u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]4587{
4588 u32 reset_mask = 0;
4589 u32 tmp;
4590
4591 /* GRBM_STATUS */
4592 tmp = RREG32(GRBM_STATUS);
4593 if (tmp & (PA_BUSY | SC_BUSY |
4594 BCI_BUSY | SX_BUSY |
4595 TA_BUSY | VGT_BUSY |
4596 DB_BUSY | CB_BUSY |
4597 GDS_BUSY | SPI_BUSY |
4598 IA_BUSY | IA_BUSY_NO_DMA))
4599 reset_mask |= RADEON_RESET_GFX;
4600
4601 if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
4602 reset_mask |= RADEON_RESET_CP;
4603
4604 /* GRBM_STATUS2 */
4605 tmp = RREG32(GRBM_STATUS2);
4606 if (tmp & RLC_BUSY)
4607 reset_mask |= RADEON_RESET_RLC;
4608
4609 /* SDMA0_STATUS_REG */
4610 tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
4611 if (!(tmp & SDMA_IDLE))
4612 reset_mask |= RADEON_RESET_DMA;
4613
4614 /* SDMA1_STATUS_REG */
4615 tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
4616 if (!(tmp & SDMA_IDLE))
4617 reset_mask |= RADEON_RESET_DMA1;
4618
4619 /* SRBM_STATUS2 */
4620 tmp = RREG32(SRBM_STATUS2);
4621 if (tmp & SDMA_BUSY)
4622 reset_mask |= RADEON_RESET_DMA;
4623
4624 if (tmp & SDMA1_BUSY)
4625 reset_mask |= RADEON_RESET_DMA1;
4626
4627 /* SRBM_STATUS */
4628 tmp = RREG32(SRBM_STATUS);
4629
4630 if (tmp & IH_BUSY)
4631 reset_mask |= RADEON_RESET_IH;
4632
4633 if (tmp & SEM_BUSY)
4634 reset_mask |= RADEON_RESET_SEM;
4635
4636 if (tmp & GRBM_RQ_PENDING)
4637 reset_mask |= RADEON_RESET_GRBM;
4638
4639 if (tmp & VMC_BUSY)
4640 reset_mask |= RADEON_RESET_VMC;
4641
4642 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
4643 MCC_BUSY | MCD_BUSY))
4644 reset_mask |= RADEON_RESET_MC;
4645
4646 if (evergreen_is_display_hung(rdev))
4647 reset_mask |= RADEON_RESET_DISPLAY;
4648
4649 /* Skip MC reset as it's mostly likely not hung, just busy */
4650 if (reset_mask & RADEON_RESET_MC) {
4651 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
4652 reset_mask &= ~RADEON_RESET_MC;
4653 }
4654
4655 return reset_mask;
4656}
4657
4658/**
4659 * cik_gpu_soft_reset - soft reset GPU
4660 *
4661 * @rdev: radeon_device pointer
4662 * @reset_mask: mask of which blocks to reset
4663 *
4664 * Soft reset the blocks specified in @reset_mask.
4665 */
4666static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
4667{
4668 struct evergreen_mc_save save;
4669 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
4670 u32 tmp;
4671
4672 if (reset_mask == 0)
4673 return;
4674
4675 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
4676
4677 cik_print_gpu_status_regs(rdev);
4678 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
4679 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
4680 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4681 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
4682
Alex Deucherfb2c7f42013-10-02 14:54:44 -0400[diff] [blame]4683 /* disable CG/PG */
4684 cik_fini_pg(rdev);
4685 cik_fini_cg(rdev);
4686
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]4687 /* stop the rlc */
4688 cik_rlc_stop(rdev);
4689
4690 /* Disable GFX parsing/prefetching */
4691 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
4692
4693 /* Disable MEC parsing/prefetching */
4694 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
4695
4696 if (reset_mask & RADEON_RESET_DMA) {
4697 /* sdma0 */
4698 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
4699 tmp |= SDMA_HALT;
4700 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
4701 }
4702 if (reset_mask & RADEON_RESET_DMA1) {
4703 /* sdma1 */
4704 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
4705 tmp |= SDMA_HALT;
4706 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
4707 }
4708
4709 evergreen_mc_stop(rdev, &save);
4710 if (evergreen_mc_wait_for_idle(rdev)) {
4711 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4712 }
4713
4714 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
4715 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
4716
4717 if (reset_mask & RADEON_RESET_CP) {
4718 grbm_soft_reset |= SOFT_RESET_CP;
4719
4720 srbm_soft_reset |= SOFT_RESET_GRBM;
4721 }
4722
4723 if (reset_mask & RADEON_RESET_DMA)
4724 srbm_soft_reset |= SOFT_RESET_SDMA;
4725
4726 if (reset_mask & RADEON_RESET_DMA1)
4727 srbm_soft_reset |= SOFT_RESET_SDMA1;
4728
4729 if (reset_mask & RADEON_RESET_DISPLAY)
4730 srbm_soft_reset |= SOFT_RESET_DC;
4731
4732 if (reset_mask & RADEON_RESET_RLC)
4733 grbm_soft_reset |= SOFT_RESET_RLC;
4734
4735 if (reset_mask & RADEON_RESET_SEM)
4736 srbm_soft_reset |= SOFT_RESET_SEM;
4737
4738 if (reset_mask & RADEON_RESET_IH)
4739 srbm_soft_reset |= SOFT_RESET_IH;
4740
4741 if (reset_mask & RADEON_RESET_GRBM)
4742 srbm_soft_reset |= SOFT_RESET_GRBM;
4743
4744 if (reset_mask & RADEON_RESET_VMC)
4745 srbm_soft_reset |= SOFT_RESET_VMC;
4746
4747 if (!(rdev->flags & RADEON_IS_IGP)) {
4748 if (reset_mask & RADEON_RESET_MC)
4749 srbm_soft_reset |= SOFT_RESET_MC;
4750 }
4751
4752 if (grbm_soft_reset) {
4753 tmp = RREG32(GRBM_SOFT_RESET);
4754 tmp |= grbm_soft_reset;
4755 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
4756 WREG32(GRBM_SOFT_RESET, tmp);
4757 tmp = RREG32(GRBM_SOFT_RESET);
4758
4759 udelay(50);
4760
4761 tmp &= ~grbm_soft_reset;
4762 WREG32(GRBM_SOFT_RESET, tmp);
4763 tmp = RREG32(GRBM_SOFT_RESET);
4764 }
4765
4766 if (srbm_soft_reset) {
4767 tmp = RREG32(SRBM_SOFT_RESET);
4768 tmp |= srbm_soft_reset;
4769 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
4770 WREG32(SRBM_SOFT_RESET, tmp);
4771 tmp = RREG32(SRBM_SOFT_RESET);
4772
4773 udelay(50);
4774
4775 tmp &= ~srbm_soft_reset;
4776 WREG32(SRBM_SOFT_RESET, tmp);
4777 tmp = RREG32(SRBM_SOFT_RESET);
4778 }
4779
4780 /* Wait a little for things to settle down */
4781 udelay(50);
4782
4783 evergreen_mc_resume(rdev, &save);
4784 udelay(50);
4785
4786 cik_print_gpu_status_regs(rdev);
4787}
4788
4789/**
4790 * cik_asic_reset - soft reset GPU
4791 *
4792 * @rdev: radeon_device pointer
4793 *
4794 * Look up which blocks are hung and attempt
4795 * to reset them.
4796 * Returns 0 for success.
4797 */
4798int cik_asic_reset(struct radeon_device *rdev)
4799{
4800 u32 reset_mask;
4801
4802 reset_mask = cik_gpu_check_soft_reset(rdev);
4803
4804 if (reset_mask)
4805 r600_set_bios_scratch_engine_hung(rdev, true);
4806
4807 cik_gpu_soft_reset(rdev, reset_mask);
4808
4809 reset_mask = cik_gpu_check_soft_reset(rdev);
4810
4811 if (!reset_mask)
4812 r600_set_bios_scratch_engine_hung(rdev, false);
4813
4814 return 0;
4815}
4816
4817/**
4818 * cik_gfx_is_lockup - check if the 3D engine is locked up
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]4819 *
4820 * @rdev: radeon_device pointer
4821 * @ring: radeon_ring structure holding ring information
4822 *
4823 * Check if the 3D engine is locked up (CIK).
4824 * Returns true if the engine is locked, false if not.
4825 */
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]4826bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]4827{
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]4828 u32 reset_mask = cik_gpu_check_soft_reset(rdev);
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]4829
Alex Deuchercc066712013-04-09 12:59:51 -0400[diff] [blame]4830 if (!(reset_mask & (RADEON_RESET_GFX |
4831 RADEON_RESET_COMPUTE |
4832 RADEON_RESET_CP))) {
Alex Deucher6f2043c2013-04-09 12:43:41 -0400[diff] [blame]4833 radeon_ring_lockup_update(ring);
4834 return false;
4835 }
4836 /* force CP activities */
4837 radeon_ring_force_activity(rdev, ring);
4838 return radeon_ring_test_lockup(rdev, ring);
4839}
4840
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]4841/* MC */
4842/**
4843 * cik_mc_program - program the GPU memory controller
4844 *
4845 * @rdev: radeon_device pointer
4846 *
4847 * Set the location of vram, gart, and AGP in the GPU's
4848 * physical address space (CIK).
4849 */
4850static void cik_mc_program(struct radeon_device *rdev)
4851{
4852 struct evergreen_mc_save save;
4853 u32 tmp;
4854 int i, j;
4855
4856 /* Initialize HDP */
4857 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
4858 WREG32((0x2c14 + j), 0x00000000);
4859 WREG32((0x2c18 + j), 0x00000000);
4860 WREG32((0x2c1c + j), 0x00000000);
4861 WREG32((0x2c20 + j), 0x00000000);
4862 WREG32((0x2c24 + j), 0x00000000);
4863 }
4864 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
4865
4866 evergreen_mc_stop(rdev, &save);
4867 if (radeon_mc_wait_for_idle(rdev)) {
4868 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4869 }
4870 /* Lockout access through VGA aperture*/
4871 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
4872 /* Update configuration */
4873 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
4874 rdev->mc.vram_start >> 12);
4875 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
4876 rdev->mc.vram_end >> 12);
4877 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
4878 rdev->vram_scratch.gpu_addr >> 12);
4879 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
4880 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
4881 WREG32(MC_VM_FB_LOCATION, tmp);
4882 /* XXX double check these! */
4883 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
4884 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
4885 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
4886 WREG32(MC_VM_AGP_BASE, 0);
4887 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
4888 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
4889 if (radeon_mc_wait_for_idle(rdev)) {
4890 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4891 }
4892 evergreen_mc_resume(rdev, &save);
4893 /* we need to own VRAM, so turn off the VGA renderer here
4894 * to stop it overwriting our objects */
4895 rv515_vga_render_disable(rdev);
4896}
4897
4898/**
4899 * cik_mc_init - initialize the memory controller driver params
4900 *
4901 * @rdev: radeon_device pointer
4902 *
4903 * Look up the amount of vram, vram width, and decide how to place
4904 * vram and gart within the GPU's physical address space (CIK).
4905 * Returns 0 for success.
4906 */
4907static int cik_mc_init(struct radeon_device *rdev)
4908{
4909 u32 tmp;
4910 int chansize, numchan;
4911
4912 /* Get VRAM informations */
4913 rdev->mc.vram_is_ddr = true;
4914 tmp = RREG32(MC_ARB_RAMCFG);
4915 if (tmp & CHANSIZE_MASK) {
4916 chansize = 64;
4917 } else {
4918 chansize = 32;
4919 }
4920 tmp = RREG32(MC_SHARED_CHMAP);
4921 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
4922 case 0:
4923 default:
4924 numchan = 1;
4925 break;
4926 case 1:
4927 numchan = 2;
4928 break;
4929 case 2:
4930 numchan = 4;
4931 break;
4932 case 3:
4933 numchan = 8;
4934 break;
4935 case 4:
4936 numchan = 3;
4937 break;
4938 case 5:
4939 numchan = 6;
4940 break;
4941 case 6:
4942 numchan = 10;
4943 break;
4944 case 7:
4945 numchan = 12;
4946 break;
4947 case 8:
4948 numchan = 16;
4949 break;
4950 }
4951 rdev->mc.vram_width = numchan * chansize;
4952 /* Could aper size report 0 ? */
4953 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
4954 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
4955 /* size in MB on si */
Alex Deucher13c5bfd2013-09-24 10:56:55 -0400[diff] [blame]4956 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
4957 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]4958 rdev->mc.visible_vram_size = rdev->mc.aper_size;
4959 si_vram_gtt_location(rdev, &rdev->mc);
4960 radeon_update_bandwidth_info(rdev);
4961
4962 return 0;
4963}
4964
4965/*
4966 * GART
4967 * VMID 0 is the physical GPU addresses as used by the kernel.
4968 * VMIDs 1-15 are used for userspace clients and are handled
4969 * by the radeon vm/hsa code.
4970 */
4971/**
4972 * cik_pcie_gart_tlb_flush - gart tlb flush callback
4973 *
4974 * @rdev: radeon_device pointer
4975 *
4976 * Flush the TLB for the VMID 0 page table (CIK).
4977 */
4978void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
4979{
4980 /* flush hdp cache */
4981 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
4982
4983 /* bits 0-15 are the VM contexts0-15 */
4984 WREG32(VM_INVALIDATE_REQUEST, 0x1);
4985}
4986
4987/**
4988 * cik_pcie_gart_enable - gart enable
4989 *
4990 * @rdev: radeon_device pointer
4991 *
4992 * This sets up the TLBs, programs the page tables for VMID0,
4993 * sets up the hw for VMIDs 1-15 which are allocated on
4994 * demand, and sets up the global locations for the LDS, GDS,
4995 * and GPUVM for FSA64 clients (CIK).
4996 * Returns 0 for success, errors for failure.
4997 */
4998static int cik_pcie_gart_enable(struct radeon_device *rdev)
4999{
5000 int r, i;
5001
5002 if (rdev->gart.robj == NULL) {
5003 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
5004 return -EINVAL;
5005 }
5006 r = radeon_gart_table_vram_pin(rdev);
5007 if (r)
5008 return r;
5009 radeon_gart_restore(rdev);
5010 /* Setup TLB control */
5011 WREG32(MC_VM_MX_L1_TLB_CNTL,
5012 (0xA << 7) |
5013 ENABLE_L1_TLB |
5014 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5015 ENABLE_ADVANCED_DRIVER_MODEL |
5016 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5017 /* Setup L2 cache */
5018 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
5019 ENABLE_L2_FRAGMENT_PROCESSING |
5020 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5021 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5022 EFFECTIVE_L2_QUEUE_SIZE(7) |
5023 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5024 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
5025 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5026 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5027 /* setup context0 */
5028 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
5029 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
5030 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
5031 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
5032 (u32)(rdev->dummy_page.addr >> 12));
5033 WREG32(VM_CONTEXT0_CNTL2, 0);
5034 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
5035 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
5036
5037 WREG32(0x15D4, 0);
5038 WREG32(0x15D8, 0);
5039 WREG32(0x15DC, 0);
5040
5041 /* empty context1-15 */
5042 /* FIXME start with 4G, once using 2 level pt switch to full
5043 * vm size space
5044 */
5045 /* set vm size, must be a multiple of 4 */
5046 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
5047 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
5048 for (i = 1; i < 16; i++) {
5049 if (i < 8)
5050 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
5051 rdev->gart.table_addr >> 12);
5052 else
5053 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
5054 rdev->gart.table_addr >> 12);
5055 }
5056
5057 /* enable context1-15 */
5058 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
5059 (u32)(rdev->dummy_page.addr >> 12));
Alex Deuchera00024b2012-09-18 16:06:01 -0400[diff] [blame]5060 WREG32(VM_CONTEXT1_CNTL2, 4);
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5061 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
Alex Deuchera00024b2012-09-18 16:06:01 -0400[diff] [blame]5062 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5063 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5064 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5065 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5066 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
5067 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
5068 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
5069 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
5070 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
5071 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
5072 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5073 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5074
5075 /* TC cache setup ??? */
5076 WREG32(TC_CFG_L1_LOAD_POLICY0, 0);
5077 WREG32(TC_CFG_L1_LOAD_POLICY1, 0);
5078 WREG32(TC_CFG_L1_STORE_POLICY, 0);
5079
5080 WREG32(TC_CFG_L2_LOAD_POLICY0, 0);
5081 WREG32(TC_CFG_L2_LOAD_POLICY1, 0);
5082 WREG32(TC_CFG_L2_STORE_POLICY0, 0);
5083 WREG32(TC_CFG_L2_STORE_POLICY1, 0);
5084 WREG32(TC_CFG_L2_ATOMIC_POLICY, 0);
5085
5086 WREG32(TC_CFG_L1_VOLATILE, 0);
5087 WREG32(TC_CFG_L2_VOLATILE, 0);
5088
5089 if (rdev->family == CHIP_KAVERI) {
5090 u32 tmp = RREG32(CHUB_CONTROL);
5091 tmp &= ~BYPASS_VM;
5092 WREG32(CHUB_CONTROL, tmp);
5093 }
5094
5095 /* XXX SH_MEM regs */
5096 /* where to put LDS, scratch, GPUVM in FSA64 space */
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]5097 mutex_lock(&rdev->srbm_mutex);
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5098 for (i = 0; i < 16; i++) {
Alex Deucherb556b122013-01-29 10:44:22 -0500[diff] [blame]5099 cik_srbm_select(rdev, 0, 0, 0, i);
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]5100 /* CP and shaders */
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5101 WREG32(SH_MEM_CONFIG, 0);
5102 WREG32(SH_MEM_APE1_BASE, 1);
5103 WREG32(SH_MEM_APE1_LIMIT, 0);
5104 WREG32(SH_MEM_BASES, 0);
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]5105 /* SDMA GFX */
5106 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
5107 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
5108 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
5109 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
5110 /* XXX SDMA RLC - todo */
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5111 }
Alex Deucherb556b122013-01-29 10:44:22 -0500[diff] [blame]5112 cik_srbm_select(rdev, 0, 0, 0, 0);
Alex Deucherf61d5b462013-08-06 12:40:16 -0400[diff] [blame]5113 mutex_unlock(&rdev->srbm_mutex);
Alex Deucher1c491652013-04-09 12:45:26 -0400[diff] [blame]5114
5115 cik_pcie_gart_tlb_flush(rdev);
5116 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
5117 (unsigned)(rdev->mc.gtt_size >> 20),
5118 (unsigned long long)rdev->gart.table_addr);
5119 rdev->gart.ready = true;
5120 return 0;
5121}
5122
5123/**
5124 * cik_pcie_gart_disable - gart disable
5125 *
5126 * @rdev: radeon_device pointer
5127 *
5128 * This disables all VM page table (CIK).
5129 */
5130static void cik_pcie_gart_disable(struct radeon_device *rdev)
5131{
5132 /* Disable all tables */
5133 WREG32(VM_CONTEXT0_CNTL, 0);
5134 WREG32(VM_CONTEXT1_CNTL, 0);
5135 /* Setup TLB control */
5136 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5137 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5138 /* Setup L2 cache */
5139 WREG32(VM_L2_CNTL,
5140 ENABLE_L2_FRAGMENT_PROCESSING |
5141 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5142 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5143 EFFECTIVE_L2_QUEUE_SIZE(7) |
5144 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5145 WREG32(VM_L2_CNTL2, 0);
5146 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5147 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5148 radeon_gart_table_vram_unpin(rdev);
5149}
5150
5151/**
5152 * cik_pcie_gart_fini - vm fini callback
5153 *
5154 * @rdev: radeon_device pointer
5155 *
5156 * Tears down the driver GART/VM setup (CIK).
5157 */
5158static void cik_pcie_gart_fini(struct radeon_device *rdev)
5159{
5160 cik_pcie_gart_disable(rdev);
5161 radeon_gart_table_vram_free(rdev);
5162 radeon_gart_fini(rdev);
5163}
5164
5165/* vm parser */
5166/**
5167 * cik_ib_parse - vm ib_parse callback
5168 *
5169 * @rdev: radeon_device pointer
5170 * @ib: indirect buffer pointer
5171 *
5172 * CIK uses hw IB checking so this is a nop (CIK).
5173 */
5174int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
5175{
5176 return 0;
5177}
5178
5179/*
5180 * vm
5181 * VMID 0 is the physical GPU addresses as used by the kernel.
5182 * VMIDs 1-15 are used for userspace clients and are handled
5183 * by the radeon vm/hsa code.
5184 */
5185/**
5186 * cik_vm_init - cik vm init callback
5187 *
5188 * @rdev: radeon_device pointer
5189 *
5190 * Inits cik specific vm parameters (number of VMs, base of vram for
5191 * VMIDs 1-15) (CIK).
5192 * Returns 0 for success.
5193 */
5194int cik_vm_init(struct radeon_device *rdev)
5195{
5196 /* number of VMs */
5197 rdev->vm_manager.nvm = 16;
5198 /* base offset of vram pages */
5199 if (rdev->flags & RADEON_IS_IGP) {
5200 u64 tmp = RREG32(MC_VM_FB_OFFSET);
5201 tmp <<= 22;
5202 rdev->vm_manager.vram_base_offset = tmp;
5203 } else
5204 rdev->vm_manager.vram_base_offset = 0;
5205
5206 return 0;
5207}
5208
5209/**
5210 * cik_vm_fini - cik vm fini callback
5211 *
5212 * @rdev: radeon_device pointer
5213 *
5214 * Tear down any asic specific VM setup (CIK).
5215 */
5216void cik_vm_fini(struct radeon_device *rdev)
5217{
5218}
5219
Alex Deucherf96ab482012-08-31 10:37:47 -0400[diff] [blame]5220/**
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]5221 * cik_vm_decode_fault - print human readable fault info
5222 *
5223 * @rdev: radeon_device pointer
5224 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
5225 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
5226 *
5227 * Print human readable fault information (CIK).
5228 */
5229static void cik_vm_decode_fault(struct radeon_device *rdev,
5230 u32 status, u32 addr, u32 mc_client)
5231{
Alex Deucher939c0d32013-09-30 18:03:06 -0400[diff] [blame]5232 u32 mc_id;
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]5233 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
5234 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
Michel Dänzer328a50c7b2013-09-18 15:39:40 +0200[diff] [blame]5235 char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
5236 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]5237
Alex Deucher939c0d32013-09-30 18:03:06 -0400[diff] [blame]5238 if (rdev->family == CHIP_HAWAII)
5239 mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5240 else
5241 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5242
Michel Dänzer328a50c7b2013-09-18 15:39:40 +0200[diff] [blame]5243 printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]5244 protections, vmid, addr,
5245 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
Michel Dänzer328a50c7b2013-09-18 15:39:40 +0200[diff] [blame]5246 block, mc_client, mc_id);
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]5247}
5248
5249/**
Alex Deucherf96ab482012-08-31 10:37:47 -0400[diff] [blame]5250 * cik_vm_flush - cik vm flush using the CP
5251 *
5252 * @rdev: radeon_device pointer
5253 *
5254 * Update the page table base and flush the VM TLB
5255 * using the CP (CIK).
5256 */
5257void cik_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
5258{
5259 struct radeon_ring *ring = &rdev->ring[ridx];
5260
5261 if (vm == NULL)
5262 return;
5263
5264 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5265 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5266 WRITE_DATA_DST_SEL(0)));
5267 if (vm->id < 8) {
5268 radeon_ring_write(ring,
5269 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
5270 } else {
5271 radeon_ring_write(ring,
5272 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
5273 }
5274 radeon_ring_write(ring, 0);
5275 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
5276
5277 /* update SH_MEM_* regs */
5278 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5279 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5280 WRITE_DATA_DST_SEL(0)));
5281 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5282 radeon_ring_write(ring, 0);
5283 radeon_ring_write(ring, VMID(vm->id));
5284
5285 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
5286 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5287 WRITE_DATA_DST_SEL(0)));
5288 radeon_ring_write(ring, SH_MEM_BASES >> 2);
5289 radeon_ring_write(ring, 0);
5290
5291 radeon_ring_write(ring, 0); /* SH_MEM_BASES */
5292 radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */
5293 radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
5294 radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
5295
5296 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5297 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5298 WRITE_DATA_DST_SEL(0)));
5299 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5300 radeon_ring_write(ring, 0);
5301 radeon_ring_write(ring, VMID(0));
5302
5303 /* HDP flush */
5304 /* We should be using the WAIT_REG_MEM packet here like in
5305 * cik_fence_ring_emit(), but it causes the CP to hang in this
5306 * context...
5307 */
5308 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5309 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5310 WRITE_DATA_DST_SEL(0)));
5311 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
5312 radeon_ring_write(ring, 0);
5313 radeon_ring_write(ring, 0);
5314
5315 /* bits 0-15 are the VM contexts0-15 */
5316 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5317 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5318 WRITE_DATA_DST_SEL(0)));
5319 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5320 radeon_ring_write(ring, 0);
5321 radeon_ring_write(ring, 1 << vm->id);
5322
Alex Deucherb07fdd32013-04-11 09:36:17 -0400[diff] [blame]5323 /* compute doesn't have PFP */
5324 if (ridx == RADEON_RING_TYPE_GFX_INDEX) {
5325 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5326 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5327 radeon_ring_write(ring, 0x0);
5328 }
Alex Deucherf96ab482012-08-31 10:37:47 -0400[diff] [blame]5329}
5330
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5331/*
5332 * RLC
5333 * The RLC is a multi-purpose microengine that handles a
5334 * variety of functions, the most important of which is
5335 * the interrupt controller.
5336 */
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5337static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
5338 bool enable)
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5339{
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5340 u32 tmp = RREG32(CP_INT_CNTL_RING0);
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5341
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5342 if (enable)
5343 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5344 else
5345 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5346 WREG32(CP_INT_CNTL_RING0, tmp);
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5347}
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5348
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5349static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)
5350{
5351 u32 tmp;
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5352
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5353 tmp = RREG32(RLC_LB_CNTL);
5354 if (enable)
5355 tmp |= LOAD_BALANCE_ENABLE;
5356 else
5357 tmp &= ~LOAD_BALANCE_ENABLE;
5358 WREG32(RLC_LB_CNTL, tmp);
5359}
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5360
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5361static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
5362{
5363 u32 i, j, k;
5364 u32 mask;
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5365
5366 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
5367 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
5368 cik_select_se_sh(rdev, i, j);
5369 for (k = 0; k < rdev->usec_timeout; k++) {
5370 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
5371 break;
5372 udelay(1);
5373 }
5374 }
5375 }
5376 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5377
5378 mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
5379 for (k = 0; k < rdev->usec_timeout; k++) {
5380 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
5381 break;
5382 udelay(1);
5383 }
5384}
5385
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5386static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)
5387{
5388 u32 tmp;
5389
5390 tmp = RREG32(RLC_CNTL);
5391 if (tmp != rlc)
5392 WREG32(RLC_CNTL, rlc);
5393}
5394
5395static u32 cik_halt_rlc(struct radeon_device *rdev)
5396{
5397 u32 data, orig;
5398
5399 orig = data = RREG32(RLC_CNTL);
5400
5401 if (data & RLC_ENABLE) {
5402 u32 i;
5403
5404 data &= ~RLC_ENABLE;
5405 WREG32(RLC_CNTL, data);
5406
5407 for (i = 0; i < rdev->usec_timeout; i++) {
5408 if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0)
5409 break;
5410 udelay(1);
5411 }
5412
5413 cik_wait_for_rlc_serdes(rdev);
5414 }
5415
5416 return orig;
5417}
5418
Alex Deuchera412fce2013-04-22 20:23:31 -0400[diff] [blame]5419void cik_enter_rlc_safe_mode(struct radeon_device *rdev)
5420{
5421 u32 tmp, i, mask;
5422
5423 tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE);
5424 WREG32(RLC_GPR_REG2, tmp);
5425
5426 mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS;
5427 for (i = 0; i < rdev->usec_timeout; i++) {
5428 if ((RREG32(RLC_GPM_STAT) & mask) == mask)
5429 break;
5430 udelay(1);
5431 }
5432
5433 for (i = 0; i < rdev->usec_timeout; i++) {
5434 if ((RREG32(RLC_GPR_REG2) & REQ) == 0)
5435 break;
5436 udelay(1);
5437 }
5438}
5439
5440void cik_exit_rlc_safe_mode(struct radeon_device *rdev)
5441{
5442 u32 tmp;
5443
5444 tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE);
5445 WREG32(RLC_GPR_REG2, tmp);
5446}
5447
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5448/**
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5449 * cik_rlc_stop - stop the RLC ME
5450 *
5451 * @rdev: radeon_device pointer
5452 *
5453 * Halt the RLC ME (MicroEngine) (CIK).
5454 */
5455static void cik_rlc_stop(struct radeon_device *rdev)
5456{
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5457 WREG32(RLC_CNTL, 0);
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5458
5459 cik_enable_gui_idle_interrupt(rdev, false);
5460
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5461 cik_wait_for_rlc_serdes(rdev);
5462}
5463
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5464/**
5465 * cik_rlc_start - start the RLC ME
5466 *
5467 * @rdev: radeon_device pointer
5468 *
5469 * Unhalt the RLC ME (MicroEngine) (CIK).
5470 */
5471static void cik_rlc_start(struct radeon_device *rdev)
5472{
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5473 WREG32(RLC_CNTL, RLC_ENABLE);
5474
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5475 cik_enable_gui_idle_interrupt(rdev, true);
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5476
5477 udelay(50);
5478}
5479
5480/**
5481 * cik_rlc_resume - setup the RLC hw
5482 *
5483 * @rdev: radeon_device pointer
5484 *
5485 * Initialize the RLC registers, load the ucode,
5486 * and start the RLC (CIK).
5487 * Returns 0 for success, -EINVAL if the ucode is not available.
5488 */
5489static int cik_rlc_resume(struct radeon_device *rdev)
5490{
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5491 u32 i, size, tmp;
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5492 const __be32 *fw_data;
5493
5494 if (!rdev->rlc_fw)
5495 return -EINVAL;
5496
5497 switch (rdev->family) {
5498 case CHIP_BONAIRE:
5499 default:
5500 size = BONAIRE_RLC_UCODE_SIZE;
5501 break;
5502 case CHIP_KAVERI:
5503 size = KV_RLC_UCODE_SIZE;
5504 break;
5505 case CHIP_KABINI:
5506 size = KB_RLC_UCODE_SIZE;
5507 break;
5508 }
5509
5510 cik_rlc_stop(rdev);
5511
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5512 /* disable CG */
5513 tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
5514 WREG32(RLC_CGCG_CGLS_CTRL, tmp);
5515
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5516 si_rlc_reset(rdev);
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5517
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5518 cik_init_pg(rdev);
5519
5520 cik_init_cg(rdev);
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5521
5522 WREG32(RLC_LB_CNTR_INIT, 0);
5523 WREG32(RLC_LB_CNTR_MAX, 0x00008000);
5524
5525 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5526 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5527 WREG32(RLC_LB_PARAMS, 0x00600408);
5528 WREG32(RLC_LB_CNTL, 0x80000004);
5529
5530 WREG32(RLC_MC_CNTL, 0);
5531 WREG32(RLC_UCODE_CNTL, 0);
5532
5533 fw_data = (const __be32 *)rdev->rlc_fw->data;
5534 WREG32(RLC_GPM_UCODE_ADDR, 0);
5535 for (i = 0; i < size; i++)
5536 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
5537 WREG32(RLC_GPM_UCODE_ADDR, 0);
5538
Alex Deucher866d83d2013-04-15 17:13:29 -0400[diff] [blame]5539 /* XXX - find out what chips support lbpw */
5540 cik_enable_lbpw(rdev, false);
5541
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5542 if (rdev->family == CHIP_BONAIRE)
5543 WREG32(RLC_DRIVER_DMA_STATUS, 0);
Alex Deucherf6796ca2012-11-09 10:44:08 -0500[diff] [blame]5544
5545 cik_rlc_start(rdev);
5546
5547 return 0;
5548}
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]5549
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5550static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)
5551{
5552 u32 data, orig, tmp, tmp2;
5553
5554 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
5555
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5556 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
Alex Deucherddc76ff2013-08-12 17:25:26 -0400[diff] [blame]5557 cik_enable_gui_idle_interrupt(rdev, true);
5558
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5559 tmp = cik_halt_rlc(rdev);
5560
5561 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5562 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5563 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5564 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
5565 WREG32(RLC_SERDES_WR_CTRL, tmp2);
5566
5567 cik_update_rlc(rdev, tmp);
5568
5569 data |= CGCG_EN | CGLS_EN;
5570 } else {
Alex Deucherddc76ff2013-08-12 17:25:26 -0400[diff] [blame]5571 cik_enable_gui_idle_interrupt(rdev, false);
5572
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5573 RREG32(CB_CGTT_SCLK_CTRL);
5574 RREG32(CB_CGTT_SCLK_CTRL);
5575 RREG32(CB_CGTT_SCLK_CTRL);
5576 RREG32(CB_CGTT_SCLK_CTRL);
5577
5578 data &= ~(CGCG_EN | CGLS_EN);
5579 }
5580
5581 if (orig != data)
5582 WREG32(RLC_CGCG_CGLS_CTRL, data);
5583
5584}
5585
5586static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
5587{
5588 u32 data, orig, tmp = 0;
5589
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5590 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
5591 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {
5592 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5593 orig = data = RREG32(CP_MEM_SLP_CNTL);
5594 data |= CP_MEM_LS_EN;
5595 if (orig != data)
5596 WREG32(CP_MEM_SLP_CNTL, data);
5597 }
5598 }
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5599
5600 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5601 data &= 0xfffffffd;
5602 if (orig != data)
5603 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5604
5605 tmp = cik_halt_rlc(rdev);
5606
5607 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5608 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5609 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5610 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
5611 WREG32(RLC_SERDES_WR_CTRL, data);
5612
5613 cik_update_rlc(rdev, tmp);
5614
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5615 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {
5616 orig = data = RREG32(CGTS_SM_CTRL_REG);
5617 data &= ~SM_MODE_MASK;
5618 data |= SM_MODE(0x2);
5619 data |= SM_MODE_ENABLE;
5620 data &= ~CGTS_OVERRIDE;
5621 if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&
5622 (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))
5623 data &= ~CGTS_LS_OVERRIDE;
5624 data &= ~ON_MONITOR_ADD_MASK;
5625 data |= ON_MONITOR_ADD_EN;
5626 data |= ON_MONITOR_ADD(0x96);
5627 if (orig != data)
5628 WREG32(CGTS_SM_CTRL_REG, data);
5629 }
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5630 } else {
5631 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5632 data |= 0x00000002;
5633 if (orig != data)
5634 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5635
5636 data = RREG32(RLC_MEM_SLP_CNTL);
5637 if (data & RLC_MEM_LS_EN) {
5638 data &= ~RLC_MEM_LS_EN;
5639 WREG32(RLC_MEM_SLP_CNTL, data);
5640 }
5641
5642 data = RREG32(CP_MEM_SLP_CNTL);
5643 if (data & CP_MEM_LS_EN) {
5644 data &= ~CP_MEM_LS_EN;
5645 WREG32(CP_MEM_SLP_CNTL, data);
5646 }
5647
5648 orig = data = RREG32(CGTS_SM_CTRL_REG);
5649 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE;
5650 if (orig != data)
5651 WREG32(CGTS_SM_CTRL_REG, data);
5652
5653 tmp = cik_halt_rlc(rdev);
5654
5655 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5656 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5657 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5658 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
5659 WREG32(RLC_SERDES_WR_CTRL, data);
5660
5661 cik_update_rlc(rdev, tmp);
5662 }
5663}
5664
5665static const u32 mc_cg_registers[] =
5666{
5667 MC_HUB_MISC_HUB_CG,
5668 MC_HUB_MISC_SIP_CG,
5669 MC_HUB_MISC_VM_CG,
5670 MC_XPB_CLK_GAT,
5671 ATC_MISC_CG,
5672 MC_CITF_MISC_WR_CG,
5673 MC_CITF_MISC_RD_CG,
5674 MC_CITF_MISC_VM_CG,
5675 VM_L2_CG,
5676};
5677
5678static void cik_enable_mc_ls(struct radeon_device *rdev,
5679 bool enable)
5680{
5681 int i;
5682 u32 orig, data;
5683
5684 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5685 orig = data = RREG32(mc_cg_registers[i]);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5686 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5687 data |= MC_LS_ENABLE;
5688 else
5689 data &= ~MC_LS_ENABLE;
5690 if (data != orig)
5691 WREG32(mc_cg_registers[i], data);
5692 }
5693}
5694
5695static void cik_enable_mc_mgcg(struct radeon_device *rdev,
5696 bool enable)
5697{
5698 int i;
5699 u32 orig, data;
5700
5701 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5702 orig = data = RREG32(mc_cg_registers[i]);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5703 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5704 data |= MC_CG_ENABLE;
5705 else
5706 data &= ~MC_CG_ENABLE;
5707 if (data != orig)
5708 WREG32(mc_cg_registers[i], data);
5709 }
5710}
5711
5712static void cik_enable_sdma_mgcg(struct radeon_device *rdev,
5713 bool enable)
5714{
5715 u32 orig, data;
5716
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5717 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5718 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);
5719 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);
5720 } else {
5721 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);
5722 data |= 0xff000000;
5723 if (data != orig)
5724 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);
5725
5726 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);
5727 data |= 0xff000000;
5728 if (data != orig)
5729 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);
5730 }
5731}
5732
5733static void cik_enable_sdma_mgls(struct radeon_device *rdev,
5734 bool enable)
5735{
5736 u32 orig, data;
5737
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5738 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5739 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
5740 data |= 0x100;
5741 if (orig != data)
5742 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
5743
5744 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
5745 data |= 0x100;
5746 if (orig != data)
5747 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
5748 } else {
5749 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
5750 data &= ~0x100;
5751 if (orig != data)
5752 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
5753
5754 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
5755 data &= ~0x100;
5756 if (orig != data)
5757 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
5758 }
5759}
5760
5761static void cik_enable_uvd_mgcg(struct radeon_device *rdev,
5762 bool enable)
5763{
5764 u32 orig, data;
5765
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5766 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5767 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5768 data = 0xfff;
5769 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
5770
5771 orig = data = RREG32(UVD_CGC_CTRL);
5772 data |= DCM;
5773 if (orig != data)
5774 WREG32(UVD_CGC_CTRL, data);
5775 } else {
5776 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5777 data &= ~0xfff;
5778 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
5779
5780 orig = data = RREG32(UVD_CGC_CTRL);
5781 data &= ~DCM;
5782 if (orig != data)
5783 WREG32(UVD_CGC_CTRL, data);
5784 }
5785}
5786
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5787static void cik_enable_bif_mgls(struct radeon_device *rdev,
5788 bool enable)
5789{
5790 u32 orig, data;
5791
5792 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
5793
5794 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
5795 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
5796 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
5797 else
5798 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
5799 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
5800
5801 if (orig != data)
5802 WREG32_PCIE_PORT(PCIE_CNTL2, data);
5803}
5804
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5805static void cik_enable_hdp_mgcg(struct radeon_device *rdev,
5806 bool enable)
5807{
5808 u32 orig, data;
5809
5810 orig = data = RREG32(HDP_HOST_PATH_CNTL);
5811
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5812 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5813 data &= ~CLOCK_GATING_DIS;
5814 else
5815 data |= CLOCK_GATING_DIS;
5816
5817 if (orig != data)
5818 WREG32(HDP_HOST_PATH_CNTL, data);
5819}
5820
5821static void cik_enable_hdp_ls(struct radeon_device *rdev,
5822 bool enable)
5823{
5824 u32 orig, data;
5825
5826 orig = data = RREG32(HDP_MEM_POWER_LS);
5827
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5828 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5829 data |= HDP_LS_ENABLE;
5830 else
5831 data &= ~HDP_LS_ENABLE;
5832
5833 if (orig != data)
5834 WREG32(HDP_MEM_POWER_LS, data);
5835}
5836
5837void cik_update_cg(struct radeon_device *rdev,
5838 u32 block, bool enable)
5839{
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]5840
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5841 if (block & RADEON_CG_BLOCK_GFX) {
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]5842 cik_enable_gui_idle_interrupt(rdev, false);
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5843 /* order matters! */
5844 if (enable) {
5845 cik_enable_mgcg(rdev, true);
5846 cik_enable_cgcg(rdev, true);
5847 } else {
5848 cik_enable_cgcg(rdev, false);
5849 cik_enable_mgcg(rdev, false);
5850 }
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]5851 cik_enable_gui_idle_interrupt(rdev, true);
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5852 }
5853
5854 if (block & RADEON_CG_BLOCK_MC) {
5855 if (!(rdev->flags & RADEON_IS_IGP)) {
5856 cik_enable_mc_mgcg(rdev, enable);
5857 cik_enable_mc_ls(rdev, enable);
5858 }
5859 }
5860
5861 if (block & RADEON_CG_BLOCK_SDMA) {
5862 cik_enable_sdma_mgcg(rdev, enable);
5863 cik_enable_sdma_mgls(rdev, enable);
5864 }
5865
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5866 if (block & RADEON_CG_BLOCK_BIF) {
5867 cik_enable_bif_mgls(rdev, enable);
5868 }
5869
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5870 if (block & RADEON_CG_BLOCK_UVD) {
5871 if (rdev->has_uvd)
5872 cik_enable_uvd_mgcg(rdev, enable);
5873 }
5874
5875 if (block & RADEON_CG_BLOCK_HDP) {
5876 cik_enable_hdp_mgcg(rdev, enable);
5877 cik_enable_hdp_ls(rdev, enable);
5878 }
5879}
5880
5881static void cik_init_cg(struct radeon_device *rdev)
5882{
5883
Alex Deucherddc76ff2013-08-12 17:25:26 -0400[diff] [blame]5884 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5885
5886 if (rdev->has_uvd)
5887 si_init_uvd_internal_cg(rdev);
5888
5889 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
5890 RADEON_CG_BLOCK_SDMA |
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5891 RADEON_CG_BLOCK_BIF |
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5892 RADEON_CG_BLOCK_UVD |
5893 RADEON_CG_BLOCK_HDP), true);
5894}
5895
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5896static void cik_fini_cg(struct radeon_device *rdev)
5897{
5898 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
5899 RADEON_CG_BLOCK_SDMA |
5900 RADEON_CG_BLOCK_BIF |
5901 RADEON_CG_BLOCK_UVD |
5902 RADEON_CG_BLOCK_HDP), false);
5903
5904 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);
5905}
5906
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5907static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,
5908 bool enable)
5909{
5910 u32 data, orig;
5911
5912 orig = data = RREG32(RLC_PG_CNTL);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5913 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5914 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
5915 else
5916 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
5917 if (orig != data)
5918 WREG32(RLC_PG_CNTL, data);
5919}
5920
5921static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,
5922 bool enable)
5923{
5924 u32 data, orig;
5925
5926 orig = data = RREG32(RLC_PG_CNTL);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5927 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5928 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
5929 else
5930 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
5931 if (orig != data)
5932 WREG32(RLC_PG_CNTL, data);
5933}
5934
5935static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)
5936{
5937 u32 data, orig;
5938
5939 orig = data = RREG32(RLC_PG_CNTL);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5940 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5941 data &= ~DISABLE_CP_PG;
5942 else
5943 data |= DISABLE_CP_PG;
5944 if (orig != data)
5945 WREG32(RLC_PG_CNTL, data);
5946}
5947
5948static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)
5949{
5950 u32 data, orig;
5951
5952 orig = data = RREG32(RLC_PG_CNTL);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]5953 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5954 data &= ~DISABLE_GDS_PG;
5955 else
5956 data |= DISABLE_GDS_PG;
5957 if (orig != data)
5958 WREG32(RLC_PG_CNTL, data);
5959}
5960
5961#define CP_ME_TABLE_SIZE 96
5962#define CP_ME_TABLE_OFFSET 2048
5963#define CP_MEC_TABLE_OFFSET 4096
5964
5965void cik_init_cp_pg_table(struct radeon_device *rdev)
5966{
5967 const __be32 *fw_data;
5968 volatile u32 *dst_ptr;
5969 int me, i, max_me = 4;
5970 u32 bo_offset = 0;
5971 u32 table_offset;
5972
5973 if (rdev->family == CHIP_KAVERI)
5974 max_me = 5;
5975
5976 if (rdev->rlc.cp_table_ptr == NULL)
5977 return;
5978
5979 /* write the cp table buffer */
5980 dst_ptr = rdev->rlc.cp_table_ptr;
5981 for (me = 0; me < max_me; me++) {
5982 if (me == 0) {
5983 fw_data = (const __be32 *)rdev->ce_fw->data;
5984 table_offset = CP_ME_TABLE_OFFSET;
5985 } else if (me == 1) {
5986 fw_data = (const __be32 *)rdev->pfp_fw->data;
5987 table_offset = CP_ME_TABLE_OFFSET;
5988 } else if (me == 2) {
5989 fw_data = (const __be32 *)rdev->me_fw->data;
5990 table_offset = CP_ME_TABLE_OFFSET;
5991 } else {
5992 fw_data = (const __be32 *)rdev->mec_fw->data;
5993 table_offset = CP_MEC_TABLE_OFFSET;
5994 }
5995
5996 for (i = 0; i < CP_ME_TABLE_SIZE; i ++) {
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]5997 dst_ptr[bo_offset + i] = cpu_to_le32(be32_to_cpu(fw_data[table_offset + i]));
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]5998 }
5999 bo_offset += CP_ME_TABLE_SIZE;
6000 }
6001}
6002
6003static void cik_enable_gfx_cgpg(struct radeon_device *rdev,
6004 bool enable)
6005{
6006 u32 data, orig;
6007
Alex Deucher2b19d172013-09-04 16:58:29 -0400[diff] [blame]6008 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6009 orig = data = RREG32(RLC_PG_CNTL);
6010 data |= GFX_PG_ENABLE;
6011 if (orig != data)
6012 WREG32(RLC_PG_CNTL, data);
6013
6014 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6015 data |= AUTO_PG_EN;
6016 if (orig != data)
6017 WREG32(RLC_AUTO_PG_CTRL, data);
6018 } else {
6019 orig = data = RREG32(RLC_PG_CNTL);
6020 data &= ~GFX_PG_ENABLE;
6021 if (orig != data)
6022 WREG32(RLC_PG_CNTL, data);
6023
6024 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6025 data &= ~AUTO_PG_EN;
6026 if (orig != data)
6027 WREG32(RLC_AUTO_PG_CTRL, data);
6028
6029 data = RREG32(DB_RENDER_CONTROL);
6030 }
6031}
6032
6033static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
6034{
6035 u32 mask = 0, tmp, tmp1;
6036 int i;
6037
6038 cik_select_se_sh(rdev, se, sh);
6039 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
6040 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
6041 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6042
6043 tmp &= 0xffff0000;
6044
6045 tmp |= tmp1;
6046 tmp >>= 16;
6047
6048 for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {
6049 mask <<= 1;
6050 mask |= 1;
6051 }
6052
6053 return (~tmp) & mask;
6054}
6055
6056static void cik_init_ao_cu_mask(struct radeon_device *rdev)
6057{
6058 u32 i, j, k, active_cu_number = 0;
6059 u32 mask, counter, cu_bitmap;
6060 u32 tmp = 0;
6061
6062 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6063 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6064 mask = 1;
6065 cu_bitmap = 0;
6066 counter = 0;
6067 for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {
6068 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {
6069 if (counter < 2)
6070 cu_bitmap |= mask;
6071 counter ++;
6072 }
6073 mask <<= 1;
6074 }
6075
6076 active_cu_number += counter;
6077 tmp |= (cu_bitmap << (i * 16 + j * 8));
6078 }
6079 }
6080
6081 WREG32(RLC_PG_AO_CU_MASK, tmp);
6082
6083 tmp = RREG32(RLC_MAX_PG_CU);
6084 tmp &= ~MAX_PU_CU_MASK;
6085 tmp |= MAX_PU_CU(active_cu_number);
6086 WREG32(RLC_MAX_PG_CU, tmp);
6087}
6088
6089static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,
6090 bool enable)
6091{
6092 u32 data, orig;
6093
6094 orig = data = RREG32(RLC_PG_CNTL);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6095 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6096 data |= STATIC_PER_CU_PG_ENABLE;
6097 else
6098 data &= ~STATIC_PER_CU_PG_ENABLE;
6099 if (orig != data)
6100 WREG32(RLC_PG_CNTL, data);
6101}
6102
6103static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,
6104 bool enable)
6105{
6106 u32 data, orig;
6107
6108 orig = data = RREG32(RLC_PG_CNTL);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6109 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6110 data |= DYN_PER_CU_PG_ENABLE;
6111 else
6112 data &= ~DYN_PER_CU_PG_ENABLE;
6113 if (orig != data)
6114 WREG32(RLC_PG_CNTL, data);
6115}
6116
6117#define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
6118#define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET 0x3D
6119
6120static void cik_init_gfx_cgpg(struct radeon_device *rdev)
6121{
6122 u32 data, orig;
6123 u32 i;
6124
6125 if (rdev->rlc.cs_data) {
6126 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6127 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6128 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6129 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);
6130 } else {
6131 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6132 for (i = 0; i < 3; i++)
6133 WREG32(RLC_GPM_SCRATCH_DATA, 0);
6134 }
6135 if (rdev->rlc.reg_list) {
6136 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
6137 for (i = 0; i < rdev->rlc.reg_list_size; i++)
6138 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);
6139 }
6140
6141 orig = data = RREG32(RLC_PG_CNTL);
6142 data |= GFX_PG_SRC;
6143 if (orig != data)
6144 WREG32(RLC_PG_CNTL, data);
6145
6146 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
6147 WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);
6148
6149 data = RREG32(CP_RB_WPTR_POLL_CNTL);
6150 data &= ~IDLE_POLL_COUNT_MASK;
6151 data |= IDLE_POLL_COUNT(0x60);
6152 WREG32(CP_RB_WPTR_POLL_CNTL, data);
6153
6154 data = 0x10101010;
6155 WREG32(RLC_PG_DELAY, data);
6156
6157 data = RREG32(RLC_PG_DELAY_2);
6158 data &= ~0xff;
6159 data |= 0x3;
6160 WREG32(RLC_PG_DELAY_2, data);
6161
6162 data = RREG32(RLC_AUTO_PG_CTRL);
6163 data &= ~GRBM_REG_SGIT_MASK;
6164 data |= GRBM_REG_SGIT(0x700);
6165 WREG32(RLC_AUTO_PG_CTRL, data);
6166
6167}
6168
6169static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)
6170{
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6171 cik_enable_gfx_cgpg(rdev, enable);
6172 cik_enable_gfx_static_mgpg(rdev, enable);
6173 cik_enable_gfx_dynamic_mgpg(rdev, enable);
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6174}
6175
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6176u32 cik_get_csb_size(struct radeon_device *rdev)
6177{
6178 u32 count = 0;
6179 const struct cs_section_def *sect = NULL;
6180 const struct cs_extent_def *ext = NULL;
6181
6182 if (rdev->rlc.cs_data == NULL)
6183 return 0;
6184
6185 /* begin clear state */
6186 count += 2;
6187 /* context control state */
6188 count += 3;
6189
6190 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6191 for (ext = sect->section; ext->extent != NULL; ++ext) {
6192 if (sect->id == SECT_CONTEXT)
6193 count += 2 + ext->reg_count;
6194 else
6195 return 0;
6196 }
6197 }
6198 /* pa_sc_raster_config/pa_sc_raster_config1 */
6199 count += 4;
6200 /* end clear state */
6201 count += 2;
6202 /* clear state */
6203 count += 2;
6204
6205 return count;
6206}
6207
6208void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
6209{
6210 u32 count = 0, i;
6211 const struct cs_section_def *sect = NULL;
6212 const struct cs_extent_def *ext = NULL;
6213
6214 if (rdev->rlc.cs_data == NULL)
6215 return;
6216 if (buffer == NULL)
6217 return;
6218
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6219 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6220 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6221
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6222 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
6223 buffer[count++] = cpu_to_le32(0x80000000);
6224 buffer[count++] = cpu_to_le32(0x80000000);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6225
6226 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6227 for (ext = sect->section; ext->extent != NULL; ++ext) {
6228 if (sect->id == SECT_CONTEXT) {
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6229 buffer[count++] =
6230 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
6231 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6232 for (i = 0; i < ext->reg_count; i++)
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6233 buffer[count++] = cpu_to_le32(ext->extent[i]);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6234 } else {
6235 return;
6236 }
6237 }
6238 }
6239
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6240 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
6241 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6242 switch (rdev->family) {
6243 case CHIP_BONAIRE:
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6244 buffer[count++] = cpu_to_le32(0x16000012);
6245 buffer[count++] = cpu_to_le32(0x00000000);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6246 break;
6247 case CHIP_KAVERI:
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6248 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6249 buffer[count++] = cpu_to_le32(0x00000000);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6250 break;
6251 case CHIP_KABINI:
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6252 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6253 buffer[count++] = cpu_to_le32(0x00000000);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6254 break;
6255 default:
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6256 buffer[count++] = cpu_to_le32(0x00000000);
6257 buffer[count++] = cpu_to_le32(0x00000000);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6258 break;
6259 }
6260
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6261 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6262 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6263
Alex Deucher6ba81e52013-10-23 18:27:10 -0400[diff] [blame]6264 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
6265 buffer[count++] = cpu_to_le32(0);
Alex Deuchera0f38602013-08-22 11:57:46 -0400[diff] [blame]6266}
6267
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6268static void cik_init_pg(struct radeon_device *rdev)
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6269{
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6270 if (rdev->pg_flags) {
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6271 cik_enable_sck_slowdown_on_pu(rdev, true);
6272 cik_enable_sck_slowdown_on_pd(rdev, true);
Alex Deucher2b19d172013-09-04 16:58:29 -0400[diff] [blame]6273 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6274 cik_init_gfx_cgpg(rdev);
6275 cik_enable_cp_pg(rdev, true);
6276 cik_enable_gds_pg(rdev, true);
6277 }
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]6278 cik_init_ao_cu_mask(rdev);
6279 cik_update_gfx_pg(rdev, true);
6280 }
6281}
6282
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6283static void cik_fini_pg(struct radeon_device *rdev)
6284{
6285 if (rdev->pg_flags) {
6286 cik_update_gfx_pg(rdev, false);
Alex Deucher2b19d172013-09-04 16:58:29 -0400[diff] [blame]6287 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]6288 cik_enable_cp_pg(rdev, false);
6289 cik_enable_gds_pg(rdev, false);
6290 }
6291 }
6292}
6293
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6294/*
6295 * Interrupts
6296 * Starting with r6xx, interrupts are handled via a ring buffer.
6297 * Ring buffers are areas of GPU accessible memory that the GPU
6298 * writes interrupt vectors into and the host reads vectors out of.
6299 * There is a rptr (read pointer) that determines where the
6300 * host is currently reading, and a wptr (write pointer)
6301 * which determines where the GPU has written. When the
6302 * pointers are equal, the ring is idle. When the GPU
6303 * writes vectors to the ring buffer, it increments the
6304 * wptr. When there is an interrupt, the host then starts
6305 * fetching commands and processing them until the pointers are
6306 * equal again at which point it updates the rptr.
6307 */
6308
6309/**
6310 * cik_enable_interrupts - Enable the interrupt ring buffer
6311 *
6312 * @rdev: radeon_device pointer
6313 *
6314 * Enable the interrupt ring buffer (CIK).
6315 */
6316static void cik_enable_interrupts(struct radeon_device *rdev)
6317{
6318 u32 ih_cntl = RREG32(IH_CNTL);
6319 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6320
6321 ih_cntl |= ENABLE_INTR;
6322 ih_rb_cntl |= IH_RB_ENABLE;
6323 WREG32(IH_CNTL, ih_cntl);
6324 WREG32(IH_RB_CNTL, ih_rb_cntl);
6325 rdev->ih.enabled = true;
6326}
6327
6328/**
6329 * cik_disable_interrupts - Disable the interrupt ring buffer
6330 *
6331 * @rdev: radeon_device pointer
6332 *
6333 * Disable the interrupt ring buffer (CIK).
6334 */
6335static void cik_disable_interrupts(struct radeon_device *rdev)
6336{
6337 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6338 u32 ih_cntl = RREG32(IH_CNTL);
6339
6340 ih_rb_cntl &= ~IH_RB_ENABLE;
6341 ih_cntl &= ~ENABLE_INTR;
6342 WREG32(IH_RB_CNTL, ih_rb_cntl);
6343 WREG32(IH_CNTL, ih_cntl);
6344 /* set rptr, wptr to 0 */
6345 WREG32(IH_RB_RPTR, 0);
6346 WREG32(IH_RB_WPTR, 0);
6347 rdev->ih.enabled = false;
6348 rdev->ih.rptr = 0;
6349}
6350
6351/**
6352 * cik_disable_interrupt_state - Disable all interrupt sources
6353 *
6354 * @rdev: radeon_device pointer
6355 *
6356 * Clear all interrupt enable bits used by the driver (CIK).
6357 */
6358static void cik_disable_interrupt_state(struct radeon_device *rdev)
6359{
6360 u32 tmp;
6361
6362 /* gfx ring */
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]6363 tmp = RREG32(CP_INT_CNTL_RING0) &
6364 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6365 WREG32(CP_INT_CNTL_RING0, tmp);
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]6366 /* sdma */
6367 tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6368 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
6369 tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6370 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6371 /* compute queues */
6372 WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
6373 WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
6374 WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
6375 WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
6376 WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
6377 WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
6378 WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
6379 WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
6380 /* grbm */
6381 WREG32(GRBM_INT_CNTL, 0);
6382 /* vline/vblank, etc. */
6383 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
6384 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
6385 if (rdev->num_crtc >= 4) {
6386 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
6387 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
6388 }
6389 if (rdev->num_crtc >= 6) {
6390 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
6391 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
6392 }
6393
6394 /* dac hotplug */
6395 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
6396
6397 /* digital hotplug */
6398 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6399 WREG32(DC_HPD1_INT_CONTROL, tmp);
6400 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6401 WREG32(DC_HPD2_INT_CONTROL, tmp);
6402 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6403 WREG32(DC_HPD3_INT_CONTROL, tmp);
6404 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6405 WREG32(DC_HPD4_INT_CONTROL, tmp);
6406 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6407 WREG32(DC_HPD5_INT_CONTROL, tmp);
6408 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6409 WREG32(DC_HPD6_INT_CONTROL, tmp);
6410
6411}
6412
6413/**
6414 * cik_irq_init - init and enable the interrupt ring
6415 *
6416 * @rdev: radeon_device pointer
6417 *
6418 * Allocate a ring buffer for the interrupt controller,
6419 * enable the RLC, disable interrupts, enable the IH
6420 * ring buffer and enable it (CIK).
6421 * Called at device load and reume.
6422 * Returns 0 for success, errors for failure.
6423 */
6424static int cik_irq_init(struct radeon_device *rdev)
6425{
6426 int ret = 0;
6427 int rb_bufsz;
6428 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
6429
6430 /* allocate ring */
6431 ret = r600_ih_ring_alloc(rdev);
6432 if (ret)
6433 return ret;
6434
6435 /* disable irqs */
6436 cik_disable_interrupts(rdev);
6437
6438 /* init rlc */
6439 ret = cik_rlc_resume(rdev);
6440 if (ret) {
6441 r600_ih_ring_fini(rdev);
6442 return ret;
6443 }
6444
6445 /* setup interrupt control */
6446 /* XXX this should actually be a bus address, not an MC address. same on older asics */
6447 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
6448 interrupt_cntl = RREG32(INTERRUPT_CNTL);
6449 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
6450 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
6451 */
6452 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
6453 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
6454 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
6455 WREG32(INTERRUPT_CNTL, interrupt_cntl);
6456
6457 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
Daniel Vetterb72a8922013-07-10 14:11:59 +0200[diff] [blame]6458 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6459
6460 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
6461 IH_WPTR_OVERFLOW_CLEAR |
6462 (rb_bufsz << 1));
6463
6464 if (rdev->wb.enabled)
6465 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
6466
6467 /* set the writeback address whether it's enabled or not */
6468 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
6469 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
6470
6471 WREG32(IH_RB_CNTL, ih_rb_cntl);
6472
6473 /* set rptr, wptr to 0 */
6474 WREG32(IH_RB_RPTR, 0);
6475 WREG32(IH_RB_WPTR, 0);
6476
6477 /* Default settings for IH_CNTL (disabled at first) */
6478 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
6479 /* RPTR_REARM only works if msi's are enabled */
6480 if (rdev->msi_enabled)
6481 ih_cntl |= RPTR_REARM;
6482 WREG32(IH_CNTL, ih_cntl);
6483
6484 /* force the active interrupt state to all disabled */
6485 cik_disable_interrupt_state(rdev);
6486
6487 pci_set_master(rdev->pdev);
6488
6489 /* enable irqs */
6490 cik_enable_interrupts(rdev);
6491
6492 return ret;
6493}
6494
6495/**
6496 * cik_irq_set - enable/disable interrupt sources
6497 *
6498 * @rdev: radeon_device pointer
6499 *
6500 * Enable interrupt sources on the GPU (vblanks, hpd,
6501 * etc.) (CIK).
6502 * Returns 0 for success, errors for failure.
6503 */
6504int cik_irq_set(struct radeon_device *rdev)
6505{
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]6506 u32 cp_int_cntl;
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]6507 u32 cp_m1p0, cp_m1p1, cp_m1p2, cp_m1p3;
6508 u32 cp_m2p0, cp_m2p1, cp_m2p2, cp_m2p3;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6509 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
6510 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
6511 u32 grbm_int_cntl = 0;
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]6512 u32 dma_cntl, dma_cntl1;
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]6513 u32 thermal_int;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6514
6515 if (!rdev->irq.installed) {
6516 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
6517 return -EINVAL;
6518 }
6519 /* don't enable anything if the ih is disabled */
6520 if (!rdev->ih.enabled) {
6521 cik_disable_interrupts(rdev);
6522 /* force the active interrupt state to all disabled */
6523 cik_disable_interrupt_state(rdev);
6524 return 0;
6525 }
6526
Alex Deucher4214faf2013-09-03 10:17:13 -0400[diff] [blame]6527 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
6528 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6529 cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
6530
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6531 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
6532 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
6533 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
6534 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
6535 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
6536 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
6537
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]6538 dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6539 dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6540
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]6541 cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6542 cp_m1p1 = RREG32(CP_ME1_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6543 cp_m1p2 = RREG32(CP_ME1_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6544 cp_m1p3 = RREG32(CP_ME1_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6545 cp_m2p0 = RREG32(CP_ME2_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6546 cp_m2p1 = RREG32(CP_ME2_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6547 cp_m2p2 = RREG32(CP_ME2_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6548 cp_m2p3 = RREG32(CP_ME2_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6549
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]6550 if (rdev->flags & RADEON_IS_IGP)
6551 thermal_int = RREG32_SMC(CG_THERMAL_INT_CTRL) &
6552 ~(THERM_INTH_MASK | THERM_INTL_MASK);
6553 else
6554 thermal_int = RREG32_SMC(CG_THERMAL_INT) &
6555 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]6556
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6557 /* enable CP interrupts on all rings */
6558 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
6559 DRM_DEBUG("cik_irq_set: sw int gfx\n");
6560 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
6561 }
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]6562 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
6563 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6564 DRM_DEBUG("si_irq_set: sw int cp1\n");
6565 if (ring->me == 1) {
6566 switch (ring->pipe) {
6567 case 0:
6568 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
6569 break;
6570 case 1:
6571 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
6572 break;
6573 case 2:
6574 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6575 break;
6576 case 3:
6577 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6578 break;
6579 default:
6580 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
6581 break;
6582 }
6583 } else if (ring->me == 2) {
6584 switch (ring->pipe) {
6585 case 0:
6586 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
6587 break;
6588 case 1:
6589 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
6590 break;
6591 case 2:
6592 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6593 break;
6594 case 3:
6595 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6596 break;
6597 default:
6598 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
6599 break;
6600 }
6601 } else {
6602 DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
6603 }
6604 }
6605 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
6606 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6607 DRM_DEBUG("si_irq_set: sw int cp2\n");
6608 if (ring->me == 1) {
6609 switch (ring->pipe) {
6610 case 0:
6611 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
6612 break;
6613 case 1:
6614 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
6615 break;
6616 case 2:
6617 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6618 break;
6619 case 3:
6620 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6621 break;
6622 default:
6623 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
6624 break;
6625 }
6626 } else if (ring->me == 2) {
6627 switch (ring->pipe) {
6628 case 0:
6629 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
6630 break;
6631 case 1:
6632 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
6633 break;
6634 case 2:
6635 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6636 break;
6637 case 3:
6638 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6639 break;
6640 default:
6641 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
6642 break;
6643 }
6644 } else {
6645 DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
6646 }
6647 }
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6648
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]6649 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
6650 DRM_DEBUG("cik_irq_set: sw int dma\n");
6651 dma_cntl |= TRAP_ENABLE;
6652 }
6653
6654 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
6655 DRM_DEBUG("cik_irq_set: sw int dma1\n");
6656 dma_cntl1 |= TRAP_ENABLE;
6657 }
6658
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6659 if (rdev->irq.crtc_vblank_int[0] ||
6660 atomic_read(&rdev->irq.pflip[0])) {
6661 DRM_DEBUG("cik_irq_set: vblank 0\n");
6662 crtc1 |= VBLANK_INTERRUPT_MASK;
6663 }
6664 if (rdev->irq.crtc_vblank_int[1] ||
6665 atomic_read(&rdev->irq.pflip[1])) {
6666 DRM_DEBUG("cik_irq_set: vblank 1\n");
6667 crtc2 |= VBLANK_INTERRUPT_MASK;
6668 }
6669 if (rdev->irq.crtc_vblank_int[2] ||
6670 atomic_read(&rdev->irq.pflip[2])) {
6671 DRM_DEBUG("cik_irq_set: vblank 2\n");
6672 crtc3 |= VBLANK_INTERRUPT_MASK;
6673 }
6674 if (rdev->irq.crtc_vblank_int[3] ||
6675 atomic_read(&rdev->irq.pflip[3])) {
6676 DRM_DEBUG("cik_irq_set: vblank 3\n");
6677 crtc4 |= VBLANK_INTERRUPT_MASK;
6678 }
6679 if (rdev->irq.crtc_vblank_int[4] ||
6680 atomic_read(&rdev->irq.pflip[4])) {
6681 DRM_DEBUG("cik_irq_set: vblank 4\n");
6682 crtc5 |= VBLANK_INTERRUPT_MASK;
6683 }
6684 if (rdev->irq.crtc_vblank_int[5] ||
6685 atomic_read(&rdev->irq.pflip[5])) {
6686 DRM_DEBUG("cik_irq_set: vblank 5\n");
6687 crtc6 |= VBLANK_INTERRUPT_MASK;
6688 }
6689 if (rdev->irq.hpd[0]) {
6690 DRM_DEBUG("cik_irq_set: hpd 1\n");
6691 hpd1 |= DC_HPDx_INT_EN;
6692 }
6693 if (rdev->irq.hpd[1]) {
6694 DRM_DEBUG("cik_irq_set: hpd 2\n");
6695 hpd2 |= DC_HPDx_INT_EN;
6696 }
6697 if (rdev->irq.hpd[2]) {
6698 DRM_DEBUG("cik_irq_set: hpd 3\n");
6699 hpd3 |= DC_HPDx_INT_EN;
6700 }
6701 if (rdev->irq.hpd[3]) {
6702 DRM_DEBUG("cik_irq_set: hpd 4\n");
6703 hpd4 |= DC_HPDx_INT_EN;
6704 }
6705 if (rdev->irq.hpd[4]) {
6706 DRM_DEBUG("cik_irq_set: hpd 5\n");
6707 hpd5 |= DC_HPDx_INT_EN;
6708 }
6709 if (rdev->irq.hpd[5]) {
6710 DRM_DEBUG("cik_irq_set: hpd 6\n");
6711 hpd6 |= DC_HPDx_INT_EN;
6712 }
6713
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]6714 if (rdev->irq.dpm_thermal) {
6715 DRM_DEBUG("dpm thermal\n");
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]6716 if (rdev->flags & RADEON_IS_IGP)
6717 thermal_int |= THERM_INTH_MASK | THERM_INTL_MASK;
6718 else
6719 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]6720 }
6721
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6722 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
6723
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]6724 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
6725 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
6726
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]6727 WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
6728 WREG32(CP_ME1_PIPE1_INT_CNTL, cp_m1p1);
6729 WREG32(CP_ME1_PIPE2_INT_CNTL, cp_m1p2);
6730 WREG32(CP_ME1_PIPE3_INT_CNTL, cp_m1p3);
6731 WREG32(CP_ME2_PIPE0_INT_CNTL, cp_m2p0);
6732 WREG32(CP_ME2_PIPE1_INT_CNTL, cp_m2p1);
6733 WREG32(CP_ME2_PIPE2_INT_CNTL, cp_m2p2);
6734 WREG32(CP_ME2_PIPE3_INT_CNTL, cp_m2p3);
6735
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6736 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
6737
6738 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
6739 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
6740 if (rdev->num_crtc >= 4) {
6741 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
6742 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
6743 }
6744 if (rdev->num_crtc >= 6) {
6745 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
6746 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
6747 }
6748
6749 WREG32(DC_HPD1_INT_CONTROL, hpd1);
6750 WREG32(DC_HPD2_INT_CONTROL, hpd2);
6751 WREG32(DC_HPD3_INT_CONTROL, hpd3);
6752 WREG32(DC_HPD4_INT_CONTROL, hpd4);
6753 WREG32(DC_HPD5_INT_CONTROL, hpd5);
6754 WREG32(DC_HPD6_INT_CONTROL, hpd6);
6755
Alex Deuchercc8dbbb2013-08-14 01:03:41 -0400[diff] [blame]6756 if (rdev->flags & RADEON_IS_IGP)
6757 WREG32_SMC(CG_THERMAL_INT_CTRL, thermal_int);
6758 else
6759 WREG32_SMC(CG_THERMAL_INT, thermal_int);
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]6760
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6761 return 0;
6762}
6763
6764/**
6765 * cik_irq_ack - ack interrupt sources
6766 *
6767 * @rdev: radeon_device pointer
6768 *
6769 * Ack interrupt sources on the GPU (vblanks, hpd,
6770 * etc.) (CIK). Certain interrupts sources are sw
6771 * generated and do not require an explicit ack.
6772 */
6773static inline void cik_irq_ack(struct radeon_device *rdev)
6774{
6775 u32 tmp;
6776
6777 rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
6778 rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
6779 rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
6780 rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
6781 rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
6782 rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
6783 rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
6784
6785 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
6786 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
6787 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
6788 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
6789 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
6790 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
6791 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
6792 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
6793
6794 if (rdev->num_crtc >= 4) {
6795 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
6796 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
6797 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
6798 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
6799 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
6800 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
6801 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
6802 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
6803 }
6804
6805 if (rdev->num_crtc >= 6) {
6806 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
6807 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
6808 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
6809 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
6810 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
6811 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
6812 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
6813 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
6814 }
6815
6816 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
6817 tmp = RREG32(DC_HPD1_INT_CONTROL);
6818 tmp |= DC_HPDx_INT_ACK;
6819 WREG32(DC_HPD1_INT_CONTROL, tmp);
6820 }
6821 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
6822 tmp = RREG32(DC_HPD2_INT_CONTROL);
6823 tmp |= DC_HPDx_INT_ACK;
6824 WREG32(DC_HPD2_INT_CONTROL, tmp);
6825 }
6826 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6827 tmp = RREG32(DC_HPD3_INT_CONTROL);
6828 tmp |= DC_HPDx_INT_ACK;
6829 WREG32(DC_HPD3_INT_CONTROL, tmp);
6830 }
6831 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6832 tmp = RREG32(DC_HPD4_INT_CONTROL);
6833 tmp |= DC_HPDx_INT_ACK;
6834 WREG32(DC_HPD4_INT_CONTROL, tmp);
6835 }
6836 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6837 tmp = RREG32(DC_HPD5_INT_CONTROL);
6838 tmp |= DC_HPDx_INT_ACK;
6839 WREG32(DC_HPD5_INT_CONTROL, tmp);
6840 }
6841 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6842 tmp = RREG32(DC_HPD5_INT_CONTROL);
6843 tmp |= DC_HPDx_INT_ACK;
6844 WREG32(DC_HPD6_INT_CONTROL, tmp);
6845 }
6846}
6847
6848/**
6849 * cik_irq_disable - disable interrupts
6850 *
6851 * @rdev: radeon_device pointer
6852 *
6853 * Disable interrupts on the hw (CIK).
6854 */
6855static void cik_irq_disable(struct radeon_device *rdev)
6856{
6857 cik_disable_interrupts(rdev);
6858 /* Wait and acknowledge irq */
6859 mdelay(1);
6860 cik_irq_ack(rdev);
6861 cik_disable_interrupt_state(rdev);
6862}
6863
6864/**
6865 * cik_irq_disable - disable interrupts for suspend
6866 *
6867 * @rdev: radeon_device pointer
6868 *
6869 * Disable interrupts and stop the RLC (CIK).
6870 * Used for suspend.
6871 */
6872static void cik_irq_suspend(struct radeon_device *rdev)
6873{
6874 cik_irq_disable(rdev);
6875 cik_rlc_stop(rdev);
6876}
6877
6878/**
6879 * cik_irq_fini - tear down interrupt support
6880 *
6881 * @rdev: radeon_device pointer
6882 *
6883 * Disable interrupts on the hw and free the IH ring
6884 * buffer (CIK).
6885 * Used for driver unload.
6886 */
6887static void cik_irq_fini(struct radeon_device *rdev)
6888{
6889 cik_irq_suspend(rdev);
6890 r600_ih_ring_fini(rdev);
6891}
6892
6893/**
6894 * cik_get_ih_wptr - get the IH ring buffer wptr
6895 *
6896 * @rdev: radeon_device pointer
6897 *
6898 * Get the IH ring buffer wptr from either the register
6899 * or the writeback memory buffer (CIK). Also check for
6900 * ring buffer overflow and deal with it.
6901 * Used by cik_irq_process().
6902 * Returns the value of the wptr.
6903 */
6904static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
6905{
6906 u32 wptr, tmp;
6907
6908 if (rdev->wb.enabled)
6909 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
6910 else
6911 wptr = RREG32(IH_RB_WPTR);
6912
6913 if (wptr & RB_OVERFLOW) {
6914 /* When a ring buffer overflow happen start parsing interrupt
6915 * from the last not overwritten vector (wptr + 16). Hopefully
6916 * this should allow us to catchup.
6917 */
6918 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
6919 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
6920 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
6921 tmp = RREG32(IH_RB_CNTL);
6922 tmp |= IH_WPTR_OVERFLOW_CLEAR;
6923 WREG32(IH_RB_CNTL, tmp);
6924 }
6925 return (wptr & rdev->ih.ptr_mask);
6926}
6927
6928/* CIK IV Ring
6929 * Each IV ring entry is 128 bits:
6930 * [7:0] - interrupt source id
6931 * [31:8] - reserved
6932 * [59:32] - interrupt source data
6933 * [63:60] - reserved
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]6934 * [71:64] - RINGID
6935 * CP:
6936 * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6937 * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
6938 * - for gfx, hw shader state (0=PS...5=LS, 6=CS)
6939 * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
6940 * PIPE_ID - ME0 0=3D
6941 * - ME1&2 compute dispatcher (4 pipes each)
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]6942 * SDMA:
6943 * INSTANCE_ID [1:0], QUEUE_ID[1:0]
6944 * INSTANCE_ID - 0 = sdma0, 1 = sdma1
6945 * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6946 * [79:72] - VMID
6947 * [95:80] - PASID
6948 * [127:96] - reserved
6949 */
6950/**
6951 * cik_irq_process - interrupt handler
6952 *
6953 * @rdev: radeon_device pointer
6954 *
6955 * Interrupt hander (CIK). Walk the IH ring,
6956 * ack interrupts and schedule work to handle
6957 * interrupt events.
6958 * Returns irq process return code.
6959 */
6960int cik_irq_process(struct radeon_device *rdev)
6961{
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]6962 struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6963 struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6964 u32 wptr;
6965 u32 rptr;
6966 u32 src_id, src_data, ring_id;
6967 u8 me_id, pipe_id, queue_id;
6968 u32 ring_index;
6969 bool queue_hotplug = false;
6970 bool queue_reset = false;
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]6971 u32 addr, status, mc_client;
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]6972 bool queue_thermal = false;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6973
6974 if (!rdev->ih.enabled || rdev->shutdown)
6975 return IRQ_NONE;
6976
6977 wptr = cik_get_ih_wptr(rdev);
6978
6979restart_ih:
6980 /* is somebody else already processing irqs? */
6981 if (atomic_xchg(&rdev->ih.lock, 1))
6982 return IRQ_NONE;
6983
6984 rptr = rdev->ih.rptr;
6985 DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
6986
6987 /* Order reading of wptr vs. reading of IH ring data */
6988 rmb();
6989
6990 /* display interrupts */
6991 cik_irq_ack(rdev);
6992
6993 while (rptr != wptr) {
6994 /* wptr/rptr are in bytes! */
6995 ring_index = rptr / 4;
6996 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
6997 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
6998 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]6999
7000 switch (src_id) {
7001 case 1: /* D1 vblank/vline */
7002 switch (src_data) {
7003 case 0: /* D1 vblank */
7004 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) {
7005 if (rdev->irq.crtc_vblank_int[0]) {
7006 drm_handle_vblank(rdev->ddev, 0);
7007 rdev->pm.vblank_sync = true;
7008 wake_up(&rdev->irq.vblank_queue);
7009 }
7010 if (atomic_read(&rdev->irq.pflip[0]))
7011 radeon_crtc_handle_flip(rdev, 0);
7012 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
7013 DRM_DEBUG("IH: D1 vblank\n");
7014 }
7015 break;
7016 case 1: /* D1 vline */
7017 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) {
7018 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
7019 DRM_DEBUG("IH: D1 vline\n");
7020 }
7021 break;
7022 default:
7023 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7024 break;
7025 }
7026 break;
7027 case 2: /* D2 vblank/vline */
7028 switch (src_data) {
7029 case 0: /* D2 vblank */
7030 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
7031 if (rdev->irq.crtc_vblank_int[1]) {
7032 drm_handle_vblank(rdev->ddev, 1);
7033 rdev->pm.vblank_sync = true;
7034 wake_up(&rdev->irq.vblank_queue);
7035 }
7036 if (atomic_read(&rdev->irq.pflip[1]))
7037 radeon_crtc_handle_flip(rdev, 1);
7038 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
7039 DRM_DEBUG("IH: D2 vblank\n");
7040 }
7041 break;
7042 case 1: /* D2 vline */
7043 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
7044 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
7045 DRM_DEBUG("IH: D2 vline\n");
7046 }
7047 break;
7048 default:
7049 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7050 break;
7051 }
7052 break;
7053 case 3: /* D3 vblank/vline */
7054 switch (src_data) {
7055 case 0: /* D3 vblank */
7056 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
7057 if (rdev->irq.crtc_vblank_int[2]) {
7058 drm_handle_vblank(rdev->ddev, 2);
7059 rdev->pm.vblank_sync = true;
7060 wake_up(&rdev->irq.vblank_queue);
7061 }
7062 if (atomic_read(&rdev->irq.pflip[2]))
7063 radeon_crtc_handle_flip(rdev, 2);
7064 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
7065 DRM_DEBUG("IH: D3 vblank\n");
7066 }
7067 break;
7068 case 1: /* D3 vline */
7069 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
7070 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
7071 DRM_DEBUG("IH: D3 vline\n");
7072 }
7073 break;
7074 default:
7075 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7076 break;
7077 }
7078 break;
7079 case 4: /* D4 vblank/vline */
7080 switch (src_data) {
7081 case 0: /* D4 vblank */
7082 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
7083 if (rdev->irq.crtc_vblank_int[3]) {
7084 drm_handle_vblank(rdev->ddev, 3);
7085 rdev->pm.vblank_sync = true;
7086 wake_up(&rdev->irq.vblank_queue);
7087 }
7088 if (atomic_read(&rdev->irq.pflip[3]))
7089 radeon_crtc_handle_flip(rdev, 3);
7090 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
7091 DRM_DEBUG("IH: D4 vblank\n");
7092 }
7093 break;
7094 case 1: /* D4 vline */
7095 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
7096 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
7097 DRM_DEBUG("IH: D4 vline\n");
7098 }
7099 break;
7100 default:
7101 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7102 break;
7103 }
7104 break;
7105 case 5: /* D5 vblank/vline */
7106 switch (src_data) {
7107 case 0: /* D5 vblank */
7108 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
7109 if (rdev->irq.crtc_vblank_int[4]) {
7110 drm_handle_vblank(rdev->ddev, 4);
7111 rdev->pm.vblank_sync = true;
7112 wake_up(&rdev->irq.vblank_queue);
7113 }
7114 if (atomic_read(&rdev->irq.pflip[4]))
7115 radeon_crtc_handle_flip(rdev, 4);
7116 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
7117 DRM_DEBUG("IH: D5 vblank\n");
7118 }
7119 break;
7120 case 1: /* D5 vline */
7121 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
7122 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
7123 DRM_DEBUG("IH: D5 vline\n");
7124 }
7125 break;
7126 default:
7127 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7128 break;
7129 }
7130 break;
7131 case 6: /* D6 vblank/vline */
7132 switch (src_data) {
7133 case 0: /* D6 vblank */
7134 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
7135 if (rdev->irq.crtc_vblank_int[5]) {
7136 drm_handle_vblank(rdev->ddev, 5);
7137 rdev->pm.vblank_sync = true;
7138 wake_up(&rdev->irq.vblank_queue);
7139 }
7140 if (atomic_read(&rdev->irq.pflip[5]))
7141 radeon_crtc_handle_flip(rdev, 5);
7142 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
7143 DRM_DEBUG("IH: D6 vblank\n");
7144 }
7145 break;
7146 case 1: /* D6 vline */
7147 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
7148 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
7149 DRM_DEBUG("IH: D6 vline\n");
7150 }
7151 break;
7152 default:
7153 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7154 break;
7155 }
7156 break;
7157 case 42: /* HPD hotplug */
7158 switch (src_data) {
7159 case 0:
7160 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7161 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
7162 queue_hotplug = true;
7163 DRM_DEBUG("IH: HPD1\n");
7164 }
7165 break;
7166 case 1:
7167 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7168 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
7169 queue_hotplug = true;
7170 DRM_DEBUG("IH: HPD2\n");
7171 }
7172 break;
7173 case 2:
7174 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7175 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
7176 queue_hotplug = true;
7177 DRM_DEBUG("IH: HPD3\n");
7178 }
7179 break;
7180 case 3:
7181 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7182 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
7183 queue_hotplug = true;
7184 DRM_DEBUG("IH: HPD4\n");
7185 }
7186 break;
7187 case 4:
7188 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7189 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
7190 queue_hotplug = true;
7191 DRM_DEBUG("IH: HPD5\n");
7192 }
7193 break;
7194 case 5:
7195 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7196 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
7197 queue_hotplug = true;
7198 DRM_DEBUG("IH: HPD6\n");
7199 }
7200 break;
7201 default:
7202 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7203 break;
7204 }
7205 break;
Christian König6a3808b2013-08-30 11:10:33 +0200[diff] [blame]7206 case 124: /* UVD */
7207 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
7208 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
7209 break;
Alex Deucher9d97c992012-09-06 14:24:48 -0400[diff] [blame]7210 case 146:
7211 case 147:
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]7212 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
7213 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
7214 mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
Alex Deucher9d97c992012-09-06 14:24:48 -0400[diff] [blame]7215 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
7216 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]7217 addr);
Alex Deucher9d97c992012-09-06 14:24:48 -0400[diff] [blame]7218 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
Alex Deucher3ec7d112013-06-14 10:42:22 -0400[diff] [blame]7219 status);
7220 cik_vm_decode_fault(rdev, status, addr, mc_client);
Alex Deucher9d97c992012-09-06 14:24:48 -0400[diff] [blame]7221 /* reset addr and status */
7222 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
7223 break;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7224 case 176: /* GFX RB CP_INT */
7225 case 177: /* GFX IB CP_INT */
7226 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7227 break;
7228 case 181: /* CP EOP event */
7229 DRM_DEBUG("IH: CP EOP\n");
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]7230 /* XXX check the bitfield order! */
7231 me_id = (ring_id & 0x60) >> 5;
7232 pipe_id = (ring_id & 0x18) >> 3;
7233 queue_id = (ring_id & 0x7) >> 0;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7234 switch (me_id) {
7235 case 0:
7236 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7237 break;
7238 case 1:
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7239 case 2:
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]7240 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
7241 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7242 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
7243 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7244 break;
7245 }
7246 break;
7247 case 184: /* CP Privileged reg access */
7248 DRM_ERROR("Illegal register access in command stream\n");
7249 /* XXX check the bitfield order! */
7250 me_id = (ring_id & 0x60) >> 5;
7251 pipe_id = (ring_id & 0x18) >> 3;
7252 queue_id = (ring_id & 0x7) >> 0;
7253 switch (me_id) {
7254 case 0:
7255 /* This results in a full GPU reset, but all we need to do is soft
7256 * reset the CP for gfx
7257 */
7258 queue_reset = true;
7259 break;
7260 case 1:
7261 /* XXX compute */
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]7262 queue_reset = true;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7263 break;
7264 case 2:
7265 /* XXX compute */
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]7266 queue_reset = true;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7267 break;
7268 }
7269 break;
7270 case 185: /* CP Privileged inst */
7271 DRM_ERROR("Illegal instruction in command stream\n");
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]7272 /* XXX check the bitfield order! */
7273 me_id = (ring_id & 0x60) >> 5;
7274 pipe_id = (ring_id & 0x18) >> 3;
7275 queue_id = (ring_id & 0x7) >> 0;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7276 switch (me_id) {
7277 case 0:
7278 /* This results in a full GPU reset, but all we need to do is soft
7279 * reset the CP for gfx
7280 */
7281 queue_reset = true;
7282 break;
7283 case 1:
7284 /* XXX compute */
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]7285 queue_reset = true;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7286 break;
7287 case 2:
7288 /* XXX compute */
Alex Deucher2b0781a2013-04-09 14:26:16 -0400[diff] [blame]7289 queue_reset = true;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7290 break;
7291 }
7292 break;
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]7293 case 224: /* SDMA trap event */
7294 /* XXX check the bitfield order! */
7295 me_id = (ring_id & 0x3) >> 0;
7296 queue_id = (ring_id & 0xc) >> 2;
7297 DRM_DEBUG("IH: SDMA trap\n");
7298 switch (me_id) {
7299 case 0:
7300 switch (queue_id) {
7301 case 0:
7302 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
7303 break;
7304 case 1:
7305 /* XXX compute */
7306 break;
7307 case 2:
7308 /* XXX compute */
7309 break;
7310 }
7311 break;
7312 case 1:
7313 switch (queue_id) {
7314 case 0:
7315 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7316 break;
7317 case 1:
7318 /* XXX compute */
7319 break;
7320 case 2:
7321 /* XXX compute */
7322 break;
7323 }
7324 break;
7325 }
7326 break;
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]7327 case 230: /* thermal low to high */
7328 DRM_DEBUG("IH: thermal low to high\n");
7329 rdev->pm.dpm.thermal.high_to_low = false;
7330 queue_thermal = true;
7331 break;
7332 case 231: /* thermal high to low */
7333 DRM_DEBUG("IH: thermal high to low\n");
7334 rdev->pm.dpm.thermal.high_to_low = true;
7335 queue_thermal = true;
7336 break;
7337 case 233: /* GUI IDLE */
7338 DRM_DEBUG("IH: GUI idle\n");
7339 break;
Alex Deucher21a93e12013-04-09 12:47:11 -0400[diff] [blame]7340 case 241: /* SDMA Privileged inst */
7341 case 247: /* SDMA Privileged inst */
7342 DRM_ERROR("Illegal instruction in SDMA command stream\n");
7343 /* XXX check the bitfield order! */
7344 me_id = (ring_id & 0x3) >> 0;
7345 queue_id = (ring_id & 0xc) >> 2;
7346 switch (me_id) {
7347 case 0:
7348 switch (queue_id) {
7349 case 0:
7350 queue_reset = true;
7351 break;
7352 case 1:
7353 /* XXX compute */
7354 queue_reset = true;
7355 break;
7356 case 2:
7357 /* XXX compute */
7358 queue_reset = true;
7359 break;
7360 }
7361 break;
7362 case 1:
7363 switch (queue_id) {
7364 case 0:
7365 queue_reset = true;
7366 break;
7367 case 1:
7368 /* XXX compute */
7369 queue_reset = true;
7370 break;
7371 case 2:
7372 /* XXX compute */
7373 queue_reset = true;
7374 break;
7375 }
7376 break;
7377 }
7378 break;
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7379 default:
7380 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7381 break;
7382 }
7383
7384 /* wptr/rptr are in bytes! */
7385 rptr += 16;
7386 rptr &= rdev->ih.ptr_mask;
7387 }
7388 if (queue_hotplug)
7389 schedule_work(&rdev->hotplug_work);
7390 if (queue_reset)
7391 schedule_work(&rdev->reset_work);
Alex Deucher41a524a2013-08-14 01:01:40 -0400[diff] [blame]7392 if (queue_thermal)
7393 schedule_work(&rdev->pm.dpm.thermal.work);
Alex Deuchera59781b2012-11-09 10:45:57 -0500[diff] [blame]7394 rdev->ih.rptr = rptr;
7395 WREG32(IH_RB_RPTR, rdev->ih.rptr);
7396 atomic_set(&rdev->ih.lock, 0);
7397
7398 /* make sure wptr hasn't changed while processing */
7399 wptr = cik_get_ih_wptr(rdev);
7400 if (wptr != rptr)
7401 goto restart_ih;
7402
7403 return IRQ_HANDLED;
7404}
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7405
7406/*
7407 * startup/shutdown callbacks
7408 */
7409/**
7410 * cik_startup - program the asic to a functional state
7411 *
7412 * @rdev: radeon_device pointer
7413 *
7414 * Programs the asic to a functional state (CIK).
7415 * Called by cik_init() and cik_resume().
7416 * Returns 0 for success, error for failure.
7417 */
7418static int cik_startup(struct radeon_device *rdev)
7419{
7420 struct radeon_ring *ring;
7421 int r;
7422
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]7423 /* enable pcie gen2/3 link */
7424 cik_pcie_gen3_enable(rdev);
Alex Deucher7235711a42013-04-04 13:58:09 -0400[diff] [blame]7425 /* enable aspm */
7426 cik_program_aspm(rdev);
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]7427
Alex Deuchere5903d32013-08-30 08:58:20 -0400[diff] [blame]7428 /* scratch needs to be initialized before MC */
7429 r = r600_vram_scratch_init(rdev);
7430 if (r)
7431 return r;
7432
Alex Deucher6fab3feb2013-08-04 12:13:17 -0400[diff] [blame]7433 cik_mc_program(rdev);
7434
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7435 if (rdev->flags & RADEON_IS_IGP) {
7436 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
7437 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
7438 r = cik_init_microcode(rdev);
7439 if (r) {
7440 DRM_ERROR("Failed to load firmware!\n");
7441 return r;
7442 }
7443 }
7444 } else {
7445 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
7446 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
7447 !rdev->mc_fw) {
7448 r = cik_init_microcode(rdev);
7449 if (r) {
7450 DRM_ERROR("Failed to load firmware!\n");
7451 return r;
7452 }
7453 }
7454
7455 r = ci_mc_load_microcode(rdev);
7456 if (r) {
7457 DRM_ERROR("Failed to load MC firmware!\n");
7458 return r;
7459 }
7460 }
7461
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7462 r = cik_pcie_gart_enable(rdev);
7463 if (r)
7464 return r;
7465 cik_gpu_init(rdev);
7466
7467 /* allocate rlc buffers */
Alex Deucher22c775c2013-07-23 09:41:05 -0400[diff] [blame]7468 if (rdev->flags & RADEON_IS_IGP) {
7469 if (rdev->family == CHIP_KAVERI) {
7470 rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;
7471 rdev->rlc.reg_list_size =
7472 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
7473 } else {
7474 rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;
7475 rdev->rlc.reg_list_size =
7476 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
7477 }
7478 }
7479 rdev->rlc.cs_data = ci_cs_data;
7480 rdev->rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4;
Alex Deucher1fd11772013-04-17 17:53:50 -0400[diff] [blame]7481 r = sumo_rlc_init(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7482 if (r) {
7483 DRM_ERROR("Failed to init rlc BOs!\n");
7484 return r;
7485 }
7486
7487 /* allocate wb buffer */
7488 r = radeon_wb_init(rdev);
7489 if (r)
7490 return r;
7491
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7492 /* allocate mec buffers */
7493 r = cik_mec_init(rdev);
7494 if (r) {
7495 DRM_ERROR("Failed to init MEC BOs!\n");
7496 return r;
7497 }
7498
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7499 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
7500 if (r) {
7501 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7502 return r;
7503 }
7504
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7505 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7506 if (r) {
7507 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7508 return r;
7509 }
7510
7511 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
7512 if (r) {
7513 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7514 return r;
7515 }
7516
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7517 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
7518 if (r) {
7519 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7520 return r;
7521 }
7522
7523 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7524 if (r) {
7525 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7526 return r;
7527 }
7528
Alex Deucher2ce529d2013-08-28 18:12:59 -0400[diff] [blame]7529 r = radeon_uvd_resume(rdev);
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7530 if (!r) {
Alex Deucher2ce529d2013-08-28 18:12:59 -0400[diff] [blame]7531 r = uvd_v4_2_resume(rdev);
7532 if (!r) {
7533 r = radeon_fence_driver_start_ring(rdev,
7534 R600_RING_TYPE_UVD_INDEX);
7535 if (r)
7536 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
7537 }
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7538 }
7539 if (r)
7540 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
7541
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7542 /* Enable IRQ */
7543 if (!rdev->irq.installed) {
7544 r = radeon_irq_kms_init(rdev);
7545 if (r)
7546 return r;
7547 }
7548
7549 r = cik_irq_init(rdev);
7550 if (r) {
7551 DRM_ERROR("radeon: IH init failed (%d).\n", r);
7552 radeon_irq_kms_fini(rdev);
7553 return r;
7554 }
7555 cik_irq_set(rdev);
7556
7557 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
7558 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
7559 CP_RB0_RPTR, CP_RB0_WPTR,
Marek Olšák1dac28e2013-10-30 14:41:35 +0100[diff] [blame]7560 PACKET3(PACKET3_NOP, 0x3FFF));
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7561 if (r)
7562 return r;
7563
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7564 /* set up the compute queues */
Alex Deucher2615b532013-06-03 11:21:58 -0400[diff] [blame]7565 /* type-2 packets are deprecated on MEC, use type-3 instead */
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7566 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7567 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
7568 CP_HQD_PQ_RPTR, CP_HQD_PQ_WPTR,
Christian König2e1e6da2013-08-13 11:56:52 +0200[diff] [blame]7569 PACKET3(PACKET3_NOP, 0x3FFF));
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7570 if (r)
7571 return r;
7572 ring->me = 1; /* first MEC */
7573 ring->pipe = 0; /* first pipe */
7574 ring->queue = 0; /* first queue */
7575 ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
7576
Alex Deucher2615b532013-06-03 11:21:58 -0400[diff] [blame]7577 /* type-2 packets are deprecated on MEC, use type-3 instead */
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7578 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7579 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
7580 CP_HQD_PQ_RPTR, CP_HQD_PQ_WPTR,
Christian König2e1e6da2013-08-13 11:56:52 +0200[diff] [blame]7581 PACKET3(PACKET3_NOP, 0x3FFF));
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7582 if (r)
7583 return r;
7584 /* dGPU only have 1 MEC */
7585 ring->me = 1; /* first MEC */
7586 ring->pipe = 0; /* first pipe */
7587 ring->queue = 1; /* second queue */
7588 ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
7589
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7590 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
7591 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
7592 SDMA0_GFX_RB_RPTR + SDMA0_REGISTER_OFFSET,
7593 SDMA0_GFX_RB_WPTR + SDMA0_REGISTER_OFFSET,
Christian König2e1e6da2013-08-13 11:56:52 +0200[diff] [blame]7594 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7595 if (r)
7596 return r;
7597
7598 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
7599 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
7600 SDMA0_GFX_RB_RPTR + SDMA1_REGISTER_OFFSET,
7601 SDMA0_GFX_RB_WPTR + SDMA1_REGISTER_OFFSET,
Christian König2e1e6da2013-08-13 11:56:52 +0200[diff] [blame]7602 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7603 if (r)
7604 return r;
7605
7606 r = cik_cp_resume(rdev);
7607 if (r)
7608 return r;
7609
7610 r = cik_sdma_resume(rdev);
7611 if (r)
7612 return r;
7613
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7614 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
7615 if (ring->ring_size) {
Christian König02c9f7f2013-08-13 11:56:51 +0200[diff] [blame]7616 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7617 UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
Christian König2e1e6da2013-08-13 11:56:52 +0200[diff] [blame]7618 RADEON_CP_PACKET2);
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7619 if (!r)
Christian Könige409b122013-08-13 11:56:53 +0200[diff] [blame]7620 r = uvd_v1_0_init(rdev);
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7621 if (r)
7622 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
7623 }
7624
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7625 r = radeon_ib_pool_init(rdev);
7626 if (r) {
7627 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
7628 return r;
7629 }
7630
7631 r = radeon_vm_manager_init(rdev);
7632 if (r) {
7633 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
7634 return r;
7635 }
7636
Alex Deucherb5306022013-07-31 16:51:33 -0400[diff] [blame]7637 r = dce6_audio_init(rdev);
7638 if (r)
7639 return r;
7640
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7641 return 0;
7642}
7643
7644/**
7645 * cik_resume - resume the asic to a functional state
7646 *
7647 * @rdev: radeon_device pointer
7648 *
7649 * Programs the asic to a functional state (CIK).
7650 * Called at resume.
7651 * Returns 0 for success, error for failure.
7652 */
7653int cik_resume(struct radeon_device *rdev)
7654{
7655 int r;
7656
7657 /* post card */
7658 atom_asic_init(rdev->mode_info.atom_context);
7659
Alex Deucher0aafd312013-04-09 14:43:30 -0400[diff] [blame]7660 /* init golden registers */
7661 cik_init_golden_registers(rdev);
7662
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7663 rdev->accel_working = true;
7664 r = cik_startup(rdev);
7665 if (r) {
7666 DRM_ERROR("cik startup failed on resume\n");
7667 rdev->accel_working = false;
7668 return r;
7669 }
7670
7671 return r;
7672
7673}
7674
7675/**
7676 * cik_suspend - suspend the asic
7677 *
7678 * @rdev: radeon_device pointer
7679 *
7680 * Bring the chip into a state suitable for suspend (CIK).
7681 * Called at suspend.
7682 * Returns 0 for success.
7683 */
7684int cik_suspend(struct radeon_device *rdev)
7685{
Alex Deucherb5306022013-07-31 16:51:33 -0400[diff] [blame]7686 dce6_audio_fini(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7687 radeon_vm_manager_fini(rdev);
7688 cik_cp_enable(rdev, false);
7689 cik_sdma_enable(rdev, false);
Christian Könige409b122013-08-13 11:56:53 +0200[diff] [blame]7690 uvd_v1_0_fini(rdev);
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7691 radeon_uvd_suspend(rdev);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]7692 cik_fini_pg(rdev);
7693 cik_fini_cg(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7694 cik_irq_suspend(rdev);
7695 radeon_wb_disable(rdev);
7696 cik_pcie_gart_disable(rdev);
7697 return 0;
7698}
7699
7700/* Plan is to move initialization in that function and use
7701 * helper function so that radeon_device_init pretty much
7702 * do nothing more than calling asic specific function. This
7703 * should also allow to remove a bunch of callback function
7704 * like vram_info.
7705 */
7706/**
7707 * cik_init - asic specific driver and hw init
7708 *
7709 * @rdev: radeon_device pointer
7710 *
7711 * Setup asic specific driver variables and program the hw
7712 * to a functional state (CIK).
7713 * Called at driver startup.
7714 * Returns 0 for success, errors for failure.
7715 */
7716int cik_init(struct radeon_device *rdev)
7717{
7718 struct radeon_ring *ring;
7719 int r;
7720
7721 /* Read BIOS */
7722 if (!radeon_get_bios(rdev)) {
7723 if (ASIC_IS_AVIVO(rdev))
7724 return -EINVAL;
7725 }
7726 /* Must be an ATOMBIOS */
7727 if (!rdev->is_atom_bios) {
7728 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
7729 return -EINVAL;
7730 }
7731 r = radeon_atombios_init(rdev);
7732 if (r)
7733 return r;
7734
7735 /* Post card if necessary */
7736 if (!radeon_card_posted(rdev)) {
7737 if (!rdev->bios) {
7738 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
7739 return -EINVAL;
7740 }
7741 DRM_INFO("GPU not posted. posting now...\n");
7742 atom_asic_init(rdev->mode_info.atom_context);
7743 }
Alex Deucher0aafd312013-04-09 14:43:30 -0400[diff] [blame]7744 /* init golden registers */
7745 cik_init_golden_registers(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7746 /* Initialize scratch registers */
7747 cik_scratch_init(rdev);
7748 /* Initialize surface registers */
7749 radeon_surface_init(rdev);
7750 /* Initialize clocks */
7751 radeon_get_clock_info(rdev->ddev);
7752
7753 /* Fence driver */
7754 r = radeon_fence_driver_init(rdev);
7755 if (r)
7756 return r;
7757
7758 /* initialize memory controller */
7759 r = cik_mc_init(rdev);
7760 if (r)
7761 return r;
7762 /* Memory manager */
7763 r = radeon_bo_init(rdev);
7764 if (r)
7765 return r;
7766
7767 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
7768 ring->ring_obj = NULL;
7769 r600_ring_init(rdev, ring, 1024 * 1024);
7770
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7771 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7772 ring->ring_obj = NULL;
7773 r600_ring_init(rdev, ring, 1024 * 1024);
7774 r = radeon_doorbell_get(rdev, &ring->doorbell_page_num);
7775 if (r)
7776 return r;
7777
7778 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7779 ring->ring_obj = NULL;
7780 r600_ring_init(rdev, ring, 1024 * 1024);
7781 r = radeon_doorbell_get(rdev, &ring->doorbell_page_num);
7782 if (r)
7783 return r;
7784
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7785 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
7786 ring->ring_obj = NULL;
7787 r600_ring_init(rdev, ring, 256 * 1024);
7788
7789 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
7790 ring->ring_obj = NULL;
7791 r600_ring_init(rdev, ring, 256 * 1024);
7792
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7793 r = radeon_uvd_init(rdev);
7794 if (!r) {
7795 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
7796 ring->ring_obj = NULL;
7797 r600_ring_init(rdev, ring, 4096);
7798 }
7799
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7800 rdev->ih.ring_obj = NULL;
7801 r600_ih_ring_init(rdev, 64 * 1024);
7802
7803 r = r600_pcie_gart_init(rdev);
7804 if (r)
7805 return r;
7806
7807 rdev->accel_working = true;
7808 r = cik_startup(rdev);
7809 if (r) {
7810 dev_err(rdev->dev, "disabling GPU acceleration\n");
7811 cik_cp_fini(rdev);
7812 cik_sdma_fini(rdev);
7813 cik_irq_fini(rdev);
Alex Deucher1fd11772013-04-17 17:53:50 -0400[diff] [blame]7814 sumo_rlc_fini(rdev);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7815 cik_mec_fini(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7816 radeon_wb_fini(rdev);
7817 radeon_ib_pool_fini(rdev);
7818 radeon_vm_manager_fini(rdev);
7819 radeon_irq_kms_fini(rdev);
7820 cik_pcie_gart_fini(rdev);
7821 rdev->accel_working = false;
7822 }
7823
7824 /* Don't start up if the MC ucode is missing.
7825 * The default clocks and voltages before the MC ucode
7826 * is loaded are not suffient for advanced operations.
7827 */
7828 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
7829 DRM_ERROR("radeon: MC ucode required for NI+.\n");
7830 return -EINVAL;
7831 }
7832
7833 return 0;
7834}
7835
7836/**
7837 * cik_fini - asic specific driver and hw fini
7838 *
7839 * @rdev: radeon_device pointer
7840 *
7841 * Tear down the asic specific driver variables and program the hw
7842 * to an idle state (CIK).
7843 * Called at driver unload.
7844 */
7845void cik_fini(struct radeon_device *rdev)
7846{
7847 cik_cp_fini(rdev);
7848 cik_sdma_fini(rdev);
Alex Deucher473359b2013-08-09 11:18:39 -0400[diff] [blame]7849 cik_fini_pg(rdev);
7850 cik_fini_cg(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7851 cik_irq_fini(rdev);
Alex Deucher1fd11772013-04-17 17:53:50 -0400[diff] [blame]7852 sumo_rlc_fini(rdev);
Alex Deucher963e81f2013-06-26 17:37:11 -0400[diff] [blame]7853 cik_mec_fini(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7854 radeon_wb_fini(rdev);
7855 radeon_vm_manager_fini(rdev);
7856 radeon_ib_pool_fini(rdev);
7857 radeon_irq_kms_fini(rdev);
Christian Könige409b122013-08-13 11:56:53 +0200[diff] [blame]7858 uvd_v1_0_fini(rdev);
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]7859 radeon_uvd_fini(rdev);
Alex Deucher7bf94a22012-08-17 11:48:29 -0400[diff] [blame]7860 cik_pcie_gart_fini(rdev);
7861 r600_vram_scratch_fini(rdev);
7862 radeon_gem_fini(rdev);
7863 radeon_fence_driver_fini(rdev);
7864 radeon_bo_fini(rdev);
7865 radeon_atombios_fini(rdev);
7866 kfree(rdev->bios);
7867 rdev->bios = NULL;
7868}
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]7869
Alex Deucher134b4802013-09-23 12:22:11 -0400[diff] [blame]7870void dce8_program_fmt(struct drm_encoder *encoder)
7871{
7872 struct drm_device *dev = encoder->dev;
7873 struct radeon_device *rdev = dev->dev_private;
7874 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
7875 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
7876 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
7877 int bpc = 0;
7878 u32 tmp = 0;
Alex Deucher6214bb72013-09-24 17:26:26 -0400[diff] [blame]7879 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
Alex Deucher134b4802013-09-23 12:22:11 -0400[diff] [blame]7880
Alex Deucher6214bb72013-09-24 17:26:26 -0400[diff] [blame]7881 if (connector) {
7882 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
Alex Deucher134b4802013-09-23 12:22:11 -0400[diff] [blame]7883 bpc = radeon_get_monitor_bpc(connector);
Alex Deucher6214bb72013-09-24 17:26:26 -0400[diff] [blame]7884 dither = radeon_connector->dither;
7885 }
Alex Deucher134b4802013-09-23 12:22:11 -0400[diff] [blame]7886
7887 /* LVDS/eDP FMT is set up by atom */
7888 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
7889 return;
7890
7891 /* not needed for analog */
7892 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
7893 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
7894 return;
7895
7896 if (bpc == 0)
7897 return;
7898
7899 switch (bpc) {
7900 case 6:
Alex Deucher6214bb72013-09-24 17:26:26 -0400[diff] [blame]7901 if (dither == RADEON_FMT_DITHER_ENABLE)
Alex Deucher134b4802013-09-23 12:22:11 -0400[diff] [blame]7902 /* XXX sort out optimal dither settings */
7903 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
7904 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0));
7905 else
7906 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0));
7907 break;
7908 case 8:
Alex Deucher6214bb72013-09-24 17:26:26 -0400[diff] [blame]7909 if (dither == RADEON_FMT_DITHER_ENABLE)
Alex Deucher134b4802013-09-23 12:22:11 -0400[diff] [blame]7910 /* XXX sort out optimal dither settings */
7911 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
7912 FMT_RGB_RANDOM_ENABLE |
7913 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1));
7914 else
7915 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1));
7916 break;
7917 case 10:
Alex Deucher6214bb72013-09-24 17:26:26 -0400[diff] [blame]7918 if (dither == RADEON_FMT_DITHER_ENABLE)
Alex Deucher134b4802013-09-23 12:22:11 -0400[diff] [blame]7919 /* XXX sort out optimal dither settings */
7920 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
7921 FMT_RGB_RANDOM_ENABLE |
7922 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2));
7923 else
7924 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2));
7925 break;
7926 default:
7927 /* not needed */
7928 break;
7929 }
7930
7931 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
7932}
7933
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]7934/* display watermark setup */
7935/**
7936 * dce8_line_buffer_adjust - Set up the line buffer
7937 *
7938 * @rdev: radeon_device pointer
7939 * @radeon_crtc: the selected display controller
7940 * @mode: the current display mode on the selected display
7941 * controller
7942 *
7943 * Setup up the line buffer allocation for
7944 * the selected display controller (CIK).
7945 * Returns the line buffer size in pixels.
7946 */
7947static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
7948 struct radeon_crtc *radeon_crtc,
7949 struct drm_display_mode *mode)
7950{
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]7951 u32 tmp, buffer_alloc, i;
7952 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]7953 /*
7954 * Line Buffer Setup
7955 * There are 6 line buffers, one for each display controllers.
7956 * There are 3 partitions per LB. Select the number of partitions
7957 * to enable based on the display width. For display widths larger
7958 * than 4096, you need use to use 2 display controllers and combine
7959 * them using the stereo blender.
7960 */
7961 if (radeon_crtc->base.enabled && mode) {
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]7962 if (mode->crtc_hdisplay < 1920) {
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]7963 tmp = 1;
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]7964 buffer_alloc = 2;
7965 } else if (mode->crtc_hdisplay < 2560) {
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]7966 tmp = 2;
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]7967 buffer_alloc = 2;
7968 } else if (mode->crtc_hdisplay < 4096) {
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]7969 tmp = 0;
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]7970 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
7971 } else {
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]7972 DRM_DEBUG_KMS("Mode too big for LB!\n");
7973 tmp = 0;
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]7974 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]7975 }
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]7976 } else {
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]7977 tmp = 1;
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]7978 buffer_alloc = 0;
7979 }
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]7980
7981 WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
7982 LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
7983
Alex Deucherbc01a8c2013-08-19 11:39:27 -0400[diff] [blame]7984 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
7985 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
7986 for (i = 0; i < rdev->usec_timeout; i++) {
7987 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
7988 DMIF_BUFFERS_ALLOCATED_COMPLETED)
7989 break;
7990 udelay(1);
7991 }
7992
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]7993 if (radeon_crtc->base.enabled && mode) {
7994 switch (tmp) {
7995 case 0:
7996 default:
7997 return 4096 * 2;
7998 case 1:
7999 return 1920 * 2;
8000 case 2:
8001 return 2560 * 2;
8002 }
8003 }
8004
8005 /* controller not enabled, so no lb used */
8006 return 0;
8007}
8008
8009/**
8010 * cik_get_number_of_dram_channels - get the number of dram channels
8011 *
8012 * @rdev: radeon_device pointer
8013 *
8014 * Look up the number of video ram channels (CIK).
8015 * Used for display watermark bandwidth calculations
8016 * Returns the number of dram channels
8017 */
8018static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
8019{
8020 u32 tmp = RREG32(MC_SHARED_CHMAP);
8021
8022 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
8023 case 0:
8024 default:
8025 return 1;
8026 case 1:
8027 return 2;
8028 case 2:
8029 return 4;
8030 case 3:
8031 return 8;
8032 case 4:
8033 return 3;
8034 case 5:
8035 return 6;
8036 case 6:
8037 return 10;
8038 case 7:
8039 return 12;
8040 case 8:
8041 return 16;
8042 }
8043}
8044
8045struct dce8_wm_params {
8046 u32 dram_channels; /* number of dram channels */
8047 u32 yclk; /* bandwidth per dram data pin in kHz */
8048 u32 sclk; /* engine clock in kHz */
8049 u32 disp_clk; /* display clock in kHz */
8050 u32 src_width; /* viewport width */
8051 u32 active_time; /* active display time in ns */
8052 u32 blank_time; /* blank time in ns */
8053 bool interlaced; /* mode is interlaced */
8054 fixed20_12 vsc; /* vertical scale ratio */
8055 u32 num_heads; /* number of active crtcs */
8056 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
8057 u32 lb_size; /* line buffer allocated to pipe */
8058 u32 vtaps; /* vertical scaler taps */
8059};
8060
8061/**
8062 * dce8_dram_bandwidth - get the dram bandwidth
8063 *
8064 * @wm: watermark calculation data
8065 *
8066 * Calculate the raw dram bandwidth (CIK).
8067 * Used for display watermark bandwidth calculations
8068 * Returns the dram bandwidth in MBytes/s
8069 */
8070static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
8071{
8072 /* Calculate raw DRAM Bandwidth */
8073 fixed20_12 dram_efficiency; /* 0.7 */
8074 fixed20_12 yclk, dram_channels, bandwidth;
8075 fixed20_12 a;
8076
8077 a.full = dfixed_const(1000);
8078 yclk.full = dfixed_const(wm->yclk);
8079 yclk.full = dfixed_div(yclk, a);
8080 dram_channels.full = dfixed_const(wm->dram_channels * 4);
8081 a.full = dfixed_const(10);
8082 dram_efficiency.full = dfixed_const(7);
8083 dram_efficiency.full = dfixed_div(dram_efficiency, a);
8084 bandwidth.full = dfixed_mul(dram_channels, yclk);
8085 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
8086
8087 return dfixed_trunc(bandwidth);
8088}
8089
8090/**
8091 * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
8092 *
8093 * @wm: watermark calculation data
8094 *
8095 * Calculate the dram bandwidth used for display (CIK).
8096 * Used for display watermark bandwidth calculations
8097 * Returns the dram bandwidth for display in MBytes/s
8098 */
8099static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
8100{
8101 /* Calculate DRAM Bandwidth and the part allocated to display. */
8102 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
8103 fixed20_12 yclk, dram_channels, bandwidth;
8104 fixed20_12 a;
8105
8106 a.full = dfixed_const(1000);
8107 yclk.full = dfixed_const(wm->yclk);
8108 yclk.full = dfixed_div(yclk, a);
8109 dram_channels.full = dfixed_const(wm->dram_channels * 4);
8110 a.full = dfixed_const(10);
8111 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
8112 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
8113 bandwidth.full = dfixed_mul(dram_channels, yclk);
8114 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
8115
8116 return dfixed_trunc(bandwidth);
8117}
8118
8119/**
8120 * dce8_data_return_bandwidth - get the data return bandwidth
8121 *
8122 * @wm: watermark calculation data
8123 *
8124 * Calculate the data return bandwidth used for display (CIK).
8125 * Used for display watermark bandwidth calculations
8126 * Returns the data return bandwidth in MBytes/s
8127 */
8128static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
8129{
8130 /* Calculate the display Data return Bandwidth */
8131 fixed20_12 return_efficiency; /* 0.8 */
8132 fixed20_12 sclk, bandwidth;
8133 fixed20_12 a;
8134
8135 a.full = dfixed_const(1000);
8136 sclk.full = dfixed_const(wm->sclk);
8137 sclk.full = dfixed_div(sclk, a);
8138 a.full = dfixed_const(10);
8139 return_efficiency.full = dfixed_const(8);
8140 return_efficiency.full = dfixed_div(return_efficiency, a);
8141 a.full = dfixed_const(32);
8142 bandwidth.full = dfixed_mul(a, sclk);
8143 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
8144
8145 return dfixed_trunc(bandwidth);
8146}
8147
8148/**
8149 * dce8_dmif_request_bandwidth - get the dmif bandwidth
8150 *
8151 * @wm: watermark calculation data
8152 *
8153 * Calculate the dmif bandwidth used for display (CIK).
8154 * Used for display watermark bandwidth calculations
8155 * Returns the dmif bandwidth in MBytes/s
8156 */
8157static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
8158{
8159 /* Calculate the DMIF Request Bandwidth */
8160 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
8161 fixed20_12 disp_clk, bandwidth;
8162 fixed20_12 a, b;
8163
8164 a.full = dfixed_const(1000);
8165 disp_clk.full = dfixed_const(wm->disp_clk);
8166 disp_clk.full = dfixed_div(disp_clk, a);
8167 a.full = dfixed_const(32);
8168 b.full = dfixed_mul(a, disp_clk);
8169
8170 a.full = dfixed_const(10);
8171 disp_clk_request_efficiency.full = dfixed_const(8);
8172 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
8173
8174 bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
8175
8176 return dfixed_trunc(bandwidth);
8177}
8178
8179/**
8180 * dce8_available_bandwidth - get the min available bandwidth
8181 *
8182 * @wm: watermark calculation data
8183 *
8184 * Calculate the min available bandwidth used for display (CIK).
8185 * Used for display watermark bandwidth calculations
8186 * Returns the min available bandwidth in MBytes/s
8187 */
8188static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
8189{
8190 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
8191 u32 dram_bandwidth = dce8_dram_bandwidth(wm);
8192 u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
8193 u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
8194
8195 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
8196}
8197
8198/**
8199 * dce8_average_bandwidth - get the average available bandwidth
8200 *
8201 * @wm: watermark calculation data
8202 *
8203 * Calculate the average available bandwidth used for display (CIK).
8204 * Used for display watermark bandwidth calculations
8205 * Returns the average available bandwidth in MBytes/s
8206 */
8207static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
8208{
8209 /* Calculate the display mode Average Bandwidth
8210 * DisplayMode should contain the source and destination dimensions,
8211 * timing, etc.
8212 */
8213 fixed20_12 bpp;
8214 fixed20_12 line_time;
8215 fixed20_12 src_width;
8216 fixed20_12 bandwidth;
8217 fixed20_12 a;
8218
8219 a.full = dfixed_const(1000);
8220 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
8221 line_time.full = dfixed_div(line_time, a);
8222 bpp.full = dfixed_const(wm->bytes_per_pixel);
8223 src_width.full = dfixed_const(wm->src_width);
8224 bandwidth.full = dfixed_mul(src_width, bpp);
8225 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
8226 bandwidth.full = dfixed_div(bandwidth, line_time);
8227
8228 return dfixed_trunc(bandwidth);
8229}
8230
8231/**
8232 * dce8_latency_watermark - get the latency watermark
8233 *
8234 * @wm: watermark calculation data
8235 *
8236 * Calculate the latency watermark (CIK).
8237 * Used for display watermark bandwidth calculations
8238 * Returns the latency watermark in ns
8239 */
8240static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
8241{
8242 /* First calculate the latency in ns */
8243 u32 mc_latency = 2000; /* 2000 ns. */
8244 u32 available_bandwidth = dce8_available_bandwidth(wm);
8245 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
8246 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
8247 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
8248 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
8249 (wm->num_heads * cursor_line_pair_return_time);
8250 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
8251 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
8252 u32 tmp, dmif_size = 12288;
8253 fixed20_12 a, b, c;
8254
8255 if (wm->num_heads == 0)
8256 return 0;
8257
8258 a.full = dfixed_const(2);
8259 b.full = dfixed_const(1);
8260 if ((wm->vsc.full > a.full) ||
8261 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
8262 (wm->vtaps >= 5) ||
8263 ((wm->vsc.full >= a.full) && wm->interlaced))
8264 max_src_lines_per_dst_line = 4;
8265 else
8266 max_src_lines_per_dst_line = 2;
8267
8268 a.full = dfixed_const(available_bandwidth);
8269 b.full = dfixed_const(wm->num_heads);
8270 a.full = dfixed_div(a, b);
8271
8272 b.full = dfixed_const(mc_latency + 512);
8273 c.full = dfixed_const(wm->disp_clk);
8274 b.full = dfixed_div(b, c);
8275
8276 c.full = dfixed_const(dmif_size);
8277 b.full = dfixed_div(c, b);
8278
8279 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
8280
8281 b.full = dfixed_const(1000);
8282 c.full = dfixed_const(wm->disp_clk);
8283 b.full = dfixed_div(c, b);
8284 c.full = dfixed_const(wm->bytes_per_pixel);
8285 b.full = dfixed_mul(b, c);
8286
8287 lb_fill_bw = min(tmp, dfixed_trunc(b));
8288
8289 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
8290 b.full = dfixed_const(1000);
8291 c.full = dfixed_const(lb_fill_bw);
8292 b.full = dfixed_div(c, b);
8293 a.full = dfixed_div(a, b);
8294 line_fill_time = dfixed_trunc(a);
8295
8296 if (line_fill_time < wm->active_time)
8297 return latency;
8298 else
8299 return latency + (line_fill_time - wm->active_time);
8300
8301}
8302
8303/**
8304 * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
8305 * average and available dram bandwidth
8306 *
8307 * @wm: watermark calculation data
8308 *
8309 * Check if the display average bandwidth fits in the display
8310 * dram bandwidth (CIK).
8311 * Used for display watermark bandwidth calculations
8312 * Returns true if the display fits, false if not.
8313 */
8314static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
8315{
8316 if (dce8_average_bandwidth(wm) <=
8317 (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
8318 return true;
8319 else
8320 return false;
8321}
8322
8323/**
8324 * dce8_average_bandwidth_vs_available_bandwidth - check
8325 * average and available bandwidth
8326 *
8327 * @wm: watermark calculation data
8328 *
8329 * Check if the display average bandwidth fits in the display
8330 * available bandwidth (CIK).
8331 * Used for display watermark bandwidth calculations
8332 * Returns true if the display fits, false if not.
8333 */
8334static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
8335{
8336 if (dce8_average_bandwidth(wm) <=
8337 (dce8_available_bandwidth(wm) / wm->num_heads))
8338 return true;
8339 else
8340 return false;
8341}
8342
8343/**
8344 * dce8_check_latency_hiding - check latency hiding
8345 *
8346 * @wm: watermark calculation data
8347 *
8348 * Check latency hiding (CIK).
8349 * Used for display watermark bandwidth calculations
8350 * Returns true if the display fits, false if not.
8351 */
8352static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
8353{
8354 u32 lb_partitions = wm->lb_size / wm->src_width;
8355 u32 line_time = wm->active_time + wm->blank_time;
8356 u32 latency_tolerant_lines;
8357 u32 latency_hiding;
8358 fixed20_12 a;
8359
8360 a.full = dfixed_const(1);
8361 if (wm->vsc.full > a.full)
8362 latency_tolerant_lines = 1;
8363 else {
8364 if (lb_partitions <= (wm->vtaps + 1))
8365 latency_tolerant_lines = 1;
8366 else
8367 latency_tolerant_lines = 2;
8368 }
8369
8370 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
8371
8372 if (dce8_latency_watermark(wm) <= latency_hiding)
8373 return true;
8374 else
8375 return false;
8376}
8377
8378/**
8379 * dce8_program_watermarks - program display watermarks
8380 *
8381 * @rdev: radeon_device pointer
8382 * @radeon_crtc: the selected display controller
8383 * @lb_size: line buffer size
8384 * @num_heads: number of display controllers in use
8385 *
8386 * Calculate and program the display watermarks for the
8387 * selected display controller (CIK).
8388 */
8389static void dce8_program_watermarks(struct radeon_device *rdev,
8390 struct radeon_crtc *radeon_crtc,
8391 u32 lb_size, u32 num_heads)
8392{
8393 struct drm_display_mode *mode = &radeon_crtc->base.mode;
Alex Deucher58ea2de2013-01-24 10:03:39 -0500[diff] [blame]8394 struct dce8_wm_params wm_low, wm_high;
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8395 u32 pixel_period;
8396 u32 line_time = 0;
8397 u32 latency_watermark_a = 0, latency_watermark_b = 0;
8398 u32 tmp, wm_mask;
8399
8400 if (radeon_crtc->base.enabled && num_heads && mode) {
8401 pixel_period = 1000000 / (u32)mode->clock;
8402 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
8403
Alex Deucher58ea2de2013-01-24 10:03:39 -0500[diff] [blame]8404 /* watermark for high clocks */
8405 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
8406 rdev->pm.dpm_enabled) {
8407 wm_high.yclk =
8408 radeon_dpm_get_mclk(rdev, false) * 10;
8409 wm_high.sclk =
8410 radeon_dpm_get_sclk(rdev, false) * 10;
8411 } else {
8412 wm_high.yclk = rdev->pm.current_mclk * 10;
8413 wm_high.sclk = rdev->pm.current_sclk * 10;
8414 }
8415
8416 wm_high.disp_clk = mode->clock;
8417 wm_high.src_width = mode->crtc_hdisplay;
8418 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
8419 wm_high.blank_time = line_time - wm_high.active_time;
8420 wm_high.interlaced = false;
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8421 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
Alex Deucher58ea2de2013-01-24 10:03:39 -0500[diff] [blame]8422 wm_high.interlaced = true;
8423 wm_high.vsc = radeon_crtc->vsc;
8424 wm_high.vtaps = 1;
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8425 if (radeon_crtc->rmx_type != RMX_OFF)
Alex Deucher58ea2de2013-01-24 10:03:39 -0500[diff] [blame]8426 wm_high.vtaps = 2;
8427 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
8428 wm_high.lb_size = lb_size;
8429 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);
8430 wm_high.num_heads = num_heads;
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8431
8432 /* set for high clocks */
Alex Deucher58ea2de2013-01-24 10:03:39 -0500[diff] [blame]8433 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535);
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8434
8435 /* possibly force display priority to high */
8436 /* should really do this at mode validation time... */
Alex Deucher58ea2de2013-01-24 10:03:39 -0500[diff] [blame]8437 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
8438 !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) ||
8439 !dce8_check_latency_hiding(&wm_high) ||
8440 (rdev->disp_priority == 2)) {
8441 DRM_DEBUG_KMS("force priority to high\n");
8442 }
8443
8444 /* watermark for low clocks */
8445 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
8446 rdev->pm.dpm_enabled) {
8447 wm_low.yclk =
8448 radeon_dpm_get_mclk(rdev, true) * 10;
8449 wm_low.sclk =
8450 radeon_dpm_get_sclk(rdev, true) * 10;
8451 } else {
8452 wm_low.yclk = rdev->pm.current_mclk * 10;
8453 wm_low.sclk = rdev->pm.current_sclk * 10;
8454 }
8455
8456 wm_low.disp_clk = mode->clock;
8457 wm_low.src_width = mode->crtc_hdisplay;
8458 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
8459 wm_low.blank_time = line_time - wm_low.active_time;
8460 wm_low.interlaced = false;
8461 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
8462 wm_low.interlaced = true;
8463 wm_low.vsc = radeon_crtc->vsc;
8464 wm_low.vtaps = 1;
8465 if (radeon_crtc->rmx_type != RMX_OFF)
8466 wm_low.vtaps = 2;
8467 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
8468 wm_low.lb_size = lb_size;
8469 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);
8470 wm_low.num_heads = num_heads;
8471
8472 /* set for low clocks */
8473 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535);
8474
8475 /* possibly force display priority to high */
8476 /* should really do this at mode validation time... */
8477 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
8478 !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) ||
8479 !dce8_check_latency_hiding(&wm_low) ||
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8480 (rdev->disp_priority == 2)) {
8481 DRM_DEBUG_KMS("force priority to high\n");
8482 }
8483 }
8484
8485 /* select wm A */
8486 wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
8487 tmp = wm_mask;
8488 tmp &= ~LATENCY_WATERMARK_MASK(3);
8489 tmp |= LATENCY_WATERMARK_MASK(1);
8490 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
8491 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
8492 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
8493 LATENCY_HIGH_WATERMARK(line_time)));
8494 /* select wm B */
8495 tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
8496 tmp &= ~LATENCY_WATERMARK_MASK(3);
8497 tmp |= LATENCY_WATERMARK_MASK(2);
8498 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
8499 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
8500 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
8501 LATENCY_HIGH_WATERMARK(line_time)));
8502 /* restore original selection */
8503 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
Alex Deucher58ea2de2013-01-24 10:03:39 -0500[diff] [blame]8504
8505 /* save values for DPM */
8506 radeon_crtc->line_time = line_time;
8507 radeon_crtc->wm_high = latency_watermark_a;
8508 radeon_crtc->wm_low = latency_watermark_b;
Alex Deuchercd84a272012-07-20 17:13:13 -0400[diff] [blame]8509}
8510
8511/**
8512 * dce8_bandwidth_update - program display watermarks
8513 *
8514 * @rdev: radeon_device pointer
8515 *
8516 * Calculate and program the display watermarks and line
8517 * buffer allocation (CIK).
8518 */
8519void dce8_bandwidth_update(struct radeon_device *rdev)
8520{
8521 struct drm_display_mode *mode = NULL;
8522 u32 num_heads = 0, lb_size;
8523 int i;
8524
8525 radeon_update_display_priority(rdev);
8526
8527 for (i = 0; i < rdev->num_crtc; i++) {
8528 if (rdev->mode_info.crtcs[i]->base.enabled)
8529 num_heads++;
8530 }
8531 for (i = 0; i < rdev->num_crtc; i++) {
8532 mode = &rdev->mode_info.crtcs[i]->base.mode;
8533 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
8534 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
8535 }
8536}
Alex Deucher44fa3462012-12-18 22:17:00 -0500[diff] [blame]8537
8538/**
8539 * cik_get_gpu_clock_counter - return GPU clock counter snapshot
8540 *
8541 * @rdev: radeon_device pointer
8542 *
8543 * Fetches a GPU clock counter snapshot (SI).
8544 * Returns the 64 bit clock counter snapshot.
8545 */
8546uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
8547{
8548 uint64_t clock;
8549
8550 mutex_lock(&rdev->gpu_clock_mutex);
8551 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
8552 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
8553 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
8554 mutex_unlock(&rdev->gpu_clock_mutex);
8555 return clock;
8556}
8557
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8558static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
8559 u32 cntl_reg, u32 status_reg)
8560{
8561 int r, i;
8562 struct atom_clock_dividers dividers;
8563 uint32_t tmp;
8564
8565 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
8566 clock, false, &dividers);
8567 if (r)
8568 return r;
8569
8570 tmp = RREG32_SMC(cntl_reg);
8571 tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
8572 tmp |= dividers.post_divider;
8573 WREG32_SMC(cntl_reg, tmp);
8574
8575 for (i = 0; i < 100; i++) {
8576 if (RREG32_SMC(status_reg) & DCLK_STATUS)
8577 break;
8578 mdelay(10);
8579 }
8580 if (i == 100)
8581 return -ETIMEDOUT;
8582
8583 return 0;
8584}
8585
8586int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
8587{
8588 int r = 0;
8589
8590 r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
8591 if (r)
8592 return r;
8593
8594 r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
8595 return r;
8596}
8597
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8598static void cik_pcie_gen3_enable(struct radeon_device *rdev)
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8599{
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8600 struct pci_dev *root = rdev->pdev->bus->self;
8601 int bridge_pos, gpu_pos;
8602 u32 speed_cntl, mask, current_data_rate;
8603 int ret, i;
8604 u16 tmp16;
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8605
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8606 if (radeon_pcie_gen2 == 0)
8607 return;
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8608
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8609 if (rdev->flags & RADEON_IS_IGP)
8610 return;
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8611
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8612 if (!(rdev->flags & RADEON_IS_PCIE))
8613 return;
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8614
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8615 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
8616 if (ret != 0)
8617 return;
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8618
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8619 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
8620 return;
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8621
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8622 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
8623 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
8624 LC_CURRENT_DATA_RATE_SHIFT;
8625 if (mask & DRM_PCIE_SPEED_80) {
8626 if (current_data_rate == 2) {
8627 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
8628 return;
8629 }
8630 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
8631 } else if (mask & DRM_PCIE_SPEED_50) {
8632 if (current_data_rate == 1) {
8633 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
8634 return;
8635 }
8636 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
8637 }
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8638
Alex Deucher8a7cd272013-08-06 11:29:39 -0400[diff] [blame]8639 bridge_pos = pci_pcie_cap(root);
8640 if (!bridge_pos)
8641 return;
8642
8643 gpu_pos = pci_pcie_cap(rdev->pdev);
8644 if (!gpu_pos)
8645 return;
8646
8647 if (mask & DRM_PCIE_SPEED_80) {
8648 /* re-try equalization if gen3 is not already enabled */
8649 if (current_data_rate != 2) {
8650 u16 bridge_cfg, gpu_cfg;
8651 u16 bridge_cfg2, gpu_cfg2;
8652 u32 max_lw, current_lw, tmp;
8653
8654 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
8655 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
8656
8657 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
8658 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
8659
8660 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
8661 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
8662
8663 tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
8664 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
8665 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
8666
8667 if (current_lw < max_lw) {
8668 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
8669 if (tmp & LC_RENEGOTIATION_SUPPORT) {
8670 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
8671 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
8672 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
8673 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
8674 }
8675 }
8676
8677 for (i = 0; i < 10; i++) {
8678 /* check status */
8679 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
8680 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
8681 break;
8682
8683 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
8684 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
8685
8686 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
8687 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
8688
8689 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
8690 tmp |= LC_SET_QUIESCE;
8691 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
8692
8693 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
8694 tmp |= LC_REDO_EQ;
8695 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
8696
8697 mdelay(100);
8698
8699 /* linkctl */
8700 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
8701 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
8702 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
8703 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
8704
8705 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
8706 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
8707 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
8708 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
8709
8710 /* linkctl2 */
8711 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
8712 tmp16 &= ~((1 << 4) | (7 << 9));
8713 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
8714 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
8715
8716 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
8717 tmp16 &= ~((1 << 4) | (7 << 9));
8718 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
8719 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
8720
8721 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
8722 tmp &= ~LC_SET_QUIESCE;
8723 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
8724 }
8725 }
8726 }
8727
8728 /* set the link speed */
8729 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
8730 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
8731 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
8732
8733 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
8734 tmp16 &= ~0xf;
8735 if (mask & DRM_PCIE_SPEED_80)
8736 tmp16 |= 3; /* gen3 */
8737 else if (mask & DRM_PCIE_SPEED_50)
8738 tmp16 |= 2; /* gen2 */
8739 else
8740 tmp16 |= 1; /* gen1 */
8741 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
8742
8743 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
8744 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
8745 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
8746
8747 for (i = 0; i < rdev->usec_timeout; i++) {
8748 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
8749 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
8750 break;
8751 udelay(1);
8752 }
8753}
Alex Deucher7235711a42013-04-04 13:58:09 -0400[diff] [blame]8754
8755static void cik_program_aspm(struct radeon_device *rdev)
8756{
8757 u32 data, orig;
8758 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
8759 bool disable_clkreq = false;
8760
8761 if (radeon_aspm == 0)
8762 return;
8763
8764 /* XXX double check IGPs */
8765 if (rdev->flags & RADEON_IS_IGP)
8766 return;
8767
8768 if (!(rdev->flags & RADEON_IS_PCIE))
8769 return;
8770
8771 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
8772 data &= ~LC_XMIT_N_FTS_MASK;
8773 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
8774 if (orig != data)
8775 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
8776
8777 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
8778 data |= LC_GO_TO_RECOVERY;
8779 if (orig != data)
8780 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
8781
8782 orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL);
8783 data |= P_IGNORE_EDB_ERR;
8784 if (orig != data)
8785 WREG32_PCIE_PORT(PCIE_P_CNTL, data);
8786
8787 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
8788 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
8789 data |= LC_PMI_TO_L1_DIS;
8790 if (!disable_l0s)
8791 data |= LC_L0S_INACTIVITY(7);
8792
8793 if (!disable_l1) {
8794 data |= LC_L1_INACTIVITY(7);
8795 data &= ~LC_PMI_TO_L1_DIS;
8796 if (orig != data)
8797 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
8798
8799 if (!disable_plloff_in_l1) {
8800 bool clk_req_support;
8801
8802 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0);
8803 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
8804 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
8805 if (orig != data)
8806 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data);
8807
8808 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1);
8809 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
8810 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
8811 if (orig != data)
8812 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data);
8813
8814 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0);
8815 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
8816 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
8817 if (orig != data)
8818 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data);
8819
8820 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1);
8821 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
8822 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
8823 if (orig != data)
8824 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data);
8825
8826 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
8827 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
8828 data |= LC_DYN_LANES_PWR_STATE(3);
8829 if (orig != data)
8830 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
8831
8832 if (!disable_clkreq) {
8833 struct pci_dev *root = rdev->pdev->bus->self;
8834 u32 lnkcap;
8835
8836 clk_req_support = false;
8837 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
8838 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
8839 clk_req_support = true;
8840 } else {
8841 clk_req_support = false;
8842 }
8843
8844 if (clk_req_support) {
8845 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
8846 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
8847 if (orig != data)
8848 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
8849
8850 orig = data = RREG32_SMC(THM_CLK_CNTL);
8851 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
8852 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
8853 if (orig != data)
8854 WREG32_SMC(THM_CLK_CNTL, data);
8855
8856 orig = data = RREG32_SMC(MISC_CLK_CTRL);
8857 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
8858 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
8859 if (orig != data)
8860 WREG32_SMC(MISC_CLK_CTRL, data);
8861
8862 orig = data = RREG32_SMC(CG_CLKPIN_CNTL);
8863 data &= ~BCLK_AS_XCLK;
8864 if (orig != data)
8865 WREG32_SMC(CG_CLKPIN_CNTL, data);
8866
8867 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2);
8868 data &= ~FORCE_BIF_REFCLK_EN;
8869 if (orig != data)
8870 WREG32_SMC(CG_CLKPIN_CNTL_2, data);
8871
8872 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL);
8873 data &= ~MPLL_CLKOUT_SEL_MASK;
8874 data |= MPLL_CLKOUT_SEL(4);
8875 if (orig != data)
8876 WREG32_SMC(MPLL_BYPASSCLK_SEL, data);
8877 }
8878 }
8879 } else {
8880 if (orig != data)
8881 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
8882 }
8883
8884 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
8885 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
8886 if (orig != data)
8887 WREG32_PCIE_PORT(PCIE_CNTL2, data);
8888
8889 if (!disable_l0s) {
8890 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
8891 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
8892 data = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
8893 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
8894 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
8895 data &= ~LC_L0S_INACTIVITY_MASK;
8896 if (orig != data)
8897 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
8898 }
8899 }
8900 }
Christian König87167bb2013-04-09 13:39:21 -0400[diff] [blame]8901}