blob: c8f359ca64d15ee3903ec0b04cd2968237056afa [file] [log] [blame]
Alex Deucher43b3cd92012-03-20 17:18:00 -04001/*
2 * Copyright 2011 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
Alex Deucher0f0de062012-03-20 17:18:17 -040024#include <linux/firmware.h>
Alex Deucher0f0de062012-03-20 17:18:17 -040025#include <linux/slab.h>
26#include <linux/module.h>
David Howells760285e2012-10-02 18:01:07 +010027#include <drm/drmP.h>
Alex Deucher43b3cd92012-03-20 17:18:00 -040028#include "radeon.h"
29#include "radeon_asic.h"
David Howells760285e2012-10-02 18:01:07 +010030#include <drm/radeon_drm.h>
Alex Deucher43b3cd92012-03-20 17:18:00 -040031#include "sid.h"
32#include "atom.h"
Alex Deucher48c0c902012-03-20 17:18:19 -040033#include "si_blit_shaders.h"
Alex Deucherbd8cd532013-04-12 16:48:21 -040034#include "clearstate_si.h"
Alex Deuchera0ceada2013-03-27 15:18:04 -040035#include "radeon_ucode.h"
Alex Deucher43b3cd92012-03-20 17:18:00 -040036
Alex Deucher0f0de062012-03-20 17:18:17 -040037
38MODULE_FIRMWARE("radeon/TAHITI_pfp.bin");
39MODULE_FIRMWARE("radeon/TAHITI_me.bin");
40MODULE_FIRMWARE("radeon/TAHITI_ce.bin");
41MODULE_FIRMWARE("radeon/TAHITI_mc.bin");
Alex Deucher1ebe9282014-04-11 11:21:49 -040042MODULE_FIRMWARE("radeon/TAHITI_mc2.bin");
Alex Deucher0f0de062012-03-20 17:18:17 -040043MODULE_FIRMWARE("radeon/TAHITI_rlc.bin");
Alex Deuchera9e61412013-06-25 17:56:16 -040044MODULE_FIRMWARE("radeon/TAHITI_smc.bin");
Alex Deucher629bd332014-06-25 18:41:34 -040045
46MODULE_FIRMWARE("radeon/tahiti_pfp.bin");
47MODULE_FIRMWARE("radeon/tahiti_me.bin");
48MODULE_FIRMWARE("radeon/tahiti_ce.bin");
49MODULE_FIRMWARE("radeon/tahiti_mc.bin");
50MODULE_FIRMWARE("radeon/tahiti_rlc.bin");
51MODULE_FIRMWARE("radeon/tahiti_smc.bin");
52
Alex Deucher0f0de062012-03-20 17:18:17 -040053MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin");
54MODULE_FIRMWARE("radeon/PITCAIRN_me.bin");
55MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin");
56MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin");
Alex Deucher1ebe9282014-04-11 11:21:49 -040057MODULE_FIRMWARE("radeon/PITCAIRN_mc2.bin");
Alex Deucher0f0de062012-03-20 17:18:17 -040058MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin");
Alex Deuchera9e61412013-06-25 17:56:16 -040059MODULE_FIRMWARE("radeon/PITCAIRN_smc.bin");
Alex Deucher629bd332014-06-25 18:41:34 -040060
61MODULE_FIRMWARE("radeon/pitcairn_pfp.bin");
62MODULE_FIRMWARE("radeon/pitcairn_me.bin");
63MODULE_FIRMWARE("radeon/pitcairn_ce.bin");
64MODULE_FIRMWARE("radeon/pitcairn_mc.bin");
65MODULE_FIRMWARE("radeon/pitcairn_rlc.bin");
66MODULE_FIRMWARE("radeon/pitcairn_smc.bin");
67
Alex Deucher0f0de062012-03-20 17:18:17 -040068MODULE_FIRMWARE("radeon/VERDE_pfp.bin");
69MODULE_FIRMWARE("radeon/VERDE_me.bin");
70MODULE_FIRMWARE("radeon/VERDE_ce.bin");
71MODULE_FIRMWARE("radeon/VERDE_mc.bin");
Alex Deucher1ebe9282014-04-11 11:21:49 -040072MODULE_FIRMWARE("radeon/VERDE_mc2.bin");
Alex Deucher0f0de062012-03-20 17:18:17 -040073MODULE_FIRMWARE("radeon/VERDE_rlc.bin");
Alex Deuchera9e61412013-06-25 17:56:16 -040074MODULE_FIRMWARE("radeon/VERDE_smc.bin");
Alex Deucher629bd332014-06-25 18:41:34 -040075
76MODULE_FIRMWARE("radeon/verde_pfp.bin");
77MODULE_FIRMWARE("radeon/verde_me.bin");
78MODULE_FIRMWARE("radeon/verde_ce.bin");
79MODULE_FIRMWARE("radeon/verde_mc.bin");
80MODULE_FIRMWARE("radeon/verde_rlc.bin");
81MODULE_FIRMWARE("radeon/verde_smc.bin");
82
Alex Deucherbcc7f5d2012-07-26 18:36:28 -040083MODULE_FIRMWARE("radeon/OLAND_pfp.bin");
84MODULE_FIRMWARE("radeon/OLAND_me.bin");
85MODULE_FIRMWARE("radeon/OLAND_ce.bin");
86MODULE_FIRMWARE("radeon/OLAND_mc.bin");
Alex Deucher1ebe9282014-04-11 11:21:49 -040087MODULE_FIRMWARE("radeon/OLAND_mc2.bin");
Alex Deucherbcc7f5d2012-07-26 18:36:28 -040088MODULE_FIRMWARE("radeon/OLAND_rlc.bin");
Alex Deuchera9e61412013-06-25 17:56:16 -040089MODULE_FIRMWARE("radeon/OLAND_smc.bin");
Alex Deucher629bd332014-06-25 18:41:34 -040090
91MODULE_FIRMWARE("radeon/oland_pfp.bin");
92MODULE_FIRMWARE("radeon/oland_me.bin");
93MODULE_FIRMWARE("radeon/oland_ce.bin");
94MODULE_FIRMWARE("radeon/oland_mc.bin");
95MODULE_FIRMWARE("radeon/oland_rlc.bin");
96MODULE_FIRMWARE("radeon/oland_smc.bin");
97
Alex Deucherc04c00b2012-07-31 12:57:45 -040098MODULE_FIRMWARE("radeon/HAINAN_pfp.bin");
99MODULE_FIRMWARE("radeon/HAINAN_me.bin");
100MODULE_FIRMWARE("radeon/HAINAN_ce.bin");
101MODULE_FIRMWARE("radeon/HAINAN_mc.bin");
Alex Deucher1ebe9282014-04-11 11:21:49 -0400102MODULE_FIRMWARE("radeon/HAINAN_mc2.bin");
Alex Deucherc04c00b2012-07-31 12:57:45 -0400103MODULE_FIRMWARE("radeon/HAINAN_rlc.bin");
Alex Deuchera9e61412013-06-25 17:56:16 -0400104MODULE_FIRMWARE("radeon/HAINAN_smc.bin");
Alex Deucher0f0de062012-03-20 17:18:17 -0400105
Alex Deucher629bd332014-06-25 18:41:34 -0400106MODULE_FIRMWARE("radeon/hainan_pfp.bin");
107MODULE_FIRMWARE("radeon/hainan_me.bin");
108MODULE_FIRMWARE("radeon/hainan_ce.bin");
109MODULE_FIRMWARE("radeon/hainan_mc.bin");
110MODULE_FIRMWARE("radeon/hainan_rlc.bin");
111MODULE_FIRMWARE("radeon/hainan_smc.bin");
112
Alex Deucher65fcf662014-06-02 16:13:21 -0400113static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
Alex Deucherb9d305d2013-02-14 17:16:51 -0500114static void si_pcie_gen3_enable(struct radeon_device *rdev);
Alex Deuchere0bcf1652013-02-15 11:56:59 -0500115static void si_program_aspm(struct radeon_device *rdev);
Alex Deucher1fd11772013-04-17 17:53:50 -0400116extern void sumo_rlc_fini(struct radeon_device *rdev);
117extern int sumo_rlc_init(struct radeon_device *rdev);
Alex Deucher25a857f2012-03-20 17:18:22 -0400118extern int r600_ih_ring_alloc(struct radeon_device *rdev);
119extern void r600_ih_ring_fini(struct radeon_device *rdev);
Alex Deucher0a96d722012-03-20 17:18:11 -0400120extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
Alex Deucherc476dde2012-03-20 17:18:12 -0400121extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
122extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
Alex Deucherca7db222012-03-20 17:18:30 -0400123extern u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev);
Alex Deucher1c534672013-01-18 15:08:38 -0500124extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
Alex Deucher014bb202013-01-18 19:36:20 -0500125extern bool evergreen_is_display_hung(struct radeon_device *rdev);
Alex Deucher811e4d52013-09-03 13:31:33 -0400126static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
127 bool enable);
Alex Deucher4a5c8ea2013-11-15 16:35:55 -0500128static void si_init_pg(struct radeon_device *rdev);
129static void si_init_cg(struct radeon_device *rdev);
Alex Deuchera6f4ae82013-10-02 14:50:57 -0400130static void si_fini_pg(struct radeon_device *rdev);
131static void si_fini_cg(struct radeon_device *rdev);
132static void si_rlc_stop(struct radeon_device *rdev);
Alex Deucher0a96d722012-03-20 17:18:11 -0400133
Alex Deucher6d8cf002013-03-06 18:48:05 -0500134static const u32 verde_rlc_save_restore_register_list[] =
135{
136 (0x8000 << 16) | (0x98f4 >> 2),
137 0x00000000,
138 (0x8040 << 16) | (0x98f4 >> 2),
139 0x00000000,
140 (0x8000 << 16) | (0xe80 >> 2),
141 0x00000000,
142 (0x8040 << 16) | (0xe80 >> 2),
143 0x00000000,
144 (0x8000 << 16) | (0x89bc >> 2),
145 0x00000000,
146 (0x8040 << 16) | (0x89bc >> 2),
147 0x00000000,
148 (0x8000 << 16) | (0x8c1c >> 2),
149 0x00000000,
150 (0x8040 << 16) | (0x8c1c >> 2),
151 0x00000000,
152 (0x9c00 << 16) | (0x98f0 >> 2),
153 0x00000000,
154 (0x9c00 << 16) | (0xe7c >> 2),
155 0x00000000,
156 (0x8000 << 16) | (0x9148 >> 2),
157 0x00000000,
158 (0x8040 << 16) | (0x9148 >> 2),
159 0x00000000,
160 (0x9c00 << 16) | (0x9150 >> 2),
161 0x00000000,
162 (0x9c00 << 16) | (0x897c >> 2),
163 0x00000000,
164 (0x9c00 << 16) | (0x8d8c >> 2),
165 0x00000000,
166 (0x9c00 << 16) | (0xac54 >> 2),
167 0X00000000,
168 0x3,
169 (0x9c00 << 16) | (0x98f8 >> 2),
170 0x00000000,
171 (0x9c00 << 16) | (0x9910 >> 2),
172 0x00000000,
173 (0x9c00 << 16) | (0x9914 >> 2),
174 0x00000000,
175 (0x9c00 << 16) | (0x9918 >> 2),
176 0x00000000,
177 (0x9c00 << 16) | (0x991c >> 2),
178 0x00000000,
179 (0x9c00 << 16) | (0x9920 >> 2),
180 0x00000000,
181 (0x9c00 << 16) | (0x9924 >> 2),
182 0x00000000,
183 (0x9c00 << 16) | (0x9928 >> 2),
184 0x00000000,
185 (0x9c00 << 16) | (0x992c >> 2),
186 0x00000000,
187 (0x9c00 << 16) | (0x9930 >> 2),
188 0x00000000,
189 (0x9c00 << 16) | (0x9934 >> 2),
190 0x00000000,
191 (0x9c00 << 16) | (0x9938 >> 2),
192 0x00000000,
193 (0x9c00 << 16) | (0x993c >> 2),
194 0x00000000,
195 (0x9c00 << 16) | (0x9940 >> 2),
196 0x00000000,
197 (0x9c00 << 16) | (0x9944 >> 2),
198 0x00000000,
199 (0x9c00 << 16) | (0x9948 >> 2),
200 0x00000000,
201 (0x9c00 << 16) | (0x994c >> 2),
202 0x00000000,
203 (0x9c00 << 16) | (0x9950 >> 2),
204 0x00000000,
205 (0x9c00 << 16) | (0x9954 >> 2),
206 0x00000000,
207 (0x9c00 << 16) | (0x9958 >> 2),
208 0x00000000,
209 (0x9c00 << 16) | (0x995c >> 2),
210 0x00000000,
211 (0x9c00 << 16) | (0x9960 >> 2),
212 0x00000000,
213 (0x9c00 << 16) | (0x9964 >> 2),
214 0x00000000,
215 (0x9c00 << 16) | (0x9968 >> 2),
216 0x00000000,
217 (0x9c00 << 16) | (0x996c >> 2),
218 0x00000000,
219 (0x9c00 << 16) | (0x9970 >> 2),
220 0x00000000,
221 (0x9c00 << 16) | (0x9974 >> 2),
222 0x00000000,
223 (0x9c00 << 16) | (0x9978 >> 2),
224 0x00000000,
225 (0x9c00 << 16) | (0x997c >> 2),
226 0x00000000,
227 (0x9c00 << 16) | (0x9980 >> 2),
228 0x00000000,
229 (0x9c00 << 16) | (0x9984 >> 2),
230 0x00000000,
231 (0x9c00 << 16) | (0x9988 >> 2),
232 0x00000000,
233 (0x9c00 << 16) | (0x998c >> 2),
234 0x00000000,
235 (0x9c00 << 16) | (0x8c00 >> 2),
236 0x00000000,
237 (0x9c00 << 16) | (0x8c14 >> 2),
238 0x00000000,
239 (0x9c00 << 16) | (0x8c04 >> 2),
240 0x00000000,
241 (0x9c00 << 16) | (0x8c08 >> 2),
242 0x00000000,
243 (0x8000 << 16) | (0x9b7c >> 2),
244 0x00000000,
245 (0x8040 << 16) | (0x9b7c >> 2),
246 0x00000000,
247 (0x8000 << 16) | (0xe84 >> 2),
248 0x00000000,
249 (0x8040 << 16) | (0xe84 >> 2),
250 0x00000000,
251 (0x8000 << 16) | (0x89c0 >> 2),
252 0x00000000,
253 (0x8040 << 16) | (0x89c0 >> 2),
254 0x00000000,
255 (0x8000 << 16) | (0x914c >> 2),
256 0x00000000,
257 (0x8040 << 16) | (0x914c >> 2),
258 0x00000000,
259 (0x8000 << 16) | (0x8c20 >> 2),
260 0x00000000,
261 (0x8040 << 16) | (0x8c20 >> 2),
262 0x00000000,
263 (0x8000 << 16) | (0x9354 >> 2),
264 0x00000000,
265 (0x8040 << 16) | (0x9354 >> 2),
266 0x00000000,
267 (0x9c00 << 16) | (0x9060 >> 2),
268 0x00000000,
269 (0x9c00 << 16) | (0x9364 >> 2),
270 0x00000000,
271 (0x9c00 << 16) | (0x9100 >> 2),
272 0x00000000,
273 (0x9c00 << 16) | (0x913c >> 2),
274 0x00000000,
275 (0x8000 << 16) | (0x90e0 >> 2),
276 0x00000000,
277 (0x8000 << 16) | (0x90e4 >> 2),
278 0x00000000,
279 (0x8000 << 16) | (0x90e8 >> 2),
280 0x00000000,
281 (0x8040 << 16) | (0x90e0 >> 2),
282 0x00000000,
283 (0x8040 << 16) | (0x90e4 >> 2),
284 0x00000000,
285 (0x8040 << 16) | (0x90e8 >> 2),
286 0x00000000,
287 (0x9c00 << 16) | (0x8bcc >> 2),
288 0x00000000,
289 (0x9c00 << 16) | (0x8b24 >> 2),
290 0x00000000,
291 (0x9c00 << 16) | (0x88c4 >> 2),
292 0x00000000,
293 (0x9c00 << 16) | (0x8e50 >> 2),
294 0x00000000,
295 (0x9c00 << 16) | (0x8c0c >> 2),
296 0x00000000,
297 (0x9c00 << 16) | (0x8e58 >> 2),
298 0x00000000,
299 (0x9c00 << 16) | (0x8e5c >> 2),
300 0x00000000,
301 (0x9c00 << 16) | (0x9508 >> 2),
302 0x00000000,
303 (0x9c00 << 16) | (0x950c >> 2),
304 0x00000000,
305 (0x9c00 << 16) | (0x9494 >> 2),
306 0x00000000,
307 (0x9c00 << 16) | (0xac0c >> 2),
308 0x00000000,
309 (0x9c00 << 16) | (0xac10 >> 2),
310 0x00000000,
311 (0x9c00 << 16) | (0xac14 >> 2),
312 0x00000000,
313 (0x9c00 << 16) | (0xae00 >> 2),
314 0x00000000,
315 (0x9c00 << 16) | (0xac08 >> 2),
316 0x00000000,
317 (0x9c00 << 16) | (0x88d4 >> 2),
318 0x00000000,
319 (0x9c00 << 16) | (0x88c8 >> 2),
320 0x00000000,
321 (0x9c00 << 16) | (0x88cc >> 2),
322 0x00000000,
323 (0x9c00 << 16) | (0x89b0 >> 2),
324 0x00000000,
325 (0x9c00 << 16) | (0x8b10 >> 2),
326 0x00000000,
327 (0x9c00 << 16) | (0x8a14 >> 2),
328 0x00000000,
329 (0x9c00 << 16) | (0x9830 >> 2),
330 0x00000000,
331 (0x9c00 << 16) | (0x9834 >> 2),
332 0x00000000,
333 (0x9c00 << 16) | (0x9838 >> 2),
334 0x00000000,
335 (0x9c00 << 16) | (0x9a10 >> 2),
336 0x00000000,
337 (0x8000 << 16) | (0x9870 >> 2),
338 0x00000000,
339 (0x8000 << 16) | (0x9874 >> 2),
340 0x00000000,
341 (0x8001 << 16) | (0x9870 >> 2),
342 0x00000000,
343 (0x8001 << 16) | (0x9874 >> 2),
344 0x00000000,
345 (0x8040 << 16) | (0x9870 >> 2),
346 0x00000000,
347 (0x8040 << 16) | (0x9874 >> 2),
348 0x00000000,
349 (0x8041 << 16) | (0x9870 >> 2),
350 0x00000000,
351 (0x8041 << 16) | (0x9874 >> 2),
352 0x00000000,
353 0x00000000
354};
355
Alex Deucher205996c2013-03-01 17:08:42 -0500356static const u32 tahiti_golden_rlc_registers[] =
357{
358 0xc424, 0xffffffff, 0x00601005,
359 0xc47c, 0xffffffff, 0x10104040,
360 0xc488, 0xffffffff, 0x0100000a,
361 0xc314, 0xffffffff, 0x00000800,
362 0xc30c, 0xffffffff, 0x800000f4,
363 0xf4a8, 0xffffffff, 0x00000000
364};
365
366static const u32 tahiti_golden_registers[] =
367{
368 0x9a10, 0x00010000, 0x00018208,
369 0x9830, 0xffffffff, 0x00000000,
370 0x9834, 0xf00fffff, 0x00000400,
371 0x9838, 0x0002021c, 0x00020200,
372 0xc78, 0x00000080, 0x00000000,
373 0xd030, 0x000300c0, 0x00800040,
374 0xd830, 0x000300c0, 0x00800040,
375 0x5bb0, 0x000000f0, 0x00000070,
376 0x5bc0, 0x00200000, 0x50100000,
377 0x7030, 0x31000311, 0x00000011,
378 0x277c, 0x00000003, 0x000007ff,
379 0x240c, 0x000007ff, 0x00000000,
380 0x8a14, 0xf000001f, 0x00000007,
381 0x8b24, 0xffffffff, 0x00ffffff,
382 0x8b10, 0x0000ff0f, 0x00000000,
383 0x28a4c, 0x07ffffff, 0x4e000000,
384 0x28350, 0x3f3f3fff, 0x2a00126a,
385 0x30, 0x000000ff, 0x0040,
386 0x34, 0x00000040, 0x00004040,
387 0x9100, 0x07ffffff, 0x03000000,
388 0x8e88, 0x01ff1f3f, 0x00000000,
389 0x8e84, 0x01ff1f3f, 0x00000000,
390 0x9060, 0x0000007f, 0x00000020,
391 0x9508, 0x00010000, 0x00010000,
392 0xac14, 0x00000200, 0x000002fb,
393 0xac10, 0xffffffff, 0x0000543b,
394 0xac0c, 0xffffffff, 0xa9210876,
395 0x88d0, 0xffffffff, 0x000fff40,
396 0x88d4, 0x0000001f, 0x00000010,
397 0x1410, 0x20000000, 0x20fffed8,
398 0x15c0, 0x000c0fc0, 0x000c0400
399};
400
401static const u32 tahiti_golden_registers2[] =
402{
403 0xc64, 0x00000001, 0x00000001
404};
405
406static const u32 pitcairn_golden_rlc_registers[] =
407{
408 0xc424, 0xffffffff, 0x00601004,
409 0xc47c, 0xffffffff, 0x10102020,
410 0xc488, 0xffffffff, 0x01000020,
411 0xc314, 0xffffffff, 0x00000800,
412 0xc30c, 0xffffffff, 0x800000a4
413};
414
415static const u32 pitcairn_golden_registers[] =
416{
417 0x9a10, 0x00010000, 0x00018208,
418 0x9830, 0xffffffff, 0x00000000,
419 0x9834, 0xf00fffff, 0x00000400,
420 0x9838, 0x0002021c, 0x00020200,
421 0xc78, 0x00000080, 0x00000000,
422 0xd030, 0x000300c0, 0x00800040,
423 0xd830, 0x000300c0, 0x00800040,
424 0x5bb0, 0x000000f0, 0x00000070,
425 0x5bc0, 0x00200000, 0x50100000,
426 0x7030, 0x31000311, 0x00000011,
427 0x2ae4, 0x00073ffe, 0x000022a2,
428 0x240c, 0x000007ff, 0x00000000,
429 0x8a14, 0xf000001f, 0x00000007,
430 0x8b24, 0xffffffff, 0x00ffffff,
431 0x8b10, 0x0000ff0f, 0x00000000,
432 0x28a4c, 0x07ffffff, 0x4e000000,
433 0x28350, 0x3f3f3fff, 0x2a00126a,
434 0x30, 0x000000ff, 0x0040,
435 0x34, 0x00000040, 0x00004040,
436 0x9100, 0x07ffffff, 0x03000000,
437 0x9060, 0x0000007f, 0x00000020,
438 0x9508, 0x00010000, 0x00010000,
439 0xac14, 0x000003ff, 0x000000f7,
440 0xac10, 0xffffffff, 0x00000000,
441 0xac0c, 0xffffffff, 0x32761054,
442 0x88d4, 0x0000001f, 0x00000010,
443 0x15c0, 0x000c0fc0, 0x000c0400
444};
445
446static const u32 verde_golden_rlc_registers[] =
447{
448 0xc424, 0xffffffff, 0x033f1005,
449 0xc47c, 0xffffffff, 0x10808020,
450 0xc488, 0xffffffff, 0x00800008,
451 0xc314, 0xffffffff, 0x00001000,
452 0xc30c, 0xffffffff, 0x80010014
453};
454
455static const u32 verde_golden_registers[] =
456{
457 0x9a10, 0x00010000, 0x00018208,
458 0x9830, 0xffffffff, 0x00000000,
459 0x9834, 0xf00fffff, 0x00000400,
460 0x9838, 0x0002021c, 0x00020200,
461 0xc78, 0x00000080, 0x00000000,
462 0xd030, 0x000300c0, 0x00800040,
463 0xd030, 0x000300c0, 0x00800040,
464 0xd830, 0x000300c0, 0x00800040,
465 0xd830, 0x000300c0, 0x00800040,
466 0x5bb0, 0x000000f0, 0x00000070,
467 0x5bc0, 0x00200000, 0x50100000,
468 0x7030, 0x31000311, 0x00000011,
469 0x2ae4, 0x00073ffe, 0x000022a2,
470 0x2ae4, 0x00073ffe, 0x000022a2,
471 0x2ae4, 0x00073ffe, 0x000022a2,
472 0x240c, 0x000007ff, 0x00000000,
473 0x240c, 0x000007ff, 0x00000000,
474 0x240c, 0x000007ff, 0x00000000,
475 0x8a14, 0xf000001f, 0x00000007,
476 0x8a14, 0xf000001f, 0x00000007,
477 0x8a14, 0xf000001f, 0x00000007,
478 0x8b24, 0xffffffff, 0x00ffffff,
479 0x8b10, 0x0000ff0f, 0x00000000,
480 0x28a4c, 0x07ffffff, 0x4e000000,
481 0x28350, 0x3f3f3fff, 0x0000124a,
482 0x28350, 0x3f3f3fff, 0x0000124a,
483 0x28350, 0x3f3f3fff, 0x0000124a,
484 0x30, 0x000000ff, 0x0040,
485 0x34, 0x00000040, 0x00004040,
486 0x9100, 0x07ffffff, 0x03000000,
487 0x9100, 0x07ffffff, 0x03000000,
488 0x8e88, 0x01ff1f3f, 0x00000000,
489 0x8e88, 0x01ff1f3f, 0x00000000,
490 0x8e88, 0x01ff1f3f, 0x00000000,
491 0x8e84, 0x01ff1f3f, 0x00000000,
492 0x8e84, 0x01ff1f3f, 0x00000000,
493 0x8e84, 0x01ff1f3f, 0x00000000,
494 0x9060, 0x0000007f, 0x00000020,
495 0x9508, 0x00010000, 0x00010000,
496 0xac14, 0x000003ff, 0x00000003,
497 0xac14, 0x000003ff, 0x00000003,
498 0xac14, 0x000003ff, 0x00000003,
499 0xac10, 0xffffffff, 0x00000000,
500 0xac10, 0xffffffff, 0x00000000,
501 0xac10, 0xffffffff, 0x00000000,
502 0xac0c, 0xffffffff, 0x00001032,
503 0xac0c, 0xffffffff, 0x00001032,
504 0xac0c, 0xffffffff, 0x00001032,
505 0x88d4, 0x0000001f, 0x00000010,
506 0x88d4, 0x0000001f, 0x00000010,
507 0x88d4, 0x0000001f, 0x00000010,
508 0x15c0, 0x000c0fc0, 0x000c0400
509};
510
511static const u32 oland_golden_rlc_registers[] =
512{
513 0xc424, 0xffffffff, 0x00601005,
514 0xc47c, 0xffffffff, 0x10104040,
515 0xc488, 0xffffffff, 0x0100000a,
516 0xc314, 0xffffffff, 0x00000800,
517 0xc30c, 0xffffffff, 0x800000f4
518};
519
520static const u32 oland_golden_registers[] =
521{
522 0x9a10, 0x00010000, 0x00018208,
523 0x9830, 0xffffffff, 0x00000000,
524 0x9834, 0xf00fffff, 0x00000400,
525 0x9838, 0x0002021c, 0x00020200,
526 0xc78, 0x00000080, 0x00000000,
527 0xd030, 0x000300c0, 0x00800040,
528 0xd830, 0x000300c0, 0x00800040,
529 0x5bb0, 0x000000f0, 0x00000070,
530 0x5bc0, 0x00200000, 0x50100000,
531 0x7030, 0x31000311, 0x00000011,
532 0x2ae4, 0x00073ffe, 0x000022a2,
533 0x240c, 0x000007ff, 0x00000000,
534 0x8a14, 0xf000001f, 0x00000007,
535 0x8b24, 0xffffffff, 0x00ffffff,
536 0x8b10, 0x0000ff0f, 0x00000000,
537 0x28a4c, 0x07ffffff, 0x4e000000,
538 0x28350, 0x3f3f3fff, 0x00000082,
539 0x30, 0x000000ff, 0x0040,
540 0x34, 0x00000040, 0x00004040,
541 0x9100, 0x07ffffff, 0x03000000,
542 0x9060, 0x0000007f, 0x00000020,
543 0x9508, 0x00010000, 0x00010000,
544 0xac14, 0x000003ff, 0x000000f3,
545 0xac10, 0xffffffff, 0x00000000,
546 0xac0c, 0xffffffff, 0x00003210,
547 0x88d4, 0x0000001f, 0x00000010,
548 0x15c0, 0x000c0fc0, 0x000c0400
549};
550
Alex Deucherfffbdda2013-05-13 13:36:23 -0400551static const u32 hainan_golden_registers[] =
552{
553 0x9a10, 0x00010000, 0x00018208,
554 0x9830, 0xffffffff, 0x00000000,
555 0x9834, 0xf00fffff, 0x00000400,
556 0x9838, 0x0002021c, 0x00020200,
557 0xd0c0, 0xff000fff, 0x00000100,
558 0xd030, 0x000300c0, 0x00800040,
559 0xd8c0, 0xff000fff, 0x00000100,
560 0xd830, 0x000300c0, 0x00800040,
561 0x2ae4, 0x00073ffe, 0x000022a2,
562 0x240c, 0x000007ff, 0x00000000,
563 0x8a14, 0xf000001f, 0x00000007,
564 0x8b24, 0xffffffff, 0x00ffffff,
565 0x8b10, 0x0000ff0f, 0x00000000,
566 0x28a4c, 0x07ffffff, 0x4e000000,
567 0x28350, 0x3f3f3fff, 0x00000000,
568 0x30, 0x000000ff, 0x0040,
569 0x34, 0x00000040, 0x00004040,
570 0x9100, 0x03e00000, 0x03600000,
571 0x9060, 0x0000007f, 0x00000020,
572 0x9508, 0x00010000, 0x00010000,
573 0xac14, 0x000003ff, 0x000000f1,
574 0xac10, 0xffffffff, 0x00000000,
575 0xac0c, 0xffffffff, 0x00003210,
576 0x88d4, 0x0000001f, 0x00000010,
577 0x15c0, 0x000c0fc0, 0x000c0400
578};
579
580static const u32 hainan_golden_registers2[] =
581{
582 0x98f8, 0xffffffff, 0x02010001
583};
584
Alex Deucher205996c2013-03-01 17:08:42 -0500585static const u32 tahiti_mgcg_cgcg_init[] =
586{
587 0xc400, 0xffffffff, 0xfffffffc,
588 0x802c, 0xffffffff, 0xe0000000,
589 0x9a60, 0xffffffff, 0x00000100,
590 0x92a4, 0xffffffff, 0x00000100,
591 0xc164, 0xffffffff, 0x00000100,
592 0x9774, 0xffffffff, 0x00000100,
593 0x8984, 0xffffffff, 0x06000100,
594 0x8a18, 0xffffffff, 0x00000100,
595 0x92a0, 0xffffffff, 0x00000100,
596 0xc380, 0xffffffff, 0x00000100,
597 0x8b28, 0xffffffff, 0x00000100,
598 0x9144, 0xffffffff, 0x00000100,
599 0x8d88, 0xffffffff, 0x00000100,
600 0x8d8c, 0xffffffff, 0x00000100,
601 0x9030, 0xffffffff, 0x00000100,
602 0x9034, 0xffffffff, 0x00000100,
603 0x9038, 0xffffffff, 0x00000100,
604 0x903c, 0xffffffff, 0x00000100,
605 0xad80, 0xffffffff, 0x00000100,
606 0xac54, 0xffffffff, 0x00000100,
607 0x897c, 0xffffffff, 0x06000100,
608 0x9868, 0xffffffff, 0x00000100,
609 0x9510, 0xffffffff, 0x00000100,
610 0xaf04, 0xffffffff, 0x00000100,
611 0xae04, 0xffffffff, 0x00000100,
612 0x949c, 0xffffffff, 0x00000100,
613 0x802c, 0xffffffff, 0xe0000000,
614 0x9160, 0xffffffff, 0x00010000,
615 0x9164, 0xffffffff, 0x00030002,
616 0x9168, 0xffffffff, 0x00040007,
617 0x916c, 0xffffffff, 0x00060005,
618 0x9170, 0xffffffff, 0x00090008,
619 0x9174, 0xffffffff, 0x00020001,
620 0x9178, 0xffffffff, 0x00040003,
621 0x917c, 0xffffffff, 0x00000007,
622 0x9180, 0xffffffff, 0x00060005,
623 0x9184, 0xffffffff, 0x00090008,
624 0x9188, 0xffffffff, 0x00030002,
625 0x918c, 0xffffffff, 0x00050004,
626 0x9190, 0xffffffff, 0x00000008,
627 0x9194, 0xffffffff, 0x00070006,
628 0x9198, 0xffffffff, 0x000a0009,
629 0x919c, 0xffffffff, 0x00040003,
630 0x91a0, 0xffffffff, 0x00060005,
631 0x91a4, 0xffffffff, 0x00000009,
632 0x91a8, 0xffffffff, 0x00080007,
633 0x91ac, 0xffffffff, 0x000b000a,
634 0x91b0, 0xffffffff, 0x00050004,
635 0x91b4, 0xffffffff, 0x00070006,
636 0x91b8, 0xffffffff, 0x0008000b,
637 0x91bc, 0xffffffff, 0x000a0009,
638 0x91c0, 0xffffffff, 0x000d000c,
639 0x91c4, 0xffffffff, 0x00060005,
640 0x91c8, 0xffffffff, 0x00080007,
641 0x91cc, 0xffffffff, 0x0000000b,
642 0x91d0, 0xffffffff, 0x000a0009,
643 0x91d4, 0xffffffff, 0x000d000c,
644 0x91d8, 0xffffffff, 0x00070006,
645 0x91dc, 0xffffffff, 0x00090008,
646 0x91e0, 0xffffffff, 0x0000000c,
647 0x91e4, 0xffffffff, 0x000b000a,
648 0x91e8, 0xffffffff, 0x000e000d,
649 0x91ec, 0xffffffff, 0x00080007,
650 0x91f0, 0xffffffff, 0x000a0009,
651 0x91f4, 0xffffffff, 0x0000000d,
652 0x91f8, 0xffffffff, 0x000c000b,
653 0x91fc, 0xffffffff, 0x000f000e,
654 0x9200, 0xffffffff, 0x00090008,
655 0x9204, 0xffffffff, 0x000b000a,
656 0x9208, 0xffffffff, 0x000c000f,
657 0x920c, 0xffffffff, 0x000e000d,
658 0x9210, 0xffffffff, 0x00110010,
659 0x9214, 0xffffffff, 0x000a0009,
660 0x9218, 0xffffffff, 0x000c000b,
661 0x921c, 0xffffffff, 0x0000000f,
662 0x9220, 0xffffffff, 0x000e000d,
663 0x9224, 0xffffffff, 0x00110010,
664 0x9228, 0xffffffff, 0x000b000a,
665 0x922c, 0xffffffff, 0x000d000c,
666 0x9230, 0xffffffff, 0x00000010,
667 0x9234, 0xffffffff, 0x000f000e,
668 0x9238, 0xffffffff, 0x00120011,
669 0x923c, 0xffffffff, 0x000c000b,
670 0x9240, 0xffffffff, 0x000e000d,
671 0x9244, 0xffffffff, 0x00000011,
672 0x9248, 0xffffffff, 0x0010000f,
673 0x924c, 0xffffffff, 0x00130012,
674 0x9250, 0xffffffff, 0x000d000c,
675 0x9254, 0xffffffff, 0x000f000e,
676 0x9258, 0xffffffff, 0x00100013,
677 0x925c, 0xffffffff, 0x00120011,
678 0x9260, 0xffffffff, 0x00150014,
679 0x9264, 0xffffffff, 0x000e000d,
680 0x9268, 0xffffffff, 0x0010000f,
681 0x926c, 0xffffffff, 0x00000013,
682 0x9270, 0xffffffff, 0x00120011,
683 0x9274, 0xffffffff, 0x00150014,
684 0x9278, 0xffffffff, 0x000f000e,
685 0x927c, 0xffffffff, 0x00110010,
686 0x9280, 0xffffffff, 0x00000014,
687 0x9284, 0xffffffff, 0x00130012,
688 0x9288, 0xffffffff, 0x00160015,
689 0x928c, 0xffffffff, 0x0010000f,
690 0x9290, 0xffffffff, 0x00120011,
691 0x9294, 0xffffffff, 0x00000015,
692 0x9298, 0xffffffff, 0x00140013,
693 0x929c, 0xffffffff, 0x00170016,
694 0x9150, 0xffffffff, 0x96940200,
695 0x8708, 0xffffffff, 0x00900100,
696 0xc478, 0xffffffff, 0x00000080,
697 0xc404, 0xffffffff, 0x0020003f,
698 0x30, 0xffffffff, 0x0000001c,
699 0x34, 0x000f0000, 0x000f0000,
700 0x160c, 0xffffffff, 0x00000100,
701 0x1024, 0xffffffff, 0x00000100,
702 0x102c, 0x00000101, 0x00000000,
703 0x20a8, 0xffffffff, 0x00000104,
704 0x264c, 0x000c0000, 0x000c0000,
705 0x2648, 0x000c0000, 0x000c0000,
706 0x55e4, 0xff000fff, 0x00000100,
707 0x55e8, 0x00000001, 0x00000001,
708 0x2f50, 0x00000001, 0x00000001,
709 0x30cc, 0xc0000fff, 0x00000104,
710 0xc1e4, 0x00000001, 0x00000001,
711 0xd0c0, 0xfffffff0, 0x00000100,
712 0xd8c0, 0xfffffff0, 0x00000100
713};
714
715static const u32 pitcairn_mgcg_cgcg_init[] =
716{
717 0xc400, 0xffffffff, 0xfffffffc,
718 0x802c, 0xffffffff, 0xe0000000,
719 0x9a60, 0xffffffff, 0x00000100,
720 0x92a4, 0xffffffff, 0x00000100,
721 0xc164, 0xffffffff, 0x00000100,
722 0x9774, 0xffffffff, 0x00000100,
723 0x8984, 0xffffffff, 0x06000100,
724 0x8a18, 0xffffffff, 0x00000100,
725 0x92a0, 0xffffffff, 0x00000100,
726 0xc380, 0xffffffff, 0x00000100,
727 0x8b28, 0xffffffff, 0x00000100,
728 0x9144, 0xffffffff, 0x00000100,
729 0x8d88, 0xffffffff, 0x00000100,
730 0x8d8c, 0xffffffff, 0x00000100,
731 0x9030, 0xffffffff, 0x00000100,
732 0x9034, 0xffffffff, 0x00000100,
733 0x9038, 0xffffffff, 0x00000100,
734 0x903c, 0xffffffff, 0x00000100,
735 0xad80, 0xffffffff, 0x00000100,
736 0xac54, 0xffffffff, 0x00000100,
737 0x897c, 0xffffffff, 0x06000100,
738 0x9868, 0xffffffff, 0x00000100,
739 0x9510, 0xffffffff, 0x00000100,
740 0xaf04, 0xffffffff, 0x00000100,
741 0xae04, 0xffffffff, 0x00000100,
742 0x949c, 0xffffffff, 0x00000100,
743 0x802c, 0xffffffff, 0xe0000000,
744 0x9160, 0xffffffff, 0x00010000,
745 0x9164, 0xffffffff, 0x00030002,
746 0x9168, 0xffffffff, 0x00040007,
747 0x916c, 0xffffffff, 0x00060005,
748 0x9170, 0xffffffff, 0x00090008,
749 0x9174, 0xffffffff, 0x00020001,
750 0x9178, 0xffffffff, 0x00040003,
751 0x917c, 0xffffffff, 0x00000007,
752 0x9180, 0xffffffff, 0x00060005,
753 0x9184, 0xffffffff, 0x00090008,
754 0x9188, 0xffffffff, 0x00030002,
755 0x918c, 0xffffffff, 0x00050004,
756 0x9190, 0xffffffff, 0x00000008,
757 0x9194, 0xffffffff, 0x00070006,
758 0x9198, 0xffffffff, 0x000a0009,
759 0x919c, 0xffffffff, 0x00040003,
760 0x91a0, 0xffffffff, 0x00060005,
761 0x91a4, 0xffffffff, 0x00000009,
762 0x91a8, 0xffffffff, 0x00080007,
763 0x91ac, 0xffffffff, 0x000b000a,
764 0x91b0, 0xffffffff, 0x00050004,
765 0x91b4, 0xffffffff, 0x00070006,
766 0x91b8, 0xffffffff, 0x0008000b,
767 0x91bc, 0xffffffff, 0x000a0009,
768 0x91c0, 0xffffffff, 0x000d000c,
769 0x9200, 0xffffffff, 0x00090008,
770 0x9204, 0xffffffff, 0x000b000a,
771 0x9208, 0xffffffff, 0x000c000f,
772 0x920c, 0xffffffff, 0x000e000d,
773 0x9210, 0xffffffff, 0x00110010,
774 0x9214, 0xffffffff, 0x000a0009,
775 0x9218, 0xffffffff, 0x000c000b,
776 0x921c, 0xffffffff, 0x0000000f,
777 0x9220, 0xffffffff, 0x000e000d,
778 0x9224, 0xffffffff, 0x00110010,
779 0x9228, 0xffffffff, 0x000b000a,
780 0x922c, 0xffffffff, 0x000d000c,
781 0x9230, 0xffffffff, 0x00000010,
782 0x9234, 0xffffffff, 0x000f000e,
783 0x9238, 0xffffffff, 0x00120011,
784 0x923c, 0xffffffff, 0x000c000b,
785 0x9240, 0xffffffff, 0x000e000d,
786 0x9244, 0xffffffff, 0x00000011,
787 0x9248, 0xffffffff, 0x0010000f,
788 0x924c, 0xffffffff, 0x00130012,
789 0x9250, 0xffffffff, 0x000d000c,
790 0x9254, 0xffffffff, 0x000f000e,
791 0x9258, 0xffffffff, 0x00100013,
792 0x925c, 0xffffffff, 0x00120011,
793 0x9260, 0xffffffff, 0x00150014,
794 0x9150, 0xffffffff, 0x96940200,
795 0x8708, 0xffffffff, 0x00900100,
796 0xc478, 0xffffffff, 0x00000080,
797 0xc404, 0xffffffff, 0x0020003f,
798 0x30, 0xffffffff, 0x0000001c,
799 0x34, 0x000f0000, 0x000f0000,
800 0x160c, 0xffffffff, 0x00000100,
801 0x1024, 0xffffffff, 0x00000100,
802 0x102c, 0x00000101, 0x00000000,
803 0x20a8, 0xffffffff, 0x00000104,
804 0x55e4, 0xff000fff, 0x00000100,
805 0x55e8, 0x00000001, 0x00000001,
806 0x2f50, 0x00000001, 0x00000001,
807 0x30cc, 0xc0000fff, 0x00000104,
808 0xc1e4, 0x00000001, 0x00000001,
809 0xd0c0, 0xfffffff0, 0x00000100,
810 0xd8c0, 0xfffffff0, 0x00000100
811};
812
813static const u32 verde_mgcg_cgcg_init[] =
814{
815 0xc400, 0xffffffff, 0xfffffffc,
816 0x802c, 0xffffffff, 0xe0000000,
817 0x9a60, 0xffffffff, 0x00000100,
818 0x92a4, 0xffffffff, 0x00000100,
819 0xc164, 0xffffffff, 0x00000100,
820 0x9774, 0xffffffff, 0x00000100,
821 0x8984, 0xffffffff, 0x06000100,
822 0x8a18, 0xffffffff, 0x00000100,
823 0x92a0, 0xffffffff, 0x00000100,
824 0xc380, 0xffffffff, 0x00000100,
825 0x8b28, 0xffffffff, 0x00000100,
826 0x9144, 0xffffffff, 0x00000100,
827 0x8d88, 0xffffffff, 0x00000100,
828 0x8d8c, 0xffffffff, 0x00000100,
829 0x9030, 0xffffffff, 0x00000100,
830 0x9034, 0xffffffff, 0x00000100,
831 0x9038, 0xffffffff, 0x00000100,
832 0x903c, 0xffffffff, 0x00000100,
833 0xad80, 0xffffffff, 0x00000100,
834 0xac54, 0xffffffff, 0x00000100,
835 0x897c, 0xffffffff, 0x06000100,
836 0x9868, 0xffffffff, 0x00000100,
837 0x9510, 0xffffffff, 0x00000100,
838 0xaf04, 0xffffffff, 0x00000100,
839 0xae04, 0xffffffff, 0x00000100,
840 0x949c, 0xffffffff, 0x00000100,
841 0x802c, 0xffffffff, 0xe0000000,
842 0x9160, 0xffffffff, 0x00010000,
843 0x9164, 0xffffffff, 0x00030002,
844 0x9168, 0xffffffff, 0x00040007,
845 0x916c, 0xffffffff, 0x00060005,
846 0x9170, 0xffffffff, 0x00090008,
847 0x9174, 0xffffffff, 0x00020001,
848 0x9178, 0xffffffff, 0x00040003,
849 0x917c, 0xffffffff, 0x00000007,
850 0x9180, 0xffffffff, 0x00060005,
851 0x9184, 0xffffffff, 0x00090008,
852 0x9188, 0xffffffff, 0x00030002,
853 0x918c, 0xffffffff, 0x00050004,
854 0x9190, 0xffffffff, 0x00000008,
855 0x9194, 0xffffffff, 0x00070006,
856 0x9198, 0xffffffff, 0x000a0009,
857 0x919c, 0xffffffff, 0x00040003,
858 0x91a0, 0xffffffff, 0x00060005,
859 0x91a4, 0xffffffff, 0x00000009,
860 0x91a8, 0xffffffff, 0x00080007,
861 0x91ac, 0xffffffff, 0x000b000a,
862 0x91b0, 0xffffffff, 0x00050004,
863 0x91b4, 0xffffffff, 0x00070006,
864 0x91b8, 0xffffffff, 0x0008000b,
865 0x91bc, 0xffffffff, 0x000a0009,
866 0x91c0, 0xffffffff, 0x000d000c,
867 0x9200, 0xffffffff, 0x00090008,
868 0x9204, 0xffffffff, 0x000b000a,
869 0x9208, 0xffffffff, 0x000c000f,
870 0x920c, 0xffffffff, 0x000e000d,
871 0x9210, 0xffffffff, 0x00110010,
872 0x9214, 0xffffffff, 0x000a0009,
873 0x9218, 0xffffffff, 0x000c000b,
874 0x921c, 0xffffffff, 0x0000000f,
875 0x9220, 0xffffffff, 0x000e000d,
876 0x9224, 0xffffffff, 0x00110010,
877 0x9228, 0xffffffff, 0x000b000a,
878 0x922c, 0xffffffff, 0x000d000c,
879 0x9230, 0xffffffff, 0x00000010,
880 0x9234, 0xffffffff, 0x000f000e,
881 0x9238, 0xffffffff, 0x00120011,
882 0x923c, 0xffffffff, 0x000c000b,
883 0x9240, 0xffffffff, 0x000e000d,
884 0x9244, 0xffffffff, 0x00000011,
885 0x9248, 0xffffffff, 0x0010000f,
886 0x924c, 0xffffffff, 0x00130012,
887 0x9250, 0xffffffff, 0x000d000c,
888 0x9254, 0xffffffff, 0x000f000e,
889 0x9258, 0xffffffff, 0x00100013,
890 0x925c, 0xffffffff, 0x00120011,
891 0x9260, 0xffffffff, 0x00150014,
892 0x9150, 0xffffffff, 0x96940200,
893 0x8708, 0xffffffff, 0x00900100,
894 0xc478, 0xffffffff, 0x00000080,
895 0xc404, 0xffffffff, 0x0020003f,
896 0x30, 0xffffffff, 0x0000001c,
897 0x34, 0x000f0000, 0x000f0000,
898 0x160c, 0xffffffff, 0x00000100,
899 0x1024, 0xffffffff, 0x00000100,
900 0x102c, 0x00000101, 0x00000000,
901 0x20a8, 0xffffffff, 0x00000104,
902 0x264c, 0x000c0000, 0x000c0000,
903 0x2648, 0x000c0000, 0x000c0000,
904 0x55e4, 0xff000fff, 0x00000100,
905 0x55e8, 0x00000001, 0x00000001,
906 0x2f50, 0x00000001, 0x00000001,
907 0x30cc, 0xc0000fff, 0x00000104,
908 0xc1e4, 0x00000001, 0x00000001,
909 0xd0c0, 0xfffffff0, 0x00000100,
910 0xd8c0, 0xfffffff0, 0x00000100
911};
912
913static const u32 oland_mgcg_cgcg_init[] =
914{
915 0xc400, 0xffffffff, 0xfffffffc,
916 0x802c, 0xffffffff, 0xe0000000,
917 0x9a60, 0xffffffff, 0x00000100,
918 0x92a4, 0xffffffff, 0x00000100,
919 0xc164, 0xffffffff, 0x00000100,
920 0x9774, 0xffffffff, 0x00000100,
921 0x8984, 0xffffffff, 0x06000100,
922 0x8a18, 0xffffffff, 0x00000100,
923 0x92a0, 0xffffffff, 0x00000100,
924 0xc380, 0xffffffff, 0x00000100,
925 0x8b28, 0xffffffff, 0x00000100,
926 0x9144, 0xffffffff, 0x00000100,
927 0x8d88, 0xffffffff, 0x00000100,
928 0x8d8c, 0xffffffff, 0x00000100,
929 0x9030, 0xffffffff, 0x00000100,
930 0x9034, 0xffffffff, 0x00000100,
931 0x9038, 0xffffffff, 0x00000100,
932 0x903c, 0xffffffff, 0x00000100,
933 0xad80, 0xffffffff, 0x00000100,
934 0xac54, 0xffffffff, 0x00000100,
935 0x897c, 0xffffffff, 0x06000100,
936 0x9868, 0xffffffff, 0x00000100,
937 0x9510, 0xffffffff, 0x00000100,
938 0xaf04, 0xffffffff, 0x00000100,
939 0xae04, 0xffffffff, 0x00000100,
940 0x949c, 0xffffffff, 0x00000100,
941 0x802c, 0xffffffff, 0xe0000000,
942 0x9160, 0xffffffff, 0x00010000,
943 0x9164, 0xffffffff, 0x00030002,
944 0x9168, 0xffffffff, 0x00040007,
945 0x916c, 0xffffffff, 0x00060005,
946 0x9170, 0xffffffff, 0x00090008,
947 0x9174, 0xffffffff, 0x00020001,
948 0x9178, 0xffffffff, 0x00040003,
949 0x917c, 0xffffffff, 0x00000007,
950 0x9180, 0xffffffff, 0x00060005,
951 0x9184, 0xffffffff, 0x00090008,
952 0x9188, 0xffffffff, 0x00030002,
953 0x918c, 0xffffffff, 0x00050004,
954 0x9190, 0xffffffff, 0x00000008,
955 0x9194, 0xffffffff, 0x00070006,
956 0x9198, 0xffffffff, 0x000a0009,
957 0x919c, 0xffffffff, 0x00040003,
958 0x91a0, 0xffffffff, 0x00060005,
959 0x91a4, 0xffffffff, 0x00000009,
960 0x91a8, 0xffffffff, 0x00080007,
961 0x91ac, 0xffffffff, 0x000b000a,
962 0x91b0, 0xffffffff, 0x00050004,
963 0x91b4, 0xffffffff, 0x00070006,
964 0x91b8, 0xffffffff, 0x0008000b,
965 0x91bc, 0xffffffff, 0x000a0009,
966 0x91c0, 0xffffffff, 0x000d000c,
967 0x91c4, 0xffffffff, 0x00060005,
968 0x91c8, 0xffffffff, 0x00080007,
969 0x91cc, 0xffffffff, 0x0000000b,
970 0x91d0, 0xffffffff, 0x000a0009,
971 0x91d4, 0xffffffff, 0x000d000c,
972 0x9150, 0xffffffff, 0x96940200,
973 0x8708, 0xffffffff, 0x00900100,
974 0xc478, 0xffffffff, 0x00000080,
975 0xc404, 0xffffffff, 0x0020003f,
976 0x30, 0xffffffff, 0x0000001c,
977 0x34, 0x000f0000, 0x000f0000,
978 0x160c, 0xffffffff, 0x00000100,
979 0x1024, 0xffffffff, 0x00000100,
980 0x102c, 0x00000101, 0x00000000,
981 0x20a8, 0xffffffff, 0x00000104,
982 0x264c, 0x000c0000, 0x000c0000,
983 0x2648, 0x000c0000, 0x000c0000,
984 0x55e4, 0xff000fff, 0x00000100,
985 0x55e8, 0x00000001, 0x00000001,
986 0x2f50, 0x00000001, 0x00000001,
987 0x30cc, 0xc0000fff, 0x00000104,
988 0xc1e4, 0x00000001, 0x00000001,
989 0xd0c0, 0xfffffff0, 0x00000100,
990 0xd8c0, 0xfffffff0, 0x00000100
991};
992
Alex Deucherfffbdda2013-05-13 13:36:23 -0400993static const u32 hainan_mgcg_cgcg_init[] =
994{
995 0xc400, 0xffffffff, 0xfffffffc,
996 0x802c, 0xffffffff, 0xe0000000,
997 0x9a60, 0xffffffff, 0x00000100,
998 0x92a4, 0xffffffff, 0x00000100,
999 0xc164, 0xffffffff, 0x00000100,
1000 0x9774, 0xffffffff, 0x00000100,
1001 0x8984, 0xffffffff, 0x06000100,
1002 0x8a18, 0xffffffff, 0x00000100,
1003 0x92a0, 0xffffffff, 0x00000100,
1004 0xc380, 0xffffffff, 0x00000100,
1005 0x8b28, 0xffffffff, 0x00000100,
1006 0x9144, 0xffffffff, 0x00000100,
1007 0x8d88, 0xffffffff, 0x00000100,
1008 0x8d8c, 0xffffffff, 0x00000100,
1009 0x9030, 0xffffffff, 0x00000100,
1010 0x9034, 0xffffffff, 0x00000100,
1011 0x9038, 0xffffffff, 0x00000100,
1012 0x903c, 0xffffffff, 0x00000100,
1013 0xad80, 0xffffffff, 0x00000100,
1014 0xac54, 0xffffffff, 0x00000100,
1015 0x897c, 0xffffffff, 0x06000100,
1016 0x9868, 0xffffffff, 0x00000100,
1017 0x9510, 0xffffffff, 0x00000100,
1018 0xaf04, 0xffffffff, 0x00000100,
1019 0xae04, 0xffffffff, 0x00000100,
1020 0x949c, 0xffffffff, 0x00000100,
1021 0x802c, 0xffffffff, 0xe0000000,
1022 0x9160, 0xffffffff, 0x00010000,
1023 0x9164, 0xffffffff, 0x00030002,
1024 0x9168, 0xffffffff, 0x00040007,
1025 0x916c, 0xffffffff, 0x00060005,
1026 0x9170, 0xffffffff, 0x00090008,
1027 0x9174, 0xffffffff, 0x00020001,
1028 0x9178, 0xffffffff, 0x00040003,
1029 0x917c, 0xffffffff, 0x00000007,
1030 0x9180, 0xffffffff, 0x00060005,
1031 0x9184, 0xffffffff, 0x00090008,
1032 0x9188, 0xffffffff, 0x00030002,
1033 0x918c, 0xffffffff, 0x00050004,
1034 0x9190, 0xffffffff, 0x00000008,
1035 0x9194, 0xffffffff, 0x00070006,
1036 0x9198, 0xffffffff, 0x000a0009,
1037 0x919c, 0xffffffff, 0x00040003,
1038 0x91a0, 0xffffffff, 0x00060005,
1039 0x91a4, 0xffffffff, 0x00000009,
1040 0x91a8, 0xffffffff, 0x00080007,
1041 0x91ac, 0xffffffff, 0x000b000a,
1042 0x91b0, 0xffffffff, 0x00050004,
1043 0x91b4, 0xffffffff, 0x00070006,
1044 0x91b8, 0xffffffff, 0x0008000b,
1045 0x91bc, 0xffffffff, 0x000a0009,
1046 0x91c0, 0xffffffff, 0x000d000c,
1047 0x91c4, 0xffffffff, 0x00060005,
1048 0x91c8, 0xffffffff, 0x00080007,
1049 0x91cc, 0xffffffff, 0x0000000b,
1050 0x91d0, 0xffffffff, 0x000a0009,
1051 0x91d4, 0xffffffff, 0x000d000c,
1052 0x9150, 0xffffffff, 0x96940200,
1053 0x8708, 0xffffffff, 0x00900100,
1054 0xc478, 0xffffffff, 0x00000080,
1055 0xc404, 0xffffffff, 0x0020003f,
1056 0x30, 0xffffffff, 0x0000001c,
1057 0x34, 0x000f0000, 0x000f0000,
1058 0x160c, 0xffffffff, 0x00000100,
1059 0x1024, 0xffffffff, 0x00000100,
1060 0x20a8, 0xffffffff, 0x00000104,
1061 0x264c, 0x000c0000, 0x000c0000,
1062 0x2648, 0x000c0000, 0x000c0000,
1063 0x2f50, 0x00000001, 0x00000001,
1064 0x30cc, 0xc0000fff, 0x00000104,
1065 0xc1e4, 0x00000001, 0x00000001,
1066 0xd0c0, 0xfffffff0, 0x00000100,
1067 0xd8c0, 0xfffffff0, 0x00000100
1068};
1069
Alex Deucher205996c2013-03-01 17:08:42 -05001070static u32 verde_pg_init[] =
1071{
1072 0x353c, 0xffffffff, 0x40000,
1073 0x3538, 0xffffffff, 0x200010ff,
1074 0x353c, 0xffffffff, 0x0,
1075 0x353c, 0xffffffff, 0x0,
1076 0x353c, 0xffffffff, 0x0,
1077 0x353c, 0xffffffff, 0x0,
1078 0x353c, 0xffffffff, 0x0,
1079 0x353c, 0xffffffff, 0x7007,
1080 0x3538, 0xffffffff, 0x300010ff,
1081 0x353c, 0xffffffff, 0x0,
1082 0x353c, 0xffffffff, 0x0,
1083 0x353c, 0xffffffff, 0x0,
1084 0x353c, 0xffffffff, 0x0,
1085 0x353c, 0xffffffff, 0x0,
1086 0x353c, 0xffffffff, 0x400000,
1087 0x3538, 0xffffffff, 0x100010ff,
1088 0x353c, 0xffffffff, 0x0,
1089 0x353c, 0xffffffff, 0x0,
1090 0x353c, 0xffffffff, 0x0,
1091 0x353c, 0xffffffff, 0x0,
1092 0x353c, 0xffffffff, 0x0,
1093 0x353c, 0xffffffff, 0x120200,
1094 0x3538, 0xffffffff, 0x500010ff,
1095 0x353c, 0xffffffff, 0x0,
1096 0x353c, 0xffffffff, 0x0,
1097 0x353c, 0xffffffff, 0x0,
1098 0x353c, 0xffffffff, 0x0,
1099 0x353c, 0xffffffff, 0x0,
1100 0x353c, 0xffffffff, 0x1e1e16,
1101 0x3538, 0xffffffff, 0x600010ff,
1102 0x353c, 0xffffffff, 0x0,
1103 0x353c, 0xffffffff, 0x0,
1104 0x353c, 0xffffffff, 0x0,
1105 0x353c, 0xffffffff, 0x0,
1106 0x353c, 0xffffffff, 0x0,
1107 0x353c, 0xffffffff, 0x171f1e,
1108 0x3538, 0xffffffff, 0x700010ff,
1109 0x353c, 0xffffffff, 0x0,
1110 0x353c, 0xffffffff, 0x0,
1111 0x353c, 0xffffffff, 0x0,
1112 0x353c, 0xffffffff, 0x0,
1113 0x353c, 0xffffffff, 0x0,
1114 0x353c, 0xffffffff, 0x0,
1115 0x3538, 0xffffffff, 0x9ff,
1116 0x3500, 0xffffffff, 0x0,
1117 0x3504, 0xffffffff, 0x10000800,
1118 0x3504, 0xffffffff, 0xf,
1119 0x3504, 0xffffffff, 0xf,
1120 0x3500, 0xffffffff, 0x4,
1121 0x3504, 0xffffffff, 0x1000051e,
1122 0x3504, 0xffffffff, 0xffff,
1123 0x3504, 0xffffffff, 0xffff,
1124 0x3500, 0xffffffff, 0x8,
1125 0x3504, 0xffffffff, 0x80500,
1126 0x3500, 0xffffffff, 0x12,
1127 0x3504, 0xffffffff, 0x9050c,
1128 0x3500, 0xffffffff, 0x1d,
1129 0x3504, 0xffffffff, 0xb052c,
1130 0x3500, 0xffffffff, 0x2a,
1131 0x3504, 0xffffffff, 0x1053e,
1132 0x3500, 0xffffffff, 0x2d,
1133 0x3504, 0xffffffff, 0x10546,
1134 0x3500, 0xffffffff, 0x30,
1135 0x3504, 0xffffffff, 0xa054e,
1136 0x3500, 0xffffffff, 0x3c,
1137 0x3504, 0xffffffff, 0x1055f,
1138 0x3500, 0xffffffff, 0x3f,
1139 0x3504, 0xffffffff, 0x10567,
1140 0x3500, 0xffffffff, 0x42,
1141 0x3504, 0xffffffff, 0x1056f,
1142 0x3500, 0xffffffff, 0x45,
1143 0x3504, 0xffffffff, 0x10572,
1144 0x3500, 0xffffffff, 0x48,
1145 0x3504, 0xffffffff, 0x20575,
1146 0x3500, 0xffffffff, 0x4c,
1147 0x3504, 0xffffffff, 0x190801,
1148 0x3500, 0xffffffff, 0x67,
1149 0x3504, 0xffffffff, 0x1082a,
1150 0x3500, 0xffffffff, 0x6a,
1151 0x3504, 0xffffffff, 0x1b082d,
1152 0x3500, 0xffffffff, 0x87,
1153 0x3504, 0xffffffff, 0x310851,
1154 0x3500, 0xffffffff, 0xba,
1155 0x3504, 0xffffffff, 0x891,
1156 0x3500, 0xffffffff, 0xbc,
1157 0x3504, 0xffffffff, 0x893,
1158 0x3500, 0xffffffff, 0xbe,
1159 0x3504, 0xffffffff, 0x20895,
1160 0x3500, 0xffffffff, 0xc2,
1161 0x3504, 0xffffffff, 0x20899,
1162 0x3500, 0xffffffff, 0xc6,
1163 0x3504, 0xffffffff, 0x2089d,
1164 0x3500, 0xffffffff, 0xca,
1165 0x3504, 0xffffffff, 0x8a1,
1166 0x3500, 0xffffffff, 0xcc,
1167 0x3504, 0xffffffff, 0x8a3,
1168 0x3500, 0xffffffff, 0xce,
1169 0x3504, 0xffffffff, 0x308a5,
1170 0x3500, 0xffffffff, 0xd3,
1171 0x3504, 0xffffffff, 0x6d08cd,
1172 0x3500, 0xffffffff, 0x142,
1173 0x3504, 0xffffffff, 0x2000095a,
1174 0x3504, 0xffffffff, 0x1,
1175 0x3500, 0xffffffff, 0x144,
1176 0x3504, 0xffffffff, 0x301f095b,
1177 0x3500, 0xffffffff, 0x165,
1178 0x3504, 0xffffffff, 0xc094d,
1179 0x3500, 0xffffffff, 0x173,
1180 0x3504, 0xffffffff, 0xf096d,
1181 0x3500, 0xffffffff, 0x184,
1182 0x3504, 0xffffffff, 0x15097f,
1183 0x3500, 0xffffffff, 0x19b,
1184 0x3504, 0xffffffff, 0xc0998,
1185 0x3500, 0xffffffff, 0x1a9,
1186 0x3504, 0xffffffff, 0x409a7,
1187 0x3500, 0xffffffff, 0x1af,
1188 0x3504, 0xffffffff, 0xcdc,
1189 0x3500, 0xffffffff, 0x1b1,
1190 0x3504, 0xffffffff, 0x800,
1191 0x3508, 0xffffffff, 0x6c9b2000,
1192 0x3510, 0xfc00, 0x2000,
1193 0x3544, 0xffffffff, 0xfc0,
1194 0x28d4, 0x00000100, 0x100
1195};
1196
1197static void si_init_golden_registers(struct radeon_device *rdev)
1198{
1199 switch (rdev->family) {
1200 case CHIP_TAHITI:
1201 radeon_program_register_sequence(rdev,
1202 tahiti_golden_registers,
1203 (const u32)ARRAY_SIZE(tahiti_golden_registers));
1204 radeon_program_register_sequence(rdev,
1205 tahiti_golden_rlc_registers,
1206 (const u32)ARRAY_SIZE(tahiti_golden_rlc_registers));
1207 radeon_program_register_sequence(rdev,
1208 tahiti_mgcg_cgcg_init,
1209 (const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init));
1210 radeon_program_register_sequence(rdev,
1211 tahiti_golden_registers2,
1212 (const u32)ARRAY_SIZE(tahiti_golden_registers2));
1213 break;
1214 case CHIP_PITCAIRN:
1215 radeon_program_register_sequence(rdev,
1216 pitcairn_golden_registers,
1217 (const u32)ARRAY_SIZE(pitcairn_golden_registers));
1218 radeon_program_register_sequence(rdev,
1219 pitcairn_golden_rlc_registers,
1220 (const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers));
1221 radeon_program_register_sequence(rdev,
1222 pitcairn_mgcg_cgcg_init,
1223 (const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
1224 break;
1225 case CHIP_VERDE:
1226 radeon_program_register_sequence(rdev,
1227 verde_golden_registers,
1228 (const u32)ARRAY_SIZE(verde_golden_registers));
1229 radeon_program_register_sequence(rdev,
1230 verde_golden_rlc_registers,
1231 (const u32)ARRAY_SIZE(verde_golden_rlc_registers));
1232 radeon_program_register_sequence(rdev,
1233 verde_mgcg_cgcg_init,
1234 (const u32)ARRAY_SIZE(verde_mgcg_cgcg_init));
1235 radeon_program_register_sequence(rdev,
1236 verde_pg_init,
1237 (const u32)ARRAY_SIZE(verde_pg_init));
1238 break;
1239 case CHIP_OLAND:
1240 radeon_program_register_sequence(rdev,
1241 oland_golden_registers,
1242 (const u32)ARRAY_SIZE(oland_golden_registers));
1243 radeon_program_register_sequence(rdev,
1244 oland_golden_rlc_registers,
1245 (const u32)ARRAY_SIZE(oland_golden_rlc_registers));
1246 radeon_program_register_sequence(rdev,
1247 oland_mgcg_cgcg_init,
1248 (const u32)ARRAY_SIZE(oland_mgcg_cgcg_init));
1249 break;
Alex Deucherfffbdda2013-05-13 13:36:23 -04001250 case CHIP_HAINAN:
1251 radeon_program_register_sequence(rdev,
1252 hainan_golden_registers,
1253 (const u32)ARRAY_SIZE(hainan_golden_registers));
1254 radeon_program_register_sequence(rdev,
1255 hainan_golden_registers2,
1256 (const u32)ARRAY_SIZE(hainan_golden_registers2));
1257 radeon_program_register_sequence(rdev,
1258 hainan_mgcg_cgcg_init,
1259 (const u32)ARRAY_SIZE(hainan_mgcg_cgcg_init));
1260 break;
Alex Deucher205996c2013-03-01 17:08:42 -05001261 default:
1262 break;
1263 }
1264}
1265
Alex Deucher454d2e22013-02-14 10:04:02 -05001266#define PCIE_BUS_CLK 10000
1267#define TCLK (PCIE_BUS_CLK / 10)
1268
1269/**
1270 * si_get_xclk - get the xclk
1271 *
1272 * @rdev: radeon_device pointer
1273 *
1274 * Returns the reference clock used by the gfx engine
1275 * (SI).
1276 */
1277u32 si_get_xclk(struct radeon_device *rdev)
1278{
1279 u32 reference_clock = rdev->clock.spll.reference_freq;
1280 u32 tmp;
1281
1282 tmp = RREG32(CG_CLKPIN_CNTL_2);
1283 if (tmp & MUX_TCLK_TO_XCLK)
1284 return TCLK;
1285
1286 tmp = RREG32(CG_CLKPIN_CNTL);
1287 if (tmp & XTALIN_DIVIDE)
1288 return reference_clock / 4;
1289
1290 return reference_clock;
1291}
1292
Alex Deucher1bd47d22012-03-20 17:18:10 -04001293/* get temperature in millidegrees */
1294int si_get_temp(struct radeon_device *rdev)
1295{
1296 u32 temp;
1297 int actual_temp = 0;
1298
1299 temp = (RREG32(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
1300 CTF_TEMP_SHIFT;
1301
1302 if (temp & 0x200)
1303 actual_temp = 255;
1304 else
1305 actual_temp = temp & 0x1ff;
1306
1307 actual_temp = (actual_temp * 1000);
1308
1309 return actual_temp;
1310}
1311
Alex Deucher8b074dd2012-03-20 17:18:18 -04001312#define TAHITI_IO_MC_REGS_SIZE 36
1313
1314static const u32 tahiti_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1315 {0x0000006f, 0x03044000},
1316 {0x00000070, 0x0480c018},
1317 {0x00000071, 0x00000040},
1318 {0x00000072, 0x01000000},
1319 {0x00000074, 0x000000ff},
1320 {0x00000075, 0x00143400},
1321 {0x00000076, 0x08ec0800},
1322 {0x00000077, 0x040000cc},
1323 {0x00000079, 0x00000000},
1324 {0x0000007a, 0x21000409},
1325 {0x0000007c, 0x00000000},
1326 {0x0000007d, 0xe8000000},
1327 {0x0000007e, 0x044408a8},
1328 {0x0000007f, 0x00000003},
1329 {0x00000080, 0x00000000},
1330 {0x00000081, 0x01000000},
1331 {0x00000082, 0x02000000},
1332 {0x00000083, 0x00000000},
1333 {0x00000084, 0xe3f3e4f4},
1334 {0x00000085, 0x00052024},
1335 {0x00000087, 0x00000000},
1336 {0x00000088, 0x66036603},
1337 {0x00000089, 0x01000000},
1338 {0x0000008b, 0x1c0a0000},
1339 {0x0000008c, 0xff010000},
1340 {0x0000008e, 0xffffefff},
1341 {0x0000008f, 0xfff3efff},
1342 {0x00000090, 0xfff3efbf},
1343 {0x00000094, 0x00101101},
1344 {0x00000095, 0x00000fff},
1345 {0x00000096, 0x00116fff},
1346 {0x00000097, 0x60010000},
1347 {0x00000098, 0x10010000},
1348 {0x00000099, 0x00006000},
1349 {0x0000009a, 0x00001000},
1350 {0x0000009f, 0x00a77400}
1351};
1352
1353static const u32 pitcairn_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1354 {0x0000006f, 0x03044000},
1355 {0x00000070, 0x0480c018},
1356 {0x00000071, 0x00000040},
1357 {0x00000072, 0x01000000},
1358 {0x00000074, 0x000000ff},
1359 {0x00000075, 0x00143400},
1360 {0x00000076, 0x08ec0800},
1361 {0x00000077, 0x040000cc},
1362 {0x00000079, 0x00000000},
1363 {0x0000007a, 0x21000409},
1364 {0x0000007c, 0x00000000},
1365 {0x0000007d, 0xe8000000},
1366 {0x0000007e, 0x044408a8},
1367 {0x0000007f, 0x00000003},
1368 {0x00000080, 0x00000000},
1369 {0x00000081, 0x01000000},
1370 {0x00000082, 0x02000000},
1371 {0x00000083, 0x00000000},
1372 {0x00000084, 0xe3f3e4f4},
1373 {0x00000085, 0x00052024},
1374 {0x00000087, 0x00000000},
1375 {0x00000088, 0x66036603},
1376 {0x00000089, 0x01000000},
1377 {0x0000008b, 0x1c0a0000},
1378 {0x0000008c, 0xff010000},
1379 {0x0000008e, 0xffffefff},
1380 {0x0000008f, 0xfff3efff},
1381 {0x00000090, 0xfff3efbf},
1382 {0x00000094, 0x00101101},
1383 {0x00000095, 0x00000fff},
1384 {0x00000096, 0x00116fff},
1385 {0x00000097, 0x60010000},
1386 {0x00000098, 0x10010000},
1387 {0x00000099, 0x00006000},
1388 {0x0000009a, 0x00001000},
1389 {0x0000009f, 0x00a47400}
1390};
1391
1392static const u32 verde_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1393 {0x0000006f, 0x03044000},
1394 {0x00000070, 0x0480c018},
1395 {0x00000071, 0x00000040},
1396 {0x00000072, 0x01000000},
1397 {0x00000074, 0x000000ff},
1398 {0x00000075, 0x00143400},
1399 {0x00000076, 0x08ec0800},
1400 {0x00000077, 0x040000cc},
1401 {0x00000079, 0x00000000},
1402 {0x0000007a, 0x21000409},
1403 {0x0000007c, 0x00000000},
1404 {0x0000007d, 0xe8000000},
1405 {0x0000007e, 0x044408a8},
1406 {0x0000007f, 0x00000003},
1407 {0x00000080, 0x00000000},
1408 {0x00000081, 0x01000000},
1409 {0x00000082, 0x02000000},
1410 {0x00000083, 0x00000000},
1411 {0x00000084, 0xe3f3e4f4},
1412 {0x00000085, 0x00052024},
1413 {0x00000087, 0x00000000},
1414 {0x00000088, 0x66036603},
1415 {0x00000089, 0x01000000},
1416 {0x0000008b, 0x1c0a0000},
1417 {0x0000008c, 0xff010000},
1418 {0x0000008e, 0xffffefff},
1419 {0x0000008f, 0xfff3efff},
1420 {0x00000090, 0xfff3efbf},
1421 {0x00000094, 0x00101101},
1422 {0x00000095, 0x00000fff},
1423 {0x00000096, 0x00116fff},
1424 {0x00000097, 0x60010000},
1425 {0x00000098, 0x10010000},
1426 {0x00000099, 0x00006000},
1427 {0x0000009a, 0x00001000},
1428 {0x0000009f, 0x00a37400}
1429};
1430
Alex Deucherbcc7f5d2012-07-26 18:36:28 -04001431static const u32 oland_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1432 {0x0000006f, 0x03044000},
1433 {0x00000070, 0x0480c018},
1434 {0x00000071, 0x00000040},
1435 {0x00000072, 0x01000000},
1436 {0x00000074, 0x000000ff},
1437 {0x00000075, 0x00143400},
1438 {0x00000076, 0x08ec0800},
1439 {0x00000077, 0x040000cc},
1440 {0x00000079, 0x00000000},
1441 {0x0000007a, 0x21000409},
1442 {0x0000007c, 0x00000000},
1443 {0x0000007d, 0xe8000000},
1444 {0x0000007e, 0x044408a8},
1445 {0x0000007f, 0x00000003},
1446 {0x00000080, 0x00000000},
1447 {0x00000081, 0x01000000},
1448 {0x00000082, 0x02000000},
1449 {0x00000083, 0x00000000},
1450 {0x00000084, 0xe3f3e4f4},
1451 {0x00000085, 0x00052024},
1452 {0x00000087, 0x00000000},
1453 {0x00000088, 0x66036603},
1454 {0x00000089, 0x01000000},
1455 {0x0000008b, 0x1c0a0000},
1456 {0x0000008c, 0xff010000},
1457 {0x0000008e, 0xffffefff},
1458 {0x0000008f, 0xfff3efff},
1459 {0x00000090, 0xfff3efbf},
1460 {0x00000094, 0x00101101},
1461 {0x00000095, 0x00000fff},
1462 {0x00000096, 0x00116fff},
1463 {0x00000097, 0x60010000},
1464 {0x00000098, 0x10010000},
1465 {0x00000099, 0x00006000},
1466 {0x0000009a, 0x00001000},
1467 {0x0000009f, 0x00a17730}
1468};
1469
Alex Deucherc04c00b2012-07-31 12:57:45 -04001470static const u32 hainan_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1471 {0x0000006f, 0x03044000},
1472 {0x00000070, 0x0480c018},
1473 {0x00000071, 0x00000040},
1474 {0x00000072, 0x01000000},
1475 {0x00000074, 0x000000ff},
1476 {0x00000075, 0x00143400},
1477 {0x00000076, 0x08ec0800},
1478 {0x00000077, 0x040000cc},
1479 {0x00000079, 0x00000000},
1480 {0x0000007a, 0x21000409},
1481 {0x0000007c, 0x00000000},
1482 {0x0000007d, 0xe8000000},
1483 {0x0000007e, 0x044408a8},
1484 {0x0000007f, 0x00000003},
1485 {0x00000080, 0x00000000},
1486 {0x00000081, 0x01000000},
1487 {0x00000082, 0x02000000},
1488 {0x00000083, 0x00000000},
1489 {0x00000084, 0xe3f3e4f4},
1490 {0x00000085, 0x00052024},
1491 {0x00000087, 0x00000000},
1492 {0x00000088, 0x66036603},
1493 {0x00000089, 0x01000000},
1494 {0x0000008b, 0x1c0a0000},
1495 {0x0000008c, 0xff010000},
1496 {0x0000008e, 0xffffefff},
1497 {0x0000008f, 0xfff3efff},
1498 {0x00000090, 0xfff3efbf},
1499 {0x00000094, 0x00101101},
1500 {0x00000095, 0x00000fff},
1501 {0x00000096, 0x00116fff},
1502 {0x00000097, 0x60010000},
1503 {0x00000098, 0x10010000},
1504 {0x00000099, 0x00006000},
1505 {0x0000009a, 0x00001000},
1506 {0x0000009f, 0x00a07730}
1507};
1508
Alex Deucher8b074dd2012-03-20 17:18:18 -04001509/* ucode loading */
Alex Deucher6c7bcce2013-12-18 14:07:14 -05001510int si_mc_load_microcode(struct radeon_device *rdev)
Alex Deucher8b074dd2012-03-20 17:18:18 -04001511{
Alex Deucher629bd332014-06-25 18:41:34 -04001512 const __be32 *fw_data = NULL;
1513 const __le32 *new_fw_data = NULL;
Alex Deucher8b074dd2012-03-20 17:18:18 -04001514 u32 running, blackout = 0;
Alex Deucher629bd332014-06-25 18:41:34 -04001515 u32 *io_mc_regs = NULL;
1516 const __le32 *new_io_mc_regs = NULL;
Alex Deucher8c79bae2014-04-16 09:42:22 -04001517 int i, regs_size, ucode_size;
Alex Deucher8b074dd2012-03-20 17:18:18 -04001518
1519 if (!rdev->mc_fw)
1520 return -EINVAL;
1521
Alex Deucher629bd332014-06-25 18:41:34 -04001522 if (rdev->new_fw) {
1523 const struct mc_firmware_header_v1_0 *hdr =
1524 (const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data;
Alex Deucher8c79bae2014-04-16 09:42:22 -04001525
Alex Deucher629bd332014-06-25 18:41:34 -04001526 radeon_ucode_print_mc_hdr(&hdr->header);
1527 regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
1528 new_io_mc_regs = (const __le32 *)
1529 (rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
1530 ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
1531 new_fw_data = (const __le32 *)
1532 (rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1533 } else {
1534 ucode_size = rdev->mc_fw->size / 4;
1535
1536 switch (rdev->family) {
1537 case CHIP_TAHITI:
1538 io_mc_regs = (u32 *)&tahiti_io_mc_regs;
1539 regs_size = TAHITI_IO_MC_REGS_SIZE;
1540 break;
1541 case CHIP_PITCAIRN:
1542 io_mc_regs = (u32 *)&pitcairn_io_mc_regs;
1543 regs_size = TAHITI_IO_MC_REGS_SIZE;
1544 break;
1545 case CHIP_VERDE:
1546 default:
1547 io_mc_regs = (u32 *)&verde_io_mc_regs;
1548 regs_size = TAHITI_IO_MC_REGS_SIZE;
1549 break;
1550 case CHIP_OLAND:
1551 io_mc_regs = (u32 *)&oland_io_mc_regs;
1552 regs_size = TAHITI_IO_MC_REGS_SIZE;
1553 break;
1554 case CHIP_HAINAN:
1555 io_mc_regs = (u32 *)&hainan_io_mc_regs;
1556 regs_size = TAHITI_IO_MC_REGS_SIZE;
1557 break;
1558 }
1559 fw_data = (const __be32 *)rdev->mc_fw->data;
Alex Deucher8b074dd2012-03-20 17:18:18 -04001560 }
1561
1562 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1563
1564 if (running == 0) {
1565 if (running) {
1566 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1567 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1568 }
1569
1570 /* reset the engine and set to writable */
1571 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1572 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1573
1574 /* load mc io regs */
1575 for (i = 0; i < regs_size; i++) {
Alex Deucher629bd332014-06-25 18:41:34 -04001576 if (rdev->new_fw) {
1577 WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++));
1578 WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++));
1579 } else {
1580 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1581 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1582 }
Alex Deucher8b074dd2012-03-20 17:18:18 -04001583 }
1584 /* load the MC ucode */
Alex Deucher629bd332014-06-25 18:41:34 -04001585 for (i = 0; i < ucode_size; i++) {
1586 if (rdev->new_fw)
1587 WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++));
1588 else
1589 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1590 }
Alex Deucher8b074dd2012-03-20 17:18:18 -04001591
1592 /* put the engine back into the active state */
1593 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1594 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1595 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1596
1597 /* wait for training to complete */
1598 for (i = 0; i < rdev->usec_timeout; i++) {
1599 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1600 break;
1601 udelay(1);
1602 }
1603 for (i = 0; i < rdev->usec_timeout; i++) {
1604 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1605 break;
1606 udelay(1);
1607 }
1608
1609 if (running)
1610 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1611 }
1612
1613 return 0;
1614}
1615
Alex Deucher0f0de062012-03-20 17:18:17 -04001616static int si_init_microcode(struct radeon_device *rdev)
1617{
Alex Deucher0f0de062012-03-20 17:18:17 -04001618 const char *chip_name;
Alex Deucher629bd332014-06-25 18:41:34 -04001619 const char *new_chip_name;
Alex Deucher0f0de062012-03-20 17:18:17 -04001620 size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size;
Alex Deucher1ebe9282014-04-11 11:21:49 -04001621 size_t smc_req_size, mc2_req_size;
Alex Deucher0f0de062012-03-20 17:18:17 -04001622 char fw_name[30];
1623 int err;
Alex Deucher629bd332014-06-25 18:41:34 -04001624 int new_fw = 0;
Alex Deucher0f0de062012-03-20 17:18:17 -04001625
1626 DRM_DEBUG("\n");
1627
Alex Deucher0f0de062012-03-20 17:18:17 -04001628 switch (rdev->family) {
1629 case CHIP_TAHITI:
1630 chip_name = "TAHITI";
Alex Deucher629bd332014-06-25 18:41:34 -04001631 new_chip_name = "tahiti";
Alex Deucher0f0de062012-03-20 17:18:17 -04001632 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1633 me_req_size = SI_PM4_UCODE_SIZE * 4;
1634 ce_req_size = SI_CE_UCODE_SIZE * 4;
1635 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1636 mc_req_size = SI_MC_UCODE_SIZE * 4;
Alex Deucher1ebe9282014-04-11 11:21:49 -04001637 mc2_req_size = TAHITI_MC_UCODE_SIZE * 4;
Alex Deuchera9e61412013-06-25 17:56:16 -04001638 smc_req_size = ALIGN(TAHITI_SMC_UCODE_SIZE, 4);
Alex Deucher0f0de062012-03-20 17:18:17 -04001639 break;
1640 case CHIP_PITCAIRN:
1641 chip_name = "PITCAIRN";
Alex Deucher629bd332014-06-25 18:41:34 -04001642 new_chip_name = "pitcairn";
Alex Deucher0f0de062012-03-20 17:18:17 -04001643 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1644 me_req_size = SI_PM4_UCODE_SIZE * 4;
1645 ce_req_size = SI_CE_UCODE_SIZE * 4;
1646 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1647 mc_req_size = SI_MC_UCODE_SIZE * 4;
Alex Deucher1ebe9282014-04-11 11:21:49 -04001648 mc2_req_size = PITCAIRN_MC_UCODE_SIZE * 4;
Alex Deuchera9e61412013-06-25 17:56:16 -04001649 smc_req_size = ALIGN(PITCAIRN_SMC_UCODE_SIZE, 4);
Alex Deucher0f0de062012-03-20 17:18:17 -04001650 break;
1651 case CHIP_VERDE:
1652 chip_name = "VERDE";
Alex Deucher629bd332014-06-25 18:41:34 -04001653 new_chip_name = "verde";
Alex Deucher0f0de062012-03-20 17:18:17 -04001654 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1655 me_req_size = SI_PM4_UCODE_SIZE * 4;
1656 ce_req_size = SI_CE_UCODE_SIZE * 4;
1657 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1658 mc_req_size = SI_MC_UCODE_SIZE * 4;
Alex Deucher1ebe9282014-04-11 11:21:49 -04001659 mc2_req_size = VERDE_MC_UCODE_SIZE * 4;
Alex Deuchera9e61412013-06-25 17:56:16 -04001660 smc_req_size = ALIGN(VERDE_SMC_UCODE_SIZE, 4);
Alex Deucher0f0de062012-03-20 17:18:17 -04001661 break;
Alex Deucherbcc7f5d2012-07-26 18:36:28 -04001662 case CHIP_OLAND:
1663 chip_name = "OLAND";
Alex Deucher629bd332014-06-25 18:41:34 -04001664 new_chip_name = "oland";
Alex Deucherbcc7f5d2012-07-26 18:36:28 -04001665 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1666 me_req_size = SI_PM4_UCODE_SIZE * 4;
1667 ce_req_size = SI_CE_UCODE_SIZE * 4;
1668 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
Alex Deucher1ebe9282014-04-11 11:21:49 -04001669 mc_req_size = mc2_req_size = OLAND_MC_UCODE_SIZE * 4;
Alex Deuchera9e61412013-06-25 17:56:16 -04001670 smc_req_size = ALIGN(OLAND_SMC_UCODE_SIZE, 4);
Alex Deucherbcc7f5d2012-07-26 18:36:28 -04001671 break;
Alex Deucherc04c00b2012-07-31 12:57:45 -04001672 case CHIP_HAINAN:
1673 chip_name = "HAINAN";
Alex Deucher629bd332014-06-25 18:41:34 -04001674 new_chip_name = "hainan";
Alex Deucherc04c00b2012-07-31 12:57:45 -04001675 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1676 me_req_size = SI_PM4_UCODE_SIZE * 4;
1677 ce_req_size = SI_CE_UCODE_SIZE * 4;
1678 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
Alex Deucher1ebe9282014-04-11 11:21:49 -04001679 mc_req_size = mc2_req_size = OLAND_MC_UCODE_SIZE * 4;
Alex Deuchera9e61412013-06-25 17:56:16 -04001680 smc_req_size = ALIGN(HAINAN_SMC_UCODE_SIZE, 4);
Alex Deucherc04c00b2012-07-31 12:57:45 -04001681 break;
Alex Deucher0f0de062012-03-20 17:18:17 -04001682 default: BUG();
1683 }
1684
Alex Deucher629bd332014-06-25 18:41:34 -04001685 DRM_INFO("Loading %s Microcode\n", new_chip_name);
Alex Deucher0f0de062012-03-20 17:18:17 -04001686
Alex Deucher629bd332014-06-25 18:41:34 -04001687 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", new_chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -04001688 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
Alex Deucher1ebe9282014-04-11 11:21:49 -04001689 if (err) {
Alex Deucher629bd332014-06-25 18:41:34 -04001690 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1691 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
Alex Deucher1ebe9282014-04-11 11:21:49 -04001692 if (err)
1693 goto out;
Alex Deucher629bd332014-06-25 18:41:34 -04001694 if (rdev->pfp_fw->size != pfp_req_size) {
1695 printk(KERN_ERR
1696 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1697 rdev->pfp_fw->size, fw_name);
1698 err = -EINVAL;
1699 goto out;
1700 }
1701 } else {
1702 err = radeon_ucode_validate(rdev->pfp_fw);
1703 if (err) {
1704 printk(KERN_ERR
1705 "si_cp: validation failed for firmware \"%s\"\n",
1706 fw_name);
1707 goto out;
1708 } else {
1709 new_fw++;
1710 }
Alex Deucher1ebe9282014-04-11 11:21:49 -04001711 }
Alex Deucher0f0de062012-03-20 17:18:17 -04001712
Alex Deucher629bd332014-06-25 18:41:34 -04001713 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", new_chip_name);
1714 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
1715 if (err) {
1716 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
1717 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
1718 if (err)
1719 goto out;
1720 if (rdev->me_fw->size != me_req_size) {
1721 printk(KERN_ERR
1722 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1723 rdev->me_fw->size, fw_name);
1724 err = -EINVAL;
1725 }
1726 } else {
1727 err = radeon_ucode_validate(rdev->me_fw);
1728 if (err) {
1729 printk(KERN_ERR
1730 "si_cp: validation failed for firmware \"%s\"\n",
1731 fw_name);
1732 goto out;
1733 } else {
1734 new_fw++;
1735 }
1736 }
1737
1738 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", new_chip_name);
1739 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1740 if (err) {
1741 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
1742 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1743 if (err)
1744 goto out;
1745 if (rdev->ce_fw->size != ce_req_size) {
1746 printk(KERN_ERR
1747 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1748 rdev->ce_fw->size, fw_name);
1749 err = -EINVAL;
1750 }
1751 } else {
1752 err = radeon_ucode_validate(rdev->ce_fw);
1753 if (err) {
1754 printk(KERN_ERR
1755 "si_cp: validation failed for firmware \"%s\"\n",
1756 fw_name);
1757 goto out;
1758 } else {
1759 new_fw++;
1760 }
1761 }
1762
1763 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", new_chip_name);
1764 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1765 if (err) {
1766 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
1767 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1768 if (err)
1769 goto out;
1770 if (rdev->rlc_fw->size != rlc_req_size) {
1771 printk(KERN_ERR
1772 "si_rlc: Bogus length %zu in firmware \"%s\"\n",
1773 rdev->rlc_fw->size, fw_name);
1774 err = -EINVAL;
1775 }
1776 } else {
1777 err = radeon_ucode_validate(rdev->rlc_fw);
1778 if (err) {
1779 printk(KERN_ERR
1780 "si_cp: validation failed for firmware \"%s\"\n",
1781 fw_name);
1782 goto out;
1783 } else {
1784 new_fw++;
1785 }
1786 }
1787
1788 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", new_chip_name);
1789 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1790 if (err) {
1791 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name);
1792 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1793 if (err) {
1794 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
1795 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1796 if (err)
1797 goto out;
1798 }
1799 if ((rdev->mc_fw->size != mc_req_size) &&
1800 (rdev->mc_fw->size != mc2_req_size)) {
1801 printk(KERN_ERR
1802 "si_mc: Bogus length %zu in firmware \"%s\"\n",
1803 rdev->mc_fw->size, fw_name);
1804 err = -EINVAL;
1805 }
1806 DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
1807 } else {
1808 err = radeon_ucode_validate(rdev->mc_fw);
1809 if (err) {
1810 printk(KERN_ERR
1811 "si_cp: validation failed for firmware \"%s\"\n",
1812 fw_name);
1813 goto out;
1814 } else {
1815 new_fw++;
1816 }
1817 }
1818
1819 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", new_chip_name);
Jerome Glisse0a168932013-07-11 15:53:01 -04001820 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
Alex Deucher8a53fa22013-08-07 16:09:08 -04001821 if (err) {
Alex Deucher629bd332014-06-25 18:41:34 -04001822 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
1823 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1824 if (err) {
1825 printk(KERN_ERR
1826 "smc: error loading firmware \"%s\"\n",
1827 fw_name);
1828 release_firmware(rdev->smc_fw);
1829 rdev->smc_fw = NULL;
1830 err = 0;
1831 } else if (rdev->smc_fw->size != smc_req_size) {
1832 printk(KERN_ERR
1833 "si_smc: Bogus length %zu in firmware \"%s\"\n",
1834 rdev->smc_fw->size, fw_name);
1835 err = -EINVAL;
1836 }
1837 } else {
1838 err = radeon_ucode_validate(rdev->smc_fw);
1839 if (err) {
1840 printk(KERN_ERR
1841 "si_cp: validation failed for firmware \"%s\"\n",
1842 fw_name);
1843 goto out;
1844 } else {
1845 new_fw++;
1846 }
Alex Deuchera9e61412013-06-25 17:56:16 -04001847 }
1848
Alex Deucher629bd332014-06-25 18:41:34 -04001849 if (new_fw == 0) {
1850 rdev->new_fw = false;
1851 } else if (new_fw < 6) {
1852 printk(KERN_ERR "si_fw: mixing new and old firmware!\n");
1853 err = -EINVAL;
1854 } else {
1855 rdev->new_fw = true;
1856 }
Alex Deucher0f0de062012-03-20 17:18:17 -04001857out:
Alex Deucher0f0de062012-03-20 17:18:17 -04001858 if (err) {
1859 if (err != -EINVAL)
1860 printk(KERN_ERR
1861 "si_cp: Failed to load firmware \"%s\"\n",
1862 fw_name);
1863 release_firmware(rdev->pfp_fw);
1864 rdev->pfp_fw = NULL;
1865 release_firmware(rdev->me_fw);
1866 rdev->me_fw = NULL;
1867 release_firmware(rdev->ce_fw);
1868 rdev->ce_fw = NULL;
1869 release_firmware(rdev->rlc_fw);
1870 rdev->rlc_fw = NULL;
1871 release_firmware(rdev->mc_fw);
1872 rdev->mc_fw = NULL;
Alex Deuchera9e61412013-06-25 17:56:16 -04001873 release_firmware(rdev->smc_fw);
1874 rdev->smc_fw = NULL;
Alex Deucher0f0de062012-03-20 17:18:17 -04001875 }
1876 return err;
1877}
1878
Alex Deucher43b3cd92012-03-20 17:18:00 -04001879/* watermark setup */
1880static u32 dce6_line_buffer_adjust(struct radeon_device *rdev,
1881 struct radeon_crtc *radeon_crtc,
1882 struct drm_display_mode *mode,
1883 struct drm_display_mode *other_mode)
1884{
Alex Deucher290d2452013-08-19 11:15:43 -04001885 u32 tmp, buffer_alloc, i;
1886 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
Alex Deucher43b3cd92012-03-20 17:18:00 -04001887 /*
1888 * Line Buffer Setup
1889 * There are 3 line buffers, each one shared by 2 display controllers.
1890 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1891 * the display controllers. The paritioning is done via one of four
1892 * preset allocations specified in bits 21:20:
1893 * 0 - half lb
1894 * 2 - whole lb, other crtc must be disabled
1895 */
1896 /* this can get tricky if we have two large displays on a paired group
1897 * of crtcs. Ideally for multiple large displays we'd assign them to
1898 * non-linked crtcs for maximum line buffer allocation.
1899 */
1900 if (radeon_crtc->base.enabled && mode) {
Alex Deucher290d2452013-08-19 11:15:43 -04001901 if (other_mode) {
Alex Deucher43b3cd92012-03-20 17:18:00 -04001902 tmp = 0; /* 1/2 */
Alex Deucher290d2452013-08-19 11:15:43 -04001903 buffer_alloc = 1;
1904 } else {
Alex Deucher43b3cd92012-03-20 17:18:00 -04001905 tmp = 2; /* whole */
Alex Deucher290d2452013-08-19 11:15:43 -04001906 buffer_alloc = 2;
1907 }
1908 } else {
Alex Deucher43b3cd92012-03-20 17:18:00 -04001909 tmp = 0;
Alex Deucher290d2452013-08-19 11:15:43 -04001910 buffer_alloc = 0;
1911 }
Alex Deucher43b3cd92012-03-20 17:18:00 -04001912
1913 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset,
1914 DC_LB_MEMORY_CONFIG(tmp));
1915
Alex Deucher290d2452013-08-19 11:15:43 -04001916 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1917 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1918 for (i = 0; i < rdev->usec_timeout; i++) {
1919 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1920 DMIF_BUFFERS_ALLOCATED_COMPLETED)
1921 break;
1922 udelay(1);
1923 }
1924
Alex Deucher43b3cd92012-03-20 17:18:00 -04001925 if (radeon_crtc->base.enabled && mode) {
1926 switch (tmp) {
1927 case 0:
1928 default:
1929 return 4096 * 2;
1930 case 2:
1931 return 8192 * 2;
1932 }
1933 }
1934
1935 /* controller not enabled, so no lb used */
1936 return 0;
1937}
1938
Alex Deucherca7db222012-03-20 17:18:30 -04001939static u32 si_get_number_of_dram_channels(struct radeon_device *rdev)
Alex Deucher43b3cd92012-03-20 17:18:00 -04001940{
1941 u32 tmp = RREG32(MC_SHARED_CHMAP);
1942
1943 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1944 case 0:
1945 default:
1946 return 1;
1947 case 1:
1948 return 2;
1949 case 2:
1950 return 4;
1951 case 3:
1952 return 8;
1953 case 4:
1954 return 3;
1955 case 5:
1956 return 6;
1957 case 6:
1958 return 10;
1959 case 7:
1960 return 12;
1961 case 8:
1962 return 16;
1963 }
1964}
1965
1966struct dce6_wm_params {
1967 u32 dram_channels; /* number of dram channels */
1968 u32 yclk; /* bandwidth per dram data pin in kHz */
1969 u32 sclk; /* engine clock in kHz */
1970 u32 disp_clk; /* display clock in kHz */
1971 u32 src_width; /* viewport width */
1972 u32 active_time; /* active display time in ns */
1973 u32 blank_time; /* blank time in ns */
1974 bool interlaced; /* mode is interlaced */
1975 fixed20_12 vsc; /* vertical scale ratio */
1976 u32 num_heads; /* number of active crtcs */
1977 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1978 u32 lb_size; /* line buffer allocated to pipe */
1979 u32 vtaps; /* vertical scaler taps */
1980};
1981
1982static u32 dce6_dram_bandwidth(struct dce6_wm_params *wm)
1983{
1984 /* Calculate raw DRAM Bandwidth */
1985 fixed20_12 dram_efficiency; /* 0.7 */
1986 fixed20_12 yclk, dram_channels, bandwidth;
1987 fixed20_12 a;
1988
1989 a.full = dfixed_const(1000);
1990 yclk.full = dfixed_const(wm->yclk);
1991 yclk.full = dfixed_div(yclk, a);
1992 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1993 a.full = dfixed_const(10);
1994 dram_efficiency.full = dfixed_const(7);
1995 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1996 bandwidth.full = dfixed_mul(dram_channels, yclk);
1997 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1998
1999 return dfixed_trunc(bandwidth);
2000}
2001
2002static u32 dce6_dram_bandwidth_for_display(struct dce6_wm_params *wm)
2003{
2004 /* Calculate DRAM Bandwidth and the part allocated to display. */
2005 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
2006 fixed20_12 yclk, dram_channels, bandwidth;
2007 fixed20_12 a;
2008
2009 a.full = dfixed_const(1000);
2010 yclk.full = dfixed_const(wm->yclk);
2011 yclk.full = dfixed_div(yclk, a);
2012 dram_channels.full = dfixed_const(wm->dram_channels * 4);
2013 a.full = dfixed_const(10);
2014 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
2015 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
2016 bandwidth.full = dfixed_mul(dram_channels, yclk);
2017 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
2018
2019 return dfixed_trunc(bandwidth);
2020}
2021
2022static u32 dce6_data_return_bandwidth(struct dce6_wm_params *wm)
2023{
2024 /* Calculate the display Data return Bandwidth */
2025 fixed20_12 return_efficiency; /* 0.8 */
2026 fixed20_12 sclk, bandwidth;
2027 fixed20_12 a;
2028
2029 a.full = dfixed_const(1000);
2030 sclk.full = dfixed_const(wm->sclk);
2031 sclk.full = dfixed_div(sclk, a);
2032 a.full = dfixed_const(10);
2033 return_efficiency.full = dfixed_const(8);
2034 return_efficiency.full = dfixed_div(return_efficiency, a);
2035 a.full = dfixed_const(32);
2036 bandwidth.full = dfixed_mul(a, sclk);
2037 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
2038
2039 return dfixed_trunc(bandwidth);
2040}
2041
2042static u32 dce6_get_dmif_bytes_per_request(struct dce6_wm_params *wm)
2043{
2044 return 32;
2045}
2046
2047static u32 dce6_dmif_request_bandwidth(struct dce6_wm_params *wm)
2048{
2049 /* Calculate the DMIF Request Bandwidth */
2050 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
2051 fixed20_12 disp_clk, sclk, bandwidth;
2052 fixed20_12 a, b1, b2;
2053 u32 min_bandwidth;
2054
2055 a.full = dfixed_const(1000);
2056 disp_clk.full = dfixed_const(wm->disp_clk);
2057 disp_clk.full = dfixed_div(disp_clk, a);
2058 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm) / 2);
2059 b1.full = dfixed_mul(a, disp_clk);
2060
2061 a.full = dfixed_const(1000);
2062 sclk.full = dfixed_const(wm->sclk);
2063 sclk.full = dfixed_div(sclk, a);
2064 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm));
2065 b2.full = dfixed_mul(a, sclk);
2066
2067 a.full = dfixed_const(10);
2068 disp_clk_request_efficiency.full = dfixed_const(8);
2069 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2070
2071 min_bandwidth = min(dfixed_trunc(b1), dfixed_trunc(b2));
2072
2073 a.full = dfixed_const(min_bandwidth);
2074 bandwidth.full = dfixed_mul(a, disp_clk_request_efficiency);
2075
2076 return dfixed_trunc(bandwidth);
2077}
2078
2079static u32 dce6_available_bandwidth(struct dce6_wm_params *wm)
2080{
2081 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2082 u32 dram_bandwidth = dce6_dram_bandwidth(wm);
2083 u32 data_return_bandwidth = dce6_data_return_bandwidth(wm);
2084 u32 dmif_req_bandwidth = dce6_dmif_request_bandwidth(wm);
2085
2086 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2087}
2088
2089static u32 dce6_average_bandwidth(struct dce6_wm_params *wm)
2090{
2091 /* Calculate the display mode Average Bandwidth
2092 * DisplayMode should contain the source and destination dimensions,
2093 * timing, etc.
2094 */
2095 fixed20_12 bpp;
2096 fixed20_12 line_time;
2097 fixed20_12 src_width;
2098 fixed20_12 bandwidth;
2099 fixed20_12 a;
2100
2101 a.full = dfixed_const(1000);
2102 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2103 line_time.full = dfixed_div(line_time, a);
2104 bpp.full = dfixed_const(wm->bytes_per_pixel);
2105 src_width.full = dfixed_const(wm->src_width);
2106 bandwidth.full = dfixed_mul(src_width, bpp);
2107 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2108 bandwidth.full = dfixed_div(bandwidth, line_time);
2109
2110 return dfixed_trunc(bandwidth);
2111}
2112
2113static u32 dce6_latency_watermark(struct dce6_wm_params *wm)
2114{
2115 /* First calcualte the latency in ns */
2116 u32 mc_latency = 2000; /* 2000 ns. */
2117 u32 available_bandwidth = dce6_available_bandwidth(wm);
2118 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2119 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2120 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2121 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2122 (wm->num_heads * cursor_line_pair_return_time);
2123 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2124 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2125 u32 tmp, dmif_size = 12288;
2126 fixed20_12 a, b, c;
2127
2128 if (wm->num_heads == 0)
2129 return 0;
2130
2131 a.full = dfixed_const(2);
2132 b.full = dfixed_const(1);
2133 if ((wm->vsc.full > a.full) ||
2134 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2135 (wm->vtaps >= 5) ||
2136 ((wm->vsc.full >= a.full) && wm->interlaced))
2137 max_src_lines_per_dst_line = 4;
2138 else
2139 max_src_lines_per_dst_line = 2;
2140
2141 a.full = dfixed_const(available_bandwidth);
2142 b.full = dfixed_const(wm->num_heads);
2143 a.full = dfixed_div(a, b);
2144
2145 b.full = dfixed_const(mc_latency + 512);
2146 c.full = dfixed_const(wm->disp_clk);
2147 b.full = dfixed_div(b, c);
2148
2149 c.full = dfixed_const(dmif_size);
2150 b.full = dfixed_div(c, b);
2151
2152 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
2153
2154 b.full = dfixed_const(1000);
2155 c.full = dfixed_const(wm->disp_clk);
2156 b.full = dfixed_div(c, b);
2157 c.full = dfixed_const(wm->bytes_per_pixel);
2158 b.full = dfixed_mul(b, c);
2159
2160 lb_fill_bw = min(tmp, dfixed_trunc(b));
2161
2162 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2163 b.full = dfixed_const(1000);
2164 c.full = dfixed_const(lb_fill_bw);
2165 b.full = dfixed_div(c, b);
2166 a.full = dfixed_div(a, b);
2167 line_fill_time = dfixed_trunc(a);
2168
2169 if (line_fill_time < wm->active_time)
2170 return latency;
2171 else
2172 return latency + (line_fill_time - wm->active_time);
2173
2174}
2175
2176static bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params *wm)
2177{
2178 if (dce6_average_bandwidth(wm) <=
2179 (dce6_dram_bandwidth_for_display(wm) / wm->num_heads))
2180 return true;
2181 else
2182 return false;
2183};
2184
2185static bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params *wm)
2186{
2187 if (dce6_average_bandwidth(wm) <=
2188 (dce6_available_bandwidth(wm) / wm->num_heads))
2189 return true;
2190 else
2191 return false;
2192};
2193
2194static bool dce6_check_latency_hiding(struct dce6_wm_params *wm)
2195{
2196 u32 lb_partitions = wm->lb_size / wm->src_width;
2197 u32 line_time = wm->active_time + wm->blank_time;
2198 u32 latency_tolerant_lines;
2199 u32 latency_hiding;
2200 fixed20_12 a;
2201
2202 a.full = dfixed_const(1);
2203 if (wm->vsc.full > a.full)
2204 latency_tolerant_lines = 1;
2205 else {
2206 if (lb_partitions <= (wm->vtaps + 1))
2207 latency_tolerant_lines = 1;
2208 else
2209 latency_tolerant_lines = 2;
2210 }
2211
2212 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2213
2214 if (dce6_latency_watermark(wm) <= latency_hiding)
2215 return true;
2216 else
2217 return false;
2218}
2219
2220static void dce6_program_watermarks(struct radeon_device *rdev,
2221 struct radeon_crtc *radeon_crtc,
2222 u32 lb_size, u32 num_heads)
2223{
2224 struct drm_display_mode *mode = &radeon_crtc->base.mode;
Alex Deucherc696e532012-05-03 10:43:25 -04002225 struct dce6_wm_params wm_low, wm_high;
2226 u32 dram_channels;
Alex Deucher43b3cd92012-03-20 17:18:00 -04002227 u32 pixel_period;
2228 u32 line_time = 0;
2229 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2230 u32 priority_a_mark = 0, priority_b_mark = 0;
2231 u32 priority_a_cnt = PRIORITY_OFF;
2232 u32 priority_b_cnt = PRIORITY_OFF;
2233 u32 tmp, arb_control3;
2234 fixed20_12 a, b, c;
2235
2236 if (radeon_crtc->base.enabled && num_heads && mode) {
2237 pixel_period = 1000000 / (u32)mode->clock;
2238 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2239 priority_a_cnt = 0;
2240 priority_b_cnt = 0;
2241
Alex Deucherca7db222012-03-20 17:18:30 -04002242 if (rdev->family == CHIP_ARUBA)
Alex Deucherc696e532012-05-03 10:43:25 -04002243 dram_channels = evergreen_get_number_of_dram_channels(rdev);
Alex Deucherca7db222012-03-20 17:18:30 -04002244 else
Alex Deucherc696e532012-05-03 10:43:25 -04002245 dram_channels = si_get_number_of_dram_channels(rdev);
2246
2247 /* watermark for high clocks */
2248 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2249 wm_high.yclk =
2250 radeon_dpm_get_mclk(rdev, false) * 10;
2251 wm_high.sclk =
2252 radeon_dpm_get_sclk(rdev, false) * 10;
2253 } else {
2254 wm_high.yclk = rdev->pm.current_mclk * 10;
2255 wm_high.sclk = rdev->pm.current_sclk * 10;
2256 }
2257
2258 wm_high.disp_clk = mode->clock;
2259 wm_high.src_width = mode->crtc_hdisplay;
2260 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2261 wm_high.blank_time = line_time - wm_high.active_time;
2262 wm_high.interlaced = false;
2263 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2264 wm_high.interlaced = true;
2265 wm_high.vsc = radeon_crtc->vsc;
2266 wm_high.vtaps = 1;
2267 if (radeon_crtc->rmx_type != RMX_OFF)
2268 wm_high.vtaps = 2;
2269 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2270 wm_high.lb_size = lb_size;
2271 wm_high.dram_channels = dram_channels;
2272 wm_high.num_heads = num_heads;
2273
2274 /* watermark for low clocks */
2275 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2276 wm_low.yclk =
2277 radeon_dpm_get_mclk(rdev, true) * 10;
2278 wm_low.sclk =
2279 radeon_dpm_get_sclk(rdev, true) * 10;
2280 } else {
2281 wm_low.yclk = rdev->pm.current_mclk * 10;
2282 wm_low.sclk = rdev->pm.current_sclk * 10;
2283 }
2284
2285 wm_low.disp_clk = mode->clock;
2286 wm_low.src_width = mode->crtc_hdisplay;
2287 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2288 wm_low.blank_time = line_time - wm_low.active_time;
2289 wm_low.interlaced = false;
2290 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2291 wm_low.interlaced = true;
2292 wm_low.vsc = radeon_crtc->vsc;
2293 wm_low.vtaps = 1;
2294 if (radeon_crtc->rmx_type != RMX_OFF)
2295 wm_low.vtaps = 2;
2296 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2297 wm_low.lb_size = lb_size;
2298 wm_low.dram_channels = dram_channels;
2299 wm_low.num_heads = num_heads;
Alex Deucher43b3cd92012-03-20 17:18:00 -04002300
2301 /* set for high clocks */
Alex Deucherc696e532012-05-03 10:43:25 -04002302 latency_watermark_a = min(dce6_latency_watermark(&wm_high), (u32)65535);
Alex Deucher43b3cd92012-03-20 17:18:00 -04002303 /* set for low clocks */
Alex Deucherc696e532012-05-03 10:43:25 -04002304 latency_watermark_b = min(dce6_latency_watermark(&wm_low), (u32)65535);
Alex Deucher43b3cd92012-03-20 17:18:00 -04002305
2306 /* possibly force display priority to high */
2307 /* should really do this at mode validation time... */
Alex Deucherc696e532012-05-03 10:43:25 -04002308 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2309 !dce6_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2310 !dce6_check_latency_hiding(&wm_high) ||
2311 (rdev->disp_priority == 2)) {
2312 DRM_DEBUG_KMS("force priority to high\n");
2313 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2314 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2315 }
2316 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2317 !dce6_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2318 !dce6_check_latency_hiding(&wm_low) ||
Alex Deucher43b3cd92012-03-20 17:18:00 -04002319 (rdev->disp_priority == 2)) {
2320 DRM_DEBUG_KMS("force priority to high\n");
2321 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2322 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2323 }
2324
2325 a.full = dfixed_const(1000);
2326 b.full = dfixed_const(mode->clock);
2327 b.full = dfixed_div(b, a);
2328 c.full = dfixed_const(latency_watermark_a);
2329 c.full = dfixed_mul(c, b);
2330 c.full = dfixed_mul(c, radeon_crtc->hsc);
2331 c.full = dfixed_div(c, a);
2332 a.full = dfixed_const(16);
2333 c.full = dfixed_div(c, a);
2334 priority_a_mark = dfixed_trunc(c);
2335 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2336
2337 a.full = dfixed_const(1000);
2338 b.full = dfixed_const(mode->clock);
2339 b.full = dfixed_div(b, a);
2340 c.full = dfixed_const(latency_watermark_b);
2341 c.full = dfixed_mul(c, b);
2342 c.full = dfixed_mul(c, radeon_crtc->hsc);
2343 c.full = dfixed_div(c, a);
2344 a.full = dfixed_const(16);
2345 c.full = dfixed_div(c, a);
2346 priority_b_mark = dfixed_trunc(c);
2347 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2348 }
2349
2350 /* select wm A */
2351 arb_control3 = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2352 tmp = arb_control3;
2353 tmp &= ~LATENCY_WATERMARK_MASK(3);
2354 tmp |= LATENCY_WATERMARK_MASK(1);
2355 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2356 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2357 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2358 LATENCY_HIGH_WATERMARK(line_time)));
2359 /* select wm B */
2360 tmp = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2361 tmp &= ~LATENCY_WATERMARK_MASK(3);
2362 tmp |= LATENCY_WATERMARK_MASK(2);
2363 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2364 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2365 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2366 LATENCY_HIGH_WATERMARK(line_time)));
2367 /* restore original selection */
2368 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, arb_control3);
2369
2370 /* write the priority marks */
2371 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2372 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2373
Alex Deucher7178d2a2013-03-21 10:38:49 -04002374 /* save values for DPM */
2375 radeon_crtc->line_time = line_time;
2376 radeon_crtc->wm_high = latency_watermark_a;
2377 radeon_crtc->wm_low = latency_watermark_b;
Alex Deucher43b3cd92012-03-20 17:18:00 -04002378}
2379
2380void dce6_bandwidth_update(struct radeon_device *rdev)
2381{
2382 struct drm_display_mode *mode0 = NULL;
2383 struct drm_display_mode *mode1 = NULL;
2384 u32 num_heads = 0, lb_size;
2385 int i;
2386
2387 radeon_update_display_priority(rdev);
2388
2389 for (i = 0; i < rdev->num_crtc; i++) {
2390 if (rdev->mode_info.crtcs[i]->base.enabled)
2391 num_heads++;
2392 }
2393 for (i = 0; i < rdev->num_crtc; i += 2) {
2394 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2395 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2396 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2397 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2398 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2399 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2400 }
2401}
2402
Alex Deucher0a96d722012-03-20 17:18:11 -04002403/*
2404 * Core functions
2405 */
Alex Deucher0a96d722012-03-20 17:18:11 -04002406static void si_tiling_mode_table_init(struct radeon_device *rdev)
2407{
2408 const u32 num_tile_mode_states = 32;
2409 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
2410
2411 switch (rdev->config.si.mem_row_size_in_kb) {
2412 case 1:
2413 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2414 break;
2415 case 2:
2416 default:
2417 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2418 break;
2419 case 4:
2420 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2421 break;
2422 }
2423
2424 if ((rdev->family == CHIP_TAHITI) ||
2425 (rdev->family == CHIP_PITCAIRN)) {
2426 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2427 switch (reg_offset) {
2428 case 0: /* non-AA compressed depth or any compressed stencil */
2429 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2430 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2431 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2432 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2433 NUM_BANKS(ADDR_SURF_16_BANK) |
2434 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2435 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2436 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2437 break;
2438 case 1: /* 2xAA/4xAA compressed depth only */
2439 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2440 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2441 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2442 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2443 NUM_BANKS(ADDR_SURF_16_BANK) |
2444 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2445 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2446 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2447 break;
2448 case 2: /* 8xAA compressed depth only */
2449 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2450 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2451 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2452 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2453 NUM_BANKS(ADDR_SURF_16_BANK) |
2454 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2455 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2456 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2457 break;
2458 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2459 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2460 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2461 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2462 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2463 NUM_BANKS(ADDR_SURF_16_BANK) |
2464 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2465 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2466 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2467 break;
2468 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2469 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2470 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2471 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2472 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2473 NUM_BANKS(ADDR_SURF_16_BANK) |
2474 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2475 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2476 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2477 break;
2478 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2479 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2480 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2481 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2482 TILE_SPLIT(split_equal_to_row_size) |
2483 NUM_BANKS(ADDR_SURF_16_BANK) |
2484 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2485 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2486 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2487 break;
2488 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2489 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2490 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2491 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2492 TILE_SPLIT(split_equal_to_row_size) |
2493 NUM_BANKS(ADDR_SURF_16_BANK) |
2494 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2495 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2496 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2497 break;
2498 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2499 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2500 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2501 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2502 TILE_SPLIT(split_equal_to_row_size) |
2503 NUM_BANKS(ADDR_SURF_16_BANK) |
2504 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2505 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2506 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2507 break;
2508 case 8: /* 1D and 1D Array Surfaces */
2509 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2510 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2511 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2512 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2513 NUM_BANKS(ADDR_SURF_16_BANK) |
2514 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2515 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2516 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2517 break;
2518 case 9: /* Displayable maps. */
2519 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2520 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2521 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2522 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2523 NUM_BANKS(ADDR_SURF_16_BANK) |
2524 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2525 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2526 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2527 break;
2528 case 10: /* Display 8bpp. */
2529 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2530 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2531 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2532 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2533 NUM_BANKS(ADDR_SURF_16_BANK) |
2534 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2535 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2536 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2537 break;
2538 case 11: /* Display 16bpp. */
2539 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2540 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2541 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2542 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2543 NUM_BANKS(ADDR_SURF_16_BANK) |
2544 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2545 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2546 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2547 break;
2548 case 12: /* Display 32bpp. */
2549 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2550 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2551 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2552 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2553 NUM_BANKS(ADDR_SURF_16_BANK) |
2554 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2555 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2556 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2557 break;
2558 case 13: /* Thin. */
2559 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2560 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2561 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2562 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2563 NUM_BANKS(ADDR_SURF_16_BANK) |
2564 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2565 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2566 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2567 break;
2568 case 14: /* Thin 8 bpp. */
2569 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2570 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2571 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2572 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2573 NUM_BANKS(ADDR_SURF_16_BANK) |
2574 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2575 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2576 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2577 break;
2578 case 15: /* Thin 16 bpp. */
2579 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2580 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2581 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2582 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2583 NUM_BANKS(ADDR_SURF_16_BANK) |
2584 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2585 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2586 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2587 break;
2588 case 16: /* Thin 32 bpp. */
2589 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2590 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2591 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2592 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2593 NUM_BANKS(ADDR_SURF_16_BANK) |
2594 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2595 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2596 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2597 break;
2598 case 17: /* Thin 64 bpp. */
2599 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2600 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2601 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2602 TILE_SPLIT(split_equal_to_row_size) |
2603 NUM_BANKS(ADDR_SURF_16_BANK) |
2604 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2605 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2606 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2607 break;
2608 case 21: /* 8 bpp PRT. */
2609 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2610 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2611 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2612 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2613 NUM_BANKS(ADDR_SURF_16_BANK) |
2614 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2615 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2616 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2617 break;
2618 case 22: /* 16 bpp PRT */
2619 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2620 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2621 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2622 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2623 NUM_BANKS(ADDR_SURF_16_BANK) |
2624 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2625 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2626 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2627 break;
2628 case 23: /* 32 bpp PRT */
2629 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2630 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2631 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2632 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2633 NUM_BANKS(ADDR_SURF_16_BANK) |
2634 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2635 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2636 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2637 break;
2638 case 24: /* 64 bpp PRT */
2639 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2640 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2641 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2642 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2643 NUM_BANKS(ADDR_SURF_16_BANK) |
2644 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2645 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2646 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2647 break;
2648 case 25: /* 128 bpp PRT */
2649 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2650 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2651 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2652 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2653 NUM_BANKS(ADDR_SURF_8_BANK) |
2654 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2655 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2656 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2657 break;
2658 default:
2659 gb_tile_moden = 0;
2660 break;
2661 }
Jerome Glisse64d7b8b2013-04-09 11:17:08 -04002662 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
Alex Deucher0a96d722012-03-20 17:18:11 -04002663 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2664 }
Alex Deucherd0ae7fc2012-07-26 17:42:25 -04002665 } else if ((rdev->family == CHIP_VERDE) ||
Alex Deucher8b028592012-07-31 12:42:48 -04002666 (rdev->family == CHIP_OLAND) ||
2667 (rdev->family == CHIP_HAINAN)) {
Alex Deucher0a96d722012-03-20 17:18:11 -04002668 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2669 switch (reg_offset) {
2670 case 0: /* non-AA compressed depth or any compressed stencil */
2671 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2672 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2673 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2674 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2675 NUM_BANKS(ADDR_SURF_16_BANK) |
2676 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2677 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2678 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2679 break;
2680 case 1: /* 2xAA/4xAA compressed depth only */
2681 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2682 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2683 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2684 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2685 NUM_BANKS(ADDR_SURF_16_BANK) |
2686 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2687 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2688 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2689 break;
2690 case 2: /* 8xAA compressed depth only */
2691 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2692 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2693 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2694 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2695 NUM_BANKS(ADDR_SURF_16_BANK) |
2696 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2697 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2698 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2699 break;
2700 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2701 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2702 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2703 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2704 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2705 NUM_BANKS(ADDR_SURF_16_BANK) |
2706 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2707 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2708 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2709 break;
2710 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2711 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2712 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2713 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2714 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2715 NUM_BANKS(ADDR_SURF_16_BANK) |
2716 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2717 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2718 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2719 break;
2720 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2721 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2722 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2723 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2724 TILE_SPLIT(split_equal_to_row_size) |
2725 NUM_BANKS(ADDR_SURF_16_BANK) |
2726 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2727 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2728 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2729 break;
2730 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2731 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2732 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2733 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2734 TILE_SPLIT(split_equal_to_row_size) |
2735 NUM_BANKS(ADDR_SURF_16_BANK) |
2736 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2737 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2738 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2739 break;
2740 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2741 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2742 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2743 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2744 TILE_SPLIT(split_equal_to_row_size) |
2745 NUM_BANKS(ADDR_SURF_16_BANK) |
2746 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2747 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2748 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2749 break;
2750 case 8: /* 1D and 1D Array Surfaces */
2751 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2752 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2753 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2754 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2755 NUM_BANKS(ADDR_SURF_16_BANK) |
2756 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2757 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2758 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2759 break;
2760 case 9: /* Displayable maps. */
2761 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2762 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2763 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2764 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2765 NUM_BANKS(ADDR_SURF_16_BANK) |
2766 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2767 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2768 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2769 break;
2770 case 10: /* Display 8bpp. */
2771 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2772 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2773 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2774 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2775 NUM_BANKS(ADDR_SURF_16_BANK) |
2776 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2777 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2778 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2779 break;
2780 case 11: /* Display 16bpp. */
2781 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2782 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2783 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2784 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2785 NUM_BANKS(ADDR_SURF_16_BANK) |
2786 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2787 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2788 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2789 break;
2790 case 12: /* Display 32bpp. */
2791 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2792 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2793 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2794 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2795 NUM_BANKS(ADDR_SURF_16_BANK) |
2796 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2797 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2798 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2799 break;
2800 case 13: /* Thin. */
2801 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2802 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2803 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2804 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2805 NUM_BANKS(ADDR_SURF_16_BANK) |
2806 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2807 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2808 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2809 break;
2810 case 14: /* Thin 8 bpp. */
2811 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2812 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2813 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2814 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2815 NUM_BANKS(ADDR_SURF_16_BANK) |
2816 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2817 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2818 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2819 break;
2820 case 15: /* Thin 16 bpp. */
2821 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2822 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2823 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2824 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2825 NUM_BANKS(ADDR_SURF_16_BANK) |
2826 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2827 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2828 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2829 break;
2830 case 16: /* Thin 32 bpp. */
2831 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2832 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2833 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2834 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2835 NUM_BANKS(ADDR_SURF_16_BANK) |
2836 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2837 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2838 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2839 break;
2840 case 17: /* Thin 64 bpp. */
2841 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2842 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2843 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2844 TILE_SPLIT(split_equal_to_row_size) |
2845 NUM_BANKS(ADDR_SURF_16_BANK) |
2846 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2847 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2848 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2849 break;
2850 case 21: /* 8 bpp PRT. */
2851 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2852 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2853 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2854 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2855 NUM_BANKS(ADDR_SURF_16_BANK) |
2856 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2857 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2858 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2859 break;
2860 case 22: /* 16 bpp PRT */
2861 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2862 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2863 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2864 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2865 NUM_BANKS(ADDR_SURF_16_BANK) |
2866 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2867 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2868 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2869 break;
2870 case 23: /* 32 bpp PRT */
2871 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2872 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2873 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2874 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2875 NUM_BANKS(ADDR_SURF_16_BANK) |
2876 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2877 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2878 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2879 break;
2880 case 24: /* 64 bpp PRT */
2881 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2882 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2883 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2884 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2885 NUM_BANKS(ADDR_SURF_16_BANK) |
2886 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2887 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2888 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2889 break;
2890 case 25: /* 128 bpp PRT */
2891 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2892 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2893 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2894 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2895 NUM_BANKS(ADDR_SURF_8_BANK) |
2896 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2897 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2898 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2899 break;
2900 default:
2901 gb_tile_moden = 0;
2902 break;
2903 }
Jerome Glisse64d7b8b2013-04-09 11:17:08 -04002904 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
Alex Deucher0a96d722012-03-20 17:18:11 -04002905 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2906 }
2907 } else
2908 DRM_ERROR("unknown asic: 0x%x\n", rdev->family);
2909}
2910
Alex Deucher1a8ca752012-06-01 18:58:22 -04002911static void si_select_se_sh(struct radeon_device *rdev,
2912 u32 se_num, u32 sh_num)
2913{
2914 u32 data = INSTANCE_BROADCAST_WRITES;
2915
2916 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
Alex Deucher79b52d6a2013-04-18 16:26:36 -04002917 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
Alex Deucher1a8ca752012-06-01 18:58:22 -04002918 else if (se_num == 0xffffffff)
2919 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
2920 else if (sh_num == 0xffffffff)
2921 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
2922 else
2923 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
2924 WREG32(GRBM_GFX_INDEX, data);
2925}
2926
2927static u32 si_create_bitmask(u32 bit_width)
2928{
2929 u32 i, mask = 0;
2930
2931 for (i = 0; i < bit_width; i++) {
2932 mask <<= 1;
2933 mask |= 1;
2934 }
2935 return mask;
2936}
2937
2938static u32 si_get_cu_enabled(struct radeon_device *rdev, u32 cu_per_sh)
2939{
2940 u32 data, mask;
2941
2942 data = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
2943 if (data & 1)
2944 data &= INACTIVE_CUS_MASK;
2945 else
2946 data = 0;
2947 data |= RREG32(GC_USER_SHADER_ARRAY_CONFIG);
2948
2949 data >>= INACTIVE_CUS_SHIFT;
2950
2951 mask = si_create_bitmask(cu_per_sh);
2952
2953 return ~data & mask;
2954}
2955
2956static void si_setup_spi(struct radeon_device *rdev,
2957 u32 se_num, u32 sh_per_se,
2958 u32 cu_per_sh)
2959{
2960 int i, j, k;
2961 u32 data, mask, active_cu;
2962
2963 for (i = 0; i < se_num; i++) {
2964 for (j = 0; j < sh_per_se; j++) {
2965 si_select_se_sh(rdev, i, j);
2966 data = RREG32(SPI_STATIC_THREAD_MGMT_3);
2967 active_cu = si_get_cu_enabled(rdev, cu_per_sh);
2968
2969 mask = 1;
2970 for (k = 0; k < 16; k++) {
2971 mask <<= k;
2972 if (active_cu & mask) {
2973 data &= ~mask;
2974 WREG32(SPI_STATIC_THREAD_MGMT_3, data);
2975 break;
2976 }
2977 }
2978 }
2979 }
2980 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2981}
2982
2983static u32 si_get_rb_disabled(struct radeon_device *rdev,
Marek Olšák9fadb352013-12-22 02:18:00 +01002984 u32 max_rb_num_per_se,
Alex Deucher1a8ca752012-06-01 18:58:22 -04002985 u32 sh_per_se)
2986{
2987 u32 data, mask;
2988
2989 data = RREG32(CC_RB_BACKEND_DISABLE);
2990 if (data & 1)
2991 data &= BACKEND_DISABLE_MASK;
2992 else
2993 data = 0;
2994 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
2995
2996 data >>= BACKEND_DISABLE_SHIFT;
2997
Marek Olšák9fadb352013-12-22 02:18:00 +01002998 mask = si_create_bitmask(max_rb_num_per_se / sh_per_se);
Alex Deucher1a8ca752012-06-01 18:58:22 -04002999
3000 return data & mask;
3001}
3002
3003static void si_setup_rb(struct radeon_device *rdev,
3004 u32 se_num, u32 sh_per_se,
Marek Olšák9fadb352013-12-22 02:18:00 +01003005 u32 max_rb_num_per_se)
Alex Deucher1a8ca752012-06-01 18:58:22 -04003006{
3007 int i, j;
3008 u32 data, mask;
3009 u32 disabled_rbs = 0;
3010 u32 enabled_rbs = 0;
3011
3012 for (i = 0; i < se_num; i++) {
3013 for (j = 0; j < sh_per_se; j++) {
3014 si_select_se_sh(rdev, i, j);
Marek Olšák9fadb352013-12-22 02:18:00 +01003015 data = si_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
Alex Deucher1a8ca752012-06-01 18:58:22 -04003016 disabled_rbs |= data << ((i * sh_per_se + j) * TAHITI_RB_BITMAP_WIDTH_PER_SH);
3017 }
3018 }
3019 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3020
3021 mask = 1;
Marek Olšák9fadb352013-12-22 02:18:00 +01003022 for (i = 0; i < max_rb_num_per_se * se_num; i++) {
Alex Deucher1a8ca752012-06-01 18:58:22 -04003023 if (!(disabled_rbs & mask))
3024 enabled_rbs |= mask;
3025 mask <<= 1;
3026 }
3027
Marek Olšák439a1cf2013-12-22 02:18:01 +01003028 rdev->config.si.backend_enable_mask = enabled_rbs;
3029
Alex Deucher1a8ca752012-06-01 18:58:22 -04003030 for (i = 0; i < se_num; i++) {
3031 si_select_se_sh(rdev, i, 0xffffffff);
3032 data = 0;
3033 for (j = 0; j < sh_per_se; j++) {
3034 switch (enabled_rbs & 3) {
3035 case 1:
3036 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3037 break;
3038 case 2:
3039 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3040 break;
3041 case 3:
3042 default:
3043 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3044 break;
3045 }
3046 enabled_rbs >>= 2;
3047 }
3048 WREG32(PA_SC_RASTER_CONFIG, data);
3049 }
3050 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3051}
3052
Alex Deucher0a96d722012-03-20 17:18:11 -04003053static void si_gpu_init(struct radeon_device *rdev)
3054{
Alex Deucher0a96d722012-03-20 17:18:11 -04003055 u32 gb_addr_config = 0;
3056 u32 mc_shared_chmap, mc_arb_ramcfg;
Alex Deucher0a96d722012-03-20 17:18:11 -04003057 u32 sx_debug_1;
Alex Deucher0a96d722012-03-20 17:18:11 -04003058 u32 hdp_host_path_cntl;
3059 u32 tmp;
Alex Deucher6101b3a2014-08-19 11:54:15 -04003060 int i, j;
Alex Deucher0a96d722012-03-20 17:18:11 -04003061
3062 switch (rdev->family) {
3063 case CHIP_TAHITI:
3064 rdev->config.si.max_shader_engines = 2;
Alex Deucher0a96d722012-03-20 17:18:11 -04003065 rdev->config.si.max_tile_pipes = 12;
Alex Deucher1a8ca752012-06-01 18:58:22 -04003066 rdev->config.si.max_cu_per_sh = 8;
3067 rdev->config.si.max_sh_per_se = 2;
Alex Deucher0a96d722012-03-20 17:18:11 -04003068 rdev->config.si.max_backends_per_se = 4;
3069 rdev->config.si.max_texture_channel_caches = 12;
3070 rdev->config.si.max_gprs = 256;
3071 rdev->config.si.max_gs_threads = 32;
3072 rdev->config.si.max_hw_contexts = 8;
3073
3074 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3075 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
3076 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3077 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
Alex Deucher1a8ca752012-06-01 18:58:22 -04003078 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
Alex Deucher0a96d722012-03-20 17:18:11 -04003079 break;
3080 case CHIP_PITCAIRN:
3081 rdev->config.si.max_shader_engines = 2;
Alex Deucher0a96d722012-03-20 17:18:11 -04003082 rdev->config.si.max_tile_pipes = 8;
Alex Deucher1a8ca752012-06-01 18:58:22 -04003083 rdev->config.si.max_cu_per_sh = 5;
3084 rdev->config.si.max_sh_per_se = 2;
Alex Deucher0a96d722012-03-20 17:18:11 -04003085 rdev->config.si.max_backends_per_se = 4;
3086 rdev->config.si.max_texture_channel_caches = 8;
3087 rdev->config.si.max_gprs = 256;
3088 rdev->config.si.max_gs_threads = 32;
3089 rdev->config.si.max_hw_contexts = 8;
3090
3091 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3092 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
3093 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3094 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
Alex Deucher1a8ca752012-06-01 18:58:22 -04003095 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
Alex Deucher0a96d722012-03-20 17:18:11 -04003096 break;
3097 case CHIP_VERDE:
3098 default:
3099 rdev->config.si.max_shader_engines = 1;
Alex Deucher0a96d722012-03-20 17:18:11 -04003100 rdev->config.si.max_tile_pipes = 4;
Alex Deucher468ef1a2013-05-21 13:35:19 -04003101 rdev->config.si.max_cu_per_sh = 5;
Alex Deucher1a8ca752012-06-01 18:58:22 -04003102 rdev->config.si.max_sh_per_se = 2;
Alex Deucher0a96d722012-03-20 17:18:11 -04003103 rdev->config.si.max_backends_per_se = 4;
3104 rdev->config.si.max_texture_channel_caches = 4;
3105 rdev->config.si.max_gprs = 256;
3106 rdev->config.si.max_gs_threads = 32;
3107 rdev->config.si.max_hw_contexts = 8;
3108
3109 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3110 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
3111 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3112 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
Alex Deucher1a8ca752012-06-01 18:58:22 -04003113 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
Alex Deucher0a96d722012-03-20 17:18:11 -04003114 break;
Alex Deucherd0ae7fc2012-07-26 17:42:25 -04003115 case CHIP_OLAND:
3116 rdev->config.si.max_shader_engines = 1;
3117 rdev->config.si.max_tile_pipes = 4;
3118 rdev->config.si.max_cu_per_sh = 6;
3119 rdev->config.si.max_sh_per_se = 1;
3120 rdev->config.si.max_backends_per_se = 2;
3121 rdev->config.si.max_texture_channel_caches = 4;
3122 rdev->config.si.max_gprs = 256;
3123 rdev->config.si.max_gs_threads = 16;
3124 rdev->config.si.max_hw_contexts = 8;
3125
3126 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3127 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
3128 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3129 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3130 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
3131 break;
Alex Deucher8b028592012-07-31 12:42:48 -04003132 case CHIP_HAINAN:
3133 rdev->config.si.max_shader_engines = 1;
3134 rdev->config.si.max_tile_pipes = 4;
3135 rdev->config.si.max_cu_per_sh = 5;
3136 rdev->config.si.max_sh_per_se = 1;
3137 rdev->config.si.max_backends_per_se = 1;
3138 rdev->config.si.max_texture_channel_caches = 2;
3139 rdev->config.si.max_gprs = 256;
3140 rdev->config.si.max_gs_threads = 16;
3141 rdev->config.si.max_hw_contexts = 8;
3142
3143 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3144 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
3145 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3146 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3147 gb_addr_config = HAINAN_GB_ADDR_CONFIG_GOLDEN;
3148 break;
Alex Deucher0a96d722012-03-20 17:18:11 -04003149 }
3150
3151 /* Initialize HDP */
3152 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3153 WREG32((0x2c14 + j), 0x00000000);
3154 WREG32((0x2c18 + j), 0x00000000);
3155 WREG32((0x2c1c + j), 0x00000000);
3156 WREG32((0x2c20 + j), 0x00000000);
3157 WREG32((0x2c24 + j), 0x00000000);
3158 }
3159
3160 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3161
3162 evergreen_fix_pci_max_read_req_size(rdev);
3163
3164 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3165
3166 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3167 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3168
Alex Deucher0a96d722012-03-20 17:18:11 -04003169 rdev->config.si.num_tile_pipes = rdev->config.si.max_tile_pipes;
Alex Deucher0a96d722012-03-20 17:18:11 -04003170 rdev->config.si.mem_max_burst_length_bytes = 256;
3171 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3172 rdev->config.si.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3173 if (rdev->config.si.mem_row_size_in_kb > 4)
3174 rdev->config.si.mem_row_size_in_kb = 4;
3175 /* XXX use MC settings? */
3176 rdev->config.si.shader_engine_tile_size = 32;
3177 rdev->config.si.num_gpus = 1;
3178 rdev->config.si.multi_gpu_tile_size = 64;
3179
Alex Deucher1a8ca752012-06-01 18:58:22 -04003180 /* fix up row size */
3181 gb_addr_config &= ~ROW_SIZE_MASK;
Alex Deucher0a96d722012-03-20 17:18:11 -04003182 switch (rdev->config.si.mem_row_size_in_kb) {
3183 case 1:
3184 default:
3185 gb_addr_config |= ROW_SIZE(0);
3186 break;
3187 case 2:
3188 gb_addr_config |= ROW_SIZE(1);
3189 break;
3190 case 4:
3191 gb_addr_config |= ROW_SIZE(2);
3192 break;
3193 }
3194
Alex Deucher0a96d722012-03-20 17:18:11 -04003195 /* setup tiling info dword. gb_addr_config is not adequate since it does
3196 * not have bank info, so create a custom tiling dword.
3197 * bits 3:0 num_pipes
3198 * bits 7:4 num_banks
3199 * bits 11:8 group_size
3200 * bits 15:12 row_size
3201 */
3202 rdev->config.si.tile_config = 0;
3203 switch (rdev->config.si.num_tile_pipes) {
3204 case 1:
3205 rdev->config.si.tile_config |= (0 << 0);
3206 break;
3207 case 2:
3208 rdev->config.si.tile_config |= (1 << 0);
3209 break;
3210 case 4:
3211 rdev->config.si.tile_config |= (2 << 0);
3212 break;
3213 case 8:
3214 default:
3215 /* XXX what about 12? */
3216 rdev->config.si.tile_config |= (3 << 0);
3217 break;
Christian Königdca571a2012-07-31 13:48:51 +02003218 }
3219 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3220 case 0: /* four banks */
Alex Deucher1a8ca752012-06-01 18:58:22 -04003221 rdev->config.si.tile_config |= 0 << 4;
Christian Königdca571a2012-07-31 13:48:51 +02003222 break;
3223 case 1: /* eight banks */
3224 rdev->config.si.tile_config |= 1 << 4;
3225 break;
3226 case 2: /* sixteen banks */
3227 default:
3228 rdev->config.si.tile_config |= 2 << 4;
3229 break;
3230 }
Alex Deucher0a96d722012-03-20 17:18:11 -04003231 rdev->config.si.tile_config |=
3232 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3233 rdev->config.si.tile_config |=
3234 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3235
Alex Deucher0a96d722012-03-20 17:18:11 -04003236 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3237 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
Alex Deucher7c1c7c12013-04-05 10:28:08 -04003238 WREG32(DMIF_ADDR_CALC, gb_addr_config);
Alex Deucher0a96d722012-03-20 17:18:11 -04003239 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
Alex Deucher8c5fd7e2012-12-04 15:28:18 -05003240 WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
3241 WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
Alex Deucher1df0d522013-04-26 18:03:44 -04003242 if (rdev->has_uvd) {
3243 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3244 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3245 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3246 }
Alex Deucher0a96d722012-03-20 17:18:11 -04003247
Alex Deucher0a96d722012-03-20 17:18:11 -04003248 si_tiling_mode_table_init(rdev);
3249
Alex Deucher1a8ca752012-06-01 18:58:22 -04003250 si_setup_rb(rdev, rdev->config.si.max_shader_engines,
3251 rdev->config.si.max_sh_per_se,
3252 rdev->config.si.max_backends_per_se);
3253
3254 si_setup_spi(rdev, rdev->config.si.max_shader_engines,
3255 rdev->config.si.max_sh_per_se,
3256 rdev->config.si.max_cu_per_sh);
3257
Alex Deucher65fcf662014-06-02 16:13:21 -04003258 for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
3259 for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
Alex Deucher6101b3a2014-08-19 11:54:15 -04003260 rdev->config.si.active_cus +=
3261 hweight32(si_get_cu_active_bitmap(rdev, i, j));
Alex Deucher65fcf662014-06-02 16:13:21 -04003262 }
3263 }
Alex Deucher1a8ca752012-06-01 18:58:22 -04003264
Alex Deucher0a96d722012-03-20 17:18:11 -04003265 /* set HW defaults for 3D engine */
3266 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3267 ROQ_IB2_START(0x2b)));
3268 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3269
3270 sx_debug_1 = RREG32(SX_DEBUG_1);
3271 WREG32(SX_DEBUG_1, sx_debug_1);
3272
3273 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3274
3275 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_frontend) |
3276 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_backend) |
3277 SC_HIZ_TILE_FIFO_SIZE(rdev->config.si.sc_hiz_tile_fifo_size) |
3278 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.si.sc_earlyz_tile_fifo_size)));
3279
3280 WREG32(VGT_NUM_INSTANCES, 1);
3281
3282 WREG32(CP_PERFMON_CNTL, 0);
3283
3284 WREG32(SQ_CONFIG, 0);
3285
3286 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3287 FORCE_EOV_MAX_REZ_CNT(255)));
3288
3289 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3290 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3291
3292 WREG32(VGT_GS_VERTEX_REUSE, 16);
3293 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3294
3295 WREG32(CB_PERFCOUNTER0_SELECT0, 0);
3296 WREG32(CB_PERFCOUNTER0_SELECT1, 0);
3297 WREG32(CB_PERFCOUNTER1_SELECT0, 0);
3298 WREG32(CB_PERFCOUNTER1_SELECT1, 0);
3299 WREG32(CB_PERFCOUNTER2_SELECT0, 0);
3300 WREG32(CB_PERFCOUNTER2_SELECT1, 0);
3301 WREG32(CB_PERFCOUNTER3_SELECT0, 0);
3302 WREG32(CB_PERFCOUNTER3_SELECT1, 0);
3303
3304 tmp = RREG32(HDP_MISC_CNTL);
3305 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3306 WREG32(HDP_MISC_CNTL, tmp);
3307
3308 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3309 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3310
3311 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3312
3313 udelay(50);
3314}
Alex Deucherc476dde2012-03-20 17:18:12 -04003315
Alex Deucher48c0c902012-03-20 17:18:19 -04003316/*
Alex Deucher2ece2e82012-03-20 17:18:20 -04003317 * GPU scratch registers helpers function.
3318 */
3319static void si_scratch_init(struct radeon_device *rdev)
3320{
3321 int i;
3322
3323 rdev->scratch.num_reg = 7;
3324 rdev->scratch.reg_base = SCRATCH_REG0;
3325 for (i = 0; i < rdev->scratch.num_reg; i++) {
3326 rdev->scratch.free[i] = true;
3327 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3328 }
3329}
3330
3331void si_fence_ring_emit(struct radeon_device *rdev,
3332 struct radeon_fence *fence)
3333{
3334 struct radeon_ring *ring = &rdev->ring[fence->ring];
3335 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3336
3337 /* flush read cache over gart */
3338 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3339 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3340 radeon_ring_write(ring, 0);
3341 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3342 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3343 PACKET3_TC_ACTION_ENA |
3344 PACKET3_SH_KCACHE_ACTION_ENA |
3345 PACKET3_SH_ICACHE_ACTION_ENA);
3346 radeon_ring_write(ring, 0xFFFFFFFF);
3347 radeon_ring_write(ring, 0);
3348 radeon_ring_write(ring, 10); /* poll interval */
3349 /* EVENT_WRITE_EOP - flush caches, send int */
3350 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3351 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5));
Christian König5e167cd2014-06-03 20:51:46 +02003352 radeon_ring_write(ring, lower_32_bits(addr));
Alex Deucher2ece2e82012-03-20 17:18:20 -04003353 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
3354 radeon_ring_write(ring, fence->seq);
3355 radeon_ring_write(ring, 0);
3356}
3357
3358/*
3359 * IB stuff
3360 */
3361void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3362{
Christian König876dc9f2012-05-08 14:24:01 +02003363 struct radeon_ring *ring = &rdev->ring[ib->ring];
Alex Deucher2ece2e82012-03-20 17:18:20 -04003364 u32 header;
3365
Alex Deuchera85a7da42012-07-17 14:02:29 -04003366 if (ib->is_const_ib) {
3367 /* set switch buffer packet before const IB */
3368 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3369 radeon_ring_write(ring, 0);
Christian König45df6802012-07-06 16:22:55 +02003370
Alex Deucher2ece2e82012-03-20 17:18:20 -04003371 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
Alex Deuchera85a7da42012-07-17 14:02:29 -04003372 } else {
Alex Deucher89d35802012-07-17 14:02:31 -04003373 u32 next_rptr;
Alex Deuchera85a7da42012-07-17 14:02:29 -04003374 if (ring->rptr_save_reg) {
Alex Deucher89d35802012-07-17 14:02:31 -04003375 next_rptr = ring->wptr + 3 + 4 + 8;
Alex Deuchera85a7da42012-07-17 14:02:29 -04003376 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3377 radeon_ring_write(ring, ((ring->rptr_save_reg -
3378 PACKET3_SET_CONFIG_REG_START) >> 2));
3379 radeon_ring_write(ring, next_rptr);
Alex Deucher89d35802012-07-17 14:02:31 -04003380 } else if (rdev->wb.enabled) {
3381 next_rptr = ring->wptr + 5 + 4 + 8;
3382 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3383 radeon_ring_write(ring, (1 << 8));
3384 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
Christian König5e167cd2014-06-03 20:51:46 +02003385 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr));
Alex Deucher89d35802012-07-17 14:02:31 -04003386 radeon_ring_write(ring, next_rptr);
Alex Deuchera85a7da42012-07-17 14:02:29 -04003387 }
3388
Alex Deucher2ece2e82012-03-20 17:18:20 -04003389 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
Alex Deuchera85a7da42012-07-17 14:02:29 -04003390 }
Alex Deucher2ece2e82012-03-20 17:18:20 -04003391
3392 radeon_ring_write(ring, header);
3393 radeon_ring_write(ring,
3394#ifdef __BIG_ENDIAN
3395 (2 << 0) |
3396#endif
3397 (ib->gpu_addr & 0xFFFFFFFC));
3398 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
Christian König4bf3dd92012-08-06 18:57:44 +02003399 radeon_ring_write(ring, ib->length_dw |
3400 (ib->vm ? (ib->vm->id << 24) : 0));
Alex Deucher2ece2e82012-03-20 17:18:20 -04003401
Alex Deuchera85a7da42012-07-17 14:02:29 -04003402 if (!ib->is_const_ib) {
3403 /* flush read cache over gart for this vmid */
3404 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3405 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
Christian König4bf3dd92012-08-06 18:57:44 +02003406 radeon_ring_write(ring, ib->vm ? ib->vm->id : 0);
Alex Deuchera85a7da42012-07-17 14:02:29 -04003407 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3408 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3409 PACKET3_TC_ACTION_ENA |
3410 PACKET3_SH_KCACHE_ACTION_ENA |
3411 PACKET3_SH_ICACHE_ACTION_ENA);
3412 radeon_ring_write(ring, 0xFFFFFFFF);
3413 radeon_ring_write(ring, 0);
3414 radeon_ring_write(ring, 10); /* poll interval */
3415 }
Alex Deucher2ece2e82012-03-20 17:18:20 -04003416}
3417
3418/*
Alex Deucher48c0c902012-03-20 17:18:19 -04003419 * CP.
3420 */
3421static void si_cp_enable(struct radeon_device *rdev, bool enable)
3422{
3423 if (enable)
3424 WREG32(CP_ME_CNTL, 0);
3425 else {
Alex Deucher50efa512014-01-27 11:26:33 -05003426 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3427 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
Alex Deucher48c0c902012-03-20 17:18:19 -04003428 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3429 WREG32(SCRATCH_UMSK, 0);
Alex Deucher8c5fd7e2012-12-04 15:28:18 -05003430 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3431 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3432 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
Alex Deucher48c0c902012-03-20 17:18:19 -04003433 }
3434 udelay(50);
3435}
3436
3437static int si_cp_load_microcode(struct radeon_device *rdev)
3438{
Alex Deucher48c0c902012-03-20 17:18:19 -04003439 int i;
3440
Alex Deucher629bd332014-06-25 18:41:34 -04003441 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
Alex Deucher48c0c902012-03-20 17:18:19 -04003442 return -EINVAL;
3443
3444 si_cp_enable(rdev, false);
3445
Alex Deucher629bd332014-06-25 18:41:34 -04003446 if (rdev->new_fw) {
3447 const struct gfx_firmware_header_v1_0 *pfp_hdr =
3448 (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
3449 const struct gfx_firmware_header_v1_0 *ce_hdr =
3450 (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
3451 const struct gfx_firmware_header_v1_0 *me_hdr =
3452 (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
3453 const __le32 *fw_data;
3454 u32 fw_size;
Alex Deucher48c0c902012-03-20 17:18:19 -04003455
Alex Deucher629bd332014-06-25 18:41:34 -04003456 radeon_ucode_print_gfx_hdr(&pfp_hdr->header);
3457 radeon_ucode_print_gfx_hdr(&ce_hdr->header);
3458 radeon_ucode_print_gfx_hdr(&me_hdr->header);
Alex Deucher48c0c902012-03-20 17:18:19 -04003459
Alex Deucher629bd332014-06-25 18:41:34 -04003460 /* PFP */
3461 fw_data = (const __le32 *)
3462 (rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
3463 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
3464 WREG32(CP_PFP_UCODE_ADDR, 0);
3465 for (i = 0; i < fw_size; i++)
3466 WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
3467 WREG32(CP_PFP_UCODE_ADDR, 0);
3468
3469 /* CE */
3470 fw_data = (const __le32 *)
3471 (rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
3472 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
3473 WREG32(CP_CE_UCODE_ADDR, 0);
3474 for (i = 0; i < fw_size; i++)
3475 WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
3476 WREG32(CP_CE_UCODE_ADDR, 0);
3477
3478 /* ME */
3479 fw_data = (const __be32 *)
3480 (rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
3481 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
3482 WREG32(CP_ME_RAM_WADDR, 0);
3483 for (i = 0; i < fw_size; i++)
3484 WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++));
3485 WREG32(CP_ME_RAM_WADDR, 0);
3486 } else {
3487 const __be32 *fw_data;
3488
3489 /* PFP */
3490 fw_data = (const __be32 *)rdev->pfp_fw->data;
3491 WREG32(CP_PFP_UCODE_ADDR, 0);
3492 for (i = 0; i < SI_PFP_UCODE_SIZE; i++)
3493 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3494 WREG32(CP_PFP_UCODE_ADDR, 0);
3495
3496 /* CE */
3497 fw_data = (const __be32 *)rdev->ce_fw->data;
3498 WREG32(CP_CE_UCODE_ADDR, 0);
3499 for (i = 0; i < SI_CE_UCODE_SIZE; i++)
3500 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3501 WREG32(CP_CE_UCODE_ADDR, 0);
3502
3503 /* ME */
3504 fw_data = (const __be32 *)rdev->me_fw->data;
3505 WREG32(CP_ME_RAM_WADDR, 0);
3506 for (i = 0; i < SI_PM4_UCODE_SIZE; i++)
3507 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3508 WREG32(CP_ME_RAM_WADDR, 0);
3509 }
Alex Deucher48c0c902012-03-20 17:18:19 -04003510
3511 WREG32(CP_PFP_UCODE_ADDR, 0);
3512 WREG32(CP_CE_UCODE_ADDR, 0);
3513 WREG32(CP_ME_RAM_WADDR, 0);
3514 WREG32(CP_ME_RAM_RADDR, 0);
3515 return 0;
3516}
3517
3518static int si_cp_start(struct radeon_device *rdev)
3519{
3520 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3521 int r, i;
3522
3523 r = radeon_ring_lock(rdev, ring, 7 + 4);
3524 if (r) {
3525 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3526 return r;
3527 }
3528 /* init the CP */
3529 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3530 radeon_ring_write(ring, 0x1);
3531 radeon_ring_write(ring, 0x0);
3532 radeon_ring_write(ring, rdev->config.si.max_hw_contexts - 1);
3533 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3534 radeon_ring_write(ring, 0);
3535 radeon_ring_write(ring, 0);
3536
3537 /* init the CE partitions */
3538 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3539 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3540 radeon_ring_write(ring, 0xc000);
3541 radeon_ring_write(ring, 0xe000);
Michel Dänzer1538a9e2014-08-18 17:34:55 +09003542 radeon_ring_unlock_commit(rdev, ring, false);
Alex Deucher48c0c902012-03-20 17:18:19 -04003543
3544 si_cp_enable(rdev, true);
3545
3546 r = radeon_ring_lock(rdev, ring, si_default_size + 10);
3547 if (r) {
3548 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3549 return r;
3550 }
3551
3552 /* setup clear context state */
3553 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3554 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3555
3556 for (i = 0; i < si_default_size; i++)
3557 radeon_ring_write(ring, si_default_state[i]);
3558
3559 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3560 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3561
3562 /* set clear context state */
3563 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3564 radeon_ring_write(ring, 0);
3565
3566 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3567 radeon_ring_write(ring, 0x00000316);
3568 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3569 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3570
Michel Dänzer1538a9e2014-08-18 17:34:55 +09003571 radeon_ring_unlock_commit(rdev, ring, false);
Alex Deucher48c0c902012-03-20 17:18:19 -04003572
3573 for (i = RADEON_RING_TYPE_GFX_INDEX; i <= CAYMAN_RING_TYPE_CP2_INDEX; ++i) {
3574 ring = &rdev->ring[i];
3575 r = radeon_ring_lock(rdev, ring, 2);
3576
3577 /* clear the compute context state */
3578 radeon_ring_write(ring, PACKET3_COMPUTE(PACKET3_CLEAR_STATE, 0));
3579 radeon_ring_write(ring, 0);
3580
Michel Dänzer1538a9e2014-08-18 17:34:55 +09003581 radeon_ring_unlock_commit(rdev, ring, false);
Alex Deucher48c0c902012-03-20 17:18:19 -04003582 }
3583
3584 return 0;
3585}
3586
3587static void si_cp_fini(struct radeon_device *rdev)
3588{
Christian König45df6802012-07-06 16:22:55 +02003589 struct radeon_ring *ring;
Alex Deucher48c0c902012-03-20 17:18:19 -04003590 si_cp_enable(rdev, false);
Christian König45df6802012-07-06 16:22:55 +02003591
3592 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3593 radeon_ring_fini(rdev, ring);
3594 radeon_scratch_free(rdev, ring->rptr_save_reg);
3595
3596 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3597 radeon_ring_fini(rdev, ring);
3598 radeon_scratch_free(rdev, ring->rptr_save_reg);
3599
3600 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3601 radeon_ring_fini(rdev, ring);
3602 radeon_scratch_free(rdev, ring->rptr_save_reg);
Alex Deucher48c0c902012-03-20 17:18:19 -04003603}
3604
3605static int si_cp_resume(struct radeon_device *rdev)
3606{
3607 struct radeon_ring *ring;
3608 u32 tmp;
3609 u32 rb_bufsz;
3610 int r;
3611
Alex Deucher811e4d52013-09-03 13:31:33 -04003612 si_enable_gui_idle_interrupt(rdev, false);
3613
Alex Deucher48c0c902012-03-20 17:18:19 -04003614 WREG32(CP_SEM_WAIT_TIMER, 0x0);
3615 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3616
3617 /* Set the write pointer delay */
3618 WREG32(CP_RB_WPTR_DELAY, 0);
3619
3620 WREG32(CP_DEBUG, 0);
3621 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3622
3623 /* ring 0 - compute and gfx */
3624 /* Set ring buffer size */
3625 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
Daniel Vetterb72a8922013-07-10 14:11:59 +02003626 rb_bufsz = order_base_2(ring->ring_size / 8);
3627 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
Alex Deucher48c0c902012-03-20 17:18:19 -04003628#ifdef __BIG_ENDIAN
3629 tmp |= BUF_SWAP_32BIT;
3630#endif
3631 WREG32(CP_RB0_CNTL, tmp);
3632
3633 /* Initialize the ring buffer's read and write pointers */
3634 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3635 ring->wptr = 0;
3636 WREG32(CP_RB0_WPTR, ring->wptr);
3637
Adam Buchbinder48fc7f72012-09-19 21:48:00 -04003638 /* set the wb address whether it's enabled or not */
Alex Deucher48c0c902012-03-20 17:18:19 -04003639 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3640 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3641
3642 if (rdev->wb.enabled)
3643 WREG32(SCRATCH_UMSK, 0xff);
3644 else {
3645 tmp |= RB_NO_UPDATE;
3646 WREG32(SCRATCH_UMSK, 0);
3647 }
3648
3649 mdelay(1);
3650 WREG32(CP_RB0_CNTL, tmp);
3651
3652 WREG32(CP_RB0_BASE, ring->gpu_addr >> 8);
3653
Alex Deucher48c0c902012-03-20 17:18:19 -04003654 /* ring1 - compute only */
3655 /* Set ring buffer size */
3656 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
Daniel Vetterb72a8922013-07-10 14:11:59 +02003657 rb_bufsz = order_base_2(ring->ring_size / 8);
3658 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
Alex Deucher48c0c902012-03-20 17:18:19 -04003659#ifdef __BIG_ENDIAN
3660 tmp |= BUF_SWAP_32BIT;
3661#endif
3662 WREG32(CP_RB1_CNTL, tmp);
3663
3664 /* Initialize the ring buffer's read and write pointers */
3665 WREG32(CP_RB1_CNTL, tmp | RB_RPTR_WR_ENA);
3666 ring->wptr = 0;
3667 WREG32(CP_RB1_WPTR, ring->wptr);
3668
Adam Buchbinder48fc7f72012-09-19 21:48:00 -04003669 /* set the wb address whether it's enabled or not */
Alex Deucher48c0c902012-03-20 17:18:19 -04003670 WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC);
3671 WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFF);
3672
3673 mdelay(1);
3674 WREG32(CP_RB1_CNTL, tmp);
3675
3676 WREG32(CP_RB1_BASE, ring->gpu_addr >> 8);
3677
Alex Deucher48c0c902012-03-20 17:18:19 -04003678 /* ring2 - compute only */
3679 /* Set ring buffer size */
3680 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
Daniel Vetterb72a8922013-07-10 14:11:59 +02003681 rb_bufsz = order_base_2(ring->ring_size / 8);
3682 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
Alex Deucher48c0c902012-03-20 17:18:19 -04003683#ifdef __BIG_ENDIAN
3684 tmp |= BUF_SWAP_32BIT;
3685#endif
3686 WREG32(CP_RB2_CNTL, tmp);
3687
3688 /* Initialize the ring buffer's read and write pointers */
3689 WREG32(CP_RB2_CNTL, tmp | RB_RPTR_WR_ENA);
3690 ring->wptr = 0;
3691 WREG32(CP_RB2_WPTR, ring->wptr);
3692
Adam Buchbinder48fc7f72012-09-19 21:48:00 -04003693 /* set the wb address whether it's enabled or not */
Alex Deucher48c0c902012-03-20 17:18:19 -04003694 WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC);
3695 WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFF);
3696
3697 mdelay(1);
3698 WREG32(CP_RB2_CNTL, tmp);
3699
3700 WREG32(CP_RB2_BASE, ring->gpu_addr >> 8);
3701
Alex Deucher48c0c902012-03-20 17:18:19 -04003702 /* start the rings */
3703 si_cp_start(rdev);
3704 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
3705 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = true;
3706 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = true;
3707 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3708 if (r) {
3709 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3710 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3711 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3712 return r;
3713 }
3714 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP1_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
3715 if (r) {
3716 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3717 }
3718 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP2_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
3719 if (r) {
3720 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3721 }
3722
Alex Deucher811e4d52013-09-03 13:31:33 -04003723 si_enable_gui_idle_interrupt(rdev, true);
3724
Alex Deucher50efa512014-01-27 11:26:33 -05003725 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3726 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
3727
Alex Deucher48c0c902012-03-20 17:18:19 -04003728 return 0;
3729}
3730
Christian König2483b4e2013-08-13 11:56:54 +02003731u32 si_gpu_check_soft_reset(struct radeon_device *rdev)
Alex Deucher014bb202013-01-18 19:36:20 -05003732{
3733 u32 reset_mask = 0;
3734 u32 tmp;
3735
3736 /* GRBM_STATUS */
3737 tmp = RREG32(GRBM_STATUS);
3738 if (tmp & (PA_BUSY | SC_BUSY |
3739 BCI_BUSY | SX_BUSY |
3740 TA_BUSY | VGT_BUSY |
3741 DB_BUSY | CB_BUSY |
3742 GDS_BUSY | SPI_BUSY |
3743 IA_BUSY | IA_BUSY_NO_DMA))
3744 reset_mask |= RADEON_RESET_GFX;
3745
3746 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3747 CP_BUSY | CP_COHERENCY_BUSY))
3748 reset_mask |= RADEON_RESET_CP;
3749
3750 if (tmp & GRBM_EE_BUSY)
3751 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3752
3753 /* GRBM_STATUS2 */
3754 tmp = RREG32(GRBM_STATUS2);
3755 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3756 reset_mask |= RADEON_RESET_RLC;
3757
3758 /* DMA_STATUS_REG 0 */
3759 tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
3760 if (!(tmp & DMA_IDLE))
3761 reset_mask |= RADEON_RESET_DMA;
3762
3763 /* DMA_STATUS_REG 1 */
3764 tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
3765 if (!(tmp & DMA_IDLE))
3766 reset_mask |= RADEON_RESET_DMA1;
3767
3768 /* SRBM_STATUS2 */
3769 tmp = RREG32(SRBM_STATUS2);
3770 if (tmp & DMA_BUSY)
3771 reset_mask |= RADEON_RESET_DMA;
3772
3773 if (tmp & DMA1_BUSY)
3774 reset_mask |= RADEON_RESET_DMA1;
3775
3776 /* SRBM_STATUS */
3777 tmp = RREG32(SRBM_STATUS);
3778
3779 if (tmp & IH_BUSY)
3780 reset_mask |= RADEON_RESET_IH;
3781
3782 if (tmp & SEM_BUSY)
3783 reset_mask |= RADEON_RESET_SEM;
3784
3785 if (tmp & GRBM_RQ_PENDING)
3786 reset_mask |= RADEON_RESET_GRBM;
3787
3788 if (tmp & VMC_BUSY)
3789 reset_mask |= RADEON_RESET_VMC;
3790
3791 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3792 MCC_BUSY | MCD_BUSY))
3793 reset_mask |= RADEON_RESET_MC;
3794
3795 if (evergreen_is_display_hung(rdev))
3796 reset_mask |= RADEON_RESET_DISPLAY;
3797
3798 /* VM_L2_STATUS */
3799 tmp = RREG32(VM_L2_STATUS);
3800 if (tmp & L2_BUSY)
3801 reset_mask |= RADEON_RESET_VMC;
3802
Alex Deucherd808fc82013-02-28 10:03:08 -05003803 /* Skip MC reset as it's mostly likely not hung, just busy */
3804 if (reset_mask & RADEON_RESET_MC) {
3805 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3806 reset_mask &= ~RADEON_RESET_MC;
3807 }
3808
Alex Deucher014bb202013-01-18 19:36:20 -05003809 return reset_mask;
3810}
3811
3812static void si_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
Alex Deucher06bc6df2013-01-03 13:15:30 -05003813{
3814 struct evergreen_mc_save save;
Alex Deucher1c534672013-01-18 15:08:38 -05003815 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3816 u32 tmp;
Alex Deucher19fc42e2013-01-14 11:04:39 -05003817
Alex Deucher06bc6df2013-01-03 13:15:30 -05003818 if (reset_mask == 0)
Alex Deucher014bb202013-01-18 19:36:20 -05003819 return;
Alex Deucher06bc6df2013-01-03 13:15:30 -05003820
3821 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3822
Alex Deucher1c534672013-01-18 15:08:38 -05003823 evergreen_print_gpu_status_regs(rdev);
Alex Deucher06bc6df2013-01-03 13:15:30 -05003824 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
3825 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
3826 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3827 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
3828
Alex Deuchera6f4ae82013-10-02 14:50:57 -04003829 /* disable PG/CG */
3830 si_fini_pg(rdev);
3831 si_fini_cg(rdev);
3832
3833 /* stop the rlc */
3834 si_rlc_stop(rdev);
3835
Alex Deucher1c534672013-01-18 15:08:38 -05003836 /* Disable CP parsing/prefetching */
3837 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
3838
3839 if (reset_mask & RADEON_RESET_DMA) {
3840 /* dma0 */
3841 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
3842 tmp &= ~DMA_RB_ENABLE;
3843 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
Alex Deucher014bb202013-01-18 19:36:20 -05003844 }
3845 if (reset_mask & RADEON_RESET_DMA1) {
Alex Deucher1c534672013-01-18 15:08:38 -05003846 /* dma1 */
3847 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
3848 tmp &= ~DMA_RB_ENABLE;
3849 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
3850 }
3851
Alex Deucherf770d782013-01-23 19:00:25 -05003852 udelay(50);
3853
3854 evergreen_mc_stop(rdev, &save);
3855 if (evergreen_mc_wait_for_idle(rdev)) {
3856 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3857 }
3858
Alex Deucher1c534672013-01-18 15:08:38 -05003859 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP)) {
3860 grbm_soft_reset = SOFT_RESET_CB |
3861 SOFT_RESET_DB |
3862 SOFT_RESET_GDS |
3863 SOFT_RESET_PA |
3864 SOFT_RESET_SC |
3865 SOFT_RESET_BCI |
3866 SOFT_RESET_SPI |
3867 SOFT_RESET_SX |
3868 SOFT_RESET_TC |
3869 SOFT_RESET_TA |
3870 SOFT_RESET_VGT |
3871 SOFT_RESET_IA;
3872 }
3873
3874 if (reset_mask & RADEON_RESET_CP) {
3875 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
3876
3877 srbm_soft_reset |= SOFT_RESET_GRBM;
3878 }
Alex Deucher06bc6df2013-01-03 13:15:30 -05003879
3880 if (reset_mask & RADEON_RESET_DMA)
Alex Deucher014bb202013-01-18 19:36:20 -05003881 srbm_soft_reset |= SOFT_RESET_DMA;
3882
3883 if (reset_mask & RADEON_RESET_DMA1)
3884 srbm_soft_reset |= SOFT_RESET_DMA1;
3885
3886 if (reset_mask & RADEON_RESET_DISPLAY)
3887 srbm_soft_reset |= SOFT_RESET_DC;
3888
3889 if (reset_mask & RADEON_RESET_RLC)
3890 grbm_soft_reset |= SOFT_RESET_RLC;
3891
3892 if (reset_mask & RADEON_RESET_SEM)
3893 srbm_soft_reset |= SOFT_RESET_SEM;
3894
3895 if (reset_mask & RADEON_RESET_IH)
3896 srbm_soft_reset |= SOFT_RESET_IH;
3897
3898 if (reset_mask & RADEON_RESET_GRBM)
3899 srbm_soft_reset |= SOFT_RESET_GRBM;
3900
3901 if (reset_mask & RADEON_RESET_VMC)
3902 srbm_soft_reset |= SOFT_RESET_VMC;
3903
3904 if (reset_mask & RADEON_RESET_MC)
3905 srbm_soft_reset |= SOFT_RESET_MC;
Alex Deucher1c534672013-01-18 15:08:38 -05003906
3907 if (grbm_soft_reset) {
3908 tmp = RREG32(GRBM_SOFT_RESET);
3909 tmp |= grbm_soft_reset;
3910 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3911 WREG32(GRBM_SOFT_RESET, tmp);
3912 tmp = RREG32(GRBM_SOFT_RESET);
3913
3914 udelay(50);
3915
3916 tmp &= ~grbm_soft_reset;
3917 WREG32(GRBM_SOFT_RESET, tmp);
3918 tmp = RREG32(GRBM_SOFT_RESET);
3919 }
3920
3921 if (srbm_soft_reset) {
3922 tmp = RREG32(SRBM_SOFT_RESET);
3923 tmp |= srbm_soft_reset;
3924 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3925 WREG32(SRBM_SOFT_RESET, tmp);
3926 tmp = RREG32(SRBM_SOFT_RESET);
3927
3928 udelay(50);
3929
3930 tmp &= ~srbm_soft_reset;
3931 WREG32(SRBM_SOFT_RESET, tmp);
3932 tmp = RREG32(SRBM_SOFT_RESET);
3933 }
Alex Deucher06bc6df2013-01-03 13:15:30 -05003934
3935 /* Wait a little for things to settle down */
3936 udelay(50);
3937
Alex Deucherc476dde2012-03-20 17:18:12 -04003938 evergreen_mc_resume(rdev, &save);
Alex Deucher1c534672013-01-18 15:08:38 -05003939 udelay(50);
Alex Deucher410a3412013-01-18 13:05:39 -05003940
Alex Deucher1c534672013-01-18 15:08:38 -05003941 evergreen_print_gpu_status_regs(rdev);
Alex Deucherc476dde2012-03-20 17:18:12 -04003942}
3943
Alex Deucher4a5c8ea2013-11-15 16:35:55 -05003944static void si_set_clk_bypass_mode(struct radeon_device *rdev)
3945{
3946 u32 tmp, i;
3947
3948 tmp = RREG32(CG_SPLL_FUNC_CNTL);
3949 tmp |= SPLL_BYPASS_EN;
3950 WREG32(CG_SPLL_FUNC_CNTL, tmp);
3951
3952 tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
3953 tmp |= SPLL_CTLREQ_CHG;
3954 WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
3955
3956 for (i = 0; i < rdev->usec_timeout; i++) {
3957 if (RREG32(SPLL_STATUS) & SPLL_CHG_STATUS)
3958 break;
3959 udelay(1);
3960 }
3961
3962 tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
3963 tmp &= ~(SPLL_CTLREQ_CHG | SCLK_MUX_UPDATE);
3964 WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
3965
3966 tmp = RREG32(MPLL_CNTL_MODE);
3967 tmp &= ~MPLL_MCLK_SEL;
3968 WREG32(MPLL_CNTL_MODE, tmp);
3969}
3970
3971static void si_spll_powerdown(struct radeon_device *rdev)
3972{
3973 u32 tmp;
3974
3975 tmp = RREG32(SPLL_CNTL_MODE);
3976 tmp |= SPLL_SW_DIR_CONTROL;
3977 WREG32(SPLL_CNTL_MODE, tmp);
3978
3979 tmp = RREG32(CG_SPLL_FUNC_CNTL);
3980 tmp |= SPLL_RESET;
3981 WREG32(CG_SPLL_FUNC_CNTL, tmp);
3982
3983 tmp = RREG32(CG_SPLL_FUNC_CNTL);
3984 tmp |= SPLL_SLEEP;
3985 WREG32(CG_SPLL_FUNC_CNTL, tmp);
3986
3987 tmp = RREG32(SPLL_CNTL_MODE);
3988 tmp &= ~SPLL_SW_DIR_CONTROL;
3989 WREG32(SPLL_CNTL_MODE, tmp);
3990}
3991
3992static void si_gpu_pci_config_reset(struct radeon_device *rdev)
3993{
3994 struct evergreen_mc_save save;
3995 u32 tmp, i;
3996
3997 dev_info(rdev->dev, "GPU pci config reset\n");
3998
3999 /* disable dpm? */
4000
4001 /* disable cg/pg */
4002 si_fini_pg(rdev);
4003 si_fini_cg(rdev);
4004
4005 /* Disable CP parsing/prefetching */
4006 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
4007 /* dma0 */
4008 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
4009 tmp &= ~DMA_RB_ENABLE;
4010 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
4011 /* dma1 */
4012 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
4013 tmp &= ~DMA_RB_ENABLE;
4014 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
4015 /* XXX other engines? */
4016
4017 /* halt the rlc, disable cp internal ints */
4018 si_rlc_stop(rdev);
4019
4020 udelay(50);
4021
4022 /* disable mem access */
4023 evergreen_mc_stop(rdev, &save);
4024 if (evergreen_mc_wait_for_idle(rdev)) {
4025 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
4026 }
4027
4028 /* set mclk/sclk to bypass */
4029 si_set_clk_bypass_mode(rdev);
4030 /* powerdown spll */
4031 si_spll_powerdown(rdev);
4032 /* disable BM */
4033 pci_clear_master(rdev->pdev);
4034 /* reset */
4035 radeon_pci_config_reset(rdev);
4036 /* wait for asic to come out of reset */
4037 for (i = 0; i < rdev->usec_timeout; i++) {
4038 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
4039 break;
4040 udelay(1);
4041 }
4042}
4043
Alex Deucherc476dde2012-03-20 17:18:12 -04004044int si_asic_reset(struct radeon_device *rdev)
4045{
Alex Deucher014bb202013-01-18 19:36:20 -05004046 u32 reset_mask;
4047
4048 reset_mask = si_gpu_check_soft_reset(rdev);
4049
4050 if (reset_mask)
4051 r600_set_bios_scratch_engine_hung(rdev, true);
4052
Alex Deucher4a5c8ea2013-11-15 16:35:55 -05004053 /* try soft reset */
Alex Deucher014bb202013-01-18 19:36:20 -05004054 si_gpu_soft_reset(rdev, reset_mask);
4055
4056 reset_mask = si_gpu_check_soft_reset(rdev);
4057
Alex Deucher4a5c8ea2013-11-15 16:35:55 -05004058 /* try pci config reset */
4059 if (reset_mask && radeon_hard_reset)
4060 si_gpu_pci_config_reset(rdev);
4061
4062 reset_mask = si_gpu_check_soft_reset(rdev);
4063
Alex Deucher014bb202013-01-18 19:36:20 -05004064 if (!reset_mask)
4065 r600_set_bios_scratch_engine_hung(rdev, false);
4066
4067 return 0;
Alex Deucherc476dde2012-03-20 17:18:12 -04004068}
4069
Alex Deucher123bc182013-01-24 11:37:19 -05004070/**
4071 * si_gfx_is_lockup - Check if the GFX engine is locked up
4072 *
4073 * @rdev: radeon_device pointer
4074 * @ring: radeon_ring structure holding ring information
4075 *
4076 * Check if the GFX engine is locked up.
4077 * Returns true if the engine appears to be locked up, false if not.
4078 */
4079bool si_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
4080{
4081 u32 reset_mask = si_gpu_check_soft_reset(rdev);
4082
4083 if (!(reset_mask & (RADEON_RESET_GFX |
4084 RADEON_RESET_COMPUTE |
4085 RADEON_RESET_CP))) {
Christian Königff212f22014-02-18 14:52:33 +01004086 radeon_ring_lockup_update(rdev, ring);
Alex Deucher123bc182013-01-24 11:37:19 -05004087 return false;
4088 }
Alex Deucher123bc182013-01-24 11:37:19 -05004089 return radeon_ring_test_lockup(rdev, ring);
4090}
4091
Alex Deucherd2800ee2012-03-20 17:18:13 -04004092/* MC */
4093static void si_mc_program(struct radeon_device *rdev)
4094{
4095 struct evergreen_mc_save save;
4096 u32 tmp;
4097 int i, j;
4098
4099 /* Initialize HDP */
4100 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
4101 WREG32((0x2c14 + j), 0x00000000);
4102 WREG32((0x2c18 + j), 0x00000000);
4103 WREG32((0x2c1c + j), 0x00000000);
4104 WREG32((0x2c20 + j), 0x00000000);
4105 WREG32((0x2c24 + j), 0x00000000);
4106 }
4107 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
4108
4109 evergreen_mc_stop(rdev, &save);
4110 if (radeon_mc_wait_for_idle(rdev)) {
4111 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4112 }
Alex Deucher51535502012-08-30 14:34:30 -04004113 if (!ASIC_IS_NODCE(rdev))
4114 /* Lockout access through VGA aperture*/
4115 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
Alex Deucherd2800ee2012-03-20 17:18:13 -04004116 /* Update configuration */
4117 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
4118 rdev->mc.vram_start >> 12);
4119 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
4120 rdev->mc.vram_end >> 12);
4121 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
4122 rdev->vram_scratch.gpu_addr >> 12);
4123 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
4124 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
4125 WREG32(MC_VM_FB_LOCATION, tmp);
4126 /* XXX double check these! */
4127 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
4128 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
4129 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
4130 WREG32(MC_VM_AGP_BASE, 0);
4131 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
4132 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
4133 if (radeon_mc_wait_for_idle(rdev)) {
4134 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4135 }
4136 evergreen_mc_resume(rdev, &save);
Alex Deucher51535502012-08-30 14:34:30 -04004137 if (!ASIC_IS_NODCE(rdev)) {
4138 /* we need to own VRAM, so turn off the VGA renderer here
4139 * to stop it overwriting our objects */
4140 rv515_vga_render_disable(rdev);
4141 }
Alex Deucherd2800ee2012-03-20 17:18:13 -04004142}
4143
Alex Deucher1c491652013-04-09 12:45:26 -04004144void si_vram_gtt_location(struct radeon_device *rdev,
4145 struct radeon_mc *mc)
Alex Deucherd2800ee2012-03-20 17:18:13 -04004146{
4147 if (mc->mc_vram_size > 0xFFC0000000ULL) {
4148 /* leave room for at least 1024M GTT */
4149 dev_warn(rdev->dev, "limiting VRAM\n");
4150 mc->real_vram_size = 0xFFC0000000ULL;
4151 mc->mc_vram_size = 0xFFC0000000ULL;
4152 }
Alex Deucher9ed8b1f2013-04-08 11:13:01 -04004153 radeon_vram_location(rdev, &rdev->mc, 0);
Alex Deucherd2800ee2012-03-20 17:18:13 -04004154 rdev->mc.gtt_base_align = 0;
Alex Deucher9ed8b1f2013-04-08 11:13:01 -04004155 radeon_gtt_location(rdev, mc);
Alex Deucherd2800ee2012-03-20 17:18:13 -04004156}
4157
4158static int si_mc_init(struct radeon_device *rdev)
4159{
4160 u32 tmp;
4161 int chansize, numchan;
4162
4163 /* Get VRAM informations */
4164 rdev->mc.vram_is_ddr = true;
4165 tmp = RREG32(MC_ARB_RAMCFG);
4166 if (tmp & CHANSIZE_OVERRIDE) {
4167 chansize = 16;
4168 } else if (tmp & CHANSIZE_MASK) {
4169 chansize = 64;
4170 } else {
4171 chansize = 32;
4172 }
4173 tmp = RREG32(MC_SHARED_CHMAP);
4174 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
4175 case 0:
4176 default:
4177 numchan = 1;
4178 break;
4179 case 1:
4180 numchan = 2;
4181 break;
4182 case 2:
4183 numchan = 4;
4184 break;
4185 case 3:
4186 numchan = 8;
4187 break;
4188 case 4:
4189 numchan = 3;
4190 break;
4191 case 5:
4192 numchan = 6;
4193 break;
4194 case 6:
4195 numchan = 10;
4196 break;
4197 case 7:
4198 numchan = 12;
4199 break;
4200 case 8:
4201 numchan = 16;
4202 break;
4203 }
4204 rdev->mc.vram_width = numchan * chansize;
4205 /* Could aper size report 0 ? */
4206 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
4207 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
4208 /* size in MB on si */
Alex Deucher0ca223b2013-12-03 09:24:30 -05004209 tmp = RREG32(CONFIG_MEMSIZE);
4210 /* some boards may have garbage in the upper 16 bits */
4211 if (tmp & 0xffff0000) {
4212 DRM_INFO("Probable bad vram size: 0x%08x\n", tmp);
4213 if (tmp & 0xffff)
4214 tmp &= 0xffff;
4215 }
4216 rdev->mc.mc_vram_size = tmp * 1024ULL * 1024ULL;
4217 rdev->mc.real_vram_size = rdev->mc.mc_vram_size;
Alex Deucherd2800ee2012-03-20 17:18:13 -04004218 rdev->mc.visible_vram_size = rdev->mc.aper_size;
4219 si_vram_gtt_location(rdev, &rdev->mc);
4220 radeon_update_bandwidth_info(rdev);
4221
4222 return 0;
4223}
4224
4225/*
4226 * GART
4227 */
4228void si_pcie_gart_tlb_flush(struct radeon_device *rdev)
4229{
4230 /* flush hdp cache */
4231 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
4232
4233 /* bits 0-15 are the VM contexts0-15 */
4234 WREG32(VM_INVALIDATE_REQUEST, 1);
4235}
4236
Lauri Kasanen1109ca02012-08-31 13:43:50 -04004237static int si_pcie_gart_enable(struct radeon_device *rdev)
Alex Deucherd2800ee2012-03-20 17:18:13 -04004238{
4239 int r, i;
4240
4241 if (rdev->gart.robj == NULL) {
4242 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
4243 return -EINVAL;
4244 }
4245 r = radeon_gart_table_vram_pin(rdev);
4246 if (r)
4247 return r;
Alex Deucherd2800ee2012-03-20 17:18:13 -04004248 /* Setup TLB control */
4249 WREG32(MC_VM_MX_L1_TLB_CNTL,
4250 (0xA << 7) |
4251 ENABLE_L1_TLB |
Christian Königec3dbbc2014-05-10 12:17:55 +02004252 ENABLE_L1_FRAGMENT_PROCESSING |
Alex Deucherd2800ee2012-03-20 17:18:13 -04004253 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4254 ENABLE_ADVANCED_DRIVER_MODEL |
4255 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4256 /* Setup L2 cache */
4257 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
Christian Königec3dbbc2014-05-10 12:17:55 +02004258 ENABLE_L2_FRAGMENT_PROCESSING |
Alex Deucherd2800ee2012-03-20 17:18:13 -04004259 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4260 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4261 EFFECTIVE_L2_QUEUE_SIZE(7) |
4262 CONTEXT1_IDENTITY_ACCESS_MODE(1));
4263 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
4264 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
Christian Königec3dbbc2014-05-10 12:17:55 +02004265 BANK_SELECT(4) |
4266 L2_CACHE_BIGK_FRAGMENT_SIZE(4));
Alex Deucherd2800ee2012-03-20 17:18:13 -04004267 /* setup context0 */
4268 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
4269 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
4270 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
4271 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
4272 (u32)(rdev->dummy_page.addr >> 12));
4273 WREG32(VM_CONTEXT0_CNTL2, 0);
4274 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
4275 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
4276
4277 WREG32(0x15D4, 0);
4278 WREG32(0x15D8, 0);
4279 WREG32(0x15DC, 0);
4280
4281 /* empty context1-15 */
Alex Deucherd2800ee2012-03-20 17:18:13 -04004282 /* set vm size, must be a multiple of 4 */
4283 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
Alex Deucherc21b3282012-06-28 17:53:07 -04004284 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
Alex Deucher23d4f1f2012-10-08 09:45:46 -04004285 /* Assign the pt base to something valid for now; the pts used for
4286 * the VMs are determined by the application and setup and assigned
4287 * on the fly in the vm part of radeon_gart.c
4288 */
Alex Deucherd2800ee2012-03-20 17:18:13 -04004289 for (i = 1; i < 16; i++) {
4290 if (i < 8)
4291 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
4292 rdev->gart.table_addr >> 12);
4293 else
4294 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
4295 rdev->gart.table_addr >> 12);
4296 }
4297
4298 /* enable context1-15 */
4299 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
4300 (u32)(rdev->dummy_page.addr >> 12));
Christian Königae133a12012-09-18 15:30:44 -04004301 WREG32(VM_CONTEXT1_CNTL2, 4);
Dmitry Cherkasovfa87e622012-09-17 19:36:19 +02004302 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
Christian König4510fb92014-06-05 23:56:50 -04004303 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
Christian Königae133a12012-09-18 15:30:44 -04004304 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4305 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4306 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4307 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4308 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
4309 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
4310 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
4311 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
4312 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
4313 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
4314 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4315 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
Alex Deucherd2800ee2012-03-20 17:18:13 -04004316
4317 si_pcie_gart_tlb_flush(rdev);
4318 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
4319 (unsigned)(rdev->mc.gtt_size >> 20),
4320 (unsigned long long)rdev->gart.table_addr);
4321 rdev->gart.ready = true;
4322 return 0;
4323}
4324
Lauri Kasanen1109ca02012-08-31 13:43:50 -04004325static void si_pcie_gart_disable(struct radeon_device *rdev)
Alex Deucherd2800ee2012-03-20 17:18:13 -04004326{
4327 /* Disable all tables */
4328 WREG32(VM_CONTEXT0_CNTL, 0);
4329 WREG32(VM_CONTEXT1_CNTL, 0);
4330 /* Setup TLB control */
4331 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4332 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4333 /* Setup L2 cache */
4334 WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4335 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4336 EFFECTIVE_L2_QUEUE_SIZE(7) |
4337 CONTEXT1_IDENTITY_ACCESS_MODE(1));
4338 WREG32(VM_L2_CNTL2, 0);
4339 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4340 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
4341 radeon_gart_table_vram_unpin(rdev);
4342}
4343
Lauri Kasanen1109ca02012-08-31 13:43:50 -04004344static void si_pcie_gart_fini(struct radeon_device *rdev)
Alex Deucherd2800ee2012-03-20 17:18:13 -04004345{
4346 si_pcie_gart_disable(rdev);
4347 radeon_gart_table_vram_free(rdev);
4348 radeon_gart_fini(rdev);
4349}
4350
Alex Deucher498dd8b2012-03-20 17:18:15 -04004351/* vm parser */
4352static bool si_vm_reg_valid(u32 reg)
4353{
4354 /* context regs are fine */
4355 if (reg >= 0x28000)
4356 return true;
4357
4358 /* check config regs */
4359 switch (reg) {
4360 case GRBM_GFX_INDEX:
Alex Deucherf418b882012-11-08 10:13:24 -05004361 case CP_STRMOUT_CNTL:
Alex Deucher498dd8b2012-03-20 17:18:15 -04004362 case VGT_VTX_VECT_EJECT_REG:
4363 case VGT_CACHE_INVALIDATION:
4364 case VGT_ESGS_RING_SIZE:
4365 case VGT_GSVS_RING_SIZE:
4366 case VGT_GS_VERTEX_REUSE:
4367 case VGT_PRIMITIVE_TYPE:
4368 case VGT_INDEX_TYPE:
4369 case VGT_NUM_INDICES:
4370 case VGT_NUM_INSTANCES:
4371 case VGT_TF_RING_SIZE:
4372 case VGT_HS_OFFCHIP_PARAM:
4373 case VGT_TF_MEMORY_BASE:
4374 case PA_CL_ENHANCE:
4375 case PA_SU_LINE_STIPPLE_VALUE:
4376 case PA_SC_LINE_STIPPLE_STATE:
4377 case PA_SC_ENHANCE:
4378 case SQC_CACHES:
4379 case SPI_STATIC_THREAD_MGMT_1:
4380 case SPI_STATIC_THREAD_MGMT_2:
4381 case SPI_STATIC_THREAD_MGMT_3:
4382 case SPI_PS_MAX_WAVE_ID:
4383 case SPI_CONFIG_CNTL:
4384 case SPI_CONFIG_CNTL_1:
4385 case TA_CNTL_AUX:
4386 return true;
4387 default:
4388 DRM_ERROR("Invalid register 0x%x in CS\n", reg);
4389 return false;
4390 }
4391}
4392
4393static int si_vm_packet3_ce_check(struct radeon_device *rdev,
4394 u32 *ib, struct radeon_cs_packet *pkt)
4395{
4396 switch (pkt->opcode) {
4397 case PACKET3_NOP:
4398 case PACKET3_SET_BASE:
4399 case PACKET3_SET_CE_DE_COUNTERS:
4400 case PACKET3_LOAD_CONST_RAM:
4401 case PACKET3_WRITE_CONST_RAM:
4402 case PACKET3_WRITE_CONST_RAM_OFFSET:
4403 case PACKET3_DUMP_CONST_RAM:
4404 case PACKET3_INCREMENT_CE_COUNTER:
4405 case PACKET3_WAIT_ON_DE_COUNTER:
4406 case PACKET3_CE_WRITE:
4407 break;
4408 default:
4409 DRM_ERROR("Invalid CE packet3: 0x%x\n", pkt->opcode);
4410 return -EINVAL;
4411 }
4412 return 0;
4413}
4414
Tom Stellarde5b9e752013-08-16 17:47:39 -04004415static int si_vm_packet3_cp_dma_check(u32 *ib, u32 idx)
4416{
4417 u32 start_reg, reg, i;
4418 u32 command = ib[idx + 4];
4419 u32 info = ib[idx + 1];
4420 u32 idx_value = ib[idx];
4421 if (command & PACKET3_CP_DMA_CMD_SAS) {
4422 /* src address space is register */
4423 if (((info & 0x60000000) >> 29) == 0) {
4424 start_reg = idx_value << 2;
4425 if (command & PACKET3_CP_DMA_CMD_SAIC) {
4426 reg = start_reg;
4427 if (!si_vm_reg_valid(reg)) {
4428 DRM_ERROR("CP DMA Bad SRC register\n");
4429 return -EINVAL;
4430 }
4431 } else {
4432 for (i = 0; i < (command & 0x1fffff); i++) {
4433 reg = start_reg + (4 * i);
4434 if (!si_vm_reg_valid(reg)) {
4435 DRM_ERROR("CP DMA Bad SRC register\n");
4436 return -EINVAL;
4437 }
4438 }
4439 }
4440 }
4441 }
4442 if (command & PACKET3_CP_DMA_CMD_DAS) {
4443 /* dst address space is register */
4444 if (((info & 0x00300000) >> 20) == 0) {
4445 start_reg = ib[idx + 2];
4446 if (command & PACKET3_CP_DMA_CMD_DAIC) {
4447 reg = start_reg;
4448 if (!si_vm_reg_valid(reg)) {
4449 DRM_ERROR("CP DMA Bad DST register\n");
4450 return -EINVAL;
4451 }
4452 } else {
4453 for (i = 0; i < (command & 0x1fffff); i++) {
4454 reg = start_reg + (4 * i);
4455 if (!si_vm_reg_valid(reg)) {
4456 DRM_ERROR("CP DMA Bad DST register\n");
4457 return -EINVAL;
4458 }
4459 }
4460 }
4461 }
4462 }
4463 return 0;
4464}
4465
Alex Deucher498dd8b2012-03-20 17:18:15 -04004466static int si_vm_packet3_gfx_check(struct radeon_device *rdev,
4467 u32 *ib, struct radeon_cs_packet *pkt)
4468{
Tom Stellarde5b9e752013-08-16 17:47:39 -04004469 int r;
Alex Deucher498dd8b2012-03-20 17:18:15 -04004470 u32 idx = pkt->idx + 1;
4471 u32 idx_value = ib[idx];
4472 u32 start_reg, end_reg, reg, i;
4473
4474 switch (pkt->opcode) {
4475 case PACKET3_NOP:
4476 case PACKET3_SET_BASE:
4477 case PACKET3_CLEAR_STATE:
4478 case PACKET3_INDEX_BUFFER_SIZE:
4479 case PACKET3_DISPATCH_DIRECT:
4480 case PACKET3_DISPATCH_INDIRECT:
4481 case PACKET3_ALLOC_GDS:
4482 case PACKET3_WRITE_GDS_RAM:
4483 case PACKET3_ATOMIC_GDS:
4484 case PACKET3_ATOMIC:
4485 case PACKET3_OCCLUSION_QUERY:
4486 case PACKET3_SET_PREDICATION:
4487 case PACKET3_COND_EXEC:
4488 case PACKET3_PRED_EXEC:
4489 case PACKET3_DRAW_INDIRECT:
4490 case PACKET3_DRAW_INDEX_INDIRECT:
4491 case PACKET3_INDEX_BASE:
4492 case PACKET3_DRAW_INDEX_2:
4493 case PACKET3_CONTEXT_CONTROL:
4494 case PACKET3_INDEX_TYPE:
4495 case PACKET3_DRAW_INDIRECT_MULTI:
4496 case PACKET3_DRAW_INDEX_AUTO:
4497 case PACKET3_DRAW_INDEX_IMMD:
4498 case PACKET3_NUM_INSTANCES:
4499 case PACKET3_DRAW_INDEX_MULTI_AUTO:
4500 case PACKET3_STRMOUT_BUFFER_UPDATE:
4501 case PACKET3_DRAW_INDEX_OFFSET_2:
4502 case PACKET3_DRAW_INDEX_MULTI_ELEMENT:
4503 case PACKET3_DRAW_INDEX_INDIRECT_MULTI:
4504 case PACKET3_MPEG_INDEX:
4505 case PACKET3_WAIT_REG_MEM:
4506 case PACKET3_MEM_WRITE:
4507 case PACKET3_PFP_SYNC_ME:
4508 case PACKET3_SURFACE_SYNC:
4509 case PACKET3_EVENT_WRITE:
4510 case PACKET3_EVENT_WRITE_EOP:
4511 case PACKET3_EVENT_WRITE_EOS:
4512 case PACKET3_SET_CONTEXT_REG:
4513 case PACKET3_SET_CONTEXT_REG_INDIRECT:
4514 case PACKET3_SET_SH_REG:
4515 case PACKET3_SET_SH_REG_OFFSET:
4516 case PACKET3_INCREMENT_DE_COUNTER:
4517 case PACKET3_WAIT_ON_CE_COUNTER:
4518 case PACKET3_WAIT_ON_AVAIL_BUFFER:
4519 case PACKET3_ME_WRITE:
4520 break;
4521 case PACKET3_COPY_DATA:
4522 if ((idx_value & 0xf00) == 0) {
4523 reg = ib[idx + 3] * 4;
4524 if (!si_vm_reg_valid(reg))
4525 return -EINVAL;
4526 }
4527 break;
4528 case PACKET3_WRITE_DATA:
4529 if ((idx_value & 0xf00) == 0) {
4530 start_reg = ib[idx + 1] * 4;
4531 if (idx_value & 0x10000) {
4532 if (!si_vm_reg_valid(start_reg))
4533 return -EINVAL;
4534 } else {
4535 for (i = 0; i < (pkt->count - 2); i++) {
4536 reg = start_reg + (4 * i);
4537 if (!si_vm_reg_valid(reg))
4538 return -EINVAL;
4539 }
4540 }
4541 }
4542 break;
4543 case PACKET3_COND_WRITE:
4544 if (idx_value & 0x100) {
4545 reg = ib[idx + 5] * 4;
4546 if (!si_vm_reg_valid(reg))
4547 return -EINVAL;
4548 }
4549 break;
4550 case PACKET3_COPY_DW:
4551 if (idx_value & 0x2) {
4552 reg = ib[idx + 3] * 4;
4553 if (!si_vm_reg_valid(reg))
4554 return -EINVAL;
4555 }
4556 break;
4557 case PACKET3_SET_CONFIG_REG:
4558 start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START;
4559 end_reg = 4 * pkt->count + start_reg - 4;
4560 if ((start_reg < PACKET3_SET_CONFIG_REG_START) ||
4561 (start_reg >= PACKET3_SET_CONFIG_REG_END) ||
4562 (end_reg >= PACKET3_SET_CONFIG_REG_END)) {
4563 DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n");
4564 return -EINVAL;
4565 }
4566 for (i = 0; i < pkt->count; i++) {
4567 reg = start_reg + (4 * i);
4568 if (!si_vm_reg_valid(reg))
4569 return -EINVAL;
4570 }
4571 break;
Alex Deucher5aa709b2012-12-03 19:42:37 -05004572 case PACKET3_CP_DMA:
Tom Stellarde5b9e752013-08-16 17:47:39 -04004573 r = si_vm_packet3_cp_dma_check(ib, idx);
4574 if (r)
4575 return r;
Alex Deucher5aa709b2012-12-03 19:42:37 -05004576 break;
Alex Deucher498dd8b2012-03-20 17:18:15 -04004577 default:
4578 DRM_ERROR("Invalid GFX packet3: 0x%x\n", pkt->opcode);
4579 return -EINVAL;
4580 }
4581 return 0;
4582}
4583
4584static int si_vm_packet3_compute_check(struct radeon_device *rdev,
4585 u32 *ib, struct radeon_cs_packet *pkt)
4586{
Tom Stellarde5b9e752013-08-16 17:47:39 -04004587 int r;
Alex Deucher498dd8b2012-03-20 17:18:15 -04004588 u32 idx = pkt->idx + 1;
4589 u32 idx_value = ib[idx];
4590 u32 start_reg, reg, i;
4591
4592 switch (pkt->opcode) {
4593 case PACKET3_NOP:
4594 case PACKET3_SET_BASE:
4595 case PACKET3_CLEAR_STATE:
4596 case PACKET3_DISPATCH_DIRECT:
4597 case PACKET3_DISPATCH_INDIRECT:
4598 case PACKET3_ALLOC_GDS:
4599 case PACKET3_WRITE_GDS_RAM:
4600 case PACKET3_ATOMIC_GDS:
4601 case PACKET3_ATOMIC:
4602 case PACKET3_OCCLUSION_QUERY:
4603 case PACKET3_SET_PREDICATION:
4604 case PACKET3_COND_EXEC:
4605 case PACKET3_PRED_EXEC:
4606 case PACKET3_CONTEXT_CONTROL:
4607 case PACKET3_STRMOUT_BUFFER_UPDATE:
4608 case PACKET3_WAIT_REG_MEM:
4609 case PACKET3_MEM_WRITE:
4610 case PACKET3_PFP_SYNC_ME:
4611 case PACKET3_SURFACE_SYNC:
4612 case PACKET3_EVENT_WRITE:
4613 case PACKET3_EVENT_WRITE_EOP:
4614 case PACKET3_EVENT_WRITE_EOS:
4615 case PACKET3_SET_CONTEXT_REG:
4616 case PACKET3_SET_CONTEXT_REG_INDIRECT:
4617 case PACKET3_SET_SH_REG:
4618 case PACKET3_SET_SH_REG_OFFSET:
4619 case PACKET3_INCREMENT_DE_COUNTER:
4620 case PACKET3_WAIT_ON_CE_COUNTER:
4621 case PACKET3_WAIT_ON_AVAIL_BUFFER:
4622 case PACKET3_ME_WRITE:
4623 break;
4624 case PACKET3_COPY_DATA:
4625 if ((idx_value & 0xf00) == 0) {
4626 reg = ib[idx + 3] * 4;
4627 if (!si_vm_reg_valid(reg))
4628 return -EINVAL;
4629 }
4630 break;
4631 case PACKET3_WRITE_DATA:
4632 if ((idx_value & 0xf00) == 0) {
4633 start_reg = ib[idx + 1] * 4;
4634 if (idx_value & 0x10000) {
4635 if (!si_vm_reg_valid(start_reg))
4636 return -EINVAL;
4637 } else {
4638 for (i = 0; i < (pkt->count - 2); i++) {
4639 reg = start_reg + (4 * i);
4640 if (!si_vm_reg_valid(reg))
4641 return -EINVAL;
4642 }
4643 }
4644 }
4645 break;
4646 case PACKET3_COND_WRITE:
4647 if (idx_value & 0x100) {
4648 reg = ib[idx + 5] * 4;
4649 if (!si_vm_reg_valid(reg))
4650 return -EINVAL;
4651 }
4652 break;
4653 case PACKET3_COPY_DW:
4654 if (idx_value & 0x2) {
4655 reg = ib[idx + 3] * 4;
4656 if (!si_vm_reg_valid(reg))
4657 return -EINVAL;
4658 }
4659 break;
Tom Stellarde5b9e752013-08-16 17:47:39 -04004660 case PACKET3_CP_DMA:
4661 r = si_vm_packet3_cp_dma_check(ib, idx);
4662 if (r)
4663 return r;
4664 break;
Alex Deucher498dd8b2012-03-20 17:18:15 -04004665 default:
4666 DRM_ERROR("Invalid Compute packet3: 0x%x\n", pkt->opcode);
4667 return -EINVAL;
4668 }
4669 return 0;
4670}
4671
4672int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
4673{
4674 int ret = 0;
4675 u32 idx = 0;
4676 struct radeon_cs_packet pkt;
4677
4678 do {
4679 pkt.idx = idx;
Ilija Hadzic4e872ae2013-01-02 18:27:48 -05004680 pkt.type = RADEON_CP_PACKET_GET_TYPE(ib->ptr[idx]);
4681 pkt.count = RADEON_CP_PACKET_GET_COUNT(ib->ptr[idx]);
Alex Deucher498dd8b2012-03-20 17:18:15 -04004682 pkt.one_reg_wr = 0;
4683 switch (pkt.type) {
Ilija Hadzic4e872ae2013-01-02 18:27:48 -05004684 case RADEON_PACKET_TYPE0:
Alex Deucher498dd8b2012-03-20 17:18:15 -04004685 dev_err(rdev->dev, "Packet0 not allowed!\n");
4686 ret = -EINVAL;
4687 break;
Ilija Hadzic4e872ae2013-01-02 18:27:48 -05004688 case RADEON_PACKET_TYPE2:
Alex Deucher498dd8b2012-03-20 17:18:15 -04004689 idx += 1;
4690 break;
Ilija Hadzic4e872ae2013-01-02 18:27:48 -05004691 case RADEON_PACKET_TYPE3:
4692 pkt.opcode = RADEON_CP_PACKET3_GET_OPCODE(ib->ptr[idx]);
Alex Deucher498dd8b2012-03-20 17:18:15 -04004693 if (ib->is_const_ib)
4694 ret = si_vm_packet3_ce_check(rdev, ib->ptr, &pkt);
4695 else {
Christian König876dc9f2012-05-08 14:24:01 +02004696 switch (ib->ring) {
Alex Deucher498dd8b2012-03-20 17:18:15 -04004697 case RADEON_RING_TYPE_GFX_INDEX:
4698 ret = si_vm_packet3_gfx_check(rdev, ib->ptr, &pkt);
4699 break;
4700 case CAYMAN_RING_TYPE_CP1_INDEX:
4701 case CAYMAN_RING_TYPE_CP2_INDEX:
4702 ret = si_vm_packet3_compute_check(rdev, ib->ptr, &pkt);
4703 break;
4704 default:
Christian König876dc9f2012-05-08 14:24:01 +02004705 dev_err(rdev->dev, "Non-PM4 ring %d !\n", ib->ring);
Alex Deucher498dd8b2012-03-20 17:18:15 -04004706 ret = -EINVAL;
4707 break;
4708 }
4709 }
4710 idx += pkt.count + 2;
4711 break;
4712 default:
4713 dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type);
4714 ret = -EINVAL;
4715 break;
4716 }
4717 if (ret)
4718 break;
4719 } while (idx < ib->length_dw);
4720
4721 return ret;
4722}
4723
Alex Deucherd2800ee2012-03-20 17:18:13 -04004724/*
4725 * vm
4726 */
4727int si_vm_init(struct radeon_device *rdev)
4728{
4729 /* number of VMs */
4730 rdev->vm_manager.nvm = 16;
4731 /* base offset of vram pages */
4732 rdev->vm_manager.vram_base_offset = 0;
4733
4734 return 0;
4735}
4736
4737void si_vm_fini(struct radeon_device *rdev)
4738{
4739}
4740
Alex Deucher82ffd922012-10-02 14:47:46 -04004741/**
Alex Deucherfbf6dc72013-06-13 18:47:58 -04004742 * si_vm_decode_fault - print human readable fault info
4743 *
4744 * @rdev: radeon_device pointer
4745 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
4746 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
4747 *
4748 * Print human readable fault information (SI).
4749 */
4750static void si_vm_decode_fault(struct radeon_device *rdev,
4751 u32 status, u32 addr)
4752{
4753 u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
4754 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
4755 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
4756 char *block;
4757
4758 if (rdev->family == CHIP_TAHITI) {
4759 switch (mc_id) {
4760 case 160:
4761 case 144:
4762 case 96:
4763 case 80:
4764 case 224:
4765 case 208:
4766 case 32:
4767 case 16:
4768 block = "CB";
4769 break;
4770 case 161:
4771 case 145:
4772 case 97:
4773 case 81:
4774 case 225:
4775 case 209:
4776 case 33:
4777 case 17:
4778 block = "CB_FMASK";
4779 break;
4780 case 162:
4781 case 146:
4782 case 98:
4783 case 82:
4784 case 226:
4785 case 210:
4786 case 34:
4787 case 18:
4788 block = "CB_CMASK";
4789 break;
4790 case 163:
4791 case 147:
4792 case 99:
4793 case 83:
4794 case 227:
4795 case 211:
4796 case 35:
4797 case 19:
4798 block = "CB_IMMED";
4799 break;
4800 case 164:
4801 case 148:
4802 case 100:
4803 case 84:
4804 case 228:
4805 case 212:
4806 case 36:
4807 case 20:
4808 block = "DB";
4809 break;
4810 case 165:
4811 case 149:
4812 case 101:
4813 case 85:
4814 case 229:
4815 case 213:
4816 case 37:
4817 case 21:
4818 block = "DB_HTILE";
4819 break;
4820 case 167:
4821 case 151:
4822 case 103:
4823 case 87:
4824 case 231:
4825 case 215:
4826 case 39:
4827 case 23:
4828 block = "DB_STEN";
4829 break;
4830 case 72:
4831 case 68:
4832 case 64:
4833 case 8:
4834 case 4:
4835 case 0:
4836 case 136:
4837 case 132:
4838 case 128:
4839 case 200:
4840 case 196:
4841 case 192:
4842 block = "TC";
4843 break;
4844 case 112:
4845 case 48:
4846 block = "CP";
4847 break;
4848 case 49:
4849 case 177:
4850 case 50:
4851 case 178:
4852 block = "SH";
4853 break;
4854 case 53:
4855 case 190:
4856 block = "VGT";
4857 break;
4858 case 117:
4859 block = "IH";
4860 break;
4861 case 51:
4862 case 115:
4863 block = "RLC";
4864 break;
4865 case 119:
4866 case 183:
4867 block = "DMA0";
4868 break;
4869 case 61:
4870 block = "DMA1";
4871 break;
4872 case 248:
4873 case 120:
4874 block = "HDP";
4875 break;
4876 default:
4877 block = "unknown";
4878 break;
4879 }
4880 } else {
4881 switch (mc_id) {
4882 case 32:
4883 case 16:
4884 case 96:
4885 case 80:
4886 case 160:
4887 case 144:
4888 case 224:
4889 case 208:
4890 block = "CB";
4891 break;
4892 case 33:
4893 case 17:
4894 case 97:
4895 case 81:
4896 case 161:
4897 case 145:
4898 case 225:
4899 case 209:
4900 block = "CB_FMASK";
4901 break;
4902 case 34:
4903 case 18:
4904 case 98:
4905 case 82:
4906 case 162:
4907 case 146:
4908 case 226:
4909 case 210:
4910 block = "CB_CMASK";
4911 break;
4912 case 35:
4913 case 19:
4914 case 99:
4915 case 83:
4916 case 163:
4917 case 147:
4918 case 227:
4919 case 211:
4920 block = "CB_IMMED";
4921 break;
4922 case 36:
4923 case 20:
4924 case 100:
4925 case 84:
4926 case 164:
4927 case 148:
4928 case 228:
4929 case 212:
4930 block = "DB";
4931 break;
4932 case 37:
4933 case 21:
4934 case 101:
4935 case 85:
4936 case 165:
4937 case 149:
4938 case 229:
4939 case 213:
4940 block = "DB_HTILE";
4941 break;
4942 case 39:
4943 case 23:
4944 case 103:
4945 case 87:
4946 case 167:
4947 case 151:
4948 case 231:
4949 case 215:
4950 block = "DB_STEN";
4951 break;
4952 case 72:
4953 case 68:
4954 case 8:
4955 case 4:
4956 case 136:
4957 case 132:
4958 case 200:
4959 case 196:
4960 block = "TC";
4961 break;
4962 case 112:
4963 case 48:
4964 block = "CP";
4965 break;
4966 case 49:
4967 case 177:
4968 case 50:
4969 case 178:
4970 block = "SH";
4971 break;
4972 case 53:
4973 block = "VGT";
4974 break;
4975 case 117:
4976 block = "IH";
4977 break;
4978 case 51:
4979 case 115:
4980 block = "RLC";
4981 break;
4982 case 119:
4983 case 183:
4984 block = "DMA0";
4985 break;
4986 case 61:
4987 block = "DMA1";
4988 break;
4989 case 248:
4990 case 120:
4991 block = "HDP";
4992 break;
4993 default:
4994 block = "unknown";
4995 break;
4996 }
4997 }
4998
4999 printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
5000 protections, vmid, addr,
5001 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
5002 block, mc_id);
5003}
5004
Alex Deucher498522b2012-10-02 14:43:38 -04005005void si_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
Alex Deucherd2800ee2012-03-20 17:18:13 -04005006{
Alex Deucher498522b2012-10-02 14:43:38 -04005007 struct radeon_ring *ring = &rdev->ring[ridx];
Alex Deucherd2800ee2012-03-20 17:18:13 -04005008
Christian Königee60e292012-08-09 16:21:08 +02005009 if (vm == NULL)
Alex Deucherd2800ee2012-03-20 17:18:13 -04005010 return;
5011
Alex Deucher76c44f22012-10-02 14:39:18 -04005012 /* write new base address */
5013 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
Christian Königf1d2a262014-07-30 17:18:12 +02005014 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
Alex Deucher76c44f22012-10-02 14:39:18 -04005015 WRITE_DATA_DST_SEL(0)));
5016
Christian Königee60e292012-08-09 16:21:08 +02005017 if (vm->id < 8) {
Alex Deucher76c44f22012-10-02 14:39:18 -04005018 radeon_ring_write(ring,
5019 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
Christian Königee60e292012-08-09 16:21:08 +02005020 } else {
Alex Deucher76c44f22012-10-02 14:39:18 -04005021 radeon_ring_write(ring,
5022 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
Christian Königee60e292012-08-09 16:21:08 +02005023 }
Alex Deucher76c44f22012-10-02 14:39:18 -04005024 radeon_ring_write(ring, 0);
Dmitry Cherkasovfa87e622012-09-17 19:36:19 +02005025 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
Christian Königee60e292012-08-09 16:21:08 +02005026
Alex Deucherd2800ee2012-03-20 17:18:13 -04005027 /* flush hdp cache */
Alex Deucher76c44f22012-10-02 14:39:18 -04005028 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
Alex Deucher4fb0bbd2014-08-07 09:57:21 -04005029 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
Alex Deucher76c44f22012-10-02 14:39:18 -04005030 WRITE_DATA_DST_SEL(0)));
5031 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
5032 radeon_ring_write(ring, 0);
Christian Königee60e292012-08-09 16:21:08 +02005033 radeon_ring_write(ring, 0x1);
5034
Alex Deucherd2800ee2012-03-20 17:18:13 -04005035 /* bits 0-15 are the VM contexts0-15 */
Alex Deucher76c44f22012-10-02 14:39:18 -04005036 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
Alex Deucher4fb0bbd2014-08-07 09:57:21 -04005037 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
Alex Deucher76c44f22012-10-02 14:39:18 -04005038 WRITE_DATA_DST_SEL(0)));
5039 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5040 radeon_ring_write(ring, 0);
Alex Deucher498522b2012-10-02 14:43:38 -04005041 radeon_ring_write(ring, 1 << vm->id);
Christian König58f8cf52012-10-22 17:42:35 +02005042
5043 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5044 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5045 radeon_ring_write(ring, 0x0);
Alex Deucherd2800ee2012-03-20 17:18:13 -04005046}
5047
Alex Deucher347e7592012-03-20 17:18:21 -04005048/*
Alex Deucherf8f84ac2013-03-07 12:56:35 -05005049 * Power and clock gating
5050 */
5051static void si_wait_for_rlc_serdes(struct radeon_device *rdev)
5052{
5053 int i;
5054
5055 for (i = 0; i < rdev->usec_timeout; i++) {
5056 if (RREG32(RLC_SERDES_MASTER_BUSY_0) == 0)
5057 break;
5058 udelay(1);
5059 }
5060
5061 for (i = 0; i < rdev->usec_timeout; i++) {
5062 if (RREG32(RLC_SERDES_MASTER_BUSY_1) == 0)
5063 break;
5064 udelay(1);
5065 }
5066}
5067
5068static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
5069 bool enable)
5070{
5071 u32 tmp = RREG32(CP_INT_CNTL_RING0);
5072 u32 mask;
5073 int i;
5074
5075 if (enable)
5076 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5077 else
5078 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5079 WREG32(CP_INT_CNTL_RING0, tmp);
5080
5081 if (!enable) {
5082 /* read a gfx register */
5083 tmp = RREG32(DB_DEPTH_INFO);
5084
5085 mask = RLC_BUSY_STATUS | GFX_POWER_STATUS | GFX_CLOCK_STATUS | GFX_LS_STATUS;
5086 for (i = 0; i < rdev->usec_timeout; i++) {
5087 if ((RREG32(RLC_STAT) & mask) == (GFX_CLOCK_STATUS | GFX_POWER_STATUS))
5088 break;
5089 udelay(1);
5090 }
5091 }
5092}
5093
5094static void si_set_uvd_dcm(struct radeon_device *rdev,
5095 bool sw_mode)
5096{
5097 u32 tmp, tmp2;
5098
5099 tmp = RREG32(UVD_CGC_CTRL);
5100 tmp &= ~(CLK_OD_MASK | CG_DT_MASK);
5101 tmp |= DCM | CG_DT(1) | CLK_OD(4);
5102
5103 if (sw_mode) {
5104 tmp &= ~0x7ffff800;
5105 tmp2 = DYN_OR_EN | DYN_RR_EN | G_DIV_ID(7);
5106 } else {
5107 tmp |= 0x7ffff800;
5108 tmp2 = 0;
5109 }
5110
5111 WREG32(UVD_CGC_CTRL, tmp);
5112 WREG32_UVD_CTX(UVD_CGC_CTRL2, tmp2);
5113}
5114
Alex Deucher22c775c2013-07-23 09:41:05 -04005115void si_init_uvd_internal_cg(struct radeon_device *rdev)
Alex Deucherf8f84ac2013-03-07 12:56:35 -05005116{
5117 bool hw_mode = true;
5118
5119 if (hw_mode) {
5120 si_set_uvd_dcm(rdev, false);
5121 } else {
5122 u32 tmp = RREG32(UVD_CGC_CTRL);
5123 tmp &= ~DCM;
5124 WREG32(UVD_CGC_CTRL, tmp);
5125 }
5126}
5127
5128static u32 si_halt_rlc(struct radeon_device *rdev)
5129{
5130 u32 data, orig;
5131
5132 orig = data = RREG32(RLC_CNTL);
5133
5134 if (data & RLC_ENABLE) {
5135 data &= ~RLC_ENABLE;
5136 WREG32(RLC_CNTL, data);
5137
5138 si_wait_for_rlc_serdes(rdev);
5139 }
5140
5141 return orig;
5142}
5143
5144static void si_update_rlc(struct radeon_device *rdev, u32 rlc)
5145{
5146 u32 tmp;
5147
5148 tmp = RREG32(RLC_CNTL);
5149 if (tmp != rlc)
5150 WREG32(RLC_CNTL, rlc);
5151}
5152
5153static void si_enable_dma_pg(struct radeon_device *rdev, bool enable)
5154{
5155 u32 data, orig;
5156
5157 orig = data = RREG32(DMA_PG);
Alex Deuchere16866e2013-08-08 19:34:07 -04005158 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA))
Alex Deucherf8f84ac2013-03-07 12:56:35 -05005159 data |= PG_CNTL_ENABLE;
5160 else
5161 data &= ~PG_CNTL_ENABLE;
5162 if (orig != data)
5163 WREG32(DMA_PG, data);
5164}
5165
5166static void si_init_dma_pg(struct radeon_device *rdev)
5167{
5168 u32 tmp;
5169
5170 WREG32(DMA_PGFSM_WRITE, 0x00002000);
5171 WREG32(DMA_PGFSM_CONFIG, 0x100010ff);
5172
5173 for (tmp = 0; tmp < 5; tmp++)
5174 WREG32(DMA_PGFSM_WRITE, 0);
5175}
5176
5177static void si_enable_gfx_cgpg(struct radeon_device *rdev,
5178 bool enable)
5179{
5180 u32 tmp;
5181
Alex Deucher2b19d172013-09-04 16:58:29 -04005182 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
Alex Deucherf8f84ac2013-03-07 12:56:35 -05005183 tmp = RLC_PUD(0x10) | RLC_PDD(0x10) | RLC_TTPD(0x10) | RLC_MSD(0x10);
5184 WREG32(RLC_TTOP_D, tmp);
5185
5186 tmp = RREG32(RLC_PG_CNTL);
5187 tmp |= GFX_PG_ENABLE;
5188 WREG32(RLC_PG_CNTL, tmp);
5189
5190 tmp = RREG32(RLC_AUTO_PG_CTRL);
5191 tmp |= AUTO_PG_EN;
5192 WREG32(RLC_AUTO_PG_CTRL, tmp);
5193 } else {
5194 tmp = RREG32(RLC_AUTO_PG_CTRL);
5195 tmp &= ~AUTO_PG_EN;
5196 WREG32(RLC_AUTO_PG_CTRL, tmp);
5197
5198 tmp = RREG32(DB_RENDER_CONTROL);
5199 }
5200}
5201
5202static void si_init_gfx_cgpg(struct radeon_device *rdev)
5203{
5204 u32 tmp;
5205
5206 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5207
5208 tmp = RREG32(RLC_PG_CNTL);
5209 tmp |= GFX_PG_SRC;
5210 WREG32(RLC_PG_CNTL, tmp);
5211
5212 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5213
5214 tmp = RREG32(RLC_AUTO_PG_CTRL);
5215
5216 tmp &= ~GRBM_REG_SGIT_MASK;
5217 tmp |= GRBM_REG_SGIT(0x700);
5218 tmp &= ~PG_AFTER_GRBM_REG_ST_MASK;
5219 WREG32(RLC_AUTO_PG_CTRL, tmp);
5220}
5221
Alex Deucherba190312013-04-17 16:27:40 -04005222static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
Alex Deucherf8f84ac2013-03-07 12:56:35 -05005223{
5224 u32 mask = 0, tmp, tmp1;
5225 int i;
5226
5227 si_select_se_sh(rdev, se, sh);
5228 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
5229 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
5230 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5231
5232 tmp &= 0xffff0000;
5233
5234 tmp |= tmp1;
5235 tmp >>= 16;
5236
5237 for (i = 0; i < rdev->config.si.max_cu_per_sh; i ++) {
5238 mask <<= 1;
5239 mask |= 1;
5240 }
5241
5242 return (~tmp) & mask;
5243}
5244
5245static void si_init_ao_cu_mask(struct radeon_device *rdev)
5246{
5247 u32 i, j, k, active_cu_number = 0;
5248 u32 mask, counter, cu_bitmap;
5249 u32 tmp = 0;
5250
5251 for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
5252 for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
5253 mask = 1;
5254 cu_bitmap = 0;
5255 counter = 0;
5256 for (k = 0; k < rdev->config.si.max_cu_per_sh; k++) {
Alex Deucherba190312013-04-17 16:27:40 -04005257 if (si_get_cu_active_bitmap(rdev, i, j) & mask) {
Alex Deucherf8f84ac2013-03-07 12:56:35 -05005258 if (counter < 2)
5259 cu_bitmap |= mask;
5260 counter++;
5261 }
5262 mask <<= 1;
5263 }
5264
5265 active_cu_number += counter;
5266 tmp |= (cu_bitmap << (i * 16 + j * 8));
5267 }
5268 }
5269
5270 WREG32(RLC_PG_AO_CU_MASK, tmp);
5271
5272 tmp = RREG32(RLC_MAX_PG_CU);
5273 tmp &= ~MAX_PU_CU_MASK;
5274 tmp |= MAX_PU_CU(active_cu_number);
5275 WREG32(RLC_MAX_PG_CU, tmp);
5276}
5277
5278static void si_enable_cgcg(struct radeon_device *rdev,
5279 bool enable)
5280{
5281 u32 data, orig, tmp;
5282
5283 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
5284
Alex Deuchere16866e2013-08-08 19:34:07 -04005285 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
Alex Deucher5594a552013-08-15 16:20:26 -04005286 si_enable_gui_idle_interrupt(rdev, true);
Alex Deucherf8f84ac2013-03-07 12:56:35 -05005287
Alex Deucherf8f84ac2013-03-07 12:56:35 -05005288 WREG32(RLC_GCPM_GENERAL_3, 0x00000080);
5289
5290 tmp = si_halt_rlc(rdev);
5291
5292 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5293 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5294 WREG32(RLC_SERDES_WR_CTRL, 0x00b000ff);
5295
5296 si_wait_for_rlc_serdes(rdev);
5297
5298 si_update_rlc(rdev, tmp);
5299
5300 WREG32(RLC_SERDES_WR_CTRL, 0x007000ff);
5301
5302 data |= CGCG_EN | CGLS_EN;
5303 } else {
Alex Deucher5594a552013-08-15 16:20:26 -04005304 si_enable_gui_idle_interrupt(rdev, false);
5305
Alex Deucherf8f84ac2013-03-07 12:56:35 -05005306 RREG32(CB_CGTT_SCLK_CTRL);
5307 RREG32(CB_CGTT_SCLK_CTRL);
5308 RREG32(CB_CGTT_SCLK_CTRL);
5309 RREG32(CB_CGTT_SCLK_CTRL);
5310
5311 data &= ~(CGCG_EN | CGLS_EN);
5312 }
5313
5314 if (orig != data)
5315 WREG32(RLC_CGCG_CGLS_CTRL, data);
5316}
5317
5318static void si_enable_mgcg(struct radeon_device *rdev,
5319 bool enable)
5320{
5321 u32 data, orig, tmp = 0;
5322
Alex Deuchere16866e2013-08-08 19:34:07 -04005323 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
Alex Deucherf8f84ac2013-03-07 12:56:35 -05005324 orig = data = RREG32(CGTS_SM_CTRL_REG);
5325 data = 0x96940200;
5326 if (orig != data)
5327 WREG32(CGTS_SM_CTRL_REG, data);
5328
Alex Deuchere16866e2013-08-08 19:34:07 -04005329 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5330 orig = data = RREG32(CP_MEM_SLP_CNTL);
5331 data |= CP_MEM_LS_EN;
5332 if (orig != data)
5333 WREG32(CP_MEM_SLP_CNTL, data);
5334 }
Alex Deucherf8f84ac2013-03-07 12:56:35 -05005335
5336 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5337 data &= 0xffffffc0;
5338 if (orig != data)
5339 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5340
5341 tmp = si_halt_rlc(rdev);
5342
5343 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5344 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5345 WREG32(RLC_SERDES_WR_CTRL, 0x00d000ff);
5346
5347 si_update_rlc(rdev, tmp);
5348 } else {
5349 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5350 data |= 0x00000003;
5351 if (orig != data)
5352 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5353
5354 data = RREG32(CP_MEM_SLP_CNTL);
5355 if (data & CP_MEM_LS_EN) {
5356 data &= ~CP_MEM_LS_EN;
5357 WREG32(CP_MEM_SLP_CNTL, data);
5358 }
5359 orig = data = RREG32(CGTS_SM_CTRL_REG);
5360 data |= LS_OVERRIDE | OVERRIDE;
5361 if (orig != data)
5362 WREG32(CGTS_SM_CTRL_REG, data);
5363
5364 tmp = si_halt_rlc(rdev);
5365
5366 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5367 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5368 WREG32(RLC_SERDES_WR_CTRL, 0x00e000ff);
5369
5370 si_update_rlc(rdev, tmp);
5371 }
5372}
5373
5374static void si_enable_uvd_mgcg(struct radeon_device *rdev,
5375 bool enable)
5376{
5377 u32 orig, data, tmp;
5378
Alex Deuchere16866e2013-08-08 19:34:07 -04005379 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
Alex Deucherf8f84ac2013-03-07 12:56:35 -05005380 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5381 tmp |= 0x3fff;
5382 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5383
5384 orig = data = RREG32(UVD_CGC_CTRL);
5385 data |= DCM;
5386 if (orig != data)
5387 WREG32(UVD_CGC_CTRL, data);
5388
5389 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0);
5390 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0);
5391 } else {
5392 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5393 tmp &= ~0x3fff;
5394 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5395
5396 orig = data = RREG32(UVD_CGC_CTRL);
5397 data &= ~DCM;
5398 if (orig != data)
5399 WREG32(UVD_CGC_CTRL, data);
5400
5401 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0xffffffff);
5402 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0xffffffff);
5403 }
5404}
5405
5406static const u32 mc_cg_registers[] =
5407{
5408 MC_HUB_MISC_HUB_CG,
5409 MC_HUB_MISC_SIP_CG,
5410 MC_HUB_MISC_VM_CG,
5411 MC_XPB_CLK_GAT,
5412 ATC_MISC_CG,
5413 MC_CITF_MISC_WR_CG,
5414 MC_CITF_MISC_RD_CG,
5415 MC_CITF_MISC_VM_CG,
5416 VM_L2_CG,
5417};
5418
5419static void si_enable_mc_ls(struct radeon_device *rdev,
5420 bool enable)
5421{
5422 int i;
5423 u32 orig, data;
5424
5425 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5426 orig = data = RREG32(mc_cg_registers[i]);
Alex Deuchere16866e2013-08-08 19:34:07 -04005427 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
Alex Deucherf8f84ac2013-03-07 12:56:35 -05005428 data |= MC_LS_ENABLE;
5429 else
5430 data &= ~MC_LS_ENABLE;
5431 if (data != orig)
5432 WREG32(mc_cg_registers[i], data);
5433 }
5434}
5435
Alex Deuchere16866e2013-08-08 19:34:07 -04005436static void si_enable_mc_mgcg(struct radeon_device *rdev,
5437 bool enable)
5438{
5439 int i;
5440 u32 orig, data;
5441
5442 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5443 orig = data = RREG32(mc_cg_registers[i]);
5444 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
5445 data |= MC_CG_ENABLE;
5446 else
5447 data &= ~MC_CG_ENABLE;
5448 if (data != orig)
5449 WREG32(mc_cg_registers[i], data);
5450 }
5451}
5452
5453static void si_enable_dma_mgcg(struct radeon_device *rdev,
5454 bool enable)
5455{
5456 u32 orig, data, offset;
5457 int i;
5458
5459 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
5460 for (i = 0; i < 2; i++) {
5461 if (i == 0)
5462 offset = DMA0_REGISTER_OFFSET;
5463 else
5464 offset = DMA1_REGISTER_OFFSET;
5465 orig = data = RREG32(DMA_POWER_CNTL + offset);
5466 data &= ~MEM_POWER_OVERRIDE;
5467 if (data != orig)
5468 WREG32(DMA_POWER_CNTL + offset, data);
5469 WREG32(DMA_CLK_CTRL + offset, 0x00000100);
5470 }
5471 } else {
5472 for (i = 0; i < 2; i++) {
5473 if (i == 0)
5474 offset = DMA0_REGISTER_OFFSET;
5475 else
5476 offset = DMA1_REGISTER_OFFSET;
5477 orig = data = RREG32(DMA_POWER_CNTL + offset);
5478 data |= MEM_POWER_OVERRIDE;
5479 if (data != orig)
5480 WREG32(DMA_POWER_CNTL + offset, data);
5481
5482 orig = data = RREG32(DMA_CLK_CTRL + offset);
5483 data = 0xff000000;
5484 if (data != orig)
5485 WREG32(DMA_CLK_CTRL + offset, data);
5486 }
5487 }
5488}
5489
5490static void si_enable_bif_mgls(struct radeon_device *rdev,
5491 bool enable)
5492{
5493 u32 orig, data;
5494
5495 orig = data = RREG32_PCIE(PCIE_CNTL2);
5496
5497 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
5498 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
5499 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
5500 else
5501 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
5502 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
5503
5504 if (orig != data)
5505 WREG32_PCIE(PCIE_CNTL2, data);
5506}
5507
5508static void si_enable_hdp_mgcg(struct radeon_device *rdev,
5509 bool enable)
5510{
5511 u32 orig, data;
5512
5513 orig = data = RREG32(HDP_HOST_PATH_CNTL);
5514
5515 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
5516 data &= ~CLOCK_GATING_DIS;
5517 else
5518 data |= CLOCK_GATING_DIS;
5519
5520 if (orig != data)
5521 WREG32(HDP_HOST_PATH_CNTL, data);
5522}
5523
5524static void si_enable_hdp_ls(struct radeon_device *rdev,
5525 bool enable)
5526{
5527 u32 orig, data;
5528
5529 orig = data = RREG32(HDP_MEM_POWER_LS);
5530
5531 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
5532 data |= HDP_LS_ENABLE;
5533 else
5534 data &= ~HDP_LS_ENABLE;
5535
5536 if (orig != data)
5537 WREG32(HDP_MEM_POWER_LS, data);
5538}
5539
Alex Deucher68e3a092013-12-18 14:11:40 -05005540static void si_update_cg(struct radeon_device *rdev,
5541 u32 block, bool enable)
Alex Deuchere16866e2013-08-08 19:34:07 -04005542{
5543 if (block & RADEON_CG_BLOCK_GFX) {
Alex Deucher811e4d52013-09-03 13:31:33 -04005544 si_enable_gui_idle_interrupt(rdev, false);
Alex Deuchere16866e2013-08-08 19:34:07 -04005545 /* order matters! */
5546 if (enable) {
5547 si_enable_mgcg(rdev, true);
5548 si_enable_cgcg(rdev, true);
5549 } else {
5550 si_enable_cgcg(rdev, false);
5551 si_enable_mgcg(rdev, false);
5552 }
Alex Deucher811e4d52013-09-03 13:31:33 -04005553 si_enable_gui_idle_interrupt(rdev, true);
Alex Deuchere16866e2013-08-08 19:34:07 -04005554 }
5555
5556 if (block & RADEON_CG_BLOCK_MC) {
5557 si_enable_mc_mgcg(rdev, enable);
5558 si_enable_mc_ls(rdev, enable);
5559 }
5560
5561 if (block & RADEON_CG_BLOCK_SDMA) {
5562 si_enable_dma_mgcg(rdev, enable);
5563 }
5564
5565 if (block & RADEON_CG_BLOCK_BIF) {
5566 si_enable_bif_mgls(rdev, enable);
5567 }
5568
5569 if (block & RADEON_CG_BLOCK_UVD) {
5570 if (rdev->has_uvd) {
5571 si_enable_uvd_mgcg(rdev, enable);
5572 }
5573 }
5574
5575 if (block & RADEON_CG_BLOCK_HDP) {
5576 si_enable_hdp_mgcg(rdev, enable);
5577 si_enable_hdp_ls(rdev, enable);
5578 }
5579}
Alex Deucherf8f84ac2013-03-07 12:56:35 -05005580
5581static void si_init_cg(struct radeon_device *rdev)
5582{
Alex Deuchere16866e2013-08-08 19:34:07 -04005583 si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5584 RADEON_CG_BLOCK_MC |
5585 RADEON_CG_BLOCK_SDMA |
5586 RADEON_CG_BLOCK_BIF |
5587 RADEON_CG_BLOCK_HDP), true);
Alex Deucherb2d70912013-07-27 17:53:25 -04005588 if (rdev->has_uvd) {
Alex Deuchere16866e2013-08-08 19:34:07 -04005589 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, true);
Alex Deucherf8f84ac2013-03-07 12:56:35 -05005590 si_init_uvd_internal_cg(rdev);
5591 }
5592}
5593
5594static void si_fini_cg(struct radeon_device *rdev)
5595{
Alex Deucher0116e1e2013-08-08 18:00:10 -04005596 if (rdev->has_uvd) {
Alex Deuchere16866e2013-08-08 19:34:07 -04005597 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, false);
Alex Deucher0116e1e2013-08-08 18:00:10 -04005598 }
Alex Deuchere16866e2013-08-08 19:34:07 -04005599 si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5600 RADEON_CG_BLOCK_MC |
5601 RADEON_CG_BLOCK_SDMA |
5602 RADEON_CG_BLOCK_BIF |
5603 RADEON_CG_BLOCK_HDP), false);
5604}
Alex Deucherf8f84ac2013-03-07 12:56:35 -05005605
Alex Deucher59a82d02013-08-13 12:48:06 -04005606u32 si_get_csb_size(struct radeon_device *rdev)
5607{
5608 u32 count = 0;
5609 const struct cs_section_def *sect = NULL;
5610 const struct cs_extent_def *ext = NULL;
5611
5612 if (rdev->rlc.cs_data == NULL)
5613 return 0;
5614
5615 /* begin clear state */
5616 count += 2;
5617 /* context control state */
5618 count += 3;
5619
5620 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5621 for (ext = sect->section; ext->extent != NULL; ++ext) {
5622 if (sect->id == SECT_CONTEXT)
5623 count += 2 + ext->reg_count;
5624 else
5625 return 0;
5626 }
5627 }
5628 /* pa_sc_raster_config */
5629 count += 3;
5630 /* end clear state */
5631 count += 2;
5632 /* clear state */
5633 count += 2;
5634
5635 return count;
5636}
5637
5638void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
5639{
5640 u32 count = 0, i;
5641 const struct cs_section_def *sect = NULL;
5642 const struct cs_extent_def *ext = NULL;
5643
5644 if (rdev->rlc.cs_data == NULL)
5645 return;
5646 if (buffer == NULL)
5647 return;
5648
Alex Deucher6ba81e52013-10-23 18:27:10 -04005649 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
5650 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
Alex Deucher59a82d02013-08-13 12:48:06 -04005651
Alex Deucher6ba81e52013-10-23 18:27:10 -04005652 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
5653 buffer[count++] = cpu_to_le32(0x80000000);
5654 buffer[count++] = cpu_to_le32(0x80000000);
Alex Deucher59a82d02013-08-13 12:48:06 -04005655
5656 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5657 for (ext = sect->section; ext->extent != NULL; ++ext) {
5658 if (sect->id == SECT_CONTEXT) {
Alex Deucher6ba81e52013-10-23 18:27:10 -04005659 buffer[count++] =
5660 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
5661 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
Alex Deucher59a82d02013-08-13 12:48:06 -04005662 for (i = 0; i < ext->reg_count; i++)
Alex Deucher6ba81e52013-10-23 18:27:10 -04005663 buffer[count++] = cpu_to_le32(ext->extent[i]);
Alex Deucher59a82d02013-08-13 12:48:06 -04005664 } else {
5665 return;
5666 }
5667 }
5668 }
5669
Alex Deucher6ba81e52013-10-23 18:27:10 -04005670 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 1));
5671 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
Alex Deucher59a82d02013-08-13 12:48:06 -04005672 switch (rdev->family) {
5673 case CHIP_TAHITI:
5674 case CHIP_PITCAIRN:
Alex Deucher6ba81e52013-10-23 18:27:10 -04005675 buffer[count++] = cpu_to_le32(0x2a00126a);
Alex Deucher59a82d02013-08-13 12:48:06 -04005676 break;
5677 case CHIP_VERDE:
Alex Deucher6ba81e52013-10-23 18:27:10 -04005678 buffer[count++] = cpu_to_le32(0x0000124a);
Alex Deucher59a82d02013-08-13 12:48:06 -04005679 break;
5680 case CHIP_OLAND:
Alex Deucher6ba81e52013-10-23 18:27:10 -04005681 buffer[count++] = cpu_to_le32(0x00000082);
Alex Deucher59a82d02013-08-13 12:48:06 -04005682 break;
5683 case CHIP_HAINAN:
Alex Deucher6ba81e52013-10-23 18:27:10 -04005684 buffer[count++] = cpu_to_le32(0x00000000);
Alex Deucher59a82d02013-08-13 12:48:06 -04005685 break;
5686 default:
Alex Deucher6ba81e52013-10-23 18:27:10 -04005687 buffer[count++] = cpu_to_le32(0x00000000);
Alex Deucher59a82d02013-08-13 12:48:06 -04005688 break;
5689 }
5690
Alex Deucher6ba81e52013-10-23 18:27:10 -04005691 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
5692 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
Alex Deucher59a82d02013-08-13 12:48:06 -04005693
Alex Deucher6ba81e52013-10-23 18:27:10 -04005694 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
5695 buffer[count++] = cpu_to_le32(0);
Alex Deucherf8f84ac2013-03-07 12:56:35 -05005696}
5697
5698static void si_init_pg(struct radeon_device *rdev)
5699{
Alex Deucher0116e1e2013-08-08 18:00:10 -04005700 if (rdev->pg_flags) {
5701 if (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA) {
5702 si_init_dma_pg(rdev);
Alex Deucher0116e1e2013-08-08 18:00:10 -04005703 }
Alex Deucherf8f84ac2013-03-07 12:56:35 -05005704 si_init_ao_cu_mask(rdev);
Alex Deucher2b19d172013-09-04 16:58:29 -04005705 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
Alex Deucher0116e1e2013-08-08 18:00:10 -04005706 si_init_gfx_cgpg(rdev);
Alex Deucheraa34dba2014-01-16 10:39:17 -05005707 } else {
5708 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5709 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
Alex Deucher0116e1e2013-08-08 18:00:10 -04005710 }
Alex Deucherf8f84ac2013-03-07 12:56:35 -05005711 si_enable_dma_pg(rdev, true);
Alex Deucherf8f84ac2013-03-07 12:56:35 -05005712 si_enable_gfx_cgpg(rdev, true);
5713 } else {
5714 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5715 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5716 }
5717}
5718
5719static void si_fini_pg(struct radeon_device *rdev)
5720{
Alex Deucher0116e1e2013-08-08 18:00:10 -04005721 if (rdev->pg_flags) {
Alex Deucherf8f84ac2013-03-07 12:56:35 -05005722 si_enable_dma_pg(rdev, false);
5723 si_enable_gfx_cgpg(rdev, false);
5724 }
5725}
5726
5727/*
Alex Deucher347e7592012-03-20 17:18:21 -04005728 * RLC
5729 */
Alex Deucher866d83d2013-04-15 17:13:29 -04005730void si_rlc_reset(struct radeon_device *rdev)
Alex Deucherd719cef2013-02-15 16:49:59 -05005731{
Alex Deucherf8f84ac2013-03-07 12:56:35 -05005732 u32 tmp = RREG32(GRBM_SOFT_RESET);
Alex Deucherd719cef2013-02-15 16:49:59 -05005733
Alex Deucherf8f84ac2013-03-07 12:56:35 -05005734 tmp |= SOFT_RESET_RLC;
5735 WREG32(GRBM_SOFT_RESET, tmp);
5736 udelay(50);
5737 tmp &= ~SOFT_RESET_RLC;
5738 WREG32(GRBM_SOFT_RESET, tmp);
5739 udelay(50);
Alex Deucherd719cef2013-02-15 16:49:59 -05005740}
5741
Alex Deucher347e7592012-03-20 17:18:21 -04005742static void si_rlc_stop(struct radeon_device *rdev)
5743{
5744 WREG32(RLC_CNTL, 0);
Alex Deucherd719cef2013-02-15 16:49:59 -05005745
5746 si_enable_gui_idle_interrupt(rdev, false);
5747
5748 si_wait_for_rlc_serdes(rdev);
Alex Deucher347e7592012-03-20 17:18:21 -04005749}
5750
5751static void si_rlc_start(struct radeon_device *rdev)
5752{
5753 WREG32(RLC_CNTL, RLC_ENABLE);
Alex Deucherd719cef2013-02-15 16:49:59 -05005754
5755 si_enable_gui_idle_interrupt(rdev, true);
5756
5757 udelay(50);
5758}
5759
5760static bool si_lbpw_supported(struct radeon_device *rdev)
5761{
5762 u32 tmp;
5763
5764 /* Enable LBPW only for DDR3 */
5765 tmp = RREG32(MC_SEQ_MISC0);
5766 if ((tmp & 0xF0000000) == 0xB0000000)
5767 return true;
5768 return false;
5769}
5770
5771static void si_enable_lbpw(struct radeon_device *rdev, bool enable)
5772{
5773 u32 tmp;
5774
5775 tmp = RREG32(RLC_LB_CNTL);
5776 if (enable)
5777 tmp |= LOAD_BALANCE_ENABLE;
5778 else
5779 tmp &= ~LOAD_BALANCE_ENABLE;
5780 WREG32(RLC_LB_CNTL, tmp);
5781
5782 if (!enable) {
5783 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5784 WREG32(SPI_LB_CU_MASK, 0x00ff);
5785 }
Alex Deucher347e7592012-03-20 17:18:21 -04005786}
5787
5788static int si_rlc_resume(struct radeon_device *rdev)
5789{
5790 u32 i;
Alex Deucher347e7592012-03-20 17:18:21 -04005791
5792 if (!rdev->rlc_fw)
5793 return -EINVAL;
5794
5795 si_rlc_stop(rdev);
5796
Alex Deucherf8f84ac2013-03-07 12:56:35 -05005797 si_rlc_reset(rdev);
5798
5799 si_init_pg(rdev);
5800
5801 si_init_cg(rdev);
5802
Alex Deucher347e7592012-03-20 17:18:21 -04005803 WREG32(RLC_RL_BASE, 0);
5804 WREG32(RLC_RL_SIZE, 0);
5805 WREG32(RLC_LB_CNTL, 0);
5806 WREG32(RLC_LB_CNTR_MAX, 0xffffffff);
5807 WREG32(RLC_LB_CNTR_INIT, 0);
Alex Deucherd719cef2013-02-15 16:49:59 -05005808 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
Alex Deucher347e7592012-03-20 17:18:21 -04005809
Alex Deucher347e7592012-03-20 17:18:21 -04005810 WREG32(RLC_MC_CNTL, 0);
5811 WREG32(RLC_UCODE_CNTL, 0);
5812
Alex Deucher629bd332014-06-25 18:41:34 -04005813 if (rdev->new_fw) {
5814 const struct rlc_firmware_header_v1_0 *hdr =
5815 (const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data;
5816 u32 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
5817 const __le32 *fw_data = (const __le32 *)
5818 (rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
5819
5820 radeon_ucode_print_rlc_hdr(&hdr->header);
5821
5822 for (i = 0; i < fw_size; i++) {
5823 WREG32(RLC_UCODE_ADDR, i);
5824 WREG32(RLC_UCODE_DATA, le32_to_cpup(fw_data++));
5825 }
5826 } else {
5827 const __be32 *fw_data =
5828 (const __be32 *)rdev->rlc_fw->data;
5829 for (i = 0; i < SI_RLC_UCODE_SIZE; i++) {
5830 WREG32(RLC_UCODE_ADDR, i);
5831 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
5832 }
Alex Deucher347e7592012-03-20 17:18:21 -04005833 }
5834 WREG32(RLC_UCODE_ADDR, 0);
5835
Alex Deucherd719cef2013-02-15 16:49:59 -05005836 si_enable_lbpw(rdev, si_lbpw_supported(rdev));
5837
Alex Deucher347e7592012-03-20 17:18:21 -04005838 si_rlc_start(rdev);
5839
5840 return 0;
5841}
5842
Alex Deucher25a857f2012-03-20 17:18:22 -04005843static void si_enable_interrupts(struct radeon_device *rdev)
5844{
5845 u32 ih_cntl = RREG32(IH_CNTL);
5846 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5847
5848 ih_cntl |= ENABLE_INTR;
5849 ih_rb_cntl |= IH_RB_ENABLE;
5850 WREG32(IH_CNTL, ih_cntl);
5851 WREG32(IH_RB_CNTL, ih_rb_cntl);
5852 rdev->ih.enabled = true;
5853}
5854
5855static void si_disable_interrupts(struct radeon_device *rdev)
5856{
5857 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5858 u32 ih_cntl = RREG32(IH_CNTL);
5859
5860 ih_rb_cntl &= ~IH_RB_ENABLE;
5861 ih_cntl &= ~ENABLE_INTR;
5862 WREG32(IH_RB_CNTL, ih_rb_cntl);
5863 WREG32(IH_CNTL, ih_cntl);
5864 /* set rptr, wptr to 0 */
5865 WREG32(IH_RB_RPTR, 0);
5866 WREG32(IH_RB_WPTR, 0);
5867 rdev->ih.enabled = false;
Alex Deucher25a857f2012-03-20 17:18:22 -04005868 rdev->ih.rptr = 0;
5869}
5870
5871static void si_disable_interrupt_state(struct radeon_device *rdev)
5872{
5873 u32 tmp;
5874
Alex Deucher811e4d52013-09-03 13:31:33 -04005875 tmp = RREG32(CP_INT_CNTL_RING0) &
5876 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5877 WREG32(CP_INT_CNTL_RING0, tmp);
Alex Deucher25a857f2012-03-20 17:18:22 -04005878 WREG32(CP_INT_CNTL_RING1, 0);
5879 WREG32(CP_INT_CNTL_RING2, 0);
Alex Deucher8c5fd7e2012-12-04 15:28:18 -05005880 tmp = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5881 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, tmp);
5882 tmp = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5883 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, tmp);
Alex Deucher25a857f2012-03-20 17:18:22 -04005884 WREG32(GRBM_INT_CNTL, 0);
Alex Deucher51535502012-08-30 14:34:30 -04005885 if (rdev->num_crtc >= 2) {
5886 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5887 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5888 }
Alex Deucher25a857f2012-03-20 17:18:22 -04005889 if (rdev->num_crtc >= 4) {
5890 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5891 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5892 }
5893 if (rdev->num_crtc >= 6) {
5894 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5895 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5896 }
5897
Alex Deucher51535502012-08-30 14:34:30 -04005898 if (rdev->num_crtc >= 2) {
5899 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5900 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5901 }
Alex Deucher25a857f2012-03-20 17:18:22 -04005902 if (rdev->num_crtc >= 4) {
5903 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5904 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5905 }
5906 if (rdev->num_crtc >= 6) {
5907 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5908 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5909 }
5910
Alex Deucher51535502012-08-30 14:34:30 -04005911 if (!ASIC_IS_NODCE(rdev)) {
Alex Deuchere9a321c2014-01-27 11:54:44 -05005912 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
Alex Deucher25a857f2012-03-20 17:18:22 -04005913
Alex Deucher51535502012-08-30 14:34:30 -04005914 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5915 WREG32(DC_HPD1_INT_CONTROL, tmp);
5916 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5917 WREG32(DC_HPD2_INT_CONTROL, tmp);
5918 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5919 WREG32(DC_HPD3_INT_CONTROL, tmp);
5920 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5921 WREG32(DC_HPD4_INT_CONTROL, tmp);
5922 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5923 WREG32(DC_HPD5_INT_CONTROL, tmp);
5924 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5925 WREG32(DC_HPD6_INT_CONTROL, tmp);
5926 }
Alex Deucher25a857f2012-03-20 17:18:22 -04005927}
5928
5929static int si_irq_init(struct radeon_device *rdev)
5930{
5931 int ret = 0;
5932 int rb_bufsz;
5933 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
5934
5935 /* allocate ring */
5936 ret = r600_ih_ring_alloc(rdev);
5937 if (ret)
5938 return ret;
5939
5940 /* disable irqs */
5941 si_disable_interrupts(rdev);
5942
5943 /* init rlc */
5944 ret = si_rlc_resume(rdev);
5945 if (ret) {
5946 r600_ih_ring_fini(rdev);
5947 return ret;
5948 }
5949
5950 /* setup interrupt control */
5951 /* set dummy read address to ring address */
5952 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
5953 interrupt_cntl = RREG32(INTERRUPT_CNTL);
5954 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
5955 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
5956 */
5957 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
5958 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
5959 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
5960 WREG32(INTERRUPT_CNTL, interrupt_cntl);
5961
5962 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
Daniel Vetterb72a8922013-07-10 14:11:59 +02005963 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
Alex Deucher25a857f2012-03-20 17:18:22 -04005964
5965 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
5966 IH_WPTR_OVERFLOW_CLEAR |
5967 (rb_bufsz << 1));
5968
5969 if (rdev->wb.enabled)
5970 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
5971
5972 /* set the writeback address whether it's enabled or not */
5973 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
5974 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
5975
5976 WREG32(IH_RB_CNTL, ih_rb_cntl);
5977
5978 /* set rptr, wptr to 0 */
5979 WREG32(IH_RB_RPTR, 0);
5980 WREG32(IH_RB_WPTR, 0);
5981
5982 /* Default settings for IH_CNTL (disabled at first) */
5983 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
5984 /* RPTR_REARM only works if msi's are enabled */
5985 if (rdev->msi_enabled)
5986 ih_cntl |= RPTR_REARM;
5987 WREG32(IH_CNTL, ih_cntl);
5988
5989 /* force the active interrupt state to all disabled */
5990 si_disable_interrupt_state(rdev);
5991
Dave Airlie20998102012-04-03 11:53:05 +01005992 pci_set_master(rdev->pdev);
5993
Alex Deucher25a857f2012-03-20 17:18:22 -04005994 /* enable irqs */
5995 si_enable_interrupts(rdev);
5996
5997 return ret;
5998}
5999
6000int si_irq_set(struct radeon_device *rdev)
6001{
Alex Deucher811e4d52013-09-03 13:31:33 -04006002 u32 cp_int_cntl;
Alex Deucher25a857f2012-03-20 17:18:22 -04006003 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
6004 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
Alex Deucher51535502012-08-30 14:34:30 -04006005 u32 hpd1 = 0, hpd2 = 0, hpd3 = 0, hpd4 = 0, hpd5 = 0, hpd6 = 0;
Alex Deucher25a857f2012-03-20 17:18:22 -04006006 u32 grbm_int_cntl = 0;
Alex Deucher8c5fd7e2012-12-04 15:28:18 -05006007 u32 dma_cntl, dma_cntl1;
Alex Deuchera9e61412013-06-25 17:56:16 -04006008 u32 thermal_int = 0;
Alex Deucher25a857f2012-03-20 17:18:22 -04006009
6010 if (!rdev->irq.installed) {
6011 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
6012 return -EINVAL;
6013 }
6014 /* don't enable anything if the ih is disabled */
6015 if (!rdev->ih.enabled) {
6016 si_disable_interrupts(rdev);
6017 /* force the active interrupt state to all disabled */
6018 si_disable_interrupt_state(rdev);
6019 return 0;
6020 }
6021
Alex Deucher811e4d52013-09-03 13:31:33 -04006022 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
6023 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6024
Alex Deucher51535502012-08-30 14:34:30 -04006025 if (!ASIC_IS_NODCE(rdev)) {
6026 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
6027 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
6028 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
6029 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
6030 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
6031 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
6032 }
Alex Deucher25a857f2012-03-20 17:18:22 -04006033
Alex Deucher8c5fd7e2012-12-04 15:28:18 -05006034 dma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6035 dma_cntl1 = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6036
Alex Deuchera9e61412013-06-25 17:56:16 -04006037 thermal_int = RREG32(CG_THERMAL_INT) &
6038 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
6039
Alex Deucher25a857f2012-03-20 17:18:22 -04006040 /* enable CP interrupts on all rings */
Christian Koenig736fc372012-05-17 19:52:00 +02006041 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
Alex Deucher25a857f2012-03-20 17:18:22 -04006042 DRM_DEBUG("si_irq_set: sw int gfx\n");
6043 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
6044 }
Christian Koenig736fc372012-05-17 19:52:00 +02006045 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
Alex Deucher25a857f2012-03-20 17:18:22 -04006046 DRM_DEBUG("si_irq_set: sw int cp1\n");
6047 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
6048 }
Christian Koenig736fc372012-05-17 19:52:00 +02006049 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
Alex Deucher25a857f2012-03-20 17:18:22 -04006050 DRM_DEBUG("si_irq_set: sw int cp2\n");
6051 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
6052 }
Alex Deucher8c5fd7e2012-12-04 15:28:18 -05006053 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
6054 DRM_DEBUG("si_irq_set: sw int dma\n");
6055 dma_cntl |= TRAP_ENABLE;
6056 }
6057
6058 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
6059 DRM_DEBUG("si_irq_set: sw int dma1\n");
6060 dma_cntl1 |= TRAP_ENABLE;
6061 }
Alex Deucher25a857f2012-03-20 17:18:22 -04006062 if (rdev->irq.crtc_vblank_int[0] ||
Christian Koenig736fc372012-05-17 19:52:00 +02006063 atomic_read(&rdev->irq.pflip[0])) {
Alex Deucher25a857f2012-03-20 17:18:22 -04006064 DRM_DEBUG("si_irq_set: vblank 0\n");
6065 crtc1 |= VBLANK_INT_MASK;
6066 }
6067 if (rdev->irq.crtc_vblank_int[1] ||
Christian Koenig736fc372012-05-17 19:52:00 +02006068 atomic_read(&rdev->irq.pflip[1])) {
Alex Deucher25a857f2012-03-20 17:18:22 -04006069 DRM_DEBUG("si_irq_set: vblank 1\n");
6070 crtc2 |= VBLANK_INT_MASK;
6071 }
6072 if (rdev->irq.crtc_vblank_int[2] ||
Christian Koenig736fc372012-05-17 19:52:00 +02006073 atomic_read(&rdev->irq.pflip[2])) {
Alex Deucher25a857f2012-03-20 17:18:22 -04006074 DRM_DEBUG("si_irq_set: vblank 2\n");
6075 crtc3 |= VBLANK_INT_MASK;
6076 }
6077 if (rdev->irq.crtc_vblank_int[3] ||
Christian Koenig736fc372012-05-17 19:52:00 +02006078 atomic_read(&rdev->irq.pflip[3])) {
Alex Deucher25a857f2012-03-20 17:18:22 -04006079 DRM_DEBUG("si_irq_set: vblank 3\n");
6080 crtc4 |= VBLANK_INT_MASK;
6081 }
6082 if (rdev->irq.crtc_vblank_int[4] ||
Christian Koenig736fc372012-05-17 19:52:00 +02006083 atomic_read(&rdev->irq.pflip[4])) {
Alex Deucher25a857f2012-03-20 17:18:22 -04006084 DRM_DEBUG("si_irq_set: vblank 4\n");
6085 crtc5 |= VBLANK_INT_MASK;
6086 }
6087 if (rdev->irq.crtc_vblank_int[5] ||
Christian Koenig736fc372012-05-17 19:52:00 +02006088 atomic_read(&rdev->irq.pflip[5])) {
Alex Deucher25a857f2012-03-20 17:18:22 -04006089 DRM_DEBUG("si_irq_set: vblank 5\n");
6090 crtc6 |= VBLANK_INT_MASK;
6091 }
6092 if (rdev->irq.hpd[0]) {
6093 DRM_DEBUG("si_irq_set: hpd 1\n");
6094 hpd1 |= DC_HPDx_INT_EN;
6095 }
6096 if (rdev->irq.hpd[1]) {
6097 DRM_DEBUG("si_irq_set: hpd 2\n");
6098 hpd2 |= DC_HPDx_INT_EN;
6099 }
6100 if (rdev->irq.hpd[2]) {
6101 DRM_DEBUG("si_irq_set: hpd 3\n");
6102 hpd3 |= DC_HPDx_INT_EN;
6103 }
6104 if (rdev->irq.hpd[3]) {
6105 DRM_DEBUG("si_irq_set: hpd 4\n");
6106 hpd4 |= DC_HPDx_INT_EN;
6107 }
6108 if (rdev->irq.hpd[4]) {
6109 DRM_DEBUG("si_irq_set: hpd 5\n");
6110 hpd5 |= DC_HPDx_INT_EN;
6111 }
6112 if (rdev->irq.hpd[5]) {
6113 DRM_DEBUG("si_irq_set: hpd 6\n");
6114 hpd6 |= DC_HPDx_INT_EN;
6115 }
Alex Deucher25a857f2012-03-20 17:18:22 -04006116
6117 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
6118 WREG32(CP_INT_CNTL_RING1, cp_int_cntl1);
6119 WREG32(CP_INT_CNTL_RING2, cp_int_cntl2);
6120
Alex Deucher8c5fd7e2012-12-04 15:28:18 -05006121 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, dma_cntl);
6122 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, dma_cntl1);
6123
Alex Deucher25a857f2012-03-20 17:18:22 -04006124 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
6125
Alex Deuchera9e61412013-06-25 17:56:16 -04006126 if (rdev->irq.dpm_thermal) {
6127 DRM_DEBUG("dpm thermal\n");
6128 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
6129 }
6130
Alex Deucher51535502012-08-30 14:34:30 -04006131 if (rdev->num_crtc >= 2) {
6132 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
6133 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
6134 }
Alex Deucher25a857f2012-03-20 17:18:22 -04006135 if (rdev->num_crtc >= 4) {
6136 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
6137 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
6138 }
6139 if (rdev->num_crtc >= 6) {
6140 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
6141 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
6142 }
6143
Alex Deucher51535502012-08-30 14:34:30 -04006144 if (rdev->num_crtc >= 2) {
Christian Königf5d636d2014-04-23 20:46:06 +02006145 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
6146 GRPH_PFLIP_INT_MASK);
6147 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
6148 GRPH_PFLIP_INT_MASK);
Alex Deucher51535502012-08-30 14:34:30 -04006149 }
Alex Deucher25a857f2012-03-20 17:18:22 -04006150 if (rdev->num_crtc >= 4) {
Christian Königf5d636d2014-04-23 20:46:06 +02006151 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
6152 GRPH_PFLIP_INT_MASK);
6153 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
6154 GRPH_PFLIP_INT_MASK);
Alex Deucher25a857f2012-03-20 17:18:22 -04006155 }
6156 if (rdev->num_crtc >= 6) {
Christian Königf5d636d2014-04-23 20:46:06 +02006157 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
6158 GRPH_PFLIP_INT_MASK);
6159 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
6160 GRPH_PFLIP_INT_MASK);
Alex Deucher25a857f2012-03-20 17:18:22 -04006161 }
6162
Alex Deucher51535502012-08-30 14:34:30 -04006163 if (!ASIC_IS_NODCE(rdev)) {
6164 WREG32(DC_HPD1_INT_CONTROL, hpd1);
6165 WREG32(DC_HPD2_INT_CONTROL, hpd2);
6166 WREG32(DC_HPD3_INT_CONTROL, hpd3);
6167 WREG32(DC_HPD4_INT_CONTROL, hpd4);
6168 WREG32(DC_HPD5_INT_CONTROL, hpd5);
6169 WREG32(DC_HPD6_INT_CONTROL, hpd6);
6170 }
Alex Deucher25a857f2012-03-20 17:18:22 -04006171
Alex Deuchera9e61412013-06-25 17:56:16 -04006172 WREG32(CG_THERMAL_INT, thermal_int);
6173
Alex Deucher25a857f2012-03-20 17:18:22 -04006174 return 0;
6175}
6176
6177static inline void si_irq_ack(struct radeon_device *rdev)
6178{
6179 u32 tmp;
6180
Alex Deucher51535502012-08-30 14:34:30 -04006181 if (ASIC_IS_NODCE(rdev))
6182 return;
6183
Alex Deucher25a857f2012-03-20 17:18:22 -04006184 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
6185 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
6186 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
6187 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
6188 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
6189 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
6190 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
6191 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
6192 if (rdev->num_crtc >= 4) {
6193 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
6194 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
6195 }
6196 if (rdev->num_crtc >= 6) {
6197 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
6198 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
6199 }
6200
6201 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
6202 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6203 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
6204 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6205 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
6206 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
6207 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
6208 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
6209 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
6210 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
6211 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
6212 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
6213
6214 if (rdev->num_crtc >= 4) {
6215 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
6216 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6217 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
6218 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6219 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
6220 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
6221 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
6222 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
6223 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
6224 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
6225 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
6226 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
6227 }
6228
6229 if (rdev->num_crtc >= 6) {
6230 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
6231 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6232 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
6233 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6234 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
6235 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
6236 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
6237 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
6238 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
6239 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
6240 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
6241 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
6242 }
6243
6244 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
6245 tmp = RREG32(DC_HPD1_INT_CONTROL);
6246 tmp |= DC_HPDx_INT_ACK;
6247 WREG32(DC_HPD1_INT_CONTROL, tmp);
6248 }
6249 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
6250 tmp = RREG32(DC_HPD2_INT_CONTROL);
6251 tmp |= DC_HPDx_INT_ACK;
6252 WREG32(DC_HPD2_INT_CONTROL, tmp);
6253 }
6254 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6255 tmp = RREG32(DC_HPD3_INT_CONTROL);
6256 tmp |= DC_HPDx_INT_ACK;
6257 WREG32(DC_HPD3_INT_CONTROL, tmp);
6258 }
6259 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6260 tmp = RREG32(DC_HPD4_INT_CONTROL);
6261 tmp |= DC_HPDx_INT_ACK;
6262 WREG32(DC_HPD4_INT_CONTROL, tmp);
6263 }
6264 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6265 tmp = RREG32(DC_HPD5_INT_CONTROL);
6266 tmp |= DC_HPDx_INT_ACK;
6267 WREG32(DC_HPD5_INT_CONTROL, tmp);
6268 }
6269 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6270 tmp = RREG32(DC_HPD5_INT_CONTROL);
6271 tmp |= DC_HPDx_INT_ACK;
6272 WREG32(DC_HPD6_INT_CONTROL, tmp);
6273 }
6274}
6275
6276static void si_irq_disable(struct radeon_device *rdev)
6277{
6278 si_disable_interrupts(rdev);
6279 /* Wait and acknowledge irq */
6280 mdelay(1);
6281 si_irq_ack(rdev);
6282 si_disable_interrupt_state(rdev);
6283}
6284
6285static void si_irq_suspend(struct radeon_device *rdev)
6286{
6287 si_irq_disable(rdev);
6288 si_rlc_stop(rdev);
6289}
6290
Alex Deucher9b136d52012-03-20 17:18:23 -04006291static void si_irq_fini(struct radeon_device *rdev)
6292{
6293 si_irq_suspend(rdev);
6294 r600_ih_ring_fini(rdev);
6295}
6296
Alex Deucher25a857f2012-03-20 17:18:22 -04006297static inline u32 si_get_ih_wptr(struct radeon_device *rdev)
6298{
6299 u32 wptr, tmp;
6300
6301 if (rdev->wb.enabled)
6302 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
6303 else
6304 wptr = RREG32(IH_RB_WPTR);
6305
6306 if (wptr & RB_OVERFLOW) {
6307 /* When a ring buffer overflow happen start parsing interrupt
6308 * from the last not overwritten vector (wptr + 16). Hopefully
6309 * this should allow us to catchup.
6310 */
6311 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
6312 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
6313 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
6314 tmp = RREG32(IH_RB_CNTL);
6315 tmp |= IH_WPTR_OVERFLOW_CLEAR;
6316 WREG32(IH_RB_CNTL, tmp);
Christian Könige8c214d2014-07-23 09:47:58 +02006317 wptr &= ~RB_OVERFLOW;
Alex Deucher25a857f2012-03-20 17:18:22 -04006318 }
6319 return (wptr & rdev->ih.ptr_mask);
6320}
6321
6322/* SI IV Ring
6323 * Each IV ring entry is 128 bits:
6324 * [7:0] - interrupt source id
6325 * [31:8] - reserved
6326 * [59:32] - interrupt source data
6327 * [63:60] - reserved
6328 * [71:64] - RINGID
6329 * [79:72] - VMID
6330 * [127:80] - reserved
6331 */
6332int si_irq_process(struct radeon_device *rdev)
6333{
6334 u32 wptr;
6335 u32 rptr;
6336 u32 src_id, src_data, ring_id;
6337 u32 ring_index;
Alex Deucher25a857f2012-03-20 17:18:22 -04006338 bool queue_hotplug = false;
Alex Deuchera9e61412013-06-25 17:56:16 -04006339 bool queue_thermal = false;
Alex Deucherfbf6dc72013-06-13 18:47:58 -04006340 u32 status, addr;
Alex Deucher25a857f2012-03-20 17:18:22 -04006341
6342 if (!rdev->ih.enabled || rdev->shutdown)
6343 return IRQ_NONE;
6344
6345 wptr = si_get_ih_wptr(rdev);
Christian Koenigc20dc362012-05-16 21:45:24 +02006346
6347restart_ih:
6348 /* is somebody else already processing irqs? */
6349 if (atomic_xchg(&rdev->ih.lock, 1))
6350 return IRQ_NONE;
6351
Alex Deucher25a857f2012-03-20 17:18:22 -04006352 rptr = rdev->ih.rptr;
6353 DRM_DEBUG("si_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
6354
Alex Deucher25a857f2012-03-20 17:18:22 -04006355 /* Order reading of wptr vs. reading of IH ring data */
6356 rmb();
6357
6358 /* display interrupts */
6359 si_irq_ack(rdev);
6360
Alex Deucher25a857f2012-03-20 17:18:22 -04006361 while (rptr != wptr) {
6362 /* wptr/rptr are in bytes! */
6363 ring_index = rptr / 4;
6364 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
6365 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
6366 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
6367
6368 switch (src_id) {
6369 case 1: /* D1 vblank/vline */
6370 switch (src_data) {
6371 case 0: /* D1 vblank */
6372 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
6373 if (rdev->irq.crtc_vblank_int[0]) {
6374 drm_handle_vblank(rdev->ddev, 0);
6375 rdev->pm.vblank_sync = true;
6376 wake_up(&rdev->irq.vblank_queue);
6377 }
Christian Koenig736fc372012-05-17 19:52:00 +02006378 if (atomic_read(&rdev->irq.pflip[0]))
Christian König1a0e7912014-05-27 16:49:21 +02006379 radeon_crtc_handle_vblank(rdev, 0);
Alex Deucher25a857f2012-03-20 17:18:22 -04006380 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
6381 DRM_DEBUG("IH: D1 vblank\n");
6382 }
6383 break;
6384 case 1: /* D1 vline */
6385 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
6386 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
6387 DRM_DEBUG("IH: D1 vline\n");
6388 }
6389 break;
6390 default:
6391 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6392 break;
6393 }
6394 break;
6395 case 2: /* D2 vblank/vline */
6396 switch (src_data) {
6397 case 0: /* D2 vblank */
6398 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
6399 if (rdev->irq.crtc_vblank_int[1]) {
6400 drm_handle_vblank(rdev->ddev, 1);
6401 rdev->pm.vblank_sync = true;
6402 wake_up(&rdev->irq.vblank_queue);
6403 }
Christian Koenig736fc372012-05-17 19:52:00 +02006404 if (atomic_read(&rdev->irq.pflip[1]))
Christian König1a0e7912014-05-27 16:49:21 +02006405 radeon_crtc_handle_vblank(rdev, 1);
Alex Deucher25a857f2012-03-20 17:18:22 -04006406 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
6407 DRM_DEBUG("IH: D2 vblank\n");
6408 }
6409 break;
6410 case 1: /* D2 vline */
6411 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
6412 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
6413 DRM_DEBUG("IH: D2 vline\n");
6414 }
6415 break;
6416 default:
6417 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6418 break;
6419 }
6420 break;
6421 case 3: /* D3 vblank/vline */
6422 switch (src_data) {
6423 case 0: /* D3 vblank */
6424 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
6425 if (rdev->irq.crtc_vblank_int[2]) {
6426 drm_handle_vblank(rdev->ddev, 2);
6427 rdev->pm.vblank_sync = true;
6428 wake_up(&rdev->irq.vblank_queue);
6429 }
Christian Koenig736fc372012-05-17 19:52:00 +02006430 if (atomic_read(&rdev->irq.pflip[2]))
Christian König1a0e7912014-05-27 16:49:21 +02006431 radeon_crtc_handle_vblank(rdev, 2);
Alex Deucher25a857f2012-03-20 17:18:22 -04006432 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
6433 DRM_DEBUG("IH: D3 vblank\n");
6434 }
6435 break;
6436 case 1: /* D3 vline */
6437 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
6438 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
6439 DRM_DEBUG("IH: D3 vline\n");
6440 }
6441 break;
6442 default:
6443 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6444 break;
6445 }
6446 break;
6447 case 4: /* D4 vblank/vline */
6448 switch (src_data) {
6449 case 0: /* D4 vblank */
6450 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
6451 if (rdev->irq.crtc_vblank_int[3]) {
6452 drm_handle_vblank(rdev->ddev, 3);
6453 rdev->pm.vblank_sync = true;
6454 wake_up(&rdev->irq.vblank_queue);
6455 }
Christian Koenig736fc372012-05-17 19:52:00 +02006456 if (atomic_read(&rdev->irq.pflip[3]))
Christian König1a0e7912014-05-27 16:49:21 +02006457 radeon_crtc_handle_vblank(rdev, 3);
Alex Deucher25a857f2012-03-20 17:18:22 -04006458 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
6459 DRM_DEBUG("IH: D4 vblank\n");
6460 }
6461 break;
6462 case 1: /* D4 vline */
6463 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
6464 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
6465 DRM_DEBUG("IH: D4 vline\n");
6466 }
6467 break;
6468 default:
6469 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6470 break;
6471 }
6472 break;
6473 case 5: /* D5 vblank/vline */
6474 switch (src_data) {
6475 case 0: /* D5 vblank */
6476 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
6477 if (rdev->irq.crtc_vblank_int[4]) {
6478 drm_handle_vblank(rdev->ddev, 4);
6479 rdev->pm.vblank_sync = true;
6480 wake_up(&rdev->irq.vblank_queue);
6481 }
Christian Koenig736fc372012-05-17 19:52:00 +02006482 if (atomic_read(&rdev->irq.pflip[4]))
Christian König1a0e7912014-05-27 16:49:21 +02006483 radeon_crtc_handle_vblank(rdev, 4);
Alex Deucher25a857f2012-03-20 17:18:22 -04006484 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
6485 DRM_DEBUG("IH: D5 vblank\n");
6486 }
6487 break;
6488 case 1: /* D5 vline */
6489 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
6490 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
6491 DRM_DEBUG("IH: D5 vline\n");
6492 }
6493 break;
6494 default:
6495 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6496 break;
6497 }
6498 break;
6499 case 6: /* D6 vblank/vline */
6500 switch (src_data) {
6501 case 0: /* D6 vblank */
6502 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
6503 if (rdev->irq.crtc_vblank_int[5]) {
6504 drm_handle_vblank(rdev->ddev, 5);
6505 rdev->pm.vblank_sync = true;
6506 wake_up(&rdev->irq.vblank_queue);
6507 }
Christian Koenig736fc372012-05-17 19:52:00 +02006508 if (atomic_read(&rdev->irq.pflip[5]))
Christian König1a0e7912014-05-27 16:49:21 +02006509 radeon_crtc_handle_vblank(rdev, 5);
Alex Deucher25a857f2012-03-20 17:18:22 -04006510 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
6511 DRM_DEBUG("IH: D6 vblank\n");
6512 }
6513 break;
6514 case 1: /* D6 vline */
6515 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
6516 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
6517 DRM_DEBUG("IH: D6 vline\n");
6518 }
6519 break;
6520 default:
6521 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6522 break;
6523 }
6524 break;
Christian Königf5d636d2014-04-23 20:46:06 +02006525 case 8: /* D1 page flip */
6526 case 10: /* D2 page flip */
6527 case 12: /* D3 page flip */
6528 case 14: /* D4 page flip */
6529 case 16: /* D5 page flip */
6530 case 18: /* D6 page flip */
6531 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
Mario Kleiner39dc5452014-07-29 06:21:44 +02006532 if (radeon_use_pflipirq > 0)
6533 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
Christian Königf5d636d2014-04-23 20:46:06 +02006534 break;
Alex Deucher25a857f2012-03-20 17:18:22 -04006535 case 42: /* HPD hotplug */
6536 switch (src_data) {
6537 case 0:
6538 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
6539 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
6540 queue_hotplug = true;
6541 DRM_DEBUG("IH: HPD1\n");
6542 }
6543 break;
6544 case 1:
6545 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
6546 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
6547 queue_hotplug = true;
6548 DRM_DEBUG("IH: HPD2\n");
6549 }
6550 break;
6551 case 2:
6552 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6553 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
6554 queue_hotplug = true;
6555 DRM_DEBUG("IH: HPD3\n");
6556 }
6557 break;
6558 case 3:
6559 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6560 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
6561 queue_hotplug = true;
6562 DRM_DEBUG("IH: HPD4\n");
6563 }
6564 break;
6565 case 4:
6566 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6567 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
6568 queue_hotplug = true;
6569 DRM_DEBUG("IH: HPD5\n");
6570 }
6571 break;
6572 case 5:
6573 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6574 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
6575 queue_hotplug = true;
6576 DRM_DEBUG("IH: HPD6\n");
6577 }
6578 break;
6579 default:
6580 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6581 break;
6582 }
6583 break;
Christian Königb927e1c2014-01-30 19:01:16 +01006584 case 124: /* UVD */
6585 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
6586 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
6587 break;
Christian Königae133a12012-09-18 15:30:44 -04006588 case 146:
6589 case 147:
Alex Deucherfbf6dc72013-06-13 18:47:58 -04006590 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
6591 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
Christian König9b7d7862014-07-07 11:16:29 +02006592 /* reset addr and status */
6593 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
6594 if (addr == 0x0 && status == 0x0)
6595 break;
Christian Königae133a12012-09-18 15:30:44 -04006596 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
6597 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
Alex Deucherfbf6dc72013-06-13 18:47:58 -04006598 addr);
Christian Königae133a12012-09-18 15:30:44 -04006599 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
Alex Deucherfbf6dc72013-06-13 18:47:58 -04006600 status);
6601 si_vm_decode_fault(rdev, status, addr);
Christian Königae133a12012-09-18 15:30:44 -04006602 break;
Alex Deucher25a857f2012-03-20 17:18:22 -04006603 case 176: /* RINGID0 CP_INT */
6604 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6605 break;
6606 case 177: /* RINGID1 CP_INT */
6607 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6608 break;
6609 case 178: /* RINGID2 CP_INT */
6610 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6611 break;
6612 case 181: /* CP EOP event */
6613 DRM_DEBUG("IH: CP EOP\n");
6614 switch (ring_id) {
6615 case 0:
6616 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6617 break;
6618 case 1:
6619 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6620 break;
6621 case 2:
6622 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6623 break;
6624 }
6625 break;
Alex Deucher8c5fd7e2012-12-04 15:28:18 -05006626 case 224: /* DMA trap event */
6627 DRM_DEBUG("IH: DMA trap\n");
6628 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
6629 break;
Alex Deuchera9e61412013-06-25 17:56:16 -04006630 case 230: /* thermal low to high */
6631 DRM_DEBUG("IH: thermal low to high\n");
6632 rdev->pm.dpm.thermal.high_to_low = false;
6633 queue_thermal = true;
6634 break;
6635 case 231: /* thermal high to low */
6636 DRM_DEBUG("IH: thermal high to low\n");
6637 rdev->pm.dpm.thermal.high_to_low = true;
6638 queue_thermal = true;
6639 break;
Alex Deucher25a857f2012-03-20 17:18:22 -04006640 case 233: /* GUI IDLE */
6641 DRM_DEBUG("IH: GUI idle\n");
Alex Deucher25a857f2012-03-20 17:18:22 -04006642 break;
Alex Deucher8c5fd7e2012-12-04 15:28:18 -05006643 case 244: /* DMA trap event */
6644 DRM_DEBUG("IH: DMA1 trap\n");
6645 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6646 break;
Alex Deucher25a857f2012-03-20 17:18:22 -04006647 default:
6648 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6649 break;
6650 }
6651
6652 /* wptr/rptr are in bytes! */
6653 rptr += 16;
6654 rptr &= rdev->ih.ptr_mask;
6655 }
Alex Deucher25a857f2012-03-20 17:18:22 -04006656 if (queue_hotplug)
6657 schedule_work(&rdev->hotplug_work);
Alex Deuchera9e61412013-06-25 17:56:16 -04006658 if (queue_thermal && rdev->pm.dpm_enabled)
6659 schedule_work(&rdev->pm.dpm.thermal.work);
Alex Deucher25a857f2012-03-20 17:18:22 -04006660 rdev->ih.rptr = rptr;
6661 WREG32(IH_RB_RPTR, rdev->ih.rptr);
Christian Koenigc20dc362012-05-16 21:45:24 +02006662 atomic_set(&rdev->ih.lock, 0);
6663
6664 /* make sure wptr hasn't changed while processing */
6665 wptr = si_get_ih_wptr(rdev);
6666 if (wptr != rptr)
6667 goto restart_ih;
6668
Alex Deucher25a857f2012-03-20 17:18:22 -04006669 return IRQ_HANDLED;
6670}
6671
Alex Deucher9b136d52012-03-20 17:18:23 -04006672/*
6673 * startup/shutdown callbacks
6674 */
6675static int si_startup(struct radeon_device *rdev)
6676{
6677 struct radeon_ring *ring;
6678 int r;
6679
Alex Deucherb9d305d2013-02-14 17:16:51 -05006680 /* enable pcie gen2/3 link */
6681 si_pcie_gen3_enable(rdev);
Alex Deuchere0bcf1652013-02-15 11:56:59 -05006682 /* enable aspm */
6683 si_program_aspm(rdev);
Alex Deucherb9d305d2013-02-14 17:16:51 -05006684
Alex Deuchere5903d32013-08-30 08:58:20 -04006685 /* scratch needs to be initialized before MC */
6686 r = r600_vram_scratch_init(rdev);
6687 if (r)
6688 return r;
6689
Alex Deucher6fab3feb2013-08-04 12:13:17 -04006690 si_mc_program(rdev);
6691
Alex Deucher6c7bcce2013-12-18 14:07:14 -05006692 if (!rdev->pm.dpm_enabled) {
6693 r = si_mc_load_microcode(rdev);
6694 if (r) {
6695 DRM_ERROR("Failed to load MC firmware!\n");
6696 return r;
6697 }
Alex Deucher9b136d52012-03-20 17:18:23 -04006698 }
6699
Alex Deucher9b136d52012-03-20 17:18:23 -04006700 r = si_pcie_gart_enable(rdev);
6701 if (r)
6702 return r;
6703 si_gpu_init(rdev);
6704
Alex Deucher9b136d52012-03-20 17:18:23 -04006705 /* allocate rlc buffers */
Alex Deucher1fd11772013-04-17 17:53:50 -04006706 if (rdev->family == CHIP_VERDE) {
6707 rdev->rlc.reg_list = verde_rlc_save_restore_register_list;
6708 rdev->rlc.reg_list_size =
6709 (u32)ARRAY_SIZE(verde_rlc_save_restore_register_list);
6710 }
6711 rdev->rlc.cs_data = si_cs_data;
6712 r = sumo_rlc_init(rdev);
Alex Deucher9b136d52012-03-20 17:18:23 -04006713 if (r) {
6714 DRM_ERROR("Failed to init rlc BOs!\n");
6715 return r;
6716 }
6717
6718 /* allocate wb buffer */
6719 r = radeon_wb_init(rdev);
6720 if (r)
6721 return r;
6722
6723 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
6724 if (r) {
6725 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6726 return r;
6727 }
6728
6729 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6730 if (r) {
6731 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6732 return r;
6733 }
6734
6735 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6736 if (r) {
6737 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6738 return r;
6739 }
6740
Alex Deucher8c5fd7e2012-12-04 15:28:18 -05006741 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
6742 if (r) {
6743 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6744 return r;
6745 }
6746
6747 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6748 if (r) {
6749 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6750 return r;
6751 }
6752
Alex Deucher1df0d522013-04-26 18:03:44 -04006753 if (rdev->has_uvd) {
Christian Könige409b122013-08-13 11:56:53 +02006754 r = uvd_v2_2_resume(rdev);
Alex Deucher1df0d522013-04-26 18:03:44 -04006755 if (!r) {
6756 r = radeon_fence_driver_start_ring(rdev,
6757 R600_RING_TYPE_UVD_INDEX);
6758 if (r)
6759 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
6760 }
Christian Königf2ba57b2013-04-08 12:41:29 +02006761 if (r)
Alex Deucher1df0d522013-04-26 18:03:44 -04006762 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
Christian Königf2ba57b2013-04-08 12:41:29 +02006763 }
Christian Königf2ba57b2013-04-08 12:41:29 +02006764
Alex Deucher9b136d52012-03-20 17:18:23 -04006765 /* Enable IRQ */
Adis Hamziće49f3952013-06-02 16:47:54 +02006766 if (!rdev->irq.installed) {
6767 r = radeon_irq_kms_init(rdev);
6768 if (r)
6769 return r;
6770 }
6771
Alex Deucher9b136d52012-03-20 17:18:23 -04006772 r = si_irq_init(rdev);
6773 if (r) {
6774 DRM_ERROR("radeon: IH init failed (%d).\n", r);
6775 radeon_irq_kms_fini(rdev);
6776 return r;
6777 }
6778 si_irq_set(rdev);
6779
6780 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6781 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
Christian König2e1e6da2013-08-13 11:56:52 +02006782 RADEON_CP_PACKET2);
Alex Deucher9b136d52012-03-20 17:18:23 -04006783 if (r)
6784 return r;
6785
6786 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6787 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
Christian König2e1e6da2013-08-13 11:56:52 +02006788 RADEON_CP_PACKET2);
Alex Deucher9b136d52012-03-20 17:18:23 -04006789 if (r)
6790 return r;
6791
6792 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6793 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
Christian König2e1e6da2013-08-13 11:56:52 +02006794 RADEON_CP_PACKET2);
Alex Deucher9b136d52012-03-20 17:18:23 -04006795 if (r)
6796 return r;
6797
Alex Deucher8c5fd7e2012-12-04 15:28:18 -05006798 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6799 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
Christian König2e1e6da2013-08-13 11:56:52 +02006800 DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
Alex Deucher8c5fd7e2012-12-04 15:28:18 -05006801 if (r)
6802 return r;
6803
6804 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6805 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
Christian König2e1e6da2013-08-13 11:56:52 +02006806 DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
Alex Deucher8c5fd7e2012-12-04 15:28:18 -05006807 if (r)
6808 return r;
6809
Alex Deucher9b136d52012-03-20 17:18:23 -04006810 r = si_cp_load_microcode(rdev);
6811 if (r)
6812 return r;
6813 r = si_cp_resume(rdev);
6814 if (r)
6815 return r;
6816
Alex Deucher8c5fd7e2012-12-04 15:28:18 -05006817 r = cayman_dma_resume(rdev);
6818 if (r)
6819 return r;
6820
Alex Deucher1df0d522013-04-26 18:03:44 -04006821 if (rdev->has_uvd) {
6822 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6823 if (ring->ring_size) {
Christian König02c9f7f2013-08-13 11:56:51 +02006824 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
Christian König2e1e6da2013-08-13 11:56:52 +02006825 RADEON_CP_PACKET2);
Alex Deucher1df0d522013-04-26 18:03:44 -04006826 if (!r)
Christian Könige409b122013-08-13 11:56:53 +02006827 r = uvd_v1_0_init(rdev);
Alex Deucher1df0d522013-04-26 18:03:44 -04006828 if (r)
6829 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
6830 }
Christian Königf2ba57b2013-04-08 12:41:29 +02006831 }
6832
Christian König2898c342012-07-05 11:55:34 +02006833 r = radeon_ib_pool_init(rdev);
6834 if (r) {
6835 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
Alex Deucher9b136d52012-03-20 17:18:23 -04006836 return r;
Christian König2898c342012-07-05 11:55:34 +02006837 }
Alex Deucher9b136d52012-03-20 17:18:23 -04006838
Christian Königc6105f22012-07-05 14:32:00 +02006839 r = radeon_vm_manager_init(rdev);
6840 if (r) {
6841 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
Alex Deucher9b136d52012-03-20 17:18:23 -04006842 return r;
Christian Königc6105f22012-07-05 14:32:00 +02006843 }
Alex Deucher9b136d52012-03-20 17:18:23 -04006844
Alex Deucherb5306022013-07-31 16:51:33 -04006845 r = dce6_audio_init(rdev);
6846 if (r)
6847 return r;
6848
Alex Deucher9b136d52012-03-20 17:18:23 -04006849 return 0;
6850}
6851
6852int si_resume(struct radeon_device *rdev)
6853{
6854 int r;
6855
6856 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
6857 * posting will perform necessary task to bring back GPU into good
6858 * shape.
6859 */
6860 /* post card */
6861 atom_asic_init(rdev->mode_info.atom_context);
6862
Alex Deucher205996c2013-03-01 17:08:42 -05006863 /* init golden registers */
6864 si_init_golden_registers(rdev);
6865
Alex Deucherbc6a6292014-02-25 12:01:28 -05006866 if (rdev->pm.pm_method == PM_METHOD_DPM)
6867 radeon_pm_resume(rdev);
Alex Deucher6c7bcce2013-12-18 14:07:14 -05006868
Alex Deucher9b136d52012-03-20 17:18:23 -04006869 rdev->accel_working = true;
6870 r = si_startup(rdev);
6871 if (r) {
6872 DRM_ERROR("si startup failed on resume\n");
6873 rdev->accel_working = false;
6874 return r;
6875 }
6876
6877 return r;
6878
6879}
6880
6881int si_suspend(struct radeon_device *rdev)
6882{
Alex Deucher6c7bcce2013-12-18 14:07:14 -05006883 radeon_pm_suspend(rdev);
Alex Deucherb5306022013-07-31 16:51:33 -04006884 dce6_audio_fini(rdev);
Alex Deucherfa3daf92013-03-11 15:32:26 -04006885 radeon_vm_manager_fini(rdev);
Alex Deucher9b136d52012-03-20 17:18:23 -04006886 si_cp_enable(rdev, false);
Alex Deucher8c5fd7e2012-12-04 15:28:18 -05006887 cayman_dma_stop(rdev);
Alex Deucher1df0d522013-04-26 18:03:44 -04006888 if (rdev->has_uvd) {
Christian Könige409b122013-08-13 11:56:53 +02006889 uvd_v1_0_fini(rdev);
Alex Deucher1df0d522013-04-26 18:03:44 -04006890 radeon_uvd_suspend(rdev);
6891 }
Alex Deuchere16866e2013-08-08 19:34:07 -04006892 si_fini_pg(rdev);
6893 si_fini_cg(rdev);
Alex Deucher9b136d52012-03-20 17:18:23 -04006894 si_irq_suspend(rdev);
6895 radeon_wb_disable(rdev);
6896 si_pcie_gart_disable(rdev);
6897 return 0;
6898}
6899
6900/* Plan is to move initialization in that function and use
6901 * helper function so that radeon_device_init pretty much
6902 * do nothing more than calling asic specific function. This
6903 * should also allow to remove a bunch of callback function
6904 * like vram_info.
6905 */
6906int si_init(struct radeon_device *rdev)
6907{
6908 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6909 int r;
6910
Alex Deucher9b136d52012-03-20 17:18:23 -04006911 /* Read BIOS */
6912 if (!radeon_get_bios(rdev)) {
6913 if (ASIC_IS_AVIVO(rdev))
6914 return -EINVAL;
6915 }
6916 /* Must be an ATOMBIOS */
6917 if (!rdev->is_atom_bios) {
6918 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
6919 return -EINVAL;
6920 }
6921 r = radeon_atombios_init(rdev);
6922 if (r)
6923 return r;
6924
6925 /* Post card if necessary */
6926 if (!radeon_card_posted(rdev)) {
6927 if (!rdev->bios) {
6928 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
6929 return -EINVAL;
6930 }
6931 DRM_INFO("GPU not posted. posting now...\n");
6932 atom_asic_init(rdev->mode_info.atom_context);
6933 }
Alex Deucher205996c2013-03-01 17:08:42 -05006934 /* init golden registers */
6935 si_init_golden_registers(rdev);
Alex Deucher9b136d52012-03-20 17:18:23 -04006936 /* Initialize scratch registers */
6937 si_scratch_init(rdev);
6938 /* Initialize surface registers */
6939 radeon_surface_init(rdev);
6940 /* Initialize clocks */
6941 radeon_get_clock_info(rdev->ddev);
6942
6943 /* Fence driver */
6944 r = radeon_fence_driver_init(rdev);
6945 if (r)
6946 return r;
6947
6948 /* initialize memory controller */
6949 r = si_mc_init(rdev);
6950 if (r)
6951 return r;
6952 /* Memory manager */
6953 r = radeon_bo_init(rdev);
6954 if (r)
6955 return r;
6956
Alex Deucher01ac8792013-12-18 19:11:27 -05006957 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
6958 !rdev->rlc_fw || !rdev->mc_fw) {
6959 r = si_init_microcode(rdev);
6960 if (r) {
6961 DRM_ERROR("Failed to load firmware!\n");
6962 return r;
6963 }
6964 }
6965
Alex Deucher6c7bcce2013-12-18 14:07:14 -05006966 /* Initialize power management */
6967 radeon_pm_init(rdev);
6968
Alex Deucher9b136d52012-03-20 17:18:23 -04006969 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6970 ring->ring_obj = NULL;
6971 r600_ring_init(rdev, ring, 1024 * 1024);
6972
6973 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6974 ring->ring_obj = NULL;
6975 r600_ring_init(rdev, ring, 1024 * 1024);
6976
6977 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6978 ring->ring_obj = NULL;
6979 r600_ring_init(rdev, ring, 1024 * 1024);
6980
Alex Deucher8c5fd7e2012-12-04 15:28:18 -05006981 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6982 ring->ring_obj = NULL;
6983 r600_ring_init(rdev, ring, 64 * 1024);
6984
6985 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6986 ring->ring_obj = NULL;
6987 r600_ring_init(rdev, ring, 64 * 1024);
6988
Alex Deucher1df0d522013-04-26 18:03:44 -04006989 if (rdev->has_uvd) {
6990 r = radeon_uvd_init(rdev);
6991 if (!r) {
6992 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6993 ring->ring_obj = NULL;
6994 r600_ring_init(rdev, ring, 4096);
6995 }
Christian Königf2ba57b2013-04-08 12:41:29 +02006996 }
6997
Alex Deucher9b136d52012-03-20 17:18:23 -04006998 rdev->ih.ring_obj = NULL;
6999 r600_ih_ring_init(rdev, 64 * 1024);
7000
7001 r = r600_pcie_gart_init(rdev);
7002 if (r)
7003 return r;
7004
Alex Deucher9b136d52012-03-20 17:18:23 -04007005 rdev->accel_working = true;
Alex Deucher9b136d52012-03-20 17:18:23 -04007006 r = si_startup(rdev);
7007 if (r) {
7008 dev_err(rdev->dev, "disabling GPU acceleration\n");
7009 si_cp_fini(rdev);
Alex Deucher8c5fd7e2012-12-04 15:28:18 -05007010 cayman_dma_fini(rdev);
Alex Deucher9b136d52012-03-20 17:18:23 -04007011 si_irq_fini(rdev);
Alex Deucher1fd11772013-04-17 17:53:50 -04007012 sumo_rlc_fini(rdev);
Alex Deucher9b136d52012-03-20 17:18:23 -04007013 radeon_wb_fini(rdev);
Christian König2898c342012-07-05 11:55:34 +02007014 radeon_ib_pool_fini(rdev);
Alex Deucher9b136d52012-03-20 17:18:23 -04007015 radeon_vm_manager_fini(rdev);
7016 radeon_irq_kms_fini(rdev);
7017 si_pcie_gart_fini(rdev);
7018 rdev->accel_working = false;
7019 }
7020
7021 /* Don't start up if the MC ucode is missing.
7022 * The default clocks and voltages before the MC ucode
7023 * is loaded are not suffient for advanced operations.
7024 */
7025 if (!rdev->mc_fw) {
7026 DRM_ERROR("radeon: MC ucode required for NI+.\n");
7027 return -EINVAL;
7028 }
7029
7030 return 0;
7031}
7032
7033void si_fini(struct radeon_device *rdev)
7034{
Alex Deucher6c7bcce2013-12-18 14:07:14 -05007035 radeon_pm_fini(rdev);
Alex Deucher9b136d52012-03-20 17:18:23 -04007036 si_cp_fini(rdev);
Alex Deucher8c5fd7e2012-12-04 15:28:18 -05007037 cayman_dma_fini(rdev);
Alex Deucherf8f84ac2013-03-07 12:56:35 -05007038 si_fini_pg(rdev);
Alex Deuchere16866e2013-08-08 19:34:07 -04007039 si_fini_cg(rdev);
Alex Deuchere0bcf1652013-02-15 11:56:59 -05007040 si_irq_fini(rdev);
Alex Deucher1fd11772013-04-17 17:53:50 -04007041 sumo_rlc_fini(rdev);
Alex Deucher9b136d52012-03-20 17:18:23 -04007042 radeon_wb_fini(rdev);
7043 radeon_vm_manager_fini(rdev);
Christian König2898c342012-07-05 11:55:34 +02007044 radeon_ib_pool_fini(rdev);
Alex Deucher9b136d52012-03-20 17:18:23 -04007045 radeon_irq_kms_fini(rdev);
Christian König2858c002013-08-01 17:34:07 +02007046 if (rdev->has_uvd) {
Christian Könige409b122013-08-13 11:56:53 +02007047 uvd_v1_0_fini(rdev);
Alex Deucher1df0d522013-04-26 18:03:44 -04007048 radeon_uvd_fini(rdev);
Christian König2858c002013-08-01 17:34:07 +02007049 }
Alex Deucher9b136d52012-03-20 17:18:23 -04007050 si_pcie_gart_fini(rdev);
7051 r600_vram_scratch_fini(rdev);
7052 radeon_gem_fini(rdev);
Alex Deucher9b136d52012-03-20 17:18:23 -04007053 radeon_fence_driver_fini(rdev);
7054 radeon_bo_fini(rdev);
7055 radeon_atombios_fini(rdev);
7056 kfree(rdev->bios);
7057 rdev->bios = NULL;
7058}
7059
Marek Olšák6759a0a2012-08-09 16:34:17 +02007060/**
Alex Deucherd0418892013-01-24 10:35:23 -05007061 * si_get_gpu_clock_counter - return GPU clock counter snapshot
Marek Olšák6759a0a2012-08-09 16:34:17 +02007062 *
7063 * @rdev: radeon_device pointer
7064 *
7065 * Fetches a GPU clock counter snapshot (SI).
7066 * Returns the 64 bit clock counter snapshot.
7067 */
Alex Deucherd0418892013-01-24 10:35:23 -05007068uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev)
Marek Olšák6759a0a2012-08-09 16:34:17 +02007069{
7070 uint64_t clock;
7071
7072 mutex_lock(&rdev->gpu_clock_mutex);
7073 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
7074 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
7075 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
7076 mutex_unlock(&rdev->gpu_clock_mutex);
7077 return clock;
7078}
Christian König2539eb02013-04-08 12:41:34 +02007079
Christian König2539eb02013-04-08 12:41:34 +02007080int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
7081{
Christian Königfacd1122013-04-29 11:55:02 +02007082 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
Christian König2539eb02013-04-08 12:41:34 +02007083 int r;
7084
Christian König4ed10832013-04-18 15:25:58 +02007085 /* bypass vclk and dclk with bclk */
7086 WREG32_P(CG_UPLL_FUNC_CNTL_2,
7087 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
7088 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
7089
7090 /* put PLL in bypass mode */
7091 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
7092
7093 if (!vclk || !dclk) {
7094 /* keep the Bypass mode, put PLL to sleep */
7095 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
7096 return 0;
7097 }
7098
Christian Königfacd1122013-04-29 11:55:02 +02007099 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
7100 16384, 0x03FFFFFF, 0, 128, 5,
7101 &fb_div, &vclk_div, &dclk_div);
7102 if (r)
7103 return r;
Christian König2539eb02013-04-08 12:41:34 +02007104
7105 /* set RESET_ANTI_MUX to 0 */
7106 WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
7107
7108 /* set VCO_MODE to 1 */
7109 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
7110
7111 /* toggle UPLL_SLEEP to 1 then back to 0 */
7112 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
7113 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
7114
7115 /* deassert UPLL_RESET */
7116 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
7117
7118 mdelay(1);
7119
Christian Königfacd1122013-04-29 11:55:02 +02007120 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
Christian König2539eb02013-04-08 12:41:34 +02007121 if (r)
7122 return r;
7123
7124 /* assert UPLL_RESET again */
7125 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
7126
7127 /* disable spread spectrum. */
7128 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
7129
7130 /* set feedback divider */
Christian Königfacd1122013-04-29 11:55:02 +02007131 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
Christian König2539eb02013-04-08 12:41:34 +02007132
7133 /* set ref divider to 0 */
7134 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
7135
Christian Königfacd1122013-04-29 11:55:02 +02007136 if (fb_div < 307200)
Christian König2539eb02013-04-08 12:41:34 +02007137 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
7138 else
7139 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
7140
7141 /* set PDIV_A and PDIV_B */
7142 WREG32_P(CG_UPLL_FUNC_CNTL_2,
Christian Königfacd1122013-04-29 11:55:02 +02007143 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
Christian König2539eb02013-04-08 12:41:34 +02007144 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
7145
7146 /* give the PLL some time to settle */
7147 mdelay(15);
7148
7149 /* deassert PLL_RESET */
7150 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
7151
7152 mdelay(15);
7153
7154 /* switch from bypass mode to normal mode */
7155 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
7156
Christian Königfacd1122013-04-29 11:55:02 +02007157 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
Christian König2539eb02013-04-08 12:41:34 +02007158 if (r)
7159 return r;
7160
7161 /* switch VCLK and DCLK selection */
7162 WREG32_P(CG_UPLL_FUNC_CNTL_2,
7163 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
7164 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
7165
7166 mdelay(100);
7167
7168 return 0;
7169}
Alex Deucherb9d305d2013-02-14 17:16:51 -05007170
7171static void si_pcie_gen3_enable(struct radeon_device *rdev)
7172{
7173 struct pci_dev *root = rdev->pdev->bus->self;
7174 int bridge_pos, gpu_pos;
7175 u32 speed_cntl, mask, current_data_rate;
7176 int ret, i;
7177 u16 tmp16;
7178
7179 if (radeon_pcie_gen2 == 0)
7180 return;
7181
7182 if (rdev->flags & RADEON_IS_IGP)
7183 return;
7184
7185 if (!(rdev->flags & RADEON_IS_PCIE))
7186 return;
7187
7188 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
7189 if (ret != 0)
7190 return;
7191
7192 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
7193 return;
7194
7195 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7196 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
7197 LC_CURRENT_DATA_RATE_SHIFT;
7198 if (mask & DRM_PCIE_SPEED_80) {
7199 if (current_data_rate == 2) {
7200 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
7201 return;
7202 }
7203 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
7204 } else if (mask & DRM_PCIE_SPEED_50) {
7205 if (current_data_rate == 1) {
7206 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
7207 return;
7208 }
7209 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
7210 }
7211
7212 bridge_pos = pci_pcie_cap(root);
7213 if (!bridge_pos)
7214 return;
7215
7216 gpu_pos = pci_pcie_cap(rdev->pdev);
7217 if (!gpu_pos)
7218 return;
7219
7220 if (mask & DRM_PCIE_SPEED_80) {
7221 /* re-try equalization if gen3 is not already enabled */
7222 if (current_data_rate != 2) {
7223 u16 bridge_cfg, gpu_cfg;
7224 u16 bridge_cfg2, gpu_cfg2;
7225 u32 max_lw, current_lw, tmp;
7226
7227 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
7228 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
7229
7230 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
7231 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
7232
7233 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
7234 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
7235
7236 tmp = RREG32_PCIE(PCIE_LC_STATUS1);
7237 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
7238 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
7239
7240 if (current_lw < max_lw) {
7241 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
7242 if (tmp & LC_RENEGOTIATION_SUPPORT) {
7243 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
7244 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
7245 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
7246 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
7247 }
7248 }
7249
7250 for (i = 0; i < 10; i++) {
7251 /* check status */
7252 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
7253 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
7254 break;
7255
7256 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
7257 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
7258
7259 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
7260 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
7261
7262 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7263 tmp |= LC_SET_QUIESCE;
7264 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7265
7266 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7267 tmp |= LC_REDO_EQ;
7268 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7269
7270 mdelay(100);
7271
7272 /* linkctl */
7273 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
7274 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
7275 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
7276 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
7277
7278 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
7279 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
7280 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
7281 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
7282
7283 /* linkctl2 */
7284 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
7285 tmp16 &= ~((1 << 4) | (7 << 9));
7286 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
7287 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
7288
7289 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
7290 tmp16 &= ~((1 << 4) | (7 << 9));
7291 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
7292 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
7293
7294 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7295 tmp &= ~LC_SET_QUIESCE;
7296 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7297 }
7298 }
7299 }
7300
7301 /* set the link speed */
7302 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
7303 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
7304 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
7305
7306 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
7307 tmp16 &= ~0xf;
7308 if (mask & DRM_PCIE_SPEED_80)
7309 tmp16 |= 3; /* gen3 */
7310 else if (mask & DRM_PCIE_SPEED_50)
7311 tmp16 |= 2; /* gen2 */
7312 else
7313 tmp16 |= 1; /* gen1 */
7314 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
7315
7316 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7317 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
7318 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
7319
7320 for (i = 0; i < rdev->usec_timeout; i++) {
7321 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7322 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
7323 break;
7324 udelay(1);
7325 }
7326}
7327
Alex Deuchere0bcf1652013-02-15 11:56:59 -05007328static void si_program_aspm(struct radeon_device *rdev)
7329{
7330 u32 data, orig;
7331 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
7332 bool disable_clkreq = false;
7333
Alex Deucher1294d4a2013-07-16 15:58:50 -04007334 if (radeon_aspm == 0)
7335 return;
7336
Alex Deuchere0bcf1652013-02-15 11:56:59 -05007337 if (!(rdev->flags & RADEON_IS_PCIE))
7338 return;
7339
7340 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7341 data &= ~LC_XMIT_N_FTS_MASK;
7342 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
7343 if (orig != data)
7344 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
7345
7346 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
7347 data |= LC_GO_TO_RECOVERY;
7348 if (orig != data)
7349 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
7350
7351 orig = data = RREG32_PCIE(PCIE_P_CNTL);
7352 data |= P_IGNORE_EDB_ERR;
7353 if (orig != data)
7354 WREG32_PCIE(PCIE_P_CNTL, data);
7355
7356 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7357 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
7358 data |= LC_PMI_TO_L1_DIS;
7359 if (!disable_l0s)
7360 data |= LC_L0S_INACTIVITY(7);
7361
7362 if (!disable_l1) {
7363 data |= LC_L1_INACTIVITY(7);
7364 data &= ~LC_PMI_TO_L1_DIS;
7365 if (orig != data)
7366 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7367
7368 if (!disable_plloff_in_l1) {
7369 bool clk_req_support;
7370
7371 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7372 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7373 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7374 if (orig != data)
7375 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7376
7377 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7378 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7379 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7380 if (orig != data)
7381 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7382
7383 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7384 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7385 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7386 if (orig != data)
7387 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7388
7389 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7390 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7391 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7392 if (orig != data)
7393 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7394
7395 if ((rdev->family != CHIP_OLAND) && (rdev->family != CHIP_HAINAN)) {
7396 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7397 data &= ~PLL_RAMP_UP_TIME_0_MASK;
7398 if (orig != data)
7399 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7400
7401 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7402 data &= ~PLL_RAMP_UP_TIME_1_MASK;
7403 if (orig != data)
7404 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7405
7406 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2);
7407 data &= ~PLL_RAMP_UP_TIME_2_MASK;
7408 if (orig != data)
7409 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2, data);
7410
7411 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3);
7412 data &= ~PLL_RAMP_UP_TIME_3_MASK;
7413 if (orig != data)
7414 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3, data);
7415
7416 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7417 data &= ~PLL_RAMP_UP_TIME_0_MASK;
7418 if (orig != data)
7419 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7420
7421 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7422 data &= ~PLL_RAMP_UP_TIME_1_MASK;
7423 if (orig != data)
7424 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7425
7426 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2);
7427 data &= ~PLL_RAMP_UP_TIME_2_MASK;
7428 if (orig != data)
7429 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2, data);
7430
7431 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3);
7432 data &= ~PLL_RAMP_UP_TIME_3_MASK;
7433 if (orig != data)
7434 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3, data);
7435 }
7436 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
7437 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
7438 data |= LC_DYN_LANES_PWR_STATE(3);
7439 if (orig != data)
7440 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
7441
7442 orig = data = RREG32_PIF_PHY0(PB0_PIF_CNTL);
7443 data &= ~LS2_EXIT_TIME_MASK;
7444 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7445 data |= LS2_EXIT_TIME(5);
7446 if (orig != data)
7447 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
7448
7449 orig = data = RREG32_PIF_PHY1(PB1_PIF_CNTL);
7450 data &= ~LS2_EXIT_TIME_MASK;
7451 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7452 data |= LS2_EXIT_TIME(5);
7453 if (orig != data)
7454 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
7455
7456 if (!disable_clkreq) {
7457 struct pci_dev *root = rdev->pdev->bus->self;
7458 u32 lnkcap;
7459
7460 clk_req_support = false;
7461 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
7462 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
7463 clk_req_support = true;
7464 } else {
7465 clk_req_support = false;
7466 }
7467
7468 if (clk_req_support) {
7469 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
7470 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
7471 if (orig != data)
7472 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
7473
7474 orig = data = RREG32(THM_CLK_CNTL);
7475 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
7476 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
7477 if (orig != data)
7478 WREG32(THM_CLK_CNTL, data);
7479
7480 orig = data = RREG32(MISC_CLK_CNTL);
7481 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
7482 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
7483 if (orig != data)
7484 WREG32(MISC_CLK_CNTL, data);
7485
7486 orig = data = RREG32(CG_CLKPIN_CNTL);
7487 data &= ~BCLK_AS_XCLK;
7488 if (orig != data)
7489 WREG32(CG_CLKPIN_CNTL, data);
7490
7491 orig = data = RREG32(CG_CLKPIN_CNTL_2);
7492 data &= ~FORCE_BIF_REFCLK_EN;
7493 if (orig != data)
7494 WREG32(CG_CLKPIN_CNTL_2, data);
7495
7496 orig = data = RREG32(MPLL_BYPASSCLK_SEL);
7497 data &= ~MPLL_CLKOUT_SEL_MASK;
7498 data |= MPLL_CLKOUT_SEL(4);
7499 if (orig != data)
7500 WREG32(MPLL_BYPASSCLK_SEL, data);
7501
7502 orig = data = RREG32(SPLL_CNTL_MODE);
7503 data &= ~SPLL_REFCLK_SEL_MASK;
7504 if (orig != data)
7505 WREG32(SPLL_CNTL_MODE, data);
7506 }
7507 }
7508 } else {
7509 if (orig != data)
7510 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7511 }
7512
7513 orig = data = RREG32_PCIE(PCIE_CNTL2);
7514 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
7515 if (orig != data)
7516 WREG32_PCIE(PCIE_CNTL2, data);
7517
7518 if (!disable_l0s) {
7519 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7520 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
7521 data = RREG32_PCIE(PCIE_LC_STATUS1);
7522 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
7523 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7524 data &= ~LC_L0S_INACTIVITY_MASK;
7525 if (orig != data)
7526 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7527 }
7528 }
7529 }
7530}