2 * Copyright 2011 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
22 * Authors: Alex Deucher
24 #include <linux/firmware.h>
25 #include <linux/slab.h>
26 #include <linux/module.h>
29 #include "radeon_asic.h"
30 #include "radeon_audio.h"
31 #include <drm/radeon_drm.h>
34 #include "si_blit_shaders.h"
35 #include "clearstate_si.h"
36 #include "radeon_ucode.h"
41 static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
42 static void si_pcie_gen3_enable(struct radeon_device *rdev);
43 static void si_program_aspm(struct radeon_device *rdev);
44 extern void sumo_rlc_fini(struct radeon_device *rdev);
45 extern int sumo_rlc_init(struct radeon_device *rdev);
46 extern int r600_ih_ring_alloc(struct radeon_device *rdev);
47 extern void r600_ih_ring_fini(struct radeon_device *rdev);
48 extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
49 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
50 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
51 extern u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev);
52 extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
53 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
54 static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
56 static void si_init_pg(struct radeon_device *rdev);
57 static void si_init_cg(struct radeon_device *rdev);
58 static void si_fini_pg(struct radeon_device *rdev);
59 static void si_fini_cg(struct radeon_device *rdev);
60 static void si_rlc_stop(struct radeon_device *rdev);
62 static const u32 verde_rlc_save_restore_register_list[] =
64 (0x8000 << 16) | (0x98f4 >> 2),
66 (0x8040 << 16) | (0x98f4 >> 2),
68 (0x8000 << 16) | (0xe80 >> 2),
70 (0x8040 << 16) | (0xe80 >> 2),
72 (0x8000 << 16) | (0x89bc >> 2),
74 (0x8040 << 16) | (0x89bc >> 2),
76 (0x8000 << 16) | (0x8c1c >> 2),
78 (0x8040 << 16) | (0x8c1c >> 2),
80 (0x9c00 << 16) | (0x98f0 >> 2),
82 (0x9c00 << 16) | (0xe7c >> 2),
84 (0x8000 << 16) | (0x9148 >> 2),
86 (0x8040 << 16) | (0x9148 >> 2),
88 (0x9c00 << 16) | (0x9150 >> 2),
90 (0x9c00 << 16) | (0x897c >> 2),
92 (0x9c00 << 16) | (0x8d8c >> 2),
94 (0x9c00 << 16) | (0xac54 >> 2),
97 (0x9c00 << 16) | (0x98f8 >> 2),
99 (0x9c00 << 16) | (0x9910 >> 2),
101 (0x9c00 << 16) | (0x9914 >> 2),
103 (0x9c00 << 16) | (0x9918 >> 2),
105 (0x9c00 << 16) | (0x991c >> 2),
107 (0x9c00 << 16) | (0x9920 >> 2),
109 (0x9c00 << 16) | (0x9924 >> 2),
111 (0x9c00 << 16) | (0x9928 >> 2),
113 (0x9c00 << 16) | (0x992c >> 2),
115 (0x9c00 << 16) | (0x9930 >> 2),
117 (0x9c00 << 16) | (0x9934 >> 2),
119 (0x9c00 << 16) | (0x9938 >> 2),
121 (0x9c00 << 16) | (0x993c >> 2),
123 (0x9c00 << 16) | (0x9940 >> 2),
125 (0x9c00 << 16) | (0x9944 >> 2),
127 (0x9c00 << 16) | (0x9948 >> 2),
129 (0x9c00 << 16) | (0x994c >> 2),
131 (0x9c00 << 16) | (0x9950 >> 2),
133 (0x9c00 << 16) | (0x9954 >> 2),
135 (0x9c00 << 16) | (0x9958 >> 2),
137 (0x9c00 << 16) | (0x995c >> 2),
139 (0x9c00 << 16) | (0x9960 >> 2),
141 (0x9c00 << 16) | (0x9964 >> 2),
143 (0x9c00 << 16) | (0x9968 >> 2),
145 (0x9c00 << 16) | (0x996c >> 2),
147 (0x9c00 << 16) | (0x9970 >> 2),
149 (0x9c00 << 16) | (0x9974 >> 2),
151 (0x9c00 << 16) | (0x9978 >> 2),
153 (0x9c00 << 16) | (0x997c >> 2),
155 (0x9c00 << 16) | (0x9980 >> 2),
157 (0x9c00 << 16) | (0x9984 >> 2),
159 (0x9c00 << 16) | (0x9988 >> 2),
161 (0x9c00 << 16) | (0x998c >> 2),
163 (0x9c00 << 16) | (0x8c00 >> 2),
165 (0x9c00 << 16) | (0x8c14 >> 2),
167 (0x9c00 << 16) | (0x8c04 >> 2),
169 (0x9c00 << 16) | (0x8c08 >> 2),
171 (0x8000 << 16) | (0x9b7c >> 2),
173 (0x8040 << 16) | (0x9b7c >> 2),
175 (0x8000 << 16) | (0xe84 >> 2),
177 (0x8040 << 16) | (0xe84 >> 2),
179 (0x8000 << 16) | (0x89c0 >> 2),
181 (0x8040 << 16) | (0x89c0 >> 2),
183 (0x8000 << 16) | (0x914c >> 2),
185 (0x8040 << 16) | (0x914c >> 2),
187 (0x8000 << 16) | (0x8c20 >> 2),
189 (0x8040 << 16) | (0x8c20 >> 2),
191 (0x8000 << 16) | (0x9354 >> 2),
193 (0x8040 << 16) | (0x9354 >> 2),
195 (0x9c00 << 16) | (0x9060 >> 2),
197 (0x9c00 << 16) | (0x9364 >> 2),
199 (0x9c00 << 16) | (0x9100 >> 2),
201 (0x9c00 << 16) | (0x913c >> 2),
203 (0x8000 << 16) | (0x90e0 >> 2),
205 (0x8000 << 16) | (0x90e4 >> 2),
207 (0x8000 << 16) | (0x90e8 >> 2),
209 (0x8040 << 16) | (0x90e0 >> 2),
211 (0x8040 << 16) | (0x90e4 >> 2),
213 (0x8040 << 16) | (0x90e8 >> 2),
215 (0x9c00 << 16) | (0x8bcc >> 2),
217 (0x9c00 << 16) | (0x8b24 >> 2),
219 (0x9c00 << 16) | (0x88c4 >> 2),
221 (0x9c00 << 16) | (0x8e50 >> 2),
223 (0x9c00 << 16) | (0x8c0c >> 2),
225 (0x9c00 << 16) | (0x8e58 >> 2),
227 (0x9c00 << 16) | (0x8e5c >> 2),
229 (0x9c00 << 16) | (0x9508 >> 2),
231 (0x9c00 << 16) | (0x950c >> 2),
233 (0x9c00 << 16) | (0x9494 >> 2),
235 (0x9c00 << 16) | (0xac0c >> 2),
237 (0x9c00 << 16) | (0xac10 >> 2),
239 (0x9c00 << 16) | (0xac14 >> 2),
241 (0x9c00 << 16) | (0xae00 >> 2),
243 (0x9c00 << 16) | (0xac08 >> 2),
245 (0x9c00 << 16) | (0x88d4 >> 2),
247 (0x9c00 << 16) | (0x88c8 >> 2),
249 (0x9c00 << 16) | (0x88cc >> 2),
251 (0x9c00 << 16) | (0x89b0 >> 2),
253 (0x9c00 << 16) | (0x8b10 >> 2),
255 (0x9c00 << 16) | (0x8a14 >> 2),
257 (0x9c00 << 16) | (0x9830 >> 2),
259 (0x9c00 << 16) | (0x9834 >> 2),
261 (0x9c00 << 16) | (0x9838 >> 2),
263 (0x9c00 << 16) | (0x9a10 >> 2),
265 (0x8000 << 16) | (0x9870 >> 2),
267 (0x8000 << 16) | (0x9874 >> 2),
269 (0x8001 << 16) | (0x9870 >> 2),
271 (0x8001 << 16) | (0x9874 >> 2),
273 (0x8040 << 16) | (0x9870 >> 2),
275 (0x8040 << 16) | (0x9874 >> 2),
277 (0x8041 << 16) | (0x9870 >> 2),
279 (0x8041 << 16) | (0x9874 >> 2),
284 static const u32 tahiti_golden_rlc_registers[] =
286 0xc424, 0xffffffff, 0x00601005,
287 0xc47c, 0xffffffff, 0x10104040,
288 0xc488, 0xffffffff, 0x0100000a,
289 0xc314, 0xffffffff, 0x00000800,
290 0xc30c, 0xffffffff, 0x800000f4,
291 0xf4a8, 0xffffffff, 0x00000000
294 static const u32 tahiti_golden_registers[] =
296 0x9a10, 0x00010000, 0x00018208,
297 0x9830, 0xffffffff, 0x00000000,
298 0x9834, 0xf00fffff, 0x00000400,
299 0x9838, 0x0002021c, 0x00020200,
300 0xc78, 0x00000080, 0x00000000,
301 0xd030, 0x000300c0, 0x00800040,
302 0xd830, 0x000300c0, 0x00800040,
303 0x5bb0, 0x000000f0, 0x00000070,
304 0x5bc0, 0x00200000, 0x50100000,
305 0x7030, 0x31000311, 0x00000011,
306 0x277c, 0x00000003, 0x000007ff,
307 0x240c, 0x000007ff, 0x00000000,
308 0x8a14, 0xf000001f, 0x00000007,
309 0x8b24, 0xffffffff, 0x00ffffff,
310 0x8b10, 0x0000ff0f, 0x00000000,
311 0x28a4c, 0x07ffffff, 0x4e000000,
312 0x28350, 0x3f3f3fff, 0x2a00126a,
313 0x30, 0x000000ff, 0x0040,
314 0x34, 0x00000040, 0x00004040,
315 0x9100, 0x07ffffff, 0x03000000,
316 0x8e88, 0x01ff1f3f, 0x00000000,
317 0x8e84, 0x01ff1f3f, 0x00000000,
318 0x9060, 0x0000007f, 0x00000020,
319 0x9508, 0x00010000, 0x00010000,
320 0xac14, 0x00000200, 0x000002fb,
321 0xac10, 0xffffffff, 0x0000543b,
322 0xac0c, 0xffffffff, 0xa9210876,
323 0x88d0, 0xffffffff, 0x000fff40,
324 0x88d4, 0x0000001f, 0x00000010,
325 0x1410, 0x20000000, 0x20fffed8,
326 0x15c0, 0x000c0fc0, 0x000c0400
329 static const u32 tahiti_golden_registers2[] =
331 0xc64, 0x00000001, 0x00000001
334 static const u32 pitcairn_golden_rlc_registers[] =
336 0xc424, 0xffffffff, 0x00601004,
337 0xc47c, 0xffffffff, 0x10102020,
338 0xc488, 0xffffffff, 0x01000020,
339 0xc314, 0xffffffff, 0x00000800,
340 0xc30c, 0xffffffff, 0x800000a4
343 static const u32 pitcairn_golden_registers[] =
345 0x9a10, 0x00010000, 0x00018208,
346 0x9830, 0xffffffff, 0x00000000,
347 0x9834, 0xf00fffff, 0x00000400,
348 0x9838, 0x0002021c, 0x00020200,
349 0xc78, 0x00000080, 0x00000000,
350 0xd030, 0x000300c0, 0x00800040,
351 0xd830, 0x000300c0, 0x00800040,
352 0x5bb0, 0x000000f0, 0x00000070,
353 0x5bc0, 0x00200000, 0x50100000,
354 0x7030, 0x31000311, 0x00000011,
355 0x2ae4, 0x00073ffe, 0x000022a2,
356 0x240c, 0x000007ff, 0x00000000,
357 0x8a14, 0xf000001f, 0x00000007,
358 0x8b24, 0xffffffff, 0x00ffffff,
359 0x8b10, 0x0000ff0f, 0x00000000,
360 0x28a4c, 0x07ffffff, 0x4e000000,
361 0x28350, 0x3f3f3fff, 0x2a00126a,
362 0x30, 0x000000ff, 0x0040,
363 0x34, 0x00000040, 0x00004040,
364 0x9100, 0x07ffffff, 0x03000000,
365 0x9060, 0x0000007f, 0x00000020,
366 0x9508, 0x00010000, 0x00010000,
367 0xac14, 0x000003ff, 0x000000f7,
368 0xac10, 0xffffffff, 0x00000000,
369 0xac0c, 0xffffffff, 0x32761054,
370 0x88d4, 0x0000001f, 0x00000010,
371 0x15c0, 0x000c0fc0, 0x000c0400
374 static const u32 verde_golden_rlc_registers[] =
376 0xc424, 0xffffffff, 0x033f1005,
377 0xc47c, 0xffffffff, 0x10808020,
378 0xc488, 0xffffffff, 0x00800008,
379 0xc314, 0xffffffff, 0x00001000,
380 0xc30c, 0xffffffff, 0x80010014
383 static const u32 verde_golden_registers[] =
385 0x9a10, 0x00010000, 0x00018208,
386 0x9830, 0xffffffff, 0x00000000,
387 0x9834, 0xf00fffff, 0x00000400,
388 0x9838, 0x0002021c, 0x00020200,
389 0xc78, 0x00000080, 0x00000000,
390 0xd030, 0x000300c0, 0x00800040,
391 0xd030, 0x000300c0, 0x00800040,
392 0xd830, 0x000300c0, 0x00800040,
393 0xd830, 0x000300c0, 0x00800040,
394 0x5bb0, 0x000000f0, 0x00000070,
395 0x5bc0, 0x00200000, 0x50100000,
396 0x7030, 0x31000311, 0x00000011,
397 0x2ae4, 0x00073ffe, 0x000022a2,
398 0x2ae4, 0x00073ffe, 0x000022a2,
399 0x2ae4, 0x00073ffe, 0x000022a2,
400 0x240c, 0x000007ff, 0x00000000,
401 0x240c, 0x000007ff, 0x00000000,
402 0x240c, 0x000007ff, 0x00000000,
403 0x8a14, 0xf000001f, 0x00000007,
404 0x8a14, 0xf000001f, 0x00000007,
405 0x8a14, 0xf000001f, 0x00000007,
406 0x8b24, 0xffffffff, 0x00ffffff,
407 0x8b10, 0x0000ff0f, 0x00000000,
408 0x28a4c, 0x07ffffff, 0x4e000000,
409 0x28350, 0x3f3f3fff, 0x0000124a,
410 0x28350, 0x3f3f3fff, 0x0000124a,
411 0x28350, 0x3f3f3fff, 0x0000124a,
412 0x30, 0x000000ff, 0x0040,
413 0x34, 0x00000040, 0x00004040,
414 0x9100, 0x07ffffff, 0x03000000,
415 0x9100, 0x07ffffff, 0x03000000,
416 0x8e88, 0x01ff1f3f, 0x00000000,
417 0x8e88, 0x01ff1f3f, 0x00000000,
418 0x8e88, 0x01ff1f3f, 0x00000000,
419 0x8e84, 0x01ff1f3f, 0x00000000,
420 0x8e84, 0x01ff1f3f, 0x00000000,
421 0x8e84, 0x01ff1f3f, 0x00000000,
422 0x9060, 0x0000007f, 0x00000020,
423 0x9508, 0x00010000, 0x00010000,
424 0xac14, 0x000003ff, 0x00000003,
425 0xac14, 0x000003ff, 0x00000003,
426 0xac14, 0x000003ff, 0x00000003,
427 0xac10, 0xffffffff, 0x00000000,
428 0xac10, 0xffffffff, 0x00000000,
429 0xac10, 0xffffffff, 0x00000000,
430 0xac0c, 0xffffffff, 0x00001032,
431 0xac0c, 0xffffffff, 0x00001032,
432 0xac0c, 0xffffffff, 0x00001032,
433 0x88d4, 0x0000001f, 0x00000010,
434 0x88d4, 0x0000001f, 0x00000010,
435 0x88d4, 0x0000001f, 0x00000010,
436 0x15c0, 0x000c0fc0, 0x000c0400
439 static const u32 oland_golden_rlc_registers[] =
441 0xc424, 0xffffffff, 0x00601005,
442 0xc47c, 0xffffffff, 0x10104040,
443 0xc488, 0xffffffff, 0x0100000a,
444 0xc314, 0xffffffff, 0x00000800,
445 0xc30c, 0xffffffff, 0x800000f4
448 static const u32 oland_golden_registers[] =
450 0x9a10, 0x00010000, 0x00018208,
451 0x9830, 0xffffffff, 0x00000000,
452 0x9834, 0xf00fffff, 0x00000400,
453 0x9838, 0x0002021c, 0x00020200,
454 0xc78, 0x00000080, 0x00000000,
455 0xd030, 0x000300c0, 0x00800040,
456 0xd830, 0x000300c0, 0x00800040,
457 0x5bb0, 0x000000f0, 0x00000070,
458 0x5bc0, 0x00200000, 0x50100000,
459 0x7030, 0x31000311, 0x00000011,
460 0x2ae4, 0x00073ffe, 0x000022a2,
461 0x240c, 0x000007ff, 0x00000000,
462 0x8a14, 0xf000001f, 0x00000007,
463 0x8b24, 0xffffffff, 0x00ffffff,
464 0x8b10, 0x0000ff0f, 0x00000000,
465 0x28a4c, 0x07ffffff, 0x4e000000,
466 0x28350, 0x3f3f3fff, 0x00000082,
467 0x30, 0x000000ff, 0x0040,
468 0x34, 0x00000040, 0x00004040,
469 0x9100, 0x07ffffff, 0x03000000,
470 0x9060, 0x0000007f, 0x00000020,
471 0x9508, 0x00010000, 0x00010000,
472 0xac14, 0x000003ff, 0x000000f3,
473 0xac10, 0xffffffff, 0x00000000,
474 0xac0c, 0xffffffff, 0x00003210,
475 0x88d4, 0x0000001f, 0x00000010,
476 0x15c0, 0x000c0fc0, 0x000c0400
479 static const u32 hainan_golden_registers[] =
481 0x9a10, 0x00010000, 0x00018208,
482 0x9830, 0xffffffff, 0x00000000,
483 0x9834, 0xf00fffff, 0x00000400,
484 0x9838, 0x0002021c, 0x00020200,
485 0xd0c0, 0xff000fff, 0x00000100,
486 0xd030, 0x000300c0, 0x00800040,
487 0xd8c0, 0xff000fff, 0x00000100,
488 0xd830, 0x000300c0, 0x00800040,
489 0x2ae4, 0x00073ffe, 0x000022a2,
490 0x240c, 0x000007ff, 0x00000000,
491 0x8a14, 0xf000001f, 0x00000007,
492 0x8b24, 0xffffffff, 0x00ffffff,
493 0x8b10, 0x0000ff0f, 0x00000000,
494 0x28a4c, 0x07ffffff, 0x4e000000,
495 0x28350, 0x3f3f3fff, 0x00000000,
496 0x30, 0x000000ff, 0x0040,
497 0x34, 0x00000040, 0x00004040,
498 0x9100, 0x03e00000, 0x03600000,
499 0x9060, 0x0000007f, 0x00000020,
500 0x9508, 0x00010000, 0x00010000,
501 0xac14, 0x000003ff, 0x000000f1,
502 0xac10, 0xffffffff, 0x00000000,
503 0xac0c, 0xffffffff, 0x00003210,
504 0x88d4, 0x0000001f, 0x00000010,
505 0x15c0, 0x000c0fc0, 0x000c0400
508 static const u32 hainan_golden_registers2[] =
510 0x98f8, 0xffffffff, 0x02010001
513 static const u32 tahiti_mgcg_cgcg_init[] =
515 0xc400, 0xffffffff, 0xfffffffc,
516 0x802c, 0xffffffff, 0xe0000000,
517 0x9a60, 0xffffffff, 0x00000100,
518 0x92a4, 0xffffffff, 0x00000100,
519 0xc164, 0xffffffff, 0x00000100,
520 0x9774, 0xffffffff, 0x00000100,
521 0x8984, 0xffffffff, 0x06000100,
522 0x8a18, 0xffffffff, 0x00000100,
523 0x92a0, 0xffffffff, 0x00000100,
524 0xc380, 0xffffffff, 0x00000100,
525 0x8b28, 0xffffffff, 0x00000100,
526 0x9144, 0xffffffff, 0x00000100,
527 0x8d88, 0xffffffff, 0x00000100,
528 0x8d8c, 0xffffffff, 0x00000100,
529 0x9030, 0xffffffff, 0x00000100,
530 0x9034, 0xffffffff, 0x00000100,
531 0x9038, 0xffffffff, 0x00000100,
532 0x903c, 0xffffffff, 0x00000100,
533 0xad80, 0xffffffff, 0x00000100,
534 0xac54, 0xffffffff, 0x00000100,
535 0x897c, 0xffffffff, 0x06000100,
536 0x9868, 0xffffffff, 0x00000100,
537 0x9510, 0xffffffff, 0x00000100,
538 0xaf04, 0xffffffff, 0x00000100,
539 0xae04, 0xffffffff, 0x00000100,
540 0x949c, 0xffffffff, 0x00000100,
541 0x802c, 0xffffffff, 0xe0000000,
542 0x9160, 0xffffffff, 0x00010000,
543 0x9164, 0xffffffff, 0x00030002,
544 0x9168, 0xffffffff, 0x00040007,
545 0x916c, 0xffffffff, 0x00060005,
546 0x9170, 0xffffffff, 0x00090008,
547 0x9174, 0xffffffff, 0x00020001,
548 0x9178, 0xffffffff, 0x00040003,
549 0x917c, 0xffffffff, 0x00000007,
550 0x9180, 0xffffffff, 0x00060005,
551 0x9184, 0xffffffff, 0x00090008,
552 0x9188, 0xffffffff, 0x00030002,
553 0x918c, 0xffffffff, 0x00050004,
554 0x9190, 0xffffffff, 0x00000008,
555 0x9194, 0xffffffff, 0x00070006,
556 0x9198, 0xffffffff, 0x000a0009,
557 0x919c, 0xffffffff, 0x00040003,
558 0x91a0, 0xffffffff, 0x00060005,
559 0x91a4, 0xffffffff, 0x00000009,
560 0x91a8, 0xffffffff, 0x00080007,
561 0x91ac, 0xffffffff, 0x000b000a,
562 0x91b0, 0xffffffff, 0x00050004,
563 0x91b4, 0xffffffff, 0x00070006,
564 0x91b8, 0xffffffff, 0x0008000b,
565 0x91bc, 0xffffffff, 0x000a0009,
566 0x91c0, 0xffffffff, 0x000d000c,
567 0x91c4, 0xffffffff, 0x00060005,
568 0x91c8, 0xffffffff, 0x00080007,
569 0x91cc, 0xffffffff, 0x0000000b,
570 0x91d0, 0xffffffff, 0x000a0009,
571 0x91d4, 0xffffffff, 0x000d000c,
572 0x91d8, 0xffffffff, 0x00070006,
573 0x91dc, 0xffffffff, 0x00090008,
574 0x91e0, 0xffffffff, 0x0000000c,
575 0x91e4, 0xffffffff, 0x000b000a,
576 0x91e8, 0xffffffff, 0x000e000d,
577 0x91ec, 0xffffffff, 0x00080007,
578 0x91f0, 0xffffffff, 0x000a0009,
579 0x91f4, 0xffffffff, 0x0000000d,
580 0x91f8, 0xffffffff, 0x000c000b,
581 0x91fc, 0xffffffff, 0x000f000e,
582 0x9200, 0xffffffff, 0x00090008,
583 0x9204, 0xffffffff, 0x000b000a,
584 0x9208, 0xffffffff, 0x000c000f,
585 0x920c, 0xffffffff, 0x000e000d,
586 0x9210, 0xffffffff, 0x00110010,
587 0x9214, 0xffffffff, 0x000a0009,
588 0x9218, 0xffffffff, 0x000c000b,
589 0x921c, 0xffffffff, 0x0000000f,
590 0x9220, 0xffffffff, 0x000e000d,
591 0x9224, 0xffffffff, 0x00110010,
592 0x9228, 0xffffffff, 0x000b000a,
593 0x922c, 0xffffffff, 0x000d000c,
594 0x9230, 0xffffffff, 0x00000010,
595 0x9234, 0xffffffff, 0x000f000e,
596 0x9238, 0xffffffff, 0x00120011,
597 0x923c, 0xffffffff, 0x000c000b,
598 0x9240, 0xffffffff, 0x000e000d,
599 0x9244, 0xffffffff, 0x00000011,
600 0x9248, 0xffffffff, 0x0010000f,
601 0x924c, 0xffffffff, 0x00130012,
602 0x9250, 0xffffffff, 0x000d000c,
603 0x9254, 0xffffffff, 0x000f000e,
604 0x9258, 0xffffffff, 0x00100013,
605 0x925c, 0xffffffff, 0x00120011,
606 0x9260, 0xffffffff, 0x00150014,
607 0x9264, 0xffffffff, 0x000e000d,
608 0x9268, 0xffffffff, 0x0010000f,
609 0x926c, 0xffffffff, 0x00000013,
610 0x9270, 0xffffffff, 0x00120011,
611 0x9274, 0xffffffff, 0x00150014,
612 0x9278, 0xffffffff, 0x000f000e,
613 0x927c, 0xffffffff, 0x00110010,
614 0x9280, 0xffffffff, 0x00000014,
615 0x9284, 0xffffffff, 0x00130012,
616 0x9288, 0xffffffff, 0x00160015,
617 0x928c, 0xffffffff, 0x0010000f,
618 0x9290, 0xffffffff, 0x00120011,
619 0x9294, 0xffffffff, 0x00000015,
620 0x9298, 0xffffffff, 0x00140013,
621 0x929c, 0xffffffff, 0x00170016,
622 0x9150, 0xffffffff, 0x96940200,
623 0x8708, 0xffffffff, 0x00900100,
624 0xc478, 0xffffffff, 0x00000080,
625 0xc404, 0xffffffff, 0x0020003f,
626 0x30, 0xffffffff, 0x0000001c,
627 0x34, 0x000f0000, 0x000f0000,
628 0x160c, 0xffffffff, 0x00000100,
629 0x1024, 0xffffffff, 0x00000100,
630 0x102c, 0x00000101, 0x00000000,
631 0x20a8, 0xffffffff, 0x00000104,
632 0x264c, 0x000c0000, 0x000c0000,
633 0x2648, 0x000c0000, 0x000c0000,
634 0x55e4, 0xff000fff, 0x00000100,
635 0x55e8, 0x00000001, 0x00000001,
636 0x2f50, 0x00000001, 0x00000001,
637 0x30cc, 0xc0000fff, 0x00000104,
638 0xc1e4, 0x00000001, 0x00000001,
639 0xd0c0, 0xfffffff0, 0x00000100,
640 0xd8c0, 0xfffffff0, 0x00000100
643 static const u32 pitcairn_mgcg_cgcg_init[] =
645 0xc400, 0xffffffff, 0xfffffffc,
646 0x802c, 0xffffffff, 0xe0000000,
647 0x9a60, 0xffffffff, 0x00000100,
648 0x92a4, 0xffffffff, 0x00000100,
649 0xc164, 0xffffffff, 0x00000100,
650 0x9774, 0xffffffff, 0x00000100,
651 0x8984, 0xffffffff, 0x06000100,
652 0x8a18, 0xffffffff, 0x00000100,
653 0x92a0, 0xffffffff, 0x00000100,
654 0xc380, 0xffffffff, 0x00000100,
655 0x8b28, 0xffffffff, 0x00000100,
656 0x9144, 0xffffffff, 0x00000100,
657 0x8d88, 0xffffffff, 0x00000100,
658 0x8d8c, 0xffffffff, 0x00000100,
659 0x9030, 0xffffffff, 0x00000100,
660 0x9034, 0xffffffff, 0x00000100,
661 0x9038, 0xffffffff, 0x00000100,
662 0x903c, 0xffffffff, 0x00000100,
663 0xad80, 0xffffffff, 0x00000100,
664 0xac54, 0xffffffff, 0x00000100,
665 0x897c, 0xffffffff, 0x06000100,
666 0x9868, 0xffffffff, 0x00000100,
667 0x9510, 0xffffffff, 0x00000100,
668 0xaf04, 0xffffffff, 0x00000100,
669 0xae04, 0xffffffff, 0x00000100,
670 0x949c, 0xffffffff, 0x00000100,
671 0x802c, 0xffffffff, 0xe0000000,
672 0x9160, 0xffffffff, 0x00010000,
673 0x9164, 0xffffffff, 0x00030002,
674 0x9168, 0xffffffff, 0x00040007,
675 0x916c, 0xffffffff, 0x00060005,
676 0x9170, 0xffffffff, 0x00090008,
677 0x9174, 0xffffffff, 0x00020001,
678 0x9178, 0xffffffff, 0x00040003,
679 0x917c, 0xffffffff, 0x00000007,
680 0x9180, 0xffffffff, 0x00060005,
681 0x9184, 0xffffffff, 0x00090008,
682 0x9188, 0xffffffff, 0x00030002,
683 0x918c, 0xffffffff, 0x00050004,
684 0x9190, 0xffffffff, 0x00000008,
685 0x9194, 0xffffffff, 0x00070006,
686 0x9198, 0xffffffff, 0x000a0009,
687 0x919c, 0xffffffff, 0x00040003,
688 0x91a0, 0xffffffff, 0x00060005,
689 0x91a4, 0xffffffff, 0x00000009,
690 0x91a8, 0xffffffff, 0x00080007,
691 0x91ac, 0xffffffff, 0x000b000a,
692 0x91b0, 0xffffffff, 0x00050004,
693 0x91b4, 0xffffffff, 0x00070006,
694 0x91b8, 0xffffffff, 0x0008000b,
695 0x91bc, 0xffffffff, 0x000a0009,
696 0x91c0, 0xffffffff, 0x000d000c,
697 0x9200, 0xffffffff, 0x00090008,
698 0x9204, 0xffffffff, 0x000b000a,
699 0x9208, 0xffffffff, 0x000c000f,
700 0x920c, 0xffffffff, 0x000e000d,
701 0x9210, 0xffffffff, 0x00110010,
702 0x9214, 0xffffffff, 0x000a0009,
703 0x9218, 0xffffffff, 0x000c000b,
704 0x921c, 0xffffffff, 0x0000000f,
705 0x9220, 0xffffffff, 0x000e000d,
706 0x9224, 0xffffffff, 0x00110010,
707 0x9228, 0xffffffff, 0x000b000a,
708 0x922c, 0xffffffff, 0x000d000c,
709 0x9230, 0xffffffff, 0x00000010,
710 0x9234, 0xffffffff, 0x000f000e,
711 0x9238, 0xffffffff, 0x00120011,
712 0x923c, 0xffffffff, 0x000c000b,
713 0x9240, 0xffffffff, 0x000e000d,
714 0x9244, 0xffffffff, 0x00000011,
715 0x9248, 0xffffffff, 0x0010000f,
716 0x924c, 0xffffffff, 0x00130012,
717 0x9250, 0xffffffff, 0x000d000c,
718 0x9254, 0xffffffff, 0x000f000e,
719 0x9258, 0xffffffff, 0x00100013,
720 0x925c, 0xffffffff, 0x00120011,
721 0x9260, 0xffffffff, 0x00150014,
722 0x9150, 0xffffffff, 0x96940200,
723 0x8708, 0xffffffff, 0x00900100,
724 0xc478, 0xffffffff, 0x00000080,
725 0xc404, 0xffffffff, 0x0020003f,
726 0x30, 0xffffffff, 0x0000001c,
727 0x34, 0x000f0000, 0x000f0000,
728 0x160c, 0xffffffff, 0x00000100,
729 0x1024, 0xffffffff, 0x00000100,
730 0x102c, 0x00000101, 0x00000000,
731 0x20a8, 0xffffffff, 0x00000104,
732 0x55e4, 0xff000fff, 0x00000100,
733 0x55e8, 0x00000001, 0x00000001,
734 0x2f50, 0x00000001, 0x00000001,
735 0x30cc, 0xc0000fff, 0x00000104,
736 0xc1e4, 0x00000001, 0x00000001,
737 0xd0c0, 0xfffffff0, 0x00000100,
738 0xd8c0, 0xfffffff0, 0x00000100
741 static const u32 verde_mgcg_cgcg_init[] =
743 0xc400, 0xffffffff, 0xfffffffc,
744 0x802c, 0xffffffff, 0xe0000000,
745 0x9a60, 0xffffffff, 0x00000100,
746 0x92a4, 0xffffffff, 0x00000100,
747 0xc164, 0xffffffff, 0x00000100,
748 0x9774, 0xffffffff, 0x00000100,
749 0x8984, 0xffffffff, 0x06000100,
750 0x8a18, 0xffffffff, 0x00000100,
751 0x92a0, 0xffffffff, 0x00000100,
752 0xc380, 0xffffffff, 0x00000100,
753 0x8b28, 0xffffffff, 0x00000100,
754 0x9144, 0xffffffff, 0x00000100,
755 0x8d88, 0xffffffff, 0x00000100,
756 0x8d8c, 0xffffffff, 0x00000100,
757 0x9030, 0xffffffff, 0x00000100,
758 0x9034, 0xffffffff, 0x00000100,
759 0x9038, 0xffffffff, 0x00000100,
760 0x903c, 0xffffffff, 0x00000100,
761 0xad80, 0xffffffff, 0x00000100,
762 0xac54, 0xffffffff, 0x00000100,
763 0x897c, 0xffffffff, 0x06000100,
764 0x9868, 0xffffffff, 0x00000100,
765 0x9510, 0xffffffff, 0x00000100,
766 0xaf04, 0xffffffff, 0x00000100,
767 0xae04, 0xffffffff, 0x00000100,
768 0x949c, 0xffffffff, 0x00000100,
769 0x802c, 0xffffffff, 0xe0000000,
770 0x9160, 0xffffffff, 0x00010000,
771 0x9164, 0xffffffff, 0x00030002,
772 0x9168, 0xffffffff, 0x00040007,
773 0x916c, 0xffffffff, 0x00060005,
774 0x9170, 0xffffffff, 0x00090008,
775 0x9174, 0xffffffff, 0x00020001,
776 0x9178, 0xffffffff, 0x00040003,
777 0x917c, 0xffffffff, 0x00000007,
778 0x9180, 0xffffffff, 0x00060005,
779 0x9184, 0xffffffff, 0x00090008,
780 0x9188, 0xffffffff, 0x00030002,
781 0x918c, 0xffffffff, 0x00050004,
782 0x9190, 0xffffffff, 0x00000008,
783 0x9194, 0xffffffff, 0x00070006,
784 0x9198, 0xffffffff, 0x000a0009,
785 0x919c, 0xffffffff, 0x00040003,
786 0x91a0, 0xffffffff, 0x00060005,
787 0x91a4, 0xffffffff, 0x00000009,
788 0x91a8, 0xffffffff, 0x00080007,
789 0x91ac, 0xffffffff, 0x000b000a,
790 0x91b0, 0xffffffff, 0x00050004,
791 0x91b4, 0xffffffff, 0x00070006,
792 0x91b8, 0xffffffff, 0x0008000b,
793 0x91bc, 0xffffffff, 0x000a0009,
794 0x91c0, 0xffffffff, 0x000d000c,
795 0x9200, 0xffffffff, 0x00090008,
796 0x9204, 0xffffffff, 0x000b000a,
797 0x9208, 0xffffffff, 0x000c000f,
798 0x920c, 0xffffffff, 0x000e000d,
799 0x9210, 0xffffffff, 0x00110010,
800 0x9214, 0xffffffff, 0x000a0009,
801 0x9218, 0xffffffff, 0x000c000b,
802 0x921c, 0xffffffff, 0x0000000f,
803 0x9220, 0xffffffff, 0x000e000d,
804 0x9224, 0xffffffff, 0x00110010,
805 0x9228, 0xffffffff, 0x000b000a,
806 0x922c, 0xffffffff, 0x000d000c,
807 0x9230, 0xffffffff, 0x00000010,
808 0x9234, 0xffffffff, 0x000f000e,
809 0x9238, 0xffffffff, 0x00120011,
810 0x923c, 0xffffffff, 0x000c000b,
811 0x9240, 0xffffffff, 0x000e000d,
812 0x9244, 0xffffffff, 0x00000011,
813 0x9248, 0xffffffff, 0x0010000f,
814 0x924c, 0xffffffff, 0x00130012,
815 0x9250, 0xffffffff, 0x000d000c,
816 0x9254, 0xffffffff, 0x000f000e,
817 0x9258, 0xffffffff, 0x00100013,
818 0x925c, 0xffffffff, 0x00120011,
819 0x9260, 0xffffffff, 0x00150014,
820 0x9150, 0xffffffff, 0x96940200,
821 0x8708, 0xffffffff, 0x00900100,
822 0xc478, 0xffffffff, 0x00000080,
823 0xc404, 0xffffffff, 0x0020003f,
824 0x30, 0xffffffff, 0x0000001c,
825 0x34, 0x000f0000, 0x000f0000,
826 0x160c, 0xffffffff, 0x00000100,
827 0x1024, 0xffffffff, 0x00000100,
828 0x102c, 0x00000101, 0x00000000,
829 0x20a8, 0xffffffff, 0x00000104,
830 0x264c, 0x000c0000, 0x000c0000,
831 0x2648, 0x000c0000, 0x000c0000,
832 0x55e4, 0xff000fff, 0x00000100,
833 0x55e8, 0x00000001, 0x00000001,
834 0x2f50, 0x00000001, 0x00000001,
835 0x30cc, 0xc0000fff, 0x00000104,
836 0xc1e4, 0x00000001, 0x00000001,
837 0xd0c0, 0xfffffff0, 0x00000100,
838 0xd8c0, 0xfffffff0, 0x00000100
841 static const u32 oland_mgcg_cgcg_init[] =
843 0xc400, 0xffffffff, 0xfffffffc,
844 0x802c, 0xffffffff, 0xe0000000,
845 0x9a60, 0xffffffff, 0x00000100,
846 0x92a4, 0xffffffff, 0x00000100,
847 0xc164, 0xffffffff, 0x00000100,
848 0x9774, 0xffffffff, 0x00000100,
849 0x8984, 0xffffffff, 0x06000100,
850 0x8a18, 0xffffffff, 0x00000100,
851 0x92a0, 0xffffffff, 0x00000100,
852 0xc380, 0xffffffff, 0x00000100,
853 0x8b28, 0xffffffff, 0x00000100,
854 0x9144, 0xffffffff, 0x00000100,
855 0x8d88, 0xffffffff, 0x00000100,
856 0x8d8c, 0xffffffff, 0x00000100,
857 0x9030, 0xffffffff, 0x00000100,
858 0x9034, 0xffffffff, 0x00000100,
859 0x9038, 0xffffffff, 0x00000100,
860 0x903c, 0xffffffff, 0x00000100,
861 0xad80, 0xffffffff, 0x00000100,
862 0xac54, 0xffffffff, 0x00000100,
863 0x897c, 0xffffffff, 0x06000100,
864 0x9868, 0xffffffff, 0x00000100,
865 0x9510, 0xffffffff, 0x00000100,
866 0xaf04, 0xffffffff, 0x00000100,
867 0xae04, 0xffffffff, 0x00000100,
868 0x949c, 0xffffffff, 0x00000100,
869 0x802c, 0xffffffff, 0xe0000000,
870 0x9160, 0xffffffff, 0x00010000,
871 0x9164, 0xffffffff, 0x00030002,
872 0x9168, 0xffffffff, 0x00040007,
873 0x916c, 0xffffffff, 0x00060005,
874 0x9170, 0xffffffff, 0x00090008,
875 0x9174, 0xffffffff, 0x00020001,
876 0x9178, 0xffffffff, 0x00040003,
877 0x917c, 0xffffffff, 0x00000007,
878 0x9180, 0xffffffff, 0x00060005,
879 0x9184, 0xffffffff, 0x00090008,
880 0x9188, 0xffffffff, 0x00030002,
881 0x918c, 0xffffffff, 0x00050004,
882 0x9190, 0xffffffff, 0x00000008,
883 0x9194, 0xffffffff, 0x00070006,
884 0x9198, 0xffffffff, 0x000a0009,
885 0x919c, 0xffffffff, 0x00040003,
886 0x91a0, 0xffffffff, 0x00060005,
887 0x91a4, 0xffffffff, 0x00000009,
888 0x91a8, 0xffffffff, 0x00080007,
889 0x91ac, 0xffffffff, 0x000b000a,
890 0x91b0, 0xffffffff, 0x00050004,
891 0x91b4, 0xffffffff, 0x00070006,
892 0x91b8, 0xffffffff, 0x0008000b,
893 0x91bc, 0xffffffff, 0x000a0009,
894 0x91c0, 0xffffffff, 0x000d000c,
895 0x91c4, 0xffffffff, 0x00060005,
896 0x91c8, 0xffffffff, 0x00080007,
897 0x91cc, 0xffffffff, 0x0000000b,
898 0x91d0, 0xffffffff, 0x000a0009,
899 0x91d4, 0xffffffff, 0x000d000c,
900 0x9150, 0xffffffff, 0x96940200,
901 0x8708, 0xffffffff, 0x00900100,
902 0xc478, 0xffffffff, 0x00000080,
903 0xc404, 0xffffffff, 0x0020003f,
904 0x30, 0xffffffff, 0x0000001c,
905 0x34, 0x000f0000, 0x000f0000,
906 0x160c, 0xffffffff, 0x00000100,
907 0x1024, 0xffffffff, 0x00000100,
908 0x102c, 0x00000101, 0x00000000,
909 0x20a8, 0xffffffff, 0x00000104,
910 0x264c, 0x000c0000, 0x000c0000,
911 0x2648, 0x000c0000, 0x000c0000,
912 0x55e4, 0xff000fff, 0x00000100,
913 0x55e8, 0x00000001, 0x00000001,
914 0x2f50, 0x00000001, 0x00000001,
915 0x30cc, 0xc0000fff, 0x00000104,
916 0xc1e4, 0x00000001, 0x00000001,
917 0xd0c0, 0xfffffff0, 0x00000100,
918 0xd8c0, 0xfffffff0, 0x00000100
921 static const u32 hainan_mgcg_cgcg_init[] =
923 0xc400, 0xffffffff, 0xfffffffc,
924 0x802c, 0xffffffff, 0xe0000000,
925 0x9a60, 0xffffffff, 0x00000100,
926 0x92a4, 0xffffffff, 0x00000100,
927 0xc164, 0xffffffff, 0x00000100,
928 0x9774, 0xffffffff, 0x00000100,
929 0x8984, 0xffffffff, 0x06000100,
930 0x8a18, 0xffffffff, 0x00000100,
931 0x92a0, 0xffffffff, 0x00000100,
932 0xc380, 0xffffffff, 0x00000100,
933 0x8b28, 0xffffffff, 0x00000100,
934 0x9144, 0xffffffff, 0x00000100,
935 0x8d88, 0xffffffff, 0x00000100,
936 0x8d8c, 0xffffffff, 0x00000100,
937 0x9030, 0xffffffff, 0x00000100,
938 0x9034, 0xffffffff, 0x00000100,
939 0x9038, 0xffffffff, 0x00000100,
940 0x903c, 0xffffffff, 0x00000100,
941 0xad80, 0xffffffff, 0x00000100,
942 0xac54, 0xffffffff, 0x00000100,
943 0x897c, 0xffffffff, 0x06000100,
944 0x9868, 0xffffffff, 0x00000100,
945 0x9510, 0xffffffff, 0x00000100,
946 0xaf04, 0xffffffff, 0x00000100,
947 0xae04, 0xffffffff, 0x00000100,
948 0x949c, 0xffffffff, 0x00000100,
949 0x802c, 0xffffffff, 0xe0000000,
950 0x9160, 0xffffffff, 0x00010000,
951 0x9164, 0xffffffff, 0x00030002,
952 0x9168, 0xffffffff, 0x00040007,
953 0x916c, 0xffffffff, 0x00060005,
954 0x9170, 0xffffffff, 0x00090008,
955 0x9174, 0xffffffff, 0x00020001,
956 0x9178, 0xffffffff, 0x00040003,
957 0x917c, 0xffffffff, 0x00000007,
958 0x9180, 0xffffffff, 0x00060005,
959 0x9184, 0xffffffff, 0x00090008,
960 0x9188, 0xffffffff, 0x00030002,
961 0x918c, 0xffffffff, 0x00050004,
962 0x9190, 0xffffffff, 0x00000008,
963 0x9194, 0xffffffff, 0x00070006,
964 0x9198, 0xffffffff, 0x000a0009,
965 0x919c, 0xffffffff, 0x00040003,
966 0x91a0, 0xffffffff, 0x00060005,
967 0x91a4, 0xffffffff, 0x00000009,
968 0x91a8, 0xffffffff, 0x00080007,
969 0x91ac, 0xffffffff, 0x000b000a,
970 0x91b0, 0xffffffff, 0x00050004,
971 0x91b4, 0xffffffff, 0x00070006,
972 0x91b8, 0xffffffff, 0x0008000b,
973 0x91bc, 0xffffffff, 0x000a0009,
974 0x91c0, 0xffffffff, 0x000d000c,
975 0x91c4, 0xffffffff, 0x00060005,
976 0x91c8, 0xffffffff, 0x00080007,
977 0x91cc, 0xffffffff, 0x0000000b,
978 0x91d0, 0xffffffff, 0x000a0009,
979 0x91d4, 0xffffffff, 0x000d000c,
980 0x9150, 0xffffffff, 0x96940200,
981 0x8708, 0xffffffff, 0x00900100,
982 0xc478, 0xffffffff, 0x00000080,
983 0xc404, 0xffffffff, 0x0020003f,
984 0x30, 0xffffffff, 0x0000001c,
985 0x34, 0x000f0000, 0x000f0000,
986 0x160c, 0xffffffff, 0x00000100,
987 0x1024, 0xffffffff, 0x00000100,
988 0x20a8, 0xffffffff, 0x00000104,
989 0x264c, 0x000c0000, 0x000c0000,
990 0x2648, 0x000c0000, 0x000c0000,
991 0x2f50, 0x00000001, 0x00000001,
992 0x30cc, 0xc0000fff, 0x00000104,
993 0xc1e4, 0x00000001, 0x00000001,
994 0xd0c0, 0xfffffff0, 0x00000100,
995 0xd8c0, 0xfffffff0, 0x00000100
998 static u32 verde_pg_init[] =
1000 0x353c, 0xffffffff, 0x40000,
1001 0x3538, 0xffffffff, 0x200010ff,
1002 0x353c, 0xffffffff, 0x0,
1003 0x353c, 0xffffffff, 0x0,
1004 0x353c, 0xffffffff, 0x0,
1005 0x353c, 0xffffffff, 0x0,
1006 0x353c, 0xffffffff, 0x0,
1007 0x353c, 0xffffffff, 0x7007,
1008 0x3538, 0xffffffff, 0x300010ff,
1009 0x353c, 0xffffffff, 0x0,
1010 0x353c, 0xffffffff, 0x0,
1011 0x353c, 0xffffffff, 0x0,
1012 0x353c, 0xffffffff, 0x0,
1013 0x353c, 0xffffffff, 0x0,
1014 0x353c, 0xffffffff, 0x400000,
1015 0x3538, 0xffffffff, 0x100010ff,
1016 0x353c, 0xffffffff, 0x0,
1017 0x353c, 0xffffffff, 0x0,
1018 0x353c, 0xffffffff, 0x0,
1019 0x353c, 0xffffffff, 0x0,
1020 0x353c, 0xffffffff, 0x0,
1021 0x353c, 0xffffffff, 0x120200,
1022 0x3538, 0xffffffff, 0x500010ff,
1023 0x353c, 0xffffffff, 0x0,
1024 0x353c, 0xffffffff, 0x0,
1025 0x353c, 0xffffffff, 0x0,
1026 0x353c, 0xffffffff, 0x0,
1027 0x353c, 0xffffffff, 0x0,
1028 0x353c, 0xffffffff, 0x1e1e16,
1029 0x3538, 0xffffffff, 0x600010ff,
1030 0x353c, 0xffffffff, 0x0,
1031 0x353c, 0xffffffff, 0x0,
1032 0x353c, 0xffffffff, 0x0,
1033 0x353c, 0xffffffff, 0x0,
1034 0x353c, 0xffffffff, 0x0,
1035 0x353c, 0xffffffff, 0x171f1e,
1036 0x3538, 0xffffffff, 0x700010ff,
1037 0x353c, 0xffffffff, 0x0,
1038 0x353c, 0xffffffff, 0x0,
1039 0x353c, 0xffffffff, 0x0,
1040 0x353c, 0xffffffff, 0x0,
1041 0x353c, 0xffffffff, 0x0,
1042 0x353c, 0xffffffff, 0x0,
1043 0x3538, 0xffffffff, 0x9ff,
1044 0x3500, 0xffffffff, 0x0,
1045 0x3504, 0xffffffff, 0x10000800,
1046 0x3504, 0xffffffff, 0xf,
1047 0x3504, 0xffffffff, 0xf,
1048 0x3500, 0xffffffff, 0x4,
1049 0x3504, 0xffffffff, 0x1000051e,
1050 0x3504, 0xffffffff, 0xffff,
1051 0x3504, 0xffffffff, 0xffff,
1052 0x3500, 0xffffffff, 0x8,
1053 0x3504, 0xffffffff, 0x80500,
1054 0x3500, 0xffffffff, 0x12,
1055 0x3504, 0xffffffff, 0x9050c,
1056 0x3500, 0xffffffff, 0x1d,
1057 0x3504, 0xffffffff, 0xb052c,
1058 0x3500, 0xffffffff, 0x2a,
1059 0x3504, 0xffffffff, 0x1053e,
1060 0x3500, 0xffffffff, 0x2d,
1061 0x3504, 0xffffffff, 0x10546,
1062 0x3500, 0xffffffff, 0x30,
1063 0x3504, 0xffffffff, 0xa054e,
1064 0x3500, 0xffffffff, 0x3c,
1065 0x3504, 0xffffffff, 0x1055f,
1066 0x3500, 0xffffffff, 0x3f,
1067 0x3504, 0xffffffff, 0x10567,
1068 0x3500, 0xffffffff, 0x42,
1069 0x3504, 0xffffffff, 0x1056f,
1070 0x3500, 0xffffffff, 0x45,
1071 0x3504, 0xffffffff, 0x10572,
1072 0x3500, 0xffffffff, 0x48,
1073 0x3504, 0xffffffff, 0x20575,
1074 0x3500, 0xffffffff, 0x4c,
1075 0x3504, 0xffffffff, 0x190801,
1076 0x3500, 0xffffffff, 0x67,
1077 0x3504, 0xffffffff, 0x1082a,
1078 0x3500, 0xffffffff, 0x6a,
1079 0x3504, 0xffffffff, 0x1b082d,
1080 0x3500, 0xffffffff, 0x87,
1081 0x3504, 0xffffffff, 0x310851,
1082 0x3500, 0xffffffff, 0xba,
1083 0x3504, 0xffffffff, 0x891,
1084 0x3500, 0xffffffff, 0xbc,
1085 0x3504, 0xffffffff, 0x893,
1086 0x3500, 0xffffffff, 0xbe,
1087 0x3504, 0xffffffff, 0x20895,
1088 0x3500, 0xffffffff, 0xc2,
1089 0x3504, 0xffffffff, 0x20899,
1090 0x3500, 0xffffffff, 0xc6,
1091 0x3504, 0xffffffff, 0x2089d,
1092 0x3500, 0xffffffff, 0xca,
1093 0x3504, 0xffffffff, 0x8a1,
1094 0x3500, 0xffffffff, 0xcc,
1095 0x3504, 0xffffffff, 0x8a3,
1096 0x3500, 0xffffffff, 0xce,
1097 0x3504, 0xffffffff, 0x308a5,
1098 0x3500, 0xffffffff, 0xd3,
1099 0x3504, 0xffffffff, 0x6d08cd,
1100 0x3500, 0xffffffff, 0x142,
1101 0x3504, 0xffffffff, 0x2000095a,
1102 0x3504, 0xffffffff, 0x1,
1103 0x3500, 0xffffffff, 0x144,
1104 0x3504, 0xffffffff, 0x301f095b,
1105 0x3500, 0xffffffff, 0x165,
1106 0x3504, 0xffffffff, 0xc094d,
1107 0x3500, 0xffffffff, 0x173,
1108 0x3504, 0xffffffff, 0xf096d,
1109 0x3500, 0xffffffff, 0x184,
1110 0x3504, 0xffffffff, 0x15097f,
1111 0x3500, 0xffffffff, 0x19b,
1112 0x3504, 0xffffffff, 0xc0998,
1113 0x3500, 0xffffffff, 0x1a9,
1114 0x3504, 0xffffffff, 0x409a7,
1115 0x3500, 0xffffffff, 0x1af,
1116 0x3504, 0xffffffff, 0xcdc,
1117 0x3500, 0xffffffff, 0x1b1,
1118 0x3504, 0xffffffff, 0x800,
1119 0x3508, 0xffffffff, 0x6c9b2000,
1120 0x3510, 0xfc00, 0x2000,
1121 0x3544, 0xffffffff, 0xfc0,
1122 0x28d4, 0x00000100, 0x100
1125 static void si_init_golden_registers(struct radeon_device *rdev)
1127 switch (rdev->family) {
1129 radeon_program_register_sequence(rdev,
1130 tahiti_golden_registers,
1131 (const u32)ARRAY_SIZE(tahiti_golden_registers));
1132 radeon_program_register_sequence(rdev,
1133 tahiti_golden_rlc_registers,
1134 (const u32)ARRAY_SIZE(tahiti_golden_rlc_registers));
1135 radeon_program_register_sequence(rdev,
1136 tahiti_mgcg_cgcg_init,
1137 (const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init));
1138 radeon_program_register_sequence(rdev,
1139 tahiti_golden_registers2,
1140 (const u32)ARRAY_SIZE(tahiti_golden_registers2));
1143 radeon_program_register_sequence(rdev,
1144 pitcairn_golden_registers,
1145 (const u32)ARRAY_SIZE(pitcairn_golden_registers));
1146 radeon_program_register_sequence(rdev,
1147 pitcairn_golden_rlc_registers,
1148 (const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers));
1149 radeon_program_register_sequence(rdev,
1150 pitcairn_mgcg_cgcg_init,
1151 (const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
1154 radeon_program_register_sequence(rdev,
1155 verde_golden_registers,
1156 (const u32)ARRAY_SIZE(verde_golden_registers));
1157 radeon_program_register_sequence(rdev,
1158 verde_golden_rlc_registers,
1159 (const u32)ARRAY_SIZE(verde_golden_rlc_registers));
1160 radeon_program_register_sequence(rdev,
1161 verde_mgcg_cgcg_init,
1162 (const u32)ARRAY_SIZE(verde_mgcg_cgcg_init));
1163 radeon_program_register_sequence(rdev,
1165 (const u32)ARRAY_SIZE(verde_pg_init));
1168 radeon_program_register_sequence(rdev,
1169 oland_golden_registers,
1170 (const u32)ARRAY_SIZE(oland_golden_registers));
1171 radeon_program_register_sequence(rdev,
1172 oland_golden_rlc_registers,
1173 (const u32)ARRAY_SIZE(oland_golden_rlc_registers));
1174 radeon_program_register_sequence(rdev,
1175 oland_mgcg_cgcg_init,
1176 (const u32)ARRAY_SIZE(oland_mgcg_cgcg_init));
1179 radeon_program_register_sequence(rdev,
1180 hainan_golden_registers,
1181 (const u32)ARRAY_SIZE(hainan_golden_registers));
1182 radeon_program_register_sequence(rdev,
1183 hainan_golden_registers2,
1184 (const u32)ARRAY_SIZE(hainan_golden_registers2));
1185 radeon_program_register_sequence(rdev,
1186 hainan_mgcg_cgcg_init,
1187 (const u32)ARRAY_SIZE(hainan_mgcg_cgcg_init));
1195 * si_get_allowed_info_register - fetch the register for the info ioctl
1197 * @rdev: radeon_device pointer
1198 * @reg: register offset in bytes
1199 * @val: register value
1201 * Returns 0 for success or -EINVAL for an invalid register
1204 int si_get_allowed_info_register(struct radeon_device *rdev,
1210 case GRBM_STATUS_SE0:
1211 case GRBM_STATUS_SE1:
1214 case (DMA_STATUS_REG + DMA0_REGISTER_OFFSET):
1215 case (DMA_STATUS_REG + DMA1_REGISTER_OFFSET):
1224 #define PCIE_BUS_CLK 10000
1225 #define TCLK (PCIE_BUS_CLK / 10)
1228 * si_get_xclk - get the xclk
1230 * @rdev: radeon_device pointer
1232 * Returns the reference clock used by the gfx engine
1235 u32 si_get_xclk(struct radeon_device *rdev)
1237 u32 reference_clock = rdev->clock.spll.reference_freq;
1240 tmp = RREG32(CG_CLKPIN_CNTL_2);
1241 if (tmp & MUX_TCLK_TO_XCLK)
1244 tmp = RREG32(CG_CLKPIN_CNTL);
1245 if (tmp & XTALIN_DIVIDE)
1246 return reference_clock / 4;
1248 return reference_clock;
1251 /* get temperature in millidegrees */
1252 int si_get_temp(struct radeon_device *rdev)
1255 int actual_temp = 0;
1257 temp = (RREG32(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
1263 actual_temp = temp & 0x1ff;
1265 actual_temp = (actual_temp * 1000);
1270 #define TAHITI_IO_MC_REGS_SIZE 36
1272 static const u32 tahiti_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1273 {0x0000006f, 0x03044000},
1274 {0x00000070, 0x0480c018},
1275 {0x00000071, 0x00000040},
1276 {0x00000072, 0x01000000},
1277 {0x00000074, 0x000000ff},
1278 {0x00000075, 0x00143400},
1279 {0x00000076, 0x08ec0800},
1280 {0x00000077, 0x040000cc},
1281 {0x00000079, 0x00000000},
1282 {0x0000007a, 0x21000409},
1283 {0x0000007c, 0x00000000},
1284 {0x0000007d, 0xe8000000},
1285 {0x0000007e, 0x044408a8},
1286 {0x0000007f, 0x00000003},
1287 {0x00000080, 0x00000000},
1288 {0x00000081, 0x01000000},
1289 {0x00000082, 0x02000000},
1290 {0x00000083, 0x00000000},
1291 {0x00000084, 0xe3f3e4f4},
1292 {0x00000085, 0x00052024},
1293 {0x00000087, 0x00000000},
1294 {0x00000088, 0x66036603},
1295 {0x00000089, 0x01000000},
1296 {0x0000008b, 0x1c0a0000},
1297 {0x0000008c, 0xff010000},
1298 {0x0000008e, 0xffffefff},
1299 {0x0000008f, 0xfff3efff},
1300 {0x00000090, 0xfff3efbf},
1301 {0x00000094, 0x00101101},
1302 {0x00000095, 0x00000fff},
1303 {0x00000096, 0x00116fff},
1304 {0x00000097, 0x60010000},
1305 {0x00000098, 0x10010000},
1306 {0x00000099, 0x00006000},
1307 {0x0000009a, 0x00001000},
1308 {0x0000009f, 0x00a77400}
1311 static const u32 pitcairn_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1312 {0x0000006f, 0x03044000},
1313 {0x00000070, 0x0480c018},
1314 {0x00000071, 0x00000040},
1315 {0x00000072, 0x01000000},
1316 {0x00000074, 0x000000ff},
1317 {0x00000075, 0x00143400},
1318 {0x00000076, 0x08ec0800},
1319 {0x00000077, 0x040000cc},
1320 {0x00000079, 0x00000000},
1321 {0x0000007a, 0x21000409},
1322 {0x0000007c, 0x00000000},
1323 {0x0000007d, 0xe8000000},
1324 {0x0000007e, 0x044408a8},
1325 {0x0000007f, 0x00000003},
1326 {0x00000080, 0x00000000},
1327 {0x00000081, 0x01000000},
1328 {0x00000082, 0x02000000},
1329 {0x00000083, 0x00000000},
1330 {0x00000084, 0xe3f3e4f4},
1331 {0x00000085, 0x00052024},
1332 {0x00000087, 0x00000000},
1333 {0x00000088, 0x66036603},
1334 {0x00000089, 0x01000000},
1335 {0x0000008b, 0x1c0a0000},
1336 {0x0000008c, 0xff010000},
1337 {0x0000008e, 0xffffefff},
1338 {0x0000008f, 0xfff3efff},
1339 {0x00000090, 0xfff3efbf},
1340 {0x00000094, 0x00101101},
1341 {0x00000095, 0x00000fff},
1342 {0x00000096, 0x00116fff},
1343 {0x00000097, 0x60010000},
1344 {0x00000098, 0x10010000},
1345 {0x00000099, 0x00006000},
1346 {0x0000009a, 0x00001000},
1347 {0x0000009f, 0x00a47400}
1350 static const u32 verde_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1351 {0x0000006f, 0x03044000},
1352 {0x00000070, 0x0480c018},
1353 {0x00000071, 0x00000040},
1354 {0x00000072, 0x01000000},
1355 {0x00000074, 0x000000ff},
1356 {0x00000075, 0x00143400},
1357 {0x00000076, 0x08ec0800},
1358 {0x00000077, 0x040000cc},
1359 {0x00000079, 0x00000000},
1360 {0x0000007a, 0x21000409},
1361 {0x0000007c, 0x00000000},
1362 {0x0000007d, 0xe8000000},
1363 {0x0000007e, 0x044408a8},
1364 {0x0000007f, 0x00000003},
1365 {0x00000080, 0x00000000},
1366 {0x00000081, 0x01000000},
1367 {0x00000082, 0x02000000},
1368 {0x00000083, 0x00000000},
1369 {0x00000084, 0xe3f3e4f4},
1370 {0x00000085, 0x00052024},
1371 {0x00000087, 0x00000000},
1372 {0x00000088, 0x66036603},
1373 {0x00000089, 0x01000000},
1374 {0x0000008b, 0x1c0a0000},
1375 {0x0000008c, 0xff010000},
1376 {0x0000008e, 0xffffefff},
1377 {0x0000008f, 0xfff3efff},
1378 {0x00000090, 0xfff3efbf},
1379 {0x00000094, 0x00101101},
1380 {0x00000095, 0x00000fff},
1381 {0x00000096, 0x00116fff},
1382 {0x00000097, 0x60010000},
1383 {0x00000098, 0x10010000},
1384 {0x00000099, 0x00006000},
1385 {0x0000009a, 0x00001000},
1386 {0x0000009f, 0x00a37400}
1389 static const u32 oland_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1390 {0x0000006f, 0x03044000},
1391 {0x00000070, 0x0480c018},
1392 {0x00000071, 0x00000040},
1393 {0x00000072, 0x01000000},
1394 {0x00000074, 0x000000ff},
1395 {0x00000075, 0x00143400},
1396 {0x00000076, 0x08ec0800},
1397 {0x00000077, 0x040000cc},
1398 {0x00000079, 0x00000000},
1399 {0x0000007a, 0x21000409},
1400 {0x0000007c, 0x00000000},
1401 {0x0000007d, 0xe8000000},
1402 {0x0000007e, 0x044408a8},
1403 {0x0000007f, 0x00000003},
1404 {0x00000080, 0x00000000},
1405 {0x00000081, 0x01000000},
1406 {0x00000082, 0x02000000},
1407 {0x00000083, 0x00000000},
1408 {0x00000084, 0xe3f3e4f4},
1409 {0x00000085, 0x00052024},
1410 {0x00000087, 0x00000000},
1411 {0x00000088, 0x66036603},
1412 {0x00000089, 0x01000000},
1413 {0x0000008b, 0x1c0a0000},
1414 {0x0000008c, 0xff010000},
1415 {0x0000008e, 0xffffefff},
1416 {0x0000008f, 0xfff3efff},
1417 {0x00000090, 0xfff3efbf},
1418 {0x00000094, 0x00101101},
1419 {0x00000095, 0x00000fff},
1420 {0x00000096, 0x00116fff},
1421 {0x00000097, 0x60010000},
1422 {0x00000098, 0x10010000},
1423 {0x00000099, 0x00006000},
1424 {0x0000009a, 0x00001000},
1425 {0x0000009f, 0x00a17730}
1428 static const u32 hainan_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1429 {0x0000006f, 0x03044000},
1430 {0x00000070, 0x0480c018},
1431 {0x00000071, 0x00000040},
1432 {0x00000072, 0x01000000},
1433 {0x00000074, 0x000000ff},
1434 {0x00000075, 0x00143400},
1435 {0x00000076, 0x08ec0800},
1436 {0x00000077, 0x040000cc},
1437 {0x00000079, 0x00000000},
1438 {0x0000007a, 0x21000409},
1439 {0x0000007c, 0x00000000},
1440 {0x0000007d, 0xe8000000},
1441 {0x0000007e, 0x044408a8},
1442 {0x0000007f, 0x00000003},
1443 {0x00000080, 0x00000000},
1444 {0x00000081, 0x01000000},
1445 {0x00000082, 0x02000000},
1446 {0x00000083, 0x00000000},
1447 {0x00000084, 0xe3f3e4f4},
1448 {0x00000085, 0x00052024},
1449 {0x00000087, 0x00000000},
1450 {0x00000088, 0x66036603},
1451 {0x00000089, 0x01000000},
1452 {0x0000008b, 0x1c0a0000},
1453 {0x0000008c, 0xff010000},
1454 {0x0000008e, 0xffffefff},
1455 {0x0000008f, 0xfff3efff},
1456 {0x00000090, 0xfff3efbf},
1457 {0x00000094, 0x00101101},
1458 {0x00000095, 0x00000fff},
1459 {0x00000096, 0x00116fff},
1460 {0x00000097, 0x60010000},
1461 {0x00000098, 0x10010000},
1462 {0x00000099, 0x00006000},
1463 {0x0000009a, 0x00001000},
1464 {0x0000009f, 0x00a07730}
1468 int si_mc_load_microcode(struct radeon_device *rdev)
1470 const __be32 *fw_data = NULL;
1471 const __le32 *new_fw_data = NULL;
1473 u32 *io_mc_regs = NULL;
1474 const __le32 *new_io_mc_regs = NULL;
1475 int i, regs_size, ucode_size;
1481 const struct mc_firmware_header_v1_0 *hdr =
1482 (const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data;
1484 radeon_ucode_print_mc_hdr(&hdr->header);
1485 regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
1486 new_io_mc_regs = (const __le32 *)
1487 (rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
1488 ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
1489 new_fw_data = (const __le32 *)
1490 (rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1492 ucode_size = rdev->mc_fw->size / 4;
1494 switch (rdev->family) {
1496 io_mc_regs = (u32 *)&tahiti_io_mc_regs;
1497 regs_size = TAHITI_IO_MC_REGS_SIZE;
1500 io_mc_regs = (u32 *)&pitcairn_io_mc_regs;
1501 regs_size = TAHITI_IO_MC_REGS_SIZE;
1505 io_mc_regs = (u32 *)&verde_io_mc_regs;
1506 regs_size = TAHITI_IO_MC_REGS_SIZE;
1509 io_mc_regs = (u32 *)&oland_io_mc_regs;
1510 regs_size = TAHITI_IO_MC_REGS_SIZE;
1513 io_mc_regs = (u32 *)&hainan_io_mc_regs;
1514 regs_size = TAHITI_IO_MC_REGS_SIZE;
1517 fw_data = (const __be32 *)rdev->mc_fw->data;
1520 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1523 /* reset the engine and set to writable */
1524 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1525 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1527 /* load mc io regs */
1528 for (i = 0; i < regs_size; i++) {
1530 WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++));
1531 WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++));
1533 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1534 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1537 /* load the MC ucode */
1538 for (i = 0; i < ucode_size; i++) {
1540 WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++));
1542 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1545 /* put the engine back into the active state */
1546 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1547 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1548 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1550 /* wait for training to complete */
1551 for (i = 0; i < rdev->usec_timeout; i++) {
1552 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1556 for (i = 0; i < rdev->usec_timeout; i++) {
1557 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1566 static int si_init_microcode(struct radeon_device *rdev)
1568 const char *chip_name;
1569 const char *new_chip_name;
1570 size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size;
1571 size_t smc_req_size, mc2_req_size;
1575 bool new_smc = false;
1579 switch (rdev->family) {
1581 chip_name = "TAHITI";
1582 /* XXX: figure out which Tahitis need the new ucode */
1585 new_chip_name = "tahiti";
1586 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1587 me_req_size = SI_PM4_UCODE_SIZE * 4;
1588 ce_req_size = SI_CE_UCODE_SIZE * 4;
1589 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1590 mc_req_size = SI_MC_UCODE_SIZE * 4;
1591 mc2_req_size = TAHITI_MC_UCODE_SIZE * 4;
1592 smc_req_size = ALIGN(TAHITI_SMC_UCODE_SIZE, 4);
1595 chip_name = "PITCAIRN";
1596 if ((rdev->pdev->revision == 0x81) ||
1597 (rdev->pdev->device == 0x6810) ||
1598 (rdev->pdev->device == 0x6811) ||
1599 (rdev->pdev->device == 0x6816) ||
1600 (rdev->pdev->device == 0x6817) ||
1601 (rdev->pdev->device == 0x6806))
1603 new_chip_name = "pitcairn";
1604 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1605 me_req_size = SI_PM4_UCODE_SIZE * 4;
1606 ce_req_size = SI_CE_UCODE_SIZE * 4;
1607 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1608 mc_req_size = SI_MC_UCODE_SIZE * 4;
1609 mc2_req_size = PITCAIRN_MC_UCODE_SIZE * 4;
1610 smc_req_size = ALIGN(PITCAIRN_SMC_UCODE_SIZE, 4);
1613 chip_name = "VERDE";
1614 if ((rdev->pdev->revision == 0x81) ||
1615 (rdev->pdev->revision == 0x83) ||
1616 (rdev->pdev->revision == 0x87) ||
1617 (rdev->pdev->device == 0x6820) ||
1618 (rdev->pdev->device == 0x6821) ||
1619 (rdev->pdev->device == 0x6822) ||
1620 (rdev->pdev->device == 0x6823) ||
1621 (rdev->pdev->device == 0x682A) ||
1622 (rdev->pdev->device == 0x682B))
1624 new_chip_name = "verde";
1625 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1626 me_req_size = SI_PM4_UCODE_SIZE * 4;
1627 ce_req_size = SI_CE_UCODE_SIZE * 4;
1628 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1629 mc_req_size = SI_MC_UCODE_SIZE * 4;
1630 mc2_req_size = VERDE_MC_UCODE_SIZE * 4;
1631 smc_req_size = ALIGN(VERDE_SMC_UCODE_SIZE, 4);
1634 chip_name = "OLAND";
1635 if ((rdev->pdev->revision == 0xC7) ||
1636 (rdev->pdev->revision == 0x80) ||
1637 (rdev->pdev->revision == 0x81) ||
1638 (rdev->pdev->revision == 0x83) ||
1639 (rdev->pdev->device == 0x6604) ||
1640 (rdev->pdev->device == 0x6605))
1642 new_chip_name = "oland";
1643 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1644 me_req_size = SI_PM4_UCODE_SIZE * 4;
1645 ce_req_size = SI_CE_UCODE_SIZE * 4;
1646 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1647 mc_req_size = mc2_req_size = OLAND_MC_UCODE_SIZE * 4;
1648 smc_req_size = ALIGN(OLAND_SMC_UCODE_SIZE, 4);
1651 chip_name = "HAINAN";
1652 if ((rdev->pdev->revision == 0x81) ||
1653 (rdev->pdev->revision == 0x83) ||
1654 (rdev->pdev->revision == 0xC3) ||
1655 (rdev->pdev->device == 0x6664) ||
1656 (rdev->pdev->device == 0x6665) ||
1657 (rdev->pdev->device == 0x6667))
1659 new_chip_name = "hainan";
1660 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1661 me_req_size = SI_PM4_UCODE_SIZE * 4;
1662 ce_req_size = SI_CE_UCODE_SIZE * 4;
1663 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1664 mc_req_size = mc2_req_size = OLAND_MC_UCODE_SIZE * 4;
1665 smc_req_size = ALIGN(HAINAN_SMC_UCODE_SIZE, 4);
1670 DRM_INFO("Loading %s Microcode\n", new_chip_name);
1672 snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", new_chip_name);
1673 err = reject_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1675 snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", chip_name);
1676 err = reject_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1679 if (rdev->pfp_fw->size != pfp_req_size) {
1681 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1682 rdev->pfp_fw->size, fw_name);
1687 err = radeon_ucode_validate(rdev->pfp_fw);
1690 "si_cp: validation failed for firmware \"%s\"\n",
1698 snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", new_chip_name);
1699 err = reject_firmware(&rdev->me_fw, fw_name, rdev->dev);
1701 snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", chip_name);
1702 err = reject_firmware(&rdev->me_fw, fw_name, rdev->dev);
1705 if (rdev->me_fw->size != me_req_size) {
1707 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1708 rdev->me_fw->size, fw_name);
1712 err = radeon_ucode_validate(rdev->me_fw);
1715 "si_cp: validation failed for firmware \"%s\"\n",
1723 snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", new_chip_name);
1724 err = reject_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1726 snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", chip_name);
1727 err = reject_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1730 if (rdev->ce_fw->size != ce_req_size) {
1732 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1733 rdev->ce_fw->size, fw_name);
1737 err = radeon_ucode_validate(rdev->ce_fw);
1740 "si_cp: validation failed for firmware \"%s\"\n",
1748 snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", new_chip_name);
1749 err = reject_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1751 snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", chip_name);
1752 err = reject_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1755 if (rdev->rlc_fw->size != rlc_req_size) {
1757 "si_rlc: Bogus length %zu in firmware \"%s\"\n",
1758 rdev->rlc_fw->size, fw_name);
1762 err = radeon_ucode_validate(rdev->rlc_fw);
1765 "si_cp: validation failed for firmware \"%s\"\n",
1773 snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", new_chip_name);
1774 err = reject_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1776 snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", chip_name);
1777 err = reject_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1779 snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", chip_name);
1780 err = reject_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1784 if ((rdev->mc_fw->size != mc_req_size) &&
1785 (rdev->mc_fw->size != mc2_req_size)) {
1787 "si_mc: Bogus length %zu in firmware \"%s\"\n",
1788 rdev->mc_fw->size, fw_name);
1791 DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
1793 err = radeon_ucode_validate(rdev->mc_fw);
1796 "si_cp: validation failed for firmware \"%s\"\n",
1805 snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", new_chip_name);
1807 snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", new_chip_name);
1808 err = reject_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1810 snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", chip_name);
1811 err = reject_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1814 "smc: error loading firmware \"%s\"\n",
1816 release_firmware(rdev->smc_fw);
1817 rdev->smc_fw = NULL;
1819 } else if (rdev->smc_fw->size != smc_req_size) {
1821 "si_smc: Bogus length %zu in firmware \"%s\"\n",
1822 rdev->smc_fw->size, fw_name);
1826 err = radeon_ucode_validate(rdev->smc_fw);
1829 "si_cp: validation failed for firmware \"%s\"\n",
1838 rdev->new_fw = false;
1839 } else if (new_fw < 6) {
1840 printk(KERN_ERR "si_fw: mixing new and old firmware!\n");
1843 rdev->new_fw = true;
1849 "si_cp: Failed to load firmware \"%s\"\n",
1851 release_firmware(rdev->pfp_fw);
1852 rdev->pfp_fw = NULL;
1853 release_firmware(rdev->me_fw);
1855 release_firmware(rdev->ce_fw);
1857 release_firmware(rdev->rlc_fw);
1858 rdev->rlc_fw = NULL;
1859 release_firmware(rdev->mc_fw);
1861 release_firmware(rdev->smc_fw);
1862 rdev->smc_fw = NULL;
1867 /* watermark setup */
1868 static u32 dce6_line_buffer_adjust(struct radeon_device *rdev,
1869 struct radeon_crtc *radeon_crtc,
1870 struct drm_display_mode *mode,
1871 struct drm_display_mode *other_mode)
1873 u32 tmp, buffer_alloc, i;
1874 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1877 * There are 3 line buffers, each one shared by 2 display controllers.
1878 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1879 * the display controllers. The paritioning is done via one of four
1880 * preset allocations specified in bits 21:20:
1882 * 2 - whole lb, other crtc must be disabled
1884 /* this can get tricky if we have two large displays on a paired group
1885 * of crtcs. Ideally for multiple large displays we'd assign them to
1886 * non-linked crtcs for maximum line buffer allocation.
1888 if (radeon_crtc->base.enabled && mode) {
1893 tmp = 2; /* whole */
1901 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset,
1902 DC_LB_MEMORY_CONFIG(tmp));
1904 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1905 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1906 for (i = 0; i < rdev->usec_timeout; i++) {
1907 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1908 DMIF_BUFFERS_ALLOCATED_COMPLETED)
1913 if (radeon_crtc->base.enabled && mode) {
1923 /* controller not enabled, so no lb used */
1927 static u32 si_get_number_of_dram_channels(struct radeon_device *rdev)
1929 u32 tmp = RREG32(MC_SHARED_CHMAP);
1931 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1954 struct dce6_wm_params {
1955 u32 dram_channels; /* number of dram channels */
1956 u32 yclk; /* bandwidth per dram data pin in kHz */
1957 u32 sclk; /* engine clock in kHz */
1958 u32 disp_clk; /* display clock in kHz */
1959 u32 src_width; /* viewport width */
1960 u32 active_time; /* active display time in ns */
1961 u32 blank_time; /* blank time in ns */
1962 bool interlaced; /* mode is interlaced */
1963 fixed20_12 vsc; /* vertical scale ratio */
1964 u32 num_heads; /* number of active crtcs */
1965 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1966 u32 lb_size; /* line buffer allocated to pipe */
1967 u32 vtaps; /* vertical scaler taps */
1970 static u32 dce6_dram_bandwidth(struct dce6_wm_params *wm)
1972 /* Calculate raw DRAM Bandwidth */
1973 fixed20_12 dram_efficiency; /* 0.7 */
1974 fixed20_12 yclk, dram_channels, bandwidth;
1977 a.full = dfixed_const(1000);
1978 yclk.full = dfixed_const(wm->yclk);
1979 yclk.full = dfixed_div(yclk, a);
1980 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1981 a.full = dfixed_const(10);
1982 dram_efficiency.full = dfixed_const(7);
1983 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1984 bandwidth.full = dfixed_mul(dram_channels, yclk);
1985 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1987 return dfixed_trunc(bandwidth);
1990 static u32 dce6_dram_bandwidth_for_display(struct dce6_wm_params *wm)
1992 /* Calculate DRAM Bandwidth and the part allocated to display. */
1993 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1994 fixed20_12 yclk, dram_channels, bandwidth;
1997 a.full = dfixed_const(1000);
1998 yclk.full = dfixed_const(wm->yclk);
1999 yclk.full = dfixed_div(yclk, a);
2000 dram_channels.full = dfixed_const(wm->dram_channels * 4);
2001 a.full = dfixed_const(10);
2002 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
2003 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
2004 bandwidth.full = dfixed_mul(dram_channels, yclk);
2005 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
2007 return dfixed_trunc(bandwidth);
2010 static u32 dce6_data_return_bandwidth(struct dce6_wm_params *wm)
2012 /* Calculate the display Data return Bandwidth */
2013 fixed20_12 return_efficiency; /* 0.8 */
2014 fixed20_12 sclk, bandwidth;
2017 a.full = dfixed_const(1000);
2018 sclk.full = dfixed_const(wm->sclk);
2019 sclk.full = dfixed_div(sclk, a);
2020 a.full = dfixed_const(10);
2021 return_efficiency.full = dfixed_const(8);
2022 return_efficiency.full = dfixed_div(return_efficiency, a);
2023 a.full = dfixed_const(32);
2024 bandwidth.full = dfixed_mul(a, sclk);
2025 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
2027 return dfixed_trunc(bandwidth);
2030 static u32 dce6_get_dmif_bytes_per_request(struct dce6_wm_params *wm)
2035 static u32 dce6_dmif_request_bandwidth(struct dce6_wm_params *wm)
2037 /* Calculate the DMIF Request Bandwidth */
2038 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
2039 fixed20_12 disp_clk, sclk, bandwidth;
2040 fixed20_12 a, b1, b2;
2043 a.full = dfixed_const(1000);
2044 disp_clk.full = dfixed_const(wm->disp_clk);
2045 disp_clk.full = dfixed_div(disp_clk, a);
2046 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm) / 2);
2047 b1.full = dfixed_mul(a, disp_clk);
2049 a.full = dfixed_const(1000);
2050 sclk.full = dfixed_const(wm->sclk);
2051 sclk.full = dfixed_div(sclk, a);
2052 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm));
2053 b2.full = dfixed_mul(a, sclk);
2055 a.full = dfixed_const(10);
2056 disp_clk_request_efficiency.full = dfixed_const(8);
2057 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2059 min_bandwidth = min(dfixed_trunc(b1), dfixed_trunc(b2));
2061 a.full = dfixed_const(min_bandwidth);
2062 bandwidth.full = dfixed_mul(a, disp_clk_request_efficiency);
2064 return dfixed_trunc(bandwidth);
2067 static u32 dce6_available_bandwidth(struct dce6_wm_params *wm)
2069 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2070 u32 dram_bandwidth = dce6_dram_bandwidth(wm);
2071 u32 data_return_bandwidth = dce6_data_return_bandwidth(wm);
2072 u32 dmif_req_bandwidth = dce6_dmif_request_bandwidth(wm);
2074 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2077 static u32 dce6_average_bandwidth(struct dce6_wm_params *wm)
2079 /* Calculate the display mode Average Bandwidth
2080 * DisplayMode should contain the source and destination dimensions,
2084 fixed20_12 line_time;
2085 fixed20_12 src_width;
2086 fixed20_12 bandwidth;
2089 a.full = dfixed_const(1000);
2090 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2091 line_time.full = dfixed_div(line_time, a);
2092 bpp.full = dfixed_const(wm->bytes_per_pixel);
2093 src_width.full = dfixed_const(wm->src_width);
2094 bandwidth.full = dfixed_mul(src_width, bpp);
2095 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2096 bandwidth.full = dfixed_div(bandwidth, line_time);
2098 return dfixed_trunc(bandwidth);
2101 static u32 dce6_latency_watermark(struct dce6_wm_params *wm)
2103 /* First calcualte the latency in ns */
2104 u32 mc_latency = 2000; /* 2000 ns. */
2105 u32 available_bandwidth = dce6_available_bandwidth(wm);
2106 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2107 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2108 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2109 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2110 (wm->num_heads * cursor_line_pair_return_time);
2111 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2112 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2113 u32 tmp, dmif_size = 12288;
2116 if (wm->num_heads == 0)
2119 a.full = dfixed_const(2);
2120 b.full = dfixed_const(1);
2121 if ((wm->vsc.full > a.full) ||
2122 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2124 ((wm->vsc.full >= a.full) && wm->interlaced))
2125 max_src_lines_per_dst_line = 4;
2127 max_src_lines_per_dst_line = 2;
2129 a.full = dfixed_const(available_bandwidth);
2130 b.full = dfixed_const(wm->num_heads);
2131 a.full = dfixed_div(a, b);
2133 b.full = dfixed_const(mc_latency + 512);
2134 c.full = dfixed_const(wm->disp_clk);
2135 b.full = dfixed_div(b, c);
2137 c.full = dfixed_const(dmif_size);
2138 b.full = dfixed_div(c, b);
2140 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
2142 b.full = dfixed_const(1000);
2143 c.full = dfixed_const(wm->disp_clk);
2144 b.full = dfixed_div(c, b);
2145 c.full = dfixed_const(wm->bytes_per_pixel);
2146 b.full = dfixed_mul(b, c);
2148 lb_fill_bw = min(tmp, dfixed_trunc(b));
2150 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2151 b.full = dfixed_const(1000);
2152 c.full = dfixed_const(lb_fill_bw);
2153 b.full = dfixed_div(c, b);
2154 a.full = dfixed_div(a, b);
2155 line_fill_time = dfixed_trunc(a);
2157 if (line_fill_time < wm->active_time)
2160 return latency + (line_fill_time - wm->active_time);
2164 static bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params *wm)
2166 if (dce6_average_bandwidth(wm) <=
2167 (dce6_dram_bandwidth_for_display(wm) / wm->num_heads))
2173 static bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params *wm)
2175 if (dce6_average_bandwidth(wm) <=
2176 (dce6_available_bandwidth(wm) / wm->num_heads))
2182 static bool dce6_check_latency_hiding(struct dce6_wm_params *wm)
2184 u32 lb_partitions = wm->lb_size / wm->src_width;
2185 u32 line_time = wm->active_time + wm->blank_time;
2186 u32 latency_tolerant_lines;
2190 a.full = dfixed_const(1);
2191 if (wm->vsc.full > a.full)
2192 latency_tolerant_lines = 1;
2194 if (lb_partitions <= (wm->vtaps + 1))
2195 latency_tolerant_lines = 1;
2197 latency_tolerant_lines = 2;
2200 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2202 if (dce6_latency_watermark(wm) <= latency_hiding)
2208 static void dce6_program_watermarks(struct radeon_device *rdev,
2209 struct radeon_crtc *radeon_crtc,
2210 u32 lb_size, u32 num_heads)
2212 struct drm_display_mode *mode = &radeon_crtc->base.mode;
2213 struct dce6_wm_params wm_low, wm_high;
2217 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2218 u32 priority_a_mark = 0, priority_b_mark = 0;
2219 u32 priority_a_cnt = PRIORITY_OFF;
2220 u32 priority_b_cnt = PRIORITY_OFF;
2221 u32 tmp, arb_control3;
2224 if (radeon_crtc->base.enabled && num_heads && mode) {
2225 pixel_period = 1000000 / (u32)mode->clock;
2226 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2230 if (rdev->family == CHIP_ARUBA)
2231 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2233 dram_channels = si_get_number_of_dram_channels(rdev);
2235 /* watermark for high clocks */
2236 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2238 radeon_dpm_get_mclk(rdev, false) * 10;
2240 radeon_dpm_get_sclk(rdev, false) * 10;
2242 wm_high.yclk = rdev->pm.current_mclk * 10;
2243 wm_high.sclk = rdev->pm.current_sclk * 10;
2246 wm_high.disp_clk = mode->clock;
2247 wm_high.src_width = mode->crtc_hdisplay;
2248 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2249 wm_high.blank_time = line_time - wm_high.active_time;
2250 wm_high.interlaced = false;
2251 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2252 wm_high.interlaced = true;
2253 wm_high.vsc = radeon_crtc->vsc;
2255 if (radeon_crtc->rmx_type != RMX_OFF)
2257 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2258 wm_high.lb_size = lb_size;
2259 wm_high.dram_channels = dram_channels;
2260 wm_high.num_heads = num_heads;
2262 /* watermark for low clocks */
2263 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2265 radeon_dpm_get_mclk(rdev, true) * 10;
2267 radeon_dpm_get_sclk(rdev, true) * 10;
2269 wm_low.yclk = rdev->pm.current_mclk * 10;
2270 wm_low.sclk = rdev->pm.current_sclk * 10;
2273 wm_low.disp_clk = mode->clock;
2274 wm_low.src_width = mode->crtc_hdisplay;
2275 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2276 wm_low.blank_time = line_time - wm_low.active_time;
2277 wm_low.interlaced = false;
2278 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2279 wm_low.interlaced = true;
2280 wm_low.vsc = radeon_crtc->vsc;
2282 if (radeon_crtc->rmx_type != RMX_OFF)
2284 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2285 wm_low.lb_size = lb_size;
2286 wm_low.dram_channels = dram_channels;
2287 wm_low.num_heads = num_heads;
2289 /* set for high clocks */
2290 latency_watermark_a = min(dce6_latency_watermark(&wm_high), (u32)65535);
2291 /* set for low clocks */
2292 latency_watermark_b = min(dce6_latency_watermark(&wm_low), (u32)65535);
2294 /* possibly force display priority to high */
2295 /* should really do this at mode validation time... */
2296 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2297 !dce6_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2298 !dce6_check_latency_hiding(&wm_high) ||
2299 (rdev->disp_priority == 2)) {
2300 DRM_DEBUG_KMS("force priority to high\n");
2301 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2302 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2304 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2305 !dce6_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2306 !dce6_check_latency_hiding(&wm_low) ||
2307 (rdev->disp_priority == 2)) {
2308 DRM_DEBUG_KMS("force priority to high\n");
2309 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2310 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2313 a.full = dfixed_const(1000);
2314 b.full = dfixed_const(mode->clock);
2315 b.full = dfixed_div(b, a);
2316 c.full = dfixed_const(latency_watermark_a);
2317 c.full = dfixed_mul(c, b);
2318 c.full = dfixed_mul(c, radeon_crtc->hsc);
2319 c.full = dfixed_div(c, a);
2320 a.full = dfixed_const(16);
2321 c.full = dfixed_div(c, a);
2322 priority_a_mark = dfixed_trunc(c);
2323 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2325 a.full = dfixed_const(1000);
2326 b.full = dfixed_const(mode->clock);
2327 b.full = dfixed_div(b, a);
2328 c.full = dfixed_const(latency_watermark_b);
2329 c.full = dfixed_mul(c, b);
2330 c.full = dfixed_mul(c, radeon_crtc->hsc);
2331 c.full = dfixed_div(c, a);
2332 a.full = dfixed_const(16);
2333 c.full = dfixed_div(c, a);
2334 priority_b_mark = dfixed_trunc(c);
2335 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2337 /* Save number of lines the linebuffer leads before the scanout */
2338 radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay);
2342 arb_control3 = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2344 tmp &= ~LATENCY_WATERMARK_MASK(3);
2345 tmp |= LATENCY_WATERMARK_MASK(1);
2346 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2347 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2348 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2349 LATENCY_HIGH_WATERMARK(line_time)));
2351 tmp = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2352 tmp &= ~LATENCY_WATERMARK_MASK(3);
2353 tmp |= LATENCY_WATERMARK_MASK(2);
2354 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2355 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2356 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2357 LATENCY_HIGH_WATERMARK(line_time)));
2358 /* restore original selection */
2359 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, arb_control3);
2361 /* write the priority marks */
2362 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2363 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2365 /* save values for DPM */
2366 radeon_crtc->line_time = line_time;
2367 radeon_crtc->wm_high = latency_watermark_a;
2368 radeon_crtc->wm_low = latency_watermark_b;
2371 void dce6_bandwidth_update(struct radeon_device *rdev)
2373 struct drm_display_mode *mode0 = NULL;
2374 struct drm_display_mode *mode1 = NULL;
2375 u32 num_heads = 0, lb_size;
2378 if (!rdev->mode_info.mode_config_initialized)
2381 radeon_update_display_priority(rdev);
2383 for (i = 0; i < rdev->num_crtc; i++) {
2384 if (rdev->mode_info.crtcs[i]->base.enabled)
2387 for (i = 0; i < rdev->num_crtc; i += 2) {
2388 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2389 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2390 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2391 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2392 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2393 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2400 static void si_tiling_mode_table_init(struct radeon_device *rdev)
2402 u32 *tile = rdev->config.si.tile_mode_array;
2403 const u32 num_tile_mode_states =
2404 ARRAY_SIZE(rdev->config.si.tile_mode_array);
2405 u32 reg_offset, split_equal_to_row_size;
2407 switch (rdev->config.si.mem_row_size_in_kb) {
2409 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2413 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2416 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2420 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2421 tile[reg_offset] = 0;
2423 switch(rdev->family) {
2426 /* non-AA compressed depth or any compressed stencil */
2427 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2428 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2429 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2430 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2431 NUM_BANKS(ADDR_SURF_16_BANK) |
2432 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2433 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2434 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2435 /* 2xAA/4xAA compressed depth only */
2436 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2437 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2438 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2439 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2440 NUM_BANKS(ADDR_SURF_16_BANK) |
2441 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2442 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2443 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2444 /* 8xAA compressed depth only */
2445 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2446 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2447 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2448 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2449 NUM_BANKS(ADDR_SURF_16_BANK) |
2450 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2451 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2452 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2453 /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2454 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2455 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2456 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2457 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2458 NUM_BANKS(ADDR_SURF_16_BANK) |
2459 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2460 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2461 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2462 /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2463 tile[4] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2464 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2465 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2466 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2467 NUM_BANKS(ADDR_SURF_16_BANK) |
2468 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2469 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2470 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2471 /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2472 tile[5] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2473 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2474 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2475 TILE_SPLIT(split_equal_to_row_size) |
2476 NUM_BANKS(ADDR_SURF_16_BANK) |
2477 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2478 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2479 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2480 /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2481 tile[6] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2482 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2483 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2484 TILE_SPLIT(split_equal_to_row_size) |
2485 NUM_BANKS(ADDR_SURF_16_BANK) |
2486 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2487 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2488 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2489 /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2490 tile[7] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2491 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2492 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2493 TILE_SPLIT(split_equal_to_row_size) |
2494 NUM_BANKS(ADDR_SURF_16_BANK) |
2495 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2496 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2497 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2498 /* 1D and 1D Array Surfaces */
2499 tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2500 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2501 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2502 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2503 NUM_BANKS(ADDR_SURF_16_BANK) |
2504 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2505 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2506 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2507 /* Displayable maps. */
2508 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2509 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2510 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2511 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2512 NUM_BANKS(ADDR_SURF_16_BANK) |
2513 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2514 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2515 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2517 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2518 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2519 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2520 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2521 NUM_BANKS(ADDR_SURF_16_BANK) |
2522 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2523 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2524 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2525 /* Display 16bpp. */
2526 tile[11] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2527 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2528 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2529 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2530 NUM_BANKS(ADDR_SURF_16_BANK) |
2531 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2532 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2533 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2534 /* Display 32bpp. */
2535 tile[12] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2536 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2537 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2538 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2539 NUM_BANKS(ADDR_SURF_16_BANK) |
2540 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2541 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2542 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2544 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2545 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2546 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2547 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2548 NUM_BANKS(ADDR_SURF_16_BANK) |
2549 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2550 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2551 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2553 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2554 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2555 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2556 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2557 NUM_BANKS(ADDR_SURF_16_BANK) |
2558 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2559 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2560 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2562 tile[15] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2563 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2564 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2565 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2566 NUM_BANKS(ADDR_SURF_16_BANK) |
2567 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2568 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2569 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2571 tile[16] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2572 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2573 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2574 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2575 NUM_BANKS(ADDR_SURF_16_BANK) |
2576 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2577 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2578 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2580 tile[17] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2581 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2582 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2583 TILE_SPLIT(split_equal_to_row_size) |
2584 NUM_BANKS(ADDR_SURF_16_BANK) |
2585 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2586 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2587 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2589 tile[21] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2590 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2591 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2592 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2593 NUM_BANKS(ADDR_SURF_16_BANK) |
2594 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2595 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2596 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2598 tile[22] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2599 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2600 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2601 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2602 NUM_BANKS(ADDR_SURF_16_BANK) |
2603 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2604 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2605 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2607 tile[23] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2608 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2609 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2610 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2611 NUM_BANKS(ADDR_SURF_16_BANK) |
2612 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2613 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2614 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2616 tile[24] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2617 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2618 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2619 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2620 NUM_BANKS(ADDR_SURF_16_BANK) |
2621 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2622 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2623 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2625 tile[25] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2626 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2627 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2628 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2629 NUM_BANKS(ADDR_SURF_8_BANK) |
2630 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2631 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2632 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2634 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2635 WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
2641 /* non-AA compressed depth or any compressed stencil */
2642 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2643 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2644 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2645 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2646 NUM_BANKS(ADDR_SURF_16_BANK) |
2647 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2648 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2649 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2650 /* 2xAA/4xAA compressed depth only */
2651 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2652 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2653 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2654 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2655 NUM_BANKS(ADDR_SURF_16_BANK) |
2656 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2657 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2658 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2659 /* 8xAA compressed depth only */
2660 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2661 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2662 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2663 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2664 NUM_BANKS(ADDR_SURF_16_BANK) |
2665 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2666 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2667 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2668 /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2669 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2670 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2671 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2672 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2673 NUM_BANKS(ADDR_SURF_16_BANK) |
2674 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2675 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2676 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2677 /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2678 tile[4] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2679 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2680 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2681 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2682 NUM_BANKS(ADDR_SURF_16_BANK) |
2683 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2684 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2685 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2686 /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2687 tile[5] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2688 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2689 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2690 TILE_SPLIT(split_equal_to_row_size) |
2691 NUM_BANKS(ADDR_SURF_16_BANK) |
2692 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2693 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2694 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2695 /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2696 tile[6] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2697 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2698 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2699 TILE_SPLIT(split_equal_to_row_size) |
2700 NUM_BANKS(ADDR_SURF_16_BANK) |
2701 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2702 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2703 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2704 /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2705 tile[7] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2706 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2707 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2708 TILE_SPLIT(split_equal_to_row_size) |
2709 NUM_BANKS(ADDR_SURF_16_BANK) |
2710 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2711 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2712 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2713 /* 1D and 1D Array Surfaces */
2714 tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2715 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2716 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2717 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2718 NUM_BANKS(ADDR_SURF_16_BANK) |
2719 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2720 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2721 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2722 /* Displayable maps. */
2723 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2724 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2725 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2726 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2727 NUM_BANKS(ADDR_SURF_16_BANK) |
2728 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2729 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2730 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2732 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2733 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2734 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2735 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2736 NUM_BANKS(ADDR_SURF_16_BANK) |
2737 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2738 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2739 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2740 /* Display 16bpp. */
2741 tile[11] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2742 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2743 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2744 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2745 NUM_BANKS(ADDR_SURF_16_BANK) |
2746 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2747 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2748 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2749 /* Display 32bpp. */
2750 tile[12] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2751 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2752 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2753 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2754 NUM_BANKS(ADDR_SURF_16_BANK) |
2755 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2756 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2757 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2759 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2760 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2761 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2762 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2763 NUM_BANKS(ADDR_SURF_16_BANK) |
2764 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2765 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2766 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2768 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2769 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2770 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2771 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2772 NUM_BANKS(ADDR_SURF_16_BANK) |
2773 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2774 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2775 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2777 tile[15] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2778 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2779 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2780 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2781 NUM_BANKS(ADDR_SURF_16_BANK) |
2782 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2783 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2784 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2786 tile[16] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2787 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2788 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2789 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2790 NUM_BANKS(ADDR_SURF_16_BANK) |
2791 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2792 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2793 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2795 tile[17] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2796 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2797 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2798 TILE_SPLIT(split_equal_to_row_size) |
2799 NUM_BANKS(ADDR_SURF_16_BANK) |
2800 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2801 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2802 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2804 tile[21] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2805 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2806 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2807 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2808 NUM_BANKS(ADDR_SURF_16_BANK) |
2809 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2810 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2811 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2813 tile[22] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2814 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2815 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2816 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2817 NUM_BANKS(ADDR_SURF_16_BANK) |
2818 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2819 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2820 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2822 tile[23] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2823 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2824 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2825 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2826 NUM_BANKS(ADDR_SURF_16_BANK) |
2827 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2828 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2829 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2831 tile[24] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2832 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2833 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2834 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2835 NUM_BANKS(ADDR_SURF_16_BANK) |
2836 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2837 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2838 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2840 tile[25] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2841 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2842 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2843 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2844 NUM_BANKS(ADDR_SURF_8_BANK) |
2845 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2846 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2847 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2849 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2850 WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
2854 DRM_ERROR("unknown asic: 0x%x\n", rdev->family);
2858 static void si_select_se_sh(struct radeon_device *rdev,
2859 u32 se_num, u32 sh_num)
2861 u32 data = INSTANCE_BROADCAST_WRITES;
2863 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
2864 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
2865 else if (se_num == 0xffffffff)
2866 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
2867 else if (sh_num == 0xffffffff)
2868 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
2870 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
2871 WREG32(GRBM_GFX_INDEX, data);
2874 static u32 si_create_bitmask(u32 bit_width)
2878 for (i = 0; i < bit_width; i++) {
2885 static u32 si_get_cu_enabled(struct radeon_device *rdev, u32 cu_per_sh)
2889 data = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
2891 data &= INACTIVE_CUS_MASK;
2894 data |= RREG32(GC_USER_SHADER_ARRAY_CONFIG);
2896 data >>= INACTIVE_CUS_SHIFT;
2898 mask = si_create_bitmask(cu_per_sh);
2900 return ~data & mask;
2903 static void si_setup_spi(struct radeon_device *rdev,
2904 u32 se_num, u32 sh_per_se,
2908 u32 data, mask, active_cu;
2910 for (i = 0; i < se_num; i++) {
2911 for (j = 0; j < sh_per_se; j++) {
2912 si_select_se_sh(rdev, i, j);
2913 data = RREG32(SPI_STATIC_THREAD_MGMT_3);
2914 active_cu = si_get_cu_enabled(rdev, cu_per_sh);
2917 for (k = 0; k < 16; k++) {
2919 if (active_cu & mask) {
2921 WREG32(SPI_STATIC_THREAD_MGMT_3, data);
2927 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2930 static u32 si_get_rb_disabled(struct radeon_device *rdev,
2931 u32 max_rb_num_per_se,
2936 data = RREG32(CC_RB_BACKEND_DISABLE);
2938 data &= BACKEND_DISABLE_MASK;
2941 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
2943 data >>= BACKEND_DISABLE_SHIFT;
2945 mask = si_create_bitmask(max_rb_num_per_se / sh_per_se);
2950 static void si_setup_rb(struct radeon_device *rdev,
2951 u32 se_num, u32 sh_per_se,
2952 u32 max_rb_num_per_se)
2956 u32 disabled_rbs = 0;
2957 u32 enabled_rbs = 0;
2959 for (i = 0; i < se_num; i++) {
2960 for (j = 0; j < sh_per_se; j++) {
2961 si_select_se_sh(rdev, i, j);
2962 data = si_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
2963 disabled_rbs |= data << ((i * sh_per_se + j) * TAHITI_RB_BITMAP_WIDTH_PER_SH);
2966 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2969 for (i = 0; i < max_rb_num_per_se * se_num; i++) {
2970 if (!(disabled_rbs & mask))
2971 enabled_rbs |= mask;
2975 rdev->config.si.backend_enable_mask = enabled_rbs;
2977 for (i = 0; i < se_num; i++) {
2978 si_select_se_sh(rdev, i, 0xffffffff);
2980 for (j = 0; j < sh_per_se; j++) {
2981 switch (enabled_rbs & 3) {
2983 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
2986 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
2990 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
2995 WREG32(PA_SC_RASTER_CONFIG, data);
2997 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3000 static void si_gpu_init(struct radeon_device *rdev)
3002 u32 gb_addr_config = 0;
3003 u32 mc_shared_chmap, mc_arb_ramcfg;
3005 u32 hdp_host_path_cntl;
3009 switch (rdev->family) {
3011 rdev->config.si.max_shader_engines = 2;
3012 rdev->config.si.max_tile_pipes = 12;
3013 rdev->config.si.max_cu_per_sh = 8;
3014 rdev->config.si.max_sh_per_se = 2;
3015 rdev->config.si.max_backends_per_se = 4;
3016 rdev->config.si.max_texture_channel_caches = 12;
3017 rdev->config.si.max_gprs = 256;
3018 rdev->config.si.max_gs_threads = 32;
3019 rdev->config.si.max_hw_contexts = 8;
3021 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3022 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
3023 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3024 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3025 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
3028 rdev->config.si.max_shader_engines = 2;
3029 rdev->config.si.max_tile_pipes = 8;
3030 rdev->config.si.max_cu_per_sh = 5;
3031 rdev->config.si.max_sh_per_se = 2;
3032 rdev->config.si.max_backends_per_se = 4;
3033 rdev->config.si.max_texture_channel_caches = 8;
3034 rdev->config.si.max_gprs = 256;
3035 rdev->config.si.max_gs_threads = 32;
3036 rdev->config.si.max_hw_contexts = 8;
3038 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3039 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
3040 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3041 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3042 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
3046 rdev->config.si.max_shader_engines = 1;
3047 rdev->config.si.max_tile_pipes = 4;
3048 rdev->config.si.max_cu_per_sh = 5;
3049 rdev->config.si.max_sh_per_se = 2;
3050 rdev->config.si.max_backends_per_se = 4;
3051 rdev->config.si.max_texture_channel_caches = 4;
3052 rdev->config.si.max_gprs = 256;
3053 rdev->config.si.max_gs_threads = 32;
3054 rdev->config.si.max_hw_contexts = 8;
3056 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3057 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
3058 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3059 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3060 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
3063 rdev->config.si.max_shader_engines = 1;
3064 rdev->config.si.max_tile_pipes = 4;
3065 rdev->config.si.max_cu_per_sh = 6;
3066 rdev->config.si.max_sh_per_se = 1;
3067 rdev->config.si.max_backends_per_se = 2;
3068 rdev->config.si.max_texture_channel_caches = 4;
3069 rdev->config.si.max_gprs = 256;
3070 rdev->config.si.max_gs_threads = 16;
3071 rdev->config.si.max_hw_contexts = 8;
3073 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3074 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
3075 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3076 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3077 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
3080 rdev->config.si.max_shader_engines = 1;
3081 rdev->config.si.max_tile_pipes = 4;
3082 rdev->config.si.max_cu_per_sh = 5;
3083 rdev->config.si.max_sh_per_se = 1;
3084 rdev->config.si.max_backends_per_se = 1;
3085 rdev->config.si.max_texture_channel_caches = 2;
3086 rdev->config.si.max_gprs = 256;
3087 rdev->config.si.max_gs_threads = 16;
3088 rdev->config.si.max_hw_contexts = 8;
3090 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3091 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
3092 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3093 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3094 gb_addr_config = HAINAN_GB_ADDR_CONFIG_GOLDEN;
3098 /* Initialize HDP */
3099 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3100 WREG32((0x2c14 + j), 0x00000000);
3101 WREG32((0x2c18 + j), 0x00000000);
3102 WREG32((0x2c1c + j), 0x00000000);
3103 WREG32((0x2c20 + j), 0x00000000);
3104 WREG32((0x2c24 + j), 0x00000000);
3107 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3108 WREG32(SRBM_INT_CNTL, 1);
3109 WREG32(SRBM_INT_ACK, 1);
3111 evergreen_fix_pci_max_read_req_size(rdev);
3113 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3115 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3116 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3118 rdev->config.si.num_tile_pipes = rdev->config.si.max_tile_pipes;
3119 rdev->config.si.mem_max_burst_length_bytes = 256;
3120 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3121 rdev->config.si.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3122 if (rdev->config.si.mem_row_size_in_kb > 4)
3123 rdev->config.si.mem_row_size_in_kb = 4;
3124 /* XXX use MC settings? */
3125 rdev->config.si.shader_engine_tile_size = 32;
3126 rdev->config.si.num_gpus = 1;
3127 rdev->config.si.multi_gpu_tile_size = 64;
3129 /* fix up row size */
3130 gb_addr_config &= ~ROW_SIZE_MASK;
3131 switch (rdev->config.si.mem_row_size_in_kb) {
3134 gb_addr_config |= ROW_SIZE(0);
3137 gb_addr_config |= ROW_SIZE(1);
3140 gb_addr_config |= ROW_SIZE(2);
3144 /* setup tiling info dword. gb_addr_config is not adequate since it does
3145 * not have bank info, so create a custom tiling dword.
3146 * bits 3:0 num_pipes
3147 * bits 7:4 num_banks
3148 * bits 11:8 group_size
3149 * bits 15:12 row_size
3151 rdev->config.si.tile_config = 0;
3152 switch (rdev->config.si.num_tile_pipes) {
3154 rdev->config.si.tile_config |= (0 << 0);
3157 rdev->config.si.tile_config |= (1 << 0);
3160 rdev->config.si.tile_config |= (2 << 0);
3164 /* XXX what about 12? */
3165 rdev->config.si.tile_config |= (3 << 0);
3168 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3169 case 0: /* four banks */
3170 rdev->config.si.tile_config |= 0 << 4;
3172 case 1: /* eight banks */
3173 rdev->config.si.tile_config |= 1 << 4;
3175 case 2: /* sixteen banks */
3177 rdev->config.si.tile_config |= 2 << 4;
3180 rdev->config.si.tile_config |=
3181 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3182 rdev->config.si.tile_config |=
3183 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3185 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3186 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3187 WREG32(DMIF_ADDR_CALC, gb_addr_config);
3188 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3189 WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
3190 WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
3191 if (rdev->has_uvd) {
3192 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3193 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3194 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3197 si_tiling_mode_table_init(rdev);
3199 si_setup_rb(rdev, rdev->config.si.max_shader_engines,
3200 rdev->config.si.max_sh_per_se,
3201 rdev->config.si.max_backends_per_se);
3203 si_setup_spi(rdev, rdev->config.si.max_shader_engines,
3204 rdev->config.si.max_sh_per_se,
3205 rdev->config.si.max_cu_per_sh);
3207 rdev->config.si.active_cus = 0;
3208 for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
3209 for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
3210 rdev->config.si.active_cus +=
3211 hweight32(si_get_cu_active_bitmap(rdev, i, j));
3215 /* set HW defaults for 3D engine */
3216 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3217 ROQ_IB2_START(0x2b)));
3218 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3220 sx_debug_1 = RREG32(SX_DEBUG_1);
3221 WREG32(SX_DEBUG_1, sx_debug_1);
3223 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3225 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_frontend) |
3226 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_backend) |
3227 SC_HIZ_TILE_FIFO_SIZE(rdev->config.si.sc_hiz_tile_fifo_size) |
3228 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.si.sc_earlyz_tile_fifo_size)));
3230 WREG32(VGT_NUM_INSTANCES, 1);
3232 WREG32(CP_PERFMON_CNTL, 0);
3234 WREG32(SQ_CONFIG, 0);
3236 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3237 FORCE_EOV_MAX_REZ_CNT(255)));
3239 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3240 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3242 WREG32(VGT_GS_VERTEX_REUSE, 16);
3243 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3245 WREG32(CB_PERFCOUNTER0_SELECT0, 0);
3246 WREG32(CB_PERFCOUNTER0_SELECT1, 0);
3247 WREG32(CB_PERFCOUNTER1_SELECT0, 0);
3248 WREG32(CB_PERFCOUNTER1_SELECT1, 0);
3249 WREG32(CB_PERFCOUNTER2_SELECT0, 0);
3250 WREG32(CB_PERFCOUNTER2_SELECT1, 0);
3251 WREG32(CB_PERFCOUNTER3_SELECT0, 0);
3252 WREG32(CB_PERFCOUNTER3_SELECT1, 0);
3254 tmp = RREG32(HDP_MISC_CNTL);
3255 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3256 WREG32(HDP_MISC_CNTL, tmp);
3258 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3259 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3261 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3267 * GPU scratch registers helpers function.
3269 static void si_scratch_init(struct radeon_device *rdev)
3273 rdev->scratch.num_reg = 7;
3274 rdev->scratch.reg_base = SCRATCH_REG0;
3275 for (i = 0; i < rdev->scratch.num_reg; i++) {
3276 rdev->scratch.free[i] = true;
3277 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3281 void si_fence_ring_emit(struct radeon_device *rdev,
3282 struct radeon_fence *fence)
3284 struct radeon_ring *ring = &rdev->ring[fence->ring];
3285 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3287 /* flush read cache over gart */
3288 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3289 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3290 radeon_ring_write(ring, 0);
3291 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3292 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3293 PACKET3_TC_ACTION_ENA |
3294 PACKET3_SH_KCACHE_ACTION_ENA |
3295 PACKET3_SH_ICACHE_ACTION_ENA);
3296 radeon_ring_write(ring, 0xFFFFFFFF);
3297 radeon_ring_write(ring, 0);
3298 radeon_ring_write(ring, 10); /* poll interval */
3299 /* EVENT_WRITE_EOP - flush caches, send int */
3300 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3301 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5));
3302 radeon_ring_write(ring, lower_32_bits(addr));
3303 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
3304 radeon_ring_write(ring, fence->seq);
3305 radeon_ring_write(ring, 0);
3311 void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3313 struct radeon_ring *ring = &rdev->ring[ib->ring];
3314 unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0;
3317 if (ib->is_const_ib) {
3318 /* set switch buffer packet before const IB */
3319 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3320 radeon_ring_write(ring, 0);
3322 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3325 if (ring->rptr_save_reg) {
3326 next_rptr = ring->wptr + 3 + 4 + 8;
3327 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3328 radeon_ring_write(ring, ((ring->rptr_save_reg -
3329 PACKET3_SET_CONFIG_REG_START) >> 2));
3330 radeon_ring_write(ring, next_rptr);
3331 } else if (rdev->wb.enabled) {
3332 next_rptr = ring->wptr + 5 + 4 + 8;
3333 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3334 radeon_ring_write(ring, (1 << 8));
3335 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3336 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr));
3337 radeon_ring_write(ring, next_rptr);
3340 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3343 radeon_ring_write(ring, header);
3344 radeon_ring_write(ring,
3348 (ib->gpu_addr & 0xFFFFFFFC));
3349 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3350 radeon_ring_write(ring, ib->length_dw | (vm_id << 24));
3352 if (!ib->is_const_ib) {
3353 /* flush read cache over gart for this vmid */
3354 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3355 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3356 radeon_ring_write(ring, vm_id);
3357 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3358 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3359 PACKET3_TC_ACTION_ENA |
3360 PACKET3_SH_KCACHE_ACTION_ENA |
3361 PACKET3_SH_ICACHE_ACTION_ENA);
3362 radeon_ring_write(ring, 0xFFFFFFFF);
3363 radeon_ring_write(ring, 0);
3364 radeon_ring_write(ring, 10); /* poll interval */
3371 static void si_cp_enable(struct radeon_device *rdev, bool enable)
3374 WREG32(CP_ME_CNTL, 0);
3376 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3377 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
3378 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3379 WREG32(SCRATCH_UMSK, 0);
3380 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3381 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3382 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3387 static int si_cp_load_microcode(struct radeon_device *rdev)
3391 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
3394 si_cp_enable(rdev, false);
3397 const struct gfx_firmware_header_v1_0 *pfp_hdr =
3398 (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
3399 const struct gfx_firmware_header_v1_0 *ce_hdr =
3400 (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
3401 const struct gfx_firmware_header_v1_0 *me_hdr =
3402 (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
3403 const __le32 *fw_data;
3406 radeon_ucode_print_gfx_hdr(&pfp_hdr->header);
3407 radeon_ucode_print_gfx_hdr(&ce_hdr->header);
3408 radeon_ucode_print_gfx_hdr(&me_hdr->header);
3411 fw_data = (const __le32 *)
3412 (rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
3413 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
3414 WREG32(CP_PFP_UCODE_ADDR, 0);
3415 for (i = 0; i < fw_size; i++)
3416 WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
3417 WREG32(CP_PFP_UCODE_ADDR, 0);
3420 fw_data = (const __le32 *)
3421 (rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
3422 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
3423 WREG32(CP_CE_UCODE_ADDR, 0);
3424 for (i = 0; i < fw_size; i++)
3425 WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
3426 WREG32(CP_CE_UCODE_ADDR, 0);
3429 fw_data = (const __be32 *)
3430 (rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
3431 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
3432 WREG32(CP_ME_RAM_WADDR, 0);
3433 for (i = 0; i < fw_size; i++)
3434 WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++));
3435 WREG32(CP_ME_RAM_WADDR, 0);
3437 const __be32 *fw_data;
3440 fw_data = (const __be32 *)rdev->pfp_fw->data;
3441 WREG32(CP_PFP_UCODE_ADDR, 0);
3442 for (i = 0; i < SI_PFP_UCODE_SIZE; i++)
3443 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3444 WREG32(CP_PFP_UCODE_ADDR, 0);
3447 fw_data = (const __be32 *)rdev->ce_fw->data;
3448 WREG32(CP_CE_UCODE_ADDR, 0);
3449 for (i = 0; i < SI_CE_UCODE_SIZE; i++)
3450 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3451 WREG32(CP_CE_UCODE_ADDR, 0);
3454 fw_data = (const __be32 *)rdev->me_fw->data;
3455 WREG32(CP_ME_RAM_WADDR, 0);
3456 for (i = 0; i < SI_PM4_UCODE_SIZE; i++)
3457 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3458 WREG32(CP_ME_RAM_WADDR, 0);
3461 WREG32(CP_PFP_UCODE_ADDR, 0);
3462 WREG32(CP_CE_UCODE_ADDR, 0);
3463 WREG32(CP_ME_RAM_WADDR, 0);
3464 WREG32(CP_ME_RAM_RADDR, 0);
3468 static int si_cp_start(struct radeon_device *rdev)
3470 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3473 r = radeon_ring_lock(rdev, ring, 7 + 4);
3475 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3479 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3480 radeon_ring_write(ring, 0x1);
3481 radeon_ring_write(ring, 0x0);
3482 radeon_ring_write(ring, rdev->config.si.max_hw_contexts - 1);
3483 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3484 radeon_ring_write(ring, 0);
3485 radeon_ring_write(ring, 0);
3487 /* init the CE partitions */
3488 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3489 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3490 radeon_ring_write(ring, 0xc000);
3491 radeon_ring_write(ring, 0xe000);
3492 radeon_ring_unlock_commit(rdev, ring, false);
3494 si_cp_enable(rdev, true);
3496 r = radeon_ring_lock(rdev, ring, si_default_size + 10);
3498 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3502 /* setup clear context state */
3503 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3504 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3506 for (i = 0; i < si_default_size; i++)
3507 radeon_ring_write(ring, si_default_state[i]);
3509 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3510 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3512 /* set clear context state */
3513 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3514 radeon_ring_write(ring, 0);
3516 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3517 radeon_ring_write(ring, 0x00000316);
3518 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3519 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3521 radeon_ring_unlock_commit(rdev, ring, false);
3523 for (i = RADEON_RING_TYPE_GFX_INDEX; i <= CAYMAN_RING_TYPE_CP2_INDEX; ++i) {
3524 ring = &rdev->ring[i];
3525 r = radeon_ring_lock(rdev, ring, 2);
3527 /* clear the compute context state */
3528 radeon_ring_write(ring, PACKET3_COMPUTE(PACKET3_CLEAR_STATE, 0));
3529 radeon_ring_write(ring, 0);
3531 radeon_ring_unlock_commit(rdev, ring, false);
3537 static void si_cp_fini(struct radeon_device *rdev)
3539 struct radeon_ring *ring;
3540 si_cp_enable(rdev, false);
3542 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3543 radeon_ring_fini(rdev, ring);
3544 radeon_scratch_free(rdev, ring->rptr_save_reg);
3546 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3547 radeon_ring_fini(rdev, ring);
3548 radeon_scratch_free(rdev, ring->rptr_save_reg);
3550 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3551 radeon_ring_fini(rdev, ring);
3552 radeon_scratch_free(rdev, ring->rptr_save_reg);
3555 static int si_cp_resume(struct radeon_device *rdev)
3557 struct radeon_ring *ring;
3562 si_enable_gui_idle_interrupt(rdev, false);
3564 WREG32(CP_SEM_WAIT_TIMER, 0x0);
3565 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3567 /* Set the write pointer delay */
3568 WREG32(CP_RB_WPTR_DELAY, 0);
3570 WREG32(CP_DEBUG, 0);
3571 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3573 /* ring 0 - compute and gfx */
3574 /* Set ring buffer size */
3575 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3576 rb_bufsz = order_base_2(ring->ring_size / 8);
3577 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3579 tmp |= BUF_SWAP_32BIT;
3581 WREG32(CP_RB0_CNTL, tmp);
3583 /* Initialize the ring buffer's read and write pointers */
3584 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3586 WREG32(CP_RB0_WPTR, ring->wptr);
3588 /* set the wb address whether it's enabled or not */
3589 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3590 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3592 if (rdev->wb.enabled)
3593 WREG32(SCRATCH_UMSK, 0xff);
3595 tmp |= RB_NO_UPDATE;
3596 WREG32(SCRATCH_UMSK, 0);
3600 WREG32(CP_RB0_CNTL, tmp);
3602 WREG32(CP_RB0_BASE, ring->gpu_addr >> 8);
3604 /* ring1 - compute only */
3605 /* Set ring buffer size */
3606 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3607 rb_bufsz = order_base_2(ring->ring_size / 8);
3608 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3610 tmp |= BUF_SWAP_32BIT;
3612 WREG32(CP_RB1_CNTL, tmp);
3614 /* Initialize the ring buffer's read and write pointers */
3615 WREG32(CP_RB1_CNTL, tmp | RB_RPTR_WR_ENA);
3617 WREG32(CP_RB1_WPTR, ring->wptr);
3619 /* set the wb address whether it's enabled or not */
3620 WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC);
3621 WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFF);
3624 WREG32(CP_RB1_CNTL, tmp);
3626 WREG32(CP_RB1_BASE, ring->gpu_addr >> 8);
3628 /* ring2 - compute only */
3629 /* Set ring buffer size */
3630 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3631 rb_bufsz = order_base_2(ring->ring_size / 8);
3632 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3634 tmp |= BUF_SWAP_32BIT;
3636 WREG32(CP_RB2_CNTL, tmp);
3638 /* Initialize the ring buffer's read and write pointers */
3639 WREG32(CP_RB2_CNTL, tmp | RB_RPTR_WR_ENA);
3641 WREG32(CP_RB2_WPTR, ring->wptr);
3643 /* set the wb address whether it's enabled or not */
3644 WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC);
3645 WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFF);
3648 WREG32(CP_RB2_CNTL, tmp);
3650 WREG32(CP_RB2_BASE, ring->gpu_addr >> 8);
3652 /* start the rings */
3654 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
3655 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = true;
3656 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = true;
3657 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3659 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3660 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3661 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3664 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP1_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
3666 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3668 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP2_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
3670 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3673 si_enable_gui_idle_interrupt(rdev, true);
3675 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3676 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
3681 u32 si_gpu_check_soft_reset(struct radeon_device *rdev)
3687 tmp = RREG32(GRBM_STATUS);
3688 if (tmp & (PA_BUSY | SC_BUSY |
3689 BCI_BUSY | SX_BUSY |
3690 TA_BUSY | VGT_BUSY |
3692 GDS_BUSY | SPI_BUSY |
3693 IA_BUSY | IA_BUSY_NO_DMA))
3694 reset_mask |= RADEON_RESET_GFX;
3696 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3697 CP_BUSY | CP_COHERENCY_BUSY))
3698 reset_mask |= RADEON_RESET_CP;
3700 if (tmp & GRBM_EE_BUSY)
3701 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3704 tmp = RREG32(GRBM_STATUS2);
3705 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3706 reset_mask |= RADEON_RESET_RLC;
3708 /* DMA_STATUS_REG 0 */
3709 tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
3710 if (!(tmp & DMA_IDLE))
3711 reset_mask |= RADEON_RESET_DMA;
3713 /* DMA_STATUS_REG 1 */
3714 tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
3715 if (!(tmp & DMA_IDLE))
3716 reset_mask |= RADEON_RESET_DMA1;
3719 tmp = RREG32(SRBM_STATUS2);
3721 reset_mask |= RADEON_RESET_DMA;
3723 if (tmp & DMA1_BUSY)
3724 reset_mask |= RADEON_RESET_DMA1;
3727 tmp = RREG32(SRBM_STATUS);
3730 reset_mask |= RADEON_RESET_IH;
3733 reset_mask |= RADEON_RESET_SEM;
3735 if (tmp & GRBM_RQ_PENDING)
3736 reset_mask |= RADEON_RESET_GRBM;
3739 reset_mask |= RADEON_RESET_VMC;
3741 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3742 MCC_BUSY | MCD_BUSY))
3743 reset_mask |= RADEON_RESET_MC;
3745 if (evergreen_is_display_hung(rdev))
3746 reset_mask |= RADEON_RESET_DISPLAY;
3749 tmp = RREG32(VM_L2_STATUS);
3751 reset_mask |= RADEON_RESET_VMC;
3753 /* Skip MC reset as it's mostly likely not hung, just busy */
3754 if (reset_mask & RADEON_RESET_MC) {
3755 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3756 reset_mask &= ~RADEON_RESET_MC;
3762 static void si_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3764 struct evergreen_mc_save save;
3765 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3768 if (reset_mask == 0)
3771 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3773 evergreen_print_gpu_status_regs(rdev);
3774 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
3775 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
3776 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3777 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
3786 /* Disable CP parsing/prefetching */
3787 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
3789 if (reset_mask & RADEON_RESET_DMA) {
3791 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
3792 tmp &= ~DMA_RB_ENABLE;
3793 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
3795 if (reset_mask & RADEON_RESET_DMA1) {
3797 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
3798 tmp &= ~DMA_RB_ENABLE;
3799 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
3804 evergreen_mc_stop(rdev, &save);
3805 if (evergreen_mc_wait_for_idle(rdev)) {
3806 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3809 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP)) {
3810 grbm_soft_reset = SOFT_RESET_CB |
3824 if (reset_mask & RADEON_RESET_CP) {
3825 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
3827 srbm_soft_reset |= SOFT_RESET_GRBM;
3830 if (reset_mask & RADEON_RESET_DMA)
3831 srbm_soft_reset |= SOFT_RESET_DMA;
3833 if (reset_mask & RADEON_RESET_DMA1)
3834 srbm_soft_reset |= SOFT_RESET_DMA1;
3836 if (reset_mask & RADEON_RESET_DISPLAY)
3837 srbm_soft_reset |= SOFT_RESET_DC;
3839 if (reset_mask & RADEON_RESET_RLC)
3840 grbm_soft_reset |= SOFT_RESET_RLC;
3842 if (reset_mask & RADEON_RESET_SEM)
3843 srbm_soft_reset |= SOFT_RESET_SEM;
3845 if (reset_mask & RADEON_RESET_IH)
3846 srbm_soft_reset |= SOFT_RESET_IH;
3848 if (reset_mask & RADEON_RESET_GRBM)
3849 srbm_soft_reset |= SOFT_RESET_GRBM;
3851 if (reset_mask & RADEON_RESET_VMC)
3852 srbm_soft_reset |= SOFT_RESET_VMC;
3854 if (reset_mask & RADEON_RESET_MC)
3855 srbm_soft_reset |= SOFT_RESET_MC;
3857 if (grbm_soft_reset) {
3858 tmp = RREG32(GRBM_SOFT_RESET);
3859 tmp |= grbm_soft_reset;
3860 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3861 WREG32(GRBM_SOFT_RESET, tmp);
3862 tmp = RREG32(GRBM_SOFT_RESET);
3866 tmp &= ~grbm_soft_reset;
3867 WREG32(GRBM_SOFT_RESET, tmp);
3868 tmp = RREG32(GRBM_SOFT_RESET);
3871 if (srbm_soft_reset) {
3872 tmp = RREG32(SRBM_SOFT_RESET);
3873 tmp |= srbm_soft_reset;
3874 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3875 WREG32(SRBM_SOFT_RESET, tmp);
3876 tmp = RREG32(SRBM_SOFT_RESET);
3880 tmp &= ~srbm_soft_reset;
3881 WREG32(SRBM_SOFT_RESET, tmp);
3882 tmp = RREG32(SRBM_SOFT_RESET);
3885 /* Wait a little for things to settle down */
3888 evergreen_mc_resume(rdev, &save);
3891 evergreen_print_gpu_status_regs(rdev);
3894 static void si_set_clk_bypass_mode(struct radeon_device *rdev)
3898 tmp = RREG32(CG_SPLL_FUNC_CNTL);
3899 tmp |= SPLL_BYPASS_EN;
3900 WREG32(CG_SPLL_FUNC_CNTL, tmp);
3902 tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
3903 tmp |= SPLL_CTLREQ_CHG;
3904 WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
3906 for (i = 0; i < rdev->usec_timeout; i++) {
3907 if (RREG32(SPLL_STATUS) & SPLL_CHG_STATUS)
3912 tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
3913 tmp &= ~(SPLL_CTLREQ_CHG | SCLK_MUX_UPDATE);
3914 WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
3916 tmp = RREG32(MPLL_CNTL_MODE);
3917 tmp &= ~MPLL_MCLK_SEL;
3918 WREG32(MPLL_CNTL_MODE, tmp);
3921 static void si_spll_powerdown(struct radeon_device *rdev)
3925 tmp = RREG32(SPLL_CNTL_MODE);
3926 tmp |= SPLL_SW_DIR_CONTROL;
3927 WREG32(SPLL_CNTL_MODE, tmp);
3929 tmp = RREG32(CG_SPLL_FUNC_CNTL);
3931 WREG32(CG_SPLL_FUNC_CNTL, tmp);
3933 tmp = RREG32(CG_SPLL_FUNC_CNTL);
3935 WREG32(CG_SPLL_FUNC_CNTL, tmp);
3937 tmp = RREG32(SPLL_CNTL_MODE);
3938 tmp &= ~SPLL_SW_DIR_CONTROL;
3939 WREG32(SPLL_CNTL_MODE, tmp);
3942 static void si_gpu_pci_config_reset(struct radeon_device *rdev)
3944 struct evergreen_mc_save save;
3947 dev_info(rdev->dev, "GPU pci config reset\n");
3955 /* Disable CP parsing/prefetching */
3956 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
3958 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
3959 tmp &= ~DMA_RB_ENABLE;
3960 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
3962 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
3963 tmp &= ~DMA_RB_ENABLE;
3964 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
3965 /* XXX other engines? */
3967 /* halt the rlc, disable cp internal ints */
3972 /* disable mem access */
3973 evergreen_mc_stop(rdev, &save);
3974 if (evergreen_mc_wait_for_idle(rdev)) {
3975 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
3978 /* set mclk/sclk to bypass */
3979 si_set_clk_bypass_mode(rdev);
3980 /* powerdown spll */
3981 si_spll_powerdown(rdev);
3983 pci_clear_master(rdev->pdev);
3985 radeon_pci_config_reset(rdev);
3986 /* wait for asic to come out of reset */
3987 for (i = 0; i < rdev->usec_timeout; i++) {
3988 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
3994 int si_asic_reset(struct radeon_device *rdev, bool hard)
3999 si_gpu_pci_config_reset(rdev);
4003 reset_mask = si_gpu_check_soft_reset(rdev);
4006 r600_set_bios_scratch_engine_hung(rdev, true);
4008 /* try soft reset */
4009 si_gpu_soft_reset(rdev, reset_mask);
4011 reset_mask = si_gpu_check_soft_reset(rdev);
4013 /* try pci config reset */
4014 if (reset_mask && radeon_hard_reset)
4015 si_gpu_pci_config_reset(rdev);
4017 reset_mask = si_gpu_check_soft_reset(rdev);
4020 r600_set_bios_scratch_engine_hung(rdev, false);
4026 * si_gfx_is_lockup - Check if the GFX engine is locked up
4028 * @rdev: radeon_device pointer
4029 * @ring: radeon_ring structure holding ring information
4031 * Check if the GFX engine is locked up.
4032 * Returns true if the engine appears to be locked up, false if not.
4034 bool si_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
4036 u32 reset_mask = si_gpu_check_soft_reset(rdev);
4038 if (!(reset_mask & (RADEON_RESET_GFX |
4039 RADEON_RESET_COMPUTE |
4040 RADEON_RESET_CP))) {
4041 radeon_ring_lockup_update(rdev, ring);
4044 return radeon_ring_test_lockup(rdev, ring);
4048 static void si_mc_program(struct radeon_device *rdev)
4050 struct evergreen_mc_save save;
4054 /* Initialize HDP */
4055 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
4056 WREG32((0x2c14 + j), 0x00000000);
4057 WREG32((0x2c18 + j), 0x00000000);
4058 WREG32((0x2c1c + j), 0x00000000);
4059 WREG32((0x2c20 + j), 0x00000000);
4060 WREG32((0x2c24 + j), 0x00000000);
4062 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
4064 evergreen_mc_stop(rdev, &save);
4065 if (radeon_mc_wait_for_idle(rdev)) {
4066 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4068 if (!ASIC_IS_NODCE(rdev))
4069 /* Lockout access through VGA aperture*/
4070 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
4071 /* Update configuration */
4072 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
4073 rdev->mc.vram_start >> 12);
4074 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
4075 rdev->mc.vram_end >> 12);
4076 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
4077 rdev->vram_scratch.gpu_addr >> 12);
4078 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
4079 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
4080 WREG32(MC_VM_FB_LOCATION, tmp);
4081 /* XXX double check these! */
4082 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
4083 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
4084 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
4085 WREG32(MC_VM_AGP_BASE, 0);
4086 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
4087 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
4088 if (radeon_mc_wait_for_idle(rdev)) {
4089 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4091 evergreen_mc_resume(rdev, &save);
4092 if (!ASIC_IS_NODCE(rdev)) {
4093 /* we need to own VRAM, so turn off the VGA renderer here
4094 * to stop it overwriting our objects */
4095 rv515_vga_render_disable(rdev);
4099 void si_vram_gtt_location(struct radeon_device *rdev,
4100 struct radeon_mc *mc)
4102 if (mc->mc_vram_size > 0xFFC0000000ULL) {
4103 /* leave room for at least 1024M GTT */
4104 dev_warn(rdev->dev, "limiting VRAM\n");
4105 mc->real_vram_size = 0xFFC0000000ULL;
4106 mc->mc_vram_size = 0xFFC0000000ULL;
4108 radeon_vram_location(rdev, &rdev->mc, 0);
4109 rdev->mc.gtt_base_align = 0;
4110 radeon_gtt_location(rdev, mc);
4113 static int si_mc_init(struct radeon_device *rdev)
4116 int chansize, numchan;
4118 /* Get VRAM informations */
4119 rdev->mc.vram_is_ddr = true;
4120 tmp = RREG32(MC_ARB_RAMCFG);
4121 if (tmp & CHANSIZE_OVERRIDE) {
4123 } else if (tmp & CHANSIZE_MASK) {
4128 tmp = RREG32(MC_SHARED_CHMAP);
4129 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
4159 rdev->mc.vram_width = numchan * chansize;
4160 /* Could aper size report 0 ? */
4161 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
4162 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
4163 /* size in MB on si */
4164 tmp = RREG32(CONFIG_MEMSIZE);
4165 /* some boards may have garbage in the upper 16 bits */
4166 if (tmp & 0xffff0000) {
4167 DRM_INFO("Probable bad vram size: 0x%08x\n", tmp);
4171 rdev->mc.mc_vram_size = tmp * 1024ULL * 1024ULL;
4172 rdev->mc.real_vram_size = rdev->mc.mc_vram_size;
4173 rdev->mc.visible_vram_size = rdev->mc.aper_size;
4174 si_vram_gtt_location(rdev, &rdev->mc);
4175 radeon_update_bandwidth_info(rdev);
4183 void si_pcie_gart_tlb_flush(struct radeon_device *rdev)
4185 /* flush hdp cache */
4186 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
4188 /* bits 0-15 are the VM contexts0-15 */
4189 WREG32(VM_INVALIDATE_REQUEST, 1);
4192 static int si_pcie_gart_enable(struct radeon_device *rdev)
4196 if (rdev->gart.robj == NULL) {
4197 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
4200 r = radeon_gart_table_vram_pin(rdev);
4203 /* Setup TLB control */
4204 WREG32(MC_VM_MX_L1_TLB_CNTL,
4207 ENABLE_L1_FRAGMENT_PROCESSING |
4208 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4209 ENABLE_ADVANCED_DRIVER_MODEL |
4210 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4211 /* Setup L2 cache */
4212 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
4213 ENABLE_L2_FRAGMENT_PROCESSING |
4214 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4215 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4216 EFFECTIVE_L2_QUEUE_SIZE(7) |
4217 CONTEXT1_IDENTITY_ACCESS_MODE(1));
4218 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
4219 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4221 L2_CACHE_BIGK_FRAGMENT_SIZE(4));
4222 /* setup context0 */
4223 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
4224 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
4225 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
4226 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
4227 (u32)(rdev->dummy_page.addr >> 12));
4228 WREG32(VM_CONTEXT0_CNTL2, 0);
4229 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
4230 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
4236 /* empty context1-15 */
4237 /* set vm size, must be a multiple of 4 */
4238 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
4239 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn - 1);
4240 /* Assign the pt base to something valid for now; the pts used for
4241 * the VMs are determined by the application and setup and assigned
4242 * on the fly in the vm part of radeon_gart.c
4244 for (i = 1; i < 16; i++) {
4246 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
4247 rdev->vm_manager.saved_table_addr[i]);
4249 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
4250 rdev->vm_manager.saved_table_addr[i]);
4253 /* enable context1-15 */
4254 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
4255 (u32)(rdev->dummy_page.addr >> 12));
4256 WREG32(VM_CONTEXT1_CNTL2, 4);
4257 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
4258 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
4259 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4260 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4261 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4262 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4263 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
4264 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
4265 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
4266 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
4267 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
4268 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
4269 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4270 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
4272 si_pcie_gart_tlb_flush(rdev);
4273 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
4274 (unsigned)(rdev->mc.gtt_size >> 20),
4275 (unsigned long long)rdev->gart.table_addr);
4276 rdev->gart.ready = true;
4280 static void si_pcie_gart_disable(struct radeon_device *rdev)
4284 for (i = 1; i < 16; ++i) {
4287 reg = VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2);
4289 reg = VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2);
4290 rdev->vm_manager.saved_table_addr[i] = RREG32(reg);
4293 /* Disable all tables */
4294 WREG32(VM_CONTEXT0_CNTL, 0);
4295 WREG32(VM_CONTEXT1_CNTL, 0);
4296 /* Setup TLB control */
4297 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4298 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4299 /* Setup L2 cache */
4300 WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4301 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4302 EFFECTIVE_L2_QUEUE_SIZE(7) |
4303 CONTEXT1_IDENTITY_ACCESS_MODE(1));
4304 WREG32(VM_L2_CNTL2, 0);
4305 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4306 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
4307 radeon_gart_table_vram_unpin(rdev);
4310 static void si_pcie_gart_fini(struct radeon_device *rdev)
4312 si_pcie_gart_disable(rdev);
4313 radeon_gart_table_vram_free(rdev);
4314 radeon_gart_fini(rdev);
4318 static bool si_vm_reg_valid(u32 reg)
4320 /* context regs are fine */
4324 /* shader regs are also fine */
4325 if (reg >= 0xB000 && reg < 0xC000)
4328 /* check config regs */
4330 case GRBM_GFX_INDEX:
4331 case CP_STRMOUT_CNTL:
4332 case VGT_VTX_VECT_EJECT_REG:
4333 case VGT_CACHE_INVALIDATION:
4334 case VGT_ESGS_RING_SIZE:
4335 case VGT_GSVS_RING_SIZE:
4336 case VGT_GS_VERTEX_REUSE:
4337 case VGT_PRIMITIVE_TYPE:
4338 case VGT_INDEX_TYPE:
4339 case VGT_NUM_INDICES:
4340 case VGT_NUM_INSTANCES:
4341 case VGT_TF_RING_SIZE:
4342 case VGT_HS_OFFCHIP_PARAM:
4343 case VGT_TF_MEMORY_BASE:
4345 case PA_SU_LINE_STIPPLE_VALUE:
4346 case PA_SC_LINE_STIPPLE_STATE:
4349 case SPI_STATIC_THREAD_MGMT_1:
4350 case SPI_STATIC_THREAD_MGMT_2:
4351 case SPI_STATIC_THREAD_MGMT_3:
4352 case SPI_PS_MAX_WAVE_ID:
4353 case SPI_CONFIG_CNTL:
4354 case SPI_CONFIG_CNTL_1:
4356 case TA_CS_BC_BASE_ADDR:
4359 DRM_ERROR("Invalid register 0x%x in CS\n", reg);
4364 static int si_vm_packet3_ce_check(struct radeon_device *rdev,
4365 u32 *ib, struct radeon_cs_packet *pkt)
4367 switch (pkt->opcode) {
4369 case PACKET3_SET_BASE:
4370 case PACKET3_SET_CE_DE_COUNTERS:
4371 case PACKET3_LOAD_CONST_RAM:
4372 case PACKET3_WRITE_CONST_RAM:
4373 case PACKET3_WRITE_CONST_RAM_OFFSET:
4374 case PACKET3_DUMP_CONST_RAM:
4375 case PACKET3_INCREMENT_CE_COUNTER:
4376 case PACKET3_WAIT_ON_DE_COUNTER:
4377 case PACKET3_CE_WRITE:
4380 DRM_ERROR("Invalid CE packet3: 0x%x\n", pkt->opcode);
4386 static int si_vm_packet3_cp_dma_check(u32 *ib, u32 idx)
4388 u32 start_reg, reg, i;
4389 u32 command = ib[idx + 4];
4390 u32 info = ib[idx + 1];
4391 u32 idx_value = ib[idx];
4392 if (command & PACKET3_CP_DMA_CMD_SAS) {
4393 /* src address space is register */
4394 if (((info & 0x60000000) >> 29) == 0) {
4395 start_reg = idx_value << 2;
4396 if (command & PACKET3_CP_DMA_CMD_SAIC) {
4398 if (!si_vm_reg_valid(reg)) {
4399 DRM_ERROR("CP DMA Bad SRC register\n");
4403 for (i = 0; i < (command & 0x1fffff); i++) {
4404 reg = start_reg + (4 * i);
4405 if (!si_vm_reg_valid(reg)) {
4406 DRM_ERROR("CP DMA Bad SRC register\n");
4413 if (command & PACKET3_CP_DMA_CMD_DAS) {
4414 /* dst address space is register */
4415 if (((info & 0x00300000) >> 20) == 0) {
4416 start_reg = ib[idx + 2];
4417 if (command & PACKET3_CP_DMA_CMD_DAIC) {
4419 if (!si_vm_reg_valid(reg)) {
4420 DRM_ERROR("CP DMA Bad DST register\n");
4424 for (i = 0; i < (command & 0x1fffff); i++) {
4425 reg = start_reg + (4 * i);
4426 if (!si_vm_reg_valid(reg)) {
4427 DRM_ERROR("CP DMA Bad DST register\n");
4437 static int si_vm_packet3_gfx_check(struct radeon_device *rdev,
4438 u32 *ib, struct radeon_cs_packet *pkt)
4441 u32 idx = pkt->idx + 1;
4442 u32 idx_value = ib[idx];
4443 u32 start_reg, end_reg, reg, i;
4445 switch (pkt->opcode) {
4447 case PACKET3_SET_BASE:
4448 case PACKET3_CLEAR_STATE:
4449 case PACKET3_INDEX_BUFFER_SIZE:
4450 case PACKET3_DISPATCH_DIRECT:
4451 case PACKET3_DISPATCH_INDIRECT:
4452 case PACKET3_ALLOC_GDS:
4453 case PACKET3_WRITE_GDS_RAM:
4454 case PACKET3_ATOMIC_GDS:
4455 case PACKET3_ATOMIC:
4456 case PACKET3_OCCLUSION_QUERY:
4457 case PACKET3_SET_PREDICATION:
4458 case PACKET3_COND_EXEC:
4459 case PACKET3_PRED_EXEC:
4460 case PACKET3_DRAW_INDIRECT:
4461 case PACKET3_DRAW_INDEX_INDIRECT:
4462 case PACKET3_INDEX_BASE:
4463 case PACKET3_DRAW_INDEX_2:
4464 case PACKET3_CONTEXT_CONTROL:
4465 case PACKET3_INDEX_TYPE:
4466 case PACKET3_DRAW_INDIRECT_MULTI:
4467 case PACKET3_DRAW_INDEX_AUTO:
4468 case PACKET3_DRAW_INDEX_IMMD:
4469 case PACKET3_NUM_INSTANCES:
4470 case PACKET3_DRAW_INDEX_MULTI_AUTO:
4471 case PACKET3_STRMOUT_BUFFER_UPDATE:
4472 case PACKET3_DRAW_INDEX_OFFSET_2:
4473 case PACKET3_DRAW_INDEX_MULTI_ELEMENT:
4474 case PACKET3_DRAW_INDEX_INDIRECT_MULTI:
4475 case PACKET3_MPEG_INDEX:
4476 case PACKET3_WAIT_REG_MEM:
4477 case PACKET3_MEM_WRITE:
4478 case PACKET3_PFP_SYNC_ME:
4479 case PACKET3_SURFACE_SYNC:
4480 case PACKET3_EVENT_WRITE:
4481 case PACKET3_EVENT_WRITE_EOP:
4482 case PACKET3_EVENT_WRITE_EOS:
4483 case PACKET3_SET_CONTEXT_REG:
4484 case PACKET3_SET_CONTEXT_REG_INDIRECT:
4485 case PACKET3_SET_SH_REG:
4486 case PACKET3_SET_SH_REG_OFFSET:
4487 case PACKET3_INCREMENT_DE_COUNTER:
4488 case PACKET3_WAIT_ON_CE_COUNTER:
4489 case PACKET3_WAIT_ON_AVAIL_BUFFER:
4490 case PACKET3_ME_WRITE:
4492 case PACKET3_COPY_DATA:
4493 if ((idx_value & 0xf00) == 0) {
4494 reg = ib[idx + 3] * 4;
4495 if (!si_vm_reg_valid(reg))
4499 case PACKET3_WRITE_DATA:
4500 if ((idx_value & 0xf00) == 0) {
4501 start_reg = ib[idx + 1] * 4;
4502 if (idx_value & 0x10000) {
4503 if (!si_vm_reg_valid(start_reg))
4506 for (i = 0; i < (pkt->count - 2); i++) {
4507 reg = start_reg + (4 * i);
4508 if (!si_vm_reg_valid(reg))
4514 case PACKET3_COND_WRITE:
4515 if (idx_value & 0x100) {
4516 reg = ib[idx + 5] * 4;
4517 if (!si_vm_reg_valid(reg))
4521 case PACKET3_COPY_DW:
4522 if (idx_value & 0x2) {
4523 reg = ib[idx + 3] * 4;
4524 if (!si_vm_reg_valid(reg))
4528 case PACKET3_SET_CONFIG_REG:
4529 start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START;
4530 end_reg = 4 * pkt->count + start_reg - 4;
4531 if ((start_reg < PACKET3_SET_CONFIG_REG_START) ||
4532 (start_reg >= PACKET3_SET_CONFIG_REG_END) ||
4533 (end_reg >= PACKET3_SET_CONFIG_REG_END)) {
4534 DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n");
4537 for (i = 0; i < pkt->count; i++) {
4538 reg = start_reg + (4 * i);
4539 if (!si_vm_reg_valid(reg))
4543 case PACKET3_CP_DMA:
4544 r = si_vm_packet3_cp_dma_check(ib, idx);
4549 DRM_ERROR("Invalid GFX packet3: 0x%x\n", pkt->opcode);
4555 static int si_vm_packet3_compute_check(struct radeon_device *rdev,
4556 u32 *ib, struct radeon_cs_packet *pkt)
4559 u32 idx = pkt->idx + 1;
4560 u32 idx_value = ib[idx];
4561 u32 start_reg, reg, i;
4563 switch (pkt->opcode) {
4565 case PACKET3_SET_BASE:
4566 case PACKET3_CLEAR_STATE:
4567 case PACKET3_DISPATCH_DIRECT:
4568 case PACKET3_DISPATCH_INDIRECT:
4569 case PACKET3_ALLOC_GDS:
4570 case PACKET3_WRITE_GDS_RAM:
4571 case PACKET3_ATOMIC_GDS:
4572 case PACKET3_ATOMIC:
4573 case PACKET3_OCCLUSION_QUERY:
4574 case PACKET3_SET_PREDICATION:
4575 case PACKET3_COND_EXEC:
4576 case PACKET3_PRED_EXEC:
4577 case PACKET3_CONTEXT_CONTROL:
4578 case PACKET3_STRMOUT_BUFFER_UPDATE:
4579 case PACKET3_WAIT_REG_MEM:
4580 case PACKET3_MEM_WRITE:
4581 case PACKET3_PFP_SYNC_ME:
4582 case PACKET3_SURFACE_SYNC:
4583 case PACKET3_EVENT_WRITE:
4584 case PACKET3_EVENT_WRITE_EOP:
4585 case PACKET3_EVENT_WRITE_EOS:
4586 case PACKET3_SET_CONTEXT_REG:
4587 case PACKET3_SET_CONTEXT_REG_INDIRECT:
4588 case PACKET3_SET_SH_REG:
4589 case PACKET3_SET_SH_REG_OFFSET:
4590 case PACKET3_INCREMENT_DE_COUNTER:
4591 case PACKET3_WAIT_ON_CE_COUNTER:
4592 case PACKET3_WAIT_ON_AVAIL_BUFFER:
4593 case PACKET3_ME_WRITE:
4595 case PACKET3_COPY_DATA:
4596 if ((idx_value & 0xf00) == 0) {
4597 reg = ib[idx + 3] * 4;
4598 if (!si_vm_reg_valid(reg))
4602 case PACKET3_WRITE_DATA:
4603 if ((idx_value & 0xf00) == 0) {
4604 start_reg = ib[idx + 1] * 4;
4605 if (idx_value & 0x10000) {
4606 if (!si_vm_reg_valid(start_reg))
4609 for (i = 0; i < (pkt->count - 2); i++) {
4610 reg = start_reg + (4 * i);
4611 if (!si_vm_reg_valid(reg))
4617 case PACKET3_COND_WRITE:
4618 if (idx_value & 0x100) {
4619 reg = ib[idx + 5] * 4;
4620 if (!si_vm_reg_valid(reg))
4624 case PACKET3_COPY_DW:
4625 if (idx_value & 0x2) {
4626 reg = ib[idx + 3] * 4;
4627 if (!si_vm_reg_valid(reg))
4631 case PACKET3_CP_DMA:
4632 r = si_vm_packet3_cp_dma_check(ib, idx);
4637 DRM_ERROR("Invalid Compute packet3: 0x%x\n", pkt->opcode);
4643 int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
4647 struct radeon_cs_packet pkt;
4651 pkt.type = RADEON_CP_PACKET_GET_TYPE(ib->ptr[idx]);
4652 pkt.count = RADEON_CP_PACKET_GET_COUNT(ib->ptr[idx]);
4655 case RADEON_PACKET_TYPE0:
4656 dev_err(rdev->dev, "Packet0 not allowed!\n");
4659 case RADEON_PACKET_TYPE2:
4662 case RADEON_PACKET_TYPE3:
4663 pkt.opcode = RADEON_CP_PACKET3_GET_OPCODE(ib->ptr[idx]);
4664 if (ib->is_const_ib)
4665 ret = si_vm_packet3_ce_check(rdev, ib->ptr, &pkt);
4668 case RADEON_RING_TYPE_GFX_INDEX:
4669 ret = si_vm_packet3_gfx_check(rdev, ib->ptr, &pkt);
4671 case CAYMAN_RING_TYPE_CP1_INDEX:
4672 case CAYMAN_RING_TYPE_CP2_INDEX:
4673 ret = si_vm_packet3_compute_check(rdev, ib->ptr, &pkt);
4676 dev_err(rdev->dev, "Non-PM4 ring %d !\n", ib->ring);
4681 idx += pkt.count + 2;
4684 dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type);
4689 for (i = 0; i < ib->length_dw; i++) {
4691 printk("\t0x%08x <---\n", ib->ptr[i]);
4693 printk("\t0x%08x\n", ib->ptr[i]);
4697 } while (idx < ib->length_dw);
4705 int si_vm_init(struct radeon_device *rdev)
4708 rdev->vm_manager.nvm = 16;
4709 /* base offset of vram pages */
4710 rdev->vm_manager.vram_base_offset = 0;
4715 void si_vm_fini(struct radeon_device *rdev)
4720 * si_vm_decode_fault - print human readable fault info
4722 * @rdev: radeon_device pointer
4723 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
4724 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
4726 * Print human readable fault information (SI).
4728 static void si_vm_decode_fault(struct radeon_device *rdev,
4729 u32 status, u32 addr)
4731 u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
4732 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
4733 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
4736 if (rdev->family == CHIP_TAHITI) {
4977 printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
4978 protections, vmid, addr,
4979 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
4983 void si_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
4984 unsigned vm_id, uint64_t pd_addr)
4986 /* write new base address */
4987 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4988 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
4989 WRITE_DATA_DST_SEL(0)));
4992 radeon_ring_write(ring,
4993 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2)) >> 2);
4995 radeon_ring_write(ring,
4996 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm_id - 8) << 2)) >> 2);
4998 radeon_ring_write(ring, 0);
4999 radeon_ring_write(ring, pd_addr >> 12);
5001 /* flush hdp cache */
5002 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5003 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
5004 WRITE_DATA_DST_SEL(0)));
5005 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
5006 radeon_ring_write(ring, 0);
5007 radeon_ring_write(ring, 0x1);
5009 /* bits 0-15 are the VM contexts0-15 */
5010 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5011 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
5012 WRITE_DATA_DST_SEL(0)));
5013 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5014 radeon_ring_write(ring, 0);
5015 radeon_ring_write(ring, 1 << vm_id);
5017 /* wait for the invalidate to complete */
5018 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
5019 radeon_ring_write(ring, (WAIT_REG_MEM_FUNCTION(0) | /* always */
5020 WAIT_REG_MEM_ENGINE(0))); /* me */
5021 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5022 radeon_ring_write(ring, 0);
5023 radeon_ring_write(ring, 0); /* ref */
5024 radeon_ring_write(ring, 0); /* mask */
5025 radeon_ring_write(ring, 0x20); /* poll interval */
5027 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5028 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5029 radeon_ring_write(ring, 0x0);
5033 * Power and clock gating
5035 static void si_wait_for_rlc_serdes(struct radeon_device *rdev)
5039 for (i = 0; i < rdev->usec_timeout; i++) {
5040 if (RREG32(RLC_SERDES_MASTER_BUSY_0) == 0)
5045 for (i = 0; i < rdev->usec_timeout; i++) {
5046 if (RREG32(RLC_SERDES_MASTER_BUSY_1) == 0)
5052 static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
5055 u32 tmp = RREG32(CP_INT_CNTL_RING0);
5060 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5062 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5063 WREG32(CP_INT_CNTL_RING0, tmp);
5066 /* read a gfx register */
5067 tmp = RREG32(DB_DEPTH_INFO);
5069 mask = RLC_BUSY_STATUS | GFX_POWER_STATUS | GFX_CLOCK_STATUS | GFX_LS_STATUS;
5070 for (i = 0; i < rdev->usec_timeout; i++) {
5071 if ((RREG32(RLC_STAT) & mask) == (GFX_CLOCK_STATUS | GFX_POWER_STATUS))
5078 static void si_set_uvd_dcm(struct radeon_device *rdev,
5083 tmp = RREG32(UVD_CGC_CTRL);
5084 tmp &= ~(CLK_OD_MASK | CG_DT_MASK);
5085 tmp |= DCM | CG_DT(1) | CLK_OD(4);
5089 tmp2 = DYN_OR_EN | DYN_RR_EN | G_DIV_ID(7);
5095 WREG32(UVD_CGC_CTRL, tmp);
5096 WREG32_UVD_CTX(UVD_CGC_CTRL2, tmp2);
5099 void si_init_uvd_internal_cg(struct radeon_device *rdev)
5101 bool hw_mode = true;
5104 si_set_uvd_dcm(rdev, false);
5106 u32 tmp = RREG32(UVD_CGC_CTRL);
5108 WREG32(UVD_CGC_CTRL, tmp);
5112 static u32 si_halt_rlc(struct radeon_device *rdev)
5116 orig = data = RREG32(RLC_CNTL);
5118 if (data & RLC_ENABLE) {
5119 data &= ~RLC_ENABLE;
5120 WREG32(RLC_CNTL, data);
5122 si_wait_for_rlc_serdes(rdev);
5128 static void si_update_rlc(struct radeon_device *rdev, u32 rlc)
5132 tmp = RREG32(RLC_CNTL);
5134 WREG32(RLC_CNTL, rlc);
5137 static void si_enable_dma_pg(struct radeon_device *rdev, bool enable)
5141 orig = data = RREG32(DMA_PG);
5142 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA))
5143 data |= PG_CNTL_ENABLE;
5145 data &= ~PG_CNTL_ENABLE;
5147 WREG32(DMA_PG, data);
5150 static void si_init_dma_pg(struct radeon_device *rdev)
5154 WREG32(DMA_PGFSM_WRITE, 0x00002000);
5155 WREG32(DMA_PGFSM_CONFIG, 0x100010ff);
5157 for (tmp = 0; tmp < 5; tmp++)
5158 WREG32(DMA_PGFSM_WRITE, 0);
5161 static void si_enable_gfx_cgpg(struct radeon_device *rdev,
5166 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
5167 tmp = RLC_PUD(0x10) | RLC_PDD(0x10) | RLC_TTPD(0x10) | RLC_MSD(0x10);
5168 WREG32(RLC_TTOP_D, tmp);
5170 tmp = RREG32(RLC_PG_CNTL);
5171 tmp |= GFX_PG_ENABLE;
5172 WREG32(RLC_PG_CNTL, tmp);
5174 tmp = RREG32(RLC_AUTO_PG_CTRL);
5176 WREG32(RLC_AUTO_PG_CTRL, tmp);
5178 tmp = RREG32(RLC_AUTO_PG_CTRL);
5180 WREG32(RLC_AUTO_PG_CTRL, tmp);
5182 tmp = RREG32(DB_RENDER_CONTROL);
5186 static void si_init_gfx_cgpg(struct radeon_device *rdev)
5190 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5192 tmp = RREG32(RLC_PG_CNTL);
5194 WREG32(RLC_PG_CNTL, tmp);
5196 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5198 tmp = RREG32(RLC_AUTO_PG_CTRL);
5200 tmp &= ~GRBM_REG_SGIT_MASK;
5201 tmp |= GRBM_REG_SGIT(0x700);
5202 tmp &= ~PG_AFTER_GRBM_REG_ST_MASK;
5203 WREG32(RLC_AUTO_PG_CTRL, tmp);
5206 static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
5208 u32 mask = 0, tmp, tmp1;
5211 si_select_se_sh(rdev, se, sh);
5212 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
5213 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
5214 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5221 for (i = 0; i < rdev->config.si.max_cu_per_sh; i ++) {
5226 return (~tmp) & mask;
5229 static void si_init_ao_cu_mask(struct radeon_device *rdev)
5231 u32 i, j, k, active_cu_number = 0;
5232 u32 mask, counter, cu_bitmap;
5235 for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
5236 for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
5240 for (k = 0; k < rdev->config.si.max_cu_per_sh; k++) {
5241 if (si_get_cu_active_bitmap(rdev, i, j) & mask) {
5249 active_cu_number += counter;
5250 tmp |= (cu_bitmap << (i * 16 + j * 8));
5254 WREG32(RLC_PG_AO_CU_MASK, tmp);
5256 tmp = RREG32(RLC_MAX_PG_CU);
5257 tmp &= ~MAX_PU_CU_MASK;
5258 tmp |= MAX_PU_CU(active_cu_number);
5259 WREG32(RLC_MAX_PG_CU, tmp);
5262 static void si_enable_cgcg(struct radeon_device *rdev,
5265 u32 data, orig, tmp;
5267 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
5269 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
5270 si_enable_gui_idle_interrupt(rdev, true);
5272 WREG32(RLC_GCPM_GENERAL_3, 0x00000080);
5274 tmp = si_halt_rlc(rdev);
5276 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5277 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5278 WREG32(RLC_SERDES_WR_CTRL, 0x00b000ff);
5280 si_wait_for_rlc_serdes(rdev);
5282 si_update_rlc(rdev, tmp);
5284 WREG32(RLC_SERDES_WR_CTRL, 0x007000ff);
5286 data |= CGCG_EN | CGLS_EN;
5288 si_enable_gui_idle_interrupt(rdev, false);
5290 RREG32(CB_CGTT_SCLK_CTRL);
5291 RREG32(CB_CGTT_SCLK_CTRL);
5292 RREG32(CB_CGTT_SCLK_CTRL);
5293 RREG32(CB_CGTT_SCLK_CTRL);
5295 data &= ~(CGCG_EN | CGLS_EN);
5299 WREG32(RLC_CGCG_CGLS_CTRL, data);
5302 static void si_enable_mgcg(struct radeon_device *rdev,
5305 u32 data, orig, tmp = 0;
5307 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
5308 orig = data = RREG32(CGTS_SM_CTRL_REG);
5311 WREG32(CGTS_SM_CTRL_REG, data);
5313 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5314 orig = data = RREG32(CP_MEM_SLP_CNTL);
5315 data |= CP_MEM_LS_EN;
5317 WREG32(CP_MEM_SLP_CNTL, data);
5320 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5323 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5325 tmp = si_halt_rlc(rdev);
5327 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5328 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5329 WREG32(RLC_SERDES_WR_CTRL, 0x00d000ff);
5331 si_update_rlc(rdev, tmp);
5333 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5336 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5338 data = RREG32(CP_MEM_SLP_CNTL);
5339 if (data & CP_MEM_LS_EN) {
5340 data &= ~CP_MEM_LS_EN;
5341 WREG32(CP_MEM_SLP_CNTL, data);
5343 orig = data = RREG32(CGTS_SM_CTRL_REG);
5344 data |= LS_OVERRIDE | OVERRIDE;
5346 WREG32(CGTS_SM_CTRL_REG, data);
5348 tmp = si_halt_rlc(rdev);
5350 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5351 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5352 WREG32(RLC_SERDES_WR_CTRL, 0x00e000ff);
5354 si_update_rlc(rdev, tmp);
5358 static void si_enable_uvd_mgcg(struct radeon_device *rdev,
5361 u32 orig, data, tmp;
5363 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
5364 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5366 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5368 orig = data = RREG32(UVD_CGC_CTRL);
5371 WREG32(UVD_CGC_CTRL, data);
5373 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0);
5374 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0);
5376 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5378 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5380 orig = data = RREG32(UVD_CGC_CTRL);
5383 WREG32(UVD_CGC_CTRL, data);
5385 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0xffffffff);
5386 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0xffffffff);
5390 static const u32 mc_cg_registers[] =
5403 static void si_enable_mc_ls(struct radeon_device *rdev,
5409 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5410 orig = data = RREG32(mc_cg_registers[i]);
5411 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
5412 data |= MC_LS_ENABLE;
5414 data &= ~MC_LS_ENABLE;
5416 WREG32(mc_cg_registers[i], data);
5420 static void si_enable_mc_mgcg(struct radeon_device *rdev,
5426 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5427 orig = data = RREG32(mc_cg_registers[i]);
5428 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
5429 data |= MC_CG_ENABLE;
5431 data &= ~MC_CG_ENABLE;
5433 WREG32(mc_cg_registers[i], data);
5437 static void si_enable_dma_mgcg(struct radeon_device *rdev,
5440 u32 orig, data, offset;
5443 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
5444 for (i = 0; i < 2; i++) {
5446 offset = DMA0_REGISTER_OFFSET;
5448 offset = DMA1_REGISTER_OFFSET;
5449 orig = data = RREG32(DMA_POWER_CNTL + offset);
5450 data &= ~MEM_POWER_OVERRIDE;
5452 WREG32(DMA_POWER_CNTL + offset, data);
5453 WREG32(DMA_CLK_CTRL + offset, 0x00000100);
5456 for (i = 0; i < 2; i++) {
5458 offset = DMA0_REGISTER_OFFSET;
5460 offset = DMA1_REGISTER_OFFSET;
5461 orig = data = RREG32(DMA_POWER_CNTL + offset);
5462 data |= MEM_POWER_OVERRIDE;
5464 WREG32(DMA_POWER_CNTL + offset, data);
5466 orig = data = RREG32(DMA_CLK_CTRL + offset);
5469 WREG32(DMA_CLK_CTRL + offset, data);
5474 static void si_enable_bif_mgls(struct radeon_device *rdev,
5479 orig = data = RREG32_PCIE(PCIE_CNTL2);
5481 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
5482 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
5483 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
5485 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
5486 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
5489 WREG32_PCIE(PCIE_CNTL2, data);
5492 static void si_enable_hdp_mgcg(struct radeon_device *rdev,
5497 orig = data = RREG32(HDP_HOST_PATH_CNTL);
5499 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
5500 data &= ~CLOCK_GATING_DIS;
5502 data |= CLOCK_GATING_DIS;
5505 WREG32(HDP_HOST_PATH_CNTL, data);
5508 static void si_enable_hdp_ls(struct radeon_device *rdev,
5513 orig = data = RREG32(HDP_MEM_POWER_LS);
5515 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
5516 data |= HDP_LS_ENABLE;
5518 data &= ~HDP_LS_ENABLE;
5521 WREG32(HDP_MEM_POWER_LS, data);
5524 static void si_update_cg(struct radeon_device *rdev,
5525 u32 block, bool enable)
5527 if (block & RADEON_CG_BLOCK_GFX) {
5528 si_enable_gui_idle_interrupt(rdev, false);
5529 /* order matters! */
5531 si_enable_mgcg(rdev, true);
5532 si_enable_cgcg(rdev, true);
5534 si_enable_cgcg(rdev, false);
5535 si_enable_mgcg(rdev, false);
5537 si_enable_gui_idle_interrupt(rdev, true);
5540 if (block & RADEON_CG_BLOCK_MC) {
5541 si_enable_mc_mgcg(rdev, enable);
5542 si_enable_mc_ls(rdev, enable);
5545 if (block & RADEON_CG_BLOCK_SDMA) {
5546 si_enable_dma_mgcg(rdev, enable);
5549 if (block & RADEON_CG_BLOCK_BIF) {
5550 si_enable_bif_mgls(rdev, enable);
5553 if (block & RADEON_CG_BLOCK_UVD) {
5554 if (rdev->has_uvd) {
5555 si_enable_uvd_mgcg(rdev, enable);
5559 if (block & RADEON_CG_BLOCK_HDP) {
5560 si_enable_hdp_mgcg(rdev, enable);
5561 si_enable_hdp_ls(rdev, enable);
5565 static void si_init_cg(struct radeon_device *rdev)
5567 si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5568 RADEON_CG_BLOCK_MC |
5569 RADEON_CG_BLOCK_SDMA |
5570 RADEON_CG_BLOCK_BIF |
5571 RADEON_CG_BLOCK_HDP), true);
5572 if (rdev->has_uvd) {
5573 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, true);
5574 si_init_uvd_internal_cg(rdev);
5578 static void si_fini_cg(struct radeon_device *rdev)
5580 if (rdev->has_uvd) {
5581 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, false);
5583 si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5584 RADEON_CG_BLOCK_MC |
5585 RADEON_CG_BLOCK_SDMA |
5586 RADEON_CG_BLOCK_BIF |
5587 RADEON_CG_BLOCK_HDP), false);
5590 u32 si_get_csb_size(struct radeon_device *rdev)
5593 const struct cs_section_def *sect = NULL;
5594 const struct cs_extent_def *ext = NULL;
5596 if (rdev->rlc.cs_data == NULL)
5599 /* begin clear state */
5601 /* context control state */
5604 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5605 for (ext = sect->section; ext->extent != NULL; ++ext) {
5606 if (sect->id == SECT_CONTEXT)
5607 count += 2 + ext->reg_count;
5612 /* pa_sc_raster_config */
5614 /* end clear state */
5622 void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
5625 const struct cs_section_def *sect = NULL;
5626 const struct cs_extent_def *ext = NULL;
5628 if (rdev->rlc.cs_data == NULL)
5633 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
5634 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
5636 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
5637 buffer[count++] = cpu_to_le32(0x80000000);
5638 buffer[count++] = cpu_to_le32(0x80000000);
5640 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5641 for (ext = sect->section; ext->extent != NULL; ++ext) {
5642 if (sect->id == SECT_CONTEXT) {
5644 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
5645 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
5646 for (i = 0; i < ext->reg_count; i++)
5647 buffer[count++] = cpu_to_le32(ext->extent[i]);
5654 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 1));
5655 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
5656 switch (rdev->family) {
5659 buffer[count++] = cpu_to_le32(0x2a00126a);
5662 buffer[count++] = cpu_to_le32(0x0000124a);
5665 buffer[count++] = cpu_to_le32(0x00000082);
5668 buffer[count++] = cpu_to_le32(0x00000000);
5671 buffer[count++] = cpu_to_le32(0x00000000);
5675 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
5676 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
5678 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
5679 buffer[count++] = cpu_to_le32(0);
5682 static void si_init_pg(struct radeon_device *rdev)
5684 if (rdev->pg_flags) {
5685 if (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA) {
5686 si_init_dma_pg(rdev);
5688 si_init_ao_cu_mask(rdev);
5689 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
5690 si_init_gfx_cgpg(rdev);
5692 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5693 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5695 si_enable_dma_pg(rdev, true);
5696 si_enable_gfx_cgpg(rdev, true);
5698 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5699 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5703 static void si_fini_pg(struct radeon_device *rdev)
5705 if (rdev->pg_flags) {
5706 si_enable_dma_pg(rdev, false);
5707 si_enable_gfx_cgpg(rdev, false);
5714 void si_rlc_reset(struct radeon_device *rdev)
5716 u32 tmp = RREG32(GRBM_SOFT_RESET);
5718 tmp |= SOFT_RESET_RLC;
5719 WREG32(GRBM_SOFT_RESET, tmp);
5721 tmp &= ~SOFT_RESET_RLC;
5722 WREG32(GRBM_SOFT_RESET, tmp);
5726 static void si_rlc_stop(struct radeon_device *rdev)
5728 WREG32(RLC_CNTL, 0);
5730 si_enable_gui_idle_interrupt(rdev, false);
5732 si_wait_for_rlc_serdes(rdev);
5735 static void si_rlc_start(struct radeon_device *rdev)
5737 WREG32(RLC_CNTL, RLC_ENABLE);
5739 si_enable_gui_idle_interrupt(rdev, true);
5744 static bool si_lbpw_supported(struct radeon_device *rdev)
5748 /* Enable LBPW only for DDR3 */
5749 tmp = RREG32(MC_SEQ_MISC0);
5750 if ((tmp & 0xF0000000) == 0xB0000000)
5755 static void si_enable_lbpw(struct radeon_device *rdev, bool enable)
5759 tmp = RREG32(RLC_LB_CNTL);
5761 tmp |= LOAD_BALANCE_ENABLE;
5763 tmp &= ~LOAD_BALANCE_ENABLE;
5764 WREG32(RLC_LB_CNTL, tmp);
5767 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5768 WREG32(SPI_LB_CU_MASK, 0x00ff);
5772 static int si_rlc_resume(struct radeon_device *rdev)
5787 WREG32(RLC_RL_BASE, 0);
5788 WREG32(RLC_RL_SIZE, 0);
5789 WREG32(RLC_LB_CNTL, 0);
5790 WREG32(RLC_LB_CNTR_MAX, 0xffffffff);
5791 WREG32(RLC_LB_CNTR_INIT, 0);
5792 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5794 WREG32(RLC_MC_CNTL, 0);
5795 WREG32(RLC_UCODE_CNTL, 0);
5798 const struct rlc_firmware_header_v1_0 *hdr =
5799 (const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data;
5800 u32 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
5801 const __le32 *fw_data = (const __le32 *)
5802 (rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
5804 radeon_ucode_print_rlc_hdr(&hdr->header);
5806 for (i = 0; i < fw_size; i++) {
5807 WREG32(RLC_UCODE_ADDR, i);
5808 WREG32(RLC_UCODE_DATA, le32_to_cpup(fw_data++));
5811 const __be32 *fw_data =
5812 (const __be32 *)rdev->rlc_fw->data;
5813 for (i = 0; i < SI_RLC_UCODE_SIZE; i++) {
5814 WREG32(RLC_UCODE_ADDR, i);
5815 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
5818 WREG32(RLC_UCODE_ADDR, 0);
5820 si_enable_lbpw(rdev, si_lbpw_supported(rdev));
5827 static void si_enable_interrupts(struct radeon_device *rdev)
5829 u32 ih_cntl = RREG32(IH_CNTL);
5830 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5832 ih_cntl |= ENABLE_INTR;
5833 ih_rb_cntl |= IH_RB_ENABLE;
5834 WREG32(IH_CNTL, ih_cntl);
5835 WREG32(IH_RB_CNTL, ih_rb_cntl);
5836 rdev->ih.enabled = true;
5839 static void si_disable_interrupts(struct radeon_device *rdev)
5841 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5842 u32 ih_cntl = RREG32(IH_CNTL);
5844 ih_rb_cntl &= ~IH_RB_ENABLE;
5845 ih_cntl &= ~ENABLE_INTR;
5846 WREG32(IH_RB_CNTL, ih_rb_cntl);
5847 WREG32(IH_CNTL, ih_cntl);
5848 /* set rptr, wptr to 0 */
5849 WREG32(IH_RB_RPTR, 0);
5850 WREG32(IH_RB_WPTR, 0);
5851 rdev->ih.enabled = false;
5855 static void si_disable_interrupt_state(struct radeon_device *rdev)
5859 tmp = RREG32(CP_INT_CNTL_RING0) &
5860 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5861 WREG32(CP_INT_CNTL_RING0, tmp);
5862 WREG32(CP_INT_CNTL_RING1, 0);
5863 WREG32(CP_INT_CNTL_RING2, 0);
5864 tmp = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5865 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, tmp);
5866 tmp = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5867 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, tmp);
5868 WREG32(GRBM_INT_CNTL, 0);
5869 WREG32(SRBM_INT_CNTL, 0);
5870 if (rdev->num_crtc >= 2) {
5871 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5872 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5874 if (rdev->num_crtc >= 4) {
5875 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5876 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5878 if (rdev->num_crtc >= 6) {
5879 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5880 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5883 if (rdev->num_crtc >= 2) {
5884 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5885 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5887 if (rdev->num_crtc >= 4) {
5888 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5889 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5891 if (rdev->num_crtc >= 6) {
5892 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5893 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5896 if (!ASIC_IS_NODCE(rdev)) {
5897 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
5899 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5900 WREG32(DC_HPD1_INT_CONTROL, tmp);
5901 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5902 WREG32(DC_HPD2_INT_CONTROL, tmp);
5903 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5904 WREG32(DC_HPD3_INT_CONTROL, tmp);
5905 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5906 WREG32(DC_HPD4_INT_CONTROL, tmp);
5907 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5908 WREG32(DC_HPD5_INT_CONTROL, tmp);
5909 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5910 WREG32(DC_HPD6_INT_CONTROL, tmp);
5914 static int si_irq_init(struct radeon_device *rdev)
5918 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
5921 ret = r600_ih_ring_alloc(rdev);
5926 si_disable_interrupts(rdev);
5929 ret = si_rlc_resume(rdev);
5931 r600_ih_ring_fini(rdev);
5935 /* setup interrupt control */
5936 /* set dummy read address to ring address */
5937 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
5938 interrupt_cntl = RREG32(INTERRUPT_CNTL);
5939 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
5940 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
5942 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
5943 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
5944 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
5945 WREG32(INTERRUPT_CNTL, interrupt_cntl);
5947 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
5948 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
5950 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
5951 IH_WPTR_OVERFLOW_CLEAR |
5954 if (rdev->wb.enabled)
5955 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
5957 /* set the writeback address whether it's enabled or not */
5958 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
5959 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
5961 WREG32(IH_RB_CNTL, ih_rb_cntl);
5963 /* set rptr, wptr to 0 */
5964 WREG32(IH_RB_RPTR, 0);
5965 WREG32(IH_RB_WPTR, 0);
5967 /* Default settings for IH_CNTL (disabled at first) */
5968 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
5969 /* RPTR_REARM only works if msi's are enabled */
5970 if (rdev->msi_enabled)
5971 ih_cntl |= RPTR_REARM;
5972 WREG32(IH_CNTL, ih_cntl);
5974 /* force the active interrupt state to all disabled */
5975 si_disable_interrupt_state(rdev);
5977 pci_set_master(rdev->pdev);
5980 si_enable_interrupts(rdev);
5985 int si_irq_set(struct radeon_device *rdev)
5988 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
5989 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
5990 u32 hpd1 = 0, hpd2 = 0, hpd3 = 0, hpd4 = 0, hpd5 = 0, hpd6 = 0;
5991 u32 grbm_int_cntl = 0;
5992 u32 dma_cntl, dma_cntl1;
5993 u32 thermal_int = 0;
5995 if (!rdev->irq.installed) {
5996 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
5999 /* don't enable anything if the ih is disabled */
6000 if (!rdev->ih.enabled) {
6001 si_disable_interrupts(rdev);
6002 /* force the active interrupt state to all disabled */
6003 si_disable_interrupt_state(rdev);
6007 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
6008 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6010 if (!ASIC_IS_NODCE(rdev)) {
6011 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
6012 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
6013 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
6014 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
6015 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
6016 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
6019 dma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6020 dma_cntl1 = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6022 thermal_int = RREG32(CG_THERMAL_INT) &
6023 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
6025 /* enable CP interrupts on all rings */
6026 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
6027 DRM_DEBUG("si_irq_set: sw int gfx\n");
6028 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
6030 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
6031 DRM_DEBUG("si_irq_set: sw int cp1\n");
6032 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
6034 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
6035 DRM_DEBUG("si_irq_set: sw int cp2\n");
6036 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
6038 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
6039 DRM_DEBUG("si_irq_set: sw int dma\n");
6040 dma_cntl |= TRAP_ENABLE;
6043 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
6044 DRM_DEBUG("si_irq_set: sw int dma1\n");
6045 dma_cntl1 |= TRAP_ENABLE;
6047 if (rdev->irq.crtc_vblank_int[0] ||
6048 atomic_read(&rdev->irq.pflip[0])) {
6049 DRM_DEBUG("si_irq_set: vblank 0\n");
6050 crtc1 |= VBLANK_INT_MASK;
6052 if (rdev->irq.crtc_vblank_int[1] ||
6053 atomic_read(&rdev->irq.pflip[1])) {
6054 DRM_DEBUG("si_irq_set: vblank 1\n");
6055 crtc2 |= VBLANK_INT_MASK;
6057 if (rdev->irq.crtc_vblank_int[2] ||
6058 atomic_read(&rdev->irq.pflip[2])) {
6059 DRM_DEBUG("si_irq_set: vblank 2\n");
6060 crtc3 |= VBLANK_INT_MASK;
6062 if (rdev->irq.crtc_vblank_int[3] ||
6063 atomic_read(&rdev->irq.pflip[3])) {
6064 DRM_DEBUG("si_irq_set: vblank 3\n");
6065 crtc4 |= VBLANK_INT_MASK;
6067 if (rdev->irq.crtc_vblank_int[4] ||
6068 atomic_read(&rdev->irq.pflip[4])) {
6069 DRM_DEBUG("si_irq_set: vblank 4\n");
6070 crtc5 |= VBLANK_INT_MASK;
6072 if (rdev->irq.crtc_vblank_int[5] ||
6073 atomic_read(&rdev->irq.pflip[5])) {
6074 DRM_DEBUG("si_irq_set: vblank 5\n");
6075 crtc6 |= VBLANK_INT_MASK;
6077 if (rdev->irq.hpd[0]) {
6078 DRM_DEBUG("si_irq_set: hpd 1\n");
6079 hpd1 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
6081 if (rdev->irq.hpd[1]) {
6082 DRM_DEBUG("si_irq_set: hpd 2\n");
6083 hpd2 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
6085 if (rdev->irq.hpd[2]) {
6086 DRM_DEBUG("si_irq_set: hpd 3\n");
6087 hpd3 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
6089 if (rdev->irq.hpd[3]) {
6090 DRM_DEBUG("si_irq_set: hpd 4\n");
6091 hpd4 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
6093 if (rdev->irq.hpd[4]) {
6094 DRM_DEBUG("si_irq_set: hpd 5\n");
6095 hpd5 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
6097 if (rdev->irq.hpd[5]) {
6098 DRM_DEBUG("si_irq_set: hpd 6\n");
6099 hpd6 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
6102 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
6103 WREG32(CP_INT_CNTL_RING1, cp_int_cntl1);
6104 WREG32(CP_INT_CNTL_RING2, cp_int_cntl2);
6106 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, dma_cntl);
6107 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, dma_cntl1);
6109 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
6111 if (rdev->irq.dpm_thermal) {
6112 DRM_DEBUG("dpm thermal\n");
6113 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
6116 if (rdev->num_crtc >= 2) {
6117 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
6118 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
6120 if (rdev->num_crtc >= 4) {
6121 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
6122 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
6124 if (rdev->num_crtc >= 6) {
6125 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
6126 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
6129 if (rdev->num_crtc >= 2) {
6130 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
6131 GRPH_PFLIP_INT_MASK);
6132 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
6133 GRPH_PFLIP_INT_MASK);
6135 if (rdev->num_crtc >= 4) {
6136 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
6137 GRPH_PFLIP_INT_MASK);
6138 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
6139 GRPH_PFLIP_INT_MASK);
6141 if (rdev->num_crtc >= 6) {
6142 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
6143 GRPH_PFLIP_INT_MASK);
6144 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
6145 GRPH_PFLIP_INT_MASK);
6148 if (!ASIC_IS_NODCE(rdev)) {
6149 WREG32(DC_HPD1_INT_CONTROL, hpd1);
6150 WREG32(DC_HPD2_INT_CONTROL, hpd2);
6151 WREG32(DC_HPD3_INT_CONTROL, hpd3);
6152 WREG32(DC_HPD4_INT_CONTROL, hpd4);
6153 WREG32(DC_HPD5_INT_CONTROL, hpd5);
6154 WREG32(DC_HPD6_INT_CONTROL, hpd6);
6157 WREG32(CG_THERMAL_INT, thermal_int);
6160 RREG32(SRBM_STATUS);
6165 static inline void si_irq_ack(struct radeon_device *rdev)
6169 if (ASIC_IS_NODCE(rdev))
6172 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
6173 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
6174 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
6175 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
6176 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
6177 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
6178 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
6179 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
6180 if (rdev->num_crtc >= 4) {
6181 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
6182 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
6184 if (rdev->num_crtc >= 6) {
6185 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
6186 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
6189 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
6190 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6191 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
6192 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6193 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
6194 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
6195 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
6196 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
6197 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
6198 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
6199 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
6200 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
6202 if (rdev->num_crtc >= 4) {
6203 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
6204 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6205 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
6206 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6207 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
6208 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
6209 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
6210 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
6211 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
6212 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
6213 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
6214 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
6217 if (rdev->num_crtc >= 6) {
6218 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
6219 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6220 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
6221 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6222 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
6223 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
6224 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
6225 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
6226 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
6227 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
6228 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
6229 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
6232 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
6233 tmp = RREG32(DC_HPD1_INT_CONTROL);
6234 tmp |= DC_HPDx_INT_ACK;
6235 WREG32(DC_HPD1_INT_CONTROL, tmp);
6237 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
6238 tmp = RREG32(DC_HPD2_INT_CONTROL);
6239 tmp |= DC_HPDx_INT_ACK;
6240 WREG32(DC_HPD2_INT_CONTROL, tmp);
6242 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6243 tmp = RREG32(DC_HPD3_INT_CONTROL);
6244 tmp |= DC_HPDx_INT_ACK;
6245 WREG32(DC_HPD3_INT_CONTROL, tmp);
6247 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6248 tmp = RREG32(DC_HPD4_INT_CONTROL);
6249 tmp |= DC_HPDx_INT_ACK;
6250 WREG32(DC_HPD4_INT_CONTROL, tmp);
6252 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6253 tmp = RREG32(DC_HPD5_INT_CONTROL);
6254 tmp |= DC_HPDx_INT_ACK;
6255 WREG32(DC_HPD5_INT_CONTROL, tmp);
6257 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6258 tmp = RREG32(DC_HPD5_INT_CONTROL);
6259 tmp |= DC_HPDx_INT_ACK;
6260 WREG32(DC_HPD6_INT_CONTROL, tmp);
6263 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_RX_INTERRUPT) {
6264 tmp = RREG32(DC_HPD1_INT_CONTROL);
6265 tmp |= DC_HPDx_RX_INT_ACK;
6266 WREG32(DC_HPD1_INT_CONTROL, tmp);
6268 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_RX_INTERRUPT) {
6269 tmp = RREG32(DC_HPD2_INT_CONTROL);
6270 tmp |= DC_HPDx_RX_INT_ACK;
6271 WREG32(DC_HPD2_INT_CONTROL, tmp);
6273 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_RX_INTERRUPT) {
6274 tmp = RREG32(DC_HPD3_INT_CONTROL);
6275 tmp |= DC_HPDx_RX_INT_ACK;
6276 WREG32(DC_HPD3_INT_CONTROL, tmp);
6278 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_RX_INTERRUPT) {
6279 tmp = RREG32(DC_HPD4_INT_CONTROL);
6280 tmp |= DC_HPDx_RX_INT_ACK;
6281 WREG32(DC_HPD4_INT_CONTROL, tmp);
6283 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_RX_INTERRUPT) {
6284 tmp = RREG32(DC_HPD5_INT_CONTROL);
6285 tmp |= DC_HPDx_RX_INT_ACK;
6286 WREG32(DC_HPD5_INT_CONTROL, tmp);
6288 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_RX_INTERRUPT) {
6289 tmp = RREG32(DC_HPD5_INT_CONTROL);
6290 tmp |= DC_HPDx_RX_INT_ACK;
6291 WREG32(DC_HPD6_INT_CONTROL, tmp);
6295 static void si_irq_disable(struct radeon_device *rdev)
6297 si_disable_interrupts(rdev);
6298 /* Wait and acknowledge irq */
6301 si_disable_interrupt_state(rdev);
6304 static void si_irq_suspend(struct radeon_device *rdev)
6306 si_irq_disable(rdev);
6310 static void si_irq_fini(struct radeon_device *rdev)
6312 si_irq_suspend(rdev);
6313 r600_ih_ring_fini(rdev);
6316 static inline u32 si_get_ih_wptr(struct radeon_device *rdev)
6320 if (rdev->wb.enabled)
6321 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
6323 wptr = RREG32(IH_RB_WPTR);
6325 if (wptr & RB_OVERFLOW) {
6326 wptr &= ~RB_OVERFLOW;
6327 /* When a ring buffer overflow happen start parsing interrupt
6328 * from the last not overwritten vector (wptr + 16). Hopefully
6329 * this should allow us to catchup.
6331 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
6332 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
6333 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
6334 tmp = RREG32(IH_RB_CNTL);
6335 tmp |= IH_WPTR_OVERFLOW_CLEAR;
6336 WREG32(IH_RB_CNTL, tmp);
6338 return (wptr & rdev->ih.ptr_mask);
6342 * Each IV ring entry is 128 bits:
6343 * [7:0] - interrupt source id
6345 * [59:32] - interrupt source data
6346 * [63:60] - reserved
6349 * [127:80] - reserved
6351 int si_irq_process(struct radeon_device *rdev)
6355 u32 src_id, src_data, ring_id;
6357 bool queue_hotplug = false;
6358 bool queue_dp = false;
6359 bool queue_thermal = false;
6362 if (!rdev->ih.enabled || rdev->shutdown)
6365 wptr = si_get_ih_wptr(rdev);
6368 /* is somebody else already processing irqs? */
6369 if (atomic_xchg(&rdev->ih.lock, 1))
6372 rptr = rdev->ih.rptr;
6373 DRM_DEBUG("si_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
6375 /* Order reading of wptr vs. reading of IH ring data */
6378 /* display interrupts */
6381 while (rptr != wptr) {
6382 /* wptr/rptr are in bytes! */
6383 ring_index = rptr / 4;
6384 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
6385 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
6386 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
6389 case 1: /* D1 vblank/vline */
6391 case 0: /* D1 vblank */
6392 if (!(rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT))
6393 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6395 if (rdev->irq.crtc_vblank_int[0]) {
6396 drm_handle_vblank(rdev->ddev, 0);
6397 rdev->pm.vblank_sync = true;
6398 wake_up(&rdev->irq.vblank_queue);
6400 if (atomic_read(&rdev->irq.pflip[0]))
6401 radeon_crtc_handle_vblank(rdev, 0);
6402 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
6403 DRM_DEBUG("IH: D1 vblank\n");
6406 case 1: /* D1 vline */
6407 if (!(rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT))
6408 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6410 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
6411 DRM_DEBUG("IH: D1 vline\n");
6415 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6419 case 2: /* D2 vblank/vline */
6421 case 0: /* D2 vblank */
6422 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT))
6423 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6425 if (rdev->irq.crtc_vblank_int[1]) {
6426 drm_handle_vblank(rdev->ddev, 1);
6427 rdev->pm.vblank_sync = true;
6428 wake_up(&rdev->irq.vblank_queue);
6430 if (atomic_read(&rdev->irq.pflip[1]))
6431 radeon_crtc_handle_vblank(rdev, 1);
6432 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
6433 DRM_DEBUG("IH: D2 vblank\n");
6436 case 1: /* D2 vline */
6437 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT))
6438 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6440 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
6441 DRM_DEBUG("IH: D2 vline\n");
6445 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6449 case 3: /* D3 vblank/vline */
6451 case 0: /* D3 vblank */
6452 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT))
6453 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6455 if (rdev->irq.crtc_vblank_int[2]) {
6456 drm_handle_vblank(rdev->ddev, 2);
6457 rdev->pm.vblank_sync = true;
6458 wake_up(&rdev->irq.vblank_queue);
6460 if (atomic_read(&rdev->irq.pflip[2]))
6461 radeon_crtc_handle_vblank(rdev, 2);
6462 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
6463 DRM_DEBUG("IH: D3 vblank\n");
6466 case 1: /* D3 vline */
6467 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT))
6468 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6470 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
6471 DRM_DEBUG("IH: D3 vline\n");
6475 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6479 case 4: /* D4 vblank/vline */
6481 case 0: /* D4 vblank */
6482 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT))
6483 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6485 if (rdev->irq.crtc_vblank_int[3]) {
6486 drm_handle_vblank(rdev->ddev, 3);
6487 rdev->pm.vblank_sync = true;
6488 wake_up(&rdev->irq.vblank_queue);
6490 if (atomic_read(&rdev->irq.pflip[3]))
6491 radeon_crtc_handle_vblank(rdev, 3);
6492 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
6493 DRM_DEBUG("IH: D4 vblank\n");
6496 case 1: /* D4 vline */
6497 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT))
6498 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6500 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
6501 DRM_DEBUG("IH: D4 vline\n");
6505 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6509 case 5: /* D5 vblank/vline */
6511 case 0: /* D5 vblank */
6512 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT))
6513 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6515 if (rdev->irq.crtc_vblank_int[4]) {
6516 drm_handle_vblank(rdev->ddev, 4);
6517 rdev->pm.vblank_sync = true;
6518 wake_up(&rdev->irq.vblank_queue);
6520 if (atomic_read(&rdev->irq.pflip[4]))
6521 radeon_crtc_handle_vblank(rdev, 4);
6522 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
6523 DRM_DEBUG("IH: D5 vblank\n");
6526 case 1: /* D5 vline */
6527 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT))
6528 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6530 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
6531 DRM_DEBUG("IH: D5 vline\n");
6535 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6539 case 6: /* D6 vblank/vline */
6541 case 0: /* D6 vblank */
6542 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT))
6543 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6545 if (rdev->irq.crtc_vblank_int[5]) {
6546 drm_handle_vblank(rdev->ddev, 5);
6547 rdev->pm.vblank_sync = true;
6548 wake_up(&rdev->irq.vblank_queue);
6550 if (atomic_read(&rdev->irq.pflip[5]))
6551 radeon_crtc_handle_vblank(rdev, 5);
6552 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
6553 DRM_DEBUG("IH: D6 vblank\n");
6556 case 1: /* D6 vline */
6557 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT))
6558 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6560 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
6561 DRM_DEBUG("IH: D6 vline\n");
6565 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6569 case 8: /* D1 page flip */
6570 case 10: /* D2 page flip */
6571 case 12: /* D3 page flip */
6572 case 14: /* D4 page flip */
6573 case 16: /* D5 page flip */
6574 case 18: /* D6 page flip */
6575 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
6576 if (radeon_use_pflipirq > 0)
6577 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
6579 case 42: /* HPD hotplug */
6582 if (!(rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT))
6583 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6585 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
6586 queue_hotplug = true;
6587 DRM_DEBUG("IH: HPD1\n");
6591 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT))
6592 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6594 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
6595 queue_hotplug = true;
6596 DRM_DEBUG("IH: HPD2\n");
6600 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT))
6601 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6603 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
6604 queue_hotplug = true;
6605 DRM_DEBUG("IH: HPD3\n");
6609 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT))
6610 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6612 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
6613 queue_hotplug = true;
6614 DRM_DEBUG("IH: HPD4\n");
6618 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT))
6619 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6621 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
6622 queue_hotplug = true;
6623 DRM_DEBUG("IH: HPD5\n");
6627 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT))
6628 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6630 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
6631 queue_hotplug = true;
6632 DRM_DEBUG("IH: HPD6\n");
6636 if (!(rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_RX_INTERRUPT))
6637 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6639 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_RX_INTERRUPT;
6641 DRM_DEBUG("IH: HPD_RX 1\n");
6645 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_RX_INTERRUPT))
6646 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6648 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_RX_INTERRUPT;
6650 DRM_DEBUG("IH: HPD_RX 2\n");
6654 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_RX_INTERRUPT))
6655 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6657 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_RX_INTERRUPT;
6659 DRM_DEBUG("IH: HPD_RX 3\n");
6663 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_RX_INTERRUPT))
6664 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6666 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_RX_INTERRUPT;
6668 DRM_DEBUG("IH: HPD_RX 4\n");
6672 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_RX_INTERRUPT))
6673 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6675 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_RX_INTERRUPT;
6677 DRM_DEBUG("IH: HPD_RX 5\n");
6681 if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_RX_INTERRUPT))
6682 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6684 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_RX_INTERRUPT;
6686 DRM_DEBUG("IH: HPD_RX 6\n");
6690 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6695 DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
6696 WREG32(SRBM_INT_ACK, 0x1);
6699 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
6700 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
6704 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
6705 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
6706 /* reset addr and status */
6707 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
6708 if (addr == 0x0 && status == 0x0)
6710 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
6711 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
6713 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
6715 si_vm_decode_fault(rdev, status, addr);
6717 case 176: /* RINGID0 CP_INT */
6718 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6720 case 177: /* RINGID1 CP_INT */
6721 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6723 case 178: /* RINGID2 CP_INT */
6724 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6726 case 181: /* CP EOP event */
6727 DRM_DEBUG("IH: CP EOP\n");
6730 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6733 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6736 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6740 case 224: /* DMA trap event */
6741 DRM_DEBUG("IH: DMA trap\n");
6742 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
6744 case 230: /* thermal low to high */
6745 DRM_DEBUG("IH: thermal low to high\n");
6746 rdev->pm.dpm.thermal.high_to_low = false;
6747 queue_thermal = true;
6749 case 231: /* thermal high to low */
6750 DRM_DEBUG("IH: thermal high to low\n");
6751 rdev->pm.dpm.thermal.high_to_low = true;
6752 queue_thermal = true;
6754 case 233: /* GUI IDLE */
6755 DRM_DEBUG("IH: GUI idle\n");
6757 case 244: /* DMA trap event */
6758 DRM_DEBUG("IH: DMA1 trap\n");
6759 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6762 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6766 /* wptr/rptr are in bytes! */
6768 rptr &= rdev->ih.ptr_mask;
6769 WREG32(IH_RB_RPTR, rptr);
6772 schedule_work(&rdev->dp_work);
6774 schedule_delayed_work(&rdev->hotplug_work, 0);
6775 if (queue_thermal && rdev->pm.dpm_enabled)
6776 schedule_work(&rdev->pm.dpm.thermal.work);
6777 rdev->ih.rptr = rptr;
6778 atomic_set(&rdev->ih.lock, 0);
6780 /* make sure wptr hasn't changed while processing */
6781 wptr = si_get_ih_wptr(rdev);
6789 * startup/shutdown callbacks
6791 static void si_uvd_init(struct radeon_device *rdev)
6798 r = radeon_uvd_init(rdev);
6800 dev_err(rdev->dev, "failed UVD (%d) init.\n", r);
6802 * At this point rdev->uvd.vcpu_bo is NULL which trickles down
6803 * to early fails uvd_v2_2_resume() and thus nothing happens
6804 * there. So it is pointless to try to go through that code
6805 * hence why we disable uvd here.
6810 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
6811 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);
6814 static void si_uvd_start(struct radeon_device *rdev)
6821 r = uvd_v2_2_resume(rdev);
6823 dev_err(rdev->dev, "failed UVD resume (%d).\n", r);
6826 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);
6828 dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);
6834 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
6837 static void si_uvd_resume(struct radeon_device *rdev)
6839 struct radeon_ring *ring;
6842 if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)
6845 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6846 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));
6848 dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
6851 r = uvd_v1_0_init(rdev);
6853 dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);
6858 static void si_vce_init(struct radeon_device *rdev)
6865 r = radeon_vce_init(rdev);
6867 dev_err(rdev->dev, "failed VCE (%d) init.\n", r);
6869 * At this point rdev->vce.vcpu_bo is NULL which trickles down
6870 * to early fails si_vce_start() and thus nothing happens
6871 * there. So it is pointless to try to go through that code
6872 * hence why we disable vce here.
6877 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_obj = NULL;
6878 r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE1_INDEX], 4096);
6879 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_obj = NULL;
6880 r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE2_INDEX], 4096);
6883 static void si_vce_start(struct radeon_device *rdev)
6890 r = radeon_vce_resume(rdev);
6892 dev_err(rdev->dev, "failed VCE resume (%d).\n", r);
6895 r = vce_v1_0_resume(rdev);
6897 dev_err(rdev->dev, "failed VCE resume (%d).\n", r);
6900 r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE1_INDEX);
6902 dev_err(rdev->dev, "failed initializing VCE1 fences (%d).\n", r);
6905 r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE2_INDEX);
6907 dev_err(rdev->dev, "failed initializing VCE2 fences (%d).\n", r);
6913 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
6914 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
6917 static void si_vce_resume(struct radeon_device *rdev)
6919 struct radeon_ring *ring;
6922 if (!rdev->has_vce || !rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size)
6925 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
6926 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, VCE_CMD_NO_OP);
6928 dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);
6931 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
6932 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, VCE_CMD_NO_OP);
6934 dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);
6937 r = vce_v1_0_init(rdev);
6939 dev_err(rdev->dev, "failed initializing VCE (%d).\n", r);
6944 static int si_startup(struct radeon_device *rdev)
6946 struct radeon_ring *ring;
6949 /* enable pcie gen2/3 link */
6950 si_pcie_gen3_enable(rdev);
6952 si_program_aspm(rdev);
6954 /* scratch needs to be initialized before MC */
6955 r = r600_vram_scratch_init(rdev);
6959 si_mc_program(rdev);
6961 if (!rdev->pm.dpm_enabled) {
6962 r = si_mc_load_microcode(rdev);
6964 DRM_ERROR("Failed to load MC firmware!\n");
6969 r = si_pcie_gart_enable(rdev);
6974 /* allocate rlc buffers */
6975 if (rdev->family == CHIP_VERDE) {
6976 rdev->rlc.reg_list = verde_rlc_save_restore_register_list;
6977 rdev->rlc.reg_list_size =
6978 (u32)ARRAY_SIZE(verde_rlc_save_restore_register_list);
6980 rdev->rlc.cs_data = si_cs_data;
6981 r = sumo_rlc_init(rdev);
6983 DRM_ERROR("Failed to init rlc BOs!\n");
6987 /* allocate wb buffer */
6988 r = radeon_wb_init(rdev);
6992 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
6994 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6998 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7000 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7004 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
7006 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7010 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
7012 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7016 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7018 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7026 if (!rdev->irq.installed) {
7027 r = radeon_irq_kms_init(rdev);
7032 r = si_irq_init(rdev);
7034 DRM_ERROR("radeon: IH init failed (%d).\n", r);
7035 radeon_irq_kms_fini(rdev);
7040 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
7041 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
7046 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7047 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
7052 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7053 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
7058 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
7059 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
7060 DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
7064 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
7065 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
7066 DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
7070 r = si_cp_load_microcode(rdev);
7073 r = si_cp_resume(rdev);
7077 r = cayman_dma_resume(rdev);
7081 si_uvd_resume(rdev);
7082 si_vce_resume(rdev);
7084 r = radeon_ib_pool_init(rdev);
7086 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
7090 r = radeon_vm_manager_init(rdev);
7092 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
7096 r = radeon_audio_init(rdev);
7103 int si_resume(struct radeon_device *rdev)
7107 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
7108 * posting will perform necessary task to bring back GPU into good
7112 atom_asic_init(rdev->mode_info.atom_context);
7114 /* init golden registers */
7115 si_init_golden_registers(rdev);
7117 if (rdev->pm.pm_method == PM_METHOD_DPM)
7118 radeon_pm_resume(rdev);
7120 rdev->accel_working = true;
7121 r = si_startup(rdev);
7123 DRM_ERROR("si startup failed on resume\n");
7124 rdev->accel_working = false;
7132 int si_suspend(struct radeon_device *rdev)
7134 radeon_pm_suspend(rdev);
7135 radeon_audio_fini(rdev);
7136 radeon_vm_manager_fini(rdev);
7137 si_cp_enable(rdev, false);
7138 cayman_dma_stop(rdev);
7139 if (rdev->has_uvd) {
7140 uvd_v1_0_fini(rdev);
7141 radeon_uvd_suspend(rdev);
7144 radeon_vce_suspend(rdev);
7147 si_irq_suspend(rdev);
7148 radeon_wb_disable(rdev);
7149 si_pcie_gart_disable(rdev);
7153 /* Plan is to move initialization in that function and use
7154 * helper function so that radeon_device_init pretty much
7155 * do nothing more than calling asic specific function. This
7156 * should also allow to remove a bunch of callback function
7159 int si_init(struct radeon_device *rdev)
7161 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
7165 if (!radeon_get_bios(rdev)) {
7166 if (ASIC_IS_AVIVO(rdev))
7169 /* Must be an ATOMBIOS */
7170 if (!rdev->is_atom_bios) {
7171 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
7174 r = radeon_atombios_init(rdev);
7178 /* Post card if necessary */
7179 if (!radeon_card_posted(rdev)) {
7181 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
7184 DRM_INFO("GPU not posted. posting now...\n");
7185 atom_asic_init(rdev->mode_info.atom_context);
7187 /* init golden registers */
7188 si_init_golden_registers(rdev);
7189 /* Initialize scratch registers */
7190 si_scratch_init(rdev);
7191 /* Initialize surface registers */
7192 radeon_surface_init(rdev);
7193 /* Initialize clocks */
7194 radeon_get_clock_info(rdev->ddev);
7197 r = radeon_fence_driver_init(rdev);
7201 /* initialize memory controller */
7202 r = si_mc_init(rdev);
7205 /* Memory manager */
7206 r = radeon_bo_init(rdev);
7210 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
7211 !rdev->rlc_fw || !rdev->mc_fw) {
7212 r = si_init_microcode(rdev);
7214 DRM_ERROR("Failed to load firmware!\n");
7219 /* Initialize power management */
7220 radeon_pm_init(rdev);
7222 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
7223 ring->ring_obj = NULL;
7224 r600_ring_init(rdev, ring, 1024 * 1024);
7226 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7227 ring->ring_obj = NULL;
7228 r600_ring_init(rdev, ring, 1024 * 1024);
7230 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7231 ring->ring_obj = NULL;
7232 r600_ring_init(rdev, ring, 1024 * 1024);
7234 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
7235 ring->ring_obj = NULL;
7236 r600_ring_init(rdev, ring, 64 * 1024);
7238 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
7239 ring->ring_obj = NULL;
7240 r600_ring_init(rdev, ring, 64 * 1024);
7245 rdev->ih.ring_obj = NULL;
7246 r600_ih_ring_init(rdev, 64 * 1024);
7248 r = r600_pcie_gart_init(rdev);
7252 rdev->accel_working = true;
7253 r = si_startup(rdev);
7255 dev_err(rdev->dev, "disabling GPU acceleration\n");
7257 cayman_dma_fini(rdev);
7259 sumo_rlc_fini(rdev);
7260 radeon_wb_fini(rdev);
7261 radeon_ib_pool_fini(rdev);
7262 radeon_vm_manager_fini(rdev);
7263 radeon_irq_kms_fini(rdev);
7264 si_pcie_gart_fini(rdev);
7265 rdev->accel_working = false;
7268 /* Don't start up if the MC ucode is missing.
7269 * The default clocks and voltages before the MC ucode
7270 * is loaded are not suffient for advanced operations.
7273 DRM_ERROR("radeon: MC ucode required for NI+.\n");
7280 void si_fini(struct radeon_device *rdev)
7282 radeon_pm_fini(rdev);
7284 cayman_dma_fini(rdev);
7288 sumo_rlc_fini(rdev);
7289 radeon_wb_fini(rdev);
7290 radeon_vm_manager_fini(rdev);
7291 radeon_ib_pool_fini(rdev);
7292 radeon_irq_kms_fini(rdev);
7293 if (rdev->has_uvd) {
7294 uvd_v1_0_fini(rdev);
7295 radeon_uvd_fini(rdev);
7298 radeon_vce_fini(rdev);
7299 si_pcie_gart_fini(rdev);
7300 r600_vram_scratch_fini(rdev);
7301 radeon_gem_fini(rdev);
7302 radeon_fence_driver_fini(rdev);
7303 radeon_bo_fini(rdev);
7304 radeon_atombios_fini(rdev);
7310 * si_get_gpu_clock_counter - return GPU clock counter snapshot
7312 * @rdev: radeon_device pointer
7314 * Fetches a GPU clock counter snapshot (SI).
7315 * Returns the 64 bit clock counter snapshot.
7317 uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev)
7321 mutex_lock(&rdev->gpu_clock_mutex);
7322 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
7323 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
7324 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
7325 mutex_unlock(&rdev->gpu_clock_mutex);
7329 int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
7331 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
7334 /* bypass vclk and dclk with bclk */
7335 WREG32_P(CG_UPLL_FUNC_CNTL_2,
7336 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
7337 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
7339 /* put PLL in bypass mode */
7340 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
7342 if (!vclk || !dclk) {
7343 /* keep the Bypass mode */
7347 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
7348 16384, 0x03FFFFFF, 0, 128, 5,
7349 &fb_div, &vclk_div, &dclk_div);
7353 /* set RESET_ANTI_MUX to 0 */
7354 WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
7356 /* set VCO_MODE to 1 */
7357 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
7359 /* disable sleep mode */
7360 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
7362 /* deassert UPLL_RESET */
7363 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
7367 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
7371 /* assert UPLL_RESET again */
7372 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
7374 /* disable spread spectrum. */
7375 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
7377 /* set feedback divider */
7378 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
7380 /* set ref divider to 0 */
7381 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
7383 if (fb_div < 307200)
7384 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
7386 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
7388 /* set PDIV_A and PDIV_B */
7389 WREG32_P(CG_UPLL_FUNC_CNTL_2,
7390 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
7391 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
7393 /* give the PLL some time to settle */
7396 /* deassert PLL_RESET */
7397 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
7401 /* switch from bypass mode to normal mode */
7402 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
7404 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
7408 /* switch VCLK and DCLK selection */
7409 WREG32_P(CG_UPLL_FUNC_CNTL_2,
7410 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
7411 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
7418 static void si_pcie_gen3_enable(struct radeon_device *rdev)
7420 struct pci_dev *root = rdev->pdev->bus->self;
7421 int bridge_pos, gpu_pos;
7422 u32 speed_cntl, mask, current_data_rate;
7426 if (pci_is_root_bus(rdev->pdev->bus))
7429 if (radeon_pcie_gen2 == 0)
7432 if (rdev->flags & RADEON_IS_IGP)
7435 if (!(rdev->flags & RADEON_IS_PCIE))
7438 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
7442 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
7445 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7446 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
7447 LC_CURRENT_DATA_RATE_SHIFT;
7448 if (mask & DRM_PCIE_SPEED_80) {
7449 if (current_data_rate == 2) {
7450 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
7453 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
7454 } else if (mask & DRM_PCIE_SPEED_50) {
7455 if (current_data_rate == 1) {
7456 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
7459 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
7462 bridge_pos = pci_pcie_cap(root);
7466 gpu_pos = pci_pcie_cap(rdev->pdev);
7470 if (mask & DRM_PCIE_SPEED_80) {
7471 /* re-try equalization if gen3 is not already enabled */
7472 if (current_data_rate != 2) {
7473 u16 bridge_cfg, gpu_cfg;
7474 u16 bridge_cfg2, gpu_cfg2;
7475 u32 max_lw, current_lw, tmp;
7477 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
7478 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
7480 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
7481 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
7483 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
7484 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
7486 tmp = RREG32_PCIE(PCIE_LC_STATUS1);
7487 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
7488 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
7490 if (current_lw < max_lw) {
7491 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
7492 if (tmp & LC_RENEGOTIATION_SUPPORT) {
7493 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
7494 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
7495 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
7496 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
7500 for (i = 0; i < 10; i++) {
7502 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
7503 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
7506 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
7507 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
7509 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
7510 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
7512 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7513 tmp |= LC_SET_QUIESCE;
7514 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7516 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7518 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7523 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
7524 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
7525 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
7526 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
7528 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
7529 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
7530 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
7531 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
7534 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
7535 tmp16 &= ~((1 << 4) | (7 << 9));
7536 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
7537 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
7539 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
7540 tmp16 &= ~((1 << 4) | (7 << 9));
7541 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
7542 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
7544 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7545 tmp &= ~LC_SET_QUIESCE;
7546 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7551 /* set the link speed */
7552 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
7553 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
7554 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
7556 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
7558 if (mask & DRM_PCIE_SPEED_80)
7559 tmp16 |= 3; /* gen3 */
7560 else if (mask & DRM_PCIE_SPEED_50)
7561 tmp16 |= 2; /* gen2 */
7563 tmp16 |= 1; /* gen1 */
7564 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
7566 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7567 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
7568 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
7570 for (i = 0; i < rdev->usec_timeout; i++) {
7571 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7572 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
7578 static void si_program_aspm(struct radeon_device *rdev)
7581 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
7582 bool disable_clkreq = false;
7584 if (radeon_aspm == 0)
7587 if (!(rdev->flags & RADEON_IS_PCIE))
7590 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7591 data &= ~LC_XMIT_N_FTS_MASK;
7592 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
7594 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
7596 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
7597 data |= LC_GO_TO_RECOVERY;
7599 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
7601 orig = data = RREG32_PCIE(PCIE_P_CNTL);
7602 data |= P_IGNORE_EDB_ERR;
7604 WREG32_PCIE(PCIE_P_CNTL, data);
7606 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7607 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
7608 data |= LC_PMI_TO_L1_DIS;
7610 data |= LC_L0S_INACTIVITY(7);
7613 data |= LC_L1_INACTIVITY(7);
7614 data &= ~LC_PMI_TO_L1_DIS;
7616 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7618 if (!disable_plloff_in_l1) {
7619 bool clk_req_support;
7621 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7622 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7623 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7625 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7627 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7628 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7629 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7631 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7633 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7634 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7635 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7637 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7639 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7640 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7641 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7643 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7645 if ((rdev->family != CHIP_OLAND) && (rdev->family != CHIP_HAINAN)) {
7646 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7647 data &= ~PLL_RAMP_UP_TIME_0_MASK;
7649 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7651 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7652 data &= ~PLL_RAMP_UP_TIME_1_MASK;
7654 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7656 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2);
7657 data &= ~PLL_RAMP_UP_TIME_2_MASK;
7659 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2, data);
7661 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3);
7662 data &= ~PLL_RAMP_UP_TIME_3_MASK;
7664 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3, data);
7666 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7667 data &= ~PLL_RAMP_UP_TIME_0_MASK;
7669 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7671 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7672 data &= ~PLL_RAMP_UP_TIME_1_MASK;
7674 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7676 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2);
7677 data &= ~PLL_RAMP_UP_TIME_2_MASK;
7679 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2, data);
7681 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3);
7682 data &= ~PLL_RAMP_UP_TIME_3_MASK;
7684 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3, data);
7686 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
7687 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
7688 data |= LC_DYN_LANES_PWR_STATE(3);
7690 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
7692 orig = data = RREG32_PIF_PHY0(PB0_PIF_CNTL);
7693 data &= ~LS2_EXIT_TIME_MASK;
7694 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7695 data |= LS2_EXIT_TIME(5);
7697 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
7699 orig = data = RREG32_PIF_PHY1(PB1_PIF_CNTL);
7700 data &= ~LS2_EXIT_TIME_MASK;
7701 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7702 data |= LS2_EXIT_TIME(5);
7704 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
7706 if (!disable_clkreq &&
7707 !pci_is_root_bus(rdev->pdev->bus)) {
7708 struct pci_dev *root = rdev->pdev->bus->self;
7711 clk_req_support = false;
7712 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
7713 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
7714 clk_req_support = true;
7716 clk_req_support = false;
7719 if (clk_req_support) {
7720 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
7721 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
7723 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
7725 orig = data = RREG32(THM_CLK_CNTL);
7726 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
7727 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
7729 WREG32(THM_CLK_CNTL, data);
7731 orig = data = RREG32(MISC_CLK_CNTL);
7732 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
7733 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
7735 WREG32(MISC_CLK_CNTL, data);
7737 orig = data = RREG32(CG_CLKPIN_CNTL);
7738 data &= ~BCLK_AS_XCLK;
7740 WREG32(CG_CLKPIN_CNTL, data);
7742 orig = data = RREG32(CG_CLKPIN_CNTL_2);
7743 data &= ~FORCE_BIF_REFCLK_EN;
7745 WREG32(CG_CLKPIN_CNTL_2, data);
7747 orig = data = RREG32(MPLL_BYPASSCLK_SEL);
7748 data &= ~MPLL_CLKOUT_SEL_MASK;
7749 data |= MPLL_CLKOUT_SEL(4);
7751 WREG32(MPLL_BYPASSCLK_SEL, data);
7753 orig = data = RREG32(SPLL_CNTL_MODE);
7754 data &= ~SPLL_REFCLK_SEL_MASK;
7756 WREG32(SPLL_CNTL_MODE, data);
7761 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7764 orig = data = RREG32_PCIE(PCIE_CNTL2);
7765 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
7767 WREG32_PCIE(PCIE_CNTL2, data);
7770 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7771 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
7772 data = RREG32_PCIE(PCIE_LC_STATUS1);
7773 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
7774 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7775 data &= ~LC_L0S_INACTIVITY_MASK;
7777 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7783 int si_vce_send_vcepll_ctlreq(struct radeon_device *rdev)
7787 /* make sure VCEPLL_CTLREQ is deasserted */
7788 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~UPLL_CTLREQ_MASK);
7792 /* assert UPLL_CTLREQ */
7793 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, UPLL_CTLREQ_MASK, ~UPLL_CTLREQ_MASK);
7795 /* wait for CTLACK and CTLACK2 to get asserted */
7796 for (i = 0; i < 100; ++i) {
7797 uint32_t mask = UPLL_CTLACK_MASK | UPLL_CTLACK2_MASK;
7798 if ((RREG32_SMC(CG_VCEPLL_FUNC_CNTL) & mask) == mask)
7803 /* deassert UPLL_CTLREQ */
7804 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~UPLL_CTLREQ_MASK);
7807 DRM_ERROR("Timeout setting UVD clocks!\n");
7814 int si_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
7816 unsigned fb_div = 0, evclk_div = 0, ecclk_div = 0;
7819 /* bypass evclk and ecclk with bclk */
7820 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
7821 EVCLK_SRC_SEL(1) | ECCLK_SRC_SEL(1),
7822 ~(EVCLK_SRC_SEL_MASK | ECCLK_SRC_SEL_MASK));
7824 /* put PLL in bypass mode */
7825 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_BYPASS_EN_MASK,
7826 ~VCEPLL_BYPASS_EN_MASK);
7828 if (!evclk || !ecclk) {
7829 /* keep the Bypass mode, put PLL to sleep */
7830 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_SLEEP_MASK,
7831 ~VCEPLL_SLEEP_MASK);
7835 r = radeon_uvd_calc_upll_dividers(rdev, evclk, ecclk, 125000, 250000,
7836 16384, 0x03FFFFFF, 0, 128, 5,
7837 &fb_div, &evclk_div, &ecclk_div);
7841 /* set RESET_ANTI_MUX to 0 */
7842 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
7844 /* set VCO_MODE to 1 */
7845 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_VCO_MODE_MASK,
7846 ~VCEPLL_VCO_MODE_MASK);
7848 /* toggle VCEPLL_SLEEP to 1 then back to 0 */
7849 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_SLEEP_MASK,
7850 ~VCEPLL_SLEEP_MASK);
7851 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_SLEEP_MASK);
7853 /* deassert VCEPLL_RESET */
7854 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_RESET_MASK);
7858 r = si_vce_send_vcepll_ctlreq(rdev);
7862 /* assert VCEPLL_RESET again */
7863 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_RESET_MASK, ~VCEPLL_RESET_MASK);
7865 /* disable spread spectrum. */
7866 WREG32_SMC_P(CG_VCEPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
7868 /* set feedback divider */
7869 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_3, VCEPLL_FB_DIV(fb_div), ~VCEPLL_FB_DIV_MASK);
7871 /* set ref divider to 0 */
7872 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_REF_DIV_MASK);
7874 /* set PDIV_A and PDIV_B */
7875 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
7876 VCEPLL_PDIV_A(evclk_div) | VCEPLL_PDIV_B(ecclk_div),
7877 ~(VCEPLL_PDIV_A_MASK | VCEPLL_PDIV_B_MASK));
7879 /* give the PLL some time to settle */
7882 /* deassert PLL_RESET */
7883 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_RESET_MASK);
7887 /* switch from bypass mode to normal mode */
7888 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_BYPASS_EN_MASK);
7890 r = si_vce_send_vcepll_ctlreq(rdev);
7894 /* switch VCLK and DCLK selection */
7895 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
7896 EVCLK_SRC_SEL(16) | ECCLK_SRC_SEL(16),
7897 ~(EVCLK_SRC_SEL_MASK | ECCLK_SRC_SEL_MASK));
projects / releases.git / blob