drivers/gpu/drm/radeon/ni.c

   1 /*
   2  * Copyright 2010 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24 #include <linux/firmware.h>
  25 #include <linux/slab.h>
  26 #include <linux/module.h>
  27 #include <drm/drmP.h>
  28 #include "radeon.h"
  29 #include "radeon_asic.h"
  30 #include "radeon_audio.h"
  31 #include <drm/radeon_drm.h>
  32 #include "nid.h"
  33 #include "atom.h"
  34 #include "ni_reg.h"
  35 #include "cayman_blit_shaders.h"
  36 #include "radeon_ucode.h"
  37 #include "clearstate_cayman.h"
  38
  39 /*
  40  * Indirect registers accessor
  41  */
  42 u32 tn_smc_rreg(struct radeon_device *rdev, u32 reg)
  43 {
  44         unsigned long flags;
  45         u32 r;
  46
  47         spin_lock_irqsave(&rdev->smc_idx_lock, flags);
  48         WREG32(TN_SMC_IND_INDEX_0, (reg));
  49         r = RREG32(TN_SMC_IND_DATA_0);
  50         spin_unlock_irqrestore(&rdev->smc_idx_lock, flags);
  51         return r;
  52 }
  53
  54 void tn_smc_wreg(struct radeon_device *rdev, u32 reg, u32 v)
  55 {
  56         unsigned long flags;
  57
  58         spin_lock_irqsave(&rdev->smc_idx_lock, flags);
  59         WREG32(TN_SMC_IND_INDEX_0, (reg));
  60         WREG32(TN_SMC_IND_DATA_0, (v));
  61         spin_unlock_irqrestore(&rdev->smc_idx_lock, flags);
  62 }
  63
  64 static const u32 tn_rlc_save_restore_register_list[] =
  65 {
  66         0x98fc,
  67         0x98f0,
  68         0x9834,
  69         0x9838,
  70         0x9870,
  71         0x9874,
  72         0x8a14,
  73         0x8b24,
  74         0x8bcc,
  75         0x8b10,
  76         0x8c30,
  77         0x8d00,
  78         0x8d04,
  79         0x8c00,
  80         0x8c04,
  81         0x8c10,
  82         0x8c14,
  83         0x8d8c,
  84         0x8cf0,
  85         0x8e38,
  86         0x9508,
  87         0x9688,
  88         0x9608,
  89         0x960c,
  90         0x9610,
  91         0x9614,
  92         0x88c4,
  93         0x8978,
  94         0x88d4,
  95         0x900c,
  96         0x9100,
  97         0x913c,
  98         0x90e8,
  99         0x9354,
 100         0xa008,
 101         0x98f8,
 102         0x9148,
 103         0x914c,
 104         0x3f94,
 105         0x98f4,
 106         0x9b7c,
 107         0x3f8c,
 108         0x8950,
 109         0x8954,
 110         0x8a18,
 111         0x8b28,
 112         0x9144,
 113         0x3f90,
 114         0x915c,
 115         0x9160,
 116         0x9178,
 117         0x917c,
 118         0x9180,
 119         0x918c,
 120         0x9190,
 121         0x9194,
 122         0x9198,
 123         0x919c,
 124         0x91a8,
 125         0x91ac,
 126         0x91b0,
 127         0x91b4,
 128         0x91b8,
 129         0x91c4,
 130         0x91c8,
 131         0x91cc,
 132         0x91d0,
 133         0x91d4,
 134         0x91e0,
 135         0x91e4,
 136         0x91ec,
 137         0x91f0,
 138         0x91f4,
 139         0x9200,
 140         0x9204,
 141         0x929c,
 142         0x8030,
 143         0x9150,
 144         0x9a60,
 145         0x920c,
 146         0x9210,
 147         0x9228,
 148         0x922c,
 149         0x9244,
 150         0x9248,
 151         0x91e8,
 152         0x9294,
 153         0x9208,
 154         0x9224,
 155         0x9240,
 156         0x9220,
 157         0x923c,
 158         0x9258,
 159         0x9744,
 160         0xa200,
 161         0xa204,
 162         0xa208,
 163         0xa20c,
 164         0x8d58,
 165         0x9030,
 166         0x9034,
 167         0x9038,
 168         0x903c,
 169         0x9040,
 170         0x9654,
 171         0x897c,
 172         0xa210,
 173         0xa214,
 174         0x9868,
 175         0xa02c,
 176         0x9664,
 177         0x9698,
 178         0x949c,
 179         0x8e10,
 180         0x8e18,
 181         0x8c50,
 182         0x8c58,
 183         0x8c60,
 184         0x8c68,
 185         0x89b4,
 186         0x9830,
 187         0x802c,
 188 };
 189
 190 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
 191 extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
 192 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
 193 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
 194 extern int evergreen_mc_wait_for_idle(struct radeon_device *rdev);
 195 extern void evergreen_mc_program(struct radeon_device *rdev);
 196 extern void evergreen_irq_suspend(struct radeon_device *rdev);
 197 extern int evergreen_mc_init(struct radeon_device *rdev);
 198 extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
 199 extern void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
 200 extern void evergreen_program_aspm(struct radeon_device *rdev);
 201 extern void sumo_rlc_fini(struct radeon_device *rdev);
 202 extern int sumo_rlc_init(struct radeon_device *rdev);
 203 extern void evergreen_gpu_pci_config_reset(struct radeon_device *rdev);
 204
 205 /* Firmware Names */
 206 /*(DEBLOBBED)*/
 207
 208
 209 static const u32 cayman_golden_registers2[] =
 210 {
 211         0x3e5c, 0xffffffff, 0x00000000,
 212         0x3e48, 0xffffffff, 0x00000000,
 213         0x3e4c, 0xffffffff, 0x00000000,
 214         0x3e64, 0xffffffff, 0x00000000,
 215         0x3e50, 0xffffffff, 0x00000000,
 216         0x3e60, 0xffffffff, 0x00000000
 217 };
 218
 219 static const u32 cayman_golden_registers[] =
 220 {
 221         0x5eb4, 0xffffffff, 0x00000002,
 222         0x5e78, 0x8f311ff1, 0x001000f0,
 223         0x3f90, 0xffff0000, 0xff000000,
 224         0x9148, 0xffff0000, 0xff000000,
 225         0x3f94, 0xffff0000, 0xff000000,
 226         0x914c, 0xffff0000, 0xff000000,
 227         0xc78, 0x00000080, 0x00000080,
 228         0xbd4, 0x70073777, 0x00011003,
 229         0xd02c, 0xbfffff1f, 0x08421000,
 230         0xd0b8, 0x73773777, 0x02011003,
 231         0x5bc0, 0x00200000, 0x50100000,
 232         0x98f8, 0x33773777, 0x02011003,
 233         0x98fc, 0xffffffff, 0x76541032,
 234         0x7030, 0x31000311, 0x00000011,
 235         0x2f48, 0x33773777, 0x42010001,
 236         0x6b28, 0x00000010, 0x00000012,
 237         0x7728, 0x00000010, 0x00000012,
 238         0x10328, 0x00000010, 0x00000012,
 239         0x10f28, 0x00000010, 0x00000012,
 240         0x11b28, 0x00000010, 0x00000012,
 241         0x12728, 0x00000010, 0x00000012,
 242         0x240c, 0x000007ff, 0x00000000,
 243         0x8a14, 0xf000001f, 0x00000007,
 244         0x8b24, 0x3fff3fff, 0x00ff0fff,
 245         0x8b10, 0x0000ff0f, 0x00000000,
 246         0x28a4c, 0x07ffffff, 0x06000000,
 247         0x10c, 0x00000001, 0x00010003,
 248         0xa02c, 0xffffffff, 0x0000009b,
 249         0x913c, 0x0000010f, 0x01000100,
 250         0x8c04, 0xf8ff00ff, 0x40600060,
 251         0x28350, 0x00000f01, 0x00000000,
 252         0x9508, 0x3700001f, 0x00000002,
 253         0x960c, 0xffffffff, 0x54763210,
 254         0x88c4, 0x001f3ae3, 0x00000082,
 255         0x88d0, 0xffffffff, 0x0f40df40,
 256         0x88d4, 0x0000001f, 0x00000010,
 257         0x8974, 0xffffffff, 0x00000000
 258 };
 259
 260 static const u32 dvst_golden_registers2[] =
 261 {
 262         0x8f8, 0xffffffff, 0,
 263         0x8fc, 0x00380000, 0,
 264         0x8f8, 0xffffffff, 1,
 265         0x8fc, 0x0e000000, 0
 266 };
 267
 268 static const u32 dvst_golden_registers[] =
 269 {
 270         0x690, 0x3fff3fff, 0x20c00033,
 271         0x918c, 0x0fff0fff, 0x00010006,
 272         0x91a8, 0x0fff0fff, 0x00010006,
 273         0x9150, 0xffffdfff, 0x6e944040,
 274         0x917c, 0x0fff0fff, 0x00030002,
 275         0x9198, 0x0fff0fff, 0x00030002,
 276         0x915c, 0x0fff0fff, 0x00010000,
 277         0x3f90, 0xffff0001, 0xff000000,
 278         0x9178, 0x0fff0fff, 0x00070000,
 279         0x9194, 0x0fff0fff, 0x00070000,
 280         0x9148, 0xffff0001, 0xff000000,
 281         0x9190, 0x0fff0fff, 0x00090008,
 282         0x91ac, 0x0fff0fff, 0x00090008,
 283         0x3f94, 0xffff0000, 0xff000000,
 284         0x914c, 0xffff0000, 0xff000000,
 285         0x929c, 0x00000fff, 0x00000001,
 286         0x55e4, 0xff607fff, 0xfc000100,
 287         0x8a18, 0xff000fff, 0x00000100,
 288         0x8b28, 0xff000fff, 0x00000100,
 289         0x9144, 0xfffc0fff, 0x00000100,
 290         0x6ed8, 0x00010101, 0x00010000,
 291         0x9830, 0xffffffff, 0x00000000,
 292         0x9834, 0xf00fffff, 0x00000400,
 293         0x9838, 0xfffffffe, 0x00000000,
 294         0xd0c0, 0xff000fff, 0x00000100,
 295         0xd02c, 0xbfffff1f, 0x08421000,
 296         0xd0b8, 0x73773777, 0x12010001,
 297         0x5bb0, 0x000000f0, 0x00000070,
 298         0x98f8, 0x73773777, 0x12010001,
 299         0x98fc, 0xffffffff, 0x00000010,
 300         0x9b7c, 0x00ff0000, 0x00fc0000,
 301         0x8030, 0x00001f0f, 0x0000100a,
 302         0x2f48, 0x73773777, 0x12010001,
 303         0x2408, 0x00030000, 0x000c007f,
 304         0x8a14, 0xf000003f, 0x00000007,
 305         0x8b24, 0x3fff3fff, 0x00ff0fff,
 306         0x8b10, 0x0000ff0f, 0x00000000,
 307         0x28a4c, 0x07ffffff, 0x06000000,
 308         0x4d8, 0x00000fff, 0x00000100,
 309         0xa008, 0xffffffff, 0x00010000,
 310         0x913c, 0xffff03ff, 0x01000100,
 311         0x8c00, 0x000000ff, 0x00000003,
 312         0x8c04, 0xf8ff00ff, 0x40600060,
 313         0x8cf0, 0x1fff1fff, 0x08e00410,
 314         0x28350, 0x00000f01, 0x00000000,
 315         0x9508, 0xf700071f, 0x00000002,
 316         0x960c, 0xffffffff, 0x54763210,
 317         0x20ef8, 0x01ff01ff, 0x00000002,
 318         0x20e98, 0xfffffbff, 0x00200000,
 319         0x2015c, 0xffffffff, 0x00000f40,
 320         0x88c4, 0x001f3ae3, 0x00000082,
 321         0x8978, 0x3fffffff, 0x04050140,
 322         0x88d4, 0x0000001f, 0x00000010,
 323         0x8974, 0xffffffff, 0x00000000
 324 };
 325
 326 static const u32 scrapper_golden_registers[] =
 327 {
 328         0x690, 0x3fff3fff, 0x20c00033,
 329         0x918c, 0x0fff0fff, 0x00010006,
 330         0x918c, 0x0fff0fff, 0x00010006,
 331         0x91a8, 0x0fff0fff, 0x00010006,
 332         0x91a8, 0x0fff0fff, 0x00010006,
 333         0x9150, 0xffffdfff, 0x6e944040,
 334         0x9150, 0xffffdfff, 0x6e944040,
 335         0x917c, 0x0fff0fff, 0x00030002,
 336         0x917c, 0x0fff0fff, 0x00030002,
 337         0x9198, 0x0fff0fff, 0x00030002,
 338         0x9198, 0x0fff0fff, 0x00030002,
 339         0x915c, 0x0fff0fff, 0x00010000,
 340         0x915c, 0x0fff0fff, 0x00010000,
 341         0x3f90, 0xffff0001, 0xff000000,
 342         0x3f90, 0xffff0001, 0xff000000,
 343         0x9178, 0x0fff0fff, 0x00070000,
 344         0x9178, 0x0fff0fff, 0x00070000,
 345         0x9194, 0x0fff0fff, 0x00070000,
 346         0x9194, 0x0fff0fff, 0x00070000,
 347         0x9148, 0xffff0001, 0xff000000,
 348         0x9148, 0xffff0001, 0xff000000,
 349         0x9190, 0x0fff0fff, 0x00090008,
 350         0x9190, 0x0fff0fff, 0x00090008,
 351         0x91ac, 0x0fff0fff, 0x00090008,
 352         0x91ac, 0x0fff0fff, 0x00090008,
 353         0x3f94, 0xffff0000, 0xff000000,
 354         0x3f94, 0xffff0000, 0xff000000,
 355         0x914c, 0xffff0000, 0xff000000,
 356         0x914c, 0xffff0000, 0xff000000,
 357         0x929c, 0x00000fff, 0x00000001,
 358         0x929c, 0x00000fff, 0x00000001,
 359         0x55e4, 0xff607fff, 0xfc000100,
 360         0x8a18, 0xff000fff, 0x00000100,
 361         0x8a18, 0xff000fff, 0x00000100,
 362         0x8b28, 0xff000fff, 0x00000100,
 363         0x8b28, 0xff000fff, 0x00000100,
 364         0x9144, 0xfffc0fff, 0x00000100,
 365         0x9144, 0xfffc0fff, 0x00000100,
 366         0x6ed8, 0x00010101, 0x00010000,
 367         0x9830, 0xffffffff, 0x00000000,
 368         0x9830, 0xffffffff, 0x00000000,
 369         0x9834, 0xf00fffff, 0x00000400,
 370         0x9834, 0xf00fffff, 0x00000400,
 371         0x9838, 0xfffffffe, 0x00000000,
 372         0x9838, 0xfffffffe, 0x00000000,
 373         0xd0c0, 0xff000fff, 0x00000100,
 374         0xd02c, 0xbfffff1f, 0x08421000,
 375         0xd02c, 0xbfffff1f, 0x08421000,
 376         0xd0b8, 0x73773777, 0x12010001,
 377         0xd0b8, 0x73773777, 0x12010001,
 378         0x5bb0, 0x000000f0, 0x00000070,
 379         0x98f8, 0x73773777, 0x12010001,
 380         0x98f8, 0x73773777, 0x12010001,
 381         0x98fc, 0xffffffff, 0x00000010,
 382         0x98fc, 0xffffffff, 0x00000010,
 383         0x9b7c, 0x00ff0000, 0x00fc0000,
 384         0x9b7c, 0x00ff0000, 0x00fc0000,
 385         0x8030, 0x00001f0f, 0x0000100a,
 386         0x8030, 0x00001f0f, 0x0000100a,
 387         0x2f48, 0x73773777, 0x12010001,
 388         0x2f48, 0x73773777, 0x12010001,
 389         0x2408, 0x00030000, 0x000c007f,
 390         0x8a14, 0xf000003f, 0x00000007,
 391         0x8a14, 0xf000003f, 0x00000007,
 392         0x8b24, 0x3fff3fff, 0x00ff0fff,
 393         0x8b24, 0x3fff3fff, 0x00ff0fff,
 394         0x8b10, 0x0000ff0f, 0x00000000,
 395         0x8b10, 0x0000ff0f, 0x00000000,
 396         0x28a4c, 0x07ffffff, 0x06000000,
 397         0x28a4c, 0x07ffffff, 0x06000000,
 398         0x4d8, 0x00000fff, 0x00000100,
 399         0x4d8, 0x00000fff, 0x00000100,
 400         0xa008, 0xffffffff, 0x00010000,
 401         0xa008, 0xffffffff, 0x00010000,
 402         0x913c, 0xffff03ff, 0x01000100,
 403         0x913c, 0xffff03ff, 0x01000100,
 404         0x90e8, 0x001fffff, 0x010400c0,
 405         0x8c00, 0x000000ff, 0x00000003,
 406         0x8c00, 0x000000ff, 0x00000003,
 407         0x8c04, 0xf8ff00ff, 0x40600060,
 408         0x8c04, 0xf8ff00ff, 0x40600060,
 409         0x8c30, 0x0000000f, 0x00040005,
 410         0x8cf0, 0x1fff1fff, 0x08e00410,
 411         0x8cf0, 0x1fff1fff, 0x08e00410,
 412         0x900c, 0x00ffffff, 0x0017071f,
 413         0x28350, 0x00000f01, 0x00000000,
 414         0x28350, 0x00000f01, 0x00000000,
 415         0x9508, 0xf700071f, 0x00000002,
 416         0x9508, 0xf700071f, 0x00000002,
 417         0x9688, 0x00300000, 0x0017000f,
 418         0x960c, 0xffffffff, 0x54763210,
 419         0x960c, 0xffffffff, 0x54763210,
 420         0x20ef8, 0x01ff01ff, 0x00000002,
 421         0x20e98, 0xfffffbff, 0x00200000,
 422         0x2015c, 0xffffffff, 0x00000f40,
 423         0x88c4, 0x001f3ae3, 0x00000082,
 424         0x88c4, 0x001f3ae3, 0x00000082,
 425         0x8978, 0x3fffffff, 0x04050140,
 426         0x8978, 0x3fffffff, 0x04050140,
 427         0x88d4, 0x0000001f, 0x00000010,
 428         0x88d4, 0x0000001f, 0x00000010,
 429         0x8974, 0xffffffff, 0x00000000,
 430         0x8974, 0xffffffff, 0x00000000
 431 };
 432
 433 static void ni_init_golden_registers(struct radeon_device *rdev)
 434 {
 435         switch (rdev->family) {
 436         case CHIP_CAYMAN:
 437                 radeon_program_register_sequence(rdev,
 438                                                  cayman_golden_registers,
 439                                                  (const u32)ARRAY_SIZE(cayman_golden_registers));
 440                 radeon_program_register_sequence(rdev,
 441                                                  cayman_golden_registers2,
 442                                                  (const u32)ARRAY_SIZE(cayman_golden_registers2));
 443                 break;
 444         case CHIP_ARUBA:
 445                 if ((rdev->pdev->device == 0x9900) ||
 446                     (rdev->pdev->device == 0x9901) ||
 447                     (rdev->pdev->device == 0x9903) ||
 448                     (rdev->pdev->device == 0x9904) ||
 449                     (rdev->pdev->device == 0x9905) ||
 450                     (rdev->pdev->device == 0x9906) ||
 451                     (rdev->pdev->device == 0x9907) ||
 452                     (rdev->pdev->device == 0x9908) ||
 453                     (rdev->pdev->device == 0x9909) ||
 454                     (rdev->pdev->device == 0x990A) ||
 455                     (rdev->pdev->device == 0x990B) ||
 456                     (rdev->pdev->device == 0x990C) ||
 457                     (rdev->pdev->device == 0x990D) ||
 458                     (rdev->pdev->device == 0x990E) ||
 459                     (rdev->pdev->device == 0x990F) ||
 460                     (rdev->pdev->device == 0x9910) ||
 461                     (rdev->pdev->device == 0x9913) ||
 462                     (rdev->pdev->device == 0x9917) ||
 463                     (rdev->pdev->device == 0x9918)) {
 464                         radeon_program_register_sequence(rdev,
 465                                                          dvst_golden_registers,
 466                                                          (const u32)ARRAY_SIZE(dvst_golden_registers));
 467                         radeon_program_register_sequence(rdev,
 468                                                          dvst_golden_registers2,
 469                                                          (const u32)ARRAY_SIZE(dvst_golden_registers2));
 470                 } else {
 471                         radeon_program_register_sequence(rdev,
 472                                                          scrapper_golden_registers,
 473                                                          (const u32)ARRAY_SIZE(scrapper_golden_registers));
 474                         radeon_program_register_sequence(rdev,
 475                                                          dvst_golden_registers2,
 476                                                          (const u32)ARRAY_SIZE(dvst_golden_registers2));
 477                 }
 478                 break;
 479         default:
 480                 break;
 481         }
 482 }
 483
 484 #define BTC_IO_MC_REGS_SIZE 29
 485
 486 static const u32 barts_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 487         {0x00000077, 0xff010100},
 488         {0x00000078, 0x00000000},
 489         {0x00000079, 0x00001434},
 490         {0x0000007a, 0xcc08ec08},
 491         {0x0000007b, 0x00040000},
 492         {0x0000007c, 0x000080c0},
 493         {0x0000007d, 0x09000000},
 494         {0x0000007e, 0x00210404},
 495         {0x00000081, 0x08a8e800},
 496         {0x00000082, 0x00030444},
 497         {0x00000083, 0x00000000},
 498         {0x00000085, 0x00000001},
 499         {0x00000086, 0x00000002},
 500         {0x00000087, 0x48490000},
 501         {0x00000088, 0x20244647},
 502         {0x00000089, 0x00000005},
 503         {0x0000008b, 0x66030000},
 504         {0x0000008c, 0x00006603},
 505         {0x0000008d, 0x00000100},
 506         {0x0000008f, 0x00001c0a},
 507         {0x00000090, 0xff000001},
 508         {0x00000094, 0x00101101},
 509         {0x00000095, 0x00000fff},
 510         {0x00000096, 0x00116fff},
 511         {0x00000097, 0x60010000},
 512         {0x00000098, 0x10010000},
 513         {0x00000099, 0x00006000},
 514         {0x0000009a, 0x00001000},
 515         {0x0000009f, 0x00946a00}
 516 };
 517
 518 static const u32 turks_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 519         {0x00000077, 0xff010100},
 520         {0x00000078, 0x00000000},
 521         {0x00000079, 0x00001434},
 522         {0x0000007a, 0xcc08ec08},
 523         {0x0000007b, 0x00040000},
 524         {0x0000007c, 0x000080c0},
 525         {0x0000007d, 0x09000000},
 526         {0x0000007e, 0x00210404},
 527         {0x00000081, 0x08a8e800},
 528         {0x00000082, 0x00030444},
 529         {0x00000083, 0x00000000},
 530         {0x00000085, 0x00000001},
 531         {0x00000086, 0x00000002},
 532         {0x00000087, 0x48490000},
 533         {0x00000088, 0x20244647},
 534         {0x00000089, 0x00000005},
 535         {0x0000008b, 0x66030000},
 536         {0x0000008c, 0x00006603},
 537         {0x0000008d, 0x00000100},
 538         {0x0000008f, 0x00001c0a},
 539         {0x00000090, 0xff000001},
 540         {0x00000094, 0x00101101},
 541         {0x00000095, 0x00000fff},
 542         {0x00000096, 0x00116fff},
 543         {0x00000097, 0x60010000},
 544         {0x00000098, 0x10010000},
 545         {0x00000099, 0x00006000},
 546         {0x0000009a, 0x00001000},
 547         {0x0000009f, 0x00936a00}
 548 };
 549
 550 static const u32 caicos_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 551         {0x00000077, 0xff010100},
 552         {0x00000078, 0x00000000},
 553         {0x00000079, 0x00001434},
 554         {0x0000007a, 0xcc08ec08},
 555         {0x0000007b, 0x00040000},
 556         {0x0000007c, 0x000080c0},
 557         {0x0000007d, 0x09000000},
 558         {0x0000007e, 0x00210404},
 559         {0x00000081, 0x08a8e800},
 560         {0x00000082, 0x00030444},
 561         {0x00000083, 0x00000000},
 562         {0x00000085, 0x00000001},
 563         {0x00000086, 0x00000002},
 564         {0x00000087, 0x48490000},
 565         {0x00000088, 0x20244647},
 566         {0x00000089, 0x00000005},
 567         {0x0000008b, 0x66030000},
 568         {0x0000008c, 0x00006603},
 569         {0x0000008d, 0x00000100},
 570         {0x0000008f, 0x00001c0a},
 571         {0x00000090, 0xff000001},
 572         {0x00000094, 0x00101101},
 573         {0x00000095, 0x00000fff},
 574         {0x00000096, 0x00116fff},
 575         {0x00000097, 0x60010000},
 576         {0x00000098, 0x10010000},
 577         {0x00000099, 0x00006000},
 578         {0x0000009a, 0x00001000},
 579         {0x0000009f, 0x00916a00}
 580 };
 581
 582 static const u32 cayman_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 583         {0x00000077, 0xff010100},
 584         {0x00000078, 0x00000000},
 585         {0x00000079, 0x00001434},
 586         {0x0000007a, 0xcc08ec08},
 587         {0x0000007b, 0x00040000},
 588         {0x0000007c, 0x000080c0},
 589         {0x0000007d, 0x09000000},
 590         {0x0000007e, 0x00210404},
 591         {0x00000081, 0x08a8e800},
 592         {0x00000082, 0x00030444},
 593         {0x00000083, 0x00000000},
 594         {0x00000085, 0x00000001},
 595         {0x00000086, 0x00000002},
 596         {0x00000087, 0x48490000},
 597         {0x00000088, 0x20244647},
 598         {0x00000089, 0x00000005},
 599         {0x0000008b, 0x66030000},
 600         {0x0000008c, 0x00006603},
 601         {0x0000008d, 0x00000100},
 602         {0x0000008f, 0x00001c0a},
 603         {0x00000090, 0xff000001},
 604         {0x00000094, 0x00101101},
 605         {0x00000095, 0x00000fff},
 606         {0x00000096, 0x00116fff},
 607         {0x00000097, 0x60010000},
 608         {0x00000098, 0x10010000},
 609         {0x00000099, 0x00006000},
 610         {0x0000009a, 0x00001000},
 611         {0x0000009f, 0x00976b00}
 612 };
 613
 614 int ni_mc_load_microcode(struct radeon_device *rdev)
 615 {
 616         const __be32 *fw_data;
 617         u32 mem_type, running, blackout = 0;
 618         u32 *io_mc_regs;
 619         int i, ucode_size, regs_size;
 620
 621         if (!rdev->mc_fw)
 622                 return -EINVAL;
 623
 624         switch (rdev->family) {
 625         case CHIP_BARTS:
 626                 io_mc_regs = (u32 *)&barts_io_mc_regs;
 627                 ucode_size = BTC_MC_UCODE_SIZE;
 628                 regs_size = BTC_IO_MC_REGS_SIZE;
 629                 break;
 630         case CHIP_TURKS:
 631                 io_mc_regs = (u32 *)&turks_io_mc_regs;
 632                 ucode_size = BTC_MC_UCODE_SIZE;
 633                 regs_size = BTC_IO_MC_REGS_SIZE;
 634                 break;
 635         case CHIP_CAICOS:
 636         default:
 637                 io_mc_regs = (u32 *)&caicos_io_mc_regs;
 638                 ucode_size = BTC_MC_UCODE_SIZE;
 639                 regs_size = BTC_IO_MC_REGS_SIZE;
 640                 break;
 641         case CHIP_CAYMAN:
 642                 io_mc_regs = (u32 *)&cayman_io_mc_regs;
 643                 ucode_size = CAYMAN_MC_UCODE_SIZE;
 644                 regs_size = BTC_IO_MC_REGS_SIZE;
 645                 break;
 646         }
 647
 648         mem_type = (RREG32(MC_SEQ_MISC0) & MC_SEQ_MISC0_GDDR5_MASK) >> MC_SEQ_MISC0_GDDR5_SHIFT;
 649         running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
 650
 651         if ((mem_type == MC_SEQ_MISC0_GDDR5_VALUE) && (running == 0)) {
 652                 if (running) {
 653                         blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
 654                         WREG32(MC_SHARED_BLACKOUT_CNTL, 1);
 655                 }
 656
 657                 /* reset the engine and set to writable */
 658                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
 659                 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
 660
 661                 /* load mc io regs */
 662                 for (i = 0; i < regs_size; i++) {
 663                         WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
 664                         WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
 665                 }
 666                 /* load the MC ucode */
 667                 fw_data = (const __be32 *)rdev->mc_fw->data;
 668                 for (i = 0; i < ucode_size; i++)
 669                         WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
 670
 671                 /* put the engine back into the active state */
 672                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
 673                 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
 674                 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
 675
 676                 /* wait for training to complete */
 677                 for (i = 0; i < rdev->usec_timeout; i++) {
 678                         if (RREG32(MC_IO_PAD_CNTL_D0) & MEM_FALL_OUT_CMD)
 679                                 break;
 680                         udelay(1);
 681                 }
 682
 683                 if (running)
 684                         WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
 685         }
 686
 687         return 0;
 688 }
 689
 690 int ni_init_microcode(struct radeon_device *rdev)
 691 {
 692         const char *chip_name;
 693         const char *rlc_chip_name;
 694         size_t pfp_req_size, me_req_size, rlc_req_size, mc_req_size;
 695         size_t smc_req_size = 0;
 696         char fw_name[30];
 697         int err;
 698
 699         DRM_DEBUG("\n");
 700
 701         switch (rdev->family) {
 702         case CHIP_BARTS:
 703                 chip_name = "BARTS";
 704                 rlc_chip_name = "BTC";
 705                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 706                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 707                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 708                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
 709                 smc_req_size = ALIGN(BARTS_SMC_UCODE_SIZE, 4);
 710                 break;
 711         case CHIP_TURKS:
 712                 chip_name = "TURKS";
 713                 rlc_chip_name = "BTC";
 714                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 715                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 716                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 717                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
 718                 smc_req_size = ALIGN(TURKS_SMC_UCODE_SIZE, 4);
 719                 break;
 720         case CHIP_CAICOS:
 721                 chip_name = "CAICOS";
 722                 rlc_chip_name = "BTC";
 723                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 724                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 725                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 726                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
 727                 smc_req_size = ALIGN(CAICOS_SMC_UCODE_SIZE, 4);
 728                 break;
 729         case CHIP_CAYMAN:
 730                 chip_name = "CAYMAN";
 731                 rlc_chip_name = "CAYMAN";
 732                 pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4;
 733                 me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
 734                 rlc_req_size = CAYMAN_RLC_UCODE_SIZE * 4;
 735                 mc_req_size = CAYMAN_MC_UCODE_SIZE * 4;
 736                 smc_req_size = ALIGN(CAYMAN_SMC_UCODE_SIZE, 4);
 737                 break;
 738         case CHIP_ARUBA:
 739                 chip_name = "ARUBA";
 740                 rlc_chip_name = "ARUBA";
 741                 /* pfp/me same size as CAYMAN */
 742                 pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4;
 743                 me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
 744                 rlc_req_size = ARUBA_RLC_UCODE_SIZE * 4;
 745                 mc_req_size = 0;
 746                 break;
 747         default: BUG();
 748         }
 749
 750         DRM_INFO("Loading %s Microcode\n", chip_name);
 751
 752         snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", chip_name);
 753         err = reject_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
 754         if (err)
 755                 goto out;
 756         if (rdev->pfp_fw->size != pfp_req_size) {
 757                 printk(KERN_ERR
 758                        "ni_cp: Bogus length %zu in firmware \"%s\"\n",
 759                        rdev->pfp_fw->size, fw_name);
 760                 err = -EINVAL;
 761                 goto out;
 762         }
 763
 764         snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", chip_name);
 765         err = reject_firmware(&rdev->me_fw, fw_name, rdev->dev);
 766         if (err)
 767                 goto out;
 768         if (rdev->me_fw->size != me_req_size) {
 769                 printk(KERN_ERR
 770                        "ni_cp: Bogus length %zu in firmware \"%s\"\n",
 771                        rdev->me_fw->size, fw_name);
 772                 err = -EINVAL;
 773         }
 774
 775         snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", rlc_chip_name);
 776         err = reject_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
 777         if (err)
 778                 goto out;
 779         if (rdev->rlc_fw->size != rlc_req_size) {
 780                 printk(KERN_ERR
 781                        "ni_rlc: Bogus length %zu in firmware \"%s\"\n",
 782                        rdev->rlc_fw->size, fw_name);
 783                 err = -EINVAL;
 784         }
 785
 786         /* no MC ucode on TN */
 787         if (!(rdev->flags & RADEON_IS_IGP)) {
 788                 snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", chip_name);
 789                 err = reject_firmware(&rdev->mc_fw, fw_name, rdev->dev);
 790                 if (err)
 791                         goto out;
 792                 if (rdev->mc_fw->size != mc_req_size) {
 793                         printk(KERN_ERR
 794                                "ni_mc: Bogus length %zu in firmware \"%s\"\n",
 795                                rdev->mc_fw->size, fw_name);
 796                         err = -EINVAL;
 797                 }
 798         }
 799
 800         if ((rdev->family >= CHIP_BARTS) && (rdev->family <= CHIP_CAYMAN)) {
 801                 snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", chip_name);
 802                 err = reject_firmware(&rdev->smc_fw, fw_name, rdev->dev);
 803                 if (err) {
 804                         printk(KERN_ERR
 805                                "smc: error loading firmware \"%s\"\n",
 806                                fw_name);
 807                         release_firmware(rdev->smc_fw);
 808                         rdev->smc_fw = NULL;
 809                         err = 0;
 810                 } else if (rdev->smc_fw->size != smc_req_size) {
 811                         printk(KERN_ERR
 812                                "ni_mc: Bogus length %zu in firmware \"%s\"\n",
 813                                rdev->mc_fw->size, fw_name);
 814                         err = -EINVAL;
 815                 }
 816         }
 817
 818 out:
 819         if (err) {
 820                 if (err != -EINVAL)
 821                         printk(KERN_ERR
 822                                "ni_cp: Failed to load firmware \"%s\"\n",
 823                                fw_name);
 824                 release_firmware(rdev->pfp_fw);
 825                 rdev->pfp_fw = NULL;
 826                 release_firmware(rdev->me_fw);
 827                 rdev->me_fw = NULL;
 828                 release_firmware(rdev->rlc_fw);
 829                 rdev->rlc_fw = NULL;
 830                 release_firmware(rdev->mc_fw);
 831                 rdev->mc_fw = NULL;
 832         }
 833         return err;
 834 }
 835
 836 /**
 837  * cayman_get_allowed_info_register - fetch the register for the info ioctl
 838  *
 839  * @rdev: radeon_device pointer
 840  * @reg: register offset in bytes
 841  * @val: register value
 842  *
 843  * Returns 0 for success or -EINVAL for an invalid register
 844  *
 845  */
 846 int cayman_get_allowed_info_register(struct radeon_device *rdev,
 847                                      u32 reg, u32 *val)
 848 {
 849         switch (reg) {
 850         case GRBM_STATUS:
 851         case GRBM_STATUS_SE0:
 852         case GRBM_STATUS_SE1:
 853         case SRBM_STATUS:
 854         case SRBM_STATUS2:
 855         case (DMA_STATUS_REG + DMA0_REGISTER_OFFSET):
 856         case (DMA_STATUS_REG + DMA1_REGISTER_OFFSET):
 857         case UVD_STATUS:
 858                 *val = RREG32(reg);
 859                 return 0;
 860         default:
 861                 return -EINVAL;
 862         }
 863 }
 864
 865 int tn_get_temp(struct radeon_device *rdev)
 866 {
 867         u32 temp = RREG32_SMC(TN_CURRENT_GNB_TEMP) & 0x7ff;
 868         int actual_temp = (temp / 8) - 49;
 869
 870         return actual_temp * 1000;
 871 }
 872
 873 /*
 874  * Core functions
 875  */
 876 static void cayman_gpu_init(struct radeon_device *rdev)
 877 {
 878         u32 gb_addr_config = 0;
 879         u32 mc_shared_chmap, mc_arb_ramcfg;
 880         u32 cgts_tcc_disable;
 881         u32 sx_debug_1;
 882         u32 smx_dc_ctl0;
 883         u32 cgts_sm_ctrl_reg;
 884         u32 hdp_host_path_cntl;
 885         u32 tmp;
 886         u32 disabled_rb_mask;
 887         int i, j;
 888
 889         switch (rdev->family) {
 890         case CHIP_CAYMAN:
 891                 rdev->config.cayman.max_shader_engines = 2;
 892                 rdev->config.cayman.max_pipes_per_simd = 4;
 893                 rdev->config.cayman.max_tile_pipes = 8;
 894                 rdev->config.cayman.max_simds_per_se = 12;
 895                 rdev->config.cayman.max_backends_per_se = 4;
 896                 rdev->config.cayman.max_texture_channel_caches = 8;
 897                 rdev->config.cayman.max_gprs = 256;
 898                 rdev->config.cayman.max_threads = 256;
 899                 rdev->config.cayman.max_gs_threads = 32;
 900                 rdev->config.cayman.max_stack_entries = 512;
 901                 rdev->config.cayman.sx_num_of_sets = 8;
 902                 rdev->config.cayman.sx_max_export_size = 256;
 903                 rdev->config.cayman.sx_max_export_pos_size = 64;
 904                 rdev->config.cayman.sx_max_export_smx_size = 192;
 905                 rdev->config.cayman.max_hw_contexts = 8;
 906                 rdev->config.cayman.sq_num_cf_insts = 2;
 907
 908                 rdev->config.cayman.sc_prim_fifo_size = 0x100;
 909                 rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;
 910                 rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;
 911                 gb_addr_config = CAYMAN_GB_ADDR_CONFIG_GOLDEN;
 912                 break;
 913         case CHIP_ARUBA:
 914         default:
 915                 rdev->config.cayman.max_shader_engines = 1;
 916                 rdev->config.cayman.max_pipes_per_simd = 4;
 917                 rdev->config.cayman.max_tile_pipes = 2;
 918                 if ((rdev->pdev->device == 0x9900) ||
 919                     (rdev->pdev->device == 0x9901) ||
 920                     (rdev->pdev->device == 0x9905) ||
 921                     (rdev->pdev->device == 0x9906) ||
 922                     (rdev->pdev->device == 0x9907) ||
 923                     (rdev->pdev->device == 0x9908) ||
 924                     (rdev->pdev->device == 0x9909) ||
 925                     (rdev->pdev->device == 0x990B) ||
 926                     (rdev->pdev->device == 0x990C) ||
 927                     (rdev->pdev->device == 0x990F) ||
 928                     (rdev->pdev->device == 0x9910) ||
 929                     (rdev->pdev->device == 0x9917) ||
 930                     (rdev->pdev->device == 0x9999) ||
 931                     (rdev->pdev->device == 0x999C)) {
 932                         rdev->config.cayman.max_simds_per_se = 6;
 933                         rdev->config.cayman.max_backends_per_se = 2;
 934                         rdev->config.cayman.max_hw_contexts = 8;
 935                         rdev->config.cayman.sx_max_export_size = 256;
 936                         rdev->config.cayman.sx_max_export_pos_size = 64;
 937                         rdev->config.cayman.sx_max_export_smx_size = 192;
 938                 } else if ((rdev->pdev->device == 0x9903) ||
 939                            (rdev->pdev->device == 0x9904) ||
 940                            (rdev->pdev->device == 0x990A) ||
 941                            (rdev->pdev->device == 0x990D) ||
 942                            (rdev->pdev->device == 0x990E) ||
 943                            (rdev->pdev->device == 0x9913) ||
 944                            (rdev->pdev->device == 0x9918) ||
 945                            (rdev->pdev->device == 0x999D)) {
 946                         rdev->config.cayman.max_simds_per_se = 4;
 947                         rdev->config.cayman.max_backends_per_se = 2;
 948                         rdev->config.cayman.max_hw_contexts = 8;
 949                         rdev->config.cayman.sx_max_export_size = 256;
 950                         rdev->config.cayman.sx_max_export_pos_size = 64;
 951                         rdev->config.cayman.sx_max_export_smx_size = 192;
 952                 } else if ((rdev->pdev->device == 0x9919) ||
 953                            (rdev->pdev->device == 0x9990) ||
 954                            (rdev->pdev->device == 0x9991) ||
 955                            (rdev->pdev->device == 0x9994) ||
 956                            (rdev->pdev->device == 0x9995) ||
 957                            (rdev->pdev->device == 0x9996) ||
 958                            (rdev->pdev->device == 0x999A) ||
 959                            (rdev->pdev->device == 0x99A0)) {
 960                         rdev->config.cayman.max_simds_per_se = 3;
 961                         rdev->config.cayman.max_backends_per_se = 1;
 962                         rdev->config.cayman.max_hw_contexts = 4;
 963                         rdev->config.cayman.sx_max_export_size = 128;
 964                         rdev->config.cayman.sx_max_export_pos_size = 32;
 965                         rdev->config.cayman.sx_max_export_smx_size = 96;
 966                 } else {
 967                         rdev->config.cayman.max_simds_per_se = 2;
 968                         rdev->config.cayman.max_backends_per_se = 1;
 969                         rdev->config.cayman.max_hw_contexts = 4;
 970                         rdev->config.cayman.sx_max_export_size = 128;
 971                         rdev->config.cayman.sx_max_export_pos_size = 32;
 972                         rdev->config.cayman.sx_max_export_smx_size = 96;
 973                 }
 974                 rdev->config.cayman.max_texture_channel_caches = 2;
 975                 rdev->config.cayman.max_gprs = 256;
 976                 rdev->config.cayman.max_threads = 256;
 977                 rdev->config.cayman.max_gs_threads = 32;
 978                 rdev->config.cayman.max_stack_entries = 512;
 979                 rdev->config.cayman.sx_num_of_sets = 8;
 980                 rdev->config.cayman.sq_num_cf_insts = 2;
 981
 982                 rdev->config.cayman.sc_prim_fifo_size = 0x40;
 983                 rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;
 984                 rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;
 985                 gb_addr_config = ARUBA_GB_ADDR_CONFIG_GOLDEN;
 986                 break;
 987         }
 988
 989         /* Initialize HDP */
 990         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
 991                 WREG32((0x2c14 + j), 0x00000000);
 992                 WREG32((0x2c18 + j), 0x00000000);
 993                 WREG32((0x2c1c + j), 0x00000000);
 994                 WREG32((0x2c20 + j), 0x00000000);
 995                 WREG32((0x2c24 + j), 0x00000000);
 996         }
 997
 998         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
 999         WREG32(SRBM_INT_CNTL, 0x1);
1000         WREG32(SRBM_INT_ACK, 0x1);
1001
1002         evergreen_fix_pci_max_read_req_size(rdev);
1003
1004         mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
1005         mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
1006
1007         tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
1008         rdev->config.cayman.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
1009         if (rdev->config.cayman.mem_row_size_in_kb > 4)
1010                 rdev->config.cayman.mem_row_size_in_kb = 4;
1011         /* XXX use MC settings? */
1012         rdev->config.cayman.shader_engine_tile_size = 32;
1013         rdev->config.cayman.num_gpus = 1;
1014         rdev->config.cayman.multi_gpu_tile_size = 64;
1015
1016         tmp = (gb_addr_config & NUM_PIPES_MASK) >> NUM_PIPES_SHIFT;
1017         rdev->config.cayman.num_tile_pipes = (1 << tmp);
1018         tmp = (gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT;
1019         rdev->config.cayman.mem_max_burst_length_bytes = (tmp + 1) * 256;
1020         tmp = (gb_addr_config & NUM_SHADER_ENGINES_MASK) >> NUM_SHADER_ENGINES_SHIFT;
1021         rdev->config.cayman.num_shader_engines = tmp + 1;
1022         tmp = (gb_addr_config & NUM_GPUS_MASK) >> NUM_GPUS_SHIFT;
1023         rdev->config.cayman.num_gpus = tmp + 1;
1024         tmp = (gb_addr_config & MULTI_GPU_TILE_SIZE_MASK) >> MULTI_GPU_TILE_SIZE_SHIFT;
1025         rdev->config.cayman.multi_gpu_tile_size = 1 << tmp;
1026         tmp = (gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT;
1027         rdev->config.cayman.mem_row_size_in_kb = 1 << tmp;
1028
1029
1030         /* setup tiling info dword.  gb_addr_config is not adequate since it does
1031          * not have bank info, so create a custom tiling dword.
1032          * bits 3:0   num_pipes
1033          * bits 7:4   num_banks
1034          * bits 11:8  group_size
1035          * bits 15:12 row_size
1036          */
1037         rdev->config.cayman.tile_config = 0;
1038         switch (rdev->config.cayman.num_tile_pipes) {
1039         case 1:
1040         default:
1041                 rdev->config.cayman.tile_config |= (0 << 0);
1042                 break;
1043         case 2:
1044                 rdev->config.cayman.tile_config |= (1 << 0);
1045                 break;
1046         case 4:
1047                 rdev->config.cayman.tile_config |= (2 << 0);
1048                 break;
1049         case 8:
1050                 rdev->config.cayman.tile_config |= (3 << 0);
1051                 break;
1052         }
1053
1054         /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
1055         if (rdev->flags & RADEON_IS_IGP)
1056                 rdev->config.cayman.tile_config |= 1 << 4;
1057         else {
1058                 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
1059                 case 0: /* four banks */
1060                         rdev->config.cayman.tile_config |= 0 << 4;
1061                         break;
1062                 case 1: /* eight banks */
1063                         rdev->config.cayman.tile_config |= 1 << 4;
1064                         break;
1065                 case 2: /* sixteen banks */
1066                 default:
1067                         rdev->config.cayman.tile_config |= 2 << 4;
1068                         break;
1069                 }
1070         }
1071         rdev->config.cayman.tile_config |=
1072                 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
1073         rdev->config.cayman.tile_config |=
1074                 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
1075
1076         tmp = 0;
1077         for (i = (rdev->config.cayman.max_shader_engines - 1); i >= 0; i--) {
1078                 u32 rb_disable_bitmap;
1079
1080                 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1081                 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1082                 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
1083                 tmp <<= 4;
1084                 tmp |= rb_disable_bitmap;
1085         }
1086         /* enabled rb are just the one not disabled :) */
1087         disabled_rb_mask = tmp;
1088         tmp = 0;
1089         for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines); i++)
1090                 tmp |= (1 << i);
1091         /* if all the backends are disabled, fix it up here */
1092         if ((disabled_rb_mask & tmp) == tmp) {
1093                 for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines); i++)
1094                         disabled_rb_mask &= ~(1 << i);
1095         }
1096
1097         for (i = 0; i < rdev->config.cayman.max_shader_engines; i++) {
1098                 u32 simd_disable_bitmap;
1099
1100                 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1101                 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1102                 simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
1103                 simd_disable_bitmap |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
1104                 tmp <<= 16;
1105                 tmp |= simd_disable_bitmap;
1106         }
1107         rdev->config.cayman.active_simds = hweight32(~tmp);
1108
1109         WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
1110         WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
1111
1112         WREG32(GB_ADDR_CONFIG, gb_addr_config);
1113         WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
1114         if (ASIC_IS_DCE6(rdev))
1115                 WREG32(DMIF_ADDR_CALC, gb_addr_config);
1116         WREG32(HDP_ADDR_CONFIG, gb_addr_config);
1117         WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
1118         WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
1119         WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
1120         WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
1121         WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
1122
1123         if ((rdev->config.cayman.max_backends_per_se == 1) &&
1124             (rdev->flags & RADEON_IS_IGP)) {
1125                 if ((disabled_rb_mask & 3) == 2) {
1126                         /* RB1 disabled, RB0 enabled */
1127                         tmp = 0x00000000;
1128                 } else {
1129                         /* RB0 disabled, RB1 enabled */
1130                         tmp = 0x11111111;
1131                 }
1132         } else {
1133                 tmp = gb_addr_config & NUM_PIPES_MASK;
1134                 tmp = r6xx_remap_render_backend(rdev, tmp,
1135                                                 rdev->config.cayman.max_backends_per_se *
1136                                                 rdev->config.cayman.max_shader_engines,
1137                                                 CAYMAN_MAX_BACKENDS, disabled_rb_mask);
1138         }
1139         WREG32(GB_BACKEND_MAP, tmp);
1140
1141         cgts_tcc_disable = 0xffff0000;
1142         for (i = 0; i < rdev->config.cayman.max_texture_channel_caches; i++)
1143                 cgts_tcc_disable &= ~(1 << (16 + i));
1144         WREG32(CGTS_TCC_DISABLE, cgts_tcc_disable);
1145         WREG32(CGTS_SYS_TCC_DISABLE, cgts_tcc_disable);
1146         WREG32(CGTS_USER_SYS_TCC_DISABLE, cgts_tcc_disable);
1147         WREG32(CGTS_USER_TCC_DISABLE, cgts_tcc_disable);
1148
1149         /* reprogram the shader complex */
1150         cgts_sm_ctrl_reg = RREG32(CGTS_SM_CTRL_REG);
1151         for (i = 0; i < 16; i++)
1152                 WREG32(CGTS_SM_CTRL_REG, OVERRIDE);
1153         WREG32(CGTS_SM_CTRL_REG, cgts_sm_ctrl_reg);
1154
1155         /* set HW defaults for 3D engine */
1156         WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
1157
1158         sx_debug_1 = RREG32(SX_DEBUG_1);
1159         sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
1160         WREG32(SX_DEBUG_1, sx_debug_1);
1161
1162         smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
1163         smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
1164         smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.cayman.sx_num_of_sets);
1165         WREG32(SMX_DC_CTL0, smx_dc_ctl0);
1166
1167         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4) | CRC_SIMD_ID_WADDR_DISABLE);
1168
1169         /* need to be explicitly zero-ed */
1170         WREG32(VGT_OFFCHIP_LDS_BASE, 0);
1171         WREG32(SQ_LSTMP_RING_BASE, 0);
1172         WREG32(SQ_HSTMP_RING_BASE, 0);
1173         WREG32(SQ_ESTMP_RING_BASE, 0);
1174         WREG32(SQ_GSTMP_RING_BASE, 0);
1175         WREG32(SQ_VSTMP_RING_BASE, 0);
1176         WREG32(SQ_PSTMP_RING_BASE, 0);
1177
1178         WREG32(TA_CNTL_AUX, DISABLE_CUBE_ANISO);
1179
1180         WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.cayman.sx_max_export_size / 4) - 1) |
1181                                         POSITION_BUFFER_SIZE((rdev->config.cayman.sx_max_export_pos_size / 4) - 1) |
1182                                         SMX_BUFFER_SIZE((rdev->config.cayman.sx_max_export_smx_size / 4) - 1)));
1183
1184         WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.cayman.sc_prim_fifo_size) |
1185                                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_hiz_tile_fifo_size) |
1186                                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_earlyz_tile_fifo_size)));
1187
1188
1189         WREG32(VGT_NUM_INSTANCES, 1);
1190
1191         WREG32(CP_PERFMON_CNTL, 0);
1192
1193         WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.cayman.sq_num_cf_insts) |
1194                                   FETCH_FIFO_HIWATER(0x4) |
1195                                   DONE_FIFO_HIWATER(0xe0) |
1196                                   ALU_UPDATE_FIFO_HIWATER(0x8)));
1197
1198         WREG32(SQ_GPR_RESOURCE_MGMT_1, NUM_CLAUSE_TEMP_GPRS(4));
1199         WREG32(SQ_CONFIG, (VC_ENABLE |
1200                            EXPORT_SRC_C |
1201                            GFX_PRIO(0) |
1202                            CS1_PRIO(0) |
1203                            CS2_PRIO(1)));
1204         WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, DYN_GPR_ENABLE);
1205
1206         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
1207                                           FORCE_EOV_MAX_REZ_CNT(255)));
1208
1209         WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
1210                AUTO_INVLD_EN(ES_AND_GS_AUTO));
1211
1212         WREG32(VGT_GS_VERTEX_REUSE, 16);
1213         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
1214
1215         WREG32(CB_PERF_CTR0_SEL_0, 0);
1216         WREG32(CB_PERF_CTR0_SEL_1, 0);
1217         WREG32(CB_PERF_CTR1_SEL_0, 0);
1218         WREG32(CB_PERF_CTR1_SEL_1, 0);
1219         WREG32(CB_PERF_CTR2_SEL_0, 0);
1220         WREG32(CB_PERF_CTR2_SEL_1, 0);
1221         WREG32(CB_PERF_CTR3_SEL_0, 0);
1222         WREG32(CB_PERF_CTR3_SEL_1, 0);
1223
1224         tmp = RREG32(HDP_MISC_CNTL);
1225         tmp |= HDP_FLUSH_INVALIDATE_CACHE;
1226         WREG32(HDP_MISC_CNTL, tmp);
1227
1228         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
1229         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
1230
1231         WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
1232
1233         udelay(50);
1234
1235         /* set clockgating golden values on TN */
1236         if (rdev->family == CHIP_ARUBA) {
1237                 tmp = RREG32_CG(CG_CGTT_LOCAL_0);
1238                 tmp &= ~0x00380000;
1239                 WREG32_CG(CG_CGTT_LOCAL_0, tmp);
1240                 tmp = RREG32_CG(CG_CGTT_LOCAL_1);
1241                 tmp &= ~0x0e000000;
1242                 WREG32_CG(CG_CGTT_LOCAL_1, tmp);
1243         }
1244 }
1245
1246 /*
1247  * GART
1248  */
1249 void cayman_pcie_gart_tlb_flush(struct radeon_device *rdev)
1250 {
1251         /* flush hdp cache */
1252         WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
1253
1254         /* bits 0-7 are the VM contexts0-7 */
1255         WREG32(VM_INVALIDATE_REQUEST, 1);
1256 }
1257
1258 static int cayman_pcie_gart_enable(struct radeon_device *rdev)
1259 {
1260         int i, r;
1261
1262         if (rdev->gart.robj == NULL) {
1263                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
1264                 return -EINVAL;
1265         }
1266         r = radeon_gart_table_vram_pin(rdev);
1267         if (r)
1268                 return r;
1269         /* Setup TLB control */
1270         WREG32(MC_VM_MX_L1_TLB_CNTL,
1271                (0xA << 7) |
1272                ENABLE_L1_TLB |
1273                ENABLE_L1_FRAGMENT_PROCESSING |
1274                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1275                ENABLE_ADVANCED_DRIVER_MODEL |
1276                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
1277         /* Setup L2 cache */
1278         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
1279                ENABLE_L2_FRAGMENT_PROCESSING |
1280                ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1281                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
1282                EFFECTIVE_L2_QUEUE_SIZE(7) |
1283                CONTEXT1_IDENTITY_ACCESS_MODE(1));
1284         WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
1285         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
1286                BANK_SELECT(6) |
1287                L2_CACHE_BIGK_FRAGMENT_SIZE(6));
1288         /* setup context0 */
1289         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
1290         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
1291         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
1292         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
1293                         (u32)(rdev->dummy_page.addr >> 12));
1294         WREG32(VM_CONTEXT0_CNTL2, 0);
1295         WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
1296                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
1297
1298         WREG32(0x15D4, 0);
1299         WREG32(0x15D8, 0);
1300         WREG32(0x15DC, 0);
1301
1302         /* empty context1-7 */
1303         /* Assign the pt base to something valid for now; the pts used for
1304          * the VMs are determined by the application and setup and assigned
1305          * on the fly in the vm part of radeon_gart.c
1306          */
1307         for (i = 1; i < 8; i++) {
1308                 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR + (i << 2), 0);
1309                 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR + (i << 2),
1310                         rdev->vm_manager.max_pfn - 1);
1311                 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
1312                        rdev->vm_manager.saved_table_addr[i]);
1313         }
1314
1315         /* enable context1-7 */
1316         WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
1317                (u32)(rdev->dummy_page.addr >> 12));
1318         WREG32(VM_CONTEXT1_CNTL2, 4);
1319         WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
1320                                 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
1321                                 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1322                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
1323                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1324                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
1325                                 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
1326                                 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
1327                                 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
1328                                 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
1329                                 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
1330                                 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
1331                                 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1332                                 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
1333
1334         cayman_pcie_gart_tlb_flush(rdev);
1335         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
1336                  (unsigned)(rdev->mc.gtt_size >> 20),
1337                  (unsigned long long)rdev->gart.table_addr);
1338         rdev->gart.ready = true;
1339         return 0;
1340 }
1341
1342 static void cayman_pcie_gart_disable(struct radeon_device *rdev)
1343 {
1344         unsigned i;
1345
1346         for (i = 1; i < 8; ++i) {
1347                 rdev->vm_manager.saved_table_addr[i] = RREG32(
1348                         VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2));
1349         }
1350
1351         /* Disable all tables */
1352         WREG32(VM_CONTEXT0_CNTL, 0);
1353         WREG32(VM_CONTEXT1_CNTL, 0);
1354         /* Setup TLB control */
1355         WREG32(MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_FRAGMENT_PROCESSING |
1356                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1357                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
1358         /* Setup L2 cache */
1359         WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1360                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
1361                EFFECTIVE_L2_QUEUE_SIZE(7) |
1362                CONTEXT1_IDENTITY_ACCESS_MODE(1));
1363         WREG32(VM_L2_CNTL2, 0);
1364         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
1365                L2_CACHE_BIGK_FRAGMENT_SIZE(6));
1366         radeon_gart_table_vram_unpin(rdev);
1367 }
1368
1369 static void cayman_pcie_gart_fini(struct radeon_device *rdev)
1370 {
1371         cayman_pcie_gart_disable(rdev);
1372         radeon_gart_table_vram_free(rdev);
1373         radeon_gart_fini(rdev);
1374 }
1375
1376 void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
1377                               int ring, u32 cp_int_cntl)
1378 {
1379         WREG32(SRBM_GFX_CNTL, RINGID(ring));
1380         WREG32(CP_INT_CNTL, cp_int_cntl);
1381 }
1382
1383 /*
1384  * CP.
1385  */
1386 void cayman_fence_ring_emit(struct radeon_device *rdev,
1387                             struct radeon_fence *fence)
1388 {
1389         struct radeon_ring *ring = &rdev->ring[fence->ring];
1390         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
1391         u32 cp_coher_cntl = PACKET3_FULL_CACHE_ENA | PACKET3_TC_ACTION_ENA |
1392                 PACKET3_SH_ACTION_ENA;
1393
1394         /* flush read cache over gart for this vmid */
1395         radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
1396         radeon_ring_write(ring, PACKET3_ENGINE_ME | cp_coher_cntl);
1397         radeon_ring_write(ring, 0xFFFFFFFF);
1398         radeon_ring_write(ring, 0);
1399         radeon_ring_write(ring, 10); /* poll interval */
1400         /* EVENT_WRITE_EOP - flush caches, send int */
1401         radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
1402         radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT_TS) | EVENT_INDEX(5));
1403         radeon_ring_write(ring, lower_32_bits(addr));
1404         radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
1405         radeon_ring_write(ring, fence->seq);
1406         radeon_ring_write(ring, 0);
1407 }
1408
1409 void cayman_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
1410 {
1411         struct radeon_ring *ring = &rdev->ring[ib->ring];
1412         unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0;
1413         u32 cp_coher_cntl = PACKET3_FULL_CACHE_ENA | PACKET3_TC_ACTION_ENA |
1414                 PACKET3_SH_ACTION_ENA;
1415
1416         /* set to DX10/11 mode */
1417         radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
1418         radeon_ring_write(ring, 1);
1419
1420         if (ring->rptr_save_reg) {
1421                 uint32_t next_rptr = ring->wptr + 3 + 4 + 8;
1422                 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
1423                 radeon_ring_write(ring, ((ring->rptr_save_reg -
1424                                           PACKET3_SET_CONFIG_REG_START) >> 2));
1425                 radeon_ring_write(ring, next_rptr);
1426         }
1427
1428         radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
1429         radeon_ring_write(ring,
1430 #ifdef __BIG_ENDIAN
1431                           (2 << 0) |
1432 #endif
1433                           (ib->gpu_addr & 0xFFFFFFFC));
1434         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
1435         radeon_ring_write(ring, ib->length_dw | (vm_id << 24));
1436
1437         /* flush read cache over gart for this vmid */
1438         radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
1439         radeon_ring_write(ring, PACKET3_ENGINE_ME | cp_coher_cntl);
1440         radeon_ring_write(ring, 0xFFFFFFFF);
1441         radeon_ring_write(ring, 0);
1442         radeon_ring_write(ring, (vm_id << 24) | 10); /* poll interval */
1443 }
1444
1445 static void cayman_cp_enable(struct radeon_device *rdev, bool enable)
1446 {
1447         if (enable)
1448                 WREG32(CP_ME_CNTL, 0);
1449         else {
1450                 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
1451                         radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
1452                 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT));
1453                 WREG32(SCRATCH_UMSK, 0);
1454                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1455         }
1456 }
1457
1458 u32 cayman_gfx_get_rptr(struct radeon_device *rdev,
1459                         struct radeon_ring *ring)
1460 {
1461         u32 rptr;
1462
1463         if (rdev->wb.enabled)
1464                 rptr = rdev->wb.wb[ring->rptr_offs/4];
1465         else {
1466                 if (ring->idx == RADEON_RING_TYPE_GFX_INDEX)
1467                         rptr = RREG32(CP_RB0_RPTR);
1468                 else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX)
1469                         rptr = RREG32(CP_RB1_RPTR);
1470                 else
1471                         rptr = RREG32(CP_RB2_RPTR);
1472         }
1473
1474         return rptr;
1475 }
1476
1477 u32 cayman_gfx_get_wptr(struct radeon_device *rdev,
1478                         struct radeon_ring *ring)
1479 {
1480         u32 wptr;
1481
1482         if (ring->idx == RADEON_RING_TYPE_GFX_INDEX)
1483                 wptr = RREG32(CP_RB0_WPTR);
1484         else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX)
1485                 wptr = RREG32(CP_RB1_WPTR);
1486         else
1487                 wptr = RREG32(CP_RB2_WPTR);
1488
1489         return wptr;
1490 }
1491
1492 void cayman_gfx_set_wptr(struct radeon_device *rdev,
1493                          struct radeon_ring *ring)
1494 {
1495         if (ring->idx == RADEON_RING_TYPE_GFX_INDEX) {
1496                 WREG32(CP_RB0_WPTR, ring->wptr);
1497                 (void)RREG32(CP_RB0_WPTR);
1498         } else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX) {
1499                 WREG32(CP_RB1_WPTR, ring->wptr);
1500                 (void)RREG32(CP_RB1_WPTR);
1501         } else {
1502                 WREG32(CP_RB2_WPTR, ring->wptr);
1503                 (void)RREG32(CP_RB2_WPTR);
1504         }
1505 }
1506
1507 static int cayman_cp_load_microcode(struct radeon_device *rdev)
1508 {
1509         const __be32 *fw_data;
1510         int i;
1511
1512         if (!rdev->me_fw || !rdev->pfp_fw)
1513                 return -EINVAL;
1514
1515         cayman_cp_enable(rdev, false);
1516
1517         fw_data = (const __be32 *)rdev->pfp_fw->data;
1518         WREG32(CP_PFP_UCODE_ADDR, 0);
1519         for (i = 0; i < CAYMAN_PFP_UCODE_SIZE; i++)
1520                 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
1521         WREG32(CP_PFP_UCODE_ADDR, 0);
1522
1523         fw_data = (const __be32 *)rdev->me_fw->data;
1524         WREG32(CP_ME_RAM_WADDR, 0);
1525         for (i = 0; i < CAYMAN_PM4_UCODE_SIZE; i++)
1526                 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
1527
1528         WREG32(CP_PFP_UCODE_ADDR, 0);
1529         WREG32(CP_ME_RAM_WADDR, 0);
1530         WREG32(CP_ME_RAM_RADDR, 0);
1531         return 0;
1532 }
1533
1534 static int cayman_cp_start(struct radeon_device *rdev)
1535 {
1536         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1537         int r, i;
1538
1539         r = radeon_ring_lock(rdev, ring, 7);
1540         if (r) {
1541                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1542                 return r;
1543         }
1544         radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
1545         radeon_ring_write(ring, 0x1);
1546         radeon_ring_write(ring, 0x0);
1547         radeon_ring_write(ring, rdev->config.cayman.max_hw_contexts - 1);
1548         radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
1549         radeon_ring_write(ring, 0);
1550         radeon_ring_write(ring, 0);
1551         radeon_ring_unlock_commit(rdev, ring, false);
1552
1553         cayman_cp_enable(rdev, true);
1554
1555         r = radeon_ring_lock(rdev, ring, cayman_default_size + 19);
1556         if (r) {
1557                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1558                 return r;
1559         }
1560
1561         /* setup clear context state */
1562         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1563         radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1564
1565         for (i = 0; i < cayman_default_size; i++)
1566                 radeon_ring_write(ring, cayman_default_state[i]);
1567
1568         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1569         radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
1570
1571         /* set clear context state */
1572         radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
1573         radeon_ring_write(ring, 0);
1574
1575         /* SQ_VTX_BASE_VTX_LOC */
1576         radeon_ring_write(ring, 0xc0026f00);
1577         radeon_ring_write(ring, 0x00000000);
1578         radeon_ring_write(ring, 0x00000000);
1579         radeon_ring_write(ring, 0x00000000);
1580
1581         /* Clear consts */
1582         radeon_ring_write(ring, 0xc0036f00);
1583         radeon_ring_write(ring, 0x00000bc4);
1584         radeon_ring_write(ring, 0xffffffff);
1585         radeon_ring_write(ring, 0xffffffff);
1586         radeon_ring_write(ring, 0xffffffff);
1587
1588         radeon_ring_write(ring, 0xc0026900);
1589         radeon_ring_write(ring, 0x00000316);
1590         radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
1591         radeon_ring_write(ring, 0x00000010); /*  */
1592
1593         radeon_ring_unlock_commit(rdev, ring, false);
1594
1595         /* XXX init other rings */
1596
1597         return 0;
1598 }
1599
1600 static void cayman_cp_fini(struct radeon_device *rdev)
1601 {
1602         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1603         cayman_cp_enable(rdev, false);
1604         radeon_ring_fini(rdev, ring);
1605         radeon_scratch_free(rdev, ring->rptr_save_reg);
1606 }
1607
1608 static int cayman_cp_resume(struct radeon_device *rdev)
1609 {
1610         static const int ridx[] = {
1611                 RADEON_RING_TYPE_GFX_INDEX,
1612                 CAYMAN_RING_TYPE_CP1_INDEX,
1613                 CAYMAN_RING_TYPE_CP2_INDEX
1614         };
1615         static const unsigned cp_rb_cntl[] = {
1616                 CP_RB0_CNTL,
1617                 CP_RB1_CNTL,
1618                 CP_RB2_CNTL,
1619         };
1620         static const unsigned cp_rb_rptr_addr[] = {
1621                 CP_RB0_RPTR_ADDR,
1622                 CP_RB1_RPTR_ADDR,
1623                 CP_RB2_RPTR_ADDR
1624         };
1625         static const unsigned cp_rb_rptr_addr_hi[] = {
1626                 CP_RB0_RPTR_ADDR_HI,
1627                 CP_RB1_RPTR_ADDR_HI,
1628                 CP_RB2_RPTR_ADDR_HI
1629         };
1630         static const unsigned cp_rb_base[] = {
1631                 CP_RB0_BASE,
1632                 CP_RB1_BASE,
1633                 CP_RB2_BASE
1634         };
1635         static const unsigned cp_rb_rptr[] = {
1636                 CP_RB0_RPTR,
1637                 CP_RB1_RPTR,
1638                 CP_RB2_RPTR
1639         };
1640         static const unsigned cp_rb_wptr[] = {
1641                 CP_RB0_WPTR,
1642                 CP_RB1_WPTR,
1643                 CP_RB2_WPTR
1644         };
1645         struct radeon_ring *ring;
1646         int i, r;
1647
1648         /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
1649         WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
1650                                  SOFT_RESET_PA |
1651                                  SOFT_RESET_SH |
1652                                  SOFT_RESET_VGT |
1653                                  SOFT_RESET_SPI |
1654                                  SOFT_RESET_SX));
1655         RREG32(GRBM_SOFT_RESET);
1656         mdelay(15);
1657         WREG32(GRBM_SOFT_RESET, 0);
1658         RREG32(GRBM_SOFT_RESET);
1659
1660         WREG32(CP_SEM_WAIT_TIMER, 0x0);
1661         WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
1662
1663         /* Set the write pointer delay */
1664         WREG32(CP_RB_WPTR_DELAY, 0);
1665
1666         WREG32(CP_DEBUG, (1 << 27));
1667
1668         /* set the wb address whether it's enabled or not */
1669         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
1670         WREG32(SCRATCH_UMSK, 0xff);
1671
1672         for (i = 0; i < 3; ++i) {
1673                 uint32_t rb_cntl;
1674                 uint64_t addr;
1675
1676                 /* Set ring buffer size */
1677                 ring = &rdev->ring[ridx[i]];
1678                 rb_cntl = order_base_2(ring->ring_size / 8);
1679                 rb_cntl |= order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8;
1680 #ifdef __BIG_ENDIAN
1681                 rb_cntl |= BUF_SWAP_32BIT;
1682 #endif
1683                 WREG32(cp_rb_cntl[i], rb_cntl);
1684
1685                 /* set the wb address whether it's enabled or not */
1686                 addr = rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET;
1687                 WREG32(cp_rb_rptr_addr[i], addr & 0xFFFFFFFC);
1688                 WREG32(cp_rb_rptr_addr_hi[i], upper_32_bits(addr) & 0xFF);
1689         }
1690
1691         /* set the rb base addr, this causes an internal reset of ALL rings */
1692         for (i = 0; i < 3; ++i) {
1693                 ring = &rdev->ring[ridx[i]];
1694                 WREG32(cp_rb_base[i], ring->gpu_addr >> 8);
1695         }
1696
1697         for (i = 0; i < 3; ++i) {
1698                 /* Initialize the ring buffer's read and write pointers */
1699                 ring = &rdev->ring[ridx[i]];
1700                 WREG32_P(cp_rb_cntl[i], RB_RPTR_WR_ENA, ~RB_RPTR_WR_ENA);
1701
1702                 ring->wptr = 0;
1703                 WREG32(cp_rb_rptr[i], 0);
1704                 WREG32(cp_rb_wptr[i], ring->wptr);
1705
1706                 mdelay(1);
1707                 WREG32_P(cp_rb_cntl[i], 0, ~RB_RPTR_WR_ENA);
1708         }
1709
1710         /* start the rings */
1711         cayman_cp_start(rdev);
1712         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
1713         rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
1714         rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
1715         /* this only test cp0 */
1716         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
1717         if (r) {
1718                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1719                 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
1720                 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
1721                 return r;
1722         }
1723
1724         if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
1725                 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
1726
1727         return 0;
1728 }
1729
1730 u32 cayman_gpu_check_soft_reset(struct radeon_device *rdev)
1731 {
1732         u32 reset_mask = 0;
1733         u32 tmp;
1734
1735         /* GRBM_STATUS */
1736         tmp = RREG32(GRBM_STATUS);
1737         if (tmp & (PA_BUSY | SC_BUSY |
1738                    SH_BUSY | SX_BUSY |
1739                    TA_BUSY | VGT_BUSY |
1740                    DB_BUSY | CB_BUSY |
1741                    GDS_BUSY | SPI_BUSY |
1742                    IA_BUSY | IA_BUSY_NO_DMA))
1743                 reset_mask |= RADEON_RESET_GFX;
1744
1745         if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
1746                    CP_BUSY | CP_COHERENCY_BUSY))
1747                 reset_mask |= RADEON_RESET_CP;
1748
1749         if (tmp & GRBM_EE_BUSY)
1750                 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
1751
1752         /* DMA_STATUS_REG 0 */
1753         tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
1754         if (!(tmp & DMA_IDLE))
1755                 reset_mask |= RADEON_RESET_DMA;
1756
1757         /* DMA_STATUS_REG 1 */
1758         tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
1759         if (!(tmp & DMA_IDLE))
1760                 reset_mask |= RADEON_RESET_DMA1;
1761
1762         /* SRBM_STATUS2 */
1763         tmp = RREG32(SRBM_STATUS2);
1764         if (tmp & DMA_BUSY)
1765                 reset_mask |= RADEON_RESET_DMA;
1766
1767         if (tmp & DMA1_BUSY)
1768                 reset_mask |= RADEON_RESET_DMA1;
1769
1770         /* SRBM_STATUS */
1771         tmp = RREG32(SRBM_STATUS);
1772         if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
1773                 reset_mask |= RADEON_RESET_RLC;
1774
1775         if (tmp & IH_BUSY)
1776                 reset_mask |= RADEON_RESET_IH;
1777
1778         if (tmp & SEM_BUSY)
1779                 reset_mask |= RADEON_RESET_SEM;
1780
1781         if (tmp & GRBM_RQ_PENDING)
1782                 reset_mask |= RADEON_RESET_GRBM;
1783
1784         if (tmp & VMC_BUSY)
1785                 reset_mask |= RADEON_RESET_VMC;
1786
1787         if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
1788                    MCC_BUSY | MCD_BUSY))
1789                 reset_mask |= RADEON_RESET_MC;
1790
1791         if (evergreen_is_display_hung(rdev))
1792                 reset_mask |= RADEON_RESET_DISPLAY;
1793
1794         /* VM_L2_STATUS */
1795         tmp = RREG32(VM_L2_STATUS);
1796         if (tmp & L2_BUSY)
1797                 reset_mask |= RADEON_RESET_VMC;
1798
1799         /* Skip MC reset as it's mostly likely not hung, just busy */
1800         if (reset_mask & RADEON_RESET_MC) {
1801                 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
1802                 reset_mask &= ~RADEON_RESET_MC;
1803         }
1804
1805         return reset_mask;
1806 }
1807
1808 static void cayman_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
1809 {
1810         struct evergreen_mc_save save;
1811         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
1812         u32 tmp;
1813
1814         if (reset_mask == 0)
1815                 return;
1816
1817         dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
1818
1819         evergreen_print_gpu_status_regs(rdev);
1820         dev_info(rdev->dev, "  VM_CONTEXT0_PROTECTION_FAULT_ADDR   0x%08X\n",
1821                  RREG32(0x14F8));
1822         dev_info(rdev->dev, "  VM_CONTEXT0_PROTECTION_FAULT_STATUS 0x%08X\n",
1823                  RREG32(0x14D8));
1824         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
1825                  RREG32(0x14FC));
1826         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
1827                  RREG32(0x14DC));
1828
1829         /* Disable CP parsing/prefetching */
1830         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
1831
1832         if (reset_mask & RADEON_RESET_DMA) {
1833                 /* dma0 */
1834                 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
1835                 tmp &= ~DMA_RB_ENABLE;
1836                 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
1837         }
1838
1839         if (reset_mask & RADEON_RESET_DMA1) {
1840                 /* dma1 */
1841                 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
1842                 tmp &= ~DMA_RB_ENABLE;
1843                 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
1844         }
1845
1846         udelay(50);
1847
1848         evergreen_mc_stop(rdev, &save);
1849         if (evergreen_mc_wait_for_idle(rdev)) {
1850                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1851         }
1852
1853         if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
1854                 grbm_soft_reset = SOFT_RESET_CB |
1855                         SOFT_RESET_DB |
1856                         SOFT_RESET_GDS |
1857                         SOFT_RESET_PA |
1858                         SOFT_RESET_SC |
1859                         SOFT_RESET_SPI |
1860                         SOFT_RESET_SH |
1861                         SOFT_RESET_SX |
1862                         SOFT_RESET_TC |
1863                         SOFT_RESET_TA |
1864                         SOFT_RESET_VGT |
1865                         SOFT_RESET_IA;
1866         }
1867
1868         if (reset_mask & RADEON_RESET_CP) {
1869                 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
1870
1871                 srbm_soft_reset |= SOFT_RESET_GRBM;
1872         }
1873
1874         if (reset_mask & RADEON_RESET_DMA)
1875                 srbm_soft_reset |= SOFT_RESET_DMA;
1876
1877         if (reset_mask & RADEON_RESET_DMA1)
1878                 srbm_soft_reset |= SOFT_RESET_DMA1;
1879
1880         if (reset_mask & RADEON_RESET_DISPLAY)
1881                 srbm_soft_reset |= SOFT_RESET_DC;
1882
1883         if (reset_mask & RADEON_RESET_RLC)
1884                 srbm_soft_reset |= SOFT_RESET_RLC;
1885
1886         if (reset_mask & RADEON_RESET_SEM)
1887                 srbm_soft_reset |= SOFT_RESET_SEM;
1888
1889         if (reset_mask & RADEON_RESET_IH)
1890                 srbm_soft_reset |= SOFT_RESET_IH;
1891
1892         if (reset_mask & RADEON_RESET_GRBM)
1893                 srbm_soft_reset |= SOFT_RESET_GRBM;
1894
1895         if (reset_mask & RADEON_RESET_VMC)
1896                 srbm_soft_reset |= SOFT_RESET_VMC;
1897
1898         if (!(rdev->flags & RADEON_IS_IGP)) {
1899                 if (reset_mask & RADEON_RESET_MC)
1900                         srbm_soft_reset |= SOFT_RESET_MC;
1901         }
1902
1903         if (grbm_soft_reset) {
1904                 tmp = RREG32(GRBM_SOFT_RESET);
1905                 tmp |= grbm_soft_reset;
1906                 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
1907                 WREG32(GRBM_SOFT_RESET, tmp);
1908                 tmp = RREG32(GRBM_SOFT_RESET);
1909
1910                 udelay(50);
1911
1912                 tmp &= ~grbm_soft_reset;
1913                 WREG32(GRBM_SOFT_RESET, tmp);
1914                 tmp = RREG32(GRBM_SOFT_RESET);
1915         }
1916
1917         if (srbm_soft_reset) {
1918                 tmp = RREG32(SRBM_SOFT_RESET);
1919                 tmp |= srbm_soft_reset;
1920                 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
1921                 WREG32(SRBM_SOFT_RESET, tmp);
1922                 tmp = RREG32(SRBM_SOFT_RESET);
1923
1924                 udelay(50);
1925
1926                 tmp &= ~srbm_soft_reset;
1927                 WREG32(SRBM_SOFT_RESET, tmp);
1928                 tmp = RREG32(SRBM_SOFT_RESET);
1929         }
1930
1931         /* Wait a little for things to settle down */
1932         udelay(50);
1933
1934         evergreen_mc_resume(rdev, &save);
1935         udelay(50);
1936
1937         evergreen_print_gpu_status_regs(rdev);
1938 }
1939
1940 int cayman_asic_reset(struct radeon_device *rdev)
1941 {
1942         u32 reset_mask;
1943
1944         reset_mask = cayman_gpu_check_soft_reset(rdev);
1945
1946         if (reset_mask)
1947                 r600_set_bios_scratch_engine_hung(rdev, true);
1948
1949         cayman_gpu_soft_reset(rdev, reset_mask);
1950
1951         reset_mask = cayman_gpu_check_soft_reset(rdev);
1952
1953         if (reset_mask)
1954                 evergreen_gpu_pci_config_reset(rdev);
1955
1956         r600_set_bios_scratch_engine_hung(rdev, false);
1957
1958         return 0;
1959 }
1960
1961 /**
1962  * cayman_gfx_is_lockup - Check if the GFX engine is locked up
1963  *
1964  * @rdev: radeon_device pointer
1965  * @ring: radeon_ring structure holding ring information
1966  *
1967  * Check if the GFX engine is locked up.
1968  * Returns true if the engine appears to be locked up, false if not.
1969  */
1970 bool cayman_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
1971 {
1972         u32 reset_mask = cayman_gpu_check_soft_reset(rdev);
1973
1974         if (!(reset_mask & (RADEON_RESET_GFX |
1975                             RADEON_RESET_COMPUTE |
1976                             RADEON_RESET_CP))) {
1977                 radeon_ring_lockup_update(rdev, ring);
1978                 return false;
1979         }
1980         return radeon_ring_test_lockup(rdev, ring);
1981 }
1982
1983 static int cayman_startup(struct radeon_device *rdev)
1984 {
1985         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1986         int r;
1987
1988         /* enable pcie gen2 link */
1989         evergreen_pcie_gen2_enable(rdev);
1990         /* enable aspm */
1991         evergreen_program_aspm(rdev);
1992
1993         /* scratch needs to be initialized before MC */
1994         r = r600_vram_scratch_init(rdev);
1995         if (r)
1996                 return r;
1997
1998         evergreen_mc_program(rdev);
1999
2000         if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
2001                 r = ni_mc_load_microcode(rdev);
2002                 if (r) {
2003                         DRM_ERROR("Failed to load MC firmware!\n");
2004                         return r;
2005                 }
2006         }
2007
2008         r = cayman_pcie_gart_enable(rdev);
2009         if (r)
2010                 return r;
2011         cayman_gpu_init(rdev);
2012
2013         /* allocate rlc buffers */
2014         if (rdev->flags & RADEON_IS_IGP) {
2015                 rdev->rlc.reg_list = tn_rlc_save_restore_register_list;
2016                 rdev->rlc.reg_list_size =
2017                         (u32)ARRAY_SIZE(tn_rlc_save_restore_register_list);
2018                 rdev->rlc.cs_data = cayman_cs_data;
2019                 r = sumo_rlc_init(rdev);
2020                 if (r) {
2021                         DRM_ERROR("Failed to init rlc BOs!\n");
2022                         return r;
2023                 }
2024         }
2025
2026         /* allocate wb buffer */
2027         r = radeon_wb_init(rdev);
2028         if (r)
2029                 return r;
2030
2031         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
2032         if (r) {
2033                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2034                 return r;
2035         }
2036
2037         r = uvd_v2_2_resume(rdev);
2038         if (!r) {
2039                 r = radeon_fence_driver_start_ring(rdev,
2040                                                    R600_RING_TYPE_UVD_INDEX);
2041                 if (r)
2042                         dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
2043         }
2044         if (r)
2045                 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
2046
2047         if (rdev->family == CHIP_ARUBA) {
2048                 r = radeon_vce_resume(rdev);
2049                 if (!r)
2050                         r = vce_v1_0_resume(rdev);
2051
2052                 if (!r)
2053                         r = radeon_fence_driver_start_ring(rdev,
2054                                                            TN_RING_TYPE_VCE1_INDEX);
2055                 if (!r)
2056                         r = radeon_fence_driver_start_ring(rdev,
2057                                                            TN_RING_TYPE_VCE2_INDEX);
2058
2059                 if (r) {
2060                         dev_err(rdev->dev, "VCE init error (%d).\n", r);
2061                         rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
2062                         rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
2063                 }
2064         }
2065
2066         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
2067         if (r) {
2068                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2069                 return r;
2070         }
2071
2072         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
2073         if (r) {
2074                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2075                 return r;
2076         }
2077
2078         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
2079         if (r) {
2080                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
2081                 return r;
2082         }
2083
2084         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
2085         if (r) {
2086                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
2087                 return r;
2088         }
2089
2090         /* Enable IRQ */
2091         if (!rdev->irq.installed) {
2092                 r = radeon_irq_kms_init(rdev);
2093                 if (r)
2094                         return r;
2095         }
2096
2097         r = r600_irq_init(rdev);
2098         if (r) {
2099                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
2100                 radeon_irq_kms_fini(rdev);
2101                 return r;
2102         }
2103         evergreen_irq_set(rdev);
2104
2105         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
2106                              RADEON_CP_PACKET2);
2107         if (r)
2108                 return r;
2109
2110         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
2111         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
2112                              DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
2113         if (r)
2114                 return r;
2115
2116         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
2117         r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
2118                              DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
2119         if (r)
2120                 return r;
2121
2122         r = cayman_cp_load_microcode(rdev);
2123         if (r)
2124                 return r;
2125         r = cayman_cp_resume(rdev);
2126         if (r)
2127                 return r;
2128
2129         r = cayman_dma_resume(rdev);
2130         if (r)
2131                 return r;
2132
2133         ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
2134         if (ring->ring_size) {
2135                 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
2136                                      RADEON_CP_PACKET2);
2137                 if (!r)
2138                         r = uvd_v1_0_init(rdev);
2139                 if (r)
2140                         DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
2141         }
2142
2143         if (rdev->family == CHIP_ARUBA) {
2144                 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
2145                 if (ring->ring_size)
2146                         r = radeon_ring_init(rdev, ring, ring->ring_size, 0, 0x0);
2147
2148                 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
2149                 if (ring->ring_size)
2150                         r = radeon_ring_init(rdev, ring, ring->ring_size, 0, 0x0);
2151
2152                 if (!r)
2153                         r = vce_v1_0_init(rdev);
2154                 if (r)
2155                         DRM_ERROR("radeon: failed initializing VCE (%d).\n", r);
2156         }
2157
2158         r = radeon_ib_pool_init(rdev);
2159         if (r) {
2160                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
2161                 return r;
2162         }
2163
2164         r = radeon_vm_manager_init(rdev);
2165         if (r) {
2166                 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
2167                 return r;
2168         }
2169
2170         r = radeon_audio_init(rdev);
2171         if (r)
2172                 return r;
2173
2174         return 0;
2175 }
2176
2177 int cayman_resume(struct radeon_device *rdev)
2178 {
2179         int r;
2180
2181         /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
2182          * posting will perform necessary task to bring back GPU into good
2183          * shape.
2184          */
2185         /* post card */
2186         atom_asic_init(rdev->mode_info.atom_context);
2187
2188         /* init golden registers */
2189         ni_init_golden_registers(rdev);
2190
2191         if (rdev->pm.pm_method == PM_METHOD_DPM)
2192                 radeon_pm_resume(rdev);
2193
2194         rdev->accel_working = true;
2195         r = cayman_startup(rdev);
2196         if (r) {
2197                 DRM_ERROR("cayman startup failed on resume\n");
2198                 rdev->accel_working = false;
2199                 return r;
2200         }
2201         return r;
2202 }
2203
2204 int cayman_suspend(struct radeon_device *rdev)
2205 {
2206         radeon_pm_suspend(rdev);
2207         radeon_audio_fini(rdev);
2208         radeon_vm_manager_fini(rdev);
2209         cayman_cp_enable(rdev, false);
2210         cayman_dma_stop(rdev);
2211         uvd_v1_0_fini(rdev);
2212         radeon_uvd_suspend(rdev);
2213         evergreen_irq_suspend(rdev);
2214         radeon_wb_disable(rdev);
2215         cayman_pcie_gart_disable(rdev);
2216         return 0;
2217 }
2218
2219 /* Plan is to move initialization in that function and use
2220  * helper function so that radeon_device_init pretty much
2221  * do nothing more than calling asic specific function. This
2222  * should also allow to remove a bunch of callback function
2223  * like vram_info.
2224  */
2225 int cayman_init(struct radeon_device *rdev)
2226 {
2227         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2228         int r;
2229
2230         /* Read BIOS */
2231         if (!radeon_get_bios(rdev)) {
2232                 if (ASIC_IS_AVIVO(rdev))
2233                         return -EINVAL;
2234         }
2235         /* Must be an ATOMBIOS */
2236         if (!rdev->is_atom_bios) {
2237                 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
2238                 return -EINVAL;
2239         }
2240         r = radeon_atombios_init(rdev);
2241         if (r)
2242                 return r;
2243
2244         /* Post card if necessary */
2245         if (!radeon_card_posted(rdev)) {
2246                 if (!rdev->bios) {
2247                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
2248                         return -EINVAL;
2249                 }
2250                 DRM_INFO("GPU not posted. posting now...\n");
2251                 atom_asic_init(rdev->mode_info.atom_context);
2252         }
2253         /* init golden registers */
2254         ni_init_golden_registers(rdev);
2255         /* Initialize scratch registers */
2256         r600_scratch_init(rdev);
2257         /* Initialize surface registers */
2258         radeon_surface_init(rdev);
2259         /* Initialize clocks */
2260         radeon_get_clock_info(rdev->ddev);
2261         /* Fence driver */
2262         r = radeon_fence_driver_init(rdev);
2263         if (r)
2264                 return r;
2265         /* initialize memory controller */
2266         r = evergreen_mc_init(rdev);
2267         if (r)
2268                 return r;
2269         /* Memory manager */
2270         r = radeon_bo_init(rdev);
2271         if (r)
2272                 return r;
2273
2274         if (rdev->flags & RADEON_IS_IGP) {
2275                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
2276                         r = ni_init_microcode(rdev);
2277                         if (r) {
2278                                 DRM_ERROR("Failed to load firmware!\n");
2279                                 return r;
2280                         }
2281                 }
2282         } else {
2283                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
2284                         r = ni_init_microcode(rdev);
2285                         if (r) {
2286                                 DRM_ERROR("Failed to load firmware!\n");
2287                                 return r;
2288                         }
2289                 }
2290         }
2291
2292         /* Initialize power management */
2293         radeon_pm_init(rdev);
2294
2295         ring->ring_obj = NULL;
2296         r600_ring_init(rdev, ring, 1024 * 1024);
2297
2298         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
2299         ring->ring_obj = NULL;
2300         r600_ring_init(rdev, ring, 64 * 1024);
2301
2302         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
2303         ring->ring_obj = NULL;
2304         r600_ring_init(rdev, ring, 64 * 1024);
2305
2306         r = radeon_uvd_init(rdev);
2307         if (!r) {
2308                 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
2309                 ring->ring_obj = NULL;
2310                 r600_ring_init(rdev, ring, 4096);
2311         }
2312
2313         if (rdev->family == CHIP_ARUBA) {
2314                 r = radeon_vce_init(rdev);
2315                 if (!r) {
2316                         ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
2317                         ring->ring_obj = NULL;
2318                         r600_ring_init(rdev, ring, 4096);
2319
2320                         ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
2321                         ring->ring_obj = NULL;
2322                         r600_ring_init(rdev, ring, 4096);
2323                 }
2324         }
2325
2326         rdev->ih.ring_obj = NULL;
2327         r600_ih_ring_init(rdev, 64 * 1024);
2328
2329         r = r600_pcie_gart_init(rdev);
2330         if (r)
2331                 return r;
2332
2333         rdev->accel_working = true;
2334         r = cayman_startup(rdev);
2335         if (r) {
2336                 dev_err(rdev->dev, "disabling GPU acceleration\n");
2337                 cayman_cp_fini(rdev);
2338                 cayman_dma_fini(rdev);
2339                 r600_irq_fini(rdev);
2340                 if (rdev->flags & RADEON_IS_IGP)
2341                         sumo_rlc_fini(rdev);
2342                 radeon_wb_fini(rdev);
2343                 radeon_ib_pool_fini(rdev);
2344                 radeon_vm_manager_fini(rdev);
2345                 radeon_irq_kms_fini(rdev);
2346                 cayman_pcie_gart_fini(rdev);
2347                 rdev->accel_working = false;
2348         }
2349
2350         /* Don't start up if the MC ucode is missing.
2351          * The default clocks and voltages before the MC ucode
2352          * is loaded are not suffient for advanced operations.
2353          *
2354          * We can skip this check for TN, because there is no MC
2355          * ucode.
2356          */
2357         if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
2358                 DRM_ERROR("radeon: MC ucode required for NI+.\n");
2359                 return -EINVAL;
2360         }
2361
2362         return 0;
2363 }
2364
2365 void cayman_fini(struct radeon_device *rdev)
2366 {
2367         radeon_pm_fini(rdev);
2368         cayman_cp_fini(rdev);
2369         cayman_dma_fini(rdev);
2370         r600_irq_fini(rdev);
2371         if (rdev->flags & RADEON_IS_IGP)
2372                 sumo_rlc_fini(rdev);
2373         radeon_wb_fini(rdev);
2374         radeon_vm_manager_fini(rdev);
2375         radeon_ib_pool_fini(rdev);
2376         radeon_irq_kms_fini(rdev);
2377         uvd_v1_0_fini(rdev);
2378         radeon_uvd_fini(rdev);
2379         if (rdev->family == CHIP_ARUBA)
2380                 radeon_vce_fini(rdev);
2381         cayman_pcie_gart_fini(rdev);
2382         r600_vram_scratch_fini(rdev);
2383         radeon_gem_fini(rdev);
2384         radeon_fence_driver_fini(rdev);
2385         radeon_bo_fini(rdev);
2386         radeon_atombios_fini(rdev);
2387         kfree(rdev->bios);
2388         rdev->bios = NULL;
2389 }
2390
2391 /*
2392  * vm
2393  */
2394 int cayman_vm_init(struct radeon_device *rdev)
2395 {
2396         /* number of VMs */
2397         rdev->vm_manager.nvm = 8;
2398         /* base offset of vram pages */
2399         if (rdev->flags & RADEON_IS_IGP) {
2400                 u64 tmp = RREG32(FUS_MC_VM_FB_OFFSET);
2401                 tmp <<= 22;
2402                 rdev->vm_manager.vram_base_offset = tmp;
2403         } else
2404                 rdev->vm_manager.vram_base_offset = 0;
2405         return 0;
2406 }
2407
2408 void cayman_vm_fini(struct radeon_device *rdev)
2409 {
2410 }
2411
2412 /**
2413  * cayman_vm_decode_fault - print human readable fault info
2414  *
2415  * @rdev: radeon_device pointer
2416  * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
2417  * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
2418  *
2419  * Print human readable fault information (cayman/TN).
2420  */
2421 void cayman_vm_decode_fault(struct radeon_device *rdev,
2422                             u32 status, u32 addr)
2423 {
2424         u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
2425         u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
2426         u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
2427         char *block;
2428
2429         switch (mc_id) {
2430         case 32:
2431         case 16:
2432         case 96:
2433         case 80:
2434         case 160:
2435         case 144:
2436         case 224:
2437         case 208:
2438                 block = "CB";
2439                 break;
2440         case 33:
2441         case 17:
2442         case 97:
2443         case 81:
2444         case 161:
2445         case 145:
2446         case 225:
2447         case 209:
2448                 block = "CB_FMASK";
2449                 break;
2450         case 34:
2451         case 18:
2452         case 98:
2453         case 82:
2454         case 162:
2455         case 146:
2456         case 226:
2457         case 210:
2458                 block = "CB_CMASK";
2459                 break;
2460         case 35:
2461         case 19:
2462         case 99:
2463         case 83:
2464         case 163:
2465         case 147:
2466         case 227:
2467         case 211:
2468                 block = "CB_IMMED";
2469                 break;
2470         case 36:
2471         case 20:
2472         case 100:
2473         case 84:
2474         case 164:
2475         case 148:
2476         case 228:
2477         case 212:
2478                 block = "DB";
2479                 break;
2480         case 37:
2481         case 21:
2482         case 101:
2483         case 85:
2484         case 165:
2485         case 149:
2486         case 229:
2487         case 213:
2488                 block = "DB_HTILE";
2489                 break;
2490         case 38:
2491         case 22:
2492         case 102:
2493         case 86:
2494         case 166:
2495         case 150:
2496         case 230:
2497         case 214:
2498                 block = "SX";
2499                 break;
2500         case 39:
2501         case 23:
2502         case 103:
2503         case 87:
2504         case 167:
2505         case 151:
2506         case 231:
2507         case 215:
2508                 block = "DB_STEN";
2509                 break;
2510         case 40:
2511         case 24:
2512         case 104:
2513         case 88:
2514         case 232:
2515         case 216:
2516         case 168:
2517         case 152:
2518                 block = "TC_TFETCH";
2519                 break;
2520         case 41:
2521         case 25:
2522         case 105:
2523         case 89:
2524         case 233:
2525         case 217:
2526         case 169:
2527         case 153:
2528                 block = "TC_VFETCH";
2529                 break;
2530         case 42:
2531         case 26:
2532         case 106:
2533         case 90:
2534         case 234:
2535         case 218:
2536         case 170:
2537         case 154:
2538                 block = "VC";
2539                 break;
2540         case 112:
2541                 block = "CP";
2542                 break;
2543         case 113:
2544         case 114:
2545                 block = "SH";
2546                 break;
2547         case 115:
2548                 block = "VGT";
2549                 break;
2550         case 178:
2551                 block = "IH";
2552                 break;
2553         case 51:
2554                 block = "RLC";
2555                 break;
2556         case 55:
2557                 block = "DMA";
2558                 break;
2559         case 56:
2560                 block = "HDP";
2561                 break;
2562         default:
2563                 block = "unknown";
2564                 break;
2565         }
2566
2567         printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
2568                protections, vmid, addr,
2569                (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
2570                block, mc_id);
2571 }
2572
2573 /**
2574  * cayman_vm_flush - vm flush using the CP
2575  *
2576  * @rdev: radeon_device pointer
2577  *
2578  * Update the page table base and flush the VM TLB
2579  * using the CP (cayman-si).
2580  */
2581 void cayman_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
2582                      unsigned vm_id, uint64_t pd_addr)
2583 {
2584         radeon_ring_write(ring, PACKET0(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2), 0));
2585         radeon_ring_write(ring, pd_addr >> 12);
2586
2587         /* flush hdp cache */
2588         radeon_ring_write(ring, PACKET0(HDP_MEM_COHERENCY_FLUSH_CNTL, 0));
2589         radeon_ring_write(ring, 0x1);
2590
2591         /* bits 0-7 are the VM contexts0-7 */
2592         radeon_ring_write(ring, PACKET0(VM_INVALIDATE_REQUEST, 0));
2593         radeon_ring_write(ring, 1 << vm_id);
2594
2595         /* wait for the invalidate to complete */
2596         radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
2597         radeon_ring_write(ring, (WAIT_REG_MEM_FUNCTION(0) |  /* always */
2598                                  WAIT_REG_MEM_ENGINE(0))); /* me */
2599         radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
2600         radeon_ring_write(ring, 0);
2601         radeon_ring_write(ring, 0); /* ref */
2602         radeon_ring_write(ring, 0); /* mask */
2603         radeon_ring_write(ring, 0x20); /* poll interval */
2604
2605         /* sync PFP to ME, otherwise we might get invalid PFP reads */
2606         radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
2607         radeon_ring_write(ring, 0x0);
2608 }
2609
2610 int tn_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
2611 {
2612         struct atom_clock_dividers dividers;
2613         int r, i;
2614
2615         r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
2616                                            ecclk, false, &dividers);
2617         if (r)
2618                 return r;
2619
2620         for (i = 0; i < 100; i++) {
2621                 if (RREG32(CG_ECLK_STATUS) & ECLK_STATUS)
2622                         break;
2623                 mdelay(10);
2624         }
2625         if (i == 100)
2626                 return -ETIMEDOUT;
2627
2628         WREG32_P(CG_ECLK_CNTL, dividers.post_div, ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK));
2629
2630         for (i = 0; i < 100; i++) {
2631                 if (RREG32(CG_ECLK_STATUS) & ECLK_STATUS)
2632                         break;
2633                 mdelay(10);
2634         }
2635         if (i == 100)
2636                 return -ETIMEDOUT;
2637
2638         return 0;
2639 }