drivers/gpu/drm/radeon/ni_dpm.c

   1 /*
   2  * Copyright 2012 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include <drm/drmP.h>
  25 #include "radeon.h"
  26 #include "radeon_asic.h"
  27 #include "nid.h"
  28 #include "r600_dpm.h"
  29 #include "ni_dpm.h"
  30 #include "atom.h"
  31 #include <linux/math64.h>
  32 #include <linux/seq_file.h>
  33
  34 #define MC_CG_ARB_FREQ_F0           0x0a
  35 #define MC_CG_ARB_FREQ_F1           0x0b
  36 #define MC_CG_ARB_FREQ_F2           0x0c
  37 #define MC_CG_ARB_FREQ_F3           0x0d
  38
  39 #define SMC_RAM_END 0xC000
  40
  41 static const struct ni_cac_weights cac_weights_cayman_xt =
  42 {
  43         0x15,
  44         0x2,
  45         0x19,
  46         0x2,
  47         0x8,
  48         0x14,
  49         0x2,
  50         0x16,
  51         0xE,
  52         0x17,
  53         0x13,
  54         0x2B,
  55         0x10,
  56         0x7,
  57         0x5,
  58         0x5,
  59         0x5,
  60         0x2,
  61         0x3,
  62         0x9,
  63         0x10,
  64         0x10,
  65         0x2B,
  66         0xA,
  67         0x9,
  68         0x4,
  69         0xD,
  70         0xD,
  71         0x3E,
  72         0x18,
  73         0x14,
  74         0,
  75         0x3,
  76         0x3,
  77         0x5,
  78         0,
  79         0x2,
  80         0,
  81         0,
  82         0,
  83         0,
  84         0,
  85         0,
  86         0,
  87         0,
  88         0,
  89         0x1CC,
  90         0,
  91         0x164,
  92         1,
  93         1,
  94         1,
  95         1,
  96         12,
  97         12,
  98         12,
  99         0x12,
 100         0x1F,
 101         132,
 102         5,
 103         7,
 104         0,
 105         { 0, 0, 0, 0, 0, 0, 0, 0 },
 106         { 0, 0, 0, 0 },
 107         true
 108 };
 109
 110 static const struct ni_cac_weights cac_weights_cayman_pro =
 111 {
 112         0x16,
 113         0x4,
 114         0x10,
 115         0x2,
 116         0xA,
 117         0x16,
 118         0x2,
 119         0x18,
 120         0x10,
 121         0x1A,
 122         0x16,
 123         0x2D,
 124         0x12,
 125         0xA,
 126         0x6,
 127         0x6,
 128         0x6,
 129         0x2,
 130         0x4,
 131         0xB,
 132         0x11,
 133         0x11,
 134         0x2D,
 135         0xC,
 136         0xC,
 137         0x7,
 138         0x10,
 139         0x10,
 140         0x3F,
 141         0x1A,
 142         0x16,
 143         0,
 144         0x7,
 145         0x4,
 146         0x6,
 147         1,
 148         0x2,
 149         0x1,
 150         0,
 151         0,
 152         0,
 153         0,
 154         0,
 155         0,
 156         0x30,
 157         0,
 158         0x1CF,
 159         0,
 160         0x166,
 161         1,
 162         1,
 163         1,
 164         1,
 165         12,
 166         12,
 167         12,
 168         0x15,
 169         0x1F,
 170         132,
 171         6,
 172         6,
 173         0,
 174         { 0, 0, 0, 0, 0, 0, 0, 0 },
 175         { 0, 0, 0, 0 },
 176         true
 177 };
 178
 179 static const struct ni_cac_weights cac_weights_cayman_le =
 180 {
 181         0x7,
 182         0xE,
 183         0x1,
 184         0xA,
 185         0x1,
 186         0x3F,
 187         0x2,
 188         0x18,
 189         0x10,
 190         0x1A,
 191         0x1,
 192         0x3F,
 193         0x1,
 194         0xE,
 195         0x6,
 196         0x6,
 197         0x6,
 198         0x2,
 199         0x4,
 200         0x9,
 201         0x1A,
 202         0x1A,
 203         0x2C,
 204         0xA,
 205         0x11,
 206         0x8,
 207         0x19,
 208         0x19,
 209         0x1,
 210         0x1,
 211         0x1A,
 212         0,
 213         0x8,
 214         0x5,
 215         0x8,
 216         0x1,
 217         0x3,
 218         0x1,
 219         0,
 220         0,
 221         0,
 222         0,
 223         0,
 224         0,
 225         0x38,
 226         0x38,
 227         0x239,
 228         0x3,
 229         0x18A,
 230         1,
 231         1,
 232         1,
 233         1,
 234         12,
 235         12,
 236         12,
 237         0x15,
 238         0x22,
 239         132,
 240         6,
 241         6,
 242         0,
 243         { 0, 0, 0, 0, 0, 0, 0, 0 },
 244         { 0, 0, 0, 0 },
 245         true
 246 };
 247
 248 #define NISLANDS_MGCG_SEQUENCE  300
 249
 250 static const u32 cayman_cgcg_cgls_default[] =
 251 {
 252         0x000008f8, 0x00000010, 0xffffffff,
 253         0x000008fc, 0x00000000, 0xffffffff,
 254         0x000008f8, 0x00000011, 0xffffffff,
 255         0x000008fc, 0x00000000, 0xffffffff,
 256         0x000008f8, 0x00000012, 0xffffffff,
 257         0x000008fc, 0x00000000, 0xffffffff,
 258         0x000008f8, 0x00000013, 0xffffffff,
 259         0x000008fc, 0x00000000, 0xffffffff,
 260         0x000008f8, 0x00000014, 0xffffffff,
 261         0x000008fc, 0x00000000, 0xffffffff,
 262         0x000008f8, 0x00000015, 0xffffffff,
 263         0x000008fc, 0x00000000, 0xffffffff,
 264         0x000008f8, 0x00000016, 0xffffffff,
 265         0x000008fc, 0x00000000, 0xffffffff,
 266         0x000008f8, 0x00000017, 0xffffffff,
 267         0x000008fc, 0x00000000, 0xffffffff,
 268         0x000008f8, 0x00000018, 0xffffffff,
 269         0x000008fc, 0x00000000, 0xffffffff,
 270         0x000008f8, 0x00000019, 0xffffffff,
 271         0x000008fc, 0x00000000, 0xffffffff,
 272         0x000008f8, 0x0000001a, 0xffffffff,
 273         0x000008fc, 0x00000000, 0xffffffff,
 274         0x000008f8, 0x0000001b, 0xffffffff,
 275         0x000008fc, 0x00000000, 0xffffffff,
 276         0x000008f8, 0x00000020, 0xffffffff,
 277         0x000008fc, 0x00000000, 0xffffffff,
 278         0x000008f8, 0x00000021, 0xffffffff,
 279         0x000008fc, 0x00000000, 0xffffffff,
 280         0x000008f8, 0x00000022, 0xffffffff,
 281         0x000008fc, 0x00000000, 0xffffffff,
 282         0x000008f8, 0x00000023, 0xffffffff,
 283         0x000008fc, 0x00000000, 0xffffffff,
 284         0x000008f8, 0x00000024, 0xffffffff,
 285         0x000008fc, 0x00000000, 0xffffffff,
 286         0x000008f8, 0x00000025, 0xffffffff,
 287         0x000008fc, 0x00000000, 0xffffffff,
 288         0x000008f8, 0x00000026, 0xffffffff,
 289         0x000008fc, 0x00000000, 0xffffffff,
 290         0x000008f8, 0x00000027, 0xffffffff,
 291         0x000008fc, 0x00000000, 0xffffffff,
 292         0x000008f8, 0x00000028, 0xffffffff,
 293         0x000008fc, 0x00000000, 0xffffffff,
 294         0x000008f8, 0x00000029, 0xffffffff,
 295         0x000008fc, 0x00000000, 0xffffffff,
 296         0x000008f8, 0x0000002a, 0xffffffff,
 297         0x000008fc, 0x00000000, 0xffffffff,
 298         0x000008f8, 0x0000002b, 0xffffffff,
 299         0x000008fc, 0x00000000, 0xffffffff
 300 };
 301 #define CAYMAN_CGCG_CGLS_DEFAULT_LENGTH sizeof(cayman_cgcg_cgls_default) / (3 * sizeof(u32))
 302
 303 static const u32 cayman_cgcg_cgls_disable[] =
 304 {
 305         0x000008f8, 0x00000010, 0xffffffff,
 306         0x000008fc, 0xffffffff, 0xffffffff,
 307         0x000008f8, 0x00000011, 0xffffffff,
 308         0x000008fc, 0xffffffff, 0xffffffff,
 309         0x000008f8, 0x00000012, 0xffffffff,
 310         0x000008fc, 0xffffffff, 0xffffffff,
 311         0x000008f8, 0x00000013, 0xffffffff,
 312         0x000008fc, 0xffffffff, 0xffffffff,
 313         0x000008f8, 0x00000014, 0xffffffff,
 314         0x000008fc, 0xffffffff, 0xffffffff,
 315         0x000008f8, 0x00000015, 0xffffffff,
 316         0x000008fc, 0xffffffff, 0xffffffff,
 317         0x000008f8, 0x00000016, 0xffffffff,
 318         0x000008fc, 0xffffffff, 0xffffffff,
 319         0x000008f8, 0x00000017, 0xffffffff,
 320         0x000008fc, 0xffffffff, 0xffffffff,
 321         0x000008f8, 0x00000018, 0xffffffff,
 322         0x000008fc, 0xffffffff, 0xffffffff,
 323         0x000008f8, 0x00000019, 0xffffffff,
 324         0x000008fc, 0xffffffff, 0xffffffff,
 325         0x000008f8, 0x0000001a, 0xffffffff,
 326         0x000008fc, 0xffffffff, 0xffffffff,
 327         0x000008f8, 0x0000001b, 0xffffffff,
 328         0x000008fc, 0xffffffff, 0xffffffff,
 329         0x000008f8, 0x00000020, 0xffffffff,
 330         0x000008fc, 0x00000000, 0xffffffff,
 331         0x000008f8, 0x00000021, 0xffffffff,
 332         0x000008fc, 0x00000000, 0xffffffff,
 333         0x000008f8, 0x00000022, 0xffffffff,
 334         0x000008fc, 0x00000000, 0xffffffff,
 335         0x000008f8, 0x00000023, 0xffffffff,
 336         0x000008fc, 0x00000000, 0xffffffff,
 337         0x000008f8, 0x00000024, 0xffffffff,
 338         0x000008fc, 0x00000000, 0xffffffff,
 339         0x000008f8, 0x00000025, 0xffffffff,
 340         0x000008fc, 0x00000000, 0xffffffff,
 341         0x000008f8, 0x00000026, 0xffffffff,
 342         0x000008fc, 0x00000000, 0xffffffff,
 343         0x000008f8, 0x00000027, 0xffffffff,
 344         0x000008fc, 0x00000000, 0xffffffff,
 345         0x000008f8, 0x00000028, 0xffffffff,
 346         0x000008fc, 0x00000000, 0xffffffff,
 347         0x000008f8, 0x00000029, 0xffffffff,
 348         0x000008fc, 0x00000000, 0xffffffff,
 349         0x000008f8, 0x0000002a, 0xffffffff,
 350         0x000008fc, 0x00000000, 0xffffffff,
 351         0x000008f8, 0x0000002b, 0xffffffff,
 352         0x000008fc, 0x00000000, 0xffffffff,
 353         0x00000644, 0x000f7902, 0x001f4180,
 354         0x00000644, 0x000f3802, 0x001f4180
 355 };
 356 #define CAYMAN_CGCG_CGLS_DISABLE_LENGTH sizeof(cayman_cgcg_cgls_disable) / (3 * sizeof(u32))
 357
 358 static const u32 cayman_cgcg_cgls_enable[] =
 359 {
 360         0x00000644, 0x000f7882, 0x001f4080,
 361         0x000008f8, 0x00000010, 0xffffffff,
 362         0x000008fc, 0x00000000, 0xffffffff,
 363         0x000008f8, 0x00000011, 0xffffffff,
 364         0x000008fc, 0x00000000, 0xffffffff,
 365         0x000008f8, 0x00000012, 0xffffffff,
 366         0x000008fc, 0x00000000, 0xffffffff,
 367         0x000008f8, 0x00000013, 0xffffffff,
 368         0x000008fc, 0x00000000, 0xffffffff,
 369         0x000008f8, 0x00000014, 0xffffffff,
 370         0x000008fc, 0x00000000, 0xffffffff,
 371         0x000008f8, 0x00000015, 0xffffffff,
 372         0x000008fc, 0x00000000, 0xffffffff,
 373         0x000008f8, 0x00000016, 0xffffffff,
 374         0x000008fc, 0x00000000, 0xffffffff,
 375         0x000008f8, 0x00000017, 0xffffffff,
 376         0x000008fc, 0x00000000, 0xffffffff,
 377         0x000008f8, 0x00000018, 0xffffffff,
 378         0x000008fc, 0x00000000, 0xffffffff,
 379         0x000008f8, 0x00000019, 0xffffffff,
 380         0x000008fc, 0x00000000, 0xffffffff,
 381         0x000008f8, 0x0000001a, 0xffffffff,
 382         0x000008fc, 0x00000000, 0xffffffff,
 383         0x000008f8, 0x0000001b, 0xffffffff,
 384         0x000008fc, 0x00000000, 0xffffffff,
 385         0x000008f8, 0x00000020, 0xffffffff,
 386         0x000008fc, 0xffffffff, 0xffffffff,
 387         0x000008f8, 0x00000021, 0xffffffff,
 388         0x000008fc, 0xffffffff, 0xffffffff,
 389         0x000008f8, 0x00000022, 0xffffffff,
 390         0x000008fc, 0xffffffff, 0xffffffff,
 391         0x000008f8, 0x00000023, 0xffffffff,
 392         0x000008fc, 0xffffffff, 0xffffffff,
 393         0x000008f8, 0x00000024, 0xffffffff,
 394         0x000008fc, 0xffffffff, 0xffffffff,
 395         0x000008f8, 0x00000025, 0xffffffff,
 396         0x000008fc, 0xffffffff, 0xffffffff,
 397         0x000008f8, 0x00000026, 0xffffffff,
 398         0x000008fc, 0xffffffff, 0xffffffff,
 399         0x000008f8, 0x00000027, 0xffffffff,
 400         0x000008fc, 0xffffffff, 0xffffffff,
 401         0x000008f8, 0x00000028, 0xffffffff,
 402         0x000008fc, 0xffffffff, 0xffffffff,
 403         0x000008f8, 0x00000029, 0xffffffff,
 404         0x000008fc, 0xffffffff, 0xffffffff,
 405         0x000008f8, 0x0000002a, 0xffffffff,
 406         0x000008fc, 0xffffffff, 0xffffffff,
 407         0x000008f8, 0x0000002b, 0xffffffff,
 408         0x000008fc, 0xffffffff, 0xffffffff
 409 };
 410 #define CAYMAN_CGCG_CGLS_ENABLE_LENGTH  sizeof(cayman_cgcg_cgls_enable) / (3 * sizeof(u32))
 411
 412 static const u32 cayman_mgcg_default[] =
 413 {
 414         0x0000802c, 0xc0000000, 0xffffffff,
 415         0x00003fc4, 0xc0000000, 0xffffffff,
 416         0x00005448, 0x00000100, 0xffffffff,
 417         0x000055e4, 0x00000100, 0xffffffff,
 418         0x0000160c, 0x00000100, 0xffffffff,
 419         0x00008984, 0x06000100, 0xffffffff,
 420         0x0000c164, 0x00000100, 0xffffffff,
 421         0x00008a18, 0x00000100, 0xffffffff,
 422         0x0000897c, 0x06000100, 0xffffffff,
 423         0x00008b28, 0x00000100, 0xffffffff,
 424         0x00009144, 0x00800200, 0xffffffff,
 425         0x00009a60, 0x00000100, 0xffffffff,
 426         0x00009868, 0x00000100, 0xffffffff,
 427         0x00008d58, 0x00000100, 0xffffffff,
 428         0x00009510, 0x00000100, 0xffffffff,
 429         0x0000949c, 0x00000100, 0xffffffff,
 430         0x00009654, 0x00000100, 0xffffffff,
 431         0x00009030, 0x00000100, 0xffffffff,
 432         0x00009034, 0x00000100, 0xffffffff,
 433         0x00009038, 0x00000100, 0xffffffff,
 434         0x0000903c, 0x00000100, 0xffffffff,
 435         0x00009040, 0x00000100, 0xffffffff,
 436         0x0000a200, 0x00000100, 0xffffffff,
 437         0x0000a204, 0x00000100, 0xffffffff,
 438         0x0000a208, 0x00000100, 0xffffffff,
 439         0x0000a20c, 0x00000100, 0xffffffff,
 440         0x00009744, 0x00000100, 0xffffffff,
 441         0x00003f80, 0x00000100, 0xffffffff,
 442         0x0000a210, 0x00000100, 0xffffffff,
 443         0x0000a214, 0x00000100, 0xffffffff,
 444         0x000004d8, 0x00000100, 0xffffffff,
 445         0x00009664, 0x00000100, 0xffffffff,
 446         0x00009698, 0x00000100, 0xffffffff,
 447         0x000004d4, 0x00000200, 0xffffffff,
 448         0x000004d0, 0x00000000, 0xffffffff,
 449         0x000030cc, 0x00000104, 0xffffffff,
 450         0x0000d0c0, 0x00000100, 0xffffffff,
 451         0x0000d8c0, 0x00000100, 0xffffffff,
 452         0x0000802c, 0x40000000, 0xffffffff,
 453         0x00003fc4, 0x40000000, 0xffffffff,
 454         0x0000915c, 0x00010000, 0xffffffff,
 455         0x00009160, 0x00030002, 0xffffffff,
 456         0x00009164, 0x00050004, 0xffffffff,
 457         0x00009168, 0x00070006, 0xffffffff,
 458         0x00009178, 0x00070000, 0xffffffff,
 459         0x0000917c, 0x00030002, 0xffffffff,
 460         0x00009180, 0x00050004, 0xffffffff,
 461         0x0000918c, 0x00010006, 0xffffffff,
 462         0x00009190, 0x00090008, 0xffffffff,
 463         0x00009194, 0x00070000, 0xffffffff,
 464         0x00009198, 0x00030002, 0xffffffff,
 465         0x0000919c, 0x00050004, 0xffffffff,
 466         0x000091a8, 0x00010006, 0xffffffff,
 467         0x000091ac, 0x00090008, 0xffffffff,
 468         0x000091b0, 0x00070000, 0xffffffff,
 469         0x000091b4, 0x00030002, 0xffffffff,
 470         0x000091b8, 0x00050004, 0xffffffff,
 471         0x000091c4, 0x00010006, 0xffffffff,
 472         0x000091c8, 0x00090008, 0xffffffff,
 473         0x000091cc, 0x00070000, 0xffffffff,
 474         0x000091d0, 0x00030002, 0xffffffff,
 475         0x000091d4, 0x00050004, 0xffffffff,
 476         0x000091e0, 0x00010006, 0xffffffff,
 477         0x000091e4, 0x00090008, 0xffffffff,
 478         0x000091e8, 0x00000000, 0xffffffff,
 479         0x000091ec, 0x00070000, 0xffffffff,
 480         0x000091f0, 0x00030002, 0xffffffff,
 481         0x000091f4, 0x00050004, 0xffffffff,
 482         0x00009200, 0x00010006, 0xffffffff,
 483         0x00009204, 0x00090008, 0xffffffff,
 484         0x00009208, 0x00070000, 0xffffffff,
 485         0x0000920c, 0x00030002, 0xffffffff,
 486         0x00009210, 0x00050004, 0xffffffff,
 487         0x0000921c, 0x00010006, 0xffffffff,
 488         0x00009220, 0x00090008, 0xffffffff,
 489         0x00009224, 0x00070000, 0xffffffff,
 490         0x00009228, 0x00030002, 0xffffffff,
 491         0x0000922c, 0x00050004, 0xffffffff,
 492         0x00009238, 0x00010006, 0xffffffff,
 493         0x0000923c, 0x00090008, 0xffffffff,
 494         0x00009240, 0x00070000, 0xffffffff,
 495         0x00009244, 0x00030002, 0xffffffff,
 496         0x00009248, 0x00050004, 0xffffffff,
 497         0x00009254, 0x00010006, 0xffffffff,
 498         0x00009258, 0x00090008, 0xffffffff,
 499         0x0000925c, 0x00070000, 0xffffffff,
 500         0x00009260, 0x00030002, 0xffffffff,
 501         0x00009264, 0x00050004, 0xffffffff,
 502         0x00009270, 0x00010006, 0xffffffff,
 503         0x00009274, 0x00090008, 0xffffffff,
 504         0x00009278, 0x00070000, 0xffffffff,
 505         0x0000927c, 0x00030002, 0xffffffff,
 506         0x00009280, 0x00050004, 0xffffffff,
 507         0x0000928c, 0x00010006, 0xffffffff,
 508         0x00009290, 0x00090008, 0xffffffff,
 509         0x000092a8, 0x00070000, 0xffffffff,
 510         0x000092ac, 0x00030002, 0xffffffff,
 511         0x000092b0, 0x00050004, 0xffffffff,
 512         0x000092bc, 0x00010006, 0xffffffff,
 513         0x000092c0, 0x00090008, 0xffffffff,
 514         0x000092c4, 0x00070000, 0xffffffff,
 515         0x000092c8, 0x00030002, 0xffffffff,
 516         0x000092cc, 0x00050004, 0xffffffff,
 517         0x000092d8, 0x00010006, 0xffffffff,
 518         0x000092dc, 0x00090008, 0xffffffff,
 519         0x00009294, 0x00000000, 0xffffffff,
 520         0x0000802c, 0x40010000, 0xffffffff,
 521         0x00003fc4, 0x40010000, 0xffffffff,
 522         0x0000915c, 0x00010000, 0xffffffff,
 523         0x00009160, 0x00030002, 0xffffffff,
 524         0x00009164, 0x00050004, 0xffffffff,
 525         0x00009168, 0x00070006, 0xffffffff,
 526         0x00009178, 0x00070000, 0xffffffff,
 527         0x0000917c, 0x00030002, 0xffffffff,
 528         0x00009180, 0x00050004, 0xffffffff,
 529         0x0000918c, 0x00010006, 0xffffffff,
 530         0x00009190, 0x00090008, 0xffffffff,
 531         0x00009194, 0x00070000, 0xffffffff,
 532         0x00009198, 0x00030002, 0xffffffff,
 533         0x0000919c, 0x00050004, 0xffffffff,
 534         0x000091a8, 0x00010006, 0xffffffff,
 535         0x000091ac, 0x00090008, 0xffffffff,
 536         0x000091b0, 0x00070000, 0xffffffff,
 537         0x000091b4, 0x00030002, 0xffffffff,
 538         0x000091b8, 0x00050004, 0xffffffff,
 539         0x000091c4, 0x00010006, 0xffffffff,
 540         0x000091c8, 0x00090008, 0xffffffff,
 541         0x000091cc, 0x00070000, 0xffffffff,
 542         0x000091d0, 0x00030002, 0xffffffff,
 543         0x000091d4, 0x00050004, 0xffffffff,
 544         0x000091e0, 0x00010006, 0xffffffff,
 545         0x000091e4, 0x00090008, 0xffffffff,
 546         0x000091e8, 0x00000000, 0xffffffff,
 547         0x000091ec, 0x00070000, 0xffffffff,
 548         0x000091f0, 0x00030002, 0xffffffff,
 549         0x000091f4, 0x00050004, 0xffffffff,
 550         0x00009200, 0x00010006, 0xffffffff,
 551         0x00009204, 0x00090008, 0xffffffff,
 552         0x00009208, 0x00070000, 0xffffffff,
 553         0x0000920c, 0x00030002, 0xffffffff,
 554         0x00009210, 0x00050004, 0xffffffff,
 555         0x0000921c, 0x00010006, 0xffffffff,
 556         0x00009220, 0x00090008, 0xffffffff,
 557         0x00009224, 0x00070000, 0xffffffff,
 558         0x00009228, 0x00030002, 0xffffffff,
 559         0x0000922c, 0x00050004, 0xffffffff,
 560         0x00009238, 0x00010006, 0xffffffff,
 561         0x0000923c, 0x00090008, 0xffffffff,
 562         0x00009240, 0x00070000, 0xffffffff,
 563         0x00009244, 0x00030002, 0xffffffff,
 564         0x00009248, 0x00050004, 0xffffffff,
 565         0x00009254, 0x00010006, 0xffffffff,
 566         0x00009258, 0x00090008, 0xffffffff,
 567         0x0000925c, 0x00070000, 0xffffffff,
 568         0x00009260, 0x00030002, 0xffffffff,
 569         0x00009264, 0x00050004, 0xffffffff,
 570         0x00009270, 0x00010006, 0xffffffff,
 571         0x00009274, 0x00090008, 0xffffffff,
 572         0x00009278, 0x00070000, 0xffffffff,
 573         0x0000927c, 0x00030002, 0xffffffff,
 574         0x00009280, 0x00050004, 0xffffffff,
 575         0x0000928c, 0x00010006, 0xffffffff,
 576         0x00009290, 0x00090008, 0xffffffff,
 577         0x000092a8, 0x00070000, 0xffffffff,
 578         0x000092ac, 0x00030002, 0xffffffff,
 579         0x000092b0, 0x00050004, 0xffffffff,
 580         0x000092bc, 0x00010006, 0xffffffff,
 581         0x000092c0, 0x00090008, 0xffffffff,
 582         0x000092c4, 0x00070000, 0xffffffff,
 583         0x000092c8, 0x00030002, 0xffffffff,
 584         0x000092cc, 0x00050004, 0xffffffff,
 585         0x000092d8, 0x00010006, 0xffffffff,
 586         0x000092dc, 0x00090008, 0xffffffff,
 587         0x00009294, 0x00000000, 0xffffffff,
 588         0x0000802c, 0xc0000000, 0xffffffff,
 589         0x00003fc4, 0xc0000000, 0xffffffff,
 590         0x000008f8, 0x00000010, 0xffffffff,
 591         0x000008fc, 0x00000000, 0xffffffff,
 592         0x000008f8, 0x00000011, 0xffffffff,
 593         0x000008fc, 0x00000000, 0xffffffff,
 594         0x000008f8, 0x00000012, 0xffffffff,
 595         0x000008fc, 0x00000000, 0xffffffff,
 596         0x000008f8, 0x00000013, 0xffffffff,
 597         0x000008fc, 0x00000000, 0xffffffff,
 598         0x000008f8, 0x00000014, 0xffffffff,
 599         0x000008fc, 0x00000000, 0xffffffff,
 600         0x000008f8, 0x00000015, 0xffffffff,
 601         0x000008fc, 0x00000000, 0xffffffff,
 602         0x000008f8, 0x00000016, 0xffffffff,
 603         0x000008fc, 0x00000000, 0xffffffff,
 604         0x000008f8, 0x00000017, 0xffffffff,
 605         0x000008fc, 0x00000000, 0xffffffff,
 606         0x000008f8, 0x00000018, 0xffffffff,
 607         0x000008fc, 0x00000000, 0xffffffff,
 608         0x000008f8, 0x00000019, 0xffffffff,
 609         0x000008fc, 0x00000000, 0xffffffff,
 610         0x000008f8, 0x0000001a, 0xffffffff,
 611         0x000008fc, 0x00000000, 0xffffffff,
 612         0x000008f8, 0x0000001b, 0xffffffff,
 613         0x000008fc, 0x00000000, 0xffffffff
 614 };
 615 #define CAYMAN_MGCG_DEFAULT_LENGTH sizeof(cayman_mgcg_default) / (3 * sizeof(u32))
 616
 617 static const u32 cayman_mgcg_disable[] =
 618 {
 619         0x0000802c, 0xc0000000, 0xffffffff,
 620         0x000008f8, 0x00000000, 0xffffffff,
 621         0x000008fc, 0xffffffff, 0xffffffff,
 622         0x000008f8, 0x00000001, 0xffffffff,
 623         0x000008fc, 0xffffffff, 0xffffffff,
 624         0x000008f8, 0x00000002, 0xffffffff,
 625         0x000008fc, 0xffffffff, 0xffffffff,
 626         0x000008f8, 0x00000003, 0xffffffff,
 627         0x000008fc, 0xffffffff, 0xffffffff,
 628         0x00009150, 0x00600000, 0xffffffff
 629 };
 630 #define CAYMAN_MGCG_DISABLE_LENGTH   sizeof(cayman_mgcg_disable) / (3 * sizeof(u32))
 631
 632 static const u32 cayman_mgcg_enable[] =
 633 {
 634         0x0000802c, 0xc0000000, 0xffffffff,
 635         0x000008f8, 0x00000000, 0xffffffff,
 636         0x000008fc, 0x00000000, 0xffffffff,
 637         0x000008f8, 0x00000001, 0xffffffff,
 638         0x000008fc, 0x00000000, 0xffffffff,
 639         0x000008f8, 0x00000002, 0xffffffff,
 640         0x000008fc, 0x00600000, 0xffffffff,
 641         0x000008f8, 0x00000003, 0xffffffff,
 642         0x000008fc, 0x00000000, 0xffffffff,
 643         0x00009150, 0x96944200, 0xffffffff
 644 };
 645
 646 #define CAYMAN_MGCG_ENABLE_LENGTH   sizeof(cayman_mgcg_enable) / (3 * sizeof(u32))
 647
 648 #define NISLANDS_SYSLS_SEQUENCE  100
 649
 650 static const u32 cayman_sysls_default[] =
 651 {
 652         /* Register,   Value,     Mask bits */
 653         0x000055e8, 0x00000000, 0xffffffff,
 654         0x0000d0bc, 0x00000000, 0xffffffff,
 655         0x0000d8bc, 0x00000000, 0xffffffff,
 656         0x000015c0, 0x000c1401, 0xffffffff,
 657         0x0000264c, 0x000c0400, 0xffffffff,
 658         0x00002648, 0x000c0400, 0xffffffff,
 659         0x00002650, 0x000c0400, 0xffffffff,
 660         0x000020b8, 0x000c0400, 0xffffffff,
 661         0x000020bc, 0x000c0400, 0xffffffff,
 662         0x000020c0, 0x000c0c80, 0xffffffff,
 663         0x0000f4a0, 0x000000c0, 0xffffffff,
 664         0x0000f4a4, 0x00680fff, 0xffffffff,
 665         0x00002f50, 0x00000404, 0xffffffff,
 666         0x000004c8, 0x00000001, 0xffffffff,
 667         0x000064ec, 0x00000000, 0xffffffff,
 668         0x00000c7c, 0x00000000, 0xffffffff,
 669         0x00008dfc, 0x00000000, 0xffffffff
 670 };
 671 #define CAYMAN_SYSLS_DEFAULT_LENGTH sizeof(cayman_sysls_default) / (3 * sizeof(u32))
 672
 673 static const u32 cayman_sysls_disable[] =
 674 {
 675         /* Register,   Value,     Mask bits */
 676         0x0000d0c0, 0x00000000, 0xffffffff,
 677         0x0000d8c0, 0x00000000, 0xffffffff,
 678         0x000055e8, 0x00000000, 0xffffffff,
 679         0x0000d0bc, 0x00000000, 0xffffffff,
 680         0x0000d8bc, 0x00000000, 0xffffffff,
 681         0x000015c0, 0x00041401, 0xffffffff,
 682         0x0000264c, 0x00040400, 0xffffffff,
 683         0x00002648, 0x00040400, 0xffffffff,
 684         0x00002650, 0x00040400, 0xffffffff,
 685         0x000020b8, 0x00040400, 0xffffffff,
 686         0x000020bc, 0x00040400, 0xffffffff,
 687         0x000020c0, 0x00040c80, 0xffffffff,
 688         0x0000f4a0, 0x000000c0, 0xffffffff,
 689         0x0000f4a4, 0x00680000, 0xffffffff,
 690         0x00002f50, 0x00000404, 0xffffffff,
 691         0x000004c8, 0x00000001, 0xffffffff,
 692         0x000064ec, 0x00007ffd, 0xffffffff,
 693         0x00000c7c, 0x0000ff00, 0xffffffff,
 694         0x00008dfc, 0x0000007f, 0xffffffff
 695 };
 696 #define CAYMAN_SYSLS_DISABLE_LENGTH sizeof(cayman_sysls_disable) / (3 * sizeof(u32))
 697
 698 static const u32 cayman_sysls_enable[] =
 699 {
 700         /* Register,   Value,     Mask bits */
 701         0x000055e8, 0x00000001, 0xffffffff,
 702         0x0000d0bc, 0x00000100, 0xffffffff,
 703         0x0000d8bc, 0x00000100, 0xffffffff,
 704         0x000015c0, 0x000c1401, 0xffffffff,
 705         0x0000264c, 0x000c0400, 0xffffffff,
 706         0x00002648, 0x000c0400, 0xffffffff,
 707         0x00002650, 0x000c0400, 0xffffffff,
 708         0x000020b8, 0x000c0400, 0xffffffff,
 709         0x000020bc, 0x000c0400, 0xffffffff,
 710         0x000020c0, 0x000c0c80, 0xffffffff,
 711         0x0000f4a0, 0x000000c0, 0xffffffff,
 712         0x0000f4a4, 0x00680fff, 0xffffffff,
 713         0x00002f50, 0x00000903, 0xffffffff,
 714         0x000004c8, 0x00000000, 0xffffffff,
 715         0x000064ec, 0x00000000, 0xffffffff,
 716         0x00000c7c, 0x00000000, 0xffffffff,
 717         0x00008dfc, 0x00000000, 0xffffffff
 718 };
 719 #define CAYMAN_SYSLS_ENABLE_LENGTH sizeof(cayman_sysls_enable) / (3 * sizeof(u32))
 720
 721 struct rv7xx_power_info *rv770_get_pi(struct radeon_device *rdev);
 722 struct evergreen_power_info *evergreen_get_pi(struct radeon_device *rdev);
 723
 724 extern int ni_mc_load_microcode(struct radeon_device *rdev);
 725
 726 struct ni_power_info *ni_get_pi(struct radeon_device *rdev)
 727 {
 728         struct ni_power_info *pi = rdev->pm.dpm.priv;
 729
 730         return pi;
 731 }
 732
 733 struct ni_ps *ni_get_ps(struct radeon_ps *rps)
 734 {
 735         struct ni_ps *ps = rps->ps_priv;
 736
 737         return ps;
 738 }
 739
 740 static void ni_calculate_leakage_for_v_and_t_formula(const struct ni_leakage_coeffients *coeff,
 741                                                      u16 v, s32 t,
 742                                                      u32 ileakage,
 743                                                      u32 *leakage)
 744 {
 745         s64 kt, kv, leakage_w, i_leakage, vddc, temperature;
 746
 747         i_leakage = div64_s64(drm_int2fixp(ileakage), 1000);
 748         vddc = div64_s64(drm_int2fixp(v), 1000);
 749         temperature = div64_s64(drm_int2fixp(t), 1000);
 750
 751         kt = drm_fixp_mul(div64_s64(drm_int2fixp(coeff->at), 1000),
 752                           drm_fixp_exp(drm_fixp_mul(div64_s64(drm_int2fixp(coeff->bt), 1000), temperature)));
 753         kv = drm_fixp_mul(div64_s64(drm_int2fixp(coeff->av), 1000),
 754                           drm_fixp_exp(drm_fixp_mul(div64_s64(drm_int2fixp(coeff->bv), 1000), vddc)));
 755
 756         leakage_w = drm_fixp_mul(drm_fixp_mul(drm_fixp_mul(i_leakage, kt), kv), vddc);
 757
 758         *leakage = drm_fixp2int(leakage_w * 1000);
 759 }
 760
 761 static void ni_calculate_leakage_for_v_and_t(struct radeon_device *rdev,
 762                                              const struct ni_leakage_coeffients *coeff,
 763                                              u16 v,
 764                                              s32 t,
 765                                              u32 i_leakage,
 766                                              u32 *leakage)
 767 {
 768         ni_calculate_leakage_for_v_and_t_formula(coeff, v, t, i_leakage, leakage);
 769 }
 770
 771 bool ni_dpm_vblank_too_short(struct radeon_device *rdev)
 772 {
 773         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
 774         u32 vblank_time = r600_dpm_get_vblank_time(rdev);
 775         /* we never hit the non-gddr5 limit so disable it */
 776         u32 switch_limit = pi->mem_gddr5 ? 450 : 0;
 777
 778         if (vblank_time < switch_limit)
 779                 return true;
 780         else
 781                 return false;
 782
 783 }
 784
 785 static void ni_apply_state_adjust_rules(struct radeon_device *rdev,
 786                                         struct radeon_ps *rps)
 787 {
 788         struct ni_ps *ps = ni_get_ps(rps);
 789         struct radeon_clock_and_voltage_limits *max_limits;
 790         bool disable_mclk_switching;
 791         u32 mclk;
 792         u16 vddci;
 793         int i;
 794
 795         if ((rdev->pm.dpm.new_active_crtc_count > 1) ||
 796             ni_dpm_vblank_too_short(rdev))
 797                 disable_mclk_switching = true;
 798         else
 799                 disable_mclk_switching = false;
 800
 801         if (rdev->pm.dpm.ac_power)
 802                 max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac;
 803         else
 804                 max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc;
 805
 806         if (rdev->pm.dpm.ac_power == false) {
 807                 for (i = 0; i < ps->performance_level_count; i++) {
 808                         if (ps->performance_levels[i].mclk > max_limits->mclk)
 809                                 ps->performance_levels[i].mclk = max_limits->mclk;
 810                         if (ps->performance_levels[i].sclk > max_limits->sclk)
 811                                 ps->performance_levels[i].sclk = max_limits->sclk;
 812                         if (ps->performance_levels[i].vddc > max_limits->vddc)
 813                                 ps->performance_levels[i].vddc = max_limits->vddc;
 814                         if (ps->performance_levels[i].vddci > max_limits->vddci)
 815                                 ps->performance_levels[i].vddci = max_limits->vddci;
 816                 }
 817         }
 818
 819         /* XXX validate the min clocks required for display */
 820
 821         /* adjust low state */
 822         if (disable_mclk_switching) {
 823                 ps->performance_levels[0].mclk =
 824                         ps->performance_levels[ps->performance_level_count - 1].mclk;
 825                 ps->performance_levels[0].vddci =
 826                         ps->performance_levels[ps->performance_level_count - 1].vddci;
 827         }
 828
 829         btc_skip_blacklist_clocks(rdev, max_limits->sclk, max_limits->mclk,
 830                                   &ps->performance_levels[0].sclk,
 831                                   &ps->performance_levels[0].mclk);
 832
 833         for (i = 1; i < ps->performance_level_count; i++) {
 834                 if (ps->performance_levels[i].sclk < ps->performance_levels[i - 1].sclk)
 835                         ps->performance_levels[i].sclk = ps->performance_levels[i - 1].sclk;
 836                 if (ps->performance_levels[i].vddc < ps->performance_levels[i - 1].vddc)
 837                         ps->performance_levels[i].vddc = ps->performance_levels[i - 1].vddc;
 838         }
 839
 840         /* adjust remaining states */
 841         if (disable_mclk_switching) {
 842                 mclk = ps->performance_levels[0].mclk;
 843                 vddci = ps->performance_levels[0].vddci;
 844                 for (i = 1; i < ps->performance_level_count; i++) {
 845                         if (mclk < ps->performance_levels[i].mclk)
 846                                 mclk = ps->performance_levels[i].mclk;
 847                         if (vddci < ps->performance_levels[i].vddci)
 848                                 vddci = ps->performance_levels[i].vddci;
 849                 }
 850                 for (i = 0; i < ps->performance_level_count; i++) {
 851                         ps->performance_levels[i].mclk = mclk;
 852                         ps->performance_levels[i].vddci = vddci;
 853                 }
 854         } else {
 855                 for (i = 1; i < ps->performance_level_count; i++) {
 856                         if (ps->performance_levels[i].mclk < ps->performance_levels[i - 1].mclk)
 857                                 ps->performance_levels[i].mclk = ps->performance_levels[i - 1].mclk;
 858                         if (ps->performance_levels[i].vddci < ps->performance_levels[i - 1].vddci)
 859                                 ps->performance_levels[i].vddci = ps->performance_levels[i - 1].vddci;
 860                 }
 861         }
 862
 863         for (i = 1; i < ps->performance_level_count; i++)
 864                 btc_skip_blacklist_clocks(rdev, max_limits->sclk, max_limits->mclk,
 865                                           &ps->performance_levels[i].sclk,
 866                                           &ps->performance_levels[i].mclk);
 867
 868         for (i = 0; i < ps->performance_level_count; i++)
 869                 btc_adjust_clock_combinations(rdev, max_limits,
 870                                               &ps->performance_levels[i]);
 871
 872         for (i = 0; i < ps->performance_level_count; i++) {
 873                 btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk,
 874                                                    ps->performance_levels[i].sclk,
 875                                                    max_limits->vddc,  &ps->performance_levels[i].vddc);
 876                 btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk,
 877                                                    ps->performance_levels[i].mclk,
 878                                                    max_limits->vddci, &ps->performance_levels[i].vddci);
 879                 btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk,
 880                                                    ps->performance_levels[i].mclk,
 881                                                    max_limits->vddc,  &ps->performance_levels[i].vddc);
 882                 btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk,
 883                                                    rdev->clock.current_dispclk,
 884                                                    max_limits->vddc,  &ps->performance_levels[i].vddc);
 885         }
 886
 887         for (i = 0; i < ps->performance_level_count; i++) {
 888                 btc_apply_voltage_delta_rules(rdev,
 889                                               max_limits->vddc, max_limits->vddci,
 890                                               &ps->performance_levels[i].vddc,
 891                                               &ps->performance_levels[i].vddci);
 892         }
 893
 894         ps->dc_compatible = true;
 895         for (i = 0; i < ps->performance_level_count; i++) {
 896                 if (ps->performance_levels[i].vddc > rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.vddc)
 897                         ps->dc_compatible = false;
 898
 899                 if (ps->performance_levels[i].vddc < rdev->pm.dpm.dyn_state.min_vddc_for_pcie_gen2)
 900                         ps->performance_levels[i].flags &= ~ATOM_PPLIB_R600_FLAGS_PCIEGEN2;
 901         }
 902 }
 903
 904 static void ni_cg_clockgating_default(struct radeon_device *rdev)
 905 {
 906         u32 count;
 907         const u32 *ps = NULL;
 908
 909         ps = (const u32 *)&cayman_cgcg_cgls_default;
 910         count = CAYMAN_CGCG_CGLS_DEFAULT_LENGTH;
 911
 912         btc_program_mgcg_hw_sequence(rdev, ps, count);
 913 }
 914
 915 static void ni_gfx_clockgating_enable(struct radeon_device *rdev,
 916                                       bool enable)
 917 {
 918         u32 count;
 919         const u32 *ps = NULL;
 920
 921         if (enable) {
 922                 ps = (const u32 *)&cayman_cgcg_cgls_enable;
 923                 count = CAYMAN_CGCG_CGLS_ENABLE_LENGTH;
 924         } else {
 925                 ps = (const u32 *)&cayman_cgcg_cgls_disable;
 926                 count = CAYMAN_CGCG_CGLS_DISABLE_LENGTH;
 927         }
 928
 929         btc_program_mgcg_hw_sequence(rdev, ps, count);
 930 }
 931
 932 static void ni_mg_clockgating_default(struct radeon_device *rdev)
 933 {
 934         u32 count;
 935         const u32 *ps = NULL;
 936
 937         ps = (const u32 *)&cayman_mgcg_default;
 938         count = CAYMAN_MGCG_DEFAULT_LENGTH;
 939
 940         btc_program_mgcg_hw_sequence(rdev, ps, count);
 941 }
 942
 943 static void ni_mg_clockgating_enable(struct radeon_device *rdev,
 944                                      bool enable)
 945 {
 946         u32 count;
 947         const u32 *ps = NULL;
 948
 949         if (enable) {
 950                 ps = (const u32 *)&cayman_mgcg_enable;
 951                 count = CAYMAN_MGCG_ENABLE_LENGTH;
 952         } else {
 953                 ps = (const u32 *)&cayman_mgcg_disable;
 954                 count = CAYMAN_MGCG_DISABLE_LENGTH;
 955         }
 956
 957         btc_program_mgcg_hw_sequence(rdev, ps, count);
 958 }
 959
 960 static void ni_ls_clockgating_default(struct radeon_device *rdev)
 961 {
 962         u32 count;
 963         const u32 *ps = NULL;
 964
 965         ps = (const u32 *)&cayman_sysls_default;
 966         count = CAYMAN_SYSLS_DEFAULT_LENGTH;
 967
 968         btc_program_mgcg_hw_sequence(rdev, ps, count);
 969 }
 970
 971 static void ni_ls_clockgating_enable(struct radeon_device *rdev,
 972                                      bool enable)
 973 {
 974         u32 count;
 975         const u32 *ps = NULL;
 976
 977         if (enable) {
 978                 ps = (const u32 *)&cayman_sysls_enable;
 979                 count = CAYMAN_SYSLS_ENABLE_LENGTH;
 980         } else {
 981                 ps = (const u32 *)&cayman_sysls_disable;
 982                 count = CAYMAN_SYSLS_DISABLE_LENGTH;
 983         }
 984
 985         btc_program_mgcg_hw_sequence(rdev, ps, count);
 986
 987 }
 988
 989 static int ni_patch_single_dependency_table_based_on_leakage(struct radeon_device *rdev,
 990                                                              struct radeon_clock_voltage_dependency_table *table)
 991 {
 992         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
 993         u32 i;
 994
 995         if (table) {
 996                 for (i = 0; i < table->count; i++) {
 997                         if (0xff01 == table->entries[i].v) {
 998                                 if (pi->max_vddc == 0)
 999                                         return -EINVAL;
1000                                 table->entries[i].v = pi->max_vddc;
1001                         }
1002                 }
1003         }
1004         return 0;
1005 }
1006
1007 static int ni_patch_dependency_tables_based_on_leakage(struct radeon_device *rdev)
1008 {
1009         int ret = 0;
1010
1011         ret = ni_patch_single_dependency_table_based_on_leakage(rdev,
1012                                                                 &rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk);
1013
1014         ret = ni_patch_single_dependency_table_based_on_leakage(rdev,
1015                                                                 &rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk);
1016         return ret;
1017 }
1018
1019 static void ni_stop_dpm(struct radeon_device *rdev)
1020 {
1021         WREG32_P(GENERAL_PWRMGT, 0, ~GLOBAL_PWRMGT_EN);
1022 }
1023
1024 #if 0
1025 static int ni_notify_hw_of_power_source(struct radeon_device *rdev,
1026                                         bool ac_power)
1027 {
1028         if (ac_power)
1029                 return (rv770_send_msg_to_smc(rdev, PPSMC_MSG_RunningOnAC) == PPSMC_Result_OK) ?
1030                         0 : -EINVAL;
1031
1032         return 0;
1033 }
1034 #endif
1035
1036 static PPSMC_Result ni_send_msg_to_smc_with_parameter(struct radeon_device *rdev,
1037                                                       PPSMC_Msg msg, u32 parameter)
1038 {
1039         WREG32(SMC_SCRATCH0, parameter);
1040         return rv770_send_msg_to_smc(rdev, msg);
1041 }
1042
1043 static int ni_restrict_performance_levels_before_switch(struct radeon_device *rdev)
1044 {
1045         if (rv770_send_msg_to_smc(rdev, PPSMC_MSG_NoForcedLevel) != PPSMC_Result_OK)
1046                 return -EINVAL;
1047
1048         return (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 1) == PPSMC_Result_OK) ?
1049                 0 : -EINVAL;
1050 }
1051
1052 int ni_dpm_force_performance_level(struct radeon_device *rdev,
1053                                    enum radeon_dpm_forced_level level)
1054 {
1055         if (level == RADEON_DPM_FORCED_LEVEL_HIGH) {
1056                 if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 0) != PPSMC_Result_OK)
1057                         return -EINVAL;
1058
1059                 if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetForcedLevels, 1) != PPSMC_Result_OK)
1060                         return -EINVAL;
1061         } else if (level == RADEON_DPM_FORCED_LEVEL_LOW) {
1062                 if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetForcedLevels, 0) != PPSMC_Result_OK)
1063                         return -EINVAL;
1064
1065                 if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 1) != PPSMC_Result_OK)
1066                         return -EINVAL;
1067         } else if (level == RADEON_DPM_FORCED_LEVEL_AUTO) {
1068                 if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetForcedLevels, 0) != PPSMC_Result_OK)
1069                         return -EINVAL;
1070
1071                 if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 0) != PPSMC_Result_OK)
1072                         return -EINVAL;
1073         }
1074
1075         rdev->pm.dpm.forced_level = level;
1076
1077         return 0;
1078 }
1079
1080 static void ni_stop_smc(struct radeon_device *rdev)
1081 {
1082         u32 tmp;
1083         int i;
1084
1085         for (i = 0; i < rdev->usec_timeout; i++) {
1086                 tmp = RREG32(LB_SYNC_RESET_SEL) & LB_SYNC_RESET_SEL_MASK;
1087                 if (tmp != 1)
1088                         break;
1089                 udelay(1);
1090         }
1091
1092         udelay(100);
1093
1094         r7xx_stop_smc(rdev);
1095 }
1096
1097 static int ni_process_firmware_header(struct radeon_device *rdev)
1098 {
1099         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1100         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1101         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1102         u32 tmp;
1103         int ret;
1104
1105         ret = rv770_read_smc_sram_dword(rdev,
1106                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1107                                         NISLANDS_SMC_FIRMWARE_HEADER_stateTable,
1108                                         &tmp, pi->sram_end);
1109
1110         if (ret)
1111                 return ret;
1112
1113         pi->state_table_start = (u16)tmp;
1114
1115         ret = rv770_read_smc_sram_dword(rdev,
1116                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1117                                         NISLANDS_SMC_FIRMWARE_HEADER_softRegisters,
1118                                         &tmp, pi->sram_end);
1119
1120         if (ret)
1121                 return ret;
1122
1123         pi->soft_regs_start = (u16)tmp;
1124
1125         ret = rv770_read_smc_sram_dword(rdev,
1126                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1127                                         NISLANDS_SMC_FIRMWARE_HEADER_mcRegisterTable,
1128                                         &tmp, pi->sram_end);
1129
1130         if (ret)
1131                 return ret;
1132
1133         eg_pi->mc_reg_table_start = (u16)tmp;
1134
1135         ret = rv770_read_smc_sram_dword(rdev,
1136                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1137                                         NISLANDS_SMC_FIRMWARE_HEADER_fanTable,
1138                                         &tmp, pi->sram_end);
1139
1140         if (ret)
1141                 return ret;
1142
1143         ni_pi->fan_table_start = (u16)tmp;
1144
1145         ret = rv770_read_smc_sram_dword(rdev,
1146                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1147                                         NISLANDS_SMC_FIRMWARE_HEADER_mcArbDramAutoRefreshTable,
1148                                         &tmp, pi->sram_end);
1149
1150         if (ret)
1151                 return ret;
1152
1153         ni_pi->arb_table_start = (u16)tmp;
1154
1155         ret = rv770_read_smc_sram_dword(rdev,
1156                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1157                                         NISLANDS_SMC_FIRMWARE_HEADER_cacTable,
1158                                         &tmp, pi->sram_end);
1159
1160         if (ret)
1161                 return ret;
1162
1163         ni_pi->cac_table_start = (u16)tmp;
1164
1165         ret = rv770_read_smc_sram_dword(rdev,
1166                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1167                                         NISLANDS_SMC_FIRMWARE_HEADER_spllTable,
1168                                         &tmp, pi->sram_end);
1169
1170         if (ret)
1171                 return ret;
1172
1173         ni_pi->spll_table_start = (u16)tmp;
1174
1175
1176         return ret;
1177 }
1178
1179 static void ni_read_clock_registers(struct radeon_device *rdev)
1180 {
1181         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1182
1183         ni_pi->clock_registers.cg_spll_func_cntl = RREG32(CG_SPLL_FUNC_CNTL);
1184         ni_pi->clock_registers.cg_spll_func_cntl_2 = RREG32(CG_SPLL_FUNC_CNTL_2);
1185         ni_pi->clock_registers.cg_spll_func_cntl_3 = RREG32(CG_SPLL_FUNC_CNTL_3);
1186         ni_pi->clock_registers.cg_spll_func_cntl_4 = RREG32(CG_SPLL_FUNC_CNTL_4);
1187         ni_pi->clock_registers.cg_spll_spread_spectrum = RREG32(CG_SPLL_SPREAD_SPECTRUM);
1188         ni_pi->clock_registers.cg_spll_spread_spectrum_2 = RREG32(CG_SPLL_SPREAD_SPECTRUM_2);
1189         ni_pi->clock_registers.mpll_ad_func_cntl = RREG32(MPLL_AD_FUNC_CNTL);
1190         ni_pi->clock_registers.mpll_ad_func_cntl_2 = RREG32(MPLL_AD_FUNC_CNTL_2);
1191         ni_pi->clock_registers.mpll_dq_func_cntl = RREG32(MPLL_DQ_FUNC_CNTL);
1192         ni_pi->clock_registers.mpll_dq_func_cntl_2 = RREG32(MPLL_DQ_FUNC_CNTL_2);
1193         ni_pi->clock_registers.mclk_pwrmgt_cntl = RREG32(MCLK_PWRMGT_CNTL);
1194         ni_pi->clock_registers.dll_cntl = RREG32(DLL_CNTL);
1195         ni_pi->clock_registers.mpll_ss1 = RREG32(MPLL_SS1);
1196         ni_pi->clock_registers.mpll_ss2 = RREG32(MPLL_SS2);
1197 }
1198
1199 #if 0
1200 static int ni_enter_ulp_state(struct radeon_device *rdev)
1201 {
1202         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1203
1204         if (pi->gfx_clock_gating) {
1205                 WREG32_P(SCLK_PWRMGT_CNTL, 0, ~DYN_GFX_CLK_OFF_EN);
1206                 WREG32_P(SCLK_PWRMGT_CNTL, GFX_CLK_FORCE_ON, ~GFX_CLK_FORCE_ON);
1207                 WREG32_P(SCLK_PWRMGT_CNTL, 0, ~GFX_CLK_FORCE_ON);
1208                 RREG32(GB_ADDR_CONFIG);
1209         }
1210
1211         WREG32_P(SMC_MSG, HOST_SMC_MSG(PPSMC_MSG_SwitchToMinimumPower),
1212                  ~HOST_SMC_MSG_MASK);
1213
1214         udelay(25000);
1215
1216         return 0;
1217 }
1218 #endif
1219
1220 static void ni_program_response_times(struct radeon_device *rdev)
1221 {
1222         u32 voltage_response_time, backbias_response_time, acpi_delay_time, vbi_time_out;
1223         u32 vddc_dly, bb_dly, acpi_dly, vbi_dly, mclk_switch_limit;
1224         u32 reference_clock;
1225
1226         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mvdd_chg_time, 1);
1227
1228         voltage_response_time = (u32)rdev->pm.dpm.voltage_response_time;
1229         backbias_response_time = (u32)rdev->pm.dpm.backbias_response_time;
1230
1231         if (voltage_response_time == 0)
1232                 voltage_response_time = 1000;
1233
1234         if (backbias_response_time == 0)
1235                 backbias_response_time = 1000;
1236
1237         acpi_delay_time = 15000;
1238         vbi_time_out = 100000;
1239
1240         reference_clock = radeon_get_xclk(rdev);
1241
1242         vddc_dly = (voltage_response_time  * reference_clock) / 1600;
1243         bb_dly   = (backbias_response_time * reference_clock) / 1600;
1244         acpi_dly = (acpi_delay_time * reference_clock) / 1600;
1245         vbi_dly  = (vbi_time_out * reference_clock) / 1600;
1246
1247         mclk_switch_limit = (460 * reference_clock) / 100;
1248
1249         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_delay_vreg,  vddc_dly);
1250         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_delay_bbias, bb_dly);
1251         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_delay_acpi,  acpi_dly);
1252         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mclk_chg_timeout, vbi_dly);
1253         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mc_block_delay, 0xAA);
1254         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mclk_switch_lim, mclk_switch_limit);
1255 }
1256
1257 static void ni_populate_smc_voltage_table(struct radeon_device *rdev,
1258                                           struct atom_voltage_table *voltage_table,
1259                                           NISLANDS_SMC_STATETABLE *table)
1260 {
1261         unsigned int i;
1262
1263         for (i = 0; i < voltage_table->count; i++) {
1264                 table->highSMIO[i] = 0;
1265                 table->lowSMIO[i] |= cpu_to_be32(voltage_table->entries[i].smio_low);
1266         }
1267 }
1268
1269 static void ni_populate_smc_voltage_tables(struct radeon_device *rdev,
1270                                            NISLANDS_SMC_STATETABLE *table)
1271 {
1272         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1273         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1274         unsigned char i;
1275
1276         if (eg_pi->vddc_voltage_table.count) {
1277                 ni_populate_smc_voltage_table(rdev, &eg_pi->vddc_voltage_table, table);
1278                 table->voltageMaskTable.highMask[NISLANDS_SMC_VOLTAGEMASK_VDDC] = 0;
1279                 table->voltageMaskTable.lowMask[NISLANDS_SMC_VOLTAGEMASK_VDDC] =
1280                         cpu_to_be32(eg_pi->vddc_voltage_table.mask_low);
1281
1282                 for (i = 0; i < eg_pi->vddc_voltage_table.count; i++) {
1283                         if (pi->max_vddc_in_table <= eg_pi->vddc_voltage_table.entries[i].value) {
1284                                 table->maxVDDCIndexInPPTable = i;
1285                                 break;
1286                         }
1287                 }
1288         }
1289
1290         if (eg_pi->vddci_voltage_table.count) {
1291                 ni_populate_smc_voltage_table(rdev, &eg_pi->vddci_voltage_table, table);
1292
1293                 table->voltageMaskTable.highMask[NISLANDS_SMC_VOLTAGEMASK_VDDCI] = 0;
1294                 table->voltageMaskTable.lowMask[NISLANDS_SMC_VOLTAGEMASK_VDDCI] =
1295                         cpu_to_be32(eg_pi->vddci_voltage_table.mask_low);
1296         }
1297 }
1298
1299 static int ni_populate_voltage_value(struct radeon_device *rdev,
1300                                      struct atom_voltage_table *table,
1301                                      u16 value,
1302                                      NISLANDS_SMC_VOLTAGE_VALUE *voltage)
1303 {
1304         unsigned int i;
1305
1306         for (i = 0; i < table->count; i++) {
1307                 if (value <= table->entries[i].value) {
1308                         voltage->index = (u8)i;
1309                         voltage->value = cpu_to_be16(table->entries[i].value);
1310                         break;
1311                 }
1312         }
1313
1314         if (i >= table->count)
1315                 return -EINVAL;
1316
1317         return 0;
1318 }
1319
1320 static void ni_populate_mvdd_value(struct radeon_device *rdev,
1321                                    u32 mclk,
1322                                    NISLANDS_SMC_VOLTAGE_VALUE *voltage)
1323 {
1324         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1325         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1326
1327         if (!pi->mvdd_control) {
1328                 voltage->index = eg_pi->mvdd_high_index;
1329                 voltage->value = cpu_to_be16(MVDD_HIGH_VALUE);
1330                 return;
1331         }
1332
1333         if (mclk <= pi->mvdd_split_frequency) {
1334                 voltage->index = eg_pi->mvdd_low_index;
1335                 voltage->value = cpu_to_be16(MVDD_LOW_VALUE);
1336         } else {
1337                 voltage->index = eg_pi->mvdd_high_index;
1338                 voltage->value = cpu_to_be16(MVDD_HIGH_VALUE);
1339         }
1340 }
1341
1342 static int ni_get_std_voltage_value(struct radeon_device *rdev,
1343                                     NISLANDS_SMC_VOLTAGE_VALUE *voltage,
1344                                     u16 *std_voltage)
1345 {
1346         if (rdev->pm.dpm.dyn_state.cac_leakage_table.entries &&
1347             ((u32)voltage->index < rdev->pm.dpm.dyn_state.cac_leakage_table.count))
1348                 *std_voltage = rdev->pm.dpm.dyn_state.cac_leakage_table.entries[voltage->index].vddc;
1349         else
1350                 *std_voltage = be16_to_cpu(voltage->value);
1351
1352         return 0;
1353 }
1354
1355 static void ni_populate_std_voltage_value(struct radeon_device *rdev,
1356                                           u16 value, u8 index,
1357                                           NISLANDS_SMC_VOLTAGE_VALUE *voltage)
1358 {
1359         voltage->index = index;
1360         voltage->value = cpu_to_be16(value);
1361 }
1362
1363 static u32 ni_get_smc_power_scaling_factor(struct radeon_device *rdev)
1364 {
1365         u32 xclk_period;
1366         u32 xclk = radeon_get_xclk(rdev);
1367         u32 tmp = RREG32(CG_CAC_CTRL) & TID_CNT_MASK;
1368
1369         xclk_period = (1000000000UL / xclk);
1370         xclk_period /= 10000UL;
1371
1372         return tmp * xclk_period;
1373 }
1374
1375 static u32 ni_scale_power_for_smc(u32 power_in_watts, u32 scaling_factor)
1376 {
1377         return (power_in_watts * scaling_factor) << 2;
1378 }
1379
1380 static u32 ni_calculate_power_boost_limit(struct radeon_device *rdev,
1381                                           struct radeon_ps *radeon_state,
1382                                           u32 near_tdp_limit)
1383 {
1384         struct ni_ps *state = ni_get_ps(radeon_state);
1385         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1386         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1387         u32 power_boost_limit = 0;
1388         int ret;
1389
1390         if (ni_pi->enable_power_containment &&
1391             ni_pi->use_power_boost_limit) {
1392                 NISLANDS_SMC_VOLTAGE_VALUE vddc;
1393                 u16 std_vddc_med;
1394                 u16 std_vddc_high;
1395                 u64 tmp, n, d;
1396
1397                 if (state->performance_level_count < 3)
1398                         return 0;
1399
1400                 ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
1401                                                 state->performance_levels[state->performance_level_count - 2].vddc,
1402                                                 &vddc);
1403                 if (ret)
1404                         return 0;
1405
1406                 ret = ni_get_std_voltage_value(rdev, &vddc, &std_vddc_med);
1407                 if (ret)
1408                         return 0;
1409
1410                 ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
1411                                                 state->performance_levels[state->performance_level_count - 1].vddc,
1412                                                 &vddc);
1413                 if (ret)
1414                         return 0;
1415
1416                 ret = ni_get_std_voltage_value(rdev, &vddc, &std_vddc_high);
1417                 if (ret)
1418                         return 0;
1419
1420                 n = ((u64)near_tdp_limit * ((u64)std_vddc_med * (u64)std_vddc_med) * 90);
1421                 d = ((u64)std_vddc_high * (u64)std_vddc_high * 100);
1422                 tmp = div64_u64(n, d);
1423
1424                 if (tmp >> 32)
1425                         return 0;
1426                 power_boost_limit = (u32)tmp;
1427         }
1428
1429         return power_boost_limit;
1430 }
1431
1432 static int ni_calculate_adjusted_tdp_limits(struct radeon_device *rdev,
1433                                             bool adjust_polarity,
1434                                             u32 tdp_adjustment,
1435                                             u32 *tdp_limit,
1436                                             u32 *near_tdp_limit)
1437 {
1438         if (tdp_adjustment > (u32)rdev->pm.dpm.tdp_od_limit)
1439                 return -EINVAL;
1440
1441         if (adjust_polarity) {
1442                 *tdp_limit = ((100 + tdp_adjustment) * rdev->pm.dpm.tdp_limit) / 100;
1443                 *near_tdp_limit = rdev->pm.dpm.near_tdp_limit + (*tdp_limit - rdev->pm.dpm.tdp_limit);
1444         } else {
1445                 *tdp_limit = ((100 - tdp_adjustment) * rdev->pm.dpm.tdp_limit) / 100;
1446                 *near_tdp_limit = rdev->pm.dpm.near_tdp_limit - (rdev->pm.dpm.tdp_limit - *tdp_limit);
1447         }
1448
1449         return 0;
1450 }
1451
1452 static int ni_populate_smc_tdp_limits(struct radeon_device *rdev,
1453                                       struct radeon_ps *radeon_state)
1454 {
1455         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1456         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1457
1458         if (ni_pi->enable_power_containment) {
1459                 NISLANDS_SMC_STATETABLE *smc_table = &ni_pi->smc_statetable;
1460                 u32 scaling_factor = ni_get_smc_power_scaling_factor(rdev);
1461                 u32 tdp_limit;
1462                 u32 near_tdp_limit;
1463                 u32 power_boost_limit;
1464                 int ret;
1465
1466                 if (scaling_factor == 0)
1467                         return -EINVAL;
1468
1469                 memset(smc_table, 0, sizeof(NISLANDS_SMC_STATETABLE));
1470
1471                 ret = ni_calculate_adjusted_tdp_limits(rdev,
1472                                                        false, /* ??? */
1473                                                        rdev->pm.dpm.tdp_adjustment,
1474                                                        &tdp_limit,
1475                                                        &near_tdp_limit);
1476                 if (ret)
1477                         return ret;
1478
1479                 power_boost_limit = ni_calculate_power_boost_limit(rdev, radeon_state,
1480                                                                    near_tdp_limit);
1481
1482                 smc_table->dpm2Params.TDPLimit =
1483                         cpu_to_be32(ni_scale_power_for_smc(tdp_limit, scaling_factor));
1484                 smc_table->dpm2Params.NearTDPLimit =
1485                         cpu_to_be32(ni_scale_power_for_smc(near_tdp_limit, scaling_factor));
1486                 smc_table->dpm2Params.SafePowerLimit =
1487                         cpu_to_be32(ni_scale_power_for_smc((near_tdp_limit * NISLANDS_DPM2_TDP_SAFE_LIMIT_PERCENT) / 100,
1488                                                            scaling_factor));
1489                 smc_table->dpm2Params.PowerBoostLimit =
1490                         cpu_to_be32(ni_scale_power_for_smc(power_boost_limit, scaling_factor));
1491
1492                 ret = rv770_copy_bytes_to_smc(rdev,
1493                                               (u16)(pi->state_table_start + offsetof(NISLANDS_SMC_STATETABLE, dpm2Params) +
1494                                                     offsetof(PP_NIslands_DPM2Parameters, TDPLimit)),
1495                                               (u8 *)(&smc_table->dpm2Params.TDPLimit),
1496                                               sizeof(u32) * 4, pi->sram_end);
1497                 if (ret)
1498                         return ret;
1499         }
1500
1501         return 0;
1502 }
1503
1504 int ni_copy_and_switch_arb_sets(struct radeon_device *rdev,
1505                                 u32 arb_freq_src, u32 arb_freq_dest)
1506 {
1507         u32 mc_arb_dram_timing;
1508         u32 mc_arb_dram_timing2;
1509         u32 burst_time;
1510         u32 mc_cg_config;
1511
1512         switch (arb_freq_src) {
1513         case MC_CG_ARB_FREQ_F0:
1514                 mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING);
1515                 mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2);
1516                 burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE0_MASK) >> STATE0_SHIFT;
1517                 break;
1518         case MC_CG_ARB_FREQ_F1:
1519                 mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING_1);
1520                 mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2_1);
1521                 burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE1_MASK) >> STATE1_SHIFT;
1522                 break;
1523         case MC_CG_ARB_FREQ_F2:
1524                 mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING_2);
1525                 mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2_2);
1526                 burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE2_MASK) >> STATE2_SHIFT;
1527                 break;
1528         case MC_CG_ARB_FREQ_F3:
1529                 mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING_3);
1530                 mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2_3);
1531                 burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE3_MASK) >> STATE3_SHIFT;
1532                 break;
1533         default:
1534                 return -EINVAL;
1535         }
1536
1537         switch (arb_freq_dest) {
1538         case MC_CG_ARB_FREQ_F0:
1539                 WREG32(MC_ARB_DRAM_TIMING, mc_arb_dram_timing);
1540                 WREG32(MC_ARB_DRAM_TIMING2, mc_arb_dram_timing2);
1541                 WREG32_P(MC_ARB_BURST_TIME, STATE0(burst_time), ~STATE0_MASK);
1542                 break;
1543         case MC_CG_ARB_FREQ_F1:
1544                 WREG32(MC_ARB_DRAM_TIMING_1, mc_arb_dram_timing);
1545                 WREG32(MC_ARB_DRAM_TIMING2_1, mc_arb_dram_timing2);
1546                 WREG32_P(MC_ARB_BURST_TIME, STATE1(burst_time), ~STATE1_MASK);
1547                 break;
1548         case MC_CG_ARB_FREQ_F2:
1549                 WREG32(MC_ARB_DRAM_TIMING_2, mc_arb_dram_timing);
1550                 WREG32(MC_ARB_DRAM_TIMING2_2, mc_arb_dram_timing2);
1551                 WREG32_P(MC_ARB_BURST_TIME, STATE2(burst_time), ~STATE2_MASK);
1552                 break;
1553         case MC_CG_ARB_FREQ_F3:
1554                 WREG32(MC_ARB_DRAM_TIMING_3, mc_arb_dram_timing);
1555                 WREG32(MC_ARB_DRAM_TIMING2_3, mc_arb_dram_timing2);
1556                 WREG32_P(MC_ARB_BURST_TIME, STATE3(burst_time), ~STATE3_MASK);
1557                 break;
1558         default:
1559                 return -EINVAL;
1560         }
1561
1562         mc_cg_config = RREG32(MC_CG_CONFIG) | 0x0000000F;
1563         WREG32(MC_CG_CONFIG, mc_cg_config);
1564         WREG32_P(MC_ARB_CG, CG_ARB_REQ(arb_freq_dest), ~CG_ARB_REQ_MASK);
1565
1566         return 0;
1567 }
1568
1569 static int ni_init_arb_table_index(struct radeon_device *rdev)
1570 {
1571         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1572         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1573         u32 tmp;
1574         int ret;
1575
1576         ret = rv770_read_smc_sram_dword(rdev, ni_pi->arb_table_start,
1577                                         &tmp, pi->sram_end);
1578         if (ret)
1579                 return ret;
1580
1581         tmp &= 0x00FFFFFF;
1582         tmp |= ((u32)MC_CG_ARB_FREQ_F1) << 24;
1583
1584         return rv770_write_smc_sram_dword(rdev, ni_pi->arb_table_start,
1585                                           tmp, pi->sram_end);
1586 }
1587
1588 static int ni_initial_switch_from_arb_f0_to_f1(struct radeon_device *rdev)
1589 {
1590         return ni_copy_and_switch_arb_sets(rdev, MC_CG_ARB_FREQ_F0, MC_CG_ARB_FREQ_F1);
1591 }
1592
1593 static int ni_force_switch_to_arb_f0(struct radeon_device *rdev)
1594 {
1595         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1596         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1597         u32 tmp;
1598         int ret;
1599
1600         ret = rv770_read_smc_sram_dword(rdev, ni_pi->arb_table_start,
1601                                         &tmp, pi->sram_end);
1602         if (ret)
1603                 return ret;
1604
1605         tmp = (tmp >> 24) & 0xff;
1606
1607         if (tmp == MC_CG_ARB_FREQ_F0)
1608                 return 0;
1609
1610         return ni_copy_and_switch_arb_sets(rdev, tmp, MC_CG_ARB_FREQ_F0);
1611 }
1612
1613 static int ni_populate_memory_timing_parameters(struct radeon_device *rdev,
1614                                                 struct rv7xx_pl *pl,
1615                                                 SMC_NIslands_MCArbDramTimingRegisterSet *arb_regs)
1616 {
1617         u32 dram_timing;
1618         u32 dram_timing2;
1619
1620         arb_regs->mc_arb_rfsh_rate =
1621                 (u8)rv770_calculate_memory_refresh_rate(rdev, pl->sclk);
1622
1623
1624         radeon_atom_set_engine_dram_timings(rdev, pl->sclk, pl->mclk);
1625
1626         dram_timing = RREG32(MC_ARB_DRAM_TIMING);
1627         dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2);
1628
1629         arb_regs->mc_arb_dram_timing  = cpu_to_be32(dram_timing);
1630         arb_regs->mc_arb_dram_timing2 = cpu_to_be32(dram_timing2);
1631
1632         return 0;
1633 }
1634
1635 static int ni_do_program_memory_timing_parameters(struct radeon_device *rdev,
1636                                                   struct radeon_ps *radeon_state,
1637                                                   unsigned int first_arb_set)
1638 {
1639         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1640         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1641         struct ni_ps *state = ni_get_ps(radeon_state);
1642         SMC_NIslands_MCArbDramTimingRegisterSet arb_regs = { 0 };
1643         int i, ret = 0;
1644
1645         for (i = 0; i < state->performance_level_count; i++) {
1646                 ret = ni_populate_memory_timing_parameters(rdev, &state->performance_levels[i], &arb_regs);
1647                 if (ret)
1648                         break;
1649
1650                 ret = rv770_copy_bytes_to_smc(rdev,
1651                                               (u16)(ni_pi->arb_table_start +
1652                                                     offsetof(SMC_NIslands_MCArbDramTimingRegisters, data) +
1653                                                     sizeof(SMC_NIslands_MCArbDramTimingRegisterSet) * (first_arb_set + i)),
1654                                               (u8 *)&arb_regs,
1655                                               (u16)sizeof(SMC_NIslands_MCArbDramTimingRegisterSet),
1656                                               pi->sram_end);
1657                 if (ret)
1658                         break;
1659         }
1660         return ret;
1661 }
1662
1663 static int ni_program_memory_timing_parameters(struct radeon_device *rdev,
1664                                                struct radeon_ps *radeon_new_state)
1665 {
1666         return ni_do_program_memory_timing_parameters(rdev, radeon_new_state,
1667                                                       NISLANDS_DRIVER_STATE_ARB_INDEX);
1668 }
1669
1670 static void ni_populate_initial_mvdd_value(struct radeon_device *rdev,
1671                                            struct NISLANDS_SMC_VOLTAGE_VALUE *voltage)
1672 {
1673         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1674
1675         voltage->index = eg_pi->mvdd_high_index;
1676         voltage->value = cpu_to_be16(MVDD_HIGH_VALUE);
1677 }
1678
1679 static int ni_populate_smc_initial_state(struct radeon_device *rdev,
1680                                          struct radeon_ps *radeon_initial_state,
1681                                          NISLANDS_SMC_STATETABLE *table)
1682 {
1683         struct ni_ps *initial_state = ni_get_ps(radeon_initial_state);
1684         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1685         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1686         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1687         u32 reg;
1688         int ret;
1689
1690         table->initialState.levels[0].mclk.vMPLL_AD_FUNC_CNTL =
1691                 cpu_to_be32(ni_pi->clock_registers.mpll_ad_func_cntl);
1692         table->initialState.levels[0].mclk.vMPLL_AD_FUNC_CNTL_2 =
1693                 cpu_to_be32(ni_pi->clock_registers.mpll_ad_func_cntl_2);
1694         table->initialState.levels[0].mclk.vMPLL_DQ_FUNC_CNTL =
1695                 cpu_to_be32(ni_pi->clock_registers.mpll_dq_func_cntl);
1696         table->initialState.levels[0].mclk.vMPLL_DQ_FUNC_CNTL_2 =
1697                 cpu_to_be32(ni_pi->clock_registers.mpll_dq_func_cntl_2);
1698         table->initialState.levels[0].mclk.vMCLK_PWRMGT_CNTL =
1699                 cpu_to_be32(ni_pi->clock_registers.mclk_pwrmgt_cntl);
1700         table->initialState.levels[0].mclk.vDLL_CNTL =
1701                 cpu_to_be32(ni_pi->clock_registers.dll_cntl);
1702         table->initialState.levels[0].mclk.vMPLL_SS =
1703                 cpu_to_be32(ni_pi->clock_registers.mpll_ss1);
1704         table->initialState.levels[0].mclk.vMPLL_SS2 =
1705                 cpu_to_be32(ni_pi->clock_registers.mpll_ss2);
1706         table->initialState.levels[0].mclk.mclk_value =
1707                 cpu_to_be32(initial_state->performance_levels[0].mclk);
1708
1709         table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL =
1710                 cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl);
1711         table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_2 =
1712                 cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl_2);
1713         table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_3 =
1714                 cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl_3);
1715         table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_4 =
1716                 cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl_4);
1717         table->initialState.levels[0].sclk.vCG_SPLL_SPREAD_SPECTRUM =
1718                 cpu_to_be32(ni_pi->clock_registers.cg_spll_spread_spectrum);
1719         table->initialState.levels[0].sclk.vCG_SPLL_SPREAD_SPECTRUM_2 =
1720                 cpu_to_be32(ni_pi->clock_registers.cg_spll_spread_spectrum_2);
1721         table->initialState.levels[0].sclk.sclk_value =
1722                 cpu_to_be32(initial_state->performance_levels[0].sclk);
1723         table->initialState.levels[0].arbRefreshState =
1724                 NISLANDS_INITIAL_STATE_ARB_INDEX;
1725
1726         table->initialState.levels[0].ACIndex = 0;
1727
1728         ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
1729                                         initial_state->performance_levels[0].vddc,
1730                                         &table->initialState.levels[0].vddc);
1731         if (!ret) {
1732                 u16 std_vddc;
1733
1734                 ret = ni_get_std_voltage_value(rdev,
1735                                                &table->initialState.levels[0].vddc,
1736                                                &std_vddc);
1737                 if (!ret)
1738                         ni_populate_std_voltage_value(rdev, std_vddc,
1739                                                       table->initialState.levels[0].vddc.index,
1740                                                       &table->initialState.levels[0].std_vddc);
1741         }
1742
1743         if (eg_pi->vddci_control)
1744                 ni_populate_voltage_value(rdev,
1745                                           &eg_pi->vddci_voltage_table,
1746                                           initial_state->performance_levels[0].vddci,
1747                                           &table->initialState.levels[0].vddci);
1748
1749         ni_populate_initial_mvdd_value(rdev, &table->initialState.levels[0].mvdd);
1750
1751         reg = CG_R(0xffff) | CG_L(0);
1752         table->initialState.levels[0].aT = cpu_to_be32(reg);
1753
1754         table->initialState.levels[0].bSP = cpu_to_be32(pi->dsp);
1755
1756         if (pi->boot_in_gen2)
1757                 table->initialState.levels[0].gen2PCIE = 1;
1758         else
1759                 table->initialState.levels[0].gen2PCIE = 0;
1760
1761         if (pi->mem_gddr5) {
1762                 table->initialState.levels[0].strobeMode =
1763                         cypress_get_strobe_mode_settings(rdev,
1764                                                          initial_state->performance_levels[0].mclk);
1765
1766                 if (initial_state->performance_levels[0].mclk > pi->mclk_edc_enable_threshold)
1767                         table->initialState.levels[0].mcFlags = NISLANDS_SMC_MC_EDC_RD_FLAG | NISLANDS_SMC_MC_EDC_WR_FLAG;
1768                 else
1769                         table->initialState.levels[0].mcFlags =  0;
1770         }
1771
1772         table->initialState.levelCount = 1;
1773
1774         table->initialState.flags |= PPSMC_SWSTATE_FLAG_DC;
1775
1776         table->initialState.levels[0].dpm2.MaxPS = 0;
1777         table->initialState.levels[0].dpm2.NearTDPDec = 0;
1778         table->initialState.levels[0].dpm2.AboveSafeInc = 0;
1779         table->initialState.levels[0].dpm2.BelowSafeInc = 0;
1780
1781         reg = MIN_POWER_MASK | MAX_POWER_MASK;
1782         table->initialState.levels[0].SQPowerThrottle = cpu_to_be32(reg);
1783
1784         reg = MAX_POWER_DELTA_MASK | STI_SIZE_MASK | LTI_RATIO_MASK;
1785         table->initialState.levels[0].SQPowerThrottle_2 = cpu_to_be32(reg);
1786
1787         return 0;
1788 }
1789
1790 static int ni_populate_smc_acpi_state(struct radeon_device *rdev,
1791                                       NISLANDS_SMC_STATETABLE *table)
1792 {
1793         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1794         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1795         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1796         u32 mpll_ad_func_cntl   = ni_pi->clock_registers.mpll_ad_func_cntl;
1797         u32 mpll_ad_func_cntl_2 = ni_pi->clock_registers.mpll_ad_func_cntl_2;
1798         u32 mpll_dq_func_cntl   = ni_pi->clock_registers.mpll_dq_func_cntl;
1799         u32 mpll_dq_func_cntl_2 = ni_pi->clock_registers.mpll_dq_func_cntl_2;
1800         u32 spll_func_cntl      = ni_pi->clock_registers.cg_spll_func_cntl;
1801         u32 spll_func_cntl_2    = ni_pi->clock_registers.cg_spll_func_cntl_2;
1802         u32 spll_func_cntl_3    = ni_pi->clock_registers.cg_spll_func_cntl_3;
1803         u32 spll_func_cntl_4    = ni_pi->clock_registers.cg_spll_func_cntl_4;
1804         u32 mclk_pwrmgt_cntl    = ni_pi->clock_registers.mclk_pwrmgt_cntl;
1805         u32 dll_cntl            = ni_pi->clock_registers.dll_cntl;
1806         u32 reg;
1807         int ret;
1808
1809         table->ACPIState = table->initialState;
1810
1811         table->ACPIState.flags &= ~PPSMC_SWSTATE_FLAG_DC;
1812
1813         if (pi->acpi_vddc) {
1814                 ret = ni_populate_voltage_value(rdev,
1815                                                 &eg_pi->vddc_voltage_table,
1816                                                 pi->acpi_vddc, &table->ACPIState.levels[0].vddc);
1817                 if (!ret) {
1818                         u16 std_vddc;
1819
1820                         ret = ni_get_std_voltage_value(rdev,
1821                                                        &table->ACPIState.levels[0].vddc, &std_vddc);
1822                         if (!ret)
1823                                 ni_populate_std_voltage_value(rdev, std_vddc,
1824                                                               table->ACPIState.levels[0].vddc.index,
1825                                                               &table->ACPIState.levels[0].std_vddc);
1826                 }
1827
1828                 if (pi->pcie_gen2) {
1829                         if (pi->acpi_pcie_gen2)
1830                                 table->ACPIState.levels[0].gen2PCIE = 1;
1831                         else
1832                                 table->ACPIState.levels[0].gen2PCIE = 0;
1833                 } else {
1834                         table->ACPIState.levels[0].gen2PCIE = 0;
1835                 }
1836         } else {
1837                 ret = ni_populate_voltage_value(rdev,
1838                                                 &eg_pi->vddc_voltage_table,
1839                                                 pi->min_vddc_in_table,
1840                                                 &table->ACPIState.levels[0].vddc);
1841                 if (!ret) {
1842                         u16 std_vddc;
1843
1844                         ret = ni_get_std_voltage_value(rdev,
1845                                                        &table->ACPIState.levels[0].vddc,
1846                                                        &std_vddc);
1847                         if (!ret)
1848                                 ni_populate_std_voltage_value(rdev, std_vddc,
1849                                                               table->ACPIState.levels[0].vddc.index,
1850                                                               &table->ACPIState.levels[0].std_vddc);
1851                 }
1852                 table->ACPIState.levels[0].gen2PCIE = 0;
1853         }
1854
1855         if (eg_pi->acpi_vddci) {
1856                 if (eg_pi->vddci_control)
1857                         ni_populate_voltage_value(rdev,
1858                                                   &eg_pi->vddci_voltage_table,
1859                                                   eg_pi->acpi_vddci,
1860                                                   &table->ACPIState.levels[0].vddci);
1861         }
1862
1863
1864         mpll_ad_func_cntl &= ~PDNB;
1865
1866         mpll_ad_func_cntl_2 |= BIAS_GEN_PDNB | RESET_EN;
1867
1868         if (pi->mem_gddr5)
1869                 mpll_dq_func_cntl &= ~PDNB;
1870         mpll_dq_func_cntl_2 |= BIAS_GEN_PDNB | RESET_EN | BYPASS;
1871
1872
1873         mclk_pwrmgt_cntl |= (MRDCKA0_RESET |
1874                              MRDCKA1_RESET |
1875                              MRDCKB0_RESET |
1876                              MRDCKB1_RESET |
1877                              MRDCKC0_RESET |
1878                              MRDCKC1_RESET |
1879                              MRDCKD0_RESET |
1880                              MRDCKD1_RESET);
1881
1882         mclk_pwrmgt_cntl &= ~(MRDCKA0_PDNB |
1883                               MRDCKA1_PDNB |
1884                               MRDCKB0_PDNB |
1885                               MRDCKB1_PDNB |
1886                               MRDCKC0_PDNB |
1887                               MRDCKC1_PDNB |
1888                               MRDCKD0_PDNB |
1889                               MRDCKD1_PDNB);
1890
1891         dll_cntl |= (MRDCKA0_BYPASS |
1892                      MRDCKA1_BYPASS |
1893                      MRDCKB0_BYPASS |
1894                      MRDCKB1_BYPASS |
1895                      MRDCKC0_BYPASS |
1896                      MRDCKC1_BYPASS |
1897                      MRDCKD0_BYPASS |
1898                      MRDCKD1_BYPASS);
1899
1900         spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK;
1901         spll_func_cntl_2 |= SCLK_MUX_SEL(4);
1902
1903         table->ACPIState.levels[0].mclk.vMPLL_AD_FUNC_CNTL = cpu_to_be32(mpll_ad_func_cntl);
1904         table->ACPIState.levels[0].mclk.vMPLL_AD_FUNC_CNTL_2 = cpu_to_be32(mpll_ad_func_cntl_2);
1905         table->ACPIState.levels[0].mclk.vMPLL_DQ_FUNC_CNTL = cpu_to_be32(mpll_dq_func_cntl);
1906         table->ACPIState.levels[0].mclk.vMPLL_DQ_FUNC_CNTL_2 = cpu_to_be32(mpll_dq_func_cntl_2);
1907         table->ACPIState.levels[0].mclk.vMCLK_PWRMGT_CNTL = cpu_to_be32(mclk_pwrmgt_cntl);
1908         table->ACPIState.levels[0].mclk.vDLL_CNTL = cpu_to_be32(dll_cntl);
1909
1910         table->ACPIState.levels[0].mclk.mclk_value = 0;
1911
1912         table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL = cpu_to_be32(spll_func_cntl);
1913         table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_2 = cpu_to_be32(spll_func_cntl_2);
1914         table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_3 = cpu_to_be32(spll_func_cntl_3);
1915         table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_4 = cpu_to_be32(spll_func_cntl_4);
1916
1917         table->ACPIState.levels[0].sclk.sclk_value = 0;
1918
1919         ni_populate_mvdd_value(rdev, 0, &table->ACPIState.levels[0].mvdd);
1920
1921         if (eg_pi->dynamic_ac_timing)
1922                 table->ACPIState.levels[0].ACIndex = 1;
1923
1924         table->ACPIState.levels[0].dpm2.MaxPS = 0;
1925         table->ACPIState.levels[0].dpm2.NearTDPDec = 0;
1926         table->ACPIState.levels[0].dpm2.AboveSafeInc = 0;
1927         table->ACPIState.levels[0].dpm2.BelowSafeInc = 0;
1928
1929         reg = MIN_POWER_MASK | MAX_POWER_MASK;
1930         table->ACPIState.levels[0].SQPowerThrottle = cpu_to_be32(reg);
1931
1932         reg = MAX_POWER_DELTA_MASK | STI_SIZE_MASK | LTI_RATIO_MASK;
1933         table->ACPIState.levels[0].SQPowerThrottle_2 = cpu_to_be32(reg);
1934
1935         return 0;
1936 }
1937
1938 static int ni_init_smc_table(struct radeon_device *rdev)
1939 {
1940         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1941         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1942         int ret;
1943         struct radeon_ps *radeon_boot_state = rdev->pm.dpm.boot_ps;
1944         NISLANDS_SMC_STATETABLE *table = &ni_pi->smc_statetable;
1945
1946         memset(table, 0, sizeof(NISLANDS_SMC_STATETABLE));
1947
1948         ni_populate_smc_voltage_tables(rdev, table);
1949
1950         switch (rdev->pm.int_thermal_type) {
1951         case THERMAL_TYPE_NI:
1952         case THERMAL_TYPE_EMC2103_WITH_INTERNAL:
1953                 table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_INTERNAL;
1954                 break;
1955         case THERMAL_TYPE_NONE:
1956                 table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_NONE;
1957                 break;
1958         default:
1959                 table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_EXTERNAL;
1960                 break;
1961         }
1962
1963         if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_HARDWAREDC)
1964                 table->systemFlags |= PPSMC_SYSTEMFLAG_GPIO_DC;
1965
1966         if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_REGULATOR_HOT)
1967                 table->systemFlags |= PPSMC_SYSTEMFLAG_REGULATOR_HOT;
1968
1969         if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_STEPVDDC)
1970                 table->systemFlags |= PPSMC_SYSTEMFLAG_STEPVDDC;
1971
1972         if (pi->mem_gddr5)
1973                 table->systemFlags |= PPSMC_SYSTEMFLAG_GDDR5;
1974
1975         ret = ni_populate_smc_initial_state(rdev, radeon_boot_state, table);
1976         if (ret)
1977                 return ret;
1978
1979         ret = ni_populate_smc_acpi_state(rdev, table);
1980         if (ret)
1981                 return ret;
1982
1983         table->driverState = table->initialState;
1984
1985         table->ULVState = table->initialState;
1986
1987         ret = ni_do_program_memory_timing_parameters(rdev, radeon_boot_state,
1988                                                      NISLANDS_INITIAL_STATE_ARB_INDEX);
1989         if (ret)
1990                 return ret;
1991
1992         return rv770_copy_bytes_to_smc(rdev, pi->state_table_start, (u8 *)table,
1993                                        sizeof(NISLANDS_SMC_STATETABLE), pi->sram_end);
1994 }
1995
1996 static int ni_calculate_sclk_params(struct radeon_device *rdev,
1997                                     u32 engine_clock,
1998                                     NISLANDS_SMC_SCLK_VALUE *sclk)
1999 {
2000         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2001         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2002         struct atom_clock_dividers dividers;
2003         u32 spll_func_cntl = ni_pi->clock_registers.cg_spll_func_cntl;
2004         u32 spll_func_cntl_2 = ni_pi->clock_registers.cg_spll_func_cntl_2;
2005         u32 spll_func_cntl_3 = ni_pi->clock_registers.cg_spll_func_cntl_3;
2006         u32 spll_func_cntl_4 = ni_pi->clock_registers.cg_spll_func_cntl_4;
2007         u32 cg_spll_spread_spectrum = ni_pi->clock_registers.cg_spll_spread_spectrum;
2008         u32 cg_spll_spread_spectrum_2 = ni_pi->clock_registers.cg_spll_spread_spectrum_2;
2009         u64 tmp;
2010         u32 reference_clock = rdev->clock.spll.reference_freq;
2011         u32 reference_divider;
2012         u32 fbdiv;
2013         int ret;
2014
2015         ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
2016                                              engine_clock, false, &dividers);
2017         if (ret)
2018                 return ret;
2019
2020         reference_divider = 1 + dividers.ref_div;
2021
2022
2023         tmp = (u64) engine_clock * reference_divider * dividers.post_div * 16834;
2024         do_div(tmp, reference_clock);
2025         fbdiv = (u32) tmp;
2026
2027         spll_func_cntl &= ~(SPLL_PDIV_A_MASK | SPLL_REF_DIV_MASK);
2028         spll_func_cntl |= SPLL_REF_DIV(dividers.ref_div);
2029         spll_func_cntl |= SPLL_PDIV_A(dividers.post_div);
2030
2031         spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK;
2032         spll_func_cntl_2 |= SCLK_MUX_SEL(2);
2033
2034         spll_func_cntl_3 &= ~SPLL_FB_DIV_MASK;
2035         spll_func_cntl_3 |= SPLL_FB_DIV(fbdiv);
2036         spll_func_cntl_3 |= SPLL_DITHEN;
2037
2038         if (pi->sclk_ss) {
2039                 struct radeon_atom_ss ss;
2040                 u32 vco_freq = engine_clock * dividers.post_div;
2041
2042                 if (radeon_atombios_get_asic_ss_info(rdev, &ss,
2043                                                      ASIC_INTERNAL_ENGINE_SS, vco_freq)) {
2044                         u32 clk_s = reference_clock * 5 / (reference_divider * ss.rate);
2045                         u32 clk_v = 4 * ss.percentage * fbdiv / (clk_s * 10000);
2046
2047                         cg_spll_spread_spectrum &= ~CLK_S_MASK;
2048                         cg_spll_spread_spectrum |= CLK_S(clk_s);
2049                         cg_spll_spread_spectrum |= SSEN;
2050
2051                         cg_spll_spread_spectrum_2 &= ~CLK_V_MASK;
2052                         cg_spll_spread_spectrum_2 |= CLK_V(clk_v);
2053                 }
2054         }
2055
2056         sclk->sclk_value = engine_clock;
2057         sclk->vCG_SPLL_FUNC_CNTL = spll_func_cntl;
2058         sclk->vCG_SPLL_FUNC_CNTL_2 = spll_func_cntl_2;
2059         sclk->vCG_SPLL_FUNC_CNTL_3 = spll_func_cntl_3;
2060         sclk->vCG_SPLL_FUNC_CNTL_4 = spll_func_cntl_4;
2061         sclk->vCG_SPLL_SPREAD_SPECTRUM = cg_spll_spread_spectrum;
2062         sclk->vCG_SPLL_SPREAD_SPECTRUM_2 = cg_spll_spread_spectrum_2;
2063
2064         return 0;
2065 }
2066
2067 static int ni_populate_sclk_value(struct radeon_device *rdev,
2068                                   u32 engine_clock,
2069                                   NISLANDS_SMC_SCLK_VALUE *sclk)
2070 {
2071         NISLANDS_SMC_SCLK_VALUE sclk_tmp;
2072         int ret;
2073
2074         ret = ni_calculate_sclk_params(rdev, engine_clock, &sclk_tmp);
2075         if (!ret) {
2076                 sclk->sclk_value = cpu_to_be32(sclk_tmp.sclk_value);
2077                 sclk->vCG_SPLL_FUNC_CNTL = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL);
2078                 sclk->vCG_SPLL_FUNC_CNTL_2 = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL_2);
2079                 sclk->vCG_SPLL_FUNC_CNTL_3 = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL_3);
2080                 sclk->vCG_SPLL_FUNC_CNTL_4 = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL_4);
2081                 sclk->vCG_SPLL_SPREAD_SPECTRUM = cpu_to_be32(sclk_tmp.vCG_SPLL_SPREAD_SPECTRUM);
2082                 sclk->vCG_SPLL_SPREAD_SPECTRUM_2 = cpu_to_be32(sclk_tmp.vCG_SPLL_SPREAD_SPECTRUM_2);
2083         }
2084
2085         return ret;
2086 }
2087
2088 static int ni_init_smc_spll_table(struct radeon_device *rdev)
2089 {
2090         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2091         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2092         SMC_NISLANDS_SPLL_DIV_TABLE *spll_table;
2093         NISLANDS_SMC_SCLK_VALUE sclk_params;
2094         u32 fb_div;
2095         u32 p_div;
2096         u32 clk_s;
2097         u32 clk_v;
2098         u32 sclk = 0;
2099         int i, ret;
2100         u32 tmp;
2101
2102         if (ni_pi->spll_table_start == 0)
2103                 return -EINVAL;
2104
2105         spll_table = kzalloc(sizeof(SMC_NISLANDS_SPLL_DIV_TABLE), GFP_KERNEL);
2106         if (spll_table == NULL)
2107                 return -ENOMEM;
2108
2109         for (i = 0; i < 256; i++) {
2110                 ret = ni_calculate_sclk_params(rdev, sclk, &sclk_params);
2111                 if (ret)
2112                         break;
2113
2114                 p_div = (sclk_params.vCG_SPLL_FUNC_CNTL & SPLL_PDIV_A_MASK) >> SPLL_PDIV_A_SHIFT;
2115                 fb_div = (sclk_params.vCG_SPLL_FUNC_CNTL_3 & SPLL_FB_DIV_MASK) >> SPLL_FB_DIV_SHIFT;
2116                 clk_s = (sclk_params.vCG_SPLL_SPREAD_SPECTRUM & CLK_S_MASK) >> CLK_S_SHIFT;
2117                 clk_v = (sclk_params.vCG_SPLL_SPREAD_SPECTRUM_2 & CLK_V_MASK) >> CLK_V_SHIFT;
2118
2119                 fb_div &= ~0x00001FFF;
2120                 fb_div >>= 1;
2121                 clk_v >>= 6;
2122
2123                 if (p_div & ~(SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_SHIFT))
2124                         ret = -EINVAL;
2125
2126                 if (clk_s & ~(SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_SHIFT))
2127                         ret = -EINVAL;
2128
2129                 if (fb_div & ~(SMC_NISLANDS_SPLL_DIV_TABLE_FBDIV_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_FBDIV_SHIFT))
2130                         ret = -EINVAL;
2131
2132                 if (clk_v & ~(SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_SHIFT))
2133                         ret = -EINVAL;
2134
2135                 if (ret)
2136                         break;
2137
2138                 tmp = ((fb_div << SMC_NISLANDS_SPLL_DIV_TABLE_FBDIV_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_FBDIV_MASK) |
2139                         ((p_div << SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_MASK);
2140                 spll_table->freq[i] = cpu_to_be32(tmp);
2141
2142                 tmp = ((clk_v << SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_MASK) |
2143                         ((clk_s << SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_MASK);
2144                 spll_table->ss[i] = cpu_to_be32(tmp);
2145
2146                 sclk += 512;
2147         }
2148
2149         if (!ret)
2150                 ret = rv770_copy_bytes_to_smc(rdev, ni_pi->spll_table_start, (u8 *)spll_table,
2151                                               sizeof(SMC_NISLANDS_SPLL_DIV_TABLE), pi->sram_end);
2152
2153         kfree(spll_table);
2154
2155         return ret;
2156 }
2157
2158 static int ni_populate_mclk_value(struct radeon_device *rdev,
2159                                   u32 engine_clock,
2160                                   u32 memory_clock,
2161                                   NISLANDS_SMC_MCLK_VALUE *mclk,
2162                                   bool strobe_mode,
2163                                   bool dll_state_on)
2164 {
2165         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2166         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2167         u32 mpll_ad_func_cntl = ni_pi->clock_registers.mpll_ad_func_cntl;
2168         u32 mpll_ad_func_cntl_2 = ni_pi->clock_registers.mpll_ad_func_cntl_2;
2169         u32 mpll_dq_func_cntl = ni_pi->clock_registers.mpll_dq_func_cntl;
2170         u32 mpll_dq_func_cntl_2 = ni_pi->clock_registers.mpll_dq_func_cntl_2;
2171         u32 mclk_pwrmgt_cntl = ni_pi->clock_registers.mclk_pwrmgt_cntl;
2172         u32 dll_cntl = ni_pi->clock_registers.dll_cntl;
2173         u32 mpll_ss1 = ni_pi->clock_registers.mpll_ss1;
2174         u32 mpll_ss2 = ni_pi->clock_registers.mpll_ss2;
2175         struct atom_clock_dividers dividers;
2176         u32 ibias;
2177         u32 dll_speed;
2178         int ret;
2179         u32 mc_seq_misc7;
2180
2181         ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_MEMORY_PLL_PARAM,
2182                                              memory_clock, strobe_mode, &dividers);
2183         if (ret)
2184                 return ret;
2185
2186         if (!strobe_mode) {
2187                 mc_seq_misc7 = RREG32(MC_SEQ_MISC7);
2188
2189                 if (mc_seq_misc7 & 0x8000000)
2190                         dividers.post_div = 1;
2191         }
2192
2193         ibias = cypress_map_clkf_to_ibias(rdev, dividers.whole_fb_div);
2194
2195         mpll_ad_func_cntl &= ~(CLKR_MASK |
2196                                YCLK_POST_DIV_MASK |
2197                                CLKF_MASK |
2198                                CLKFRAC_MASK |
2199                                IBIAS_MASK);
2200         mpll_ad_func_cntl |= CLKR(dividers.ref_div);
2201         mpll_ad_func_cntl |= YCLK_POST_DIV(dividers.post_div);
2202         mpll_ad_func_cntl |= CLKF(dividers.whole_fb_div);
2203         mpll_ad_func_cntl |= CLKFRAC(dividers.frac_fb_div);
2204         mpll_ad_func_cntl |= IBIAS(ibias);
2205
2206         if (dividers.vco_mode)
2207                 mpll_ad_func_cntl_2 |= VCO_MODE;
2208         else
2209                 mpll_ad_func_cntl_2 &= ~VCO_MODE;
2210
2211         if (pi->mem_gddr5) {
2212                 mpll_dq_func_cntl &= ~(CLKR_MASK |
2213                                        YCLK_POST_DIV_MASK |
2214                                        CLKF_MASK |
2215                                        CLKFRAC_MASK |
2216                                        IBIAS_MASK);
2217                 mpll_dq_func_cntl |= CLKR(dividers.ref_div);
2218                 mpll_dq_func_cntl |= YCLK_POST_DIV(dividers.post_div);
2219                 mpll_dq_func_cntl |= CLKF(dividers.whole_fb_div);
2220                 mpll_dq_func_cntl |= CLKFRAC(dividers.frac_fb_div);
2221                 mpll_dq_func_cntl |= IBIAS(ibias);
2222
2223                 if (strobe_mode)
2224                         mpll_dq_func_cntl &= ~PDNB;
2225                 else
2226                         mpll_dq_func_cntl |= PDNB;
2227
2228                 if (dividers.vco_mode)
2229                         mpll_dq_func_cntl_2 |= VCO_MODE;
2230                 else
2231                         mpll_dq_func_cntl_2 &= ~VCO_MODE;
2232         }
2233
2234         if (pi->mclk_ss) {
2235                 struct radeon_atom_ss ss;
2236                 u32 vco_freq = memory_clock * dividers.post_div;
2237
2238                 if (radeon_atombios_get_asic_ss_info(rdev, &ss,
2239                                                      ASIC_INTERNAL_MEMORY_SS, vco_freq)) {
2240                         u32 reference_clock = rdev->clock.mpll.reference_freq;
2241                         u32 decoded_ref = rv740_get_decoded_reference_divider(dividers.ref_div);
2242                         u32 clk_s = reference_clock * 5 / (decoded_ref * ss.rate);
2243                         u32 clk_v = ss.percentage *
2244                                 (0x4000 * dividers.whole_fb_div + 0x800 * dividers.frac_fb_div) / (clk_s * 625);
2245
2246                         mpll_ss1 &= ~CLKV_MASK;
2247                         mpll_ss1 |= CLKV(clk_v);
2248
2249                         mpll_ss2 &= ~CLKS_MASK;
2250                         mpll_ss2 |= CLKS(clk_s);
2251                 }
2252         }
2253
2254         dll_speed = rv740_get_dll_speed(pi->mem_gddr5,
2255                                         memory_clock);
2256
2257         mclk_pwrmgt_cntl &= ~DLL_SPEED_MASK;
2258         mclk_pwrmgt_cntl |= DLL_SPEED(dll_speed);
2259         if (dll_state_on)
2260                 mclk_pwrmgt_cntl |= (MRDCKA0_PDNB |
2261                                      MRDCKA1_PDNB |
2262                                      MRDCKB0_PDNB |
2263                                      MRDCKB1_PDNB |
2264                                      MRDCKC0_PDNB |
2265                                      MRDCKC1_PDNB |
2266                                      MRDCKD0_PDNB |
2267                                      MRDCKD1_PDNB);
2268         else
2269                 mclk_pwrmgt_cntl &= ~(MRDCKA0_PDNB |
2270                                       MRDCKA1_PDNB |
2271                                       MRDCKB0_PDNB |
2272                                       MRDCKB1_PDNB |
2273                                       MRDCKC0_PDNB |
2274                                       MRDCKC1_PDNB |
2275                                       MRDCKD0_PDNB |
2276                                       MRDCKD1_PDNB);
2277
2278
2279         mclk->mclk_value = cpu_to_be32(memory_clock);
2280         mclk->vMPLL_AD_FUNC_CNTL = cpu_to_be32(mpll_ad_func_cntl);
2281         mclk->vMPLL_AD_FUNC_CNTL_2 = cpu_to_be32(mpll_ad_func_cntl_2);
2282         mclk->vMPLL_DQ_FUNC_CNTL = cpu_to_be32(mpll_dq_func_cntl);
2283         mclk->vMPLL_DQ_FUNC_CNTL_2 = cpu_to_be32(mpll_dq_func_cntl_2);
2284         mclk->vMCLK_PWRMGT_CNTL = cpu_to_be32(mclk_pwrmgt_cntl);
2285         mclk->vDLL_CNTL = cpu_to_be32(dll_cntl);
2286         mclk->vMPLL_SS = cpu_to_be32(mpll_ss1);
2287         mclk->vMPLL_SS2 = cpu_to_be32(mpll_ss2);
2288
2289         return 0;
2290 }
2291
2292 static void ni_populate_smc_sp(struct radeon_device *rdev,
2293                                struct radeon_ps *radeon_state,
2294                                NISLANDS_SMC_SWSTATE *smc_state)
2295 {
2296         struct ni_ps *ps = ni_get_ps(radeon_state);
2297         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2298         int i;
2299
2300         for (i = 0; i < ps->performance_level_count - 1; i++)
2301                 smc_state->levels[i].bSP = cpu_to_be32(pi->dsp);
2302
2303         smc_state->levels[ps->performance_level_count - 1].bSP =
2304                 cpu_to_be32(pi->psp);
2305 }
2306
2307 static int ni_convert_power_level_to_smc(struct radeon_device *rdev,
2308                                          struct rv7xx_pl *pl,
2309                                          NISLANDS_SMC_HW_PERFORMANCE_LEVEL *level)
2310 {
2311         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2312         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2313         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2314         int ret;
2315         bool dll_state_on;
2316         u16 std_vddc;
2317         u32 tmp = RREG32(DC_STUTTER_CNTL);
2318
2319         level->gen2PCIE = pi->pcie_gen2 ?
2320                 ((pl->flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2) ? 1 : 0) : 0;
2321
2322         ret = ni_populate_sclk_value(rdev, pl->sclk, &level->sclk);
2323         if (ret)
2324                 return ret;
2325
2326         level->mcFlags =  0;
2327         if (pi->mclk_stutter_mode_threshold &&
2328             (pl->mclk <= pi->mclk_stutter_mode_threshold) &&
2329             !eg_pi->uvd_enabled &&
2330             (tmp & DC_STUTTER_ENABLE_A) &&
2331             (tmp & DC_STUTTER_ENABLE_B))
2332                 level->mcFlags |= NISLANDS_SMC_MC_STUTTER_EN;
2333
2334         if (pi->mem_gddr5) {
2335                 if (pl->mclk > pi->mclk_edc_enable_threshold)
2336                         level->mcFlags |= NISLANDS_SMC_MC_EDC_RD_FLAG;
2337                 if (pl->mclk > eg_pi->mclk_edc_wr_enable_threshold)
2338                         level->mcFlags |= NISLANDS_SMC_MC_EDC_WR_FLAG;
2339
2340                 level->strobeMode = cypress_get_strobe_mode_settings(rdev, pl->mclk);
2341
2342                 if (level->strobeMode & NISLANDS_SMC_STROBE_ENABLE) {
2343                         if (cypress_get_mclk_frequency_ratio(rdev, pl->mclk, true) >=
2344                             ((RREG32(MC_SEQ_MISC7) >> 16) & 0xf))
2345                                 dll_state_on = ((RREG32(MC_SEQ_MISC5) >> 1) & 0x1) ? true : false;
2346                         else
2347                                 dll_state_on = ((RREG32(MC_SEQ_MISC6) >> 1) & 0x1) ? true : false;
2348                 } else {
2349                         dll_state_on = false;
2350                         if (pl->mclk > ni_pi->mclk_rtt_mode_threshold)
2351                                 level->mcFlags |= NISLANDS_SMC_MC_RTT_ENABLE;
2352                 }
2353
2354                 ret = ni_populate_mclk_value(rdev, pl->sclk, pl->mclk,
2355                                              &level->mclk,
2356                                              (level->strobeMode & NISLANDS_SMC_STROBE_ENABLE) != 0,
2357                                              dll_state_on);
2358         } else
2359                 ret = ni_populate_mclk_value(rdev, pl->sclk, pl->mclk, &level->mclk, 1, 1);
2360
2361         if (ret)
2362                 return ret;
2363
2364         ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
2365                                         pl->vddc, &level->vddc);
2366         if (ret)
2367                 return ret;
2368
2369         ret = ni_get_std_voltage_value(rdev, &level->vddc, &std_vddc);
2370         if (ret)
2371                 return ret;
2372
2373         ni_populate_std_voltage_value(rdev, std_vddc,
2374                                       level->vddc.index, &level->std_vddc);
2375
2376         if (eg_pi->vddci_control) {
2377                 ret = ni_populate_voltage_value(rdev, &eg_pi->vddci_voltage_table,
2378                                                 pl->vddci, &level->vddci);
2379                 if (ret)
2380                         return ret;
2381         }
2382
2383         ni_populate_mvdd_value(rdev, pl->mclk, &level->mvdd);
2384
2385         return ret;
2386 }
2387
2388 static int ni_populate_smc_t(struct radeon_device *rdev,
2389                              struct radeon_ps *radeon_state,
2390                              NISLANDS_SMC_SWSTATE *smc_state)
2391 {
2392         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2393         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2394         struct ni_ps *state = ni_get_ps(radeon_state);
2395         u32 a_t;
2396         u32 t_l, t_h;
2397         u32 high_bsp;
2398         int i, ret;
2399
2400         if (state->performance_level_count >= 9)
2401                 return -EINVAL;
2402
2403         if (state->performance_level_count < 2) {
2404                 a_t = CG_R(0xffff) | CG_L(0);
2405                 smc_state->levels[0].aT = cpu_to_be32(a_t);
2406                 return 0;
2407         }
2408
2409         smc_state->levels[0].aT = cpu_to_be32(0);
2410
2411         for (i = 0; i <= state->performance_level_count - 2; i++) {
2412                 if (eg_pi->uvd_enabled)
2413                         ret = r600_calculate_at(
2414                                 1000 * (i * (eg_pi->smu_uvd_hs ? 2 : 8) + 2),
2415                                 100 * R600_AH_DFLT,
2416                                 state->performance_levels[i + 1].sclk,
2417                                 state->performance_levels[i].sclk,
2418                                 &t_l,
2419                                 &t_h);
2420                 else
2421                         ret = r600_calculate_at(
2422                                 1000 * (i + 1),
2423                                 100 * R600_AH_DFLT,
2424                                 state->performance_levels[i + 1].sclk,
2425                                 state->performance_levels[i].sclk,
2426                                 &t_l,
2427                                 &t_h);
2428
2429                 if (ret) {
2430                         t_h = (i + 1) * 1000 - 50 * R600_AH_DFLT;
2431                         t_l = (i + 1) * 1000 + 50 * R600_AH_DFLT;
2432                 }
2433
2434                 a_t = be32_to_cpu(smc_state->levels[i].aT) & ~CG_R_MASK;
2435                 a_t |= CG_R(t_l * pi->bsp / 20000);
2436                 smc_state->levels[i].aT = cpu_to_be32(a_t);
2437
2438                 high_bsp = (i == state->performance_level_count - 2) ?
2439                         pi->pbsp : pi->bsp;
2440
2441                 a_t = CG_R(0xffff) | CG_L(t_h * high_bsp / 20000);
2442                 smc_state->levels[i + 1].aT = cpu_to_be32(a_t);
2443         }
2444
2445         return 0;
2446 }
2447
2448 static int ni_populate_power_containment_values(struct radeon_device *rdev,
2449                                                 struct radeon_ps *radeon_state,
2450                                                 NISLANDS_SMC_SWSTATE *smc_state)
2451 {
2452         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2453         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2454         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2455         struct ni_ps *state = ni_get_ps(radeon_state);
2456         u32 prev_sclk;
2457         u32 max_sclk;
2458         u32 min_sclk;
2459         int i, ret;
2460         u32 tdp_limit;
2461         u32 near_tdp_limit;
2462         u32 power_boost_limit;
2463         u8 max_ps_percent;
2464
2465         if (ni_pi->enable_power_containment == false)
2466                 return 0;
2467
2468         if (state->performance_level_count == 0)
2469                 return -EINVAL;
2470
2471         if (smc_state->levelCount != state->performance_level_count)
2472                 return -EINVAL;
2473
2474         ret = ni_calculate_adjusted_tdp_limits(rdev,
2475                                                false, /* ??? */
2476                                                rdev->pm.dpm.tdp_adjustment,
2477                                                &tdp_limit,
2478                                                &near_tdp_limit);
2479         if (ret)
2480                 return ret;
2481
2482         power_boost_limit = ni_calculate_power_boost_limit(rdev, radeon_state, near_tdp_limit);
2483
2484         ret = rv770_write_smc_sram_dword(rdev,
2485                                          pi->state_table_start +
2486                                          offsetof(NISLANDS_SMC_STATETABLE, dpm2Params) +
2487                                          offsetof(PP_NIslands_DPM2Parameters, PowerBoostLimit),
2488                                          ni_scale_power_for_smc(power_boost_limit, ni_get_smc_power_scaling_factor(rdev)),
2489                                          pi->sram_end);
2490         if (ret)
2491                 power_boost_limit = 0;
2492
2493         smc_state->levels[0].dpm2.MaxPS = 0;
2494         smc_state->levels[0].dpm2.NearTDPDec = 0;
2495         smc_state->levels[0].dpm2.AboveSafeInc = 0;
2496         smc_state->levels[0].dpm2.BelowSafeInc = 0;
2497         smc_state->levels[0].stateFlags |= power_boost_limit ? PPSMC_STATEFLAG_POWERBOOST : 0;
2498
2499         for (i = 1; i < state->performance_level_count; i++) {
2500                 prev_sclk = state->performance_levels[i-1].sclk;
2501                 max_sclk  = state->performance_levels[i].sclk;
2502                 max_ps_percent = (i != (state->performance_level_count - 1)) ?
2503                         NISLANDS_DPM2_MAXPS_PERCENT_M : NISLANDS_DPM2_MAXPS_PERCENT_H;
2504
2505                 if (max_sclk < prev_sclk)
2506                         return -EINVAL;
2507
2508                 if ((max_ps_percent == 0) || (prev_sclk == max_sclk) || eg_pi->uvd_enabled)
2509                         min_sclk = max_sclk;
2510                 else if (1 == i)
2511                         min_sclk = prev_sclk;
2512                 else
2513                         min_sclk = (prev_sclk * (u32)max_ps_percent) / 100;
2514
2515                 if (min_sclk < state->performance_levels[0].sclk)
2516                         min_sclk = state->performance_levels[0].sclk;
2517
2518                 if (min_sclk == 0)
2519                         return -EINVAL;
2520
2521                 smc_state->levels[i].dpm2.MaxPS =
2522                         (u8)((NISLANDS_DPM2_MAX_PULSE_SKIP * (max_sclk - min_sclk)) / max_sclk);
2523                 smc_state->levels[i].dpm2.NearTDPDec = NISLANDS_DPM2_NEAR_TDP_DEC;
2524                 smc_state->levels[i].dpm2.AboveSafeInc = NISLANDS_DPM2_ABOVE_SAFE_INC;
2525                 smc_state->levels[i].dpm2.BelowSafeInc = NISLANDS_DPM2_BELOW_SAFE_INC;
2526                 smc_state->levels[i].stateFlags |=
2527                         ((i != (state->performance_level_count - 1)) && power_boost_limit) ?
2528                         PPSMC_STATEFLAG_POWERBOOST : 0;
2529         }
2530
2531         return 0;
2532 }
2533
2534 static int ni_populate_sq_ramping_values(struct radeon_device *rdev,
2535                                          struct radeon_ps *radeon_state,
2536                                          NISLANDS_SMC_SWSTATE *smc_state)
2537 {
2538         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2539         struct ni_ps *state = ni_get_ps(radeon_state);
2540         u32 sq_power_throttle;
2541         u32 sq_power_throttle2;
2542         bool enable_sq_ramping = ni_pi->enable_sq_ramping;
2543         int i;
2544
2545         if (state->performance_level_count == 0)
2546                 return -EINVAL;
2547
2548         if (smc_state->levelCount != state->performance_level_count)
2549                 return -EINVAL;
2550
2551         if (rdev->pm.dpm.sq_ramping_threshold == 0)
2552                 return -EINVAL;
2553
2554         if (NISLANDS_DPM2_SQ_RAMP_MAX_POWER > (MAX_POWER_MASK >> MAX_POWER_SHIFT))
2555                 enable_sq_ramping = false;
2556
2557         if (NISLANDS_DPM2_SQ_RAMP_MIN_POWER > (MIN_POWER_MASK >> MIN_POWER_SHIFT))
2558                 enable_sq_ramping = false;
2559
2560         if (NISLANDS_DPM2_SQ_RAMP_MAX_POWER_DELTA > (MAX_POWER_DELTA_MASK >> MAX_POWER_DELTA_SHIFT))
2561                 enable_sq_ramping = false;
2562
2563         if (NISLANDS_DPM2_SQ_RAMP_STI_SIZE > (STI_SIZE_MASK >> STI_SIZE_SHIFT))
2564                 enable_sq_ramping = false;
2565
2566         if (NISLANDS_DPM2_SQ_RAMP_LTI_RATIO > (LTI_RATIO_MASK >> LTI_RATIO_SHIFT))
2567                 enable_sq_ramping = false;
2568
2569         for (i = 0; i < state->performance_level_count; i++) {
2570                 sq_power_throttle  = 0;
2571                 sq_power_throttle2 = 0;
2572
2573                 if ((state->performance_levels[i].sclk >= rdev->pm.dpm.sq_ramping_threshold) &&
2574                     enable_sq_ramping) {
2575                         sq_power_throttle |= MAX_POWER(NISLANDS_DPM2_SQ_RAMP_MAX_POWER);
2576                         sq_power_throttle |= MIN_POWER(NISLANDS_DPM2_SQ_RAMP_MIN_POWER);
2577                         sq_power_throttle2 |= MAX_POWER_DELTA(NISLANDS_DPM2_SQ_RAMP_MAX_POWER_DELTA);
2578                         sq_power_throttle2 |= STI_SIZE(NISLANDS_DPM2_SQ_RAMP_STI_SIZE);
2579                         sq_power_throttle2 |= LTI_RATIO(NISLANDS_DPM2_SQ_RAMP_LTI_RATIO);
2580                 } else {
2581                         sq_power_throttle |= MAX_POWER_MASK | MIN_POWER_MASK;
2582                         sq_power_throttle2 |= MAX_POWER_DELTA_MASK | STI_SIZE_MASK | LTI_RATIO_MASK;
2583                 }
2584
2585                 smc_state->levels[i].SQPowerThrottle   = cpu_to_be32(sq_power_throttle);
2586                 smc_state->levels[i].SQPowerThrottle_2 = cpu_to_be32(sq_power_throttle2);
2587         }
2588
2589         return 0;
2590 }
2591
2592 static int ni_enable_power_containment(struct radeon_device *rdev,
2593                                        struct radeon_ps *radeon_new_state,
2594                                        bool enable)
2595 {
2596         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2597         PPSMC_Result smc_result;
2598         int ret = 0;
2599
2600         if (ni_pi->enable_power_containment) {
2601                 if (enable) {
2602                         if (!r600_is_uvd_state(radeon_new_state->class, radeon_new_state->class2)) {
2603                                 smc_result = rv770_send_msg_to_smc(rdev, PPSMC_TDPClampingActive);
2604                                 if (smc_result != PPSMC_Result_OK) {
2605                                         ret = -EINVAL;
2606                                         ni_pi->pc_enabled = false;
2607                                 } else {
2608                                         ni_pi->pc_enabled = true;
2609                                 }
2610                         }
2611                 } else {
2612                         smc_result = rv770_send_msg_to_smc(rdev, PPSMC_TDPClampingInactive);
2613                         if (smc_result != PPSMC_Result_OK)
2614                                 ret = -EINVAL;
2615                         ni_pi->pc_enabled = false;
2616                 }
2617         }
2618
2619         return ret;
2620 }
2621
2622 static int ni_convert_power_state_to_smc(struct radeon_device *rdev,
2623                                          struct radeon_ps *radeon_state,
2624                                          NISLANDS_SMC_SWSTATE *smc_state)
2625 {
2626         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2627         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2628         struct ni_ps *state = ni_get_ps(radeon_state);
2629         int i, ret;
2630         u32 threshold = state->performance_levels[state->performance_level_count - 1].sclk * 100 / 100;
2631
2632         if (!(radeon_state->caps & ATOM_PPLIB_DISALLOW_ON_DC))
2633                 smc_state->flags |= PPSMC_SWSTATE_FLAG_DC;
2634
2635         smc_state->levelCount = 0;
2636
2637         if (state->performance_level_count > NISLANDS_MAX_SMC_PERFORMANCE_LEVELS_PER_SWSTATE)
2638                 return -EINVAL;
2639
2640         for (i = 0; i < state->performance_level_count; i++) {
2641                 ret = ni_convert_power_level_to_smc(rdev, &state->performance_levels[i],
2642                                                     &smc_state->levels[i]);
2643                 smc_state->levels[i].arbRefreshState =
2644                         (u8)(NISLANDS_DRIVER_STATE_ARB_INDEX + i);
2645
2646                 if (ret)
2647                         return ret;
2648
2649                 if (ni_pi->enable_power_containment)
2650                         smc_state->levels[i].displayWatermark =
2651                                 (state->performance_levels[i].sclk < threshold) ?
2652                                 PPSMC_DISPLAY_WATERMARK_LOW : PPSMC_DISPLAY_WATERMARK_HIGH;
2653                 else
2654                         smc_state->levels[i].displayWatermark = (i < 2) ?
2655                                 PPSMC_DISPLAY_WATERMARK_LOW : PPSMC_DISPLAY_WATERMARK_HIGH;
2656
2657                 if (eg_pi->dynamic_ac_timing)
2658                         smc_state->levels[i].ACIndex = NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT + i;
2659                 else
2660                         smc_state->levels[i].ACIndex = 0;
2661
2662                 smc_state->levelCount++;
2663         }
2664
2665         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_watermark_threshold,
2666                                       cpu_to_be32(threshold / 512));
2667
2668         ni_populate_smc_sp(rdev, radeon_state, smc_state);
2669
2670         ret = ni_populate_power_containment_values(rdev, radeon_state, smc_state);
2671         if (ret)
2672                 ni_pi->enable_power_containment = false;
2673
2674         ret = ni_populate_sq_ramping_values(rdev, radeon_state, smc_state);
2675         if (ret)
2676                 ni_pi->enable_sq_ramping = false;
2677
2678         return ni_populate_smc_t(rdev, radeon_state, smc_state);
2679 }
2680
2681 static int ni_upload_sw_state(struct radeon_device *rdev,
2682                               struct radeon_ps *radeon_new_state)
2683 {
2684         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2685         u16 address = pi->state_table_start +
2686                 offsetof(NISLANDS_SMC_STATETABLE, driverState);
2687         u16 state_size = sizeof(NISLANDS_SMC_SWSTATE) +
2688                 ((NISLANDS_MAX_SMC_PERFORMANCE_LEVELS_PER_SWSTATE - 1) * sizeof(NISLANDS_SMC_HW_PERFORMANCE_LEVEL));
2689         int ret;
2690         NISLANDS_SMC_SWSTATE *smc_state = kzalloc(state_size, GFP_KERNEL);
2691
2692         if (smc_state == NULL)
2693                 return -ENOMEM;
2694
2695         ret = ni_convert_power_state_to_smc(rdev, radeon_new_state, smc_state);
2696         if (ret)
2697                 goto done;
2698
2699         ret = rv770_copy_bytes_to_smc(rdev, address, (u8 *)smc_state, state_size, pi->sram_end);
2700
2701 done:
2702         kfree(smc_state);
2703
2704         return ret;
2705 }
2706
2707 static int ni_set_mc_special_registers(struct radeon_device *rdev,
2708                                        struct ni_mc_reg_table *table)
2709 {
2710         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2711         u8 i, j, k;
2712         u32 temp_reg;
2713
2714         for (i = 0, j = table->last; i < table->last; i++) {
2715                 switch (table->mc_reg_address[i].s1) {
2716                 case MC_SEQ_MISC1 >> 2:
2717                         if (j >= SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2718                                 return -EINVAL;
2719                         temp_reg = RREG32(MC_PMG_CMD_EMRS);
2720                         table->mc_reg_address[j].s1 = MC_PMG_CMD_EMRS >> 2;
2721                         table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_EMRS_LP >> 2;
2722                         for (k = 0; k < table->num_entries; k++)
2723                                 table->mc_reg_table_entry[k].mc_data[j] =
2724                                         ((temp_reg & 0xffff0000)) |
2725                                         ((table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16);
2726                         j++;
2727                         if (j >= SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2728                                 return -EINVAL;
2729
2730                         temp_reg = RREG32(MC_PMG_CMD_MRS);
2731                         table->mc_reg_address[j].s1 = MC_PMG_CMD_MRS >> 2;
2732                         table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_MRS_LP >> 2;
2733                         for(k = 0; k < table->num_entries; k++) {
2734                                 table->mc_reg_table_entry[k].mc_data[j] =
2735                                         (temp_reg & 0xffff0000) |
2736                                         (table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff);
2737                                 if (!pi->mem_gddr5)
2738                                         table->mc_reg_table_entry[k].mc_data[j] |= 0x100;
2739                         }
2740                         j++;
2741                         break;
2742                 case MC_SEQ_RESERVE_M >> 2:
2743                         if (j >= SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2744                                 return -EINVAL;
2745                         temp_reg = RREG32(MC_PMG_CMD_MRS1);
2746                         table->mc_reg_address[j].s1 = MC_PMG_CMD_MRS1 >> 2;
2747                         table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_MRS1_LP >> 2;
2748                         for (k = 0; k < table->num_entries; k++)
2749                                 table->mc_reg_table_entry[k].mc_data[j] =
2750                                         (temp_reg & 0xffff0000) |
2751                                         (table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff);
2752                         j++;
2753                         break;
2754                 default:
2755                         break;
2756                 }
2757         }
2758
2759         table->last = j;
2760
2761         return 0;
2762 }
2763
2764 static bool ni_check_s0_mc_reg_index(u16 in_reg, u16 *out_reg)
2765 {
2766         bool result = true;
2767
2768         switch (in_reg) {
2769         case  MC_SEQ_RAS_TIMING >> 2:
2770                 *out_reg = MC_SEQ_RAS_TIMING_LP >> 2;
2771                 break;
2772         case MC_SEQ_CAS_TIMING >> 2:
2773                 *out_reg = MC_SEQ_CAS_TIMING_LP >> 2;
2774                 break;
2775         case MC_SEQ_MISC_TIMING >> 2:
2776                 *out_reg = MC_SEQ_MISC_TIMING_LP >> 2;
2777                 break;
2778         case MC_SEQ_MISC_TIMING2 >> 2:
2779                 *out_reg = MC_SEQ_MISC_TIMING2_LP >> 2;
2780                 break;
2781         case MC_SEQ_RD_CTL_D0 >> 2:
2782                 *out_reg = MC_SEQ_RD_CTL_D0_LP >> 2;
2783                 break;
2784         case MC_SEQ_RD_CTL_D1 >> 2:
2785                 *out_reg = MC_SEQ_RD_CTL_D1_LP >> 2;
2786                 break;
2787         case MC_SEQ_WR_CTL_D0 >> 2:
2788                 *out_reg = MC_SEQ_WR_CTL_D0_LP >> 2;
2789                 break;
2790         case MC_SEQ_WR_CTL_D1 >> 2:
2791                 *out_reg = MC_SEQ_WR_CTL_D1_LP >> 2;
2792                 break;
2793         case MC_PMG_CMD_EMRS >> 2:
2794                 *out_reg = MC_SEQ_PMG_CMD_EMRS_LP >> 2;
2795                 break;
2796         case MC_PMG_CMD_MRS >> 2:
2797                 *out_reg = MC_SEQ_PMG_CMD_MRS_LP >> 2;
2798                 break;
2799         case MC_PMG_CMD_MRS1 >> 2:
2800                 *out_reg = MC_SEQ_PMG_CMD_MRS1_LP >> 2;
2801                 break;
2802         case MC_SEQ_PMG_TIMING >> 2:
2803                 *out_reg = MC_SEQ_PMG_TIMING_LP >> 2;
2804                 break;
2805         case MC_PMG_CMD_MRS2 >> 2:
2806                 *out_reg = MC_SEQ_PMG_CMD_MRS2_LP >> 2;
2807                 break;
2808         default:
2809                 result = false;
2810                 break;
2811         }
2812
2813         return result;
2814 }
2815
2816 static void ni_set_valid_flag(struct ni_mc_reg_table *table)
2817 {
2818         u8 i, j;
2819
2820         for (i = 0; i < table->last; i++) {
2821                 for (j = 1; j < table->num_entries; j++) {
2822                         if (table->mc_reg_table_entry[j-1].mc_data[i] != table->mc_reg_table_entry[j].mc_data[i]) {
2823                                 table->valid_flag |= 1 << i;
2824                                 break;
2825                         }
2826                 }
2827         }
2828 }
2829
2830 static void ni_set_s0_mc_reg_index(struct ni_mc_reg_table *table)
2831 {
2832         u32 i;
2833         u16 address;
2834
2835         for (i = 0; i < table->last; i++)
2836                 table->mc_reg_address[i].s0 =
2837                         ni_check_s0_mc_reg_index(table->mc_reg_address[i].s1, &address) ?
2838                         address : table->mc_reg_address[i].s1;
2839 }
2840
2841 static int ni_copy_vbios_mc_reg_table(struct atom_mc_reg_table *table,
2842                                       struct ni_mc_reg_table *ni_table)
2843 {
2844         u8 i, j;
2845
2846         if (table->last > SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2847                 return -EINVAL;
2848         if (table->num_entries > MAX_AC_TIMING_ENTRIES)
2849                 return -EINVAL;
2850
2851         for (i = 0; i < table->last; i++)
2852                 ni_table->mc_reg_address[i].s1 = table->mc_reg_address[i].s1;
2853         ni_table->last = table->last;
2854
2855         for (i = 0; i < table->num_entries; i++) {
2856                 ni_table->mc_reg_table_entry[i].mclk_max =
2857                         table->mc_reg_table_entry[i].mclk_max;
2858                 for (j = 0; j < table->last; j++)
2859                         ni_table->mc_reg_table_entry[i].mc_data[j] =
2860                                 table->mc_reg_table_entry[i].mc_data[j];
2861         }
2862         ni_table->num_entries = table->num_entries;
2863
2864         return 0;
2865 }
2866
2867 static int ni_initialize_mc_reg_table(struct radeon_device *rdev)
2868 {
2869         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2870         int ret;
2871         struct atom_mc_reg_table *table;
2872         struct ni_mc_reg_table *ni_table = &ni_pi->mc_reg_table;
2873         u8 module_index = rv770_get_memory_module_index(rdev);
2874
2875         table = kzalloc(sizeof(struct atom_mc_reg_table), GFP_KERNEL);
2876         if (!table)
2877                 return -ENOMEM;
2878
2879         WREG32(MC_SEQ_RAS_TIMING_LP, RREG32(MC_SEQ_RAS_TIMING));
2880         WREG32(MC_SEQ_CAS_TIMING_LP, RREG32(MC_SEQ_CAS_TIMING));
2881         WREG32(MC_SEQ_MISC_TIMING_LP, RREG32(MC_SEQ_MISC_TIMING));
2882         WREG32(MC_SEQ_MISC_TIMING2_LP, RREG32(MC_SEQ_MISC_TIMING2));
2883         WREG32(MC_SEQ_PMG_CMD_EMRS_LP, RREG32(MC_PMG_CMD_EMRS));
2884         WREG32(MC_SEQ_PMG_CMD_MRS_LP, RREG32(MC_PMG_CMD_MRS));
2885         WREG32(MC_SEQ_PMG_CMD_MRS1_LP, RREG32(MC_PMG_CMD_MRS1));
2886         WREG32(MC_SEQ_WR_CTL_D0_LP, RREG32(MC_SEQ_WR_CTL_D0));
2887         WREG32(MC_SEQ_WR_CTL_D1_LP, RREG32(MC_SEQ_WR_CTL_D1));
2888         WREG32(MC_SEQ_RD_CTL_D0_LP, RREG32(MC_SEQ_RD_CTL_D0));
2889         WREG32(MC_SEQ_RD_CTL_D1_LP, RREG32(MC_SEQ_RD_CTL_D1));
2890         WREG32(MC_SEQ_PMG_TIMING_LP, RREG32(MC_SEQ_PMG_TIMING));
2891         WREG32(MC_SEQ_PMG_CMD_MRS2_LP, RREG32(MC_PMG_CMD_MRS2));
2892
2893         ret = radeon_atom_init_mc_reg_table(rdev, module_index, table);
2894
2895         if (ret)
2896                 goto init_mc_done;
2897
2898         ret = ni_copy_vbios_mc_reg_table(table, ni_table);
2899
2900         if (ret)
2901                 goto init_mc_done;
2902
2903         ni_set_s0_mc_reg_index(ni_table);
2904
2905         ret = ni_set_mc_special_registers(rdev, ni_table);
2906
2907         if (ret)
2908                 goto init_mc_done;
2909
2910         ni_set_valid_flag(ni_table);
2911
2912 init_mc_done:
2913         kfree(table);
2914
2915         return ret;
2916 }
2917
2918 static void ni_populate_mc_reg_addresses(struct radeon_device *rdev,
2919                                          SMC_NIslands_MCRegisters *mc_reg_table)
2920 {
2921         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2922         u32 i, j;
2923
2924         for (i = 0, j = 0; j < ni_pi->mc_reg_table.last; j++) {
2925                 if (ni_pi->mc_reg_table.valid_flag & (1 << j)) {
2926                         if (i >= SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2927                                 break;
2928                         mc_reg_table->address[i].s0 =
2929                                 cpu_to_be16(ni_pi->mc_reg_table.mc_reg_address[j].s0);
2930                         mc_reg_table->address[i].s1 =
2931                                 cpu_to_be16(ni_pi->mc_reg_table.mc_reg_address[j].s1);
2932                         i++;
2933                 }
2934         }
2935         mc_reg_table->last = (u8)i;
2936 }
2937
2938
2939 static void ni_convert_mc_registers(struct ni_mc_reg_entry *entry,
2940                                     SMC_NIslands_MCRegisterSet *data,
2941                                     u32 num_entries, u32 valid_flag)
2942 {
2943         u32 i, j;
2944
2945         for (i = 0, j = 0; j < num_entries; j++) {
2946                 if (valid_flag & (1 << j)) {
2947                         data->value[i] = cpu_to_be32(entry->mc_data[j]);
2948                         i++;
2949                 }
2950         }
2951 }
2952
2953 static void ni_convert_mc_reg_table_entry_to_smc(struct radeon_device *rdev,
2954                                                  struct rv7xx_pl *pl,
2955                                                  SMC_NIslands_MCRegisterSet *mc_reg_table_data)
2956 {
2957         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2958         u32 i = 0;
2959
2960         for (i = 0; i < ni_pi->mc_reg_table.num_entries; i++) {
2961                 if (pl->mclk <= ni_pi->mc_reg_table.mc_reg_table_entry[i].mclk_max)
2962                         break;
2963         }
2964
2965         if ((i == ni_pi->mc_reg_table.num_entries) && (i > 0))
2966                 --i;
2967
2968         ni_convert_mc_registers(&ni_pi->mc_reg_table.mc_reg_table_entry[i],
2969                                 mc_reg_table_data,
2970                                 ni_pi->mc_reg_table.last,
2971                                 ni_pi->mc_reg_table.valid_flag);
2972 }
2973
2974 static void ni_convert_mc_reg_table_to_smc(struct radeon_device *rdev,
2975                                            struct radeon_ps *radeon_state,
2976                                            SMC_NIslands_MCRegisters *mc_reg_table)
2977 {
2978         struct ni_ps *state = ni_get_ps(radeon_state);
2979         int i;
2980
2981         for (i = 0; i < state->performance_level_count; i++) {
2982                 ni_convert_mc_reg_table_entry_to_smc(rdev,
2983                                                      &state->performance_levels[i],
2984                                                      &mc_reg_table->data[NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT + i]);
2985         }
2986 }
2987
2988 static int ni_populate_mc_reg_table(struct radeon_device *rdev,
2989                                     struct radeon_ps *radeon_boot_state)
2990 {
2991         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2992         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2993         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2994         struct ni_ps *boot_state = ni_get_ps(radeon_boot_state);
2995         SMC_NIslands_MCRegisters *mc_reg_table = &ni_pi->smc_mc_reg_table;
2996
2997         memset(mc_reg_table, 0, sizeof(SMC_NIslands_MCRegisters));
2998
2999         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_seq_index, 1);
3000
3001         ni_populate_mc_reg_addresses(rdev, mc_reg_table);
3002
3003         ni_convert_mc_reg_table_entry_to_smc(rdev, &boot_state->performance_levels[0],
3004                                              &mc_reg_table->data[0]);
3005
3006         ni_convert_mc_registers(&ni_pi->mc_reg_table.mc_reg_table_entry[0],
3007                                 &mc_reg_table->data[1],
3008                                 ni_pi->mc_reg_table.last,
3009                                 ni_pi->mc_reg_table.valid_flag);
3010
3011         ni_convert_mc_reg_table_to_smc(rdev, radeon_boot_state, mc_reg_table);
3012
3013         return rv770_copy_bytes_to_smc(rdev, eg_pi->mc_reg_table_start,
3014                                        (u8 *)mc_reg_table,
3015                                        sizeof(SMC_NIslands_MCRegisters),
3016                                        pi->sram_end);
3017 }
3018
3019 static int ni_upload_mc_reg_table(struct radeon_device *rdev,
3020                                   struct radeon_ps *radeon_new_state)
3021 {
3022         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3023         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3024         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3025         struct ni_ps *ni_new_state = ni_get_ps(radeon_new_state);
3026         SMC_NIslands_MCRegisters *mc_reg_table = &ni_pi->smc_mc_reg_table;
3027         u16 address;
3028
3029         memset(mc_reg_table, 0, sizeof(SMC_NIslands_MCRegisters));
3030
3031         ni_convert_mc_reg_table_to_smc(rdev, radeon_new_state, mc_reg_table);
3032
3033         address = eg_pi->mc_reg_table_start +
3034                 (u16)offsetof(SMC_NIslands_MCRegisters, data[NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT]);
3035
3036         return rv770_copy_bytes_to_smc(rdev, address,
3037                                        (u8 *)&mc_reg_table->data[NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT],
3038                                        sizeof(SMC_NIslands_MCRegisterSet) * ni_new_state->performance_level_count,
3039                                        pi->sram_end);
3040 }
3041
3042 static int ni_init_driver_calculated_leakage_table(struct radeon_device *rdev,
3043                                                    PP_NIslands_CACTABLES *cac_tables)
3044 {
3045         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3046         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3047         u32 leakage = 0;
3048         unsigned int i, j, table_size;
3049         s32 t;
3050         u32 smc_leakage, max_leakage = 0;
3051         u32 scaling_factor;
3052
3053         table_size = eg_pi->vddc_voltage_table.count;
3054
3055         if (SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES < table_size)
3056                 table_size = SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES;
3057
3058         scaling_factor = ni_get_smc_power_scaling_factor(rdev);
3059
3060         for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++) {
3061                 for (j = 0; j < table_size; j++) {
3062                         t = (1000 * ((i + 1) * 8));
3063
3064                         if (t < ni_pi->cac_data.leakage_minimum_temperature)
3065                                 t = ni_pi->cac_data.leakage_minimum_temperature;
3066
3067                         ni_calculate_leakage_for_v_and_t(rdev,
3068                                                          &ni_pi->cac_data.leakage_coefficients,
3069                                                          eg_pi->vddc_voltage_table.entries[j].value,
3070                                                          t,
3071                                                          ni_pi->cac_data.i_leakage,
3072                                                          &leakage);
3073
3074                         smc_leakage = ni_scale_power_for_smc(leakage, scaling_factor) / 1000;
3075                         if (smc_leakage > max_leakage)
3076                                 max_leakage = smc_leakage;
3077
3078                         cac_tables->cac_lkge_lut[i][j] = cpu_to_be32(smc_leakage);
3079                 }
3080         }
3081
3082         for (j = table_size; j < SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES; j++) {
3083                 for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++)
3084                         cac_tables->cac_lkge_lut[i][j] = cpu_to_be32(max_leakage);
3085         }
3086         return 0;
3087 }
3088
3089 static int ni_init_simplified_leakage_table(struct radeon_device *rdev,
3090                                             PP_NIslands_CACTABLES *cac_tables)
3091 {
3092         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3093         struct radeon_cac_leakage_table *leakage_table =
3094                 &rdev->pm.dpm.dyn_state.cac_leakage_table;
3095         u32 i, j, table_size;
3096         u32 smc_leakage, max_leakage = 0;
3097         u32 scaling_factor;
3098
3099         if (!leakage_table)
3100                 return -EINVAL;
3101
3102         table_size = leakage_table->count;
3103
3104         if (eg_pi->vddc_voltage_table.count != table_size)
3105                 table_size = (eg_pi->vddc_voltage_table.count < leakage_table->count) ?
3106                         eg_pi->vddc_voltage_table.count : leakage_table->count;
3107
3108         if (SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES < table_size)
3109                 table_size = SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES;
3110
3111         if (table_size == 0)
3112                 return -EINVAL;
3113
3114         scaling_factor = ni_get_smc_power_scaling_factor(rdev);
3115
3116         for (j = 0; j < table_size; j++) {
3117                 smc_leakage = leakage_table->entries[j].leakage;
3118
3119                 if (smc_leakage > max_leakage)
3120                         max_leakage = smc_leakage;
3121
3122                 for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++)
3123                         cac_tables->cac_lkge_lut[i][j] =
3124                                 cpu_to_be32(ni_scale_power_for_smc(smc_leakage, scaling_factor));
3125         }
3126
3127         for (j = table_size; j < SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES; j++) {
3128                 for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++)
3129                         cac_tables->cac_lkge_lut[i][j] =
3130                                 cpu_to_be32(ni_scale_power_for_smc(max_leakage, scaling_factor));
3131         }
3132         return 0;
3133 }
3134
3135 static int ni_initialize_smc_cac_tables(struct radeon_device *rdev)
3136 {
3137         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3138         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3139         PP_NIslands_CACTABLES *cac_tables = NULL;
3140         int i, ret;
3141         u32 reg;
3142
3143         if (ni_pi->enable_cac == false)
3144                 return 0;
3145
3146         cac_tables = kzalloc(sizeof(PP_NIslands_CACTABLES), GFP_KERNEL);
3147         if (!cac_tables)
3148                 return -ENOMEM;
3149
3150         reg = RREG32(CG_CAC_CTRL) & ~(TID_CNT_MASK | TID_UNIT_MASK);
3151         reg |= (TID_CNT(ni_pi->cac_weights->tid_cnt) |
3152                 TID_UNIT(ni_pi->cac_weights->tid_unit));
3153         WREG32(CG_CAC_CTRL, reg);
3154
3155         for (i = 0; i < NISLANDS_DCCAC_MAX_LEVELS; i++)
3156                 ni_pi->dc_cac_table[i] = ni_pi->cac_weights->dc_cac[i];
3157
3158         for (i = 0; i < SMC_NISLANDS_BIF_LUT_NUM_OF_ENTRIES; i++)
3159                 cac_tables->cac_bif_lut[i] = ni_pi->cac_weights->pcie_cac[i];
3160
3161         ni_pi->cac_data.i_leakage = rdev->pm.dpm.cac_leakage;
3162         ni_pi->cac_data.pwr_const = 0;
3163         ni_pi->cac_data.dc_cac_value = ni_pi->dc_cac_table[NISLANDS_DCCAC_LEVEL_0];
3164         ni_pi->cac_data.bif_cac_value = 0;
3165         ni_pi->cac_data.mc_wr_weight = ni_pi->cac_weights->mc_write_weight;
3166         ni_pi->cac_data.mc_rd_weight = ni_pi->cac_weights->mc_read_weight;
3167         ni_pi->cac_data.allow_ovrflw = 0;
3168         ni_pi->cac_data.l2num_win_tdp = ni_pi->lta_window_size;
3169         ni_pi->cac_data.num_win_tdp = 0;
3170         ni_pi->cac_data.lts_truncate_n = ni_pi->lts_truncate;
3171
3172         if (ni_pi->driver_calculate_cac_leakage)
3173                 ret = ni_init_driver_calculated_leakage_table(rdev, cac_tables);
3174         else
3175                 ret = ni_init_simplified_leakage_table(rdev, cac_tables);
3176
3177         if (ret)
3178                 goto done_free;
3179
3180         cac_tables->pwr_const      = cpu_to_be32(ni_pi->cac_data.pwr_const);
3181         cac_tables->dc_cacValue    = cpu_to_be32(ni_pi->cac_data.dc_cac_value);
3182         cac_tables->bif_cacValue   = cpu_to_be32(ni_pi->cac_data.bif_cac_value);
3183         cac_tables->AllowOvrflw    = ni_pi->cac_data.allow_ovrflw;
3184         cac_tables->MCWrWeight     = ni_pi->cac_data.mc_wr_weight;
3185         cac_tables->MCRdWeight     = ni_pi->cac_data.mc_rd_weight;
3186         cac_tables->numWin_TDP     = ni_pi->cac_data.num_win_tdp;
3187         cac_tables->l2numWin_TDP   = ni_pi->cac_data.l2num_win_tdp;
3188         cac_tables->lts_truncate_n = ni_pi->cac_data.lts_truncate_n;
3189
3190         ret = rv770_copy_bytes_to_smc(rdev, ni_pi->cac_table_start, (u8 *)cac_tables,
3191                                       sizeof(PP_NIslands_CACTABLES), pi->sram_end);
3192
3193 done_free:
3194         if (ret) {
3195                 ni_pi->enable_cac = false;
3196                 ni_pi->enable_power_containment = false;
3197         }
3198
3199         kfree(cac_tables);
3200
3201         return 0;
3202 }
3203
3204 static int ni_initialize_hardware_cac_manager(struct radeon_device *rdev)
3205 {
3206         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3207         u32 reg;
3208
3209         if (!ni_pi->enable_cac ||
3210             !ni_pi->cac_configuration_required)
3211                 return 0;
3212
3213         if (ni_pi->cac_weights == NULL)
3214                 return -EINVAL;
3215
3216         reg = RREG32_CG(CG_CAC_REGION_1_WEIGHT_0) & ~(WEIGHT_TCP_SIG0_MASK |
3217                                                       WEIGHT_TCP_SIG1_MASK |
3218                                                       WEIGHT_TA_SIG_MASK);
3219         reg |= (WEIGHT_TCP_SIG0(ni_pi->cac_weights->weight_tcp_sig0) |
3220                 WEIGHT_TCP_SIG1(ni_pi->cac_weights->weight_tcp_sig1) |
3221                 WEIGHT_TA_SIG(ni_pi->cac_weights->weight_ta_sig));
3222         WREG32_CG(CG_CAC_REGION_1_WEIGHT_0, reg);
3223
3224         reg = RREG32_CG(CG_CAC_REGION_1_WEIGHT_1) & ~(WEIGHT_TCC_EN0_MASK |
3225                                                       WEIGHT_TCC_EN1_MASK |
3226                                                       WEIGHT_TCC_EN2_MASK);
3227         reg |= (WEIGHT_TCC_EN0(ni_pi->cac_weights->weight_tcc_en0) |
3228                 WEIGHT_TCC_EN1(ni_pi->cac_weights->weight_tcc_en1) |
3229                 WEIGHT_TCC_EN2(ni_pi->cac_weights->weight_tcc_en2));
3230         WREG32_CG(CG_CAC_REGION_1_WEIGHT_1, reg);
3231
3232         reg = RREG32_CG(CG_CAC_REGION_2_WEIGHT_0) & ~(WEIGHT_CB_EN0_MASK |
3233                                                       WEIGHT_CB_EN1_MASK |
3234                                                       WEIGHT_CB_EN2_MASK |
3235                                                       WEIGHT_CB_EN3_MASK);
3236         reg |= (WEIGHT_CB_EN0(ni_pi->cac_weights->weight_cb_en0) |
3237                 WEIGHT_CB_EN1(ni_pi->cac_weights->weight_cb_en1) |
3238                 WEIGHT_CB_EN2(ni_pi->cac_weights->weight_cb_en2) |
3239                 WEIGHT_CB_EN3(ni_pi->cac_weights->weight_cb_en3));
3240         WREG32_CG(CG_CAC_REGION_2_WEIGHT_0, reg);
3241
3242         reg = RREG32_CG(CG_CAC_REGION_2_WEIGHT_1) & ~(WEIGHT_DB_SIG0_MASK |
3243                                                       WEIGHT_DB_SIG1_MASK |
3244                                                       WEIGHT_DB_SIG2_MASK |
3245                                                       WEIGHT_DB_SIG3_MASK);
3246         reg |= (WEIGHT_DB_SIG0(ni_pi->cac_weights->weight_db_sig0) |
3247                 WEIGHT_DB_SIG1(ni_pi->cac_weights->weight_db_sig1) |
3248                 WEIGHT_DB_SIG2(ni_pi->cac_weights->weight_db_sig2) |
3249                 WEIGHT_DB_SIG3(ni_pi->cac_weights->weight_db_sig3));
3250         WREG32_CG(CG_CAC_REGION_2_WEIGHT_1, reg);
3251
3252         reg = RREG32_CG(CG_CAC_REGION_2_WEIGHT_2) & ~(WEIGHT_SXM_SIG0_MASK |
3253                                                       WEIGHT_SXM_SIG1_MASK |
3254                                                       WEIGHT_SXM_SIG2_MASK |
3255                                                       WEIGHT_SXS_SIG0_MASK |
3256                                                       WEIGHT_SXS_SIG1_MASK);
3257         reg |= (WEIGHT_SXM_SIG0(ni_pi->cac_weights->weight_sxm_sig0) |
3258                 WEIGHT_SXM_SIG1(ni_pi->cac_weights->weight_sxm_sig1) |
3259                 WEIGHT_SXM_SIG2(ni_pi->cac_weights->weight_sxm_sig2) |
3260                 WEIGHT_SXS_SIG0(ni_pi->cac_weights->weight_sxs_sig0) |
3261                 WEIGHT_SXS_SIG1(ni_pi->cac_weights->weight_sxs_sig1));
3262         WREG32_CG(CG_CAC_REGION_2_WEIGHT_2, reg);
3263
3264         reg = RREG32_CG(CG_CAC_REGION_3_WEIGHT_0) & ~(WEIGHT_XBR_0_MASK |
3265                                                       WEIGHT_XBR_1_MASK |
3266                                                       WEIGHT_XBR_2_MASK |
3267                                                       WEIGHT_SPI_SIG0_MASK);
3268         reg |= (WEIGHT_XBR_0(ni_pi->cac_weights->weight_xbr_0) |
3269                 WEIGHT_XBR_1(ni_pi->cac_weights->weight_xbr_1) |
3270                 WEIGHT_XBR_2(ni_pi->cac_weights->weight_xbr_2) |
3271                 WEIGHT_SPI_SIG0(ni_pi->cac_weights->weight_spi_sig0));
3272         WREG32_CG(CG_CAC_REGION_3_WEIGHT_0, reg);
3273
3274         reg = RREG32_CG(CG_CAC_REGION_3_WEIGHT_1) & ~(WEIGHT_SPI_SIG1_MASK |
3275                                                       WEIGHT_SPI_SIG2_MASK |
3276                                                       WEIGHT_SPI_SIG3_MASK |
3277                                                       WEIGHT_SPI_SIG4_MASK |
3278                                                       WEIGHT_SPI_SIG5_MASK);
3279         reg |= (WEIGHT_SPI_SIG1(ni_pi->cac_weights->weight_spi_sig1) |
3280                 WEIGHT_SPI_SIG2(ni_pi->cac_weights->weight_spi_sig2) |
3281                 WEIGHT_SPI_SIG3(ni_pi->cac_weights->weight_spi_sig3) |
3282                 WEIGHT_SPI_SIG4(ni_pi->cac_weights->weight_spi_sig4) |
3283                 WEIGHT_SPI_SIG5(ni_pi->cac_weights->weight_spi_sig5));
3284         WREG32_CG(CG_CAC_REGION_3_WEIGHT_1, reg);
3285
3286         reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_0) & ~(WEIGHT_LDS_SIG0_MASK |
3287                                                       WEIGHT_LDS_SIG1_MASK |
3288                                                       WEIGHT_SC_MASK);
3289         reg |= (WEIGHT_LDS_SIG0(ni_pi->cac_weights->weight_lds_sig0) |
3290                 WEIGHT_LDS_SIG1(ni_pi->cac_weights->weight_lds_sig1) |
3291                 WEIGHT_SC(ni_pi->cac_weights->weight_sc));
3292         WREG32_CG(CG_CAC_REGION_4_WEIGHT_0, reg);
3293
3294         reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_1) & ~(WEIGHT_BIF_MASK |
3295                                                       WEIGHT_CP_MASK |
3296                                                       WEIGHT_PA_SIG0_MASK |
3297                                                       WEIGHT_PA_SIG1_MASK |
3298                                                       WEIGHT_VGT_SIG0_MASK);
3299         reg |= (WEIGHT_BIF(ni_pi->cac_weights->weight_bif) |
3300                 WEIGHT_CP(ni_pi->cac_weights->weight_cp) |
3301                 WEIGHT_PA_SIG0(ni_pi->cac_weights->weight_pa_sig0) |
3302                 WEIGHT_PA_SIG1(ni_pi->cac_weights->weight_pa_sig1) |
3303                 WEIGHT_VGT_SIG0(ni_pi->cac_weights->weight_vgt_sig0));
3304         WREG32_CG(CG_CAC_REGION_4_WEIGHT_1, reg);
3305
3306         reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_2) & ~(WEIGHT_VGT_SIG1_MASK |
3307                                                       WEIGHT_VGT_SIG2_MASK |
3308                                                       WEIGHT_DC_SIG0_MASK |
3309                                                       WEIGHT_DC_SIG1_MASK |
3310                                                       WEIGHT_DC_SIG2_MASK);
3311         reg |= (WEIGHT_VGT_SIG1(ni_pi->cac_weights->weight_vgt_sig1) |
3312                 WEIGHT_VGT_SIG2(ni_pi->cac_weights->weight_vgt_sig2) |
3313                 WEIGHT_DC_SIG0(ni_pi->cac_weights->weight_dc_sig0) |
3314                 WEIGHT_DC_SIG1(ni_pi->cac_weights->weight_dc_sig1) |
3315                 WEIGHT_DC_SIG2(ni_pi->cac_weights->weight_dc_sig2));
3316         WREG32_CG(CG_CAC_REGION_4_WEIGHT_2, reg);
3317
3318         reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_3) & ~(WEIGHT_DC_SIG3_MASK |
3319                                                       WEIGHT_UVD_SIG0_MASK |
3320                                                       WEIGHT_UVD_SIG1_MASK |
3321                                                       WEIGHT_SPARE0_MASK |
3322                                                       WEIGHT_SPARE1_MASK);
3323         reg |= (WEIGHT_DC_SIG3(ni_pi->cac_weights->weight_dc_sig3) |
3324                 WEIGHT_UVD_SIG0(ni_pi->cac_weights->weight_uvd_sig0) |
3325                 WEIGHT_UVD_SIG1(ni_pi->cac_weights->weight_uvd_sig1) |
3326                 WEIGHT_SPARE0(ni_pi->cac_weights->weight_spare0) |
3327                 WEIGHT_SPARE1(ni_pi->cac_weights->weight_spare1));
3328         WREG32_CG(CG_CAC_REGION_4_WEIGHT_3, reg);
3329
3330         reg = RREG32_CG(CG_CAC_REGION_5_WEIGHT_0) & ~(WEIGHT_SQ_VSP_MASK |
3331                                                       WEIGHT_SQ_VSP0_MASK);
3332         reg |= (WEIGHT_SQ_VSP(ni_pi->cac_weights->weight_sq_vsp) |
3333                 WEIGHT_SQ_VSP0(ni_pi->cac_weights->weight_sq_vsp0));
3334         WREG32_CG(CG_CAC_REGION_5_WEIGHT_0, reg);
3335
3336         reg = RREG32_CG(CG_CAC_REGION_5_WEIGHT_1) & ~(WEIGHT_SQ_GPR_MASK);
3337         reg |= WEIGHT_SQ_GPR(ni_pi->cac_weights->weight_sq_gpr);
3338         WREG32_CG(CG_CAC_REGION_5_WEIGHT_1, reg);
3339
3340         reg = RREG32_CG(CG_CAC_REGION_4_OVERRIDE_4) & ~(OVR_MODE_SPARE_0_MASK |
3341                                                         OVR_VAL_SPARE_0_MASK |
3342                                                         OVR_MODE_SPARE_1_MASK |
3343                                                         OVR_VAL_SPARE_1_MASK);
3344         reg |= (OVR_MODE_SPARE_0(ni_pi->cac_weights->ovr_mode_spare_0) |
3345                 OVR_VAL_SPARE_0(ni_pi->cac_weights->ovr_val_spare_0) |
3346                 OVR_MODE_SPARE_1(ni_pi->cac_weights->ovr_mode_spare_1) |
3347                 OVR_VAL_SPARE_1(ni_pi->cac_weights->ovr_val_spare_1));
3348         WREG32_CG(CG_CAC_REGION_4_OVERRIDE_4, reg);
3349
3350         reg = RREG32(SQ_CAC_THRESHOLD) & ~(VSP_MASK |
3351                                            VSP0_MASK |
3352                                            GPR_MASK);
3353         reg |= (VSP(ni_pi->cac_weights->vsp) |
3354                 VSP0(ni_pi->cac_weights->vsp0) |
3355                 GPR(ni_pi->cac_weights->gpr));
3356         WREG32(SQ_CAC_THRESHOLD, reg);
3357
3358         reg = (MCDW_WR_ENABLE |
3359                MCDX_WR_ENABLE |
3360                MCDY_WR_ENABLE |
3361                MCDZ_WR_ENABLE |
3362                INDEX(0x09D4));
3363         WREG32(MC_CG_CONFIG, reg);
3364
3365         reg = (READ_WEIGHT(ni_pi->cac_weights->mc_read_weight) |
3366                WRITE_WEIGHT(ni_pi->cac_weights->mc_write_weight) |
3367                ALLOW_OVERFLOW);
3368         WREG32(MC_CG_DATAPORT, reg);
3369
3370         return 0;
3371 }
3372
3373 static int ni_enable_smc_cac(struct radeon_device *rdev,
3374                              struct radeon_ps *radeon_new_state,
3375                              bool enable)
3376 {
3377         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3378         int ret = 0;
3379         PPSMC_Result smc_result;
3380
3381         if (ni_pi->enable_cac) {
3382                 if (enable) {
3383                         if (!r600_is_uvd_state(radeon_new_state->class, radeon_new_state->class2)) {
3384                                 smc_result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_CollectCAC_PowerCorreln);
3385
3386                                 if (ni_pi->support_cac_long_term_average) {
3387                                         smc_result = rv770_send_msg_to_smc(rdev, PPSMC_CACLongTermAvgEnable);
3388                                         if (PPSMC_Result_OK != smc_result)
3389                                                 ni_pi->support_cac_long_term_average = false;
3390                                 }
3391
3392                                 smc_result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_EnableCac);
3393                                 if (PPSMC_Result_OK != smc_result)
3394                                         ret = -EINVAL;
3395
3396                                 ni_pi->cac_enabled = (PPSMC_Result_OK == smc_result) ? true : false;
3397                         }
3398                 } else if (ni_pi->cac_enabled) {
3399                         smc_result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_DisableCac);
3400
3401                         ni_pi->cac_enabled = false;
3402
3403                         if (ni_pi->support_cac_long_term_average) {
3404                                 smc_result = rv770_send_msg_to_smc(rdev, PPSMC_CACLongTermAvgDisable);
3405                                 if (PPSMC_Result_OK != smc_result)
3406                                         ni_pi->support_cac_long_term_average = false;
3407                         }
3408                 }
3409         }
3410
3411         return ret;
3412 }
3413
3414 static int ni_pcie_performance_request(struct radeon_device *rdev,
3415                                        u8 perf_req, bool advertise)
3416 {
3417 #if defined(CONFIG_ACPI)
3418         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3419
3420         if ((perf_req == PCIE_PERF_REQ_PECI_GEN1) ||
3421             (perf_req == PCIE_PERF_REQ_PECI_GEN2)) {
3422                 if (eg_pi->pcie_performance_request_registered == false)
3423                         radeon_acpi_pcie_notify_device_ready(rdev);
3424                 eg_pi->pcie_performance_request_registered = true;
3425                 return radeon_acpi_pcie_performance_request(rdev, perf_req, advertise);
3426         } else if ((perf_req == PCIE_PERF_REQ_REMOVE_REGISTRY) &&
3427                     eg_pi->pcie_performance_request_registered) {
3428                 eg_pi->pcie_performance_request_registered = false;
3429                 return radeon_acpi_pcie_performance_request(rdev, perf_req, advertise);
3430         }
3431 #endif
3432         return 0;
3433 }
3434
3435 static int ni_advertise_gen2_capability(struct radeon_device *rdev)
3436 {
3437         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3438         u32 tmp;
3439
3440         tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
3441
3442         if ((tmp & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
3443             (tmp & LC_OTHER_SIDE_SUPPORTS_GEN2))
3444                 pi->pcie_gen2 = true;
3445         else
3446                 pi->pcie_gen2 = false;
3447
3448         if (!pi->pcie_gen2)
3449                 ni_pcie_performance_request(rdev, PCIE_PERF_REQ_PECI_GEN2, true);
3450
3451         return 0;
3452 }
3453
3454 static void ni_enable_bif_dynamic_pcie_gen2(struct radeon_device *rdev,
3455                                             bool enable)
3456 {
3457         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3458         u32 tmp, bif;
3459
3460         tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
3461
3462         if ((tmp & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
3463             (tmp & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
3464                 if (enable) {
3465                         if (!pi->boot_in_gen2) {
3466                                 bif = RREG32(CG_BIF_REQ_AND_RSP) & ~CG_CLIENT_REQ_MASK;
3467                                 bif |= CG_CLIENT_REQ(0xd);
3468                                 WREG32(CG_BIF_REQ_AND_RSP, bif);
3469                         }
3470                         tmp &= ~LC_HW_VOLTAGE_IF_CONTROL_MASK;
3471                         tmp |= LC_HW_VOLTAGE_IF_CONTROL(1);
3472                         tmp |= LC_GEN2_EN_STRAP;
3473
3474                         tmp |= LC_CLR_FAILED_SPD_CHANGE_CNT;
3475                         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
3476                         udelay(10);
3477                         tmp &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
3478                         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
3479                 } else {
3480                         if (!pi->boot_in_gen2) {
3481                                 bif = RREG32(CG_BIF_REQ_AND_RSP) & ~CG_CLIENT_REQ_MASK;
3482                                 bif |= CG_CLIENT_REQ(0xd);
3483                                 WREG32(CG_BIF_REQ_AND_RSP, bif);
3484
3485                                 tmp &= ~LC_HW_VOLTAGE_IF_CONTROL_MASK;
3486                                 tmp &= ~LC_GEN2_EN_STRAP;
3487                         }
3488                         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
3489                 }
3490         }
3491 }
3492
3493 static void ni_enable_dynamic_pcie_gen2(struct radeon_device *rdev,
3494                                         bool enable)
3495 {
3496         ni_enable_bif_dynamic_pcie_gen2(rdev, enable);
3497
3498         if (enable)
3499                 WREG32_P(GENERAL_PWRMGT, ENABLE_GEN2PCIE, ~ENABLE_GEN2PCIE);
3500         else
3501                 WREG32_P(GENERAL_PWRMGT, 0, ~ENABLE_GEN2PCIE);
3502 }
3503
3504 void ni_set_uvd_clock_before_set_eng_clock(struct radeon_device *rdev,
3505                                            struct radeon_ps *new_ps,
3506                                            struct radeon_ps *old_ps)
3507 {
3508         struct ni_ps *new_state = ni_get_ps(new_ps);
3509         struct ni_ps *current_state = ni_get_ps(old_ps);
3510
3511         if ((new_ps->vclk == old_ps->vclk) &&
3512             (new_ps->dclk == old_ps->dclk))
3513                 return;
3514
3515         if (new_state->performance_levels[new_state->performance_level_count - 1].sclk >=
3516             current_state->performance_levels[current_state->performance_level_count - 1].sclk)
3517                 return;
3518
3519         radeon_set_uvd_clocks(rdev, new_ps->vclk, new_ps->dclk);
3520 }
3521
3522 void ni_set_uvd_clock_after_set_eng_clock(struct radeon_device *rdev,
3523                                           struct radeon_ps *new_ps,
3524                                           struct radeon_ps *old_ps)
3525 {
3526         struct ni_ps *new_state = ni_get_ps(new_ps);
3527         struct ni_ps *current_state = ni_get_ps(old_ps);
3528
3529         if ((new_ps->vclk == old_ps->vclk) &&
3530             (new_ps->dclk == old_ps->dclk))
3531                 return;
3532
3533         if (new_state->performance_levels[new_state->performance_level_count - 1].sclk <
3534             current_state->performance_levels[current_state->performance_level_count - 1].sclk)
3535                 return;
3536
3537         radeon_set_uvd_clocks(rdev, new_ps->vclk, new_ps->dclk);
3538 }
3539
3540 void ni_dpm_setup_asic(struct radeon_device *rdev)
3541 {
3542         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3543         int r;
3544
3545         r = ni_mc_load_microcode(rdev);
3546         if (r)
3547                 DRM_ERROR("Failed to load MC firmware!\n");
3548         ni_read_clock_registers(rdev);
3549         btc_read_arb_registers(rdev);
3550         rv770_get_memory_type(rdev);
3551         if (eg_pi->pcie_performance_request)
3552                 ni_advertise_gen2_capability(rdev);
3553         rv770_get_pcie_gen2_status(rdev);
3554         rv770_enable_acpi_pm(rdev);
3555 }
3556
3557 void ni_update_current_ps(struct radeon_device *rdev,
3558                           struct radeon_ps *rps)
3559 {
3560         struct ni_ps *new_ps = ni_get_ps(rps);
3561         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3562         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3563
3564         eg_pi->current_rps = *rps;
3565         ni_pi->current_ps = *new_ps;
3566         eg_pi->current_rps.ps_priv = &ni_pi->current_ps;
3567 }
3568
3569 void ni_update_requested_ps(struct radeon_device *rdev,
3570                             struct radeon_ps *rps)
3571 {
3572         struct ni_ps *new_ps = ni_get_ps(rps);
3573         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3574         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3575
3576         eg_pi->requested_rps = *rps;
3577         ni_pi->requested_ps = *new_ps;
3578         eg_pi->requested_rps.ps_priv = &ni_pi->requested_ps;
3579 }
3580
3581 int ni_dpm_enable(struct radeon_device *rdev)
3582 {
3583         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3584         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3585         struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps;
3586         int ret;
3587
3588         if (pi->gfx_clock_gating)
3589                 ni_cg_clockgating_default(rdev);
3590         if (btc_dpm_enabled(rdev))
3591                 return -EINVAL;
3592         if (pi->mg_clock_gating)
3593                 ni_mg_clockgating_default(rdev);
3594         if (eg_pi->ls_clock_gating)
3595                 ni_ls_clockgating_default(rdev);
3596         if (pi->voltage_control) {
3597                 rv770_enable_voltage_control(rdev, true);
3598                 ret = cypress_construct_voltage_tables(rdev);
3599                 if (ret) {
3600                         DRM_ERROR("cypress_construct_voltage_tables failed\n");
3601                         return ret;
3602                 }
3603         }
3604         if (eg_pi->dynamic_ac_timing) {
3605                 ret = ni_initialize_mc_reg_table(rdev);
3606                 if (ret)
3607                         eg_pi->dynamic_ac_timing = false;
3608         }
3609         if (pi->dynamic_ss)
3610                 cypress_enable_spread_spectrum(rdev, true);
3611         if (pi->thermal_protection)
3612                 rv770_enable_thermal_protection(rdev, true);
3613         rv770_setup_bsp(rdev);
3614         rv770_program_git(rdev);
3615         rv770_program_tp(rdev);
3616         rv770_program_tpp(rdev);
3617         rv770_program_sstp(rdev);
3618         cypress_enable_display_gap(rdev);
3619         rv770_program_vc(rdev);
3620         if (pi->dynamic_pcie_gen2)
3621                 ni_enable_dynamic_pcie_gen2(rdev, true);
3622         ret = rv770_upload_firmware(rdev);
3623         if (ret) {
3624                 DRM_ERROR("rv770_upload_firmware failed\n");
3625                 return ret;
3626         }
3627         ret = ni_process_firmware_header(rdev);
3628         if (ret) {
3629                 DRM_ERROR("ni_process_firmware_header failed\n");
3630                 return ret;
3631         }
3632         ret = ni_initial_switch_from_arb_f0_to_f1(rdev);
3633         if (ret) {
3634                 DRM_ERROR("ni_initial_switch_from_arb_f0_to_f1 failed\n");
3635                 return ret;
3636         }
3637         ret = ni_init_smc_table(rdev);
3638         if (ret) {
3639                 DRM_ERROR("ni_init_smc_table failed\n");
3640                 return ret;
3641         }
3642         ret = ni_init_smc_spll_table(rdev);
3643         if (ret) {
3644                 DRM_ERROR("ni_init_smc_spll_table failed\n");
3645                 return ret;
3646         }
3647         ret = ni_init_arb_table_index(rdev);
3648         if (ret) {
3649                 DRM_ERROR("ni_init_arb_table_index failed\n");
3650                 return ret;
3651         }
3652         if (eg_pi->dynamic_ac_timing) {
3653                 ret = ni_populate_mc_reg_table(rdev, boot_ps);
3654                 if (ret) {
3655                         DRM_ERROR("ni_populate_mc_reg_table failed\n");
3656                         return ret;
3657                 }
3658         }
3659         ret = ni_initialize_smc_cac_tables(rdev);
3660         if (ret) {
3661                 DRM_ERROR("ni_initialize_smc_cac_tables failed\n");
3662                 return ret;
3663         }
3664         ret = ni_initialize_hardware_cac_manager(rdev);
3665         if (ret) {
3666                 DRM_ERROR("ni_initialize_hardware_cac_manager failed\n");
3667                 return ret;
3668         }
3669         ret = ni_populate_smc_tdp_limits(rdev, boot_ps);
3670         if (ret) {
3671                 DRM_ERROR("ni_populate_smc_tdp_limits failed\n");
3672                 return ret;
3673         }
3674         ni_program_response_times(rdev);
3675         r7xx_start_smc(rdev);
3676         ret = cypress_notify_smc_display_change(rdev, false);
3677         if (ret) {
3678                 DRM_ERROR("cypress_notify_smc_display_change failed\n");
3679                 return ret;
3680         }
3681         cypress_enable_sclk_control(rdev, true);
3682         if (eg_pi->memory_transition)
3683                 cypress_enable_mclk_control(rdev, true);
3684         cypress_start_dpm(rdev);
3685         if (pi->gfx_clock_gating)
3686                 ni_gfx_clockgating_enable(rdev, true);
3687         if (pi->mg_clock_gating)
3688                 ni_mg_clockgating_enable(rdev, true);
3689         if (eg_pi->ls_clock_gating)
3690                 ni_ls_clockgating_enable(rdev, true);
3691
3692         rv770_enable_auto_throttle_source(rdev, RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL, true);
3693
3694         ni_update_current_ps(rdev, boot_ps);
3695
3696         return 0;
3697 }
3698
3699 void ni_dpm_disable(struct radeon_device *rdev)
3700 {
3701         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3702         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3703         struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps;
3704
3705         if (!btc_dpm_enabled(rdev))
3706                 return;
3707         rv770_clear_vc(rdev);
3708         if (pi->thermal_protection)
3709                 rv770_enable_thermal_protection(rdev, false);
3710         ni_enable_power_containment(rdev, boot_ps, false);
3711         ni_enable_smc_cac(rdev, boot_ps, false);
3712         cypress_enable_spread_spectrum(rdev, false);
3713         rv770_enable_auto_throttle_source(rdev, RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL, false);
3714         if (pi->dynamic_pcie_gen2)
3715                 ni_enable_dynamic_pcie_gen2(rdev, false);
3716
3717         if (rdev->irq.installed &&
3718             r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) {
3719                 rdev->irq.dpm_thermal = false;
3720                 radeon_irq_set(rdev);
3721         }
3722
3723         if (pi->gfx_clock_gating)
3724                 ni_gfx_clockgating_enable(rdev, false);
3725         if (pi->mg_clock_gating)
3726                 ni_mg_clockgating_enable(rdev, false);
3727         if (eg_pi->ls_clock_gating)
3728                 ni_ls_clockgating_enable(rdev, false);
3729         ni_stop_dpm(rdev);
3730         btc_reset_to_default(rdev);
3731         ni_stop_smc(rdev);
3732         ni_force_switch_to_arb_f0(rdev);
3733
3734         ni_update_current_ps(rdev, boot_ps);
3735 }
3736
3737 static int ni_power_control_set_level(struct radeon_device *rdev)
3738 {
3739         struct radeon_ps *new_ps = rdev->pm.dpm.requested_ps;
3740         int ret;
3741
3742         ret = ni_restrict_performance_levels_before_switch(rdev);
3743         if (ret)
3744                 return ret;
3745         ret = rv770_halt_smc(rdev);
3746         if (ret)
3747                 return ret;
3748         ret = ni_populate_smc_tdp_limits(rdev, new_ps);
3749         if (ret)
3750                 return ret;
3751         ret = rv770_resume_smc(rdev);
3752         if (ret)
3753                 return ret;
3754         ret = rv770_set_sw_state(rdev);
3755         if (ret)
3756                 return ret;
3757
3758         return 0;
3759 }
3760
3761 int ni_dpm_pre_set_power_state(struct radeon_device *rdev)
3762 {
3763         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3764         struct radeon_ps requested_ps = *rdev->pm.dpm.requested_ps;
3765         struct radeon_ps *new_ps = &requested_ps;
3766
3767         ni_update_requested_ps(rdev, new_ps);
3768
3769         ni_apply_state_adjust_rules(rdev, &eg_pi->requested_rps);
3770
3771         return 0;
3772 }
3773
3774 int ni_dpm_set_power_state(struct radeon_device *rdev)
3775 {
3776         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3777         struct radeon_ps *new_ps = &eg_pi->requested_rps;
3778         struct radeon_ps *old_ps = &eg_pi->current_rps;
3779         int ret;
3780
3781         ret = ni_restrict_performance_levels_before_switch(rdev);
3782         if (ret) {
3783                 DRM_ERROR("ni_restrict_performance_levels_before_switch failed\n");
3784                 return ret;
3785         }
3786         ni_set_uvd_clock_before_set_eng_clock(rdev, new_ps, old_ps);
3787         ret = ni_enable_power_containment(rdev, new_ps, false);
3788         if (ret) {
3789                 DRM_ERROR("ni_enable_power_containment failed\n");
3790                 return ret;
3791         }
3792         ret = ni_enable_smc_cac(rdev, new_ps, false);
3793         if (ret) {
3794                 DRM_ERROR("ni_enable_smc_cac failed\n");
3795                 return ret;
3796         }
3797         ret = rv770_halt_smc(rdev);
3798         if (ret) {
3799                 DRM_ERROR("rv770_halt_smc failed\n");
3800                 return ret;
3801         }
3802         if (eg_pi->smu_uvd_hs)
3803                 btc_notify_uvd_to_smc(rdev, new_ps);
3804         ret = ni_upload_sw_state(rdev, new_ps);
3805         if (ret) {
3806                 DRM_ERROR("ni_upload_sw_state failed\n");
3807                 return ret;
3808         }
3809         if (eg_pi->dynamic_ac_timing) {
3810                 ret = ni_upload_mc_reg_table(rdev, new_ps);
3811                 if (ret) {
3812                         DRM_ERROR("ni_upload_mc_reg_table failed\n");
3813                         return ret;
3814                 }
3815         }
3816         ret = ni_program_memory_timing_parameters(rdev, new_ps);
3817         if (ret) {
3818                 DRM_ERROR("ni_program_memory_timing_parameters failed\n");
3819                 return ret;
3820         }
3821         ret = rv770_resume_smc(rdev);
3822         if (ret) {
3823                 DRM_ERROR("rv770_resume_smc failed\n");
3824                 return ret;
3825         }
3826         ret = rv770_set_sw_state(rdev);
3827         if (ret) {
3828                 DRM_ERROR("rv770_set_sw_state failed\n");
3829                 return ret;
3830         }
3831         ni_set_uvd_clock_after_set_eng_clock(rdev, new_ps, old_ps);
3832         ret = ni_enable_smc_cac(rdev, new_ps, true);
3833         if (ret) {
3834                 DRM_ERROR("ni_enable_smc_cac failed\n");
3835                 return ret;
3836         }
3837         ret = ni_enable_power_containment(rdev, new_ps, true);
3838         if (ret) {
3839                 DRM_ERROR("ni_enable_power_containment failed\n");
3840                 return ret;
3841         }
3842
3843         /* update tdp */
3844         ret = ni_power_control_set_level(rdev);
3845         if (ret) {
3846                 DRM_ERROR("ni_power_control_set_level failed\n");
3847                 return ret;
3848         }
3849
3850         return 0;
3851 }
3852
3853 void ni_dpm_post_set_power_state(struct radeon_device *rdev)
3854 {
3855         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3856         struct radeon_ps *new_ps = &eg_pi->requested_rps;
3857
3858         ni_update_current_ps(rdev, new_ps);
3859 }
3860
3861 #if 0
3862 void ni_dpm_reset_asic(struct radeon_device *rdev)
3863 {
3864         ni_restrict_performance_levels_before_switch(rdev);
3865         rv770_set_boot_state(rdev);
3866 }
3867 #endif
3868
3869 union power_info {
3870         struct _ATOM_POWERPLAY_INFO info;
3871         struct _ATOM_POWERPLAY_INFO_V2 info_2;
3872         struct _ATOM_POWERPLAY_INFO_V3 info_3;
3873         struct _ATOM_PPLIB_POWERPLAYTABLE pplib;
3874         struct _ATOM_PPLIB_POWERPLAYTABLE2 pplib2;
3875         struct _ATOM_PPLIB_POWERPLAYTABLE3 pplib3;
3876 };
3877
3878 union pplib_clock_info {
3879         struct _ATOM_PPLIB_R600_CLOCK_INFO r600;
3880         struct _ATOM_PPLIB_RS780_CLOCK_INFO rs780;
3881         struct _ATOM_PPLIB_EVERGREEN_CLOCK_INFO evergreen;
3882         struct _ATOM_PPLIB_SUMO_CLOCK_INFO sumo;
3883 };
3884
3885 union pplib_power_state {
3886         struct _ATOM_PPLIB_STATE v1;
3887         struct _ATOM_PPLIB_STATE_V2 v2;
3888 };
3889
3890 static void ni_parse_pplib_non_clock_info(struct radeon_device *rdev,
3891                                           struct radeon_ps *rps,
3892                                           struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info,
3893                                           u8 table_rev)
3894 {
3895         rps->caps = le32_to_cpu(non_clock_info->ulCapsAndSettings);
3896         rps->class = le16_to_cpu(non_clock_info->usClassification);
3897         rps->class2 = le16_to_cpu(non_clock_info->usClassification2);
3898
3899         if (ATOM_PPLIB_NONCLOCKINFO_VER1 < table_rev) {
3900                 rps->vclk = le32_to_cpu(non_clock_info->ulVCLK);
3901                 rps->dclk = le32_to_cpu(non_clock_info->ulDCLK);
3902         } else if (r600_is_uvd_state(rps->class, rps->class2)) {
3903                 rps->vclk = RV770_DEFAULT_VCLK_FREQ;
3904                 rps->dclk = RV770_DEFAULT_DCLK_FREQ;
3905         } else {
3906                 rps->vclk = 0;
3907                 rps->dclk = 0;
3908         }
3909
3910         if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT)
3911                 rdev->pm.dpm.boot_ps = rps;
3912         if (rps->class & ATOM_PPLIB_CLASSIFICATION_UVDSTATE)
3913                 rdev->pm.dpm.uvd_ps = rps;
3914 }
3915
3916 static void ni_parse_pplib_clock_info(struct radeon_device *rdev,
3917                                       struct radeon_ps *rps, int index,
3918                                       union pplib_clock_info *clock_info)
3919 {
3920         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3921         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3922         struct ni_ps *ps = ni_get_ps(rps);
3923         struct rv7xx_pl *pl = &ps->performance_levels[index];
3924
3925         ps->performance_level_count = index + 1;
3926
3927         pl->sclk = le16_to_cpu(clock_info->evergreen.usEngineClockLow);
3928         pl->sclk |= clock_info->evergreen.ucEngineClockHigh << 16;
3929         pl->mclk = le16_to_cpu(clock_info->evergreen.usMemoryClockLow);
3930         pl->mclk |= clock_info->evergreen.ucMemoryClockHigh << 16;
3931
3932         pl->vddc = le16_to_cpu(clock_info->evergreen.usVDDC);
3933         pl->vddci = le16_to_cpu(clock_info->evergreen.usVDDCI);
3934         pl->flags = le32_to_cpu(clock_info->evergreen.ulFlags);
3935
3936         /* patch up vddc if necessary */
3937         if (pl->vddc == 0xff01) {
3938                 if (pi->max_vddc)
3939                         pl->vddc = pi->max_vddc;
3940         }
3941
3942         if (rps->class & ATOM_PPLIB_CLASSIFICATION_ACPI) {
3943                 pi->acpi_vddc = pl->vddc;
3944                 eg_pi->acpi_vddci = pl->vddci;
3945                 if (ps->performance_levels[0].flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2)
3946                         pi->acpi_pcie_gen2 = true;
3947                 else
3948                         pi->acpi_pcie_gen2 = false;
3949         }
3950
3951         if (rps->class2 & ATOM_PPLIB_CLASSIFICATION2_ULV) {
3952                 eg_pi->ulv.supported = true;
3953                 eg_pi->ulv.pl = pl;
3954         }
3955
3956         if (pi->min_vddc_in_table > pl->vddc)
3957                 pi->min_vddc_in_table = pl->vddc;
3958
3959         if (pi->max_vddc_in_table < pl->vddc)
3960                 pi->max_vddc_in_table = pl->vddc;
3961
3962         /* patch up boot state */
3963         if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT) {
3964                 u16 vddc, vddci, mvdd;
3965                 radeon_atombios_get_default_voltages(rdev, &vddc, &vddci, &mvdd);
3966                 pl->mclk = rdev->clock.default_mclk;
3967                 pl->sclk = rdev->clock.default_sclk;
3968                 pl->vddc = vddc;
3969                 pl->vddci = vddci;
3970         }
3971
3972         if ((rps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) ==
3973             ATOM_PPLIB_CLASSIFICATION_UI_PERFORMANCE) {
3974                 rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.sclk = pl->sclk;
3975                 rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.mclk = pl->mclk;
3976                 rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddc = pl->vddc;
3977                 rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddci = pl->vddci;
3978         }
3979 }
3980
3981 static int ni_parse_power_table(struct radeon_device *rdev)
3982 {
3983         struct radeon_mode_info *mode_info = &rdev->mode_info;
3984         struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info;
3985         union pplib_power_state *power_state;
3986         int i, j;
3987         union pplib_clock_info *clock_info;
3988         union power_info *power_info;
3989         int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo);
3990         u16 data_offset;
3991         u8 frev, crev;
3992         struct ni_ps *ps;
3993
3994         if (!atom_parse_data_header(mode_info->atom_context, index, NULL,
3995                                    &frev, &crev, &data_offset))
3996                 return -EINVAL;
3997         power_info = (union power_info *)(mode_info->atom_context->bios + data_offset);
3998
3999         rdev->pm.dpm.ps = kcalloc(power_info->pplib.ucNumStates,
4000                                   sizeof(struct radeon_ps),
4001                                   GFP_KERNEL);
4002         if (!rdev->pm.dpm.ps)
4003                 return -ENOMEM;
4004
4005         for (i = 0; i < power_info->pplib.ucNumStates; i++) {
4006                 power_state = (union pplib_power_state *)
4007                         (mode_info->atom_context->bios + data_offset +
4008                          le16_to_cpu(power_info->pplib.usStateArrayOffset) +
4009                          i * power_info->pplib.ucStateEntrySize);
4010                 non_clock_info = (struct _ATOM_PPLIB_NONCLOCK_INFO *)
4011                         (mode_info->atom_context->bios + data_offset +
4012                          le16_to_cpu(power_info->pplib.usNonClockInfoArrayOffset) +
4013                          (power_state->v1.ucNonClockStateIndex *
4014                           power_info->pplib.ucNonClockSize));
4015                 if (power_info->pplib.ucStateEntrySize - 1) {
4016                         u8 *idx;
4017                         ps = kzalloc(sizeof(struct ni_ps), GFP_KERNEL);
4018                         if (ps == NULL) {
4019                                 kfree(rdev->pm.dpm.ps);
4020                                 return -ENOMEM;
4021                         }
4022                         rdev->pm.dpm.ps[i].ps_priv = ps;
4023                         ni_parse_pplib_non_clock_info(rdev, &rdev->pm.dpm.ps[i],
4024                                                          non_clock_info,
4025                                                          power_info->pplib.ucNonClockSize);
4026                         idx = (u8 *)&power_state->v1.ucClockStateIndices[0];
4027                         for (j = 0; j < (power_info->pplib.ucStateEntrySize - 1); j++) {
4028                                 clock_info = (union pplib_clock_info *)
4029                                         (mode_info->atom_context->bios + data_offset +
4030                                          le16_to_cpu(power_info->pplib.usClockInfoArrayOffset) +
4031                                          (idx[j] * power_info->pplib.ucClockInfoSize));
4032                                 ni_parse_pplib_clock_info(rdev,
4033                                                           &rdev->pm.dpm.ps[i], j,
4034                                                           clock_info);
4035                         }
4036                 }
4037         }
4038         rdev->pm.dpm.num_ps = power_info->pplib.ucNumStates;
4039         return 0;
4040 }
4041
4042 int ni_dpm_init(struct radeon_device *rdev)
4043 {
4044         struct rv7xx_power_info *pi;
4045         struct evergreen_power_info *eg_pi;
4046         struct ni_power_info *ni_pi;
4047         struct atom_clock_dividers dividers;
4048         int ret;
4049
4050         ni_pi = kzalloc(sizeof(struct ni_power_info), GFP_KERNEL);
4051         if (ni_pi == NULL)
4052                 return -ENOMEM;
4053         rdev->pm.dpm.priv = ni_pi;
4054         eg_pi = &ni_pi->eg;
4055         pi = &eg_pi->rv7xx;
4056
4057         rv770_get_max_vddc(rdev);
4058
4059         eg_pi->ulv.supported = false;
4060         pi->acpi_vddc = 0;
4061         eg_pi->acpi_vddci = 0;
4062         pi->min_vddc_in_table = 0;
4063         pi->max_vddc_in_table = 0;
4064
4065         ret = r600_get_platform_caps(rdev);
4066         if (ret)
4067                 return ret;
4068
4069         ret = ni_parse_power_table(rdev);
4070         if (ret)
4071                 return ret;
4072         ret = r600_parse_extended_power_table(rdev);
4073         if (ret)
4074                 return ret;
4075
4076         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries =
4077                 kcalloc(4,
4078                         sizeof(struct radeon_clock_voltage_dependency_entry),
4079                         GFP_KERNEL);
4080         if (!rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries) {
4081                 r600_free_extended_power_table(rdev);
4082                 return -ENOMEM;
4083         }
4084         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.count = 4;
4085         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[0].clk = 0;
4086         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[0].v = 0;
4087         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[1].clk = 36000;
4088         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[1].v = 720;
4089         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[2].clk = 54000;
4090         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[2].v = 810;
4091         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[3].clk = 72000;
4092         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[3].v = 900;
4093
4094         ni_patch_dependency_tables_based_on_leakage(rdev);
4095
4096         if (rdev->pm.dpm.voltage_response_time == 0)
4097                 rdev->pm.dpm.voltage_response_time = R600_VOLTAGERESPONSETIME_DFLT;
4098         if (rdev->pm.dpm.backbias_response_time == 0)
4099                 rdev->pm.dpm.backbias_response_time = R600_BACKBIASRESPONSETIME_DFLT;
4100
4101         ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
4102                                              0, false, &dividers);
4103         if (ret)
4104                 pi->ref_div = dividers.ref_div + 1;
4105         else
4106                 pi->ref_div = R600_REFERENCEDIVIDER_DFLT;
4107
4108         pi->rlp = RV770_RLP_DFLT;
4109         pi->rmp = RV770_RMP_DFLT;
4110         pi->lhp = RV770_LHP_DFLT;
4111         pi->lmp = RV770_LMP_DFLT;
4112
4113         eg_pi->ats[0].rlp = RV770_RLP_DFLT;
4114         eg_pi->ats[0].rmp = RV770_RMP_DFLT;
4115         eg_pi->ats[0].lhp = RV770_LHP_DFLT;
4116         eg_pi->ats[0].lmp = RV770_LMP_DFLT;
4117
4118         eg_pi->ats[1].rlp = BTC_RLP_UVD_DFLT;
4119         eg_pi->ats[1].rmp = BTC_RMP_UVD_DFLT;
4120         eg_pi->ats[1].lhp = BTC_LHP_UVD_DFLT;
4121         eg_pi->ats[1].lmp = BTC_LMP_UVD_DFLT;
4122
4123         eg_pi->smu_uvd_hs = true;
4124
4125         if (rdev->pdev->device == 0x6707) {
4126                 pi->mclk_strobe_mode_threshold = 55000;
4127                 pi->mclk_edc_enable_threshold = 55000;
4128                 eg_pi->mclk_edc_wr_enable_threshold = 55000;
4129         } else {
4130                 pi->mclk_strobe_mode_threshold = 40000;
4131                 pi->mclk_edc_enable_threshold = 40000;
4132                 eg_pi->mclk_edc_wr_enable_threshold = 40000;
4133         }
4134         ni_pi->mclk_rtt_mode_threshold = eg_pi->mclk_edc_wr_enable_threshold;
4135
4136         pi->voltage_control =
4137                 radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC, 0);
4138
4139         pi->mvdd_control =
4140                 radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_MVDDC, 0);
4141
4142         eg_pi->vddci_control =
4143                 radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_VDDCI, 0);
4144
4145         rv770_get_engine_memory_ss(rdev);
4146
4147         pi->asi = RV770_ASI_DFLT;
4148         pi->pasi = CYPRESS_HASI_DFLT;
4149         pi->vrc = CYPRESS_VRC_DFLT;
4150
4151         pi->power_gating = false;
4152
4153         pi->gfx_clock_gating = true;
4154
4155         pi->mg_clock_gating = true;
4156         pi->mgcgtssm = true;
4157         eg_pi->ls_clock_gating = false;
4158         eg_pi->sclk_deep_sleep = false;
4159
4160         pi->dynamic_pcie_gen2 = true;
4161
4162         if (rdev->pm.int_thermal_type != THERMAL_TYPE_NONE)
4163                 pi->thermal_protection = true;
4164         else
4165                 pi->thermal_protection = false;
4166
4167         pi->display_gap = true;
4168
4169         pi->dcodt = true;
4170
4171         pi->ulps = true;
4172
4173         eg_pi->dynamic_ac_timing = true;
4174         eg_pi->abm = true;
4175         eg_pi->mcls = true;
4176         eg_pi->light_sleep = true;
4177         eg_pi->memory_transition = true;
4178 #if defined(CONFIG_ACPI)
4179         eg_pi->pcie_performance_request =
4180                 radeon_acpi_is_pcie_performance_request_supported(rdev);
4181 #else
4182         eg_pi->pcie_performance_request = false;
4183 #endif
4184
4185         eg_pi->dll_default_on = false;
4186
4187         eg_pi->sclk_deep_sleep = false;
4188
4189         pi->mclk_stutter_mode_threshold = 0;
4190
4191         pi->sram_end = SMC_RAM_END;
4192
4193         rdev->pm.dpm.dyn_state.mclk_sclk_ratio = 3;
4194         rdev->pm.dpm.dyn_state.vddc_vddci_delta = 200;
4195         rdev->pm.dpm.dyn_state.min_vddc_for_pcie_gen2 = 900;
4196         rdev->pm.dpm.dyn_state.valid_sclk_values.count = ARRAY_SIZE(btc_valid_sclk);
4197         rdev->pm.dpm.dyn_state.valid_sclk_values.values = btc_valid_sclk;
4198         rdev->pm.dpm.dyn_state.valid_mclk_values.count = 0;
4199         rdev->pm.dpm.dyn_state.valid_mclk_values.values = NULL;
4200         rdev->pm.dpm.dyn_state.sclk_mclk_delta = 12500;
4201
4202         ni_pi->cac_data.leakage_coefficients.at = 516;
4203         ni_pi->cac_data.leakage_coefficients.bt = 18;
4204         ni_pi->cac_data.leakage_coefficients.av = 51;
4205         ni_pi->cac_data.leakage_coefficients.bv = 2957;
4206
4207         switch (rdev->pdev->device) {
4208         case 0x6700:
4209         case 0x6701:
4210         case 0x6702:
4211         case 0x6703:
4212         case 0x6718:
4213                 ni_pi->cac_weights = &cac_weights_cayman_xt;
4214                 break;
4215         case 0x6705:
4216         case 0x6719:
4217         case 0x671D:
4218         case 0x671C:
4219         default:
4220                 ni_pi->cac_weights = &cac_weights_cayman_pro;
4221                 break;
4222         case 0x6704:
4223         case 0x6706:
4224         case 0x6707:
4225         case 0x6708:
4226         case 0x6709:
4227                 ni_pi->cac_weights = &cac_weights_cayman_le;
4228                 break;
4229         }
4230
4231         if (ni_pi->cac_weights->enable_power_containment_by_default) {
4232                 ni_pi->enable_power_containment = true;
4233                 ni_pi->enable_cac = true;
4234                 ni_pi->enable_sq_ramping = true;
4235         } else {
4236                 ni_pi->enable_power_containment = false;
4237                 ni_pi->enable_cac = false;
4238                 ni_pi->enable_sq_ramping = false;
4239         }
4240
4241         ni_pi->driver_calculate_cac_leakage = false;
4242         ni_pi->cac_configuration_required = true;
4243
4244         if (ni_pi->cac_configuration_required) {
4245                 ni_pi->support_cac_long_term_average = true;
4246                 ni_pi->lta_window_size = ni_pi->cac_weights->l2_lta_window_size;
4247                 ni_pi->lts_truncate = ni_pi->cac_weights->lts_truncate;
4248         } else {
4249                 ni_pi->support_cac_long_term_average = false;
4250                 ni_pi->lta_window_size = 0;
4251                 ni_pi->lts_truncate = 0;
4252         }
4253
4254         ni_pi->use_power_boost_limit = true;
4255
4256         /* make sure dc limits are valid */
4257         if ((rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.sclk == 0) ||
4258             (rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.mclk == 0))
4259                 rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc =
4260                         rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac;
4261
4262         return 0;
4263 }
4264
4265 void ni_dpm_fini(struct radeon_device *rdev)
4266 {
4267         int i;
4268
4269         for (i = 0; i < rdev->pm.dpm.num_ps; i++) {
4270                 kfree(rdev->pm.dpm.ps[i].ps_priv);
4271         }
4272         kfree(rdev->pm.dpm.ps);
4273         kfree(rdev->pm.dpm.priv);
4274         kfree(rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries);
4275         r600_free_extended_power_table(rdev);
4276 }
4277
4278 void ni_dpm_print_power_state(struct radeon_device *rdev,
4279                               struct radeon_ps *rps)
4280 {
4281         struct ni_ps *ps = ni_get_ps(rps);
4282         struct rv7xx_pl *pl;
4283         int i;
4284
4285         r600_dpm_print_class_info(rps->class, rps->class2);
4286         r600_dpm_print_cap_info(rps->caps);
4287         printk("\tuvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
4288         for (i = 0; i < ps->performance_level_count; i++) {
4289                 pl = &ps->performance_levels[i];
4290                 if (rdev->family >= CHIP_TAHITI)
4291                         printk("\t\tpower level %d    sclk: %u mclk: %u vddc: %u vddci: %u pcie gen: %u\n",
4292                                i, pl->sclk, pl->mclk, pl->vddc, pl->vddci, pl->pcie_gen + 1);
4293                 else
4294                         printk("\t\tpower level %d    sclk: %u mclk: %u vddc: %u vddci: %u\n",
4295                                i, pl->sclk, pl->mclk, pl->vddc, pl->vddci);
4296         }
4297         r600_dpm_print_ps_status(rdev, rps);
4298 }
4299
4300 void ni_dpm_debugfs_print_current_performance_level(struct radeon_device *rdev,
4301                                                     struct seq_file *m)
4302 {
4303         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
4304         struct radeon_ps *rps = &eg_pi->current_rps;
4305         struct ni_ps *ps = ni_get_ps(rps);
4306         struct rv7xx_pl *pl;
4307         u32 current_index =
4308                 (RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_INDEX_MASK) >>
4309                 CURRENT_STATE_INDEX_SHIFT;
4310
4311         if (current_index >= ps->performance_level_count) {
4312                 seq_printf(m, "invalid dpm profile %d\n", current_index);
4313         } else {
4314                 pl = &ps->performance_levels[current_index];
4315                 seq_printf(m, "uvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
4316                 seq_printf(m, "power level %d    sclk: %u mclk: %u vddc: %u vddci: %u\n",
4317                            current_index, pl->sclk, pl->mclk, pl->vddc, pl->vddci);
4318         }
4319 }
4320
4321 u32 ni_dpm_get_current_sclk(struct radeon_device *rdev)
4322 {
4323         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
4324         struct radeon_ps *rps = &eg_pi->current_rps;
4325         struct ni_ps *ps = ni_get_ps(rps);
4326         struct rv7xx_pl *pl;
4327         u32 current_index =
4328                 (RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_INDEX_MASK) >>
4329                 CURRENT_STATE_INDEX_SHIFT;
4330
4331         if (current_index >= ps->performance_level_count) {
4332                 return 0;
4333         } else {
4334                 pl = &ps->performance_levels[current_index];
4335                 return pl->sclk;
4336         }
4337 }
4338
4339 u32 ni_dpm_get_current_mclk(struct radeon_device *rdev)
4340 {
4341         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
4342         struct radeon_ps *rps = &eg_pi->current_rps;
4343         struct ni_ps *ps = ni_get_ps(rps);
4344         struct rv7xx_pl *pl;
4345         u32 current_index =
4346                 (RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_INDEX_MASK) >>
4347                 CURRENT_STATE_INDEX_SHIFT;
4348
4349         if (current_index >= ps->performance_level_count) {
4350                 return 0;
4351         } else {
4352                 pl = &ps->performance_levels[current_index];
4353                 return pl->mclk;
4354         }
4355 }
4356
4357 u32 ni_dpm_get_sclk(struct radeon_device *rdev, bool low)
4358 {
4359         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
4360         struct ni_ps *requested_state = ni_get_ps(&eg_pi->requested_rps);
4361
4362         if (low)
4363                 return requested_state->performance_levels[0].sclk;
4364         else
4365                 return requested_state->performance_levels[requested_state->performance_level_count - 1].sclk;
4366 }
4367
4368 u32 ni_dpm_get_mclk(struct radeon_device *rdev, bool low)
4369 {
4370         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
4371         struct ni_ps *requested_state = ni_get_ps(&eg_pi->requested_rps);
4372
4373         if (low)
4374                 return requested_state->performance_levels[0].mclk;
4375         else
4376                 return requested_state->performance_levels[requested_state->performance_level_count - 1].mclk;
4377 }
4378