sys/dev/drm2/radeon/ni.c

   1 /*
   2  * Copyright 2010 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24
  25 #include <sys/cdefs.h>
  26 __FBSDID("$FreeBSD$");
  27
  28 #include <dev/drm2/drmP.h>
  29 #include "radeon.h"
  30 #include "radeon_asic.h"
  31 #include <dev/drm2/radeon/radeon_drm.h>
  32 #include "nid.h"
  33 #include "atom.h"
  34 #include "ni_reg.h"
  35 #include "cayman_blit_shaders.h"
  36
  37 #ifdef FREEBSD_WIP /* FreeBSD: to please GCC 4.2. */
  38 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
  39 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
  40 extern int evergreen_mc_wait_for_idle(struct radeon_device *rdev);
  41 extern void evergreen_mc_program(struct radeon_device *rdev);
  42 extern void evergreen_irq_suspend(struct radeon_device *rdev);
  43 extern int evergreen_mc_init(struct radeon_device *rdev);
  44 extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
  45 #endif
  46 extern void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
  47 #ifdef FREEBSD_WIP /* FreeBSD: to please GCC 4.2. */
  48 extern void si_rlc_fini(struct radeon_device *rdev);
  49 extern int si_rlc_init(struct radeon_device *rdev);
  50 #endif
  51
  52 #define EVERGREEN_PFP_UCODE_SIZE 1120
  53 #define EVERGREEN_PM4_UCODE_SIZE 1376
  54 #define EVERGREEN_RLC_UCODE_SIZE 768
  55 #define BTC_MC_UCODE_SIZE 6024
  56
  57 #define CAYMAN_PFP_UCODE_SIZE 2176
  58 #define CAYMAN_PM4_UCODE_SIZE 2176
  59 #define CAYMAN_RLC_UCODE_SIZE 1024
  60 #define CAYMAN_MC_UCODE_SIZE 6037
  61
  62 #define ARUBA_RLC_UCODE_SIZE 1536
  63
  64 #ifdef __linux__
  65 /* Firmware Names */
  66 MODULE_FIRMWARE("radeon/BARTS_pfp.bin");
  67 MODULE_FIRMWARE("radeon/BARTS_me.bin");
  68 MODULE_FIRMWARE("radeon/BARTS_mc.bin");
  69 MODULE_FIRMWARE("radeon/BTC_rlc.bin");
  70 MODULE_FIRMWARE("radeon/TURKS_pfp.bin");
  71 MODULE_FIRMWARE("radeon/TURKS_me.bin");
  72 MODULE_FIRMWARE("radeon/TURKS_mc.bin");
  73 MODULE_FIRMWARE("radeon/CAICOS_pfp.bin");
  74 MODULE_FIRMWARE("radeon/CAICOS_me.bin");
  75 MODULE_FIRMWARE("radeon/CAICOS_mc.bin");
  76 MODULE_FIRMWARE("radeon/CAYMAN_pfp.bin");
  77 MODULE_FIRMWARE("radeon/CAYMAN_me.bin");
  78 MODULE_FIRMWARE("radeon/CAYMAN_mc.bin");
  79 MODULE_FIRMWARE("radeon/CAYMAN_rlc.bin");
  80 MODULE_FIRMWARE("radeon/ARUBA_pfp.bin");
  81 MODULE_FIRMWARE("radeon/ARUBA_me.bin");
  82 MODULE_FIRMWARE("radeon/ARUBA_rlc.bin");
  83 #endif
  84
  85 #define BTC_IO_MC_REGS_SIZE 29
  86
  87 static const u32 barts_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
  88         {0x00000077, 0xff010100},
  89         {0x00000078, 0x00000000},
  90         {0x00000079, 0x00001434},
  91         {0x0000007a, 0xcc08ec08},
  92         {0x0000007b, 0x00040000},
  93         {0x0000007c, 0x000080c0},
  94         {0x0000007d, 0x09000000},
  95         {0x0000007e, 0x00210404},
  96         {0x00000081, 0x08a8e800},
  97         {0x00000082, 0x00030444},
  98         {0x00000083, 0x00000000},
  99         {0x00000085, 0x00000001},
 100         {0x00000086, 0x00000002},
 101         {0x00000087, 0x48490000},
 102         {0x00000088, 0x20244647},
 103         {0x00000089, 0x00000005},
 104         {0x0000008b, 0x66030000},
 105         {0x0000008c, 0x00006603},
 106         {0x0000008d, 0x00000100},
 107         {0x0000008f, 0x00001c0a},
 108         {0x00000090, 0xff000001},
 109         {0x00000094, 0x00101101},
 110         {0x00000095, 0x00000fff},
 111         {0x00000096, 0x00116fff},
 112         {0x00000097, 0x60010000},
 113         {0x00000098, 0x10010000},
 114         {0x00000099, 0x00006000},
 115         {0x0000009a, 0x00001000},
 116         {0x0000009f, 0x00946a00}
 117 };
 118
 119 static const u32 turks_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 120         {0x00000077, 0xff010100},
 121         {0x00000078, 0x00000000},
 122         {0x00000079, 0x00001434},
 123         {0x0000007a, 0xcc08ec08},
 124         {0x0000007b, 0x00040000},
 125         {0x0000007c, 0x000080c0},
 126         {0x0000007d, 0x09000000},
 127         {0x0000007e, 0x00210404},
 128         {0x00000081, 0x08a8e800},
 129         {0x00000082, 0x00030444},
 130         {0x00000083, 0x00000000},
 131         {0x00000085, 0x00000001},
 132         {0x00000086, 0x00000002},
 133         {0x00000087, 0x48490000},
 134         {0x00000088, 0x20244647},
 135         {0x00000089, 0x00000005},
 136         {0x0000008b, 0x66030000},
 137         {0x0000008c, 0x00006603},
 138         {0x0000008d, 0x00000100},
 139         {0x0000008f, 0x00001c0a},
 140         {0x00000090, 0xff000001},
 141         {0x00000094, 0x00101101},
 142         {0x00000095, 0x00000fff},
 143         {0x00000096, 0x00116fff},
 144         {0x00000097, 0x60010000},
 145         {0x00000098, 0x10010000},
 146         {0x00000099, 0x00006000},
 147         {0x0000009a, 0x00001000},
 148         {0x0000009f, 0x00936a00}
 149 };
 150
 151 static const u32 caicos_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 152         {0x00000077, 0xff010100},
 153         {0x00000078, 0x00000000},
 154         {0x00000079, 0x00001434},
 155         {0x0000007a, 0xcc08ec08},
 156         {0x0000007b, 0x00040000},
 157         {0x0000007c, 0x000080c0},
 158         {0x0000007d, 0x09000000},
 159         {0x0000007e, 0x00210404},
 160         {0x00000081, 0x08a8e800},
 161         {0x00000082, 0x00030444},
 162         {0x00000083, 0x00000000},
 163         {0x00000085, 0x00000001},
 164         {0x00000086, 0x00000002},
 165         {0x00000087, 0x48490000},
 166         {0x00000088, 0x20244647},
 167         {0x00000089, 0x00000005},
 168         {0x0000008b, 0x66030000},
 169         {0x0000008c, 0x00006603},
 170         {0x0000008d, 0x00000100},
 171         {0x0000008f, 0x00001c0a},
 172         {0x00000090, 0xff000001},
 173         {0x00000094, 0x00101101},
 174         {0x00000095, 0x00000fff},
 175         {0x00000096, 0x00116fff},
 176         {0x00000097, 0x60010000},
 177         {0x00000098, 0x10010000},
 178         {0x00000099, 0x00006000},
 179         {0x0000009a, 0x00001000},
 180         {0x0000009f, 0x00916a00}
 181 };
 182
 183 static const u32 cayman_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 184         {0x00000077, 0xff010100},
 185         {0x00000078, 0x00000000},
 186         {0x00000079, 0x00001434},
 187         {0x0000007a, 0xcc08ec08},
 188         {0x0000007b, 0x00040000},
 189         {0x0000007c, 0x000080c0},
 190         {0x0000007d, 0x09000000},
 191         {0x0000007e, 0x00210404},
 192         {0x00000081, 0x08a8e800},
 193         {0x00000082, 0x00030444},
 194         {0x00000083, 0x00000000},
 195         {0x00000085, 0x00000001},
 196         {0x00000086, 0x00000002},
 197         {0x00000087, 0x48490000},
 198         {0x00000088, 0x20244647},
 199         {0x00000089, 0x00000005},
 200         {0x0000008b, 0x66030000},
 201         {0x0000008c, 0x00006603},
 202         {0x0000008d, 0x00000100},
 203         {0x0000008f, 0x00001c0a},
 204         {0x00000090, 0xff000001},
 205         {0x00000094, 0x00101101},
 206         {0x00000095, 0x00000fff},
 207         {0x00000096, 0x00116fff},
 208         {0x00000097, 0x60010000},
 209         {0x00000098, 0x10010000},
 210         {0x00000099, 0x00006000},
 211         {0x0000009a, 0x00001000},
 212         {0x0000009f, 0x00976b00}
 213 };
 214
 215 int ni_mc_load_microcode(struct radeon_device *rdev)
 216 {
 217         const __be32 *fw_data;
 218         u32 mem_type, running, blackout = 0;
 219         const u32 *io_mc_regs;
 220         int i, ucode_size, regs_size;
 221
 222         if (!rdev->mc_fw)
 223                 return -EINVAL;
 224
 225         switch (rdev->family) {
 226         case CHIP_BARTS:
 227                 io_mc_regs = &barts_io_mc_regs[0][0];
 228                 ucode_size = BTC_MC_UCODE_SIZE;
 229                 regs_size = BTC_IO_MC_REGS_SIZE;
 230                 break;
 231         case CHIP_TURKS:
 232                 io_mc_regs = &turks_io_mc_regs[0][0];
 233                 ucode_size = BTC_MC_UCODE_SIZE;
 234                 regs_size = BTC_IO_MC_REGS_SIZE;
 235                 break;
 236         case CHIP_CAICOS:
 237         default:
 238                 io_mc_regs = &caicos_io_mc_regs[0][0];
 239                 ucode_size = BTC_MC_UCODE_SIZE;
 240                 regs_size = BTC_IO_MC_REGS_SIZE;
 241                 break;
 242         case CHIP_CAYMAN:
 243                 io_mc_regs = &cayman_io_mc_regs[0][0];
 244                 ucode_size = CAYMAN_MC_UCODE_SIZE;
 245                 regs_size = BTC_IO_MC_REGS_SIZE;
 246                 break;
 247         }
 248
 249         mem_type = (RREG32(MC_SEQ_MISC0) & MC_SEQ_MISC0_GDDR5_MASK) >> MC_SEQ_MISC0_GDDR5_SHIFT;
 250         running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
 251
 252         if ((mem_type == MC_SEQ_MISC0_GDDR5_VALUE) && (running == 0)) {
 253                 if (running) {
 254                         blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
 255                         WREG32(MC_SHARED_BLACKOUT_CNTL, 1);
 256                 }
 257
 258                 /* reset the engine and set to writable */
 259                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
 260                 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
 261
 262                 /* load mc io regs */
 263                 for (i = 0; i < regs_size; i++) {
 264                         WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
 265                         WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
 266                 }
 267                 /* load the MC ucode */
 268                 fw_data = (const __be32 *)rdev->mc_fw->data;
 269                 for (i = 0; i < ucode_size; i++)
 270                         WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
 271
 272                 /* put the engine back into the active state */
 273                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
 274                 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
 275                 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
 276
 277                 /* wait for training to complete */
 278                 for (i = 0; i < rdev->usec_timeout; i++) {
 279                         if (RREG32(MC_IO_PAD_CNTL_D0) & MEM_FALL_OUT_CMD)
 280                                 break;
 281                         udelay(1);
 282                 }
 283
 284                 if (running)
 285                         WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
 286         }
 287
 288         return 0;
 289 }
 290
 291 int ni_init_microcode(struct radeon_device *rdev)
 292 {
 293         const char *chip_name;
 294         const char *rlc_chip_name;
 295         size_t pfp_req_size, me_req_size, rlc_req_size, mc_req_size;
 296         char fw_name[30];
 297         int err;
 298
 299         DRM_DEBUG("\n");
 300
 301         switch (rdev->family) {
 302         case CHIP_BARTS:
 303                 chip_name = "BARTS";
 304                 rlc_chip_name = "BTC";
 305                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 306                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 307                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 308                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
 309                 break;
 310         case CHIP_TURKS:
 311                 chip_name = "TURKS";
 312                 rlc_chip_name = "BTC";
 313                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 314                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 315                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 316                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
 317                 break;
 318         case CHIP_CAICOS:
 319                 chip_name = "CAICOS";
 320                 rlc_chip_name = "BTC";
 321                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 322                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 323                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 324                 mc_req_size = BTC_MC_UCODE_SIZE * 4;
 325                 break;
 326         case CHIP_CAYMAN:
 327                 chip_name = "CAYMAN";
 328                 rlc_chip_name = "CAYMAN";
 329                 pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4;
 330                 me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
 331                 rlc_req_size = CAYMAN_RLC_UCODE_SIZE * 4;
 332                 mc_req_size = CAYMAN_MC_UCODE_SIZE * 4;
 333                 break;
 334         case CHIP_ARUBA:
 335                 chip_name = "ARUBA";
 336                 rlc_chip_name = "ARUBA";
 337                 /* pfp/me same size as CAYMAN */
 338                 pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4;
 339                 me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
 340                 rlc_req_size = ARUBA_RLC_UCODE_SIZE * 4;
 341                 mc_req_size = 0;
 342                 break;
 343         default: panic("%s: Unsupported family %d", __func__, rdev->family);
 344         }
 345
 346         DRM_INFO("Loading %s Microcode\n", chip_name);
 347         err = 0;
 348
 349         snprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_pfp", chip_name);
 350         rdev->pfp_fw = firmware_get(fw_name);
 351         if (rdev->pfp_fw == NULL) {
 352                 err = -ENOENT;
 353                 goto out;
 354         }
 355         if (rdev->pfp_fw->datasize != pfp_req_size) {
 356                 DRM_ERROR(
 357                        "ni_cp: Bogus length %zu in firmware \"%s\"\n",
 358                        rdev->pfp_fw->datasize, fw_name);
 359                 err = -EINVAL;
 360                 goto out;
 361         }
 362
 363         snprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_me", chip_name);
 364         rdev->me_fw = firmware_get(fw_name);
 365         if (rdev->me_fw == NULL) {
 366                 err = -ENOENT;
 367                 goto out;
 368         }
 369         if (rdev->me_fw->datasize != me_req_size) {
 370                 DRM_ERROR(
 371                        "ni_cp: Bogus length %zu in firmware \"%s\"\n",
 372                        rdev->me_fw->datasize, fw_name);
 373                 err = -EINVAL;
 374         }
 375
 376         snprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_rlc", rlc_chip_name);
 377         rdev->rlc_fw = firmware_get(fw_name);
 378         if (rdev->rlc_fw == NULL) {
 379                 err = -ENOENT;
 380                 goto out;
 381         }
 382         if (rdev->rlc_fw->datasize != rlc_req_size) {
 383                 DRM_ERROR(
 384                        "ni_rlc: Bogus length %zu in firmware \"%s\"\n",
 385                        rdev->rlc_fw->datasize, fw_name);
 386                 err = -EINVAL;
 387         }
 388
 389         /* no MC ucode on TN */
 390         if (!(rdev->flags & RADEON_IS_IGP)) {
 391                 snprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_mc", chip_name);
 392                 rdev->mc_fw = firmware_get(fw_name);
 393                 if (rdev->mc_fw == NULL) {
 394                         err = -ENOENT;
 395                         goto out;
 396                 }
 397                 if (rdev->mc_fw->datasize != mc_req_size) {
 398                         DRM_ERROR(
 399                                "ni_mc: Bogus length %zu in firmware \"%s\"\n",
 400                                rdev->mc_fw->datasize, fw_name);
 401                         err = -EINVAL;
 402                 }
 403         }
 404 out:
 405         if (err) {
 406                 if (err != -EINVAL)
 407                         DRM_ERROR(
 408                                "ni_cp: Failed to load firmware \"%s\"\n",
 409                                fw_name);
 410                 if (rdev->pfp_fw != NULL) {
 411                         firmware_put(rdev->pfp_fw, FIRMWARE_UNLOAD);
 412                         rdev->pfp_fw = NULL;
 413                 }
 414                 if (rdev->me_fw != NULL) {
 415                         firmware_put(rdev->me_fw, FIRMWARE_UNLOAD);
 416                         rdev->me_fw = NULL;
 417                 }
 418                 if (rdev->rlc_fw != NULL) {
 419                         firmware_put(rdev->rlc_fw, FIRMWARE_UNLOAD);
 420                         rdev->rlc_fw = NULL;
 421                 }
 422                 if (rdev->mc_fw != NULL) {
 423                         firmware_put(rdev->mc_fw, FIRMWARE_UNLOAD);
 424                         rdev->mc_fw = NULL;
 425                 }
 426         }
 427         return err;
 428 }
 429
 430 /**
 431  * ni_fini_microcode - drop the firmwares image references
 432  *
 433  * @rdev: radeon_device pointer
 434  *
 435  * Drop the pfp, me, mc and rlc firmwares image references.
 436  * Called at driver shutdown.
 437  */
 438 void ni_fini_microcode(struct radeon_device *rdev)
 439 {
 440
 441         if (rdev->pfp_fw != NULL) {
 442                 firmware_put(rdev->pfp_fw, FIRMWARE_UNLOAD);
 443                 rdev->pfp_fw = NULL;
 444         }
 445
 446         if (rdev->me_fw != NULL) {
 447                 firmware_put(rdev->me_fw, FIRMWARE_UNLOAD);
 448                 rdev->me_fw = NULL;
 449         }
 450
 451         if (rdev->rlc_fw != NULL) {
 452                 firmware_put(rdev->rlc_fw, FIRMWARE_UNLOAD);
 453                 rdev->rlc_fw = NULL;
 454         }
 455
 456         if (rdev->mc_fw != NULL) {
 457                 firmware_put(rdev->mc_fw, FIRMWARE_UNLOAD);
 458                 rdev->mc_fw = NULL;
 459         }
 460 }
 461
 462
 463 /*
 464  * Core functions
 465  */
 466 static void cayman_gpu_init(struct radeon_device *rdev)
 467 {
 468         u32 gb_addr_config = 0;
 469         u32 mc_shared_chmap, mc_arb_ramcfg;
 470         u32 cgts_tcc_disable;
 471         u32 sx_debug_1;
 472         u32 smx_dc_ctl0;
 473         u32 cgts_sm_ctrl_reg;
 474         u32 hdp_host_path_cntl;
 475         u32 tmp;
 476         u32 disabled_rb_mask;
 477         int i, j;
 478
 479         switch (rdev->family) {
 480         case CHIP_CAYMAN:
 481                 rdev->config.cayman.max_shader_engines = 2;
 482                 rdev->config.cayman.max_pipes_per_simd = 4;
 483                 rdev->config.cayman.max_tile_pipes = 8;
 484                 rdev->config.cayman.max_simds_per_se = 12;
 485                 rdev->config.cayman.max_backends_per_se = 4;
 486                 rdev->config.cayman.max_texture_channel_caches = 8;
 487                 rdev->config.cayman.max_gprs = 256;
 488                 rdev->config.cayman.max_threads = 256;
 489                 rdev->config.cayman.max_gs_threads = 32;
 490                 rdev->config.cayman.max_stack_entries = 512;
 491                 rdev->config.cayman.sx_num_of_sets = 8;
 492                 rdev->config.cayman.sx_max_export_size = 256;
 493                 rdev->config.cayman.sx_max_export_pos_size = 64;
 494                 rdev->config.cayman.sx_max_export_smx_size = 192;
 495                 rdev->config.cayman.max_hw_contexts = 8;
 496                 rdev->config.cayman.sq_num_cf_insts = 2;
 497
 498                 rdev->config.cayman.sc_prim_fifo_size = 0x100;
 499                 rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;
 500                 rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;
 501                 gb_addr_config = CAYMAN_GB_ADDR_CONFIG_GOLDEN;
 502                 break;
 503         case CHIP_ARUBA:
 504         default:
 505                 rdev->config.cayman.max_shader_engines = 1;
 506                 rdev->config.cayman.max_pipes_per_simd = 4;
 507                 rdev->config.cayman.max_tile_pipes = 2;
 508                 if ((rdev->ddev->pci_device == 0x9900) ||
 509                     (rdev->ddev->pci_device == 0x9901) ||
 510                     (rdev->ddev->pci_device == 0x9905) ||
 511                     (rdev->ddev->pci_device == 0x9906) ||
 512                     (rdev->ddev->pci_device == 0x9907) ||
 513                     (rdev->ddev->pci_device == 0x9908) ||
 514                     (rdev->ddev->pci_device == 0x9909) ||
 515                     (rdev->ddev->pci_device == 0x990B) ||
 516                     (rdev->ddev->pci_device == 0x990C) ||
 517                     (rdev->ddev->pci_device == 0x990F) ||
 518                     (rdev->ddev->pci_device == 0x9910) ||
 519                     (rdev->ddev->pci_device == 0x9917) ||
 520                     (rdev->ddev->pci_device == 0x9999) ||
 521                     (rdev->ddev->pci_device == 0x999C)) {
 522                         rdev->config.cayman.max_simds_per_se = 6;
 523                         rdev->config.cayman.max_backends_per_se = 2;
 524                 } else if ((rdev->ddev->pci_device == 0x9903) ||
 525                            (rdev->ddev->pci_device == 0x9904) ||
 526                            (rdev->ddev->pci_device == 0x990A) ||
 527                            (rdev->ddev->pci_device == 0x990D) ||
 528                            (rdev->ddev->pci_device == 0x990E) ||
 529                            (rdev->ddev->pci_device == 0x9913) ||
 530                            (rdev->ddev->pci_device == 0x9918) ||
 531                            (rdev->ddev->pci_device == 0x999D)) {
 532                         rdev->config.cayman.max_simds_per_se = 4;
 533                         rdev->config.cayman.max_backends_per_se = 2;
 534                 } else if ((rdev->ddev->pci_device == 0x9919) ||
 535                            (rdev->ddev->pci_device == 0x9990) ||
 536                            (rdev->ddev->pci_device == 0x9991) ||
 537                            (rdev->ddev->pci_device == 0x9994) ||
 538                            (rdev->ddev->pci_device == 0x9995) ||
 539                            (rdev->ddev->pci_device == 0x9996) ||
 540                            (rdev->ddev->pci_device == 0x999A) ||
 541                            (rdev->ddev->pci_device == 0x99A0)) {
 542                         rdev->config.cayman.max_simds_per_se = 3;
 543                         rdev->config.cayman.max_backends_per_se = 1;
 544                 } else {
 545                         rdev->config.cayman.max_simds_per_se = 2;
 546                         rdev->config.cayman.max_backends_per_se = 1;
 547                 }
 548                 rdev->config.cayman.max_texture_channel_caches = 2;
 549                 rdev->config.cayman.max_gprs = 256;
 550                 rdev->config.cayman.max_threads = 256;
 551                 rdev->config.cayman.max_gs_threads = 32;
 552                 rdev->config.cayman.max_stack_entries = 512;
 553                 rdev->config.cayman.sx_num_of_sets = 8;
 554                 rdev->config.cayman.sx_max_export_size = 256;
 555                 rdev->config.cayman.sx_max_export_pos_size = 64;
 556                 rdev->config.cayman.sx_max_export_smx_size = 192;
 557                 rdev->config.cayman.max_hw_contexts = 8;
 558                 rdev->config.cayman.sq_num_cf_insts = 2;
 559
 560                 rdev->config.cayman.sc_prim_fifo_size = 0x40;
 561                 rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;
 562                 rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;
 563                 gb_addr_config = ARUBA_GB_ADDR_CONFIG_GOLDEN;
 564                 break;
 565         }
 566
 567         /* Initialize HDP */
 568         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
 569                 WREG32((0x2c14 + j), 0x00000000);
 570                 WREG32((0x2c18 + j), 0x00000000);
 571                 WREG32((0x2c1c + j), 0x00000000);
 572                 WREG32((0x2c20 + j), 0x00000000);
 573                 WREG32((0x2c24 + j), 0x00000000);
 574         }
 575
 576         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
 577
 578         evergreen_fix_pci_max_read_req_size(rdev);
 579
 580         mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
 581         mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
 582
 583         tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
 584         rdev->config.cayman.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
 585         if (rdev->config.cayman.mem_row_size_in_kb > 4)
 586                 rdev->config.cayman.mem_row_size_in_kb = 4;
 587         /* XXX use MC settings? */
 588         rdev->config.cayman.shader_engine_tile_size = 32;
 589         rdev->config.cayman.num_gpus = 1;
 590         rdev->config.cayman.multi_gpu_tile_size = 64;
 591
 592         tmp = (gb_addr_config & NUM_PIPES_MASK) >> NUM_PIPES_SHIFT;
 593         rdev->config.cayman.num_tile_pipes = (1 << tmp);
 594         tmp = (gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT;
 595         rdev->config.cayman.mem_max_burst_length_bytes = (tmp + 1) * 256;
 596         tmp = (gb_addr_config & NUM_SHADER_ENGINES_MASK) >> NUM_SHADER_ENGINES_SHIFT;
 597         rdev->config.cayman.num_shader_engines = tmp + 1;
 598         tmp = (gb_addr_config & NUM_GPUS_MASK) >> NUM_GPUS_SHIFT;
 599         rdev->config.cayman.num_gpus = tmp + 1;
 600         tmp = (gb_addr_config & MULTI_GPU_TILE_SIZE_MASK) >> MULTI_GPU_TILE_SIZE_SHIFT;
 601         rdev->config.cayman.multi_gpu_tile_size = 1 << tmp;
 602         tmp = (gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT;
 603         rdev->config.cayman.mem_row_size_in_kb = 1 << tmp;
 604
 605
 606         /* setup tiling info dword.  gb_addr_config is not adequate since it does
 607          * not have bank info, so create a custom tiling dword.
 608          * bits 3:0   num_pipes
 609          * bits 7:4   num_banks
 610          * bits 11:8  group_size
 611          * bits 15:12 row_size
 612          */
 613         rdev->config.cayman.tile_config = 0;
 614         switch (rdev->config.cayman.num_tile_pipes) {
 615         case 1:
 616         default:
 617                 rdev->config.cayman.tile_config |= (0 << 0);
 618                 break;
 619         case 2:
 620                 rdev->config.cayman.tile_config |= (1 << 0);
 621                 break;
 622         case 4:
 623                 rdev->config.cayman.tile_config |= (2 << 0);
 624                 break;
 625         case 8:
 626                 rdev->config.cayman.tile_config |= (3 << 0);
 627                 break;
 628         }
 629
 630         /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
 631         if (rdev->flags & RADEON_IS_IGP)
 632                 rdev->config.cayman.tile_config |= 1 << 4;
 633         else {
 634                 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
 635                 case 0: /* four banks */
 636                         rdev->config.cayman.tile_config |= 0 << 4;
 637                         break;
 638                 case 1: /* eight banks */
 639                         rdev->config.cayman.tile_config |= 1 << 4;
 640                         break;
 641                 case 2: /* sixteen banks */
 642                 default:
 643                         rdev->config.cayman.tile_config |= 2 << 4;
 644                         break;
 645                 }
 646         }
 647         rdev->config.cayman.tile_config |=
 648                 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
 649         rdev->config.cayman.tile_config |=
 650                 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
 651
 652         tmp = 0;
 653         for (i = (rdev->config.cayman.max_shader_engines - 1); i >= 0; i--) {
 654                 u32 rb_disable_bitmap;
 655
 656                 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
 657                 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
 658                 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
 659                 tmp <<= 4;
 660                 tmp |= rb_disable_bitmap;
 661         }
 662         /* enabled rb are just the one not disabled :) */
 663         disabled_rb_mask = tmp;
 664
 665         WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
 666         WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
 667
 668         WREG32(GB_ADDR_CONFIG, gb_addr_config);
 669         WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
 670         if (ASIC_IS_DCE6(rdev))
 671                 WREG32(DMIF_ADDR_CALC, gb_addr_config);
 672         WREG32(HDP_ADDR_CONFIG, gb_addr_config);
 673         WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
 674         WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
 675
 676         if ((rdev->config.cayman.max_backends_per_se == 1) &&
 677             (rdev->flags & RADEON_IS_IGP)) {
 678                 if ((disabled_rb_mask & 3) == 1) {
 679                         /* RB0 disabled, RB1 enabled */
 680                         tmp = 0x11111111;
 681                 } else {
 682                         /* RB1 disabled, RB0 enabled */
 683                         tmp = 0x00000000;
 684                 }
 685         } else {
 686                 tmp = gb_addr_config & NUM_PIPES_MASK;
 687                 tmp = r6xx_remap_render_backend(rdev, tmp,
 688                                                 rdev->config.cayman.max_backends_per_se *
 689                                                 rdev->config.cayman.max_shader_engines,
 690                                                 CAYMAN_MAX_BACKENDS, disabled_rb_mask);
 691         }
 692         WREG32(GB_BACKEND_MAP, tmp);
 693
 694         cgts_tcc_disable = 0xffff0000;
 695         for (i = 0; i < rdev->config.cayman.max_texture_channel_caches; i++)
 696                 cgts_tcc_disable &= ~(1 << (16 + i));
 697         WREG32(CGTS_TCC_DISABLE, cgts_tcc_disable);
 698         WREG32(CGTS_SYS_TCC_DISABLE, cgts_tcc_disable);
 699         WREG32(CGTS_USER_SYS_TCC_DISABLE, cgts_tcc_disable);
 700         WREG32(CGTS_USER_TCC_DISABLE, cgts_tcc_disable);
 701
 702         /* reprogram the shader complex */
 703         cgts_sm_ctrl_reg = RREG32(CGTS_SM_CTRL_REG);
 704         for (i = 0; i < 16; i++)
 705                 WREG32(CGTS_SM_CTRL_REG, OVERRIDE);
 706         WREG32(CGTS_SM_CTRL_REG, cgts_sm_ctrl_reg);
 707
 708         /* set HW defaults for 3D engine */
 709         WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
 710
 711         sx_debug_1 = RREG32(SX_DEBUG_1);
 712         sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
 713         WREG32(SX_DEBUG_1, sx_debug_1);
 714
 715         smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
 716         smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
 717         smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.cayman.sx_num_of_sets);
 718         WREG32(SMX_DC_CTL0, smx_dc_ctl0);
 719
 720         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4) | CRC_SIMD_ID_WADDR_DISABLE);
 721
 722         /* need to be explicitly zero-ed */
 723         WREG32(VGT_OFFCHIP_LDS_BASE, 0);
 724         WREG32(SQ_LSTMP_RING_BASE, 0);
 725         WREG32(SQ_HSTMP_RING_BASE, 0);
 726         WREG32(SQ_ESTMP_RING_BASE, 0);
 727         WREG32(SQ_GSTMP_RING_BASE, 0);
 728         WREG32(SQ_VSTMP_RING_BASE, 0);
 729         WREG32(SQ_PSTMP_RING_BASE, 0);
 730
 731         WREG32(TA_CNTL_AUX, DISABLE_CUBE_ANISO);
 732
 733         WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.cayman.sx_max_export_size / 4) - 1) |
 734                                         POSITION_BUFFER_SIZE((rdev->config.cayman.sx_max_export_pos_size / 4) - 1) |
 735                                         SMX_BUFFER_SIZE((rdev->config.cayman.sx_max_export_smx_size / 4) - 1)));
 736
 737         WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.cayman.sc_prim_fifo_size) |
 738                                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_hiz_tile_fifo_size) |
 739                                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_earlyz_tile_fifo_size)));
 740
 741
 742         WREG32(VGT_NUM_INSTANCES, 1);
 743
 744         WREG32(CP_PERFMON_CNTL, 0);
 745
 746         WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.cayman.sq_num_cf_insts) |
 747                                   FETCH_FIFO_HIWATER(0x4) |
 748                                   DONE_FIFO_HIWATER(0xe0) |
 749                                   ALU_UPDATE_FIFO_HIWATER(0x8)));
 750
 751         WREG32(SQ_GPR_RESOURCE_MGMT_1, NUM_CLAUSE_TEMP_GPRS(4));
 752         WREG32(SQ_CONFIG, (VC_ENABLE |
 753                            EXPORT_SRC_C |
 754                            GFX_PRIO(0) |
 755                            CS1_PRIO(0) |
 756                            CS2_PRIO(1)));
 757         WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, DYN_GPR_ENABLE);
 758
 759         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
 760                                           FORCE_EOV_MAX_REZ_CNT(255)));
 761
 762         WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
 763                AUTO_INVLD_EN(ES_AND_GS_AUTO));
 764
 765         WREG32(VGT_GS_VERTEX_REUSE, 16);
 766         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
 767
 768         WREG32(CB_PERF_CTR0_SEL_0, 0);
 769         WREG32(CB_PERF_CTR0_SEL_1, 0);
 770         WREG32(CB_PERF_CTR1_SEL_0, 0);
 771         WREG32(CB_PERF_CTR1_SEL_1, 0);
 772         WREG32(CB_PERF_CTR2_SEL_0, 0);
 773         WREG32(CB_PERF_CTR2_SEL_1, 0);
 774         WREG32(CB_PERF_CTR3_SEL_0, 0);
 775         WREG32(CB_PERF_CTR3_SEL_1, 0);
 776
 777         tmp = RREG32(HDP_MISC_CNTL);
 778         tmp |= HDP_FLUSH_INVALIDATE_CACHE;
 779         WREG32(HDP_MISC_CNTL, tmp);
 780
 781         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
 782         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
 783
 784         WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
 785
 786         udelay(50);
 787 }
 788
 789 /*
 790  * GART
 791  */
 792 void cayman_pcie_gart_tlb_flush(struct radeon_device *rdev)
 793 {
 794         /* flush hdp cache */
 795         WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
 796
 797         /* bits 0-7 are the VM contexts0-7 */
 798         WREG32(VM_INVALIDATE_REQUEST, 1);
 799 }
 800
 801 static int cayman_pcie_gart_enable(struct radeon_device *rdev)
 802 {
 803         int i, r;
 804
 805         if (rdev->gart.robj == NULL) {
 806                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
 807                 return -EINVAL;
 808         }
 809         r = radeon_gart_table_vram_pin(rdev);
 810         if (r)
 811                 return r;
 812         radeon_gart_restore(rdev);
 813         /* Setup TLB control */
 814         WREG32(MC_VM_MX_L1_TLB_CNTL,
 815                (0xA << 7) |
 816                ENABLE_L1_TLB |
 817                ENABLE_L1_FRAGMENT_PROCESSING |
 818                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
 819                ENABLE_ADVANCED_DRIVER_MODEL |
 820                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
 821         /* Setup L2 cache */
 822         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
 823                ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
 824                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
 825                EFFECTIVE_L2_QUEUE_SIZE(7) |
 826                CONTEXT1_IDENTITY_ACCESS_MODE(1));
 827         WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
 828         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
 829                L2_CACHE_BIGK_FRAGMENT_SIZE(6));
 830         /* setup context0 */
 831         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
 832         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
 833         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
 834         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
 835                         (u32)(rdev->dummy_page.addr >> 12));
 836         WREG32(VM_CONTEXT0_CNTL2, 0);
 837         WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
 838                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
 839
 840         WREG32(0x15D4, 0);
 841         WREG32(0x15D8, 0);
 842         WREG32(0x15DC, 0);
 843
 844         /* empty context1-7 */
 845         /* Assign the pt base to something valid for now; the pts used for
 846          * the VMs are determined by the application and setup and assigned
 847          * on the fly in the vm part of radeon_gart.c
 848          */
 849         for (i = 1; i < 8; i++) {
 850                 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR + (i << 2), 0);
 851                 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR + (i << 2), rdev->vm_manager.max_pfn);
 852                 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
 853                         rdev->gart.table_addr >> 12);
 854         }
 855
 856         /* enable context1-7 */
 857         WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
 858                (u32)(rdev->dummy_page.addr >> 12));
 859         WREG32(VM_CONTEXT1_CNTL2, 4);
 860         WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
 861                                 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
 862                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
 863                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
 864                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
 865                                 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
 866                                 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
 867                                 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
 868                                 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
 869                                 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
 870                                 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
 871                                 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
 872                                 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
 873
 874         cayman_pcie_gart_tlb_flush(rdev);
 875         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
 876                  (unsigned)(rdev->mc.gtt_size >> 20),
 877                  (unsigned long long)rdev->gart.table_addr);
 878         rdev->gart.ready = true;
 879         return 0;
 880 }
 881
 882 static void cayman_pcie_gart_disable(struct radeon_device *rdev)
 883 {
 884         /* Disable all tables */
 885         WREG32(VM_CONTEXT0_CNTL, 0);
 886         WREG32(VM_CONTEXT1_CNTL, 0);
 887         /* Setup TLB control */
 888         WREG32(MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_FRAGMENT_PROCESSING |
 889                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
 890                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
 891         /* Setup L2 cache */
 892         WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
 893                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
 894                EFFECTIVE_L2_QUEUE_SIZE(7) |
 895                CONTEXT1_IDENTITY_ACCESS_MODE(1));
 896         WREG32(VM_L2_CNTL2, 0);
 897         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
 898                L2_CACHE_BIGK_FRAGMENT_SIZE(6));
 899         radeon_gart_table_vram_unpin(rdev);
 900 }
 901
 902 static void cayman_pcie_gart_fini(struct radeon_device *rdev)
 903 {
 904         cayman_pcie_gart_disable(rdev);
 905         radeon_gart_table_vram_free(rdev);
 906         radeon_gart_fini(rdev);
 907 }
 908
 909 void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
 910                               int ring, u32 cp_int_cntl)
 911 {
 912         u32 srbm_gfx_cntl = RREG32(SRBM_GFX_CNTL) & ~3;
 913
 914         WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl | (ring & 3));
 915         WREG32(CP_INT_CNTL, cp_int_cntl);
 916 }
 917
 918 /*
 919  * CP.
 920  */
 921 void cayman_fence_ring_emit(struct radeon_device *rdev,
 922                             struct radeon_fence *fence)
 923 {
 924         struct radeon_ring *ring = &rdev->ring[fence->ring];
 925         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
 926
 927         /* flush read cache over gart for this vmid */
 928         radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
 929         radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
 930         radeon_ring_write(ring, 0);
 931         radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
 932         radeon_ring_write(ring, PACKET3_TC_ACTION_ENA | PACKET3_SH_ACTION_ENA);
 933         radeon_ring_write(ring, 0xFFFFFFFF);
 934         radeon_ring_write(ring, 0);
 935         radeon_ring_write(ring, 10); /* poll interval */
 936         /* EVENT_WRITE_EOP - flush caches, send int */
 937         radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
 938         radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT_TS) | EVENT_INDEX(5));
 939         radeon_ring_write(ring, addr & 0xffffffff);
 940         radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
 941         radeon_ring_write(ring, fence->seq);
 942         radeon_ring_write(ring, 0);
 943 }
 944
 945 void cayman_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
 946 {
 947         struct radeon_ring *ring = &rdev->ring[ib->ring];
 948
 949         /* set to DX10/11 mode */
 950         radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
 951         radeon_ring_write(ring, 1);
 952
 953         if (ring->rptr_save_reg) {
 954                 uint32_t next_rptr = ring->wptr + 3 + 4 + 8;
 955                 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
 956                 radeon_ring_write(ring, ((ring->rptr_save_reg -
 957                                           PACKET3_SET_CONFIG_REG_START) >> 2));
 958                 radeon_ring_write(ring, next_rptr);
 959         }
 960
 961         radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
 962         radeon_ring_write(ring,
 963 #ifdef __BIG_ENDIAN
 964                           (2 << 0) |
 965 #endif
 966                           (ib->gpu_addr & 0xFFFFFFFC));
 967         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
 968         radeon_ring_write(ring, ib->length_dw |
 969                           (ib->vm ? (ib->vm->id << 24) : 0));
 970
 971         /* flush read cache over gart for this vmid */
 972         radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
 973         radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
 974         radeon_ring_write(ring, ib->vm ? ib->vm->id : 0);
 975         radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
 976         radeon_ring_write(ring, PACKET3_TC_ACTION_ENA | PACKET3_SH_ACTION_ENA);
 977         radeon_ring_write(ring, 0xFFFFFFFF);
 978         radeon_ring_write(ring, 0);
 979         radeon_ring_write(ring, 10); /* poll interval */
 980 }
 981
 982 static void cayman_cp_enable(struct radeon_device *rdev, bool enable)
 983 {
 984         if (enable)
 985                 WREG32(CP_ME_CNTL, 0);
 986         else {
 987                 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
 988                 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT));
 989                 WREG32(SCRATCH_UMSK, 0);
 990                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
 991         }
 992 }
 993
 994 static int cayman_cp_load_microcode(struct radeon_device *rdev)
 995 {
 996         const __be32 *fw_data;
 997         int i;
 998
 999         if (!rdev->me_fw || !rdev->pfp_fw)
1000                 return -EINVAL;
1001
1002         cayman_cp_enable(rdev, false);
1003
1004         fw_data = (const __be32 *)rdev->pfp_fw->data;
1005         WREG32(CP_PFP_UCODE_ADDR, 0);
1006         for (i = 0; i < CAYMAN_PFP_UCODE_SIZE; i++)
1007                 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
1008         WREG32(CP_PFP_UCODE_ADDR, 0);
1009
1010         fw_data = (const __be32 *)rdev->me_fw->data;
1011         WREG32(CP_ME_RAM_WADDR, 0);
1012         for (i = 0; i < CAYMAN_PM4_UCODE_SIZE; i++)
1013                 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
1014
1015         WREG32(CP_PFP_UCODE_ADDR, 0);
1016         WREG32(CP_ME_RAM_WADDR, 0);
1017         WREG32(CP_ME_RAM_RADDR, 0);
1018         return 0;
1019 }
1020
1021 static int cayman_cp_start(struct radeon_device *rdev)
1022 {
1023         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1024         int r, i;
1025
1026         r = radeon_ring_lock(rdev, ring, 7);
1027         if (r) {
1028                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1029                 return r;
1030         }
1031         radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
1032         radeon_ring_write(ring, 0x1);
1033         radeon_ring_write(ring, 0x0);
1034         radeon_ring_write(ring, rdev->config.cayman.max_hw_contexts - 1);
1035         radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
1036         radeon_ring_write(ring, 0);
1037         radeon_ring_write(ring, 0);
1038         radeon_ring_unlock_commit(rdev, ring);
1039
1040         cayman_cp_enable(rdev, true);
1041
1042         r = radeon_ring_lock(rdev, ring, cayman_default_size + 19);
1043         if (r) {
1044                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1045                 return r;
1046         }
1047
1048         /* setup clear context state */
1049         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1050         radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1051
1052         for (i = 0; i < cayman_default_size; i++)
1053                 radeon_ring_write(ring, cayman_default_state[i]);
1054
1055         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1056         radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
1057
1058         /* set clear context state */
1059         radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
1060         radeon_ring_write(ring, 0);
1061
1062         /* SQ_VTX_BASE_VTX_LOC */
1063         radeon_ring_write(ring, 0xc0026f00);
1064         radeon_ring_write(ring, 0x00000000);
1065         radeon_ring_write(ring, 0x00000000);
1066         radeon_ring_write(ring, 0x00000000);
1067
1068         /* Clear consts */
1069         radeon_ring_write(ring, 0xc0036f00);
1070         radeon_ring_write(ring, 0x00000bc4);
1071         radeon_ring_write(ring, 0xffffffff);
1072         radeon_ring_write(ring, 0xffffffff);
1073         radeon_ring_write(ring, 0xffffffff);
1074
1075         radeon_ring_write(ring, 0xc0026900);
1076         radeon_ring_write(ring, 0x00000316);
1077         radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
1078         radeon_ring_write(ring, 0x00000010); /*  */
1079
1080         radeon_ring_unlock_commit(rdev, ring);
1081
1082         /* XXX init other rings */
1083
1084         return 0;
1085 }
1086
1087 static void cayman_cp_fini(struct radeon_device *rdev)
1088 {
1089         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1090         cayman_cp_enable(rdev, false);
1091         radeon_ring_fini(rdev, ring);
1092         radeon_scratch_free(rdev, ring->rptr_save_reg);
1093 }
1094
1095 static int cayman_cp_resume(struct radeon_device *rdev)
1096 {
1097         static const int ridx[] = {
1098                 RADEON_RING_TYPE_GFX_INDEX,
1099                 CAYMAN_RING_TYPE_CP1_INDEX,
1100                 CAYMAN_RING_TYPE_CP2_INDEX
1101         };
1102         static const unsigned cp_rb_cntl[] = {
1103                 CP_RB0_CNTL,
1104                 CP_RB1_CNTL,
1105                 CP_RB2_CNTL,
1106         };
1107         static const unsigned cp_rb_rptr_addr[] = {
1108                 CP_RB0_RPTR_ADDR,
1109                 CP_RB1_RPTR_ADDR,
1110                 CP_RB2_RPTR_ADDR
1111         };
1112         static const unsigned cp_rb_rptr_addr_hi[] = {
1113                 CP_RB0_RPTR_ADDR_HI,
1114                 CP_RB1_RPTR_ADDR_HI,
1115                 CP_RB2_RPTR_ADDR_HI
1116         };
1117         static const unsigned cp_rb_base[] = {
1118                 CP_RB0_BASE,
1119                 CP_RB1_BASE,
1120                 CP_RB2_BASE
1121         };
1122         struct radeon_ring *ring;
1123         int i, r;
1124
1125         /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
1126         WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
1127                                  SOFT_RESET_PA |
1128                                  SOFT_RESET_SH |
1129                                  SOFT_RESET_VGT |
1130                                  SOFT_RESET_SPI |
1131                                  SOFT_RESET_SX));
1132         RREG32(GRBM_SOFT_RESET);
1133         mdelay(15);
1134         WREG32(GRBM_SOFT_RESET, 0);
1135         RREG32(GRBM_SOFT_RESET);
1136
1137         WREG32(CP_SEM_WAIT_TIMER, 0x0);
1138         WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
1139
1140         /* Set the write pointer delay */
1141         WREG32(CP_RB_WPTR_DELAY, 0);
1142
1143         WREG32(CP_DEBUG, (1 << 27));
1144
1145         /* set the wb address whether it's enabled or not */
1146         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
1147         WREG32(SCRATCH_UMSK, 0xff);
1148
1149         for (i = 0; i < 3; ++i) {
1150                 uint32_t rb_cntl;
1151                 uint64_t addr;
1152
1153                 /* Set ring buffer size */
1154                 ring = &rdev->ring[ridx[i]];
1155                 rb_cntl = drm_order(ring->ring_size / 8);
1156                 rb_cntl |= drm_order(RADEON_GPU_PAGE_SIZE/8) << 8;
1157 #ifdef __BIG_ENDIAN
1158                 rb_cntl |= BUF_SWAP_32BIT;
1159 #endif
1160                 WREG32(cp_rb_cntl[i], rb_cntl);
1161
1162                 /* set the wb address whether it's enabled or not */
1163                 addr = rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET;
1164                 WREG32(cp_rb_rptr_addr[i], addr & 0xFFFFFFFC);
1165                 WREG32(cp_rb_rptr_addr_hi[i], upper_32_bits(addr) & 0xFF);
1166         }
1167
1168         /* set the rb base addr, this causes an internal reset of ALL rings */
1169         for (i = 0; i < 3; ++i) {
1170                 ring = &rdev->ring[ridx[i]];
1171                 WREG32(cp_rb_base[i], ring->gpu_addr >> 8);
1172         }
1173
1174         for (i = 0; i < 3; ++i) {
1175                 /* Initialize the ring buffer's read and write pointers */
1176                 ring = &rdev->ring[ridx[i]];
1177                 WREG32_P(cp_rb_cntl[i], RB_RPTR_WR_ENA, ~RB_RPTR_WR_ENA);
1178
1179                 ring->rptr = ring->wptr = 0;
1180                 WREG32(ring->rptr_reg, ring->rptr);
1181                 WREG32(ring->wptr_reg, ring->wptr);
1182
1183                 mdelay(1);
1184                 WREG32_P(cp_rb_cntl[i], 0, ~RB_RPTR_WR_ENA);
1185         }
1186
1187         /* start the rings */
1188         cayman_cp_start(rdev);
1189         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
1190         rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
1191         rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
1192         /* this only test cp0 */
1193         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
1194         if (r) {
1195                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1196                 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
1197                 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
1198                 return r;
1199         }
1200
1201         return 0;
1202 }
1203
1204 /*
1205  * DMA
1206  * Starting with R600, the GPU has an asynchronous
1207  * DMA engine.  The programming model is very similar
1208  * to the 3D engine (ring buffer, IBs, etc.), but the
1209  * DMA controller has it's own packet format that is
1210  * different form the PM4 format used by the 3D engine.
1211  * It supports copying data, writing embedded data,
1212  * solid fills, and a number of other things.  It also
1213  * has support for tiling/detiling of buffers.
1214  * Cayman and newer support two asynchronous DMA engines.
1215  */
1216 /**
1217  * cayman_dma_ring_ib_execute - Schedule an IB on the DMA engine
1218  *
1219  * @rdev: radeon_device pointer
1220  * @ib: IB object to schedule
1221  *
1222  * Schedule an IB in the DMA ring (cayman-SI).
1223  */
1224 void cayman_dma_ring_ib_execute(struct radeon_device *rdev,
1225                                 struct radeon_ib *ib)
1226 {
1227         struct radeon_ring *ring = &rdev->ring[ib->ring];
1228
1229         if (rdev->wb.enabled) {
1230                 u32 next_rptr = ring->wptr + 4;
1231                 while ((next_rptr & 7) != 5)
1232                         next_rptr++;
1233                 next_rptr += 3;
1234                 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 1));
1235                 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
1236                 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xff);
1237                 radeon_ring_write(ring, next_rptr);
1238         }
1239
1240         /* The indirect buffer packet must end on an 8 DW boundary in the DMA ring.
1241          * Pad as necessary with NOPs.
1242          */
1243         while ((ring->wptr & 7) != 5)
1244                 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
1245         radeon_ring_write(ring, DMA_IB_PACKET(DMA_PACKET_INDIRECT_BUFFER, ib->vm ? ib->vm->id : 0, 0));
1246         radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFE0));
1247         radeon_ring_write(ring, (ib->length_dw << 12) | (upper_32_bits(ib->gpu_addr) & 0xFF));
1248
1249 }
1250
1251 /**
1252  * cayman_dma_stop - stop the async dma engines
1253  *
1254  * @rdev: radeon_device pointer
1255  *
1256  * Stop the async dma engines (cayman-SI).
1257  */
1258 void cayman_dma_stop(struct radeon_device *rdev)
1259 {
1260         u32 rb_cntl;
1261
1262         radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
1263
1264         /* dma0 */
1265         rb_cntl = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
1266         rb_cntl &= ~DMA_RB_ENABLE;
1267         WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, rb_cntl);
1268
1269         /* dma1 */
1270         rb_cntl = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
1271         rb_cntl &= ~DMA_RB_ENABLE;
1272         WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, rb_cntl);
1273
1274         rdev->ring[R600_RING_TYPE_DMA_INDEX].ready = false;
1275         rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX].ready = false;
1276 }
1277
1278 /**
1279  * cayman_dma_resume - setup and start the async dma engines
1280  *
1281  * @rdev: radeon_device pointer
1282  *
1283  * Set up the DMA ring buffers and enable them. (cayman-SI).
1284  * Returns 0 for success, error for failure.
1285  */
1286 int cayman_dma_resume(struct radeon_device *rdev)
1287 {
1288         struct radeon_ring *ring;
1289         u32 rb_cntl, dma_cntl, ib_cntl;
1290         u32 rb_bufsz;
1291         u32 reg_offset, wb_offset;
1292         int i, r;
1293
1294         /* Reset dma */
1295         WREG32(SRBM_SOFT_RESET, SOFT_RESET_DMA | SOFT_RESET_DMA1);
1296         RREG32(SRBM_SOFT_RESET);
1297         udelay(50);
1298         WREG32(SRBM_SOFT_RESET, 0);
1299
1300         for (i = 0; i < 2; i++) {
1301                 if (i == 0) {
1302                         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
1303                         reg_offset = DMA0_REGISTER_OFFSET;
1304                         wb_offset = R600_WB_DMA_RPTR_OFFSET;
1305                 } else {
1306                         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
1307                         reg_offset = DMA1_REGISTER_OFFSET;
1308                         wb_offset = CAYMAN_WB_DMA1_RPTR_OFFSET;
1309                 }
1310
1311                 WREG32(DMA_SEM_INCOMPLETE_TIMER_CNTL + reg_offset, 0);
1312                 WREG32(DMA_SEM_WAIT_FAIL_TIMER_CNTL + reg_offset, 0);
1313
1314                 /* Set ring buffer size in dwords */
1315                 rb_bufsz = drm_order(ring->ring_size / 4);
1316                 rb_cntl = rb_bufsz << 1;
1317 #ifdef __BIG_ENDIAN
1318                 rb_cntl |= DMA_RB_SWAP_ENABLE | DMA_RPTR_WRITEBACK_SWAP_ENABLE;
1319 #endif
1320                 WREG32(DMA_RB_CNTL + reg_offset, rb_cntl);
1321
1322                 /* Initialize the ring buffer's read and write pointers */
1323                 WREG32(DMA_RB_RPTR + reg_offset, 0);
1324                 WREG32(DMA_RB_WPTR + reg_offset, 0);
1325
1326                 /* set the wb address whether it's enabled or not */
1327                 WREG32(DMA_RB_RPTR_ADDR_HI + reg_offset,
1328                        upper_32_bits(rdev->wb.gpu_addr + wb_offset) & 0xFF);
1329                 WREG32(DMA_RB_RPTR_ADDR_LO + reg_offset,
1330                        ((rdev->wb.gpu_addr + wb_offset) & 0xFFFFFFFC));
1331
1332                 if (rdev->wb.enabled)
1333                         rb_cntl |= DMA_RPTR_WRITEBACK_ENABLE;
1334
1335                 WREG32(DMA_RB_BASE + reg_offset, ring->gpu_addr >> 8);
1336
1337                 /* enable DMA IBs */
1338                 ib_cntl = DMA_IB_ENABLE | CMD_VMID_FORCE;
1339 #ifdef __BIG_ENDIAN
1340                 ib_cntl |= DMA_IB_SWAP_ENABLE;
1341 #endif
1342                 WREG32(DMA_IB_CNTL + reg_offset, ib_cntl);
1343
1344                 dma_cntl = RREG32(DMA_CNTL + reg_offset);
1345                 dma_cntl &= ~CTXEMPTY_INT_ENABLE;
1346                 WREG32(DMA_CNTL + reg_offset, dma_cntl);
1347
1348                 ring->wptr = 0;
1349                 WREG32(DMA_RB_WPTR + reg_offset, ring->wptr << 2);
1350
1351                 ring->rptr = RREG32(DMA_RB_RPTR + reg_offset) >> 2;
1352
1353                 WREG32(DMA_RB_CNTL + reg_offset, rb_cntl | DMA_RB_ENABLE);
1354
1355                 ring->ready = true;
1356
1357                 r = radeon_ring_test(rdev, ring->idx, ring);
1358                 if (r) {
1359                         ring->ready = false;
1360                         return r;
1361                 }
1362         }
1363
1364         radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
1365
1366         return 0;
1367 }
1368
1369 /**
1370  * cayman_dma_fini - tear down the async dma engines
1371  *
1372  * @rdev: radeon_device pointer
1373  *
1374  * Stop the async dma engines and free the rings (cayman-SI).
1375  */
1376 void cayman_dma_fini(struct radeon_device *rdev)
1377 {
1378         cayman_dma_stop(rdev);
1379         radeon_ring_fini(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX]);
1380         radeon_ring_fini(rdev, &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]);
1381 }
1382
1383 static void cayman_gpu_soft_reset_gfx(struct radeon_device *rdev)
1384 {
1385         u32 grbm_reset = 0;
1386
1387         if (!(RREG32(GRBM_STATUS) & GUI_ACTIVE))
1388                 return;
1389
1390         dev_info(rdev->dev, "  GRBM_STATUS               = 0x%08X\n",
1391                 RREG32(GRBM_STATUS));
1392         dev_info(rdev->dev, "  GRBM_STATUS_SE0           = 0x%08X\n",
1393                 RREG32(GRBM_STATUS_SE0));
1394         dev_info(rdev->dev, "  GRBM_STATUS_SE1           = 0x%08X\n",
1395                 RREG32(GRBM_STATUS_SE1));
1396         dev_info(rdev->dev, "  SRBM_STATUS               = 0x%08X\n",
1397                 RREG32(SRBM_STATUS));
1398         dev_info(rdev->dev, "  R_008674_CP_STALLED_STAT1 = 0x%08X\n",
1399                 RREG32(CP_STALLED_STAT1));
1400         dev_info(rdev->dev, "  R_008678_CP_STALLED_STAT2 = 0x%08X\n",
1401                 RREG32(CP_STALLED_STAT2));
1402         dev_info(rdev->dev, "  R_00867C_CP_BUSY_STAT     = 0x%08X\n",
1403                 RREG32(CP_BUSY_STAT));
1404         dev_info(rdev->dev, "  R_008680_CP_STAT          = 0x%08X\n",
1405                 RREG32(CP_STAT));
1406
1407         /* Disable CP parsing/prefetching */
1408         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
1409
1410         /* reset all the gfx blocks */
1411         grbm_reset = (SOFT_RESET_CP |
1412                       SOFT_RESET_CB |
1413                       SOFT_RESET_DB |
1414                       SOFT_RESET_GDS |
1415                       SOFT_RESET_PA |
1416                       SOFT_RESET_SC |
1417                       SOFT_RESET_SPI |
1418                       SOFT_RESET_SH |
1419                       SOFT_RESET_SX |
1420                       SOFT_RESET_TC |
1421                       SOFT_RESET_TA |
1422                       SOFT_RESET_VGT |
1423                       SOFT_RESET_IA);
1424
1425         dev_info(rdev->dev, "  GRBM_SOFT_RESET=0x%08X\n", grbm_reset);
1426         WREG32(GRBM_SOFT_RESET, grbm_reset);
1427         (void)RREG32(GRBM_SOFT_RESET);
1428         udelay(50);
1429         WREG32(GRBM_SOFT_RESET, 0);
1430         (void)RREG32(GRBM_SOFT_RESET);
1431
1432         dev_info(rdev->dev, "  GRBM_STATUS               = 0x%08X\n",
1433                 RREG32(GRBM_STATUS));
1434         dev_info(rdev->dev, "  GRBM_STATUS_SE0           = 0x%08X\n",
1435                 RREG32(GRBM_STATUS_SE0));
1436         dev_info(rdev->dev, "  GRBM_STATUS_SE1           = 0x%08X\n",
1437                 RREG32(GRBM_STATUS_SE1));
1438         dev_info(rdev->dev, "  SRBM_STATUS               = 0x%08X\n",
1439                 RREG32(SRBM_STATUS));
1440         dev_info(rdev->dev, "  R_008674_CP_STALLED_STAT1 = 0x%08X\n",
1441                 RREG32(CP_STALLED_STAT1));
1442         dev_info(rdev->dev, "  R_008678_CP_STALLED_STAT2 = 0x%08X\n",
1443                 RREG32(CP_STALLED_STAT2));
1444         dev_info(rdev->dev, "  R_00867C_CP_BUSY_STAT     = 0x%08X\n",
1445                 RREG32(CP_BUSY_STAT));
1446         dev_info(rdev->dev, "  R_008680_CP_STAT          = 0x%08X\n",
1447                 RREG32(CP_STAT));
1448
1449 }
1450
1451 static void cayman_gpu_soft_reset_dma(struct radeon_device *rdev)
1452 {
1453         u32 tmp;
1454
1455         if (RREG32(DMA_STATUS_REG) & DMA_IDLE)
1456                 return;
1457
1458         dev_info(rdev->dev, "  R_00D034_DMA_STATUS_REG   = 0x%08X\n",
1459                 RREG32(DMA_STATUS_REG));
1460
1461         /* dma0 */
1462         tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
1463         tmp &= ~DMA_RB_ENABLE;
1464         WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
1465
1466         /* dma1 */
1467         tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
1468         tmp &= ~DMA_RB_ENABLE;
1469         WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
1470
1471         /* Reset dma */
1472         WREG32(SRBM_SOFT_RESET, SOFT_RESET_DMA | SOFT_RESET_DMA1);
1473         RREG32(SRBM_SOFT_RESET);
1474         udelay(50);
1475         WREG32(SRBM_SOFT_RESET, 0);
1476
1477         dev_info(rdev->dev, "  R_00D034_DMA_STATUS_REG   = 0x%08X\n",
1478                 RREG32(DMA_STATUS_REG));
1479
1480 }
1481
1482 static int cayman_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
1483 {
1484         struct evergreen_mc_save save;
1485
1486         if (!(RREG32(GRBM_STATUS) & GUI_ACTIVE))
1487                 reset_mask &= ~(RADEON_RESET_GFX | RADEON_RESET_COMPUTE);
1488
1489         if (RREG32(DMA_STATUS_REG) & DMA_IDLE)
1490                 reset_mask &= ~RADEON_RESET_DMA;
1491
1492         if (reset_mask == 0)
1493                 return 0;
1494
1495         dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
1496
1497         dev_info(rdev->dev, "  VM_CONTEXT0_PROTECTION_FAULT_ADDR   0x%08X\n",
1498                  RREG32(0x14F8));
1499         dev_info(rdev->dev, "  VM_CONTEXT0_PROTECTION_FAULT_STATUS 0x%08X\n",
1500                  RREG32(0x14D8));
1501         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
1502                  RREG32(0x14FC));
1503         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
1504                  RREG32(0x14DC));
1505
1506         evergreen_mc_stop(rdev, &save);
1507         if (evergreen_mc_wait_for_idle(rdev)) {
1508                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1509         }
1510
1511         if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE))
1512                 cayman_gpu_soft_reset_gfx(rdev);
1513
1514         if (reset_mask & RADEON_RESET_DMA)
1515                 cayman_gpu_soft_reset_dma(rdev);
1516
1517         /* Wait a little for things to settle down */
1518         udelay(50);
1519
1520         evergreen_mc_resume(rdev, &save);
1521         return 0;
1522 }
1523
1524 int cayman_asic_reset(struct radeon_device *rdev)
1525 {
1526         return cayman_gpu_soft_reset(rdev, (RADEON_RESET_GFX |
1527                                             RADEON_RESET_COMPUTE |
1528                                             RADEON_RESET_DMA));
1529 }
1530
1531 /**
1532  * cayman_dma_is_lockup - Check if the DMA engine is locked up
1533  *
1534  * @rdev: radeon_device pointer
1535  * @ring: radeon_ring structure holding ring information
1536  *
1537  * Check if the async DMA engine is locked up (cayman-SI).
1538  * Returns true if the engine appears to be locked up, false if not.
1539  */
1540 bool cayman_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
1541 {
1542         u32 dma_status_reg;
1543
1544         if (ring->idx == R600_RING_TYPE_DMA_INDEX)
1545                 dma_status_reg = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
1546         else
1547                 dma_status_reg = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
1548         if (dma_status_reg & DMA_IDLE) {
1549                 radeon_ring_lockup_update(ring);
1550                 return false;
1551         }
1552         /* force ring activities */
1553         radeon_ring_force_activity(rdev, ring);
1554         return radeon_ring_test_lockup(rdev, ring);
1555 }
1556
1557 static int cayman_startup(struct radeon_device *rdev)
1558 {
1559         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1560         int r;
1561
1562         /* enable pcie gen2 link */
1563         evergreen_pcie_gen2_enable(rdev);
1564
1565         if (rdev->flags & RADEON_IS_IGP) {
1566                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
1567                         r = ni_init_microcode(rdev);
1568                         if (r) {
1569                                 DRM_ERROR("Failed to load firmware!\n");
1570                                 return r;
1571                         }
1572                 }
1573         } else {
1574                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
1575                         r = ni_init_microcode(rdev);
1576                         if (r) {
1577                                 DRM_ERROR("Failed to load firmware!\n");
1578                                 return r;
1579                         }
1580                 }
1581
1582                 r = ni_mc_load_microcode(rdev);
1583                 if (r) {
1584                         DRM_ERROR("Failed to load MC firmware!\n");
1585                         return r;
1586                 }
1587         }
1588
1589         r = r600_vram_scratch_init(rdev);
1590         if (r)
1591                 return r;
1592
1593         evergreen_mc_program(rdev);
1594         r = cayman_pcie_gart_enable(rdev);
1595         if (r)
1596                 return r;
1597         cayman_gpu_init(rdev);
1598
1599         r = evergreen_blit_init(rdev);
1600         if (r) {
1601                 r600_blit_fini(rdev);
1602                 rdev->asic->copy.copy = NULL;
1603                 dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
1604         }
1605
1606         /* allocate rlc buffers */
1607         if (rdev->flags & RADEON_IS_IGP) {
1608                 r = si_rlc_init(rdev);
1609                 if (r) {
1610                         DRM_ERROR("Failed to init rlc BOs!\n");
1611                         return r;
1612                 }
1613         }
1614
1615         /* allocate wb buffer */
1616         r = radeon_wb_init(rdev);
1617         if (r)
1618                 return r;
1619
1620         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
1621         if (r) {
1622                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
1623                 return r;
1624         }
1625
1626         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
1627         if (r) {
1628                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
1629                 return r;
1630         }
1631
1632         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
1633         if (r) {
1634                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
1635                 return r;
1636         }
1637
1638         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
1639         if (r) {
1640                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
1641                 return r;
1642         }
1643
1644         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
1645         if (r) {
1646                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
1647                 return r;
1648         }
1649
1650         /* Enable IRQ */
1651         r = r600_irq_init(rdev);
1652         if (r) {
1653                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
1654                 radeon_irq_kms_fini(rdev);
1655                 return r;
1656         }
1657         evergreen_irq_set(rdev);
1658
1659         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
1660                              CP_RB0_RPTR, CP_RB0_WPTR,
1661                              0, 0xfffff, RADEON_CP_PACKET2);
1662         if (r)
1663                 return r;
1664
1665         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
1666         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
1667                              DMA_RB_RPTR + DMA0_REGISTER_OFFSET,
1668                              DMA_RB_WPTR + DMA0_REGISTER_OFFSET,
1669                              2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
1670         if (r)
1671                 return r;
1672
1673         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
1674         r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
1675                              DMA_RB_RPTR + DMA1_REGISTER_OFFSET,
1676                              DMA_RB_WPTR + DMA1_REGISTER_OFFSET,
1677                              2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
1678         if (r)
1679                 return r;
1680
1681         r = cayman_cp_load_microcode(rdev);
1682         if (r)
1683                 return r;
1684         r = cayman_cp_resume(rdev);
1685         if (r)
1686                 return r;
1687
1688         r = cayman_dma_resume(rdev);
1689         if (r)
1690                 return r;
1691
1692         r = radeon_ib_pool_init(rdev);
1693         if (r) {
1694                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
1695                 return r;
1696         }
1697
1698         r = radeon_vm_manager_init(rdev);
1699         if (r) {
1700                 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
1701                 return r;
1702         }
1703
1704         r = r600_audio_init(rdev);
1705         if (r)
1706                 return r;
1707
1708         return 0;
1709 }
1710
1711 int cayman_resume(struct radeon_device *rdev)
1712 {
1713         int r;
1714
1715         /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
1716          * posting will perform necessary task to bring back GPU into good
1717          * shape.
1718          */
1719         /* post card */
1720         atom_asic_init(rdev->mode_info.atom_context);
1721
1722         rdev->accel_working = true;
1723         r = cayman_startup(rdev);
1724         if (r) {
1725                 DRM_ERROR("cayman startup failed on resume\n");
1726                 rdev->accel_working = false;
1727                 return r;
1728         }
1729         return r;
1730 }
1731
1732 int cayman_suspend(struct radeon_device *rdev)
1733 {
1734         r600_audio_fini(rdev);
1735         radeon_vm_manager_fini(rdev);
1736         cayman_cp_enable(rdev, false);
1737         cayman_dma_stop(rdev);
1738         evergreen_irq_suspend(rdev);
1739         radeon_wb_disable(rdev);
1740         cayman_pcie_gart_disable(rdev);
1741         return 0;
1742 }
1743
1744 /* Plan is to move initialization in that function and use
1745  * helper function so that radeon_device_init pretty much
1746  * do nothing more than calling asic specific function. This
1747  * should also allow to remove a bunch of callback function
1748  * like vram_info.
1749  */
1750 int cayman_init(struct radeon_device *rdev)
1751 {
1752         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1753         int r;
1754
1755         /* Read BIOS */
1756         if (!radeon_get_bios(rdev)) {
1757                 if (ASIC_IS_AVIVO(rdev))
1758                         return -EINVAL;
1759         }
1760         /* Must be an ATOMBIOS */
1761         if (!rdev->is_atom_bios) {
1762                 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
1763                 return -EINVAL;
1764         }
1765         r = radeon_atombios_init(rdev);
1766         if (r)
1767                 return r;
1768
1769         /* Post card if necessary */
1770         if (!radeon_card_posted(rdev)) {
1771                 if (!rdev->bios) {
1772                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
1773                         return -EINVAL;
1774                 }
1775                 DRM_INFO("GPU not posted. posting now...\n");
1776                 atom_asic_init(rdev->mode_info.atom_context);
1777         }
1778         /* Initialize scratch registers */
1779         r600_scratch_init(rdev);
1780         /* Initialize surface registers */
1781         radeon_surface_init(rdev);
1782         /* Initialize clocks */
1783         radeon_get_clock_info(rdev->ddev);
1784         /* Fence driver */
1785         r = radeon_fence_driver_init(rdev);
1786         if (r)
1787                 return r;
1788         /* initialize memory controller */
1789         r = evergreen_mc_init(rdev);
1790         if (r)
1791                 return r;
1792         /* Memory manager */
1793         r = radeon_bo_init(rdev);
1794         if (r)
1795                 return r;
1796
1797         r = radeon_irq_kms_init(rdev);
1798         if (r)
1799                 return r;
1800
1801         ring->ring_obj = NULL;
1802         r600_ring_init(rdev, ring, 1024 * 1024);
1803
1804         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
1805         ring->ring_obj = NULL;
1806         r600_ring_init(rdev, ring, 64 * 1024);
1807
1808         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
1809         ring->ring_obj = NULL;
1810         r600_ring_init(rdev, ring, 64 * 1024);
1811
1812         rdev->ih.ring_obj = NULL;
1813         r600_ih_ring_init(rdev, 64 * 1024);
1814
1815         r = r600_pcie_gart_init(rdev);
1816         if (r)
1817                 return r;
1818
1819         rdev->accel_working = true;
1820         r = cayman_startup(rdev);
1821         if (r) {
1822                 dev_err(rdev->dev, "disabling GPU acceleration\n");
1823                 cayman_cp_fini(rdev);
1824                 cayman_dma_fini(rdev);
1825                 r600_irq_fini(rdev);
1826                 if (rdev->flags & RADEON_IS_IGP)
1827                         si_rlc_fini(rdev);
1828                 radeon_wb_fini(rdev);
1829                 radeon_ib_pool_fini(rdev);
1830                 radeon_vm_manager_fini(rdev);
1831                 radeon_irq_kms_fini(rdev);
1832                 cayman_pcie_gart_fini(rdev);
1833                 rdev->accel_working = false;
1834         }
1835
1836         /* Don't start up if the MC ucode is missing.
1837          * The default clocks and voltages before the MC ucode
1838          * is loaded are not suffient for advanced operations.
1839          *
1840          * We can skip this check for TN, because there is no MC
1841          * ucode.
1842          */
1843         if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
1844                 DRM_ERROR("radeon: MC ucode required for NI+.\n");
1845                 return -EINVAL;
1846         }
1847
1848         return 0;
1849 }
1850
1851 void cayman_fini(struct radeon_device *rdev)
1852 {
1853         r600_blit_fini(rdev);
1854         cayman_cp_fini(rdev);
1855         cayman_dma_fini(rdev);
1856         r600_irq_fini(rdev);
1857         if (rdev->flags & RADEON_IS_IGP)
1858                 si_rlc_fini(rdev);
1859         radeon_wb_fini(rdev);
1860         radeon_vm_manager_fini(rdev);
1861         radeon_ib_pool_fini(rdev);
1862         radeon_irq_kms_fini(rdev);
1863         cayman_pcie_gart_fini(rdev);
1864         r600_vram_scratch_fini(rdev);
1865         radeon_gem_fini(rdev);
1866         radeon_fence_driver_fini(rdev);
1867         radeon_bo_fini(rdev);
1868         radeon_atombios_fini(rdev);
1869         ni_fini_microcode(rdev);
1870         free(rdev->bios, DRM_MEM_DRIVER);
1871         rdev->bios = NULL;
1872 }
1873
1874 /*
1875  * vm
1876  */
1877 int cayman_vm_init(struct radeon_device *rdev)
1878 {
1879         /* number of VMs */
1880         rdev->vm_manager.nvm = 8;
1881         /* base offset of vram pages */
1882         if (rdev->flags & RADEON_IS_IGP) {
1883                 u64 tmp = RREG32(FUS_MC_VM_FB_OFFSET);
1884                 tmp <<= 22;
1885                 rdev->vm_manager.vram_base_offset = tmp;
1886         } else
1887                 rdev->vm_manager.vram_base_offset = 0;
1888         return 0;
1889 }
1890
1891 void cayman_vm_fini(struct radeon_device *rdev)
1892 {
1893 }
1894
1895 #define R600_ENTRY_VALID   (1 << 0)
1896 #define R600_PTE_SYSTEM    (1 << 1)
1897 #define R600_PTE_SNOOPED   (1 << 2)
1898 #define R600_PTE_READABLE  (1 << 5)
1899 #define R600_PTE_WRITEABLE (1 << 6)
1900
1901 uint32_t cayman_vm_page_flags(struct radeon_device *rdev, uint32_t flags)
1902 {
1903         uint32_t r600_flags = 0;
1904         r600_flags |= (flags & RADEON_VM_PAGE_VALID) ? R600_ENTRY_VALID : 0;
1905         r600_flags |= (flags & RADEON_VM_PAGE_READABLE) ? R600_PTE_READABLE : 0;
1906         r600_flags |= (flags & RADEON_VM_PAGE_WRITEABLE) ? R600_PTE_WRITEABLE : 0;
1907         if (flags & RADEON_VM_PAGE_SYSTEM) {
1908                 r600_flags |= R600_PTE_SYSTEM;
1909                 r600_flags |= (flags & RADEON_VM_PAGE_SNOOPED) ? R600_PTE_SNOOPED : 0;
1910         }
1911         return r600_flags;
1912 }
1913
1914 /**
1915  * cayman_vm_set_page - update the page tables using the CP
1916  *
1917  * @rdev: radeon_device pointer
1918  * @pe: addr of the page entry
1919  * @addr: dst addr to write into pe
1920  * @count: number of page entries to update
1921  * @incr: increase next addr by incr bytes
1922  * @flags: access flags
1923  *
1924  * Update the page tables using the CP (cayman-si).
1925  */
1926 void cayman_vm_set_page(struct radeon_device *rdev, uint64_t pe,
1927                         uint64_t addr, unsigned count,
1928                         uint32_t incr, uint32_t flags)
1929 {
1930         struct radeon_ring *ring = &rdev->ring[rdev->asic->vm.pt_ring_index];
1931         uint32_t r600_flags = cayman_vm_page_flags(rdev, flags);
1932         uint64_t value;
1933         unsigned ndw;
1934
1935         if (rdev->asic->vm.pt_ring_index == RADEON_RING_TYPE_GFX_INDEX) {
1936                 while (count) {
1937                         ndw = 1 + count * 2;
1938                         if (ndw > 0x3FFF)
1939                                 ndw = 0x3FFF;
1940
1941                         radeon_ring_write(ring, PACKET3(PACKET3_ME_WRITE, ndw));
1942                         radeon_ring_write(ring, pe);
1943                         radeon_ring_write(ring, upper_32_bits(pe) & 0xff);
1944                         for (; ndw > 1; ndw -= 2, --count, pe += 8) {
1945                                 if (flags & RADEON_VM_PAGE_SYSTEM) {
1946                                         value = radeon_vm_map_gart(rdev, addr);
1947                                         value &= 0xFFFFFFFFFFFFF000ULL;
1948                                 } else if (flags & RADEON_VM_PAGE_VALID) {
1949                                         value = addr;
1950                                 } else {
1951                                         value = 0;
1952                                 }
1953                                 addr += incr;
1954                                 value |= r600_flags;
1955                                 radeon_ring_write(ring, value);
1956                                 radeon_ring_write(ring, upper_32_bits(value));
1957                         }
1958                 }
1959         } else {
1960                 while (count) {
1961                         ndw = count * 2;
1962                         if (ndw > 0xFFFFE)
1963                                 ndw = 0xFFFFE;
1964
1965                         /* for non-physically contiguous pages (system) */
1966                         radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 0, ndw));
1967                         radeon_ring_write(ring, pe);
1968                         radeon_ring_write(ring, upper_32_bits(pe) & 0xff);
1969                         for (; ndw > 0; ndw -= 2, --count, pe += 8) {
1970                                 if (flags & RADEON_VM_PAGE_SYSTEM) {
1971                                         value = radeon_vm_map_gart(rdev, addr);
1972                                         value &= 0xFFFFFFFFFFFFF000ULL;
1973                                 } else if (flags & RADEON_VM_PAGE_VALID) {
1974                                         value = addr;
1975                                 } else {
1976                                         value = 0;
1977                                 }
1978                                 addr += incr;
1979                                 value |= r600_flags;
1980                                 radeon_ring_write(ring, value);
1981                                 radeon_ring_write(ring, upper_32_bits(value));
1982                         }
1983                 }
1984         }
1985 }
1986
1987 /**
1988  * cayman_vm_flush - vm flush using the CP
1989  *
1990  * @rdev: radeon_device pointer
1991  *
1992  * Update the page table base and flush the VM TLB
1993  * using the CP (cayman-si).
1994  */
1995 void cayman_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
1996 {
1997         struct radeon_ring *ring = &rdev->ring[ridx];
1998
1999         if (vm == NULL)
2000                 return;
2001
2002         radeon_ring_write(ring, PACKET0(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2), 0));
2003         radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
2004
2005         /* flush hdp cache */
2006         radeon_ring_write(ring, PACKET0(HDP_MEM_COHERENCY_FLUSH_CNTL, 0));
2007         radeon_ring_write(ring, 0x1);
2008
2009         /* bits 0-7 are the VM contexts0-7 */
2010         radeon_ring_write(ring, PACKET0(VM_INVALIDATE_REQUEST, 0));
2011         radeon_ring_write(ring, 1 << vm->id);
2012
2013         /* sync PFP to ME, otherwise we might get invalid PFP reads */
2014         radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
2015         radeon_ring_write(ring, 0x0);
2016 }
2017
2018 void cayman_dma_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
2019 {
2020         struct radeon_ring *ring = &rdev->ring[ridx];
2021
2022         if (vm == NULL)
2023                 return;
2024
2025         radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0));
2026         radeon_ring_write(ring, (0xf << 16) | ((VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2));
2027         radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
2028
2029         /* flush hdp cache */
2030         radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0));
2031         radeon_ring_write(ring, (0xf << 16) | (HDP_MEM_COHERENCY_FLUSH_CNTL >> 2));
2032         radeon_ring_write(ring, 1);
2033
2034         /* bits 0-7 are the VM contexts0-7 */
2035         radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0));
2036         radeon_ring_write(ring, (0xf << 16) | (VM_INVALIDATE_REQUEST >> 2));
2037         radeon_ring_write(ring, 1 << vm->id);
2038 }
2039