sys/dev/drm2/radeon/r600.c

   1 /*
   2  * Copyright 2008 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  * Copyright 2009 Jerome Glisse.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22  * OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  * Authors: Dave Airlie
  25  *          Alex Deucher
  26  *          Jerome Glisse
  27  */
  28
  29 #include <sys/cdefs.h>
  30 __FBSDID("$FreeBSD$");
  31
  32 #include <dev/drm2/drmP.h>
  33 #include <dev/drm2/radeon/radeon_drm.h>
  34 #include "radeon.h"
  35 #include "radeon_asic.h"
  36 #include "radeon_mode.h"
  37 #include "r600d.h"
  38 #include "atom.h"
  39 #include "avivod.h"
  40
  41 #define PFP_UCODE_SIZE 576
  42 #define PM4_UCODE_SIZE 1792
  43 #define RLC_UCODE_SIZE 768
  44 #define R700_PFP_UCODE_SIZE 848
  45 #define R700_PM4_UCODE_SIZE 1360
  46 #define R700_RLC_UCODE_SIZE 1024
  47 #define EVERGREEN_PFP_UCODE_SIZE 1120
  48 #define EVERGREEN_PM4_UCODE_SIZE 1376
  49 #define EVERGREEN_RLC_UCODE_SIZE 768
  50 #define CAYMAN_RLC_UCODE_SIZE 1024
  51 #define ARUBA_RLC_UCODE_SIZE 1536
  52
  53 #ifdef __linux__
  54 /* Firmware Names */
  55 MODULE_FIRMWARE("radeon/R600_pfp.bin");
  56 MODULE_FIRMWARE("radeon/R600_me.bin");
  57 MODULE_FIRMWARE("radeon/RV610_pfp.bin");
  58 MODULE_FIRMWARE("radeon/RV610_me.bin");
  59 MODULE_FIRMWARE("radeon/RV630_pfp.bin");
  60 MODULE_FIRMWARE("radeon/RV630_me.bin");
  61 MODULE_FIRMWARE("radeon/RV620_pfp.bin");
  62 MODULE_FIRMWARE("radeon/RV620_me.bin");
  63 MODULE_FIRMWARE("radeon/RV635_pfp.bin");
  64 MODULE_FIRMWARE("radeon/RV635_me.bin");
  65 MODULE_FIRMWARE("radeon/RV670_pfp.bin");
  66 MODULE_FIRMWARE("radeon/RV670_me.bin");
  67 MODULE_FIRMWARE("radeon/RS780_pfp.bin");
  68 MODULE_FIRMWARE("radeon/RS780_me.bin");
  69 MODULE_FIRMWARE("radeon/RV770_pfp.bin");
  70 MODULE_FIRMWARE("radeon/RV770_me.bin");
  71 MODULE_FIRMWARE("radeon/RV730_pfp.bin");
  72 MODULE_FIRMWARE("radeon/RV730_me.bin");
  73 MODULE_FIRMWARE("radeon/RV710_pfp.bin");
  74 MODULE_FIRMWARE("radeon/RV710_me.bin");
  75 MODULE_FIRMWARE("radeon/R600_rlc.bin");
  76 MODULE_FIRMWARE("radeon/R700_rlc.bin");
  77 MODULE_FIRMWARE("radeon/CEDAR_pfp.bin");
  78 MODULE_FIRMWARE("radeon/CEDAR_me.bin");
  79 MODULE_FIRMWARE("radeon/CEDAR_rlc.bin");
  80 MODULE_FIRMWARE("radeon/REDWOOD_pfp.bin");
  81 MODULE_FIRMWARE("radeon/REDWOOD_me.bin");
  82 MODULE_FIRMWARE("radeon/REDWOOD_rlc.bin");
  83 MODULE_FIRMWARE("radeon/JUNIPER_pfp.bin");
  84 MODULE_FIRMWARE("radeon/JUNIPER_me.bin");
  85 MODULE_FIRMWARE("radeon/JUNIPER_rlc.bin");
  86 MODULE_FIRMWARE("radeon/CYPRESS_pfp.bin");
  87 MODULE_FIRMWARE("radeon/CYPRESS_me.bin");
  88 MODULE_FIRMWARE("radeon/CYPRESS_rlc.bin");
  89 MODULE_FIRMWARE("radeon/PALM_pfp.bin");
  90 MODULE_FIRMWARE("radeon/PALM_me.bin");
  91 MODULE_FIRMWARE("radeon/SUMO_rlc.bin");
  92 MODULE_FIRMWARE("radeon/SUMO_pfp.bin");
  93 MODULE_FIRMWARE("radeon/SUMO_me.bin");
  94 MODULE_FIRMWARE("radeon/SUMO2_pfp.bin");
  95 MODULE_FIRMWARE("radeon/SUMO2_me.bin");
  96 #endif
  97
  98 int r600_debugfs_mc_info_init(struct radeon_device *rdev);
  99
 100 /* r600,rv610,rv630,rv620,rv635,rv670 */
 101 #ifdef FREEBSD_WIP /* FreeBSD: to please GCC 4.2. */
 102 int r600_mc_wait_for_idle(struct radeon_device *rdev);
 103 #endif
 104 static void r600_gpu_init(struct radeon_device *rdev);
 105 #ifdef FREEBSD_WIP /* FreeBSD: to please GCC 4.2. */
 106 void r600_fini(struct radeon_device *rdev);
 107 #endif
 108 void r600_irq_disable(struct radeon_device *rdev);
 109 static void r600_pcie_gen2_enable(struct radeon_device *rdev);
 110
 111 /* get temperature in millidegrees */
 112 int rv6xx_get_temp(struct radeon_device *rdev)
 113 {
 114         u32 temp = (RREG32(CG_THERMAL_STATUS) & ASIC_T_MASK) >>
 115                 ASIC_T_SHIFT;
 116         int actual_temp = temp & 0xff;
 117
 118         if (temp & 0x100)
 119                 actual_temp -= 256;
 120
 121         return actual_temp * 1000;
 122 }
 123
 124 void r600_pm_get_dynpm_state(struct radeon_device *rdev)
 125 {
 126         int i;
 127
 128         rdev->pm.dynpm_can_upclock = true;
 129         rdev->pm.dynpm_can_downclock = true;
 130
 131         /* power state array is low to high, default is first */
 132         if ((rdev->flags & RADEON_IS_IGP) || (rdev->family == CHIP_R600)) {
 133                 int min_power_state_index = 0;
 134
 135                 if (rdev->pm.num_power_states > 2)
 136                         min_power_state_index = 1;
 137
 138                 switch (rdev->pm.dynpm_planned_action) {
 139                 case DYNPM_ACTION_MINIMUM:
 140                         rdev->pm.requested_power_state_index = min_power_state_index;
 141                         rdev->pm.requested_clock_mode_index = 0;
 142                         rdev->pm.dynpm_can_downclock = false;
 143                         break;
 144                 case DYNPM_ACTION_DOWNCLOCK:
 145                         if (rdev->pm.current_power_state_index == min_power_state_index) {
 146                                 rdev->pm.requested_power_state_index = rdev->pm.current_power_state_index;
 147                                 rdev->pm.dynpm_can_downclock = false;
 148                         } else {
 149                                 if (rdev->pm.active_crtc_count > 1) {
 150                                         for (i = 0; i < rdev->pm.num_power_states; i++) {
 151                                                 if (rdev->pm.power_state[i].flags & RADEON_PM_STATE_SINGLE_DISPLAY_ONLY)
 152                                                         continue;
 153                                                 else if (i >= rdev->pm.current_power_state_index) {
 154                                                         rdev->pm.requested_power_state_index =
 155                                                                 rdev->pm.current_power_state_index;
 156                                                         break;
 157                                                 } else {
 158                                                         rdev->pm.requested_power_state_index = i;
 159                                                         break;
 160                                                 }
 161                                         }
 162                                 } else {
 163                                         if (rdev->pm.current_power_state_index == 0)
 164                                                 rdev->pm.requested_power_state_index =
 165                                                         rdev->pm.num_power_states - 1;
 166                                         else
 167                                                 rdev->pm.requested_power_state_index =
 168                                                         rdev->pm.current_power_state_index - 1;
 169                                 }
 170                         }
 171                         rdev->pm.requested_clock_mode_index = 0;
 172                         /* don't use the power state if crtcs are active and no display flag is set */
 173                         if ((rdev->pm.active_crtc_count > 0) &&
 174                             (rdev->pm.power_state[rdev->pm.requested_power_state_index].
 175                              clock_info[rdev->pm.requested_clock_mode_index].flags &
 176                              RADEON_PM_MODE_NO_DISPLAY)) {
 177                                 rdev->pm.requested_power_state_index++;
 178                         }
 179                         break;
 180                 case DYNPM_ACTION_UPCLOCK:
 181                         if (rdev->pm.current_power_state_index == (rdev->pm.num_power_states - 1)) {
 182                                 rdev->pm.requested_power_state_index = rdev->pm.current_power_state_index;
 183                                 rdev->pm.dynpm_can_upclock = false;
 184                         } else {
 185                                 if (rdev->pm.active_crtc_count > 1) {
 186                                         for (i = (rdev->pm.num_power_states - 1); i >= 0; i--) {
 187                                                 if (rdev->pm.power_state[i].flags & RADEON_PM_STATE_SINGLE_DISPLAY_ONLY)
 188                                                         continue;
 189                                                 else if (i <= rdev->pm.current_power_state_index) {
 190                                                         rdev->pm.requested_power_state_index =
 191                                                                 rdev->pm.current_power_state_index;
 192                                                         break;
 193                                                 } else {
 194                                                         rdev->pm.requested_power_state_index = i;
 195                                                         break;
 196                                                 }
 197                                         }
 198                                 } else
 199                                         rdev->pm.requested_power_state_index =
 200                                                 rdev->pm.current_power_state_index + 1;
 201                         }
 202                         rdev->pm.requested_clock_mode_index = 0;
 203                         break;
 204                 case DYNPM_ACTION_DEFAULT:
 205                         rdev->pm.requested_power_state_index = rdev->pm.default_power_state_index;
 206                         rdev->pm.requested_clock_mode_index = 0;
 207                         rdev->pm.dynpm_can_upclock = false;
 208                         break;
 209                 case DYNPM_ACTION_NONE:
 210                 default:
 211                         DRM_ERROR("Requested mode for not defined action\n");
 212                         return;
 213                 }
 214         } else {
 215                 /* XXX select a power state based on AC/DC, single/dualhead, etc. */
 216                 /* for now just select the first power state and switch between clock modes */
 217                 /* power state array is low to high, default is first (0) */
 218                 if (rdev->pm.active_crtc_count > 1) {
 219                         rdev->pm.requested_power_state_index = -1;
 220                         /* start at 1 as we don't want the default mode */
 221                         for (i = 1; i < rdev->pm.num_power_states; i++) {
 222                                 if (rdev->pm.power_state[i].flags & RADEON_PM_STATE_SINGLE_DISPLAY_ONLY)
 223                                         continue;
 224                                 else if ((rdev->pm.power_state[i].type == POWER_STATE_TYPE_PERFORMANCE) ||
 225                                          (rdev->pm.power_state[i].type == POWER_STATE_TYPE_BATTERY)) {
 226                                         rdev->pm.requested_power_state_index = i;
 227                                         break;
 228                                 }
 229                         }
 230                         /* if nothing selected, grab the default state. */
 231                         if (rdev->pm.requested_power_state_index == -1)
 232                                 rdev->pm.requested_power_state_index = 0;
 233                 } else
 234                         rdev->pm.requested_power_state_index = 1;
 235
 236                 switch (rdev->pm.dynpm_planned_action) {
 237                 case DYNPM_ACTION_MINIMUM:
 238                         rdev->pm.requested_clock_mode_index = 0;
 239                         rdev->pm.dynpm_can_downclock = false;
 240                         break;
 241                 case DYNPM_ACTION_DOWNCLOCK:
 242                         if (rdev->pm.requested_power_state_index == rdev->pm.current_power_state_index) {
 243                                 if (rdev->pm.current_clock_mode_index == 0) {
 244                                         rdev->pm.requested_clock_mode_index = 0;
 245                                         rdev->pm.dynpm_can_downclock = false;
 246                                 } else
 247                                         rdev->pm.requested_clock_mode_index =
 248                                                 rdev->pm.current_clock_mode_index - 1;
 249                         } else {
 250                                 rdev->pm.requested_clock_mode_index = 0;
 251                                 rdev->pm.dynpm_can_downclock = false;
 252                         }
 253                         /* don't use the power state if crtcs are active and no display flag is set */
 254                         if ((rdev->pm.active_crtc_count > 0) &&
 255                             (rdev->pm.power_state[rdev->pm.requested_power_state_index].
 256                              clock_info[rdev->pm.requested_clock_mode_index].flags &
 257                              RADEON_PM_MODE_NO_DISPLAY)) {
 258                                 rdev->pm.requested_clock_mode_index++;
 259                         }
 260                         break;
 261                 case DYNPM_ACTION_UPCLOCK:
 262                         if (rdev->pm.requested_power_state_index == rdev->pm.current_power_state_index) {
 263                                 if (rdev->pm.current_clock_mode_index ==
 264                                     (rdev->pm.power_state[rdev->pm.requested_power_state_index].num_clock_modes - 1)) {
 265                                         rdev->pm.requested_clock_mode_index = rdev->pm.current_clock_mode_index;
 266                                         rdev->pm.dynpm_can_upclock = false;
 267                                 } else
 268                                         rdev->pm.requested_clock_mode_index =
 269                                                 rdev->pm.current_clock_mode_index + 1;
 270                         } else {
 271                                 rdev->pm.requested_clock_mode_index =
 272                                         rdev->pm.power_state[rdev->pm.requested_power_state_index].num_clock_modes - 1;
 273                                 rdev->pm.dynpm_can_upclock = false;
 274                         }
 275                         break;
 276                 case DYNPM_ACTION_DEFAULT:
 277                         rdev->pm.requested_power_state_index = rdev->pm.default_power_state_index;
 278                         rdev->pm.requested_clock_mode_index = 0;
 279                         rdev->pm.dynpm_can_upclock = false;
 280                         break;
 281                 case DYNPM_ACTION_NONE:
 282                 default:
 283                         DRM_ERROR("Requested mode for not defined action\n");
 284                         return;
 285                 }
 286         }
 287
 288         DRM_DEBUG_DRIVER("Requested: e: %d m: %d p: %d\n",
 289                   rdev->pm.power_state[rdev->pm.requested_power_state_index].
 290                   clock_info[rdev->pm.requested_clock_mode_index].sclk,
 291                   rdev->pm.power_state[rdev->pm.requested_power_state_index].
 292                   clock_info[rdev->pm.requested_clock_mode_index].mclk,
 293                   rdev->pm.power_state[rdev->pm.requested_power_state_index].
 294                   pcie_lanes);
 295 }
 296
 297 void rs780_pm_init_profile(struct radeon_device *rdev)
 298 {
 299         if (rdev->pm.num_power_states == 2) {
 300                 /* default */
 301                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
 302                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
 303                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
 304                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
 305                 /* low sh */
 306                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = 0;
 307                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = 0;
 308                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
 309                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
 310                 /* mid sh */
 311                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = 0;
 312                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = 0;
 313                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
 314                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
 315                 /* high sh */
 316                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = 0;
 317                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = 1;
 318                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
 319                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 0;
 320                 /* low mh */
 321                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = 0;
 322                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = 0;
 323                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
 324                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
 325                 /* mid mh */
 326                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = 0;
 327                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = 0;
 328                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
 329                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
 330                 /* high mh */
 331                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = 0;
 332                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = 1;
 333                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
 334                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 0;
 335         } else if (rdev->pm.num_power_states == 3) {
 336                 /* default */
 337                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
 338                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
 339                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
 340                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
 341                 /* low sh */
 342                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = 1;
 343                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = 1;
 344                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
 345                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
 346                 /* mid sh */
 347                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = 1;
 348                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = 1;
 349                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
 350                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
 351                 /* high sh */
 352                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = 1;
 353                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = 2;
 354                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
 355                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 0;
 356                 /* low mh */
 357                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = 1;
 358                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = 1;
 359                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
 360                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
 361                 /* mid mh */
 362                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = 1;
 363                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = 1;
 364                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
 365                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
 366                 /* high mh */
 367                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = 1;
 368                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = 2;
 369                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
 370                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 0;
 371         } else {
 372                 /* default */
 373                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
 374                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
 375                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
 376                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
 377                 /* low sh */
 378                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = 2;
 379                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = 2;
 380                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
 381                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
 382                 /* mid sh */
 383                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = 2;
 384                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = 2;
 385                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
 386                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
 387                 /* high sh */
 388                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = 2;
 389                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = 3;
 390                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
 391                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 0;
 392                 /* low mh */
 393                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = 2;
 394                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = 0;
 395                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
 396                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
 397                 /* mid mh */
 398                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = 2;
 399                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = 0;
 400                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
 401                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
 402                 /* high mh */
 403                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = 2;
 404                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = 3;
 405                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
 406                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 0;
 407         }
 408 }
 409
 410 void r600_pm_init_profile(struct radeon_device *rdev)
 411 {
 412         int idx;
 413
 414         if (rdev->family == CHIP_R600) {
 415                 /* XXX */
 416                 /* default */
 417                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
 418                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
 419                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
 420                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
 421                 /* low sh */
 422                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
 423                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
 424                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
 425                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
 426                 /* mid sh */
 427                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
 428                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
 429                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
 430                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
 431                 /* high sh */
 432                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
 433                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
 434                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
 435                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 0;
 436                 /* low mh */
 437                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
 438                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
 439                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
 440                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
 441                 /* mid mh */
 442                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
 443                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
 444                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
 445                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
 446                 /* high mh */
 447                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
 448                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
 449                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
 450                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 0;
 451         } else {
 452                 if (rdev->pm.num_power_states < 4) {
 453                         /* default */
 454                         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
 455                         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
 456                         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
 457                         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
 458                         /* low sh */
 459                         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = 1;
 460                         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = 1;
 461                         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
 462                         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
 463                         /* mid sh */
 464                         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = 1;
 465                         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = 1;
 466                         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
 467                         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
 468                         /* high sh */
 469                         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = 1;
 470                         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = 1;
 471                         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
 472                         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
 473                         /* low mh */
 474                         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = 2;
 475                         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = 2;
 476                         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
 477                         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
 478                         /* low mh */
 479                         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = 2;
 480                         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = 2;
 481                         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
 482                         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
 483                         /* high mh */
 484                         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = 2;
 485                         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = 2;
 486                         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
 487                         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
 488                 } else {
 489                         /* default */
 490                         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
 491                         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
 492                         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
 493                         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
 494                         /* low sh */
 495                         if (rdev->flags & RADEON_IS_MOBILITY)
 496                                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
 497                         else
 498                                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
 499                         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
 500                         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
 501                         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
 502                         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
 503                         /* mid sh */
 504                         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
 505                         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
 506                         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
 507                         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
 508                         /* high sh */
 509                         idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
 510                         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
 511                         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
 512                         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
 513                         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
 514                         /* low mh */
 515                         if (rdev->flags & RADEON_IS_MOBILITY)
 516                                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 1);
 517                         else
 518                                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 1);
 519                         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
 520                         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
 521                         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
 522                         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
 523                         /* mid mh */
 524                         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
 525                         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
 526                         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
 527                         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
 528                         /* high mh */
 529                         idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 1);
 530                         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
 531                         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
 532                         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
 533                         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
 534                 }
 535         }
 536 }
 537
 538 void r600_pm_misc(struct radeon_device *rdev)
 539 {
 540         int req_ps_idx = rdev->pm.requested_power_state_index;
 541         int req_cm_idx = rdev->pm.requested_clock_mode_index;
 542         struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
 543         struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
 544
 545         if ((voltage->type == VOLTAGE_SW) && voltage->voltage) {
 546                 /* 0xff01 is a flag rather then an actual voltage */
 547                 if (voltage->voltage == 0xff01)
 548                         return;
 549                 if (voltage->voltage != rdev->pm.current_vddc) {
 550                         radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
 551                         rdev->pm.current_vddc = voltage->voltage;
 552                         DRM_DEBUG_DRIVER("Setting: v: %d\n", voltage->voltage);
 553                 }
 554         }
 555 }
 556
 557 bool r600_gui_idle(struct radeon_device *rdev)
 558 {
 559         if (RREG32(GRBM_STATUS) & GUI_ACTIVE)
 560                 return false;
 561         else
 562                 return true;
 563 }
 564
 565 /* hpd for digital panel detect/disconnect */
 566 bool r600_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
 567 {
 568         bool connected = false;
 569
 570         if (ASIC_IS_DCE3(rdev)) {
 571                 switch (hpd) {
 572                 case RADEON_HPD_1:
 573                         if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
 574                                 connected = true;
 575                         break;
 576                 case RADEON_HPD_2:
 577                         if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
 578                                 connected = true;
 579                         break;
 580                 case RADEON_HPD_3:
 581                         if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
 582                                 connected = true;
 583                         break;
 584                 case RADEON_HPD_4:
 585                         if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
 586                                 connected = true;
 587                         break;
 588                         /* DCE 3.2 */
 589                 case RADEON_HPD_5:
 590                         if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
 591                                 connected = true;
 592                         break;
 593                 case RADEON_HPD_6:
 594                         if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
 595                                 connected = true;
 596                         break;
 597                 default:
 598                         break;
 599                 }
 600         } else {
 601                 switch (hpd) {
 602                 case RADEON_HPD_1:
 603                         if (RREG32(DC_HOT_PLUG_DETECT1_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
 604                                 connected = true;
 605                         break;
 606                 case RADEON_HPD_2:
 607                         if (RREG32(DC_HOT_PLUG_DETECT2_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
 608                                 connected = true;
 609                         break;
 610                 case RADEON_HPD_3:
 611                         if (RREG32(DC_HOT_PLUG_DETECT3_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
 612                                 connected = true;
 613                         break;
 614                 default:
 615                         break;
 616                 }
 617         }
 618         return connected;
 619 }
 620
 621 void r600_hpd_set_polarity(struct radeon_device *rdev,
 622                            enum radeon_hpd_id hpd)
 623 {
 624         u32 tmp;
 625         bool connected = r600_hpd_sense(rdev, hpd);
 626
 627         if (ASIC_IS_DCE3(rdev)) {
 628                 switch (hpd) {
 629                 case RADEON_HPD_1:
 630                         tmp = RREG32(DC_HPD1_INT_CONTROL);
 631                         if (connected)
 632                                 tmp &= ~DC_HPDx_INT_POLARITY;
 633                         else
 634                                 tmp |= DC_HPDx_INT_POLARITY;
 635                         WREG32(DC_HPD1_INT_CONTROL, tmp);
 636                         break;
 637                 case RADEON_HPD_2:
 638                         tmp = RREG32(DC_HPD2_INT_CONTROL);
 639                         if (connected)
 640                                 tmp &= ~DC_HPDx_INT_POLARITY;
 641                         else
 642                                 tmp |= DC_HPDx_INT_POLARITY;
 643                         WREG32(DC_HPD2_INT_CONTROL, tmp);
 644                         break;
 645                 case RADEON_HPD_3:
 646                         tmp = RREG32(DC_HPD3_INT_CONTROL);
 647                         if (connected)
 648                                 tmp &= ~DC_HPDx_INT_POLARITY;
 649                         else
 650                                 tmp |= DC_HPDx_INT_POLARITY;
 651                         WREG32(DC_HPD3_INT_CONTROL, tmp);
 652                         break;
 653                 case RADEON_HPD_4:
 654                         tmp = RREG32(DC_HPD4_INT_CONTROL);
 655                         if (connected)
 656                                 tmp &= ~DC_HPDx_INT_POLARITY;
 657                         else
 658                                 tmp |= DC_HPDx_INT_POLARITY;
 659                         WREG32(DC_HPD4_INT_CONTROL, tmp);
 660                         break;
 661                 case RADEON_HPD_5:
 662                         tmp = RREG32(DC_HPD5_INT_CONTROL);
 663                         if (connected)
 664                                 tmp &= ~DC_HPDx_INT_POLARITY;
 665                         else
 666                                 tmp |= DC_HPDx_INT_POLARITY;
 667                         WREG32(DC_HPD5_INT_CONTROL, tmp);
 668                         break;
 669                         /* DCE 3.2 */
 670                 case RADEON_HPD_6:
 671                         tmp = RREG32(DC_HPD6_INT_CONTROL);
 672                         if (connected)
 673                                 tmp &= ~DC_HPDx_INT_POLARITY;
 674                         else
 675                                 tmp |= DC_HPDx_INT_POLARITY;
 676                         WREG32(DC_HPD6_INT_CONTROL, tmp);
 677                         break;
 678                 default:
 679                         break;
 680                 }
 681         } else {
 682                 switch (hpd) {
 683                 case RADEON_HPD_1:
 684                         tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL);
 685                         if (connected)
 686                                 tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
 687                         else
 688                                 tmp |= DC_HOT_PLUG_DETECTx_INT_POLARITY;
 689                         WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
 690                         break;
 691                 case RADEON_HPD_2:
 692                         tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL);
 693                         if (connected)
 694                                 tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
 695                         else
 696                                 tmp |= DC_HOT_PLUG_DETECTx_INT_POLARITY;
 697                         WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
 698                         break;
 699                 case RADEON_HPD_3:
 700                         tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL);
 701                         if (connected)
 702                                 tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
 703                         else
 704                                 tmp |= DC_HOT_PLUG_DETECTx_INT_POLARITY;
 705                         WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
 706                         break;
 707                 default:
 708                         break;
 709                 }
 710         }
 711 }
 712
 713 void r600_hpd_init(struct radeon_device *rdev)
 714 {
 715         struct drm_device *dev = rdev->ddev;
 716         struct drm_connector *connector;
 717         unsigned enable = 0;
 718
 719         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
 720                 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
 721
 722                 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
 723                     connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
 724                         /* don't try to enable hpd on eDP or LVDS avoid breaking the
 725                          * aux dp channel on imac and help (but not completely fix)
 726                          * https://bugzilla.redhat.com/show_bug.cgi?id=726143
 727                          */
 728                         continue;
 729                 }
 730                 if (ASIC_IS_DCE3(rdev)) {
 731                         u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) | DC_HPDx_RX_INT_TIMER(0xfa);
 732                         if (ASIC_IS_DCE32(rdev))
 733                                 tmp |= DC_HPDx_EN;
 734
 735                         switch (radeon_connector->hpd.hpd) {
 736                         case RADEON_HPD_1:
 737                                 WREG32(DC_HPD1_CONTROL, tmp);
 738                                 break;
 739                         case RADEON_HPD_2:
 740                                 WREG32(DC_HPD2_CONTROL, tmp);
 741                                 break;
 742                         case RADEON_HPD_3:
 743                                 WREG32(DC_HPD3_CONTROL, tmp);
 744                                 break;
 745                         case RADEON_HPD_4:
 746                                 WREG32(DC_HPD4_CONTROL, tmp);
 747                                 break;
 748                                 /* DCE 3.2 */
 749                         case RADEON_HPD_5:
 750                                 WREG32(DC_HPD5_CONTROL, tmp);
 751                                 break;
 752                         case RADEON_HPD_6:
 753                                 WREG32(DC_HPD6_CONTROL, tmp);
 754                                 break;
 755                         default:
 756                                 break;
 757                         }
 758                 } else {
 759                         switch (radeon_connector->hpd.hpd) {
 760                         case RADEON_HPD_1:
 761                                 WREG32(DC_HOT_PLUG_DETECT1_CONTROL, DC_HOT_PLUG_DETECTx_EN);
 762                                 break;
 763                         case RADEON_HPD_2:
 764                                 WREG32(DC_HOT_PLUG_DETECT2_CONTROL, DC_HOT_PLUG_DETECTx_EN);
 765                                 break;
 766                         case RADEON_HPD_3:
 767                                 WREG32(DC_HOT_PLUG_DETECT3_CONTROL, DC_HOT_PLUG_DETECTx_EN);
 768                                 break;
 769                         default:
 770                                 break;
 771                         }
 772                 }
 773                 enable |= 1 << radeon_connector->hpd.hpd;
 774                 radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
 775         }
 776         radeon_irq_kms_enable_hpd(rdev, enable);
 777 }
 778
 779 void r600_hpd_fini(struct radeon_device *rdev)
 780 {
 781         struct drm_device *dev = rdev->ddev;
 782         struct drm_connector *connector;
 783         unsigned disable = 0;
 784
 785         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
 786                 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
 787                 if (ASIC_IS_DCE3(rdev)) {
 788                         switch (radeon_connector->hpd.hpd) {
 789                         case RADEON_HPD_1:
 790                                 WREG32(DC_HPD1_CONTROL, 0);
 791                                 break;
 792                         case RADEON_HPD_2:
 793                                 WREG32(DC_HPD2_CONTROL, 0);
 794                                 break;
 795                         case RADEON_HPD_3:
 796                                 WREG32(DC_HPD3_CONTROL, 0);
 797                                 break;
 798                         case RADEON_HPD_4:
 799                                 WREG32(DC_HPD4_CONTROL, 0);
 800                                 break;
 801                                 /* DCE 3.2 */
 802                         case RADEON_HPD_5:
 803                                 WREG32(DC_HPD5_CONTROL, 0);
 804                                 break;
 805                         case RADEON_HPD_6:
 806                                 WREG32(DC_HPD6_CONTROL, 0);
 807                                 break;
 808                         default:
 809                                 break;
 810                         }
 811                 } else {
 812                         switch (radeon_connector->hpd.hpd) {
 813                         case RADEON_HPD_1:
 814                                 WREG32(DC_HOT_PLUG_DETECT1_CONTROL, 0);
 815                                 break;
 816                         case RADEON_HPD_2:
 817                                 WREG32(DC_HOT_PLUG_DETECT2_CONTROL, 0);
 818                                 break;
 819                         case RADEON_HPD_3:
 820                                 WREG32(DC_HOT_PLUG_DETECT3_CONTROL, 0);
 821                                 break;
 822                         default:
 823                                 break;
 824                         }
 825                 }
 826                 disable |= 1 << radeon_connector->hpd.hpd;
 827         }
 828         radeon_irq_kms_disable_hpd(rdev, disable);
 829 }
 830
 831 /*
 832  * R600 PCIE GART
 833  */
 834 void r600_pcie_gart_tlb_flush(struct radeon_device *rdev)
 835 {
 836         unsigned i;
 837         u32 tmp;
 838
 839         /* flush hdp cache so updates hit vram */
 840         if ((rdev->family >= CHIP_RV770) && (rdev->family <= CHIP_RV740) &&
 841             !(rdev->flags & RADEON_IS_AGP)) {
 842                 volatile uint32_t *ptr = rdev->gart.ptr;
 843                 u32 tmp;
 844
 845                 /* r7xx hw bug.  write to HDP_DEBUG1 followed by fb read
 846                  * rather than write to HDP_REG_COHERENCY_FLUSH_CNTL
 847                  * This seems to cause problems on some AGP cards. Just use the old
 848                  * method for them.
 849                  */
 850                 WREG32(HDP_DEBUG1, 0);
 851                 tmp = *ptr;
 852         } else
 853                 WREG32(R_005480_HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
 854
 855         WREG32(VM_CONTEXT0_INVALIDATION_LOW_ADDR, rdev->mc.gtt_start >> 12);
 856         WREG32(VM_CONTEXT0_INVALIDATION_HIGH_ADDR, (rdev->mc.gtt_end - 1) >> 12);
 857         WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
 858         for (i = 0; i < rdev->usec_timeout; i++) {
 859                 /* read MC_STATUS */
 860                 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
 861                 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
 862                 if (tmp == 2) {
 863                         DRM_ERROR("[drm] r600 flush TLB failed\n");
 864                         return;
 865                 }
 866                 if (tmp) {
 867                         return;
 868                 }
 869                 udelay(1);
 870         }
 871 }
 872
 873 int r600_pcie_gart_init(struct radeon_device *rdev)
 874 {
 875         int r;
 876
 877         if (rdev->gart.robj) {
 878                 DRM_ERROR("R600 PCIE GART already initialized\n");
 879                 return 0;
 880         }
 881         /* Initialize common gart structure */
 882         r = radeon_gart_init(rdev);
 883         if (r)
 884                 return r;
 885         rdev->gart.table_size = rdev->gart.num_gpu_pages * 8;
 886         return radeon_gart_table_vram_alloc(rdev);
 887 }
 888
 889 static int r600_pcie_gart_enable(struct radeon_device *rdev)
 890 {
 891         u32 tmp;
 892         int r, i;
 893
 894         if (rdev->gart.robj == NULL) {
 895                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
 896                 return -EINVAL;
 897         }
 898         r = radeon_gart_table_vram_pin(rdev);
 899         if (r)
 900                 return r;
 901         radeon_gart_restore(rdev);
 902
 903         /* Setup L2 cache */
 904         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
 905                                 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
 906                                 EFFECTIVE_L2_QUEUE_SIZE(7));
 907         WREG32(VM_L2_CNTL2, 0);
 908         WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) | BANK_SELECT_1(1));
 909         /* Setup TLB control */
 910         tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
 911                 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
 912                 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5) |
 913                 ENABLE_WAIT_L2_QUERY;
 914         WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
 915         WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
 916         WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp | ENABLE_L1_STRICT_ORDERING);
 917         WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
 918         WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
 919         WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
 920         WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
 921         WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
 922         WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
 923         WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
 924         WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
 925         WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
 926         WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE);
 927         WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE);
 928         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
 929         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
 930         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
 931         WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
 932                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
 933         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
 934                         (u32)(rdev->dummy_page.addr >> 12));
 935         for (i = 1; i < 7; i++)
 936                 WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
 937
 938         r600_pcie_gart_tlb_flush(rdev);
 939         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
 940                  (unsigned)(rdev->mc.gtt_size >> 20),
 941                  (unsigned long long)rdev->gart.table_addr);
 942         rdev->gart.ready = true;
 943         return 0;
 944 }
 945
 946 static void r600_pcie_gart_disable(struct radeon_device *rdev)
 947 {
 948         u32 tmp;
 949         int i;
 950
 951         /* Disable all tables */
 952         for (i = 0; i < 7; i++)
 953                 WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
 954
 955         /* Disable L2 cache */
 956         WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
 957                                 EFFECTIVE_L2_QUEUE_SIZE(7));
 958         WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) | BANK_SELECT_1(1));
 959         /* Setup L1 TLB control */
 960         tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5) |
 961                 ENABLE_WAIT_L2_QUERY;
 962         WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
 963         WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
 964         WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
 965         WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
 966         WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
 967         WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
 968         WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
 969         WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
 970         WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp);
 971         WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp);
 972         WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
 973         WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
 974         WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp);
 975         WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
 976         radeon_gart_table_vram_unpin(rdev);
 977 }
 978
 979 static void r600_pcie_gart_fini(struct radeon_device *rdev)
 980 {
 981         radeon_gart_fini(rdev);
 982         r600_pcie_gart_disable(rdev);
 983         radeon_gart_table_vram_free(rdev);
 984 }
 985
 986 static void r600_agp_enable(struct radeon_device *rdev)
 987 {
 988         u32 tmp;
 989         int i;
 990
 991         /* Setup L2 cache */
 992         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
 993                                 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
 994                                 EFFECTIVE_L2_QUEUE_SIZE(7));
 995         WREG32(VM_L2_CNTL2, 0);
 996         WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) | BANK_SELECT_1(1));
 997         /* Setup TLB control */
 998         tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
 999                 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1000                 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5) |
1001                 ENABLE_WAIT_L2_QUERY;
1002         WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
1003         WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
1004         WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp | ENABLE_L1_STRICT_ORDERING);
1005         WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
1006         WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
1007         WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
1008         WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
1009         WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
1010         WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
1011         WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
1012         WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
1013         WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
1014         WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE);
1015         WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE);
1016         for (i = 0; i < 7; i++)
1017                 WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
1018 }
1019
1020 int r600_mc_wait_for_idle(struct radeon_device *rdev)
1021 {
1022         unsigned i;
1023         u32 tmp;
1024
1025         for (i = 0; i < rdev->usec_timeout; i++) {
1026                 /* read MC_STATUS */
1027                 tmp = RREG32(R_000E50_SRBM_STATUS) & 0x3F00;
1028                 if (!tmp)
1029                         return 0;
1030                 udelay(1);
1031         }
1032         return -1;
1033 }
1034
1035 static void r600_mc_program(struct radeon_device *rdev)
1036 {
1037         struct rv515_mc_save save;
1038         u32 tmp;
1039         int i, j;
1040
1041         /* Initialize HDP */
1042         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
1043                 WREG32((0x2c14 + j), 0x00000000);
1044                 WREG32((0x2c18 + j), 0x00000000);
1045                 WREG32((0x2c1c + j), 0x00000000);
1046                 WREG32((0x2c20 + j), 0x00000000);
1047                 WREG32((0x2c24 + j), 0x00000000);
1048         }
1049         WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
1050
1051         rv515_mc_stop(rdev, &save);
1052         if (r600_mc_wait_for_idle(rdev)) {
1053                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1054         }
1055         /* Lockout access through VGA aperture (doesn't exist before R600) */
1056         WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
1057         /* Update configuration */
1058         if (rdev->flags & RADEON_IS_AGP) {
1059                 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
1060                         /* VRAM before AGP */
1061                         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1062                                 rdev->mc.vram_start >> 12);
1063                         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1064                                 rdev->mc.gtt_end >> 12);
1065                 } else {
1066                         /* VRAM after AGP */
1067                         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1068                                 rdev->mc.gtt_start >> 12);
1069                         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1070                                 rdev->mc.vram_end >> 12);
1071                 }
1072         } else {
1073                 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, rdev->mc.vram_start >> 12);
1074                 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, rdev->mc.vram_end >> 12);
1075         }
1076         WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
1077         tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
1078         tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
1079         WREG32(MC_VM_FB_LOCATION, tmp);
1080         WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
1081         WREG32(HDP_NONSURFACE_INFO, (2 << 7));
1082         WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
1083         if (rdev->flags & RADEON_IS_AGP) {
1084                 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 22);
1085                 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 22);
1086                 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
1087         } else {
1088                 WREG32(MC_VM_AGP_BASE, 0);
1089                 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
1090                 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
1091         }
1092         if (r600_mc_wait_for_idle(rdev)) {
1093                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1094         }
1095         rv515_mc_resume(rdev, &save);
1096         /* we need to own VRAM, so turn off the VGA renderer here
1097          * to stop it overwriting our objects */
1098         rv515_vga_render_disable(rdev);
1099 }
1100
1101 /**
1102  * r600_vram_gtt_location - try to find VRAM & GTT location
1103  * @rdev: radeon device structure holding all necessary informations
1104  * @mc: memory controller structure holding memory informations
1105  *
1106  * Function will place try to place VRAM at same place as in CPU (PCI)
1107  * address space as some GPU seems to have issue when we reprogram at
1108  * different address space.
1109  *
1110  * If there is not enough space to fit the unvisible VRAM after the
1111  * aperture then we limit the VRAM size to the aperture.
1112  *
1113  * If we are using AGP then place VRAM adjacent to AGP aperture are we need
1114  * them to be in one from GPU point of view so that we can program GPU to
1115  * catch access outside them (weird GPU policy see ??).
1116  *
1117  * This function will never fails, worst case are limiting VRAM or GTT.
1118  *
1119  * Note: GTT start, end, size should be initialized before calling this
1120  * function on AGP platform.
1121  */
1122 static void r600_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc)
1123 {
1124         u64 size_bf, size_af;
1125
1126         if (mc->mc_vram_size > 0xE0000000) {
1127                 /* leave room for at least 512M GTT */
1128                 dev_warn(rdev->dev, "limiting VRAM\n");
1129                 mc->real_vram_size = 0xE0000000;
1130                 mc->mc_vram_size = 0xE0000000;
1131         }
1132         if (rdev->flags & RADEON_IS_AGP) {
1133                 size_bf = mc->gtt_start;
1134                 size_af = 0xFFFFFFFF - mc->gtt_end;
1135                 if (size_bf > size_af) {
1136                         if (mc->mc_vram_size > size_bf) {
1137                                 dev_warn(rdev->dev, "limiting VRAM\n");
1138                                 mc->real_vram_size = size_bf;
1139                                 mc->mc_vram_size = size_bf;
1140                         }
1141                         mc->vram_start = mc->gtt_start - mc->mc_vram_size;
1142                 } else {
1143                         if (mc->mc_vram_size > size_af) {
1144                                 dev_warn(rdev->dev, "limiting VRAM\n");
1145                                 mc->real_vram_size = size_af;
1146                                 mc->mc_vram_size = size_af;
1147                         }
1148                         mc->vram_start = mc->gtt_end + 1;
1149                 }
1150                 mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
1151                 dev_info(rdev->dev, "VRAM: %juM 0x%08jX - 0x%08jX (%juM used)\n",
1152                                 (uintmax_t)mc->mc_vram_size >> 20, (uintmax_t)mc->vram_start,
1153                                 (uintmax_t)mc->vram_end, (uintmax_t)mc->real_vram_size >> 20);
1154         } else {
1155                 u64 base = 0;
1156                 if (rdev->flags & RADEON_IS_IGP) {
1157                         base = RREG32(MC_VM_FB_LOCATION) & 0xFFFF;
1158                         base <<= 24;
1159                 }
1160                 radeon_vram_location(rdev, &rdev->mc, base);
1161                 rdev->mc.gtt_base_align = 0;
1162                 radeon_gtt_location(rdev, mc);
1163         }
1164 }
1165
1166 static int r600_mc_init(struct radeon_device *rdev)
1167 {
1168         u32 tmp;
1169         int chansize, numchan;
1170
1171         /* Get VRAM informations */
1172         rdev->mc.vram_is_ddr = true;
1173         tmp = RREG32(RAMCFG);
1174         if (tmp & CHANSIZE_OVERRIDE) {
1175                 chansize = 16;
1176         } else if (tmp & CHANSIZE_MASK) {
1177                 chansize = 64;
1178         } else {
1179                 chansize = 32;
1180         }
1181         tmp = RREG32(CHMAP);
1182         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1183         case 0:
1184         default:
1185                 numchan = 1;
1186                 break;
1187         case 1:
1188                 numchan = 2;
1189                 break;
1190         case 2:
1191                 numchan = 4;
1192                 break;
1193         case 3:
1194                 numchan = 8;
1195                 break;
1196         }
1197         rdev->mc.vram_width = numchan * chansize;
1198         /* Could aper size report 0 ? */
1199         rdev->mc.aper_base = drm_get_resource_start(rdev->ddev, 0);
1200         rdev->mc.aper_size = drm_get_resource_len(rdev->ddev, 0);
1201         /* Setup GPU memory space */
1202         rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
1203         rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
1204         rdev->mc.visible_vram_size = rdev->mc.aper_size;
1205         r600_vram_gtt_location(rdev, &rdev->mc);
1206
1207         if (rdev->flags & RADEON_IS_IGP) {
1208                 rs690_pm_info(rdev);
1209                 rdev->mc.igp_sideport_enabled = radeon_atombios_sideport_present(rdev);
1210         }
1211         radeon_update_bandwidth_info(rdev);
1212         return 0;
1213 }
1214
1215 int r600_vram_scratch_init(struct radeon_device *rdev)
1216 {
1217         int r;
1218         void *vram_scratch_ptr_ptr; /* FreeBSD: to please GCC 4.2. */
1219
1220         if (rdev->vram_scratch.robj == NULL) {
1221                 r = radeon_bo_create(rdev, RADEON_GPU_PAGE_SIZE,
1222                                      PAGE_SIZE, true, RADEON_GEM_DOMAIN_VRAM,
1223                                      NULL, &rdev->vram_scratch.robj);
1224                 if (r) {
1225                         return r;
1226                 }
1227         }
1228
1229         r = radeon_bo_reserve(rdev->vram_scratch.robj, false);
1230         if (unlikely(r != 0)) {
1231                 radeon_bo_unref(&rdev->vram_scratch.robj);
1232                 return r;
1233         }
1234         r = radeon_bo_pin(rdev->vram_scratch.robj,
1235                           RADEON_GEM_DOMAIN_VRAM, &rdev->vram_scratch.gpu_addr);
1236         if (r) {
1237                 radeon_bo_unreserve(rdev->vram_scratch.robj);
1238                 radeon_bo_unref(&rdev->vram_scratch.robj);
1239                 return r;
1240         }
1241         vram_scratch_ptr_ptr = &rdev->vram_scratch.ptr;
1242         r = radeon_bo_kmap(rdev->vram_scratch.robj,
1243                                 vram_scratch_ptr_ptr);
1244         if (r)
1245                 radeon_bo_unpin(rdev->vram_scratch.robj);
1246         radeon_bo_unreserve(rdev->vram_scratch.robj);
1247         if (r)
1248                 radeon_bo_unref(&rdev->vram_scratch.robj);
1249
1250         return r;
1251 }
1252
1253 void r600_vram_scratch_fini(struct radeon_device *rdev)
1254 {
1255         int r;
1256
1257         if (rdev->vram_scratch.robj == NULL) {
1258                 return;
1259         }
1260         r = radeon_bo_reserve(rdev->vram_scratch.robj, false);
1261         if (likely(r == 0)) {
1262                 radeon_bo_kunmap(rdev->vram_scratch.robj);
1263                 radeon_bo_unpin(rdev->vram_scratch.robj);
1264                 radeon_bo_unreserve(rdev->vram_scratch.robj);
1265         }
1266         radeon_bo_unref(&rdev->vram_scratch.robj);
1267 }
1268
1269 /* We doesn't check that the GPU really needs a reset we simply do the
1270  * reset, it's up to the caller to determine if the GPU needs one. We
1271  * might add an helper function to check that.
1272  */
1273 static void r600_gpu_soft_reset_gfx(struct radeon_device *rdev)
1274 {
1275         u32 grbm_busy_mask = S_008010_VC_BUSY(1) | S_008010_VGT_BUSY_NO_DMA(1) |
1276                                 S_008010_VGT_BUSY(1) | S_008010_TA03_BUSY(1) |
1277                                 S_008010_TC_BUSY(1) | S_008010_SX_BUSY(1) |
1278                                 S_008010_SH_BUSY(1) | S_008010_SPI03_BUSY(1) |
1279                                 S_008010_SMX_BUSY(1) | S_008010_SC_BUSY(1) |
1280                                 S_008010_PA_BUSY(1) | S_008010_DB03_BUSY(1) |
1281                                 S_008010_CR_BUSY(1) | S_008010_CB03_BUSY(1) |
1282                                 S_008010_GUI_ACTIVE(1);
1283         u32 grbm2_busy_mask = S_008014_SPI0_BUSY(1) | S_008014_SPI1_BUSY(1) |
1284                         S_008014_SPI2_BUSY(1) | S_008014_SPI3_BUSY(1) |
1285                         S_008014_TA0_BUSY(1) | S_008014_TA1_BUSY(1) |
1286                         S_008014_TA2_BUSY(1) | S_008014_TA3_BUSY(1) |
1287                         S_008014_DB0_BUSY(1) | S_008014_DB1_BUSY(1) |
1288                         S_008014_DB2_BUSY(1) | S_008014_DB3_BUSY(1) |
1289                         S_008014_CB0_BUSY(1) | S_008014_CB1_BUSY(1) |
1290                         S_008014_CB2_BUSY(1) | S_008014_CB3_BUSY(1);
1291         u32 tmp;
1292
1293         if (!(RREG32(GRBM_STATUS) & GUI_ACTIVE))
1294                 return;
1295
1296         dev_info(rdev->dev, "  R_008010_GRBM_STATUS      = 0x%08X\n",
1297                 RREG32(R_008010_GRBM_STATUS));
1298         dev_info(rdev->dev, "  R_008014_GRBM_STATUS2     = 0x%08X\n",
1299                 RREG32(R_008014_GRBM_STATUS2));
1300         dev_info(rdev->dev, "  R_000E50_SRBM_STATUS      = 0x%08X\n",
1301                 RREG32(R_000E50_SRBM_STATUS));
1302         dev_info(rdev->dev, "  R_008674_CP_STALLED_STAT1 = 0x%08X\n",
1303                 RREG32(CP_STALLED_STAT1));
1304         dev_info(rdev->dev, "  R_008678_CP_STALLED_STAT2 = 0x%08X\n",
1305                 RREG32(CP_STALLED_STAT2));
1306         dev_info(rdev->dev, "  R_00867C_CP_BUSY_STAT     = 0x%08X\n",
1307                 RREG32(CP_BUSY_STAT));
1308         dev_info(rdev->dev, "  R_008680_CP_STAT          = 0x%08X\n",
1309                 RREG32(CP_STAT));
1310
1311         /* Disable CP parsing/prefetching */
1312         WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1));
1313
1314         /* Check if any of the rendering block is busy and reset it */
1315         if ((RREG32(R_008010_GRBM_STATUS) & grbm_busy_mask) ||
1316             (RREG32(R_008014_GRBM_STATUS2) & grbm2_busy_mask)) {
1317                 tmp = S_008020_SOFT_RESET_CR(1) |
1318                         S_008020_SOFT_RESET_DB(1) |
1319                         S_008020_SOFT_RESET_CB(1) |
1320                         S_008020_SOFT_RESET_PA(1) |
1321                         S_008020_SOFT_RESET_SC(1) |
1322                         S_008020_SOFT_RESET_SMX(1) |
1323                         S_008020_SOFT_RESET_SPI(1) |
1324                         S_008020_SOFT_RESET_SX(1) |
1325                         S_008020_SOFT_RESET_SH(1) |
1326                         S_008020_SOFT_RESET_TC(1) |
1327                         S_008020_SOFT_RESET_TA(1) |
1328                         S_008020_SOFT_RESET_VC(1) |
1329                         S_008020_SOFT_RESET_VGT(1);
1330                 dev_info(rdev->dev, "  R_008020_GRBM_SOFT_RESET=0x%08X\n", tmp);
1331                 WREG32(R_008020_GRBM_SOFT_RESET, tmp);
1332                 RREG32(R_008020_GRBM_SOFT_RESET);
1333                 mdelay(15);
1334                 WREG32(R_008020_GRBM_SOFT_RESET, 0);
1335         }
1336         /* Reset CP (we always reset CP) */
1337         tmp = S_008020_SOFT_RESET_CP(1);
1338         dev_info(rdev->dev, "R_008020_GRBM_SOFT_RESET=0x%08X\n", tmp);
1339         WREG32(R_008020_GRBM_SOFT_RESET, tmp);
1340         RREG32(R_008020_GRBM_SOFT_RESET);
1341         mdelay(15);
1342         WREG32(R_008020_GRBM_SOFT_RESET, 0);
1343
1344         dev_info(rdev->dev, "  R_008010_GRBM_STATUS      = 0x%08X\n",
1345                 RREG32(R_008010_GRBM_STATUS));
1346         dev_info(rdev->dev, "  R_008014_GRBM_STATUS2     = 0x%08X\n",
1347                 RREG32(R_008014_GRBM_STATUS2));
1348         dev_info(rdev->dev, "  R_000E50_SRBM_STATUS      = 0x%08X\n",
1349                 RREG32(R_000E50_SRBM_STATUS));
1350         dev_info(rdev->dev, "  R_008674_CP_STALLED_STAT1 = 0x%08X\n",
1351                 RREG32(CP_STALLED_STAT1));
1352         dev_info(rdev->dev, "  R_008678_CP_STALLED_STAT2 = 0x%08X\n",
1353                 RREG32(CP_STALLED_STAT2));
1354         dev_info(rdev->dev, "  R_00867C_CP_BUSY_STAT     = 0x%08X\n",
1355                 RREG32(CP_BUSY_STAT));
1356         dev_info(rdev->dev, "  R_008680_CP_STAT          = 0x%08X\n",
1357                 RREG32(CP_STAT));
1358
1359 }
1360
1361 static void r600_gpu_soft_reset_dma(struct radeon_device *rdev)
1362 {
1363         u32 tmp;
1364
1365         if (RREG32(DMA_STATUS_REG) & DMA_IDLE)
1366                 return;
1367
1368         dev_info(rdev->dev, "  R_00D034_DMA_STATUS_REG   = 0x%08X\n",
1369                 RREG32(DMA_STATUS_REG));
1370
1371         /* Disable DMA */
1372         tmp = RREG32(DMA_RB_CNTL);
1373         tmp &= ~DMA_RB_ENABLE;
1374         WREG32(DMA_RB_CNTL, tmp);
1375
1376         /* Reset dma */
1377         if (rdev->family >= CHIP_RV770)
1378                 WREG32(SRBM_SOFT_RESET, RV770_SOFT_RESET_DMA);
1379         else
1380                 WREG32(SRBM_SOFT_RESET, SOFT_RESET_DMA);
1381         RREG32(SRBM_SOFT_RESET);
1382         udelay(50);
1383         WREG32(SRBM_SOFT_RESET, 0);
1384
1385         dev_info(rdev->dev, "  R_00D034_DMA_STATUS_REG   = 0x%08X\n",
1386                 RREG32(DMA_STATUS_REG));
1387 }
1388
1389 static int r600_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
1390 {
1391         struct rv515_mc_save save;
1392
1393         if (!(RREG32(GRBM_STATUS) & GUI_ACTIVE))
1394                 reset_mask &= ~(RADEON_RESET_GFX | RADEON_RESET_COMPUTE);
1395
1396         if (RREG32(DMA_STATUS_REG) & DMA_IDLE)
1397                 reset_mask &= ~RADEON_RESET_DMA;
1398
1399         if (reset_mask == 0)
1400                 return 0;
1401
1402         dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
1403
1404         rv515_mc_stop(rdev, &save);
1405         if (r600_mc_wait_for_idle(rdev)) {
1406                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1407         }
1408
1409         if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE))
1410                 r600_gpu_soft_reset_gfx(rdev);
1411
1412         if (reset_mask & RADEON_RESET_DMA)
1413                 r600_gpu_soft_reset_dma(rdev);
1414
1415         /* Wait a little for things to settle down */
1416         mdelay(1);
1417
1418         rv515_mc_resume(rdev, &save);
1419         return 0;
1420 }
1421
1422 bool r600_gpu_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
1423 {
1424         u32 srbm_status;
1425         u32 grbm_status;
1426         u32 grbm_status2;
1427
1428         srbm_status = RREG32(R_000E50_SRBM_STATUS);
1429         grbm_status = RREG32(R_008010_GRBM_STATUS);
1430         grbm_status2 = RREG32(R_008014_GRBM_STATUS2);
1431         if (!G_008010_GUI_ACTIVE(grbm_status)) {
1432                 radeon_ring_lockup_update(ring);
1433                 return false;
1434         }
1435         /* force CP activities */
1436         radeon_ring_force_activity(rdev, ring);
1437         return radeon_ring_test_lockup(rdev, ring);
1438 }
1439
1440 /**
1441  * r600_dma_is_lockup - Check if the DMA engine is locked up
1442  *
1443  * @rdev: radeon_device pointer
1444  * @ring: radeon_ring structure holding ring information
1445  *
1446  * Check if the async DMA engine is locked up (r6xx-evergreen).
1447  * Returns true if the engine appears to be locked up, false if not.
1448  */
1449 bool r600_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
1450 {
1451         u32 dma_status_reg;
1452
1453         dma_status_reg = RREG32(DMA_STATUS_REG);
1454         if (dma_status_reg & DMA_IDLE) {
1455                 radeon_ring_lockup_update(ring);
1456                 return false;
1457         }
1458         /* force ring activities */
1459         radeon_ring_force_activity(rdev, ring);
1460         return radeon_ring_test_lockup(rdev, ring);
1461 }
1462
1463 int r600_asic_reset(struct radeon_device *rdev)
1464 {
1465         return r600_gpu_soft_reset(rdev, (RADEON_RESET_GFX |
1466                                           RADEON_RESET_COMPUTE |
1467                                           RADEON_RESET_DMA));
1468 }
1469
1470 u32 r6xx_remap_render_backend(struct radeon_device *rdev,
1471                               u32 tiling_pipe_num,
1472                               u32 max_rb_num,
1473                               u32 total_max_rb_num,
1474                               u32 disabled_rb_mask)
1475 {
1476         u32 rendering_pipe_num, rb_num_width, req_rb_num;
1477         u32 pipe_rb_ratio, pipe_rb_remain, tmp;
1478         u32 data = 0, mask = 1 << (max_rb_num - 1);
1479         unsigned i, j;
1480
1481         /* mask out the RBs that don't exist on that asic */
1482         tmp = disabled_rb_mask | ((0xff << max_rb_num) & 0xff);
1483         /* make sure at least one RB is available */
1484         if ((tmp & 0xff) != 0xff)
1485                 disabled_rb_mask = tmp;
1486
1487         rendering_pipe_num = 1 << tiling_pipe_num;
1488         req_rb_num = total_max_rb_num - r600_count_pipe_bits(disabled_rb_mask);
1489         KASSERT(rendering_pipe_num >= req_rb_num, ("rendering_pipe_num < req_rb_num"));
1490
1491         pipe_rb_ratio = rendering_pipe_num / req_rb_num;
1492         pipe_rb_remain = rendering_pipe_num - pipe_rb_ratio * req_rb_num;
1493
1494         if (rdev->family <= CHIP_RV740) {
1495                 /* r6xx/r7xx */
1496                 rb_num_width = 2;
1497         } else {
1498                 /* eg+ */
1499                 rb_num_width = 4;
1500         }
1501
1502         for (i = 0; i < max_rb_num; i++) {
1503                 if (!(mask & disabled_rb_mask)) {
1504                         for (j = 0; j < pipe_rb_ratio; j++) {
1505                                 data <<= rb_num_width;
1506                                 data |= max_rb_num - i - 1;
1507                         }
1508                         if (pipe_rb_remain) {
1509                                 data <<= rb_num_width;
1510                                 data |= max_rb_num - i - 1;
1511                                 pipe_rb_remain--;
1512                         }
1513                 }
1514                 mask >>= 1;
1515         }
1516
1517         return data;
1518 }
1519
1520 int r600_count_pipe_bits(uint32_t val)
1521 {
1522         return hweight32(val);
1523 }
1524
1525 static void r600_gpu_init(struct radeon_device *rdev)
1526 {
1527         u32 tiling_config;
1528         u32 ramcfg;
1529         u32 cc_rb_backend_disable;
1530         u32 cc_gc_shader_pipe_config;
1531         u32 tmp;
1532         int i, j;
1533         u32 sq_config;
1534         u32 sq_gpr_resource_mgmt_1 = 0;
1535         u32 sq_gpr_resource_mgmt_2 = 0;
1536         u32 sq_thread_resource_mgmt = 0;
1537         u32 sq_stack_resource_mgmt_1 = 0;
1538         u32 sq_stack_resource_mgmt_2 = 0;
1539         u32 disabled_rb_mask;
1540
1541         rdev->config.r600.tiling_group_size = 256;
1542         switch (rdev->family) {
1543         case CHIP_R600:
1544                 rdev->config.r600.max_pipes = 4;
1545                 rdev->config.r600.max_tile_pipes = 8;
1546                 rdev->config.r600.max_simds = 4;
1547                 rdev->config.r600.max_backends = 4;
1548                 rdev->config.r600.max_gprs = 256;
1549                 rdev->config.r600.max_threads = 192;
1550                 rdev->config.r600.max_stack_entries = 256;
1551                 rdev->config.r600.max_hw_contexts = 8;
1552                 rdev->config.r600.max_gs_threads = 16;
1553                 rdev->config.r600.sx_max_export_size = 128;
1554                 rdev->config.r600.sx_max_export_pos_size = 16;
1555                 rdev->config.r600.sx_max_export_smx_size = 128;
1556                 rdev->config.r600.sq_num_cf_insts = 2;
1557                 break;
1558         case CHIP_RV630:
1559         case CHIP_RV635:
1560                 rdev->config.r600.max_pipes = 2;
1561                 rdev->config.r600.max_tile_pipes = 2;
1562                 rdev->config.r600.max_simds = 3;
1563                 rdev->config.r600.max_backends = 1;
1564                 rdev->config.r600.max_gprs = 128;
1565                 rdev->config.r600.max_threads = 192;
1566                 rdev->config.r600.max_stack_entries = 128;
1567                 rdev->config.r600.max_hw_contexts = 8;
1568                 rdev->config.r600.max_gs_threads = 4;
1569                 rdev->config.r600.sx_max_export_size = 128;
1570                 rdev->config.r600.sx_max_export_pos_size = 16;
1571                 rdev->config.r600.sx_max_export_smx_size = 128;
1572                 rdev->config.r600.sq_num_cf_insts = 2;
1573                 break;
1574         case CHIP_RV610:
1575         case CHIP_RV620:
1576         case CHIP_RS780:
1577         case CHIP_RS880:
1578                 rdev->config.r600.max_pipes = 1;
1579                 rdev->config.r600.max_tile_pipes = 1;
1580                 rdev->config.r600.max_simds = 2;
1581                 rdev->config.r600.max_backends = 1;
1582                 rdev->config.r600.max_gprs = 128;
1583                 rdev->config.r600.max_threads = 192;
1584                 rdev->config.r600.max_stack_entries = 128;
1585                 rdev->config.r600.max_hw_contexts = 4;
1586                 rdev->config.r600.max_gs_threads = 4;
1587                 rdev->config.r600.sx_max_export_size = 128;
1588                 rdev->config.r600.sx_max_export_pos_size = 16;
1589                 rdev->config.r600.sx_max_export_smx_size = 128;
1590                 rdev->config.r600.sq_num_cf_insts = 1;
1591                 break;
1592         case CHIP_RV670:
1593                 rdev->config.r600.max_pipes = 4;
1594                 rdev->config.r600.max_tile_pipes = 4;
1595                 rdev->config.r600.max_simds = 4;
1596                 rdev->config.r600.max_backends = 4;
1597                 rdev->config.r600.max_gprs = 192;
1598                 rdev->config.r600.max_threads = 192;
1599                 rdev->config.r600.max_stack_entries = 256;
1600                 rdev->config.r600.max_hw_contexts = 8;
1601                 rdev->config.r600.max_gs_threads = 16;
1602                 rdev->config.r600.sx_max_export_size = 128;
1603                 rdev->config.r600.sx_max_export_pos_size = 16;
1604                 rdev->config.r600.sx_max_export_smx_size = 128;
1605                 rdev->config.r600.sq_num_cf_insts = 2;
1606                 break;
1607         default:
1608                 break;
1609         }
1610
1611         /* Initialize HDP */
1612         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
1613                 WREG32((0x2c14 + j), 0x00000000);
1614                 WREG32((0x2c18 + j), 0x00000000);
1615                 WREG32((0x2c1c + j), 0x00000000);
1616                 WREG32((0x2c20 + j), 0x00000000);
1617                 WREG32((0x2c24 + j), 0x00000000);
1618         }
1619
1620         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
1621
1622         /* Setup tiling */
1623         tiling_config = 0;
1624         ramcfg = RREG32(RAMCFG);
1625         switch (rdev->config.r600.max_tile_pipes) {
1626         case 1:
1627                 tiling_config |= PIPE_TILING(0);
1628                 break;
1629         case 2:
1630                 tiling_config |= PIPE_TILING(1);
1631                 break;
1632         case 4:
1633                 tiling_config |= PIPE_TILING(2);
1634                 break;
1635         case 8:
1636                 tiling_config |= PIPE_TILING(3);
1637                 break;
1638         default:
1639                 break;
1640         }
1641         rdev->config.r600.tiling_npipes = rdev->config.r600.max_tile_pipes;
1642         rdev->config.r600.tiling_nbanks = 4 << ((ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT);
1643         tiling_config |= BANK_TILING((ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT);
1644         tiling_config |= GROUP_SIZE((ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT);
1645
1646         tmp = (ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT;
1647         if (tmp > 3) {
1648                 tiling_config |= ROW_TILING(3);
1649                 tiling_config |= SAMPLE_SPLIT(3);
1650         } else {
1651                 tiling_config |= ROW_TILING(tmp);
1652                 tiling_config |= SAMPLE_SPLIT(tmp);
1653         }
1654         tiling_config |= BANK_SWAPS(1);
1655
1656         cc_rb_backend_disable = RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000;
1657         tmp = R6XX_MAX_BACKENDS -
1658                 r600_count_pipe_bits((cc_rb_backend_disable >> 16) & R6XX_MAX_BACKENDS_MASK);
1659         if (tmp < rdev->config.r600.max_backends) {
1660                 rdev->config.r600.max_backends = tmp;
1661         }
1662
1663         cc_gc_shader_pipe_config = RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0x00ffff00;
1664         tmp = R6XX_MAX_PIPES -
1665                 r600_count_pipe_bits((cc_gc_shader_pipe_config >> 8) & R6XX_MAX_PIPES_MASK);
1666         if (tmp < rdev->config.r600.max_pipes) {
1667                 rdev->config.r600.max_pipes = tmp;
1668         }
1669         tmp = R6XX_MAX_SIMDS -
1670                 r600_count_pipe_bits((cc_gc_shader_pipe_config >> 16) & R6XX_MAX_SIMDS_MASK);
1671         if (tmp < rdev->config.r600.max_simds) {
1672                 rdev->config.r600.max_simds = tmp;
1673         }
1674
1675         disabled_rb_mask = (RREG32(CC_RB_BACKEND_DISABLE) >> 16) & R6XX_MAX_BACKENDS_MASK;
1676         tmp = (tiling_config & PIPE_TILING__MASK) >> PIPE_TILING__SHIFT;
1677         tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.r600.max_backends,
1678                                         R6XX_MAX_BACKENDS, disabled_rb_mask);
1679         tiling_config |= tmp << 16;
1680         rdev->config.r600.backend_map = tmp;
1681
1682         rdev->config.r600.tile_config = tiling_config;
1683         WREG32(GB_TILING_CONFIG, tiling_config);
1684         WREG32(DCP_TILING_CONFIG, tiling_config & 0xffff);
1685         WREG32(HDP_TILING_CONFIG, tiling_config & 0xffff);
1686         WREG32(DMA_TILING_CONFIG, tiling_config & 0xffff);
1687
1688         tmp = R6XX_MAX_PIPES - r600_count_pipe_bits((cc_gc_shader_pipe_config & INACTIVE_QD_PIPES_MASK) >> 8);
1689         WREG32(VGT_OUT_DEALLOC_CNTL, (tmp * 4) & DEALLOC_DIST_MASK);
1690         WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, ((tmp * 4) - 2) & VTX_REUSE_DEPTH_MASK);
1691
1692         /* Setup some CP states */
1693         WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) | ROQ_IB2_START(0x2b)));
1694         WREG32(CP_MEQ_THRESHOLDS, (MEQ_END(0x40) | ROQ_END(0x40)));
1695
1696         WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO | SYNC_GRADIENT |
1697                              SYNC_WALKER | SYNC_ALIGNER));
1698         /* Setup various GPU states */
1699         if (rdev->family == CHIP_RV670)
1700                 WREG32(ARB_GDEC_RD_CNTL, 0x00000021);
1701
1702         tmp = RREG32(SX_DEBUG_1);
1703         tmp |= SMX_EVENT_RELEASE;
1704         if ((rdev->family > CHIP_R600))
1705                 tmp |= ENABLE_NEW_SMX_ADDRESS;
1706         WREG32(SX_DEBUG_1, tmp);
1707
1708         if (((rdev->family) == CHIP_R600) ||
1709             ((rdev->family) == CHIP_RV630) ||
1710             ((rdev->family) == CHIP_RV610) ||
1711             ((rdev->family) == CHIP_RV620) ||
1712             ((rdev->family) == CHIP_RS780) ||
1713             ((rdev->family) == CHIP_RS880)) {
1714                 WREG32(DB_DEBUG, PREZ_MUST_WAIT_FOR_POSTZ_DONE);
1715         } else {
1716                 WREG32(DB_DEBUG, 0);
1717         }
1718         WREG32(DB_WATERMARKS, (DEPTH_FREE(4) | DEPTH_CACHELINE_FREE(16) |
1719                                DEPTH_FLUSH(16) | DEPTH_PENDING_FREE(4)));
1720
1721         WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
1722         WREG32(VGT_NUM_INSTANCES, 0);
1723
1724         WREG32(SPI_CONFIG_CNTL, GPR_WRITE_PRIORITY(0));
1725         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(0));
1726
1727         tmp = RREG32(SQ_MS_FIFO_SIZES);
1728         if (((rdev->family) == CHIP_RV610) ||
1729             ((rdev->family) == CHIP_RV620) ||
1730             ((rdev->family) == CHIP_RS780) ||
1731             ((rdev->family) == CHIP_RS880)) {
1732                 tmp = (CACHE_FIFO_SIZE(0xa) |
1733                        FETCH_FIFO_HIWATER(0xa) |
1734                        DONE_FIFO_HIWATER(0xe0) |
1735                        ALU_UPDATE_FIFO_HIWATER(0x8));
1736         } else if (((rdev->family) == CHIP_R600) ||
1737                    ((rdev->family) == CHIP_RV630)) {
1738                 tmp &= ~DONE_FIFO_HIWATER(0xff);
1739                 tmp |= DONE_FIFO_HIWATER(0x4);
1740         }
1741         WREG32(SQ_MS_FIFO_SIZES, tmp);
1742
1743         /* SQ_CONFIG, SQ_GPR_RESOURCE_MGMT, SQ_THREAD_RESOURCE_MGMT, SQ_STACK_RESOURCE_MGMT
1744          * should be adjusted as needed by the 2D/3D drivers.  This just sets default values
1745          */
1746         sq_config = RREG32(SQ_CONFIG);
1747         sq_config &= ~(PS_PRIO(3) |
1748                        VS_PRIO(3) |
1749                        GS_PRIO(3) |
1750                        ES_PRIO(3));
1751         sq_config |= (DX9_CONSTS |
1752                       VC_ENABLE |
1753                       PS_PRIO(0) |
1754                       VS_PRIO(1) |
1755                       GS_PRIO(2) |
1756                       ES_PRIO(3));
1757
1758         if ((rdev->family) == CHIP_R600) {
1759                 sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(124) |
1760                                           NUM_VS_GPRS(124) |
1761                                           NUM_CLAUSE_TEMP_GPRS(4));
1762                 sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(0) |
1763                                           NUM_ES_GPRS(0));
1764                 sq_thread_resource_mgmt = (NUM_PS_THREADS(136) |
1765                                            NUM_VS_THREADS(48) |
1766                                            NUM_GS_THREADS(4) |
1767                                            NUM_ES_THREADS(4));
1768                 sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(128) |
1769                                             NUM_VS_STACK_ENTRIES(128));
1770                 sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(0) |
1771                                             NUM_ES_STACK_ENTRIES(0));
1772         } else if (((rdev->family) == CHIP_RV610) ||
1773                    ((rdev->family) == CHIP_RV620) ||
1774                    ((rdev->family) == CHIP_RS780) ||
1775                    ((rdev->family) == CHIP_RS880)) {
1776                 /* no vertex cache */
1777                 sq_config &= ~VC_ENABLE;
1778
1779                 sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) |
1780                                           NUM_VS_GPRS(44) |
1781                                           NUM_CLAUSE_TEMP_GPRS(2));
1782                 sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(17) |
1783                                           NUM_ES_GPRS(17));
1784                 sq_thread_resource_mgmt = (NUM_PS_THREADS(79) |
1785                                            NUM_VS_THREADS(78) |
1786                                            NUM_GS_THREADS(4) |
1787                                            NUM_ES_THREADS(31));
1788                 sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(40) |
1789                                             NUM_VS_STACK_ENTRIES(40));
1790                 sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(32) |
1791                                             NUM_ES_STACK_ENTRIES(16));
1792         } else if (((rdev->family) == CHIP_RV630) ||
1793                    ((rdev->family) == CHIP_RV635)) {
1794                 sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) |
1795                                           NUM_VS_GPRS(44) |
1796                                           NUM_CLAUSE_TEMP_GPRS(2));
1797                 sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(18) |
1798                                           NUM_ES_GPRS(18));
1799                 sq_thread_resource_mgmt = (NUM_PS_THREADS(79) |
1800                                            NUM_VS_THREADS(78) |
1801                                            NUM_GS_THREADS(4) |
1802                                            NUM_ES_THREADS(31));
1803                 sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(40) |
1804                                             NUM_VS_STACK_ENTRIES(40));
1805                 sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(32) |
1806                                             NUM_ES_STACK_ENTRIES(16));
1807         } else if ((rdev->family) == CHIP_RV670) {
1808                 sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) |
1809                                           NUM_VS_GPRS(44) |
1810                                           NUM_CLAUSE_TEMP_GPRS(2));
1811                 sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(17) |
1812                                           NUM_ES_GPRS(17));
1813                 sq_thread_resource_mgmt = (NUM_PS_THREADS(79) |
1814                                            NUM_VS_THREADS(78) |
1815                                            NUM_GS_THREADS(4) |
1816                                            NUM_ES_THREADS(31));
1817                 sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(64) |
1818                                             NUM_VS_STACK_ENTRIES(64));
1819                 sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(64) |
1820                                             NUM_ES_STACK_ENTRIES(64));
1821         }
1822
1823         WREG32(SQ_CONFIG, sq_config);
1824         WREG32(SQ_GPR_RESOURCE_MGMT_1,  sq_gpr_resource_mgmt_1);
1825         WREG32(SQ_GPR_RESOURCE_MGMT_2,  sq_gpr_resource_mgmt_2);
1826         WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
1827         WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
1828         WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
1829
1830         if (((rdev->family) == CHIP_RV610) ||
1831             ((rdev->family) == CHIP_RV620) ||
1832             ((rdev->family) == CHIP_RS780) ||
1833             ((rdev->family) == CHIP_RS880)) {
1834                 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(TC_ONLY));
1835         } else {
1836                 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC));
1837         }
1838
1839         /* More default values. 2D/3D driver should adjust as needed */
1840         WREG32(PA_SC_AA_SAMPLE_LOCS_2S, (S0_X(0xc) | S0_Y(0x4) |
1841                                          S1_X(0x4) | S1_Y(0xc)));
1842         WREG32(PA_SC_AA_SAMPLE_LOCS_4S, (S0_X(0xe) | S0_Y(0xe) |
1843                                          S1_X(0x2) | S1_Y(0x2) |
1844                                          S2_X(0xa) | S2_Y(0x6) |
1845                                          S3_X(0x6) | S3_Y(0xa)));
1846         WREG32(PA_SC_AA_SAMPLE_LOCS_8S_WD0, (S0_X(0xe) | S0_Y(0xb) |
1847                                              S1_X(0x4) | S1_Y(0xc) |
1848                                              S2_X(0x1) | S2_Y(0x6) |
1849                                              S3_X(0xa) | S3_Y(0xe)));
1850         WREG32(PA_SC_AA_SAMPLE_LOCS_8S_WD1, (S4_X(0x6) | S4_Y(0x1) |
1851                                              S5_X(0x0) | S5_Y(0x0) |
1852                                              S6_X(0xb) | S6_Y(0x4) |
1853                                              S7_X(0x7) | S7_Y(0x8)));
1854
1855         WREG32(VGT_STRMOUT_EN, 0);
1856         tmp = rdev->config.r600.max_pipes * 16;
1857         switch (rdev->family) {
1858         case CHIP_RV610:
1859         case CHIP_RV620:
1860         case CHIP_RS780:
1861         case CHIP_RS880:
1862                 tmp += 32;
1863                 break;
1864         case CHIP_RV670:
1865                 tmp += 128;
1866                 break;
1867         default:
1868                 break;
1869         }
1870         if (tmp > 256) {
1871                 tmp = 256;
1872         }
1873         WREG32(VGT_ES_PER_GS, 128);
1874         WREG32(VGT_GS_PER_ES, tmp);
1875         WREG32(VGT_GS_PER_VS, 2);
1876         WREG32(VGT_GS_VERTEX_REUSE, 16);
1877
1878         /* more default values. 2D/3D driver should adjust as needed */
1879         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
1880         WREG32(VGT_STRMOUT_EN, 0);
1881         WREG32(SX_MISC, 0);
1882         WREG32(PA_SC_MODE_CNTL, 0);
1883         WREG32(PA_SC_AA_CONFIG, 0);
1884         WREG32(PA_SC_LINE_STIPPLE, 0);
1885         WREG32(SPI_INPUT_Z, 0);
1886         WREG32(SPI_PS_IN_CONTROL_0, NUM_INTERP(2));
1887         WREG32(CB_COLOR7_FRAG, 0);
1888
1889         /* Clear render buffer base addresses */
1890         WREG32(CB_COLOR0_BASE, 0);
1891         WREG32(CB_COLOR1_BASE, 0);
1892         WREG32(CB_COLOR2_BASE, 0);
1893         WREG32(CB_COLOR3_BASE, 0);
1894         WREG32(CB_COLOR4_BASE, 0);
1895         WREG32(CB_COLOR5_BASE, 0);
1896         WREG32(CB_COLOR6_BASE, 0);
1897         WREG32(CB_COLOR7_BASE, 0);
1898         WREG32(CB_COLOR7_FRAG, 0);
1899
1900         switch (rdev->family) {
1901         case CHIP_RV610:
1902         case CHIP_RV620:
1903         case CHIP_RS780:
1904         case CHIP_RS880:
1905                 tmp = TC_L2_SIZE(8);
1906                 break;
1907         case CHIP_RV630:
1908         case CHIP_RV635:
1909                 tmp = TC_L2_SIZE(4);
1910                 break;
1911         case CHIP_R600:
1912                 tmp = TC_L2_SIZE(0) | L2_DISABLE_LATE_HIT;
1913                 break;
1914         default:
1915                 tmp = TC_L2_SIZE(0);
1916                 break;
1917         }
1918         WREG32(TC_CNTL, tmp);
1919
1920         tmp = RREG32(HDP_HOST_PATH_CNTL);
1921         WREG32(HDP_HOST_PATH_CNTL, tmp);
1922
1923         tmp = RREG32(ARB_POP);
1924         tmp |= ENABLE_TC128;
1925         WREG32(ARB_POP, tmp);
1926
1927         WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
1928         WREG32(PA_CL_ENHANCE, (CLIP_VTX_REORDER_ENA |
1929                                NUM_CLIP_SEQ(3)));
1930         WREG32(PA_SC_ENHANCE, FORCE_EOV_MAX_CLK_CNT(4095));
1931         WREG32(VC_ENHANCE, 0);
1932 }
1933
1934
1935 /*
1936  * Indirect registers accessor
1937  */
1938 u32 r600_pciep_rreg(struct radeon_device *rdev, u32 reg)
1939 {
1940         u32 r;
1941
1942         WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
1943         (void)RREG32(PCIE_PORT_INDEX);
1944         r = RREG32(PCIE_PORT_DATA);
1945         return r;
1946 }
1947
1948 void r600_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
1949 {
1950         WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
1951         (void)RREG32(PCIE_PORT_INDEX);
1952         WREG32(PCIE_PORT_DATA, (v));
1953         (void)RREG32(PCIE_PORT_DATA);
1954 }
1955
1956 /*
1957  * CP & Ring
1958  */
1959 void r600_cp_stop(struct radeon_device *rdev)
1960 {
1961         radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
1962         WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1));
1963         WREG32(SCRATCH_UMSK, 0);
1964         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1965 }
1966
1967 int r600_init_microcode(struct radeon_device *rdev)
1968 {
1969         const char *chip_name;
1970         const char *rlc_chip_name;
1971         size_t pfp_req_size, me_req_size, rlc_req_size;
1972         char fw_name[30];
1973         int err;
1974
1975         DRM_DEBUG("\n");
1976
1977         switch (rdev->family) {
1978         case CHIP_R600:
1979                 chip_name = "R600";
1980                 rlc_chip_name = "R600";
1981                 break;
1982         case CHIP_RV610:
1983                 chip_name = "RV610";
1984                 rlc_chip_name = "R600";
1985                 break;
1986         case CHIP_RV630:
1987                 chip_name = "RV630";
1988                 rlc_chip_name = "R600";
1989                 break;
1990         case CHIP_RV620:
1991                 chip_name = "RV620";
1992                 rlc_chip_name = "R600";
1993                 break;
1994         case CHIP_RV635:
1995                 chip_name = "RV635";
1996                 rlc_chip_name = "R600";
1997                 break;
1998         case CHIP_RV670:
1999                 chip_name = "RV670";
2000                 rlc_chip_name = "R600";
2001                 break;
2002         case CHIP_RS780:
2003         case CHIP_RS880:
2004                 chip_name = "RS780";
2005                 rlc_chip_name = "R600";
2006                 break;
2007         case CHIP_RV770:
2008                 chip_name = "RV770";
2009                 rlc_chip_name = "R700";
2010                 break;
2011         case CHIP_RV730:
2012         case CHIP_RV740:
2013                 chip_name = "RV730";
2014                 rlc_chip_name = "R700";
2015                 break;
2016         case CHIP_RV710:
2017                 chip_name = "RV710";
2018                 rlc_chip_name = "R700";
2019                 break;
2020         case CHIP_CEDAR:
2021                 chip_name = "CEDAR";
2022                 rlc_chip_name = "CEDAR";
2023                 break;
2024         case CHIP_REDWOOD:
2025                 chip_name = "REDWOOD";
2026                 rlc_chip_name = "REDWOOD";
2027                 break;
2028         case CHIP_JUNIPER:
2029                 chip_name = "JUNIPER";
2030                 rlc_chip_name = "JUNIPER";
2031                 break;
2032         case CHIP_CYPRESS:
2033         case CHIP_HEMLOCK:
2034                 chip_name = "CYPRESS";
2035                 rlc_chip_name = "CYPRESS";
2036                 break;
2037         case CHIP_PALM:
2038                 chip_name = "PALM";
2039                 rlc_chip_name = "SUMO";
2040                 break;
2041         case CHIP_SUMO:
2042                 chip_name = "SUMO";
2043                 rlc_chip_name = "SUMO";
2044                 break;
2045         case CHIP_SUMO2:
2046                 chip_name = "SUMO2";
2047                 rlc_chip_name = "SUMO";
2048                 break;
2049         default: panic("%s: Unsupported family %d", __func__, rdev->family);
2050         }
2051
2052         if (rdev->family >= CHIP_CEDAR) {
2053                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
2054                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
2055                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
2056         } else if (rdev->family >= CHIP_RV770) {
2057                 pfp_req_size = R700_PFP_UCODE_SIZE * 4;
2058                 me_req_size = R700_PM4_UCODE_SIZE * 4;
2059                 rlc_req_size = R700_RLC_UCODE_SIZE * 4;
2060         } else {
2061                 pfp_req_size = PFP_UCODE_SIZE * 4;
2062                 me_req_size = PM4_UCODE_SIZE * 12;
2063                 rlc_req_size = RLC_UCODE_SIZE * 4;
2064         }
2065
2066         DRM_INFO("Loading %s Microcode\n", chip_name);
2067         err = 0;
2068
2069         snprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_pfp", chip_name);
2070         rdev->pfp_fw = firmware_get(fw_name);
2071         if (rdev->pfp_fw == NULL) {
2072                 err = -ENOENT;
2073                 goto out;
2074         }
2075         if (rdev->pfp_fw->datasize != pfp_req_size) {
2076                 DRM_ERROR(
2077                        "r600_cp: Bogus length %zu in firmware \"%s\"\n",
2078                        rdev->pfp_fw->datasize, fw_name);
2079                 err = -EINVAL;
2080                 goto out;
2081         }
2082
2083         snprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_me", chip_name);
2084         rdev->me_fw = firmware_get(fw_name);
2085         if (rdev->me_fw == NULL) {
2086                 err = -ENOENT;
2087                 goto out;
2088         }
2089         if (rdev->me_fw->datasize != me_req_size) {
2090                 DRM_ERROR(
2091                        "r600_cp: Bogus length %zu in firmware \"%s\"\n",
2092                        rdev->me_fw->datasize, fw_name);
2093                 err = -EINVAL;
2094         }
2095
2096         snprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_rlc", rlc_chip_name);
2097         rdev->rlc_fw = firmware_get(fw_name);
2098         if (rdev->rlc_fw == NULL) {
2099                 err = -ENOENT;
2100                 goto out;
2101         }
2102         if (rdev->rlc_fw->datasize != rlc_req_size) {
2103                 DRM_ERROR(
2104                        "r600_rlc: Bogus length %zu in firmware \"%s\"\n",
2105                        rdev->rlc_fw->datasize, fw_name);
2106                 err = -EINVAL;
2107         }
2108
2109 out:
2110         if (err) {
2111                 if (err != -EINVAL)
2112                         DRM_ERROR(
2113                                "r600_cp: Failed to load firmware \"%s\"\n",
2114                                fw_name);
2115                 if (rdev->pfp_fw != NULL) {
2116                         firmware_put(rdev->pfp_fw, FIRMWARE_UNLOAD);
2117                         rdev->pfp_fw = NULL;
2118                 }
2119                 if (rdev->me_fw != NULL) {
2120                         firmware_put(rdev->me_fw, FIRMWARE_UNLOAD);
2121                         rdev->me_fw = NULL;
2122                 }
2123                 if (rdev->rlc_fw != NULL) {
2124                         firmware_put(rdev->rlc_fw, FIRMWARE_UNLOAD);
2125                         rdev->rlc_fw = NULL;
2126                 }
2127         }
2128         return err;
2129 }
2130
2131 /**
2132  * r600_fini_microcode - drop the firmwares image references
2133  *
2134  * @rdev: radeon_device pointer
2135  *
2136  * Drop the pfp, me and rlc firmwares image references.
2137  * Called at driver shutdown.
2138  */
2139 void r600_fini_microcode(struct radeon_device *rdev)
2140 {
2141
2142         if (rdev->pfp_fw != NULL) {
2143                 firmware_put(rdev->pfp_fw, FIRMWARE_UNLOAD);
2144                 rdev->pfp_fw = NULL;
2145         }
2146
2147         if (rdev->me_fw != NULL) {
2148                 firmware_put(rdev->me_fw, FIRMWARE_UNLOAD);
2149                 rdev->me_fw = NULL;
2150         }
2151
2152         if (rdev->rlc_fw != NULL) {
2153                 firmware_put(rdev->rlc_fw, FIRMWARE_UNLOAD);
2154                 rdev->rlc_fw = NULL;
2155         }
2156 }
2157
2158 static int r600_cp_load_microcode(struct radeon_device *rdev)
2159 {
2160         const __be32 *fw_data;
2161         int i;
2162
2163         if (!rdev->me_fw || !rdev->pfp_fw)
2164                 return -EINVAL;
2165
2166         r600_cp_stop(rdev);
2167
2168         WREG32(CP_RB_CNTL,
2169 #ifdef __BIG_ENDIAN
2170                BUF_SWAP_32BIT |
2171 #endif
2172                RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
2173
2174         /* Reset cp */
2175         WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
2176         RREG32(GRBM_SOFT_RESET);
2177         mdelay(15);
2178         WREG32(GRBM_SOFT_RESET, 0);
2179
2180         WREG32(CP_ME_RAM_WADDR, 0);
2181
2182         fw_data = (const __be32 *)rdev->me_fw->data;
2183         WREG32(CP_ME_RAM_WADDR, 0);
2184         for (i = 0; i < PM4_UCODE_SIZE * 3; i++)
2185                 WREG32(CP_ME_RAM_DATA,
2186                        be32_to_cpup(fw_data++));
2187
2188         fw_data = (const __be32 *)rdev->pfp_fw->data;
2189         WREG32(CP_PFP_UCODE_ADDR, 0);
2190         for (i = 0; i < PFP_UCODE_SIZE; i++)
2191                 WREG32(CP_PFP_UCODE_DATA,
2192                        be32_to_cpup(fw_data++));
2193
2194         WREG32(CP_PFP_UCODE_ADDR, 0);
2195         WREG32(CP_ME_RAM_WADDR, 0);
2196         WREG32(CP_ME_RAM_RADDR, 0);
2197         return 0;
2198 }
2199
2200 int r600_cp_start(struct radeon_device *rdev)
2201 {
2202         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2203         int r;
2204         uint32_t cp_me;
2205
2206         r = radeon_ring_lock(rdev, ring, 7);
2207         if (r) {
2208                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2209                 return r;
2210         }
2211         radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
2212         radeon_ring_write(ring, 0x1);
2213         if (rdev->family >= CHIP_RV770) {
2214                 radeon_ring_write(ring, 0x0);
2215                 radeon_ring_write(ring, rdev->config.rv770.max_hw_contexts - 1);
2216         } else {
2217                 radeon_ring_write(ring, 0x3);
2218                 radeon_ring_write(ring, rdev->config.r600.max_hw_contexts - 1);
2219         }
2220         radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
2221         radeon_ring_write(ring, 0);
2222         radeon_ring_write(ring, 0);
2223         radeon_ring_unlock_commit(rdev, ring);
2224
2225         cp_me = 0xff;
2226         WREG32(R_0086D8_CP_ME_CNTL, cp_me);
2227         return 0;
2228 }
2229
2230 int r600_cp_resume(struct radeon_device *rdev)
2231 {
2232         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2233         u32 tmp;
2234         u32 rb_bufsz;
2235         int r;
2236
2237         /* Reset cp */
2238         WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
2239         RREG32(GRBM_SOFT_RESET);
2240         mdelay(15);
2241         WREG32(GRBM_SOFT_RESET, 0);
2242
2243         /* Set ring buffer size */
2244         rb_bufsz = drm_order(ring->ring_size / 8);
2245         tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
2246 #ifdef __BIG_ENDIAN
2247         tmp |= BUF_SWAP_32BIT;
2248 #endif
2249         WREG32(CP_RB_CNTL, tmp);
2250         WREG32(CP_SEM_WAIT_TIMER, 0x0);
2251
2252         /* Set the write pointer delay */
2253         WREG32(CP_RB_WPTR_DELAY, 0);
2254
2255         /* Initialize the ring buffer's read and write pointers */
2256         WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
2257         WREG32(CP_RB_RPTR_WR, 0);
2258         ring->wptr = 0;
2259         WREG32(CP_RB_WPTR, ring->wptr);
2260
2261         /* set the wb address whether it's enabled or not */
2262         WREG32(CP_RB_RPTR_ADDR,
2263                ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
2264         WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
2265         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
2266
2267         if (rdev->wb.enabled)
2268                 WREG32(SCRATCH_UMSK, 0xff);
2269         else {
2270                 tmp |= RB_NO_UPDATE;
2271                 WREG32(SCRATCH_UMSK, 0);
2272         }
2273
2274         mdelay(1);
2275         WREG32(CP_RB_CNTL, tmp);
2276
2277         WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
2278         WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
2279
2280         ring->rptr = RREG32(CP_RB_RPTR);
2281
2282         r600_cp_start(rdev);
2283         ring->ready = true;
2284         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
2285         if (r) {
2286                 ring->ready = false;
2287                 return r;
2288         }
2289         return 0;
2290 }
2291
2292 void r600_ring_init(struct radeon_device *rdev, struct radeon_ring *ring, unsigned ring_size)
2293 {
2294         u32 rb_bufsz;
2295         int r;
2296
2297         /* Align ring size */
2298         rb_bufsz = drm_order(ring_size / 8);
2299         ring_size = (1 << (rb_bufsz + 1)) * 4;
2300         ring->ring_size = ring_size;
2301         ring->align_mask = 16 - 1;
2302
2303         if (radeon_ring_supports_scratch_reg(rdev, ring)) {
2304                 r = radeon_scratch_get(rdev, &ring->rptr_save_reg);
2305                 if (r) {
2306                         DRM_ERROR("failed to get scratch reg for rptr save (%d).\n", r);
2307                         ring->rptr_save_reg = 0;
2308                 }
2309         }
2310 }
2311
2312 void r600_cp_fini(struct radeon_device *rdev)
2313 {
2314         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2315         r600_cp_stop(rdev);
2316         radeon_ring_fini(rdev, ring);
2317         radeon_scratch_free(rdev, ring->rptr_save_reg);
2318 }
2319
2320 /*
2321  * DMA
2322  * Starting with R600, the GPU has an asynchronous
2323  * DMA engine.  The programming model is very similar
2324  * to the 3D engine (ring buffer, IBs, etc.), but the
2325  * DMA controller has it's own packet format that is
2326  * different form the PM4 format used by the 3D engine.
2327  * It supports copying data, writing embedded data,
2328  * solid fills, and a number of other things.  It also
2329  * has support for tiling/detiling of buffers.
2330  */
2331 /**
2332  * r600_dma_stop - stop the async dma engine
2333  *
2334  * @rdev: radeon_device pointer
2335  *
2336  * Stop the async dma engine (r6xx-evergreen).
2337  */
2338 void r600_dma_stop(struct radeon_device *rdev)
2339 {
2340         u32 rb_cntl = RREG32(DMA_RB_CNTL);
2341
2342         radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
2343
2344         rb_cntl &= ~DMA_RB_ENABLE;
2345         WREG32(DMA_RB_CNTL, rb_cntl);
2346
2347         rdev->ring[R600_RING_TYPE_DMA_INDEX].ready = false;
2348 }
2349
2350 /**
2351  * r600_dma_resume - setup and start the async dma engine
2352  *
2353  * @rdev: radeon_device pointer
2354  *
2355  * Set up the DMA ring buffer and enable it. (r6xx-evergreen).
2356  * Returns 0 for success, error for failure.
2357  */
2358 int r600_dma_resume(struct radeon_device *rdev)
2359 {
2360         struct radeon_ring *ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
2361         u32 rb_cntl, dma_cntl, ib_cntl;
2362         u32 rb_bufsz;
2363         int r;
2364
2365         /* Reset dma */
2366         if (rdev->family >= CHIP_RV770)
2367                 WREG32(SRBM_SOFT_RESET, RV770_SOFT_RESET_DMA);
2368         else
2369                 WREG32(SRBM_SOFT_RESET, SOFT_RESET_DMA);
2370         RREG32(SRBM_SOFT_RESET);
2371         udelay(50);
2372         WREG32(SRBM_SOFT_RESET, 0);
2373
2374         WREG32(DMA_SEM_INCOMPLETE_TIMER_CNTL, 0);
2375         WREG32(DMA_SEM_WAIT_FAIL_TIMER_CNTL, 0);
2376
2377         /* Set ring buffer size in dwords */
2378         rb_bufsz = drm_order(ring->ring_size / 4);
2379         rb_cntl = rb_bufsz << 1;
2380 #ifdef __BIG_ENDIAN
2381         rb_cntl |= DMA_RB_SWAP_ENABLE | DMA_RPTR_WRITEBACK_SWAP_ENABLE;
2382 #endif
2383         WREG32(DMA_RB_CNTL, rb_cntl);
2384
2385         /* Initialize the ring buffer's read and write pointers */
2386         WREG32(DMA_RB_RPTR, 0);
2387         WREG32(DMA_RB_WPTR, 0);
2388
2389         /* set the wb address whether it's enabled or not */
2390         WREG32(DMA_RB_RPTR_ADDR_HI,
2391                upper_32_bits(rdev->wb.gpu_addr + R600_WB_DMA_RPTR_OFFSET) & 0xFF);
2392         WREG32(DMA_RB_RPTR_ADDR_LO,
2393                ((rdev->wb.gpu_addr + R600_WB_DMA_RPTR_OFFSET) & 0xFFFFFFFC));
2394
2395         if (rdev->wb.enabled)
2396                 rb_cntl |= DMA_RPTR_WRITEBACK_ENABLE;
2397
2398         WREG32(DMA_RB_BASE, ring->gpu_addr >> 8);
2399
2400         /* enable DMA IBs */
2401         ib_cntl = DMA_IB_ENABLE;
2402 #ifdef __BIG_ENDIAN
2403         ib_cntl |= DMA_IB_SWAP_ENABLE;
2404 #endif
2405         WREG32(DMA_IB_CNTL, ib_cntl);
2406
2407         dma_cntl = RREG32(DMA_CNTL);
2408         dma_cntl &= ~CTXEMPTY_INT_ENABLE;
2409         WREG32(DMA_CNTL, dma_cntl);
2410
2411         if (rdev->family >= CHIP_RV770)
2412                 WREG32(DMA_MODE, 1);
2413
2414         ring->wptr = 0;
2415         WREG32(DMA_RB_WPTR, ring->wptr << 2);
2416
2417         ring->rptr = RREG32(DMA_RB_RPTR) >> 2;
2418
2419         WREG32(DMA_RB_CNTL, rb_cntl | DMA_RB_ENABLE);
2420
2421         ring->ready = true;
2422
2423         r = radeon_ring_test(rdev, R600_RING_TYPE_DMA_INDEX, ring);
2424         if (r) {
2425                 ring->ready = false;
2426                 return r;
2427         }
2428
2429         radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
2430
2431         return 0;
2432 }
2433
2434 /**
2435  * r600_dma_fini - tear down the async dma engine
2436  *
2437  * @rdev: radeon_device pointer
2438  *
2439  * Stop the async dma engine and free the ring (r6xx-evergreen).
2440  */
2441 void r600_dma_fini(struct radeon_device *rdev)
2442 {
2443         r600_dma_stop(rdev);
2444         radeon_ring_fini(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX]);
2445 }
2446
2447 /*
2448  * GPU scratch registers helpers function.
2449  */
2450 void r600_scratch_init(struct radeon_device *rdev)
2451 {
2452         int i;
2453
2454         rdev->scratch.num_reg = 7;
2455         rdev->scratch.reg_base = SCRATCH_REG0;
2456         for (i = 0; i < rdev->scratch.num_reg; i++) {
2457                 rdev->scratch.free[i] = true;
2458                 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
2459         }
2460 }
2461
2462 int r600_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
2463 {
2464         uint32_t scratch;
2465         uint32_t tmp = 0;
2466         unsigned i;
2467         int r;
2468
2469         r = radeon_scratch_get(rdev, &scratch);
2470         if (r) {
2471                 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
2472                 return r;
2473         }
2474         WREG32(scratch, 0xCAFEDEAD);
2475         r = radeon_ring_lock(rdev, ring, 3);
2476         if (r) {
2477                 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
2478                 radeon_scratch_free(rdev, scratch);
2479                 return r;
2480         }
2481         radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2482         radeon_ring_write(ring, ((scratch - PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
2483         radeon_ring_write(ring, 0xDEADBEEF);
2484         radeon_ring_unlock_commit(rdev, ring);
2485         for (i = 0; i < rdev->usec_timeout; i++) {
2486                 tmp = RREG32(scratch);
2487                 if (tmp == 0xDEADBEEF)
2488                         break;
2489                 DRM_UDELAY(1);
2490         }
2491         if (i < rdev->usec_timeout) {
2492                 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
2493         } else {
2494                 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
2495                           ring->idx, scratch, tmp);
2496                 r = -EINVAL;
2497         }
2498         radeon_scratch_free(rdev, scratch);
2499         return r;
2500 }
2501
2502 /**
2503  * r600_dma_ring_test - simple async dma engine test
2504  *
2505  * @rdev: radeon_device pointer
2506  * @ring: radeon_ring structure holding ring information
2507  *
2508  * Test the DMA engine by writing using it to write an
2509  * value to memory. (r6xx-SI).
2510  * Returns 0 for success, error for failure.
2511  */
2512 int r600_dma_ring_test(struct radeon_device *rdev,
2513                        struct radeon_ring *ring)
2514 {
2515         unsigned i;
2516         int r;
2517         volatile uint32_t *ptr = rdev->vram_scratch.ptr;
2518         u32 tmp;
2519
2520         if (!ptr) {
2521                 DRM_ERROR("invalid vram scratch pointer\n");
2522                 return -EINVAL;
2523         }
2524
2525         tmp = 0xCAFEDEAD;
2526         *ptr = tmp;
2527
2528         r = radeon_ring_lock(rdev, ring, 4);
2529         if (r) {
2530                 DRM_ERROR("radeon: dma failed to lock ring %d (%d).\n", ring->idx, r);
2531                 return r;
2532         }
2533         radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 1));
2534         radeon_ring_write(ring, rdev->vram_scratch.gpu_addr & 0xfffffffc);
2535         radeon_ring_write(ring, upper_32_bits(rdev->vram_scratch.gpu_addr) & 0xff);
2536         radeon_ring_write(ring, 0xDEADBEEF);
2537         radeon_ring_unlock_commit(rdev, ring);
2538
2539         for (i = 0; i < rdev->usec_timeout; i++) {
2540                 tmp = *ptr;
2541                 if (tmp == 0xDEADBEEF)
2542                         break;
2543                 DRM_UDELAY(1);
2544         }
2545
2546         if (i < rdev->usec_timeout) {
2547                 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
2548         } else {
2549                 DRM_ERROR("radeon: ring %d test failed (0x%08X)\n",
2550                           ring->idx, tmp);
2551                 r = -EINVAL;
2552         }
2553         return r;
2554 }
2555
2556 /*
2557  * CP fences/semaphores
2558  */
2559
2560 void r600_fence_ring_emit(struct radeon_device *rdev,
2561                           struct radeon_fence *fence)
2562 {
2563         struct radeon_ring *ring = &rdev->ring[fence->ring];
2564
2565         if (rdev->wb.use_event) {
2566                 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
2567                 /* flush read cache over gart */
2568                 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
2569                 radeon_ring_write(ring, PACKET3_TC_ACTION_ENA |
2570                                         PACKET3_VC_ACTION_ENA |
2571                                         PACKET3_SH_ACTION_ENA);
2572                 radeon_ring_write(ring, 0xFFFFFFFF);
2573                 radeon_ring_write(ring, 0);
2574                 radeon_ring_write(ring, 10); /* poll interval */
2575                 /* EVENT_WRITE_EOP - flush caches, send int */
2576                 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
2577                 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT_TS) | EVENT_INDEX(5));
2578                 radeon_ring_write(ring, addr & 0xffffffff);
2579                 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
2580                 radeon_ring_write(ring, fence->seq);
2581                 radeon_ring_write(ring, 0);
2582         } else {
2583                 /* flush read cache over gart */
2584                 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
2585                 radeon_ring_write(ring, PACKET3_TC_ACTION_ENA |
2586                                         PACKET3_VC_ACTION_ENA |
2587                                         PACKET3_SH_ACTION_ENA);
2588                 radeon_ring_write(ring, 0xFFFFFFFF);
2589                 radeon_ring_write(ring, 0);
2590                 radeon_ring_write(ring, 10); /* poll interval */
2591                 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE, 0));
2592                 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT) | EVENT_INDEX(0));
2593                 /* wait for 3D idle clean */
2594                 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2595                 radeon_ring_write(ring, (WAIT_UNTIL - PACKET3_SET_CONFIG_REG_OFFSET) >> 2);
2596                 radeon_ring_write(ring, WAIT_3D_IDLE_bit | WAIT_3D_IDLECLEAN_bit);
2597                 /* Emit fence sequence & fire IRQ */
2598                 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2599                 radeon_ring_write(ring, ((rdev->fence_drv[fence->ring].scratch_reg - PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
2600                 radeon_ring_write(ring, fence->seq);
2601                 /* CP_INTERRUPT packet 3 no longer exists, use packet 0 */
2602                 radeon_ring_write(ring, PACKET0(CP_INT_STATUS, 0));
2603                 radeon_ring_write(ring, RB_INT_STAT);
2604         }
2605 }
2606
2607 void r600_semaphore_ring_emit(struct radeon_device *rdev,
2608                               struct radeon_ring *ring,
2609                               struct radeon_semaphore *semaphore,
2610                               bool emit_wait)
2611 {
2612         uint64_t addr = semaphore->gpu_addr;
2613         unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
2614
2615         if (rdev->family < CHIP_CAYMAN)
2616                 sel |= PACKET3_SEM_WAIT_ON_SIGNAL;
2617
2618         radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
2619         radeon_ring_write(ring, addr & 0xffffffff);
2620         radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | sel);
2621 }
2622
2623 /*
2624  * DMA fences/semaphores
2625  */
2626
2627 /**
2628  * r600_dma_fence_ring_emit - emit a fence on the DMA ring
2629  *
2630  * @rdev: radeon_device pointer
2631  * @fence: radeon fence object
2632  *
2633  * Add a DMA fence packet to the ring to write
2634  * the fence seq number and DMA trap packet to generate
2635  * an interrupt if needed (r6xx-r7xx).
2636  */
2637 void r600_dma_fence_ring_emit(struct radeon_device *rdev,
2638                               struct radeon_fence *fence)
2639 {
2640         struct radeon_ring *ring = &rdev->ring[fence->ring];
2641         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
2642
2643         /* write the fence */
2644         radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_FENCE, 0, 0, 0));
2645         radeon_ring_write(ring, addr & 0xfffffffc);
2646         radeon_ring_write(ring, (upper_32_bits(addr) & 0xff));
2647         radeon_ring_write(ring, lower_32_bits(fence->seq));
2648         /* generate an interrupt */
2649         radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_TRAP, 0, 0, 0));
2650 }
2651
2652 /**
2653  * r600_dma_semaphore_ring_emit - emit a semaphore on the dma ring
2654  *
2655  * @rdev: radeon_device pointer
2656  * @ring: radeon_ring structure holding ring information
2657  * @semaphore: radeon semaphore object
2658  * @emit_wait: wait or signal semaphore
2659  *
2660  * Add a DMA semaphore packet to the ring wait on or signal
2661  * other rings (r6xx-SI).
2662  */
2663 void r600_dma_semaphore_ring_emit(struct radeon_device *rdev,
2664                                   struct radeon_ring *ring,
2665                                   struct radeon_semaphore *semaphore,
2666                                   bool emit_wait)
2667 {
2668         u64 addr = semaphore->gpu_addr;
2669         u32 s = emit_wait ? 0 : 1;
2670
2671         radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SEMAPHORE, 0, s, 0));
2672         radeon_ring_write(ring, addr & 0xfffffffc);
2673         radeon_ring_write(ring, upper_32_bits(addr) & 0xff);
2674 }
2675
2676 int r600_copy_blit(struct radeon_device *rdev,
2677                    uint64_t src_offset,
2678                    uint64_t dst_offset,
2679                    unsigned num_gpu_pages,
2680                    struct radeon_fence **fence)
2681 {
2682         struct radeon_semaphore *sem = NULL;
2683         struct radeon_sa_bo *vb = NULL;
2684         int r;
2685
2686         r = r600_blit_prepare_copy(rdev, num_gpu_pages, fence, &vb, &sem);
2687         if (r) {
2688                 return r;
2689         }
2690         r600_kms_blit_copy(rdev, src_offset, dst_offset, num_gpu_pages, vb);
2691         r600_blit_done_copy(rdev, fence, vb, sem);
2692         return 0;
2693 }
2694
2695 /**
2696  * r600_copy_dma - copy pages using the DMA engine
2697  *
2698  * @rdev: radeon_device pointer
2699  * @src_offset: src GPU address
2700  * @dst_offset: dst GPU address
2701  * @num_gpu_pages: number of GPU pages to xfer
2702  * @fence: radeon fence object
2703  *
2704  * Copy GPU paging using the DMA engine (r6xx).
2705  * Used by the radeon ttm implementation to move pages if
2706  * registered as the asic copy callback.
2707  */
2708 int r600_copy_dma(struct radeon_device *rdev,
2709                   uint64_t src_offset, uint64_t dst_offset,
2710                   unsigned num_gpu_pages,
2711                   struct radeon_fence **fence)
2712 {
2713         struct radeon_semaphore *sem = NULL;
2714         int ring_index = rdev->asic->copy.dma_ring_index;
2715         struct radeon_ring *ring = &rdev->ring[ring_index];
2716         u32 size_in_dw, cur_size_in_dw;
2717         int i, num_loops;
2718         int r = 0;
2719
2720         r = radeon_semaphore_create(rdev, &sem);
2721         if (r) {
2722                 DRM_ERROR("radeon: moving bo (%d).\n", r);
2723                 return r;
2724         }
2725
2726         size_in_dw = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT) / 4;
2727         num_loops = DIV_ROUND_UP(size_in_dw, 0xFFFE);
2728         r = radeon_ring_lock(rdev, ring, num_loops * 4 + 8);
2729         if (r) {
2730                 DRM_ERROR("radeon: moving bo (%d).\n", r);
2731                 radeon_semaphore_free(rdev, &sem, NULL);
2732                 return r;
2733         }
2734
2735         if (radeon_fence_need_sync(*fence, ring->idx)) {
2736                 radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring,
2737                                             ring->idx);
2738                 radeon_fence_note_sync(*fence, ring->idx);
2739         } else {
2740                 radeon_semaphore_free(rdev, &sem, NULL);
2741         }
2742
2743         for (i = 0; i < num_loops; i++) {
2744                 cur_size_in_dw = size_in_dw;
2745                 if (cur_size_in_dw > 0xFFFE)
2746                         cur_size_in_dw = 0xFFFE;
2747                 size_in_dw -= cur_size_in_dw;
2748                 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_COPY, 0, 0, cur_size_in_dw));
2749                 radeon_ring_write(ring, dst_offset & 0xfffffffc);
2750                 radeon_ring_write(ring, src_offset & 0xfffffffc);
2751                 radeon_ring_write(ring, (((upper_32_bits(dst_offset) & 0xff) << 16) |
2752                                          (upper_32_bits(src_offset) & 0xff)));
2753                 src_offset += cur_size_in_dw * 4;
2754                 dst_offset += cur_size_in_dw * 4;
2755         }
2756
2757         r = radeon_fence_emit(rdev, fence, ring->idx);
2758         if (r) {
2759                 radeon_ring_unlock_undo(rdev, ring);
2760                 return r;
2761         }
2762
2763         radeon_ring_unlock_commit(rdev, ring);
2764         radeon_semaphore_free(rdev, &sem, *fence);
2765
2766         return r;
2767 }
2768
2769 int r600_set_surface_reg(struct radeon_device *rdev, int reg,
2770                          uint32_t tiling_flags, uint32_t pitch,
2771                          uint32_t offset, uint32_t obj_size)
2772 {
2773         /* FIXME: implement */
2774         return 0;
2775 }
2776
2777 void r600_clear_surface_reg(struct radeon_device *rdev, int reg)
2778 {
2779         /* FIXME: implement */
2780 }
2781
2782 static int r600_startup(struct radeon_device *rdev)
2783 {
2784         struct radeon_ring *ring;
2785         int r;
2786
2787         /* enable pcie gen2 link */
2788         r600_pcie_gen2_enable(rdev);
2789
2790         if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
2791                 r = r600_init_microcode(rdev);
2792                 if (r) {
2793                         DRM_ERROR("Failed to load firmware!\n");
2794                         return r;
2795                 }
2796         }
2797
2798         r = r600_vram_scratch_init(rdev);
2799         if (r)
2800                 return r;
2801
2802         r600_mc_program(rdev);
2803         if (rdev->flags & RADEON_IS_AGP) {
2804                 r600_agp_enable(rdev);
2805         } else {
2806                 r = r600_pcie_gart_enable(rdev);
2807                 if (r)
2808                         return r;
2809         }
2810         r600_gpu_init(rdev);
2811         r = r600_blit_init(rdev);
2812         if (r) {
2813                 r600_blit_fini(rdev);
2814                 rdev->asic->copy.copy = NULL;
2815                 dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
2816         }
2817
2818         /* allocate wb buffer */
2819         r = radeon_wb_init(rdev);
2820         if (r)
2821                 return r;
2822
2823         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
2824         if (r) {
2825                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2826                 return r;
2827         }
2828
2829         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
2830         if (r) {
2831                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
2832                 return r;
2833         }
2834
2835         /* Enable IRQ */
2836         r = r600_irq_init(rdev);
2837         if (r) {
2838                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
2839                 radeon_irq_kms_fini(rdev);
2840                 return r;
2841         }
2842         r600_irq_set(rdev);
2843
2844         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2845         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
2846                              R600_CP_RB_RPTR, R600_CP_RB_WPTR,
2847                              0, 0xfffff, RADEON_CP_PACKET2);
2848         if (r)
2849                 return r;
2850
2851         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
2852         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
2853                              DMA_RB_RPTR, DMA_RB_WPTR,
2854                              2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
2855         if (r)
2856                 return r;
2857
2858         r = r600_cp_load_microcode(rdev);
2859         if (r)
2860                 return r;
2861         r = r600_cp_resume(rdev);
2862         if (r)
2863                 return r;
2864
2865         r = r600_dma_resume(rdev);
2866         if (r)
2867                 return r;
2868
2869         r = radeon_ib_pool_init(rdev);
2870         if (r) {
2871                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
2872                 return r;
2873         }
2874
2875         r = r600_audio_init(rdev);
2876         if (r) {
2877                 DRM_ERROR("radeon: audio init failed\n");
2878                 return r;
2879         }
2880
2881         return 0;
2882 }
2883
2884 void r600_vga_set_state(struct radeon_device *rdev, bool state)
2885 {
2886         uint32_t temp;
2887
2888         temp = RREG32(CONFIG_CNTL);
2889         if (state == false) {
2890                 temp &= ~(1<<0);
2891                 temp |= (1<<1);
2892         } else {
2893                 temp &= ~(1<<1);
2894         }
2895         WREG32(CONFIG_CNTL, temp);
2896 }
2897
2898 int r600_resume(struct radeon_device *rdev)
2899 {
2900         int r;
2901
2902         /* Do not reset GPU before posting, on r600 hw unlike on r500 hw,
2903          * posting will perform necessary task to bring back GPU into good
2904          * shape.
2905          */
2906         /* post card */
2907         atom_asic_init(rdev->mode_info.atom_context);
2908
2909         rdev->accel_working = true;
2910         r = r600_startup(rdev);
2911         if (r) {
2912                 DRM_ERROR("r600 startup failed on resume\n");
2913                 rdev->accel_working = false;
2914                 return r;
2915         }
2916
2917         return r;
2918 }
2919
2920 int r600_suspend(struct radeon_device *rdev)
2921 {
2922         r600_audio_fini(rdev);
2923         r600_cp_stop(rdev);
2924         r600_dma_stop(rdev);
2925         r600_irq_suspend(rdev);
2926         radeon_wb_disable(rdev);
2927         r600_pcie_gart_disable(rdev);
2928
2929         return 0;
2930 }
2931
2932 /* Plan is to move initialization in that function and use
2933  * helper function so that radeon_device_init pretty much
2934  * do nothing more than calling asic specific function. This
2935  * should also allow to remove a bunch of callback function
2936  * like vram_info.
2937  */
2938 int r600_init(struct radeon_device *rdev)
2939 {
2940         int r;
2941
2942         if (r600_debugfs_mc_info_init(rdev)) {
2943                 DRM_ERROR("Failed to register debugfs file for mc !\n");
2944         }
2945         /* Read BIOS */
2946         if (!radeon_get_bios(rdev)) {
2947                 if (ASIC_IS_AVIVO(rdev))
2948                         return -EINVAL;
2949         }
2950         /* Must be an ATOMBIOS */
2951         if (!rdev->is_atom_bios) {
2952                 dev_err(rdev->dev, "Expecting atombios for R600 GPU\n");
2953                 return -EINVAL;
2954         }
2955         r = radeon_atombios_init(rdev);
2956         if (r)
2957                 return r;
2958         /* Post card if necessary */
2959         if (!radeon_card_posted(rdev)) {
2960                 if (!rdev->bios) {
2961                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
2962                         return -EINVAL;
2963                 }
2964                 DRM_INFO("GPU not posted. posting now...\n");
2965                 atom_asic_init(rdev->mode_info.atom_context);
2966         }
2967         /* Initialize scratch registers */
2968         r600_scratch_init(rdev);
2969         /* Initialize surface registers */
2970         radeon_surface_init(rdev);
2971         /* Initialize clocks */
2972         radeon_get_clock_info(rdev->ddev);
2973         /* Fence driver */
2974         r = radeon_fence_driver_init(rdev);
2975         if (r)
2976                 return r;
2977         if (rdev->flags & RADEON_IS_AGP) {
2978                 r = radeon_agp_init(rdev);
2979                 if (r)
2980                         radeon_agp_disable(rdev);
2981         }
2982         r = r600_mc_init(rdev);
2983         if (r)
2984                 return r;
2985         /* Memory manager */
2986         r = radeon_bo_init(rdev);
2987         if (r)
2988                 return r;
2989
2990         r = radeon_irq_kms_init(rdev);
2991         if (r)
2992                 return r;
2993
2994         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
2995         r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
2996
2997         rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
2998         r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
2999
3000         rdev->ih.ring_obj = NULL;
3001         r600_ih_ring_init(rdev, 64 * 1024);
3002
3003         r = r600_pcie_gart_init(rdev);
3004         if (r)
3005                 return r;
3006
3007         rdev->accel_working = true;
3008         r = r600_startup(rdev);
3009         if (r) {
3010                 dev_err(rdev->dev, "disabling GPU acceleration\n");
3011                 r600_cp_fini(rdev);
3012                 r600_dma_fini(rdev);
3013                 r600_irq_fini(rdev);
3014                 radeon_wb_fini(rdev);
3015                 radeon_ib_pool_fini(rdev);
3016                 radeon_irq_kms_fini(rdev);
3017                 r600_pcie_gart_fini(rdev);
3018                 rdev->accel_working = false;
3019         }
3020
3021         /* Don't start up if the ucode is missing. */
3022         if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
3023                 DRM_ERROR("radeon: ucode required for R600+.\n");
3024                 return -EINVAL;
3025         }
3026
3027         return 0;
3028 }
3029
3030 void r600_fini(struct radeon_device *rdev)
3031 {
3032         r600_audio_fini(rdev);
3033         r600_blit_fini(rdev);
3034         r600_cp_fini(rdev);
3035         r600_dma_fini(rdev);
3036         r600_irq_fini(rdev);
3037         radeon_wb_fini(rdev);
3038         radeon_ib_pool_fini(rdev);
3039         radeon_irq_kms_fini(rdev);
3040         r600_pcie_gart_fini(rdev);
3041         r600_vram_scratch_fini(rdev);
3042         radeon_agp_fini(rdev);
3043         radeon_gem_fini(rdev);
3044         radeon_fence_driver_fini(rdev);
3045         radeon_bo_fini(rdev);
3046         radeon_atombios_fini(rdev);
3047         r600_fini_microcode(rdev);
3048         free(rdev->bios, DRM_MEM_DRIVER);
3049         rdev->bios = NULL;
3050 }
3051
3052
3053 /*
3054  * CS stuff
3055  */
3056 void r600_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3057 {
3058         struct radeon_ring *ring = &rdev->ring[ib->ring];
3059         u32 next_rptr;
3060
3061         if (ring->rptr_save_reg) {
3062                 next_rptr = ring->wptr + 3 + 4;
3063                 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3064                 radeon_ring_write(ring, ((ring->rptr_save_reg -
3065                                          PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
3066                 radeon_ring_write(ring, next_rptr);
3067         } else if (rdev->wb.enabled) {
3068                 next_rptr = ring->wptr + 5 + 4;
3069                 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
3070                 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3071                 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
3072                 radeon_ring_write(ring, next_rptr);
3073                 radeon_ring_write(ring, 0);
3074         }
3075
3076         radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
3077         radeon_ring_write(ring,
3078 #ifdef __BIG_ENDIAN
3079                           (2 << 0) |
3080 #endif
3081                           (ib->gpu_addr & 0xFFFFFFFC));
3082         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
3083         radeon_ring_write(ring, ib->length_dw);
3084 }
3085
3086 int r600_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3087 {
3088         struct radeon_ib ib;
3089         uint32_t scratch;
3090         uint32_t tmp = 0;
3091         unsigned i;
3092         int r;
3093
3094         r = radeon_scratch_get(rdev, &scratch);
3095         if (r) {
3096                 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
3097                 return r;
3098         }
3099         WREG32(scratch, 0xCAFEDEAD);
3100         r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3101         if (r) {
3102                 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
3103                 goto free_scratch;
3104         }
3105         ib.ptr[0] = PACKET3(PACKET3_SET_CONFIG_REG, 1);
3106         ib.ptr[1] = ((scratch - PACKET3_SET_CONFIG_REG_OFFSET) >> 2);
3107         ib.ptr[2] = 0xDEADBEEF;
3108         ib.length_dw = 3;
3109         r = radeon_ib_schedule(rdev, &ib, NULL);
3110         if (r) {
3111                 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
3112                 goto free_ib;
3113         }
3114         r = radeon_fence_wait(ib.fence, false);
3115         if (r) {
3116                 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
3117                 goto free_ib;
3118         }
3119         for (i = 0; i < rdev->usec_timeout; i++) {
3120                 tmp = RREG32(scratch);
3121                 if (tmp == 0xDEADBEEF)
3122                         break;
3123                 DRM_UDELAY(1);
3124         }
3125         if (i < rdev->usec_timeout) {
3126                 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3127         } else {
3128                 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
3129                           scratch, tmp);
3130                 r = -EINVAL;
3131         }
3132 free_ib:
3133         radeon_ib_free(rdev, &ib);
3134 free_scratch:
3135         radeon_scratch_free(rdev, scratch);
3136         return r;
3137 }
3138
3139 /**
3140  * r600_dma_ib_test - test an IB on the DMA engine
3141  *
3142  * @rdev: radeon_device pointer
3143  * @ring: radeon_ring structure holding ring information
3144  *
3145  * Test a simple IB in the DMA ring (r6xx-SI).
3146  * Returns 0 on success, error on failure.
3147  */
3148 int r600_dma_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3149 {
3150         struct radeon_ib ib;
3151         unsigned i;
3152         int r;
3153         volatile uint32_t *ptr = rdev->vram_scratch.ptr;
3154         u32 tmp = 0;
3155
3156         if (!ptr) {
3157                 DRM_ERROR("invalid vram scratch pointer\n");
3158                 return -EINVAL;
3159         }
3160
3161         tmp = 0xCAFEDEAD;
3162         *ptr = tmp;
3163
3164         r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3165         if (r) {
3166                 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
3167                 return r;
3168         }
3169
3170         ib.ptr[0] = DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 1);
3171         ib.ptr[1] = rdev->vram_scratch.gpu_addr & 0xfffffffc;
3172         ib.ptr[2] = upper_32_bits(rdev->vram_scratch.gpu_addr) & 0xff;
3173         ib.ptr[3] = 0xDEADBEEF;
3174         ib.length_dw = 4;
3175
3176         r = radeon_ib_schedule(rdev, &ib, NULL);
3177         if (r) {
3178                 radeon_ib_free(rdev, &ib);
3179                 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
3180                 return r;
3181         }
3182         r = radeon_fence_wait(ib.fence, false);
3183         if (r) {
3184                 radeon_ib_free(rdev, &ib);
3185                 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
3186                 return r;
3187         }
3188         for (i = 0; i < rdev->usec_timeout; i++) {
3189                 tmp = *ptr;
3190                 if (tmp == 0xDEADBEEF)
3191                         break;
3192                 DRM_UDELAY(1);
3193         }
3194         if (i < rdev->usec_timeout) {
3195                 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3196         } else {
3197                 DRM_ERROR("radeon: ib test failed (0x%08X)\n", tmp);
3198                 r = -EINVAL;
3199         }
3200         radeon_ib_free(rdev, &ib);
3201         return r;
3202 }
3203
3204 /**
3205  * r600_dma_ring_ib_execute - Schedule an IB on the DMA engine
3206  *
3207  * @rdev: radeon_device pointer
3208  * @ib: IB object to schedule
3209  *
3210  * Schedule an IB in the DMA ring (r6xx-r7xx).
3211  */
3212 void r600_dma_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3213 {
3214         struct radeon_ring *ring = &rdev->ring[ib->ring];
3215
3216         if (rdev->wb.enabled) {
3217                 u32 next_rptr = ring->wptr + 4;
3218                 while ((next_rptr & 7) != 5)
3219                         next_rptr++;
3220                 next_rptr += 3;
3221                 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 1));
3222                 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3223                 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xff);
3224                 radeon_ring_write(ring, next_rptr);
3225         }
3226
3227         /* The indirect buffer packet must end on an 8 DW boundary in the DMA ring.
3228          * Pad as necessary with NOPs.
3229          */
3230         while ((ring->wptr & 7) != 5)
3231                 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
3232         radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_INDIRECT_BUFFER, 0, 0, 0));
3233         radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFE0));
3234         radeon_ring_write(ring, (ib->length_dw << 16) | (upper_32_bits(ib->gpu_addr) & 0xFF));
3235
3236 }
3237
3238 /*
3239  * Interrupts
3240  *
3241  * Interrupts use a ring buffer on r6xx/r7xx hardware.  It works pretty
3242  * the same as the CP ring buffer, but in reverse.  Rather than the CPU
3243  * writing to the ring and the GPU consuming, the GPU writes to the ring
3244  * and host consumes.  As the host irq handler processes interrupts, it
3245  * increments the rptr.  When the rptr catches up with the wptr, all the
3246  * current interrupts have been processed.
3247  */
3248
3249 void r600_ih_ring_init(struct radeon_device *rdev, unsigned ring_size)
3250 {
3251         u32 rb_bufsz;
3252
3253         /* Align ring size */
3254         rb_bufsz = drm_order(ring_size / 4);
3255         ring_size = (1 << rb_bufsz) * 4;
3256         rdev->ih.ring_size = ring_size;
3257         rdev->ih.ptr_mask = rdev->ih.ring_size - 1;
3258         rdev->ih.rptr = 0;
3259 }
3260
3261 int r600_ih_ring_alloc(struct radeon_device *rdev)
3262 {
3263         int r;
3264         void *ring_ptr; /* FreeBSD: to please GCC 4.2. */
3265
3266         /* Allocate ring buffer */
3267         if (rdev->ih.ring_obj == NULL) {
3268                 r = radeon_bo_create(rdev, rdev->ih.ring_size,
3269                                      PAGE_SIZE, true,
3270                                      RADEON_GEM_DOMAIN_GTT,
3271                                      NULL, &rdev->ih.ring_obj);
3272                 if (r) {
3273                         DRM_ERROR("radeon: failed to create ih ring buffer (%d).\n", r);
3274                         return r;
3275                 }
3276                 r = radeon_bo_reserve(rdev->ih.ring_obj, false);
3277                 if (unlikely(r != 0)) {
3278                         radeon_bo_unref(&rdev->ih.ring_obj);
3279                         return r;
3280                 }
3281                 r = radeon_bo_pin(rdev->ih.ring_obj,
3282                                   RADEON_GEM_DOMAIN_GTT,
3283                                   &rdev->ih.gpu_addr);
3284                 if (r) {
3285                         radeon_bo_unreserve(rdev->ih.ring_obj);
3286                         radeon_bo_unref(&rdev->ih.ring_obj);
3287                         DRM_ERROR("radeon: failed to pin ih ring buffer (%d).\n", r);
3288                         return r;
3289                 }
3290                 ring_ptr = &rdev->ih.ring;
3291                 r = radeon_bo_kmap(rdev->ih.ring_obj,
3292                                    ring_ptr);
3293                 if (r)
3294                         radeon_bo_unpin(rdev->ih.ring_obj);
3295                 radeon_bo_unreserve(rdev->ih.ring_obj);
3296                 if (r) {
3297                         DRM_ERROR("radeon: failed to map ih ring buffer (%d).\n", r);
3298                         radeon_bo_unref(&rdev->ih.ring_obj);
3299                         return r;
3300                 }
3301         }
3302         return 0;
3303 }
3304
3305 void r600_ih_ring_fini(struct radeon_device *rdev)
3306 {
3307         int r;
3308         if (rdev->ih.ring_obj) {
3309                 r = radeon_bo_reserve(rdev->ih.ring_obj, false);
3310                 if (likely(r == 0)) {
3311                         radeon_bo_kunmap(rdev->ih.ring_obj);
3312                         radeon_bo_unpin(rdev->ih.ring_obj);
3313                         radeon_bo_unreserve(rdev->ih.ring_obj);
3314                 }
3315                 radeon_bo_unref(&rdev->ih.ring_obj);
3316                 rdev->ih.ring = NULL;
3317                 rdev->ih.ring_obj = NULL;
3318         }
3319 }
3320
3321 void r600_rlc_stop(struct radeon_device *rdev)
3322 {
3323
3324         if ((rdev->family >= CHIP_RV770) &&
3325             (rdev->family <= CHIP_RV740)) {
3326                 /* r7xx asics need to soft reset RLC before halting */
3327                 WREG32(SRBM_SOFT_RESET, SOFT_RESET_RLC);
3328                 RREG32(SRBM_SOFT_RESET);
3329                 mdelay(15);
3330                 WREG32(SRBM_SOFT_RESET, 0);
3331                 RREG32(SRBM_SOFT_RESET);
3332         }
3333
3334         WREG32(RLC_CNTL, 0);
3335 }
3336
3337 static void r600_rlc_start(struct radeon_device *rdev)
3338 {
3339         WREG32(RLC_CNTL, RLC_ENABLE);
3340 }
3341
3342 static int r600_rlc_init(struct radeon_device *rdev)
3343 {
3344         u32 i;
3345         const __be32 *fw_data;
3346
3347         if (!rdev->rlc_fw)
3348                 return -EINVAL;
3349
3350         r600_rlc_stop(rdev);
3351
3352         WREG32(RLC_HB_CNTL, 0);
3353
3354         if (rdev->family == CHIP_ARUBA) {
3355                 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
3356                 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
3357         }
3358         if (rdev->family <= CHIP_CAYMAN) {
3359                 WREG32(RLC_HB_BASE, 0);
3360                 WREG32(RLC_HB_RPTR, 0);
3361                 WREG32(RLC_HB_WPTR, 0);
3362         }
3363         if (rdev->family <= CHIP_CAICOS) {
3364                 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
3365                 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
3366         }
3367         WREG32(RLC_MC_CNTL, 0);
3368         WREG32(RLC_UCODE_CNTL, 0);
3369
3370         fw_data = (const __be32 *)rdev->rlc_fw->data;
3371         if (rdev->family >= CHIP_ARUBA) {
3372                 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
3373                         WREG32(RLC_UCODE_ADDR, i);
3374                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
3375                 }
3376         } else if (rdev->family >= CHIP_CAYMAN) {
3377                 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
3378                         WREG32(RLC_UCODE_ADDR, i);
3379                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
3380                 }
3381         } else if (rdev->family >= CHIP_CEDAR) {
3382                 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
3383                         WREG32(RLC_UCODE_ADDR, i);
3384                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
3385                 }
3386         } else if (rdev->family >= CHIP_RV770) {
3387                 for (i = 0; i < R700_RLC_UCODE_SIZE; i++) {
3388                         WREG32(RLC_UCODE_ADDR, i);
3389                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
3390                 }
3391         } else {
3392                 for (i = 0; i < RLC_UCODE_SIZE; i++) {
3393                         WREG32(RLC_UCODE_ADDR, i);
3394                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
3395                 }
3396         }
3397         WREG32(RLC_UCODE_ADDR, 0);
3398
3399         r600_rlc_start(rdev);
3400
3401         return 0;
3402 }
3403
3404 static void r600_enable_interrupts(struct radeon_device *rdev)
3405 {
3406         u32 ih_cntl = RREG32(IH_CNTL);
3407         u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
3408
3409         ih_cntl |= ENABLE_INTR;
3410         ih_rb_cntl |= IH_RB_ENABLE;
3411         WREG32(IH_CNTL, ih_cntl);
3412         WREG32(IH_RB_CNTL, ih_rb_cntl);
3413         rdev->ih.enabled = true;
3414 }
3415
3416 void r600_disable_interrupts(struct radeon_device *rdev)
3417 {
3418         u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
3419         u32 ih_cntl = RREG32(IH_CNTL);
3420
3421         ih_rb_cntl &= ~IH_RB_ENABLE;
3422         ih_cntl &= ~ENABLE_INTR;
3423         WREG32(IH_RB_CNTL, ih_rb_cntl);
3424         WREG32(IH_CNTL, ih_cntl);
3425         /* set rptr, wptr to 0 */
3426         WREG32(IH_RB_RPTR, 0);
3427         WREG32(IH_RB_WPTR, 0);
3428         rdev->ih.enabled = false;
3429         rdev->ih.rptr = 0;
3430 }
3431
3432 static void r600_disable_interrupt_state(struct radeon_device *rdev)
3433 {
3434         u32 tmp;
3435
3436         WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
3437         tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
3438         WREG32(DMA_CNTL, tmp);
3439         WREG32(GRBM_INT_CNTL, 0);
3440         WREG32(DxMODE_INT_MASK, 0);
3441         WREG32(D1GRPH_INTERRUPT_CONTROL, 0);
3442         WREG32(D2GRPH_INTERRUPT_CONTROL, 0);
3443         if (ASIC_IS_DCE3(rdev)) {
3444                 WREG32(DCE3_DACA_AUTODETECT_INT_CONTROL, 0);
3445                 WREG32(DCE3_DACB_AUTODETECT_INT_CONTROL, 0);
3446                 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3447                 WREG32(DC_HPD1_INT_CONTROL, tmp);
3448                 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3449                 WREG32(DC_HPD2_INT_CONTROL, tmp);
3450                 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3451                 WREG32(DC_HPD3_INT_CONTROL, tmp);
3452                 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3453                 WREG32(DC_HPD4_INT_CONTROL, tmp);
3454                 if (ASIC_IS_DCE32(rdev)) {
3455                         tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3456                         WREG32(DC_HPD5_INT_CONTROL, tmp);
3457                         tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3458                         WREG32(DC_HPD6_INT_CONTROL, tmp);
3459                         tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3460                         WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0, tmp);
3461                         tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3462                         WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1, tmp);
3463                 } else {
3464                         tmp = RREG32(HDMI0_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3465                         WREG32(HDMI0_AUDIO_PACKET_CONTROL, tmp);
3466                         tmp = RREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3467                         WREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL, tmp);
3468                 }
3469         } else {
3470                 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
3471                 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
3472                 tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
3473                 WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
3474                 tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
3475                 WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
3476                 tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
3477                 WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
3478                 tmp = RREG32(HDMI0_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3479                 WREG32(HDMI0_AUDIO_PACKET_CONTROL, tmp);
3480                 tmp = RREG32(HDMI1_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3481                 WREG32(HDMI1_AUDIO_PACKET_CONTROL, tmp);
3482         }
3483 }
3484
3485 int r600_irq_init(struct radeon_device *rdev)
3486 {
3487         int ret = 0;
3488         int rb_bufsz;
3489         u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
3490
3491         /* allocate ring */
3492         ret = r600_ih_ring_alloc(rdev);
3493         if (ret)
3494                 return ret;
3495
3496         /* disable irqs */
3497         r600_disable_interrupts(rdev);
3498
3499         /* init rlc */
3500         ret = r600_rlc_init(rdev);
3501         if (ret) {
3502                 r600_ih_ring_fini(rdev);
3503                 return ret;
3504         }
3505
3506         /* setup interrupt control */
3507         /* set dummy read address to ring address */
3508         WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
3509         interrupt_cntl = RREG32(INTERRUPT_CNTL);
3510         /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
3511          * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
3512          */
3513         interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
3514         /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
3515         interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
3516         WREG32(INTERRUPT_CNTL, interrupt_cntl);
3517
3518         WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
3519         rb_bufsz = drm_order(rdev->ih.ring_size / 4);
3520
3521         ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
3522                       IH_WPTR_OVERFLOW_CLEAR |
3523                       (rb_bufsz << 1));
3524
3525         if (rdev->wb.enabled)
3526                 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
3527
3528         /* set the writeback address whether it's enabled or not */
3529         WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
3530         WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
3531
3532         WREG32(IH_RB_CNTL, ih_rb_cntl);
3533
3534         /* set rptr, wptr to 0 */
3535         WREG32(IH_RB_RPTR, 0);
3536         WREG32(IH_RB_WPTR, 0);
3537
3538         /* Default settings for IH_CNTL (disabled at first) */
3539         ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10);
3540         /* RPTR_REARM only works if msi's are enabled */
3541         if (rdev->msi_enabled)
3542                 ih_cntl |= RPTR_REARM;
3543         WREG32(IH_CNTL, ih_cntl);
3544
3545         /* force the active interrupt state to all disabled */
3546         if (rdev->family >= CHIP_CEDAR)
3547                 evergreen_disable_interrupt_state(rdev);
3548         else
3549                 r600_disable_interrupt_state(rdev);
3550
3551         /* at this point everything should be setup correctly to enable master */
3552         pci_enable_busmaster(rdev->dev);
3553
3554         /* enable irqs */
3555         r600_enable_interrupts(rdev);
3556
3557         return ret;
3558 }
3559
3560 void r600_irq_suspend(struct radeon_device *rdev)
3561 {
3562         r600_irq_disable(rdev);
3563         r600_rlc_stop(rdev);
3564 }
3565
3566 void r600_irq_fini(struct radeon_device *rdev)
3567 {
3568         r600_irq_suspend(rdev);
3569         r600_ih_ring_fini(rdev);
3570 }
3571
3572 int r600_irq_set(struct radeon_device *rdev)
3573 {
3574         u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
3575         u32 mode_int = 0;
3576         u32 hpd1, hpd2, hpd3, hpd4 = 0, hpd5 = 0, hpd6 = 0;
3577         u32 grbm_int_cntl = 0;
3578         u32 hdmi0, hdmi1;
3579         u32 d1grph = 0, d2grph = 0;
3580         u32 dma_cntl;
3581
3582         if (!rdev->irq.installed) {
3583                 DRM_ERROR("Can't enable IRQ/MSI because no handler is installed\n");
3584                 return -EINVAL;
3585         }
3586         /* don't enable anything if the ih is disabled */
3587         if (!rdev->ih.enabled) {
3588                 r600_disable_interrupts(rdev);
3589                 /* force the active interrupt state to all disabled */
3590                 r600_disable_interrupt_state(rdev);
3591                 return 0;
3592         }
3593
3594         if (ASIC_IS_DCE3(rdev)) {
3595                 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
3596                 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
3597                 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
3598                 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
3599                 if (ASIC_IS_DCE32(rdev)) {
3600                         hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
3601                         hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
3602                         hdmi0 = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
3603                         hdmi1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
3604                 } else {
3605                         hdmi0 = RREG32(HDMI0_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3606                         hdmi1 = RREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3607                 }
3608         } else {
3609                 hpd1 = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL) & ~DC_HPDx_INT_EN;
3610                 hpd2 = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL) & ~DC_HPDx_INT_EN;
3611                 hpd3 = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL) & ~DC_HPDx_INT_EN;
3612                 hdmi0 = RREG32(HDMI0_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3613                 hdmi1 = RREG32(HDMI1_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3614         }
3615         dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
3616
3617         if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
3618                 DRM_DEBUG("r600_irq_set: sw int\n");
3619                 cp_int_cntl |= RB_INT_ENABLE;
3620                 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
3621         }
3622
3623         if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
3624                 DRM_DEBUG("r600_irq_set: sw int dma\n");
3625                 dma_cntl |= TRAP_ENABLE;
3626         }
3627
3628         if (rdev->irq.crtc_vblank_int[0] ||
3629             atomic_read(&rdev->irq.pflip[0])) {
3630                 DRM_DEBUG("r600_irq_set: vblank 0\n");
3631                 mode_int |= D1MODE_VBLANK_INT_MASK;
3632         }
3633         if (rdev->irq.crtc_vblank_int[1] ||
3634             atomic_read(&rdev->irq.pflip[1])) {
3635                 DRM_DEBUG("r600_irq_set: vblank 1\n");
3636                 mode_int |= D2MODE_VBLANK_INT_MASK;
3637         }
3638         if (rdev->irq.hpd[0]) {
3639                 DRM_DEBUG("r600_irq_set: hpd 1\n");
3640                 hpd1 |= DC_HPDx_INT_EN;
3641         }
3642         if (rdev->irq.hpd[1]) {
3643                 DRM_DEBUG("r600_irq_set: hpd 2\n");
3644                 hpd2 |= DC_HPDx_INT_EN;
3645         }
3646         if (rdev->irq.hpd[2]) {
3647                 DRM_DEBUG("r600_irq_set: hpd 3\n");
3648                 hpd3 |= DC_HPDx_INT_EN;
3649         }
3650         if (rdev->irq.hpd[3]) {
3651                 DRM_DEBUG("r600_irq_set: hpd 4\n");
3652                 hpd4 |= DC_HPDx_INT_EN;
3653         }
3654         if (rdev->irq.hpd[4]) {
3655                 DRM_DEBUG("r600_irq_set: hpd 5\n");
3656                 hpd5 |= DC_HPDx_INT_EN;
3657         }
3658         if (rdev->irq.hpd[5]) {
3659                 DRM_DEBUG("r600_irq_set: hpd 6\n");
3660                 hpd6 |= DC_HPDx_INT_EN;
3661         }
3662         if (rdev->irq.afmt[0]) {
3663                 DRM_DEBUG("r600_irq_set: hdmi 0\n");
3664                 hdmi0 |= HDMI0_AZ_FORMAT_WTRIG_MASK;
3665         }
3666         if (rdev->irq.afmt[1]) {
3667                 DRM_DEBUG("r600_irq_set: hdmi 0\n");
3668                 hdmi1 |= HDMI0_AZ_FORMAT_WTRIG_MASK;
3669         }
3670
3671         WREG32(CP_INT_CNTL, cp_int_cntl);
3672         WREG32(DMA_CNTL, dma_cntl);
3673         WREG32(DxMODE_INT_MASK, mode_int);
3674         WREG32(D1GRPH_INTERRUPT_CONTROL, d1grph);
3675         WREG32(D2GRPH_INTERRUPT_CONTROL, d2grph);
3676         WREG32(GRBM_INT_CNTL, grbm_int_cntl);
3677         if (ASIC_IS_DCE3(rdev)) {
3678                 WREG32(DC_HPD1_INT_CONTROL, hpd1);
3679                 WREG32(DC_HPD2_INT_CONTROL, hpd2);
3680                 WREG32(DC_HPD3_INT_CONTROL, hpd3);
3681                 WREG32(DC_HPD4_INT_CONTROL, hpd4);
3682                 if (ASIC_IS_DCE32(rdev)) {
3683                         WREG32(DC_HPD5_INT_CONTROL, hpd5);
3684                         WREG32(DC_HPD6_INT_CONTROL, hpd6);
3685                         WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0, hdmi0);
3686                         WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1, hdmi1);
3687                 } else {
3688                         WREG32(HDMI0_AUDIO_PACKET_CONTROL, hdmi0);
3689                         WREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL, hdmi1);
3690                 }
3691         } else {
3692                 WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, hpd1);
3693                 WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, hpd2);
3694                 WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, hpd3);
3695                 WREG32(HDMI0_AUDIO_PACKET_CONTROL, hdmi0);
3696                 WREG32(HDMI1_AUDIO_PACKET_CONTROL, hdmi1);
3697         }
3698
3699         return 0;
3700 }
3701
3702 static void r600_irq_ack(struct radeon_device *rdev)
3703 {
3704         u32 tmp;
3705
3706         if (ASIC_IS_DCE3(rdev)) {
3707                 rdev->irq.stat_regs.r600.disp_int = RREG32(DCE3_DISP_INTERRUPT_STATUS);
3708                 rdev->irq.stat_regs.r600.disp_int_cont = RREG32(DCE3_DISP_INTERRUPT_STATUS_CONTINUE);
3709                 rdev->irq.stat_regs.r600.disp_int_cont2 = RREG32(DCE3_DISP_INTERRUPT_STATUS_CONTINUE2);
3710                 if (ASIC_IS_DCE32(rdev)) {
3711                         rdev->irq.stat_regs.r600.hdmi0_status = RREG32(AFMT_STATUS + DCE3_HDMI_OFFSET0);
3712                         rdev->irq.stat_regs.r600.hdmi1_status = RREG32(AFMT_STATUS + DCE3_HDMI_OFFSET1);
3713                 } else {
3714                         rdev->irq.stat_regs.r600.hdmi0_status = RREG32(HDMI0_STATUS);
3715                         rdev->irq.stat_regs.r600.hdmi1_status = RREG32(DCE3_HDMI1_STATUS);
3716                 }
3717         } else {
3718                 rdev->irq.stat_regs.r600.disp_int = RREG32(DISP_INTERRUPT_STATUS);
3719                 rdev->irq.stat_regs.r600.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
3720                 rdev->irq.stat_regs.r600.disp_int_cont2 = 0;
3721                 rdev->irq.stat_regs.r600.hdmi0_status = RREG32(HDMI0_STATUS);
3722                 rdev->irq.stat_regs.r600.hdmi1_status = RREG32(HDMI1_STATUS);
3723         }
3724         rdev->irq.stat_regs.r600.d1grph_int = RREG32(D1GRPH_INTERRUPT_STATUS);
3725         rdev->irq.stat_regs.r600.d2grph_int = RREG32(D2GRPH_INTERRUPT_STATUS);
3726
3727         if (rdev->irq.stat_regs.r600.d1grph_int & DxGRPH_PFLIP_INT_OCCURRED)
3728                 WREG32(D1GRPH_INTERRUPT_STATUS, DxGRPH_PFLIP_INT_CLEAR);
3729         if (rdev->irq.stat_regs.r600.d2grph_int & DxGRPH_PFLIP_INT_OCCURRED)
3730                 WREG32(D2GRPH_INTERRUPT_STATUS, DxGRPH_PFLIP_INT_CLEAR);
3731         if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VBLANK_INTERRUPT)
3732                 WREG32(D1MODE_VBLANK_STATUS, DxMODE_VBLANK_ACK);
3733         if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VLINE_INTERRUPT)
3734                 WREG32(D1MODE_VLINE_STATUS, DxMODE_VLINE_ACK);
3735         if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VBLANK_INTERRUPT)
3736                 WREG32(D2MODE_VBLANK_STATUS, DxMODE_VBLANK_ACK);
3737         if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VLINE_INTERRUPT)
3738                 WREG32(D2MODE_VLINE_STATUS, DxMODE_VLINE_ACK);
3739         if (rdev->irq.stat_regs.r600.disp_int & DC_HPD1_INTERRUPT) {
3740                 if (ASIC_IS_DCE3(rdev)) {
3741                         tmp = RREG32(DC_HPD1_INT_CONTROL);
3742                         tmp |= DC_HPDx_INT_ACK;
3743                         WREG32(DC_HPD1_INT_CONTROL, tmp);
3744                 } else {
3745                         tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL);
3746                         tmp |= DC_HPDx_INT_ACK;
3747                         WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
3748                 }
3749         }
3750         if (rdev->irq.stat_regs.r600.disp_int & DC_HPD2_INTERRUPT) {
3751                 if (ASIC_IS_DCE3(rdev)) {
3752                         tmp = RREG32(DC_HPD2_INT_CONTROL);
3753                         tmp |= DC_HPDx_INT_ACK;
3754                         WREG32(DC_HPD2_INT_CONTROL, tmp);
3755                 } else {
3756                         tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL);
3757                         tmp |= DC_HPDx_INT_ACK;
3758                         WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
3759                 }
3760         }
3761         if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD3_INTERRUPT) {
3762                 if (ASIC_IS_DCE3(rdev)) {
3763                         tmp = RREG32(DC_HPD3_INT_CONTROL);
3764                         tmp |= DC_HPDx_INT_ACK;
3765                         WREG32(DC_HPD3_INT_CONTROL, tmp);
3766                 } else {
3767                         tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL);
3768                         tmp |= DC_HPDx_INT_ACK;
3769                         WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
3770                 }
3771         }
3772         if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD4_INTERRUPT) {
3773                 tmp = RREG32(DC_HPD4_INT_CONTROL);
3774                 tmp |= DC_HPDx_INT_ACK;
3775                 WREG32(DC_HPD4_INT_CONTROL, tmp);
3776         }
3777         if (ASIC_IS_DCE32(rdev)) {
3778                 if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD5_INTERRUPT) {
3779                         tmp = RREG32(DC_HPD5_INT_CONTROL);
3780                         tmp |= DC_HPDx_INT_ACK;
3781                         WREG32(DC_HPD5_INT_CONTROL, tmp);
3782                 }
3783                 if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD6_INTERRUPT) {
3784                         tmp = RREG32(DC_HPD5_INT_CONTROL);
3785                         tmp |= DC_HPDx_INT_ACK;
3786                         WREG32(DC_HPD6_INT_CONTROL, tmp);
3787                 }
3788                 if (rdev->irq.stat_regs.r600.hdmi0_status & AFMT_AZ_FORMAT_WTRIG) {
3789                         tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0);
3790                         tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
3791                         WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0, tmp);
3792                 }
3793                 if (rdev->irq.stat_regs.r600.hdmi1_status & AFMT_AZ_FORMAT_WTRIG) {
3794                         tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1);
3795                         tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
3796                         WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1, tmp);
3797                 }
3798         } else {
3799                 if (rdev->irq.stat_regs.r600.hdmi0_status & HDMI0_AZ_FORMAT_WTRIG) {
3800                         tmp = RREG32(HDMI0_AUDIO_PACKET_CONTROL);
3801                         tmp |= HDMI0_AZ_FORMAT_WTRIG_ACK;
3802                         WREG32(HDMI0_AUDIO_PACKET_CONTROL, tmp);
3803                 }
3804                 if (rdev->irq.stat_regs.r600.hdmi1_status & HDMI0_AZ_FORMAT_WTRIG) {
3805                         if (ASIC_IS_DCE3(rdev)) {
3806                                 tmp = RREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL);
3807                                 tmp |= HDMI0_AZ_FORMAT_WTRIG_ACK;
3808                                 WREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL, tmp);
3809                         } else {
3810                                 tmp = RREG32(HDMI1_AUDIO_PACKET_CONTROL);
3811                                 tmp |= HDMI0_AZ_FORMAT_WTRIG_ACK;
3812                                 WREG32(HDMI1_AUDIO_PACKET_CONTROL, tmp);
3813                         }
3814                 }
3815         }
3816 }
3817
3818 void r600_irq_disable(struct radeon_device *rdev)
3819 {
3820         r600_disable_interrupts(rdev);
3821         /* Wait and acknowledge irq */
3822         mdelay(1);
3823         r600_irq_ack(rdev);
3824         r600_disable_interrupt_state(rdev);
3825 }
3826
3827 static u32 r600_get_ih_wptr(struct radeon_device *rdev)
3828 {
3829         u32 wptr, tmp;
3830
3831         if (rdev->wb.enabled)
3832                 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
3833         else
3834                 wptr = RREG32(IH_RB_WPTR);
3835
3836         if (wptr & RB_OVERFLOW) {
3837                 /* When a ring buffer overflow happen start parsing interrupt
3838                  * from the last not overwritten vector (wptr + 16). Hopefully
3839                  * this should allow us to catchup.
3840                  */
3841                 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
3842                         wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
3843                 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
3844                 tmp = RREG32(IH_RB_CNTL);
3845                 tmp |= IH_WPTR_OVERFLOW_CLEAR;
3846                 WREG32(IH_RB_CNTL, tmp);
3847         }
3848         return (wptr & rdev->ih.ptr_mask);
3849 }
3850
3851 /*        r600 IV Ring
3852  * Each IV ring entry is 128 bits:
3853  * [7:0]    - interrupt source id
3854  * [31:8]   - reserved
3855  * [59:32]  - interrupt source data
3856  * [127:60]  - reserved
3857  *
3858  * The basic interrupt vector entries
3859  * are decoded as follows:
3860  * src_id  src_data  description
3861  *      1         0  D1 Vblank
3862  *      1         1  D1 Vline
3863  *      5         0  D2 Vblank
3864  *      5         1  D2 Vline
3865  *     19         0  FP Hot plug detection A
3866  *     19         1  FP Hot plug detection B
3867  *     19         2  DAC A auto-detection
3868  *     19         3  DAC B auto-detection
3869  *     21         4  HDMI block A
3870  *     21         5  HDMI block B
3871  *    176         -  CP_INT RB
3872  *    177         -  CP_INT IB1
3873  *    178         -  CP_INT IB2
3874  *    181         -  EOP Interrupt
3875  *    233         -  GUI Idle
3876  *
3877  * Note, these are based on r600 and may need to be
3878  * adjusted or added to on newer asics
3879  */
3880
3881 irqreturn_t r600_irq_process(struct radeon_device *rdev)
3882 {
3883         u32 wptr;
3884         u32 rptr;
3885         u32 src_id, src_data;
3886         u32 ring_index;
3887         bool queue_hotplug = false;
3888         bool queue_hdmi = false;
3889
3890         if (!rdev->ih.enabled || rdev->shutdown)
3891                 return IRQ_NONE;
3892
3893         /* No MSIs, need a dummy read to flush PCI DMAs */
3894         if (!rdev->msi_enabled)
3895                 RREG32(IH_RB_WPTR);
3896
3897         wptr = r600_get_ih_wptr(rdev);
3898
3899 restart_ih:
3900         /* is somebody else already processing irqs? */
3901         if (atomic_xchg(&rdev->ih.lock, 1))
3902                 return IRQ_NONE;
3903
3904         rptr = rdev->ih.rptr;
3905         DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
3906
3907         /* Order reading of wptr vs. reading of IH ring data */
3908         rmb();
3909
3910         /* display interrupts */
3911         r600_irq_ack(rdev);
3912
3913         while (rptr != wptr) {
3914                 /* wptr/rptr are in bytes! */
3915                 ring_index = rptr / 4;
3916                 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
3917                 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
3918
3919                 switch (src_id) {
3920                 case 1: /* D1 vblank/vline */
3921                         switch (src_data) {
3922                         case 0: /* D1 vblank */
3923                                 if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VBLANK_INTERRUPT) {
3924                                         if (rdev->irq.crtc_vblank_int[0]) {
3925                                                 drm_handle_vblank(rdev->ddev, 0);
3926                                                 rdev->pm.vblank_sync = true;
3927                                                 DRM_WAKEUP(&rdev->irq.vblank_queue);
3928                                         }
3929                                         if (atomic_read(&rdev->irq.pflip[0]))
3930                                                 radeon_crtc_handle_flip(rdev, 0);
3931                                         rdev->irq.stat_regs.r600.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
3932                                         DRM_DEBUG("IH: D1 vblank\n");
3933                                 }
3934                                 break;
3935                         case 1: /* D1 vline */
3936                                 if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VLINE_INTERRUPT) {
3937                                         rdev->irq.stat_regs.r600.disp_int &= ~LB_D1_VLINE_INTERRUPT;
3938                                         DRM_DEBUG("IH: D1 vline\n");
3939                                 }
3940                                 break;
3941                         default:
3942                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
3943                                 break;
3944                         }
3945                         break;
3946                 case 5: /* D2 vblank/vline */
3947                         switch (src_data) {
3948                         case 0: /* D2 vblank */
3949                                 if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VBLANK_INTERRUPT) {
3950                                         if (rdev->irq.crtc_vblank_int[1]) {
3951                                                 drm_handle_vblank(rdev->ddev, 1);
3952                                                 rdev->pm.vblank_sync = true;
3953                                                 DRM_WAKEUP(&rdev->irq.vblank_queue);
3954                                         }
3955                                         if (atomic_read(&rdev->irq.pflip[1]))
3956                                                 radeon_crtc_handle_flip(rdev, 1);
3957                                         rdev->irq.stat_regs.r600.disp_int &= ~LB_D2_VBLANK_INTERRUPT;
3958                                         DRM_DEBUG("IH: D2 vblank\n");
3959                                 }
3960                                 break;
3961                         case 1: /* D1 vline */
3962                                 if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VLINE_INTERRUPT) {
3963                                         rdev->irq.stat_regs.r600.disp_int &= ~LB_D2_VLINE_INTERRUPT;
3964                                         DRM_DEBUG("IH: D2 vline\n");
3965                                 }
3966                                 break;
3967                         default:
3968                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
3969                                 break;
3970                         }
3971                         break;
3972                 case 19: /* HPD/DAC hotplug */
3973                         switch (src_data) {
3974                         case 0:
3975                                 if (rdev->irq.stat_regs.r600.disp_int & DC_HPD1_INTERRUPT) {
3976                                         rdev->irq.stat_regs.r600.disp_int &= ~DC_HPD1_INTERRUPT;
3977                                         queue_hotplug = true;
3978                                         DRM_DEBUG("IH: HPD1\n");
3979                                 }
3980                                 break;
3981                         case 1:
3982                                 if (rdev->irq.stat_regs.r600.disp_int & DC_HPD2_INTERRUPT) {
3983                                         rdev->irq.stat_regs.r600.disp_int &= ~DC_HPD2_INTERRUPT;
3984                                         queue_hotplug = true;
3985                                         DRM_DEBUG("IH: HPD2\n");
3986                                 }
3987                                 break;
3988                         case 4:
3989                                 if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD3_INTERRUPT) {
3990                                         rdev->irq.stat_regs.r600.disp_int_cont &= ~DC_HPD3_INTERRUPT;
3991                                         queue_hotplug = true;
3992                                         DRM_DEBUG("IH: HPD3\n");
3993                                 }
3994                                 break;
3995                         case 5:
3996                                 if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD4_INTERRUPT) {
3997                                         rdev->irq.stat_regs.r600.disp_int_cont &= ~DC_HPD4_INTERRUPT;
3998                                         queue_hotplug = true;
3999                                         DRM_DEBUG("IH: HPD4\n");
4000                                 }
4001                                 break;
4002                         case 10:
4003                                 if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD5_INTERRUPT) {
4004                                         rdev->irq.stat_regs.r600.disp_int_cont2 &= ~DC_HPD5_INTERRUPT;
4005                                         queue_hotplug = true;
4006                                         DRM_DEBUG("IH: HPD5\n");
4007                                 }
4008                                 break;
4009                         case 12:
4010                                 if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD6_INTERRUPT) {
4011                                         rdev->irq.stat_regs.r600.disp_int_cont2 &= ~DC_HPD6_INTERRUPT;
4012                                         queue_hotplug = true;
4013                                         DRM_DEBUG("IH: HPD6\n");
4014                                 }
4015                                 break;
4016                         default:
4017                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4018                                 break;
4019                         }
4020                         break;
4021                 case 21: /* hdmi */
4022                         switch (src_data) {
4023                         case 4:
4024                                 if (rdev->irq.stat_regs.r600.hdmi0_status & HDMI0_AZ_FORMAT_WTRIG) {
4025                                         rdev->irq.stat_regs.r600.hdmi0_status &= ~HDMI0_AZ_FORMAT_WTRIG;
4026                                         queue_hdmi = true;
4027                                         DRM_DEBUG("IH: HDMI0\n");
4028                                 }
4029                                 break;
4030                         case 5:
4031                                 if (rdev->irq.stat_regs.r600.hdmi1_status & HDMI0_AZ_FORMAT_WTRIG) {
4032                                         rdev->irq.stat_regs.r600.hdmi1_status &= ~HDMI0_AZ_FORMAT_WTRIG;
4033                                         queue_hdmi = true;
4034                                         DRM_DEBUG("IH: HDMI1\n");
4035                                 }
4036                                 break;
4037                         default:
4038                                 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
4039                                 break;
4040                         }
4041                         break;
4042                 case 176: /* CP_INT in ring buffer */
4043                 case 177: /* CP_INT in IB1 */
4044                 case 178: /* CP_INT in IB2 */
4045                         DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
4046                         radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4047                         break;
4048                 case 181: /* CP EOP event */
4049                         DRM_DEBUG("IH: CP EOP\n");
4050                         radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4051                         break;
4052                 case 224: /* DMA trap event */
4053                         DRM_DEBUG("IH: DMA trap\n");
4054                         radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
4055                         break;
4056                 case 233: /* GUI IDLE */
4057                         DRM_DEBUG("IH: GUI idle\n");
4058                         break;
4059                 default:
4060                         DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4061                         break;
4062                 }
4063
4064                 /* wptr/rptr are in bytes! */
4065                 rptr += 16;
4066                 rptr &= rdev->ih.ptr_mask;
4067         }
4068         if (queue_hotplug)
4069                 taskqueue_enqueue(rdev->tq, &rdev->hotplug_work);
4070         if (queue_hdmi)
4071                 taskqueue_enqueue(rdev->tq, &rdev->audio_work);
4072         rdev->ih.rptr = rptr;
4073         WREG32(IH_RB_RPTR, rdev->ih.rptr);
4074         atomic_set(&rdev->ih.lock, 0);
4075
4076         /* make sure wptr hasn't changed while processing */
4077         wptr = r600_get_ih_wptr(rdev);
4078         if (wptr != rptr)
4079                 goto restart_ih;
4080
4081         return IRQ_HANDLED;
4082 }
4083
4084 /*
4085  * Debugfs info
4086  */
4087 #if defined(CONFIG_DEBUG_FS)
4088
4089 static int r600_debugfs_mc_info(struct seq_file *m, void *data)
4090 {
4091         struct drm_info_node *node = (struct drm_info_node *) m->private;
4092         struct drm_device *dev = node->minor->dev;
4093         struct radeon_device *rdev = dev->dev_private;
4094
4095         DREG32_SYS(m, rdev, R_000E50_SRBM_STATUS);
4096         DREG32_SYS(m, rdev, VM_L2_STATUS);
4097         return 0;
4098 }
4099
4100 static struct drm_info_list r600_mc_info_list[] = {
4101         {"r600_mc_info", r600_debugfs_mc_info, 0, NULL},
4102 };
4103 #endif
4104
4105 int r600_debugfs_mc_info_init(struct radeon_device *rdev)
4106 {
4107 #if defined(CONFIG_DEBUG_FS)
4108         return radeon_debugfs_add_files(rdev, r600_mc_info_list, ARRAY_SIZE(r600_mc_info_list));
4109 #else
4110         return 0;
4111 #endif
4112 }
4113
4114 /**
4115  * r600_ioctl_wait_idle - flush host path cache on wait idle ioctl
4116  * rdev: radeon device structure
4117  * bo: buffer object struct which userspace is waiting for idle
4118  *
4119  * Some R6XX/R7XX doesn't seems to take into account HDP flush performed
4120  * through ring buffer, this leads to corruption in rendering, see
4121  * http://bugzilla.kernel.org/show_bug.cgi?id=15186 to avoid this we
4122  * directly perform HDP flush by writing register through MMIO.
4123  */
4124 void r600_ioctl_wait_idle(struct radeon_device *rdev, struct radeon_bo *bo)
4125 {
4126         /* r7xx hw bug.  write to HDP_DEBUG1 followed by fb read
4127          * rather than write to HDP_REG_COHERENCY_FLUSH_CNTL.
4128          * This seems to cause problems on some AGP cards. Just use the old
4129          * method for them.
4130          */
4131         if ((rdev->family >= CHIP_RV770) && (rdev->family <= CHIP_RV740) &&
4132             rdev->vram_scratch.ptr && !(rdev->flags & RADEON_IS_AGP)) {
4133                 volatile uint32_t *ptr = rdev->vram_scratch.ptr;
4134                 u32 tmp;
4135
4136                 WREG32(HDP_DEBUG1, 0);
4137                 tmp = *ptr;
4138         } else
4139                 WREG32(R_005480_HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
4140 }
4141
4142 void r600_set_pcie_lanes(struct radeon_device *rdev, int lanes)
4143 {
4144         u32 link_width_cntl, mask, target_reg;
4145
4146         if (rdev->flags & RADEON_IS_IGP)
4147                 return;
4148
4149         if (!(rdev->flags & RADEON_IS_PCIE))
4150                 return;
4151
4152         /* x2 cards have a special sequence */
4153         if (ASIC_IS_X2(rdev))
4154                 return;
4155
4156         /* FIXME wait for idle */
4157
4158         switch (lanes) {
4159         case 0:
4160                 mask = RADEON_PCIE_LC_LINK_WIDTH_X0;
4161                 break;
4162         case 1:
4163                 mask = RADEON_PCIE_LC_LINK_WIDTH_X1;
4164                 break;
4165         case 2:
4166                 mask = RADEON_PCIE_LC_LINK_WIDTH_X2;
4167                 break;
4168         case 4:
4169                 mask = RADEON_PCIE_LC_LINK_WIDTH_X4;
4170                 break;
4171         case 8:
4172                 mask = RADEON_PCIE_LC_LINK_WIDTH_X8;
4173                 break;
4174         case 12:
4175                 mask = RADEON_PCIE_LC_LINK_WIDTH_X12;
4176                 break;
4177         case 16:
4178         default:
4179                 mask = RADEON_PCIE_LC_LINK_WIDTH_X16;
4180                 break;
4181         }
4182
4183         link_width_cntl = RREG32_PCIE_P(RADEON_PCIE_LC_LINK_WIDTH_CNTL);
4184
4185         if ((link_width_cntl & RADEON_PCIE_LC_LINK_WIDTH_RD_MASK) ==
4186             (mask << RADEON_PCIE_LC_LINK_WIDTH_RD_SHIFT))
4187                 return;
4188
4189         if (link_width_cntl & R600_PCIE_LC_UPCONFIGURE_DIS)
4190                 return;
4191
4192         link_width_cntl &= ~(RADEON_PCIE_LC_LINK_WIDTH_MASK |
4193                              RADEON_PCIE_LC_RECONFIG_NOW |
4194                              R600_PCIE_LC_RENEGOTIATE_EN |
4195                              R600_PCIE_LC_RECONFIG_ARC_MISSING_ESCAPE);
4196         link_width_cntl |= mask;
4197
4198         WREG32_PCIE_P(RADEON_PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
4199
4200         /* some northbridges can renegotiate the link rather than requiring
4201          * a complete re-config.
4202          * e.g., AMD 780/790 northbridges (pci ids: 0x5956, 0x5957, 0x5958, etc.)
4203          */
4204         if (link_width_cntl & R600_PCIE_LC_RENEGOTIATION_SUPPORT)
4205                 link_width_cntl |= R600_PCIE_LC_RENEGOTIATE_EN | R600_PCIE_LC_UPCONFIGURE_SUPPORT;
4206         else
4207                 link_width_cntl |= R600_PCIE_LC_RECONFIG_ARC_MISSING_ESCAPE;
4208
4209         WREG32_PCIE_P(RADEON_PCIE_LC_LINK_WIDTH_CNTL, (link_width_cntl |
4210                                                        RADEON_PCIE_LC_RECONFIG_NOW));
4211
4212         if (rdev->family >= CHIP_RV770)
4213                 target_reg = R700_TARGET_AND_CURRENT_PROFILE_INDEX;
4214         else
4215                 target_reg = R600_TARGET_AND_CURRENT_PROFILE_INDEX;
4216
4217         /* wait for lane set to complete */
4218         link_width_cntl = RREG32(target_reg);
4219         while (link_width_cntl == 0xffffffff)
4220                 link_width_cntl = RREG32(target_reg);
4221
4222 }
4223
4224 int r600_get_pcie_lanes(struct radeon_device *rdev)
4225 {
4226         u32 link_width_cntl;
4227
4228         if (rdev->flags & RADEON_IS_IGP)
4229                 return 0;
4230
4231         if (!(rdev->flags & RADEON_IS_PCIE))
4232                 return 0;
4233
4234         /* x2 cards have a special sequence */
4235         if (ASIC_IS_X2(rdev))
4236                 return 0;
4237
4238         /* FIXME wait for idle */
4239
4240         link_width_cntl = RREG32_PCIE_P(RADEON_PCIE_LC_LINK_WIDTH_CNTL);
4241
4242         switch ((link_width_cntl & RADEON_PCIE_LC_LINK_WIDTH_RD_MASK) >> RADEON_PCIE_LC_LINK_WIDTH_RD_SHIFT) {
4243         case RADEON_PCIE_LC_LINK_WIDTH_X0:
4244                 return 0;
4245         case RADEON_PCIE_LC_LINK_WIDTH_X1:
4246                 return 1;
4247         case RADEON_PCIE_LC_LINK_WIDTH_X2:
4248                 return 2;
4249         case RADEON_PCIE_LC_LINK_WIDTH_X4:
4250                 return 4;
4251         case RADEON_PCIE_LC_LINK_WIDTH_X8:
4252                 return 8;
4253         case RADEON_PCIE_LC_LINK_WIDTH_X16:
4254         default:
4255                 return 16;
4256         }
4257 }
4258
4259 static void r600_pcie_gen2_enable(struct radeon_device *rdev)
4260 {
4261         u32 link_width_cntl, lanes, speed_cntl, training_cntl, tmp;
4262         u16 link_cntl2;
4263         u32 mask;
4264         int ret;
4265
4266         if (radeon_pcie_gen2 == 0)
4267                 return;
4268
4269         if (rdev->flags & RADEON_IS_IGP)
4270                 return;
4271
4272         if (!(rdev->flags & RADEON_IS_PCIE))
4273                 return;
4274
4275         /* x2 cards have a special sequence */
4276         if (ASIC_IS_X2(rdev))
4277                 return;
4278
4279         /* only RV6xx+ chips are supported */
4280         if (rdev->family <= CHIP_R600)
4281                 return;
4282
4283         ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
4284         if (ret != 0)
4285                 return;
4286
4287         if (!(mask & DRM_PCIE_SPEED_50))
4288                 return;
4289
4290         speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
4291         if (speed_cntl & LC_CURRENT_DATA_RATE) {
4292                 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
4293                 return;
4294         }
4295
4296         DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
4297
4298         /* 55 nm r6xx asics */
4299         if ((rdev->family == CHIP_RV670) ||
4300             (rdev->family == CHIP_RV620) ||
4301             (rdev->family == CHIP_RV635)) {
4302                 /* advertise upconfig capability */
4303                 link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
4304                 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
4305                 WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
4306                 link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
4307                 if (link_width_cntl & LC_RENEGOTIATION_SUPPORT) {
4308                         lanes = (link_width_cntl & LC_LINK_WIDTH_RD_MASK) >> LC_LINK_WIDTH_RD_SHIFT;
4309                         link_width_cntl &= ~(LC_LINK_WIDTH_MASK |
4310                                              LC_RECONFIG_ARC_MISSING_ESCAPE);
4311                         link_width_cntl |= lanes | LC_RECONFIG_NOW | LC_RENEGOTIATE_EN;
4312                         WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
4313                 } else {
4314                         link_width_cntl |= LC_UPCONFIGURE_DIS;
4315                         WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
4316                 }
4317         }
4318
4319         speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
4320         if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
4321             (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
4322
4323                 /* 55 nm r6xx asics */
4324                 if ((rdev->family == CHIP_RV670) ||
4325                     (rdev->family == CHIP_RV620) ||
4326                     (rdev->family == CHIP_RV635)) {
4327                         WREG32(MM_CFGREGS_CNTL, 0x8);
4328                         link_cntl2 = RREG32(0x4088);
4329                         WREG32(MM_CFGREGS_CNTL, 0);
4330                         /* not supported yet */
4331                         if (link_cntl2 & SELECTABLE_DEEMPHASIS)
4332                                 return;
4333                 }
4334
4335                 speed_cntl &= ~LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_MASK;
4336                 speed_cntl |= (0x3 << LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_SHIFT);
4337                 speed_cntl &= ~LC_VOLTAGE_TIMER_SEL_MASK;
4338                 speed_cntl &= ~LC_FORCE_DIS_HW_SPEED_CHANGE;
4339                 speed_cntl |= LC_FORCE_EN_HW_SPEED_CHANGE;
4340                 WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
4341
4342                 tmp = RREG32(0x541c);
4343                 WREG32(0x541c, tmp | 0x8);
4344                 WREG32(MM_CFGREGS_CNTL, MM_WR_TO_CFG_EN);
4345                 link_cntl2 = RREG16(0x4088);
4346                 link_cntl2 &= ~TARGET_LINK_SPEED_MASK;
4347                 link_cntl2 |= 0x2;
4348                 WREG16(0x4088, link_cntl2);
4349                 WREG32(MM_CFGREGS_CNTL, 0);
4350
4351                 if ((rdev->family == CHIP_RV670) ||
4352                     (rdev->family == CHIP_RV620) ||
4353                     (rdev->family == CHIP_RV635)) {
4354                         training_cntl = RREG32_PCIE_P(PCIE_LC_TRAINING_CNTL);
4355                         training_cntl &= ~LC_POINT_7_PLUS_EN;
4356                         WREG32_PCIE_P(PCIE_LC_TRAINING_CNTL, training_cntl);
4357                 } else {
4358                         speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
4359                         speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
4360                         WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
4361                 }
4362
4363                 speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
4364                 speed_cntl |= LC_GEN2_EN_STRAP;
4365                 WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
4366
4367         } else {
4368                 link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
4369                 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
4370                 if (1)
4371                         link_width_cntl |= LC_UPCONFIGURE_DIS;
4372                 else
4373                         link_width_cntl &= ~LC_UPCONFIGURE_DIS;
4374                 WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
4375         }
4376 }
4377
4378 /**
4379  * r600_get_gpu_clock - return GPU clock counter snapshot
4380  *
4381  * @rdev: radeon_device pointer
4382  *
4383  * Fetches a GPU clock counter snapshot (R6xx-cayman).
4384  * Returns the 64 bit clock counter snapshot.
4385  */
4386 uint64_t r600_get_gpu_clock(struct radeon_device *rdev)
4387 {
4388         uint64_t clock;
4389
4390         sx_xlock(&rdev->gpu_clock_mutex);
4391         WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
4392         clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
4393                 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
4394         sx_xunlock(&rdev->gpu_clock_mutex);
4395         return clock;
4396 }