sys/dev/drm/r600_cp.c

   1 /*-
   2  * Copyright 2008-2009 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  22  * DEALINGS IN THE SOFTWARE.
  23  *
  24  * Authors:
  25  *     Dave Airlie <airlied@redhat.com>
  26  *     Alex Deucher <alexander.deucher@amd.com>
  27  */
  28
  29 #include <sys/cdefs.h>
  30 __FBSDID("$FreeBSD$");
  31
  32 #include "dev/drm/drmP.h"
  33 #include "dev/drm/drm.h"
  34 #include "dev/drm/radeon_drm.h"
  35 #include "dev/drm/radeon_drv.h"
  36
  37 #include "dev/drm/r600_microcode.h"
  38
  39 # define ATI_PCIGART_PAGE_SIZE          4096    /**< PCI GART page size */
  40 # define ATI_PCIGART_PAGE_MASK          (~(ATI_PCIGART_PAGE_SIZE-1))
  41
  42 #define R600_PTE_VALID     (1 << 0)
  43 #define R600_PTE_SYSTEM    (1 << 1)
  44 #define R600_PTE_SNOOPED   (1 << 2)
  45 #define R600_PTE_READABLE  (1 << 5)
  46 #define R600_PTE_WRITEABLE (1 << 6)
  47
  48 /* MAX values used for gfx init */
  49 #define R6XX_MAX_SH_GPRS           256
  50 #define R6XX_MAX_TEMP_GPRS         16
  51 #define R6XX_MAX_SH_THREADS        256
  52 #define R6XX_MAX_SH_STACK_ENTRIES  4096
  53 #define R6XX_MAX_BACKENDS          8
  54 #define R6XX_MAX_BACKENDS_MASK     0xff
  55 #define R6XX_MAX_SIMDS             8
  56 #define R6XX_MAX_SIMDS_MASK        0xff
  57 #define R6XX_MAX_PIPES             8
  58 #define R6XX_MAX_PIPES_MASK        0xff
  59
  60 #define R7XX_MAX_SH_GPRS           256
  61 #define R7XX_MAX_TEMP_GPRS         16
  62 #define R7XX_MAX_SH_THREADS        256
  63 #define R7XX_MAX_SH_STACK_ENTRIES  4096
  64 #define R7XX_MAX_BACKENDS          8
  65 #define R7XX_MAX_BACKENDS_MASK     0xff
  66 #define R7XX_MAX_SIMDS             16
  67 #define R7XX_MAX_SIMDS_MASK        0xffff
  68 #define R7XX_MAX_PIPES             8
  69 #define R7XX_MAX_PIPES_MASK        0xff
  70
  71 static int r600_do_wait_for_fifo(drm_radeon_private_t *dev_priv, int entries)
  72 {
  73         int i;
  74
  75         dev_priv->stats.boxes |= RADEON_BOX_WAIT_IDLE;
  76
  77         for (i = 0; i < dev_priv->usec_timeout; i++) {
  78                 int slots;
  79                 if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV770)
  80                         slots = (RADEON_READ(R600_GRBM_STATUS)
  81                                  & R700_CMDFIFO_AVAIL_MASK);
  82                 else
  83                         slots = (RADEON_READ(R600_GRBM_STATUS)
  84                                  & R600_CMDFIFO_AVAIL_MASK);
  85                 if (slots >= entries)
  86                         return 0;
  87                 DRM_UDELAY(1);
  88         }
  89         DRM_INFO("wait for fifo failed status : 0x%08X 0x%08X\n",
  90                  RADEON_READ(R600_GRBM_STATUS),
  91                  RADEON_READ(R600_GRBM_STATUS2));
  92
  93         return -EBUSY;
  94 }
  95
  96 static int r600_do_wait_for_idle(drm_radeon_private_t *dev_priv)
  97 {
  98         int i, ret;
  99
 100         dev_priv->stats.boxes |= RADEON_BOX_WAIT_IDLE;
 101
 102         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV770)
 103                 ret = r600_do_wait_for_fifo(dev_priv, 8);
 104         else
 105                 ret = r600_do_wait_for_fifo(dev_priv, 16);
 106         if (ret)
 107                 return ret;
 108         for (i = 0; i < dev_priv->usec_timeout; i++) {
 109                 if (!(RADEON_READ(R600_GRBM_STATUS) & R600_GUI_ACTIVE))
 110                         return 0;
 111                 DRM_UDELAY(1);
 112         }
 113         DRM_INFO("wait idle failed status : 0x%08X 0x%08X\n",
 114                  RADEON_READ(R600_GRBM_STATUS),
 115                  RADEON_READ(R600_GRBM_STATUS2));
 116
 117         return -EBUSY;
 118 }
 119
 120 void r600_page_table_cleanup(struct drm_device *dev, struct drm_ati_pcigart_info *gart_info)
 121 {
 122 #ifdef __linux__
 123         struct drm_sg_mem *entry = dev->sg;
 124         int max_pages;
 125         int pages;
 126         int i;
 127 #endif
 128         if (gart_info->bus_addr) {
 129 #ifdef __linux__
 130                 max_pages = (gart_info->table_size / sizeof(u32));
 131                 pages = (entry->pages <= max_pages)
 132                   ? entry->pages : max_pages;
 133
 134                 for (i = 0; i < pages; i++) {
 135                         if (!entry->busaddr[i])
 136                                 break;
 137                         pci_unmap_single(dev->pdev, entry->busaddr[i],
 138                                          PAGE_SIZE, PCI_DMA_TODEVICE);
 139                 }
 140 #endif
 141                 if (gart_info->gart_table_location == DRM_ATI_GART_MAIN)
 142                         gart_info->bus_addr = 0;
 143         }
 144 }
 145
 146 /* R600 has page table setup */
 147 int r600_page_table_init(struct drm_device *dev)
 148 {
 149         drm_radeon_private_t *dev_priv = dev->dev_private;
 150         struct drm_ati_pcigart_info *gart_info = &dev_priv->gart_info;
 151         struct drm_sg_mem *entry = dev->sg;
 152         int ret = 0;
 153         int i, j;
 154         int max_pages, pages;
 155         u64 *pci_gart, page_base;
 156         dma_addr_t entry_addr;
 157
 158         /* okay page table is available - lets rock */
 159
 160         /* PTEs are 64-bits */
 161         pci_gart = (u64 *)gart_info->addr;
 162
 163         max_pages = (gart_info->table_size / sizeof(u64));
 164         pages = (entry->pages <= max_pages) ? entry->pages : max_pages;
 165
 166         memset(pci_gart, 0, max_pages * sizeof(u64));
 167
 168         for (i = 0; i < pages; i++) {
 169 #ifdef __linux__
 170                 entry->busaddr[i] = pci_map_single(dev->pdev,
 171                                                    page_address(entry->
 172                                                                 pagelist[i]),
 173                                                    PAGE_SIZE, PCI_DMA_TODEVICE);
 174                 if (entry->busaddr[i] == 0) {
 175                         DRM_ERROR("unable to map PCIGART pages!\n");
 176                         r600_page_table_cleanup(dev, gart_info);
 177                         goto done;
 178                 }
 179 #endif
 180                 entry_addr = entry->busaddr[i];
 181                 for (j = 0; j < (PAGE_SIZE / ATI_PCIGART_PAGE_SIZE); j++) {
 182                         page_base = (u64) entry_addr & ATI_PCIGART_PAGE_MASK;
 183                         page_base |= R600_PTE_VALID | R600_PTE_SYSTEM | R600_PTE_SNOOPED;
 184                         page_base |= R600_PTE_READABLE | R600_PTE_WRITEABLE;
 185
 186                         *pci_gart = page_base;
 187
 188                         if ((i % 128) == 0)
 189                                 DRM_DEBUG("page entry %d: 0x%016llx\n",
 190                                     i, (unsigned long long)page_base);
 191                         pci_gart++;
 192                         entry_addr += ATI_PCIGART_PAGE_SIZE;
 193                 }
 194         }
 195         ret = 1;
 196 #ifdef __linux__
 197 done:
 198 #endif
 199         return ret;
 200 }
 201
 202 static void r600_vm_flush_gart_range(struct drm_device *dev)
 203 {
 204         drm_radeon_private_t *dev_priv = dev->dev_private;
 205         u32 resp, countdown = 1000;
 206         RADEON_WRITE(R600_VM_CONTEXT0_INVALIDATION_LOW_ADDR, dev_priv->gart_vm_start >> 12);
 207         RADEON_WRITE(R600_VM_CONTEXT0_INVALIDATION_HIGH_ADDR, (dev_priv->gart_vm_start + dev_priv->gart_size - 1) >> 12);
 208         RADEON_WRITE(R600_VM_CONTEXT0_REQUEST_RESPONSE, 2);
 209
 210         do {
 211                 resp = RADEON_READ(R600_VM_CONTEXT0_REQUEST_RESPONSE);
 212                 countdown--;
 213                 DRM_UDELAY(1);
 214         } while (((resp & 0xf0) == 0) && countdown);
 215 }
 216
 217 static void r600_vm_init(struct drm_device *dev)
 218 {
 219         drm_radeon_private_t *dev_priv = dev->dev_private;
 220         /* initialise the VM to use the page table we constructed up there */
 221         u32 vm_c0, i;
 222         u32 mc_rd_a;
 223         u32 vm_l2_cntl, vm_l2_cntl3;
 224         /* okay set up the PCIE aperture type thingo */
 225         RADEON_WRITE(R600_MC_VM_SYSTEM_APERTURE_LOW_ADDR, dev_priv->gart_vm_start >> 12);
 226         RADEON_WRITE(R600_MC_VM_SYSTEM_APERTURE_HIGH_ADDR, (dev_priv->gart_vm_start + dev_priv->gart_size - 1) >> 12);
 227         RADEON_WRITE(R600_MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, 0);
 228
 229         /* setup MC RD a */
 230         mc_rd_a = R600_MCD_L1_TLB | R600_MCD_L1_FRAG_PROC | R600_MCD_SYSTEM_ACCESS_MODE_IN_SYS |
 231                 R600_MCD_SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU | R600_MCD_EFFECTIVE_L1_TLB_SIZE(5) |
 232                 R600_MCD_EFFECTIVE_L1_QUEUE_SIZE(5) | R600_MCD_WAIT_L2_QUERY;
 233
 234         RADEON_WRITE(R600_MCD_RD_A_CNTL, mc_rd_a);
 235         RADEON_WRITE(R600_MCD_RD_B_CNTL, mc_rd_a);
 236
 237         RADEON_WRITE(R600_MCD_WR_A_CNTL, mc_rd_a);
 238         RADEON_WRITE(R600_MCD_WR_B_CNTL, mc_rd_a);
 239
 240         RADEON_WRITE(R600_MCD_RD_GFX_CNTL, mc_rd_a);
 241         RADEON_WRITE(R600_MCD_WR_GFX_CNTL, mc_rd_a);
 242
 243         RADEON_WRITE(R600_MCD_RD_SYS_CNTL, mc_rd_a);
 244         RADEON_WRITE(R600_MCD_WR_SYS_CNTL, mc_rd_a);
 245
 246         RADEON_WRITE(R600_MCD_RD_HDP_CNTL, mc_rd_a | R600_MCD_L1_STRICT_ORDERING);
 247         RADEON_WRITE(R600_MCD_WR_HDP_CNTL, mc_rd_a /*| R600_MCD_L1_STRICT_ORDERING*/);
 248
 249         RADEON_WRITE(R600_MCD_RD_PDMA_CNTL, mc_rd_a);
 250         RADEON_WRITE(R600_MCD_WR_PDMA_CNTL, mc_rd_a);
 251
 252         RADEON_WRITE(R600_MCD_RD_SEM_CNTL, mc_rd_a | R600_MCD_SEMAPHORE_MODE);
 253         RADEON_WRITE(R600_MCD_WR_SEM_CNTL, mc_rd_a);
 254
 255         vm_l2_cntl = R600_VM_L2_CACHE_EN | R600_VM_L2_FRAG_PROC | R600_VM_ENABLE_PTE_CACHE_LRU_W;
 256         vm_l2_cntl |= R600_VM_L2_CNTL_QUEUE_SIZE(7);
 257         RADEON_WRITE(R600_VM_L2_CNTL, vm_l2_cntl);
 258
 259         RADEON_WRITE(R600_VM_L2_CNTL2, 0);
 260         vm_l2_cntl3 = (R600_VM_L2_CNTL3_BANK_SELECT_0(0) |
 261                        R600_VM_L2_CNTL3_BANK_SELECT_1(1) |
 262                        R600_VM_L2_CNTL3_CACHE_UPDATE_MODE(2));
 263         RADEON_WRITE(R600_VM_L2_CNTL3, vm_l2_cntl3);
 264
 265         vm_c0 = R600_VM_ENABLE_CONTEXT | R600_VM_PAGE_TABLE_DEPTH_FLAT;
 266
 267         RADEON_WRITE(R600_VM_CONTEXT0_CNTL, vm_c0);
 268
 269         vm_c0 &= ~R600_VM_ENABLE_CONTEXT;
 270
 271         /* disable all other contexts */
 272         for (i = 1; i < 8; i++)
 273                 RADEON_WRITE(R600_VM_CONTEXT0_CNTL + (i * 4), vm_c0);
 274
 275         RADEON_WRITE(R600_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, dev_priv->gart_info.bus_addr >> 12);
 276         RADEON_WRITE(R600_VM_CONTEXT0_PAGE_TABLE_START_ADDR, dev_priv->gart_vm_start >> 12);
 277         RADEON_WRITE(R600_VM_CONTEXT0_PAGE_TABLE_END_ADDR, (dev_priv->gart_vm_start + dev_priv->gart_size - 1) >> 12);
 278
 279         r600_vm_flush_gart_range(dev);
 280 }
 281
 282 /* load r600 microcode */
 283 static void r600_cp_load_microcode(drm_radeon_private_t *dev_priv)
 284 {
 285         int i;
 286
 287         r600_do_cp_stop(dev_priv);
 288
 289         RADEON_WRITE(R600_CP_RB_CNTL,
 290                      R600_RB_NO_UPDATE |
 291                      R600_RB_BLKSZ(15) |
 292                      R600_RB_BUFSZ(3));
 293
 294         RADEON_WRITE(R600_GRBM_SOFT_RESET, R600_SOFT_RESET_CP);
 295         RADEON_READ(R600_GRBM_SOFT_RESET);
 296         DRM_UDELAY(15000);
 297         RADEON_WRITE(R600_GRBM_SOFT_RESET, 0);
 298
 299         RADEON_WRITE(R600_CP_ME_RAM_WADDR, 0);
 300
 301         if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R600)) {
 302                 DRM_INFO("Loading R600 CP Microcode\n");
 303                 for (i = 0; i < PM4_UCODE_SIZE; i++) {
 304                         RADEON_WRITE(R600_CP_ME_RAM_DATA,
 305                                      R600_cp_microcode[i][0]);
 306                         RADEON_WRITE(R600_CP_ME_RAM_DATA,
 307                                      R600_cp_microcode[i][1]);
 308                         RADEON_WRITE(R600_CP_ME_RAM_DATA,
 309                                      R600_cp_microcode[i][2]);
 310                 }
 311
 312                 RADEON_WRITE(R600_CP_PFP_UCODE_ADDR, 0);
 313                 DRM_INFO("Loading R600 PFP Microcode\n");
 314                 for (i = 0; i < PFP_UCODE_SIZE; i++)
 315                         RADEON_WRITE(R600_CP_PFP_UCODE_DATA, R600_pfp_microcode[i]);
 316         } else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV610)) {
 317                 DRM_INFO("Loading RV610 CP Microcode\n");
 318                 for (i = 0; i < PM4_UCODE_SIZE; i++) {
 319                         RADEON_WRITE(R600_CP_ME_RAM_DATA,
 320                                      RV610_cp_microcode[i][0]);
 321                         RADEON_WRITE(R600_CP_ME_RAM_DATA,
 322                                      RV610_cp_microcode[i][1]);
 323                         RADEON_WRITE(R600_CP_ME_RAM_DATA,
 324                                      RV610_cp_microcode[i][2]);
 325                 }
 326
 327                 RADEON_WRITE(R600_CP_PFP_UCODE_ADDR, 0);
 328                 DRM_INFO("Loading RV610 PFP Microcode\n");
 329                 for (i = 0; i < PFP_UCODE_SIZE; i++)
 330                         RADEON_WRITE(R600_CP_PFP_UCODE_DATA, RV610_pfp_microcode[i]);
 331         } else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV630)) {
 332                 DRM_INFO("Loading RV630 CP Microcode\n");
 333                 for (i = 0; i < PM4_UCODE_SIZE; i++) {
 334                         RADEON_WRITE(R600_CP_ME_RAM_DATA,
 335                                      RV630_cp_microcode[i][0]);
 336                         RADEON_WRITE(R600_CP_ME_RAM_DATA,
 337                                      RV630_cp_microcode[i][1]);
 338                         RADEON_WRITE(R600_CP_ME_RAM_DATA,
 339                                      RV630_cp_microcode[i][2]);
 340                 }
 341
 342                 RADEON_WRITE(R600_CP_PFP_UCODE_ADDR, 0);
 343                 DRM_INFO("Loading RV630 PFP Microcode\n");
 344                 for (i = 0; i < PFP_UCODE_SIZE; i++)
 345                         RADEON_WRITE(R600_CP_PFP_UCODE_DATA, RV630_pfp_microcode[i]);
 346         } else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV620)) {
 347                 DRM_INFO("Loading RV620 CP Microcode\n");
 348                 for (i = 0; i < PM4_UCODE_SIZE; i++) {
 349                         RADEON_WRITE(R600_CP_ME_RAM_DATA,
 350                                      RV620_cp_microcode[i][0]);
 351                         RADEON_WRITE(R600_CP_ME_RAM_DATA,
 352                                      RV620_cp_microcode[i][1]);
 353                         RADEON_WRITE(R600_CP_ME_RAM_DATA,
 354                                      RV620_cp_microcode[i][2]);
 355                 }
 356
 357                 RADEON_WRITE(R600_CP_PFP_UCODE_ADDR, 0);
 358                 DRM_INFO("Loading RV620 PFP Microcode\n");
 359                 for (i = 0; i < PFP_UCODE_SIZE; i++)
 360                         RADEON_WRITE(R600_CP_PFP_UCODE_DATA, RV620_pfp_microcode[i]);
 361         } else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV635)) {
 362                 DRM_INFO("Loading RV635 CP Microcode\n");
 363                 for (i = 0; i < PM4_UCODE_SIZE; i++) {
 364                         RADEON_WRITE(R600_CP_ME_RAM_DATA,
 365                                      RV635_cp_microcode[i][0]);
 366                         RADEON_WRITE(R600_CP_ME_RAM_DATA,
 367                                      RV635_cp_microcode[i][1]);
 368                         RADEON_WRITE(R600_CP_ME_RAM_DATA,
 369                                      RV635_cp_microcode[i][2]);
 370                 }
 371
 372                 RADEON_WRITE(R600_CP_PFP_UCODE_ADDR, 0);
 373                 DRM_INFO("Loading RV635 PFP Microcode\n");
 374                 for (i = 0; i < PFP_UCODE_SIZE; i++)
 375                         RADEON_WRITE(R600_CP_PFP_UCODE_DATA, RV635_pfp_microcode[i]);
 376         } else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV670)) {
 377                 DRM_INFO("Loading RV670 CP Microcode\n");
 378                 for (i = 0; i < PM4_UCODE_SIZE; i++) {
 379                         RADEON_WRITE(R600_CP_ME_RAM_DATA,
 380                                      RV670_cp_microcode[i][0]);
 381                         RADEON_WRITE(R600_CP_ME_RAM_DATA,
 382                                      RV670_cp_microcode[i][1]);
 383                         RADEON_WRITE(R600_CP_ME_RAM_DATA,
 384                                      RV670_cp_microcode[i][2]);
 385                 }
 386
 387                 RADEON_WRITE(R600_CP_PFP_UCODE_ADDR, 0);
 388                 DRM_INFO("Loading RV670 PFP Microcode\n");
 389                 for (i = 0; i < PFP_UCODE_SIZE; i++)
 390                         RADEON_WRITE(R600_CP_PFP_UCODE_DATA, RV670_pfp_microcode[i]);
 391         } else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS780)) {
 392                 DRM_INFO("Loading RS780 CP Microcode\n");
 393                 for (i = 0; i < PM4_UCODE_SIZE; i++) {
 394                         RADEON_WRITE(R600_CP_ME_RAM_DATA,
 395                                      RS780_cp_microcode[i][0]);
 396                         RADEON_WRITE(R600_CP_ME_RAM_DATA,
 397                                      RS780_cp_microcode[i][1]);
 398                         RADEON_WRITE(R600_CP_ME_RAM_DATA,
 399                                      RS780_cp_microcode[i][2]);
 400                 }
 401
 402                 RADEON_WRITE(R600_CP_PFP_UCODE_ADDR, 0);
 403                 DRM_INFO("Loading RS780 PFP Microcode\n");
 404                 for (i = 0; i < PFP_UCODE_SIZE; i++)
 405                         RADEON_WRITE(R600_CP_PFP_UCODE_DATA, RS780_pfp_microcode[i]);
 406         }
 407         RADEON_WRITE(R600_CP_PFP_UCODE_ADDR, 0);
 408         RADEON_WRITE(R600_CP_ME_RAM_WADDR, 0);
 409         RADEON_WRITE(R600_CP_ME_RAM_RADDR, 0);
 410
 411 }
 412
 413 static void r700_vm_init(struct drm_device *dev)
 414 {
 415         drm_radeon_private_t *dev_priv = dev->dev_private;
 416         /* initialise the VM to use the page table we constructed up there */
 417         u32 vm_c0, i;
 418         u32 mc_vm_md_l1;
 419         u32 vm_l2_cntl, vm_l2_cntl3;
 420         /* okay set up the PCIE aperture type thingo */
 421         RADEON_WRITE(R700_MC_VM_SYSTEM_APERTURE_LOW_ADDR, dev_priv->gart_vm_start >> 12);
 422         RADEON_WRITE(R700_MC_VM_SYSTEM_APERTURE_HIGH_ADDR, (dev_priv->gart_vm_start + dev_priv->gart_size - 1) >> 12);
 423         RADEON_WRITE(R700_MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, 0);
 424
 425         mc_vm_md_l1 = R700_ENABLE_L1_TLB |
 426             R700_ENABLE_L1_FRAGMENT_PROCESSING |
 427             R700_SYSTEM_ACCESS_MODE_IN_SYS |
 428             R700_SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
 429             R700_EFFECTIVE_L1_TLB_SIZE(5) |
 430             R700_EFFECTIVE_L1_QUEUE_SIZE(5);
 431
 432         RADEON_WRITE(R700_MC_VM_MD_L1_TLB0_CNTL, mc_vm_md_l1);
 433         RADEON_WRITE(R700_MC_VM_MD_L1_TLB1_CNTL, mc_vm_md_l1);
 434         RADEON_WRITE(R700_MC_VM_MD_L1_TLB2_CNTL, mc_vm_md_l1);
 435         RADEON_WRITE(R700_MC_VM_MB_L1_TLB0_CNTL, mc_vm_md_l1);
 436         RADEON_WRITE(R700_MC_VM_MB_L1_TLB1_CNTL, mc_vm_md_l1);
 437         RADEON_WRITE(R700_MC_VM_MB_L1_TLB2_CNTL, mc_vm_md_l1);
 438         RADEON_WRITE(R700_MC_VM_MB_L1_TLB3_CNTL, mc_vm_md_l1);
 439
 440         vm_l2_cntl = R600_VM_L2_CACHE_EN | R600_VM_L2_FRAG_PROC | R600_VM_ENABLE_PTE_CACHE_LRU_W;
 441         vm_l2_cntl |= R700_VM_L2_CNTL_QUEUE_SIZE(7);
 442         RADEON_WRITE(R600_VM_L2_CNTL, vm_l2_cntl);
 443
 444         RADEON_WRITE(R600_VM_L2_CNTL2, 0);
 445         vm_l2_cntl3 = R700_VM_L2_CNTL3_BANK_SELECT(0) | R700_VM_L2_CNTL3_CACHE_UPDATE_MODE(2);
 446         RADEON_WRITE(R600_VM_L2_CNTL3, vm_l2_cntl3);
 447
 448         vm_c0 = R600_VM_ENABLE_CONTEXT | R600_VM_PAGE_TABLE_DEPTH_FLAT;
 449
 450         RADEON_WRITE(R600_VM_CONTEXT0_CNTL, vm_c0);
 451
 452         vm_c0 &= ~R600_VM_ENABLE_CONTEXT;
 453
 454         /* disable all other contexts */
 455         for (i = 1; i < 8; i++)
 456                 RADEON_WRITE(R600_VM_CONTEXT0_CNTL + (i * 4), vm_c0);
 457
 458         RADEON_WRITE(R700_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, dev_priv->gart_info.bus_addr >> 12);
 459         RADEON_WRITE(R700_VM_CONTEXT0_PAGE_TABLE_START_ADDR, dev_priv->gart_vm_start >> 12);
 460         RADEON_WRITE(R700_VM_CONTEXT0_PAGE_TABLE_END_ADDR, (dev_priv->gart_vm_start + dev_priv->gart_size - 1) >> 12);
 461
 462         r600_vm_flush_gart_range(dev);
 463 }
 464
 465 /* load r600 microcode */
 466 static void r700_cp_load_microcode(drm_radeon_private_t *dev_priv)
 467 {
 468         int i;
 469
 470         r600_do_cp_stop(dev_priv);
 471
 472         RADEON_WRITE(R600_CP_RB_CNTL,
 473                      R600_RB_NO_UPDATE |
 474                      (15 << 8) |
 475                      (3 << 0));
 476
 477         RADEON_WRITE(R600_GRBM_SOFT_RESET, R600_SOFT_RESET_CP);
 478         RADEON_READ(R600_GRBM_SOFT_RESET);
 479         DRM_UDELAY(15000);
 480         RADEON_WRITE(R600_GRBM_SOFT_RESET, 0);
 481
 482
 483         if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV770)) {
 484                 RADEON_WRITE(R600_CP_PFP_UCODE_ADDR, 0);
 485                 DRM_INFO("Loading RV770 PFP Microcode\n");
 486                 for (i = 0; i < R700_PFP_UCODE_SIZE; i++)
 487                         RADEON_WRITE(R600_CP_PFP_UCODE_DATA, RV770_pfp_microcode[i]);
 488                 RADEON_WRITE(R600_CP_PFP_UCODE_ADDR, 0);
 489
 490                 RADEON_WRITE(R600_CP_ME_RAM_WADDR, 0);
 491                 DRM_INFO("Loading RV770 CP Microcode\n");
 492                 for (i = 0; i < R700_PM4_UCODE_SIZE; i++)
 493                         RADEON_WRITE(R600_CP_ME_RAM_DATA, RV770_cp_microcode[i]);
 494                 RADEON_WRITE(R600_CP_ME_RAM_WADDR, 0);
 495
 496         } else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV730)) {
 497                 RADEON_WRITE(R600_CP_PFP_UCODE_ADDR, 0);
 498                 DRM_INFO("Loading RV730 PFP Microcode\n");
 499                 for (i = 0; i < R700_PFP_UCODE_SIZE; i++)
 500                         RADEON_WRITE(R600_CP_PFP_UCODE_DATA, RV730_pfp_microcode[i]);
 501                 RADEON_WRITE(R600_CP_PFP_UCODE_ADDR, 0);
 502
 503                 RADEON_WRITE(R600_CP_ME_RAM_WADDR, 0);
 504                 DRM_INFO("Loading RV730 CP Microcode\n");
 505                 for (i = 0; i < R700_PM4_UCODE_SIZE; i++)
 506                         RADEON_WRITE(R600_CP_ME_RAM_DATA, RV730_cp_microcode[i]);
 507                 RADEON_WRITE(R600_CP_ME_RAM_WADDR, 0);
 508
 509         } else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV710)) {
 510                 RADEON_WRITE(R600_CP_PFP_UCODE_ADDR, 0);
 511                 DRM_INFO("Loading RV710 PFP Microcode\n");
 512                 for (i = 0; i < R700_PFP_UCODE_SIZE; i++)
 513                         RADEON_WRITE(R600_CP_PFP_UCODE_DATA, RV710_pfp_microcode[i]);
 514                 RADEON_WRITE(R600_CP_PFP_UCODE_ADDR, 0);
 515
 516                 RADEON_WRITE(R600_CP_ME_RAM_WADDR, 0);
 517                 DRM_INFO("Loading RV710 CP Microcode\n");
 518                 for (i = 0; i < R700_PM4_UCODE_SIZE; i++)
 519                         RADEON_WRITE(R600_CP_ME_RAM_DATA, RV710_cp_microcode[i]);
 520                 RADEON_WRITE(R600_CP_ME_RAM_WADDR, 0);
 521
 522         }
 523         RADEON_WRITE(R600_CP_PFP_UCODE_ADDR, 0);
 524         RADEON_WRITE(R600_CP_ME_RAM_WADDR, 0);
 525         RADEON_WRITE(R600_CP_ME_RAM_RADDR, 0);
 526
 527 }
 528
 529 static void r600_test_writeback(drm_radeon_private_t *dev_priv)
 530 {
 531         u32 tmp;
 532
 533         /* Start with assuming that writeback doesn't work */
 534         dev_priv->writeback_works = 0;
 535
 536         /* Writeback doesn't seem to work everywhere, test it here and possibly
 537          * enable it if it appears to work
 538          */
 539         radeon_write_ring_rptr(dev_priv, R600_SCRATCHOFF(1), 0);
 540
 541         RADEON_WRITE(R600_SCRATCH_REG1, 0xdeadbeef);
 542
 543         for (tmp = 0; tmp < dev_priv->usec_timeout; tmp++) {
 544                 u32 val;
 545
 546                 val = radeon_read_ring_rptr(dev_priv, R600_SCRATCHOFF(1));
 547                 if (val == 0xdeadbeef)
 548                         break;
 549                 DRM_UDELAY(1);
 550         }
 551
 552         if (tmp < dev_priv->usec_timeout) {
 553                 dev_priv->writeback_works = 1;
 554                 DRM_INFO("writeback test succeeded in %d usecs\n", tmp);
 555         } else {
 556                 dev_priv->writeback_works = 0;
 557                 DRM_INFO("writeback test failed\n");
 558         }
 559         if (radeon_no_wb == 1) {
 560                 dev_priv->writeback_works = 0;
 561                 DRM_INFO("writeback forced off\n");
 562         }
 563
 564         if (!dev_priv->writeback_works) {
 565                 /* Disable writeback to avoid unnecessary bus master transfer */
 566                 RADEON_WRITE(R600_CP_RB_CNTL, RADEON_READ(R600_CP_RB_CNTL) |
 567                              RADEON_RB_NO_UPDATE);
 568                 RADEON_WRITE(R600_SCRATCH_UMSK, 0);
 569         }
 570 }
 571
 572 int r600_do_engine_reset(struct drm_device *dev)
 573 {
 574         drm_radeon_private_t *dev_priv = dev->dev_private;
 575         u32 cp_ptr, cp_me_cntl, cp_rb_cntl;
 576
 577         DRM_INFO("Resetting GPU\n");
 578
 579         cp_ptr = RADEON_READ(R600_CP_RB_WPTR);
 580         cp_me_cntl = RADEON_READ(R600_CP_ME_CNTL);
 581         RADEON_WRITE(R600_CP_ME_CNTL, R600_CP_ME_HALT);
 582
 583         RADEON_WRITE(R600_GRBM_SOFT_RESET, 0x7fff);
 584         RADEON_READ(R600_GRBM_SOFT_RESET);
 585         DRM_UDELAY(50);
 586         RADEON_WRITE(R600_GRBM_SOFT_RESET, 0);
 587         RADEON_READ(R600_GRBM_SOFT_RESET);
 588
 589         RADEON_WRITE(R600_CP_RB_WPTR_DELAY, 0);
 590         cp_rb_cntl = RADEON_READ(R600_CP_RB_CNTL);
 591         RADEON_WRITE(R600_CP_RB_CNTL, R600_RB_RPTR_WR_ENA);
 592
 593         RADEON_WRITE(R600_CP_RB_RPTR_WR, cp_ptr);
 594         RADEON_WRITE(R600_CP_RB_WPTR, cp_ptr);
 595         RADEON_WRITE(R600_CP_RB_CNTL, cp_rb_cntl);
 596         RADEON_WRITE(R600_CP_ME_CNTL, cp_me_cntl);
 597
 598         /* Reset the CP ring */
 599         r600_do_cp_reset(dev_priv);
 600
 601         /* The CP is no longer running after an engine reset */
 602         dev_priv->cp_running = 0;
 603
 604         /* Reset any pending vertex, indirect buffers */
 605         radeon_freelist_reset(dev);
 606
 607         return 0;
 608
 609 }
 610
 611 static u32 r600_get_tile_pipe_to_backend_map(u32 num_tile_pipes,
 612                                              u32 num_backends,
 613                                              u32 backend_disable_mask)
 614 {
 615         u32 backend_map = 0;
 616         u32 enabled_backends_mask;
 617         u32 enabled_backends_count;
 618         u32 cur_pipe;
 619         u32 swizzle_pipe[R6XX_MAX_PIPES];
 620         u32 cur_backend;
 621         u32 i;
 622
 623         if (num_tile_pipes > R6XX_MAX_PIPES)
 624                 num_tile_pipes = R6XX_MAX_PIPES;
 625         if (num_tile_pipes < 1)
 626                 num_tile_pipes = 1;
 627         if (num_backends > R6XX_MAX_BACKENDS)
 628                 num_backends = R6XX_MAX_BACKENDS;
 629         if (num_backends < 1)
 630                 num_backends = 1;
 631
 632         enabled_backends_mask = 0;
 633         enabled_backends_count = 0;
 634         for (i = 0; i < R6XX_MAX_BACKENDS; ++i) {
 635                 if (((backend_disable_mask >> i) & 1) == 0) {
 636                         enabled_backends_mask |= (1 << i);
 637                         ++enabled_backends_count;
 638                 }
 639                 if (enabled_backends_count == num_backends)
 640                         break;
 641         }
 642
 643         if (enabled_backends_count == 0) {
 644                 enabled_backends_mask = 1;
 645                 enabled_backends_count = 1;
 646         }
 647
 648         if (enabled_backends_count != num_backends)
 649                 num_backends = enabled_backends_count;
 650
 651         memset((uint8_t *)&swizzle_pipe[0], 0, sizeof(u32) * R6XX_MAX_PIPES);
 652         switch (num_tile_pipes) {
 653         case 1:
 654                 swizzle_pipe[0] = 0;
 655                 break;
 656         case 2:
 657                 swizzle_pipe[0] = 0;
 658                 swizzle_pipe[1] = 1;
 659                 break;
 660         case 3:
 661                 swizzle_pipe[0] = 0;
 662                 swizzle_pipe[1] = 1;
 663                 swizzle_pipe[2] = 2;
 664                 break;
 665         case 4:
 666                 swizzle_pipe[0] = 0;
 667                 swizzle_pipe[1] = 1;
 668                 swizzle_pipe[2] = 2;
 669                 swizzle_pipe[3] = 3;
 670                 break;
 671         case 5:
 672                 swizzle_pipe[0] = 0;
 673                 swizzle_pipe[1] = 1;
 674                 swizzle_pipe[2] = 2;
 675                 swizzle_pipe[3] = 3;
 676                 swizzle_pipe[4] = 4;
 677                 break;
 678         case 6:
 679                 swizzle_pipe[0] = 0;
 680                 swizzle_pipe[1] = 2;
 681                 swizzle_pipe[2] = 4;
 682                 swizzle_pipe[3] = 5;
 683                 swizzle_pipe[4] = 1;
 684                 swizzle_pipe[5] = 3;
 685                 break;
 686         case 7:
 687                 swizzle_pipe[0] = 0;
 688                 swizzle_pipe[1] = 2;
 689                 swizzle_pipe[2] = 4;
 690                 swizzle_pipe[3] = 6;
 691                 swizzle_pipe[4] = 1;
 692                 swizzle_pipe[5] = 3;
 693                 swizzle_pipe[6] = 5;
 694                 break;
 695         case 8:
 696                 swizzle_pipe[0] = 0;
 697                 swizzle_pipe[1] = 2;
 698                 swizzle_pipe[2] = 4;
 699                 swizzle_pipe[3] = 6;
 700                 swizzle_pipe[4] = 1;
 701                 swizzle_pipe[5] = 3;
 702                 swizzle_pipe[6] = 5;
 703                 swizzle_pipe[7] = 7;
 704                 break;
 705         }
 706
 707         cur_backend = 0;
 708         for (cur_pipe = 0; cur_pipe < num_tile_pipes; ++cur_pipe) {
 709                 while (((1 << cur_backend) & enabled_backends_mask) == 0)
 710                         cur_backend = (cur_backend + 1) % R6XX_MAX_BACKENDS;
 711
 712                 backend_map |= (u32)(((cur_backend & 3) << (swizzle_pipe[cur_pipe] * 2)));
 713
 714                 cur_backend = (cur_backend + 1) % R6XX_MAX_BACKENDS;
 715         }
 716
 717         return backend_map;
 718 }
 719
 720 static int r600_count_pipe_bits(uint32_t val)
 721 {
 722         int i, ret = 0;
 723         for (i = 0; i < 32; i++) {
 724                 ret += val & 1;
 725                 val >>= 1;
 726         }
 727         return ret;
 728 }
 729
 730 static void r600_gfx_init(struct drm_device *dev,
 731                           drm_radeon_private_t *dev_priv)
 732 {
 733         int i, j, num_qd_pipes;
 734         u32 sx_debug_1;
 735         u32 tc_cntl;
 736         u32 arb_pop;
 737         u32 num_gs_verts_per_thread;
 738         u32 vgt_gs_per_es;
 739         u32 gs_prim_buffer_depth = 0;
 740         u32 sq_ms_fifo_sizes;
 741         u32 sq_config;
 742         u32 sq_gpr_resource_mgmt_1 = 0;
 743         u32 sq_gpr_resource_mgmt_2 = 0;
 744         u32 sq_thread_resource_mgmt = 0;
 745         u32 sq_stack_resource_mgmt_1 = 0;
 746         u32 sq_stack_resource_mgmt_2 = 0;
 747         u32 hdp_host_path_cntl;
 748         u32 backend_map;
 749         u32 gb_tiling_config = 0;
 750         u32 cc_rb_backend_disable = 0;
 751         u32 cc_gc_shader_pipe_config = 0;
 752         u32 ramcfg;
 753
 754         /* setup chip specs */
 755         switch (dev_priv->flags & RADEON_FAMILY_MASK) {
 756         case CHIP_R600:
 757                 dev_priv->r600_max_pipes = 4;
 758                 dev_priv->r600_max_tile_pipes = 8;
 759                 dev_priv->r600_max_simds = 4;
 760                 dev_priv->r600_max_backends = 4;
 761                 dev_priv->r600_max_gprs = 256;
 762                 dev_priv->r600_max_threads = 192;
 763                 dev_priv->r600_max_stack_entries = 256;
 764                 dev_priv->r600_max_hw_contexts = 8;
 765                 dev_priv->r600_max_gs_threads = 16;
 766                 dev_priv->r600_sx_max_export_size = 128;
 767                 dev_priv->r600_sx_max_export_pos_size = 16;
 768                 dev_priv->r600_sx_max_export_smx_size = 128;
 769                 dev_priv->r600_sq_num_cf_insts = 2;
 770                 break;
 771         case CHIP_RV630:
 772         case CHIP_RV635:
 773                 dev_priv->r600_max_pipes = 2;
 774                 dev_priv->r600_max_tile_pipes = 2;
 775                 dev_priv->r600_max_simds = 3;
 776                 dev_priv->r600_max_backends = 1;
 777                 dev_priv->r600_max_gprs = 128;
 778                 dev_priv->r600_max_threads = 192;
 779                 dev_priv->r600_max_stack_entries = 128;
 780                 dev_priv->r600_max_hw_contexts = 8;
 781                 dev_priv->r600_max_gs_threads = 4;
 782                 dev_priv->r600_sx_max_export_size = 128;
 783                 dev_priv->r600_sx_max_export_pos_size = 16;
 784                 dev_priv->r600_sx_max_export_smx_size = 128;
 785                 dev_priv->r600_sq_num_cf_insts = 2;
 786                 break;
 787         case CHIP_RV610:
 788         case CHIP_RS780:
 789         case CHIP_RV620:
 790                 dev_priv->r600_max_pipes = 1;
 791                 dev_priv->r600_max_tile_pipes = 1;
 792                 dev_priv->r600_max_simds = 2;
 793                 dev_priv->r600_max_backends = 1;
 794                 dev_priv->r600_max_gprs = 128;
 795                 dev_priv->r600_max_threads = 192;
 796                 dev_priv->r600_max_stack_entries = 128;
 797                 dev_priv->r600_max_hw_contexts = 4;
 798                 dev_priv->r600_max_gs_threads = 4;
 799                 dev_priv->r600_sx_max_export_size = 128;
 800                 dev_priv->r600_sx_max_export_pos_size = 16;
 801                 dev_priv->r600_sx_max_export_smx_size = 128;
 802                 dev_priv->r600_sq_num_cf_insts = 1;
 803                 break;
 804         case CHIP_RV670:
 805                 dev_priv->r600_max_pipes = 4;
 806                 dev_priv->r600_max_tile_pipes = 4;
 807                 dev_priv->r600_max_simds = 4;
 808                 dev_priv->r600_max_backends = 4;
 809                 dev_priv->r600_max_gprs = 192;
 810                 dev_priv->r600_max_threads = 192;
 811                 dev_priv->r600_max_stack_entries = 256;
 812                 dev_priv->r600_max_hw_contexts = 8;
 813                 dev_priv->r600_max_gs_threads = 16;
 814                 dev_priv->r600_sx_max_export_size = 128;
 815                 dev_priv->r600_sx_max_export_pos_size = 16;
 816                 dev_priv->r600_sx_max_export_smx_size = 128;
 817                 dev_priv->r600_sq_num_cf_insts = 2;
 818                 break;
 819         default:
 820                 break;
 821         }
 822
 823         /* Initialize HDP */
 824         j = 0;
 825         for (i = 0; i < 32; i++) {
 826                 RADEON_WRITE((0x2c14 + j), 0x00000000);
 827                 RADEON_WRITE((0x2c18 + j), 0x00000000);
 828                 RADEON_WRITE((0x2c1c + j), 0x00000000);
 829                 RADEON_WRITE((0x2c20 + j), 0x00000000);
 830                 RADEON_WRITE((0x2c24 + j), 0x00000000);
 831                 j += 0x18;
 832         }
 833
 834         RADEON_WRITE(R600_GRBM_CNTL, R600_GRBM_READ_TIMEOUT(0xff));
 835
 836         /* setup tiling, simd, pipe config */
 837         ramcfg = RADEON_READ(R600_RAMCFG);
 838
 839         switch (dev_priv->r600_max_tile_pipes) {
 840         case 1:
 841                 gb_tiling_config |= R600_PIPE_TILING(0);
 842                 break;
 843         case 2:
 844                 gb_tiling_config |= R600_PIPE_TILING(1);
 845                 break;
 846         case 4:
 847                 gb_tiling_config |= R600_PIPE_TILING(2);
 848                 break;
 849         case 8:
 850                 gb_tiling_config |= R600_PIPE_TILING(3);
 851                 break;
 852         default:
 853                 break;
 854         }
 855
 856         gb_tiling_config |= R600_BANK_TILING((ramcfg >> R600_NOOFBANK_SHIFT) & R600_NOOFBANK_MASK);
 857
 858         gb_tiling_config |= R600_GROUP_SIZE(0);
 859
 860         if (((ramcfg >> R600_NOOFROWS_SHIFT) & R600_NOOFROWS_MASK) > 3) {
 861                 gb_tiling_config |= R600_ROW_TILING(3);
 862                 gb_tiling_config |= R600_SAMPLE_SPLIT(3);
 863         } else {
 864                 gb_tiling_config |=
 865                         R600_ROW_TILING(((ramcfg >> R600_NOOFROWS_SHIFT) & R600_NOOFROWS_MASK));
 866                 gb_tiling_config |=
 867                         R600_SAMPLE_SPLIT(((ramcfg >> R600_NOOFROWS_SHIFT) & R600_NOOFROWS_MASK));
 868         }
 869
 870         gb_tiling_config |= R600_BANK_SWAPS(1);
 871
 872         backend_map = r600_get_tile_pipe_to_backend_map(dev_priv->r600_max_tile_pipes,
 873                                                         dev_priv->r600_max_backends,
 874                                                         (0xff << dev_priv->r600_max_backends) & 0xff);
 875         gb_tiling_config |= R600_BACKEND_MAP(backend_map);
 876
 877         cc_gc_shader_pipe_config =
 878                 R600_INACTIVE_QD_PIPES((R6XX_MAX_PIPES_MASK << dev_priv->r600_max_pipes) & R6XX_MAX_PIPES_MASK);
 879         cc_gc_shader_pipe_config |=
 880                 R600_INACTIVE_SIMDS((R6XX_MAX_SIMDS_MASK << dev_priv->r600_max_simds) & R6XX_MAX_SIMDS_MASK);
 881
 882         cc_rb_backend_disable =
 883                 R600_BACKEND_DISABLE((R6XX_MAX_BACKENDS_MASK << dev_priv->r600_max_backends) & R6XX_MAX_BACKENDS_MASK);
 884
 885         RADEON_WRITE(R600_GB_TILING_CONFIG,      gb_tiling_config);
 886         RADEON_WRITE(R600_DCP_TILING_CONFIG,    (gb_tiling_config & 0xffff));
 887         RADEON_WRITE(R600_HDP_TILING_CONFIG,    (gb_tiling_config & 0xffff));
 888
 889         RADEON_WRITE(R600_CC_RB_BACKEND_DISABLE,      cc_rb_backend_disable);
 890         RADEON_WRITE(R600_CC_GC_SHADER_PIPE_CONFIG,   cc_gc_shader_pipe_config);
 891         RADEON_WRITE(R600_GC_USER_SHADER_PIPE_CONFIG, cc_gc_shader_pipe_config);
 892
 893         num_qd_pipes =
 894                 R6XX_MAX_BACKENDS - r600_count_pipe_bits(cc_gc_shader_pipe_config & R600_INACTIVE_QD_PIPES_MASK);
 895         RADEON_WRITE(R600_VGT_OUT_DEALLOC_CNTL, (num_qd_pipes * 4) & R600_DEALLOC_DIST_MASK);
 896         RADEON_WRITE(R600_VGT_VERTEX_REUSE_BLOCK_CNTL, ((num_qd_pipes * 4) - 2) & R600_VTX_REUSE_DEPTH_MASK);
 897
 898         /* set HW defaults for 3D engine */
 899         RADEON_WRITE(R600_CP_QUEUE_THRESHOLDS, (R600_ROQ_IB1_START(0x16) |
 900                                                 R600_ROQ_IB2_START(0x2b)));
 901
 902         RADEON_WRITE(R600_CP_MEQ_THRESHOLDS, (R600_MEQ_END(0x40) |
 903                                               R600_ROQ_END(0x40)));
 904
 905         RADEON_WRITE(R600_TA_CNTL_AUX, (R600_DISABLE_CUBE_ANISO |
 906                                         R600_SYNC_GRADIENT |
 907                                         R600_SYNC_WALKER |
 908                                         R600_SYNC_ALIGNER));
 909
 910         if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV670)
 911                 RADEON_WRITE(R600_ARB_GDEC_RD_CNTL, 0x00000021);
 912
 913         sx_debug_1 = RADEON_READ(R600_SX_DEBUG_1);
 914         sx_debug_1 |= R600_SMX_EVENT_RELEASE;
 915         if (((dev_priv->flags & RADEON_FAMILY_MASK) > CHIP_R600))
 916                 sx_debug_1 |= R600_ENABLE_NEW_SMX_ADDRESS;
 917         RADEON_WRITE(R600_SX_DEBUG_1, sx_debug_1);
 918
 919         if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R600) ||
 920             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV630) ||
 921             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV610) ||
 922             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV620) ||
 923             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS780))
 924                 RADEON_WRITE(R600_DB_DEBUG, R600_PREZ_MUST_WAIT_FOR_POSTZ_DONE);
 925         else
 926                 RADEON_WRITE(R600_DB_DEBUG, 0);
 927
 928         RADEON_WRITE(R600_DB_WATERMARKS, (R600_DEPTH_FREE(4) |
 929                                           R600_DEPTH_FLUSH(16) |
 930                                           R600_DEPTH_PENDING_FREE(4) |
 931                                           R600_DEPTH_CACHELINE_FREE(16)));
 932         RADEON_WRITE(R600_PA_SC_MULTI_CHIP_CNTL, 0);
 933         RADEON_WRITE(R600_VGT_NUM_INSTANCES, 0);
 934
 935         RADEON_WRITE(R600_SPI_CONFIG_CNTL, R600_GPR_WRITE_PRIORITY(0));
 936         RADEON_WRITE(R600_SPI_CONFIG_CNTL_1, R600_VTX_DONE_DELAY(0));
 937
 938         sq_ms_fifo_sizes = RADEON_READ(R600_SQ_MS_FIFO_SIZES);
 939         if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV610) ||
 940             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV620) ||
 941             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS780)) {
 942                 sq_ms_fifo_sizes = (R600_CACHE_FIFO_SIZE(0xa) |
 943                                     R600_FETCH_FIFO_HIWATER(0xa) |
 944                                     R600_DONE_FIFO_HIWATER(0xe0) |
 945                                     R600_ALU_UPDATE_FIFO_HIWATER(0x8));
 946         } else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R600) ||
 947                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV630)) {
 948                 sq_ms_fifo_sizes &= ~R600_DONE_FIFO_HIWATER(0xff);
 949                 sq_ms_fifo_sizes |= R600_DONE_FIFO_HIWATER(0x4);
 950         }
 951         RADEON_WRITE(R600_SQ_MS_FIFO_SIZES, sq_ms_fifo_sizes);
 952
 953         /* SQ_CONFIG, SQ_GPR_RESOURCE_MGMT, SQ_THREAD_RESOURCE_MGMT, SQ_STACK_RESOURCE_MGMT
 954          * should be adjusted as needed by the 2D/3D drivers.  This just sets default values
 955          */
 956         sq_config = RADEON_READ(R600_SQ_CONFIG);
 957         sq_config &= ~(R600_PS_PRIO(3) |
 958                        R600_VS_PRIO(3) |
 959                        R600_GS_PRIO(3) |
 960                        R600_ES_PRIO(3));
 961         sq_config |= (R600_DX9_CONSTS |
 962                       R600_VC_ENABLE |
 963                       R600_PS_PRIO(0) |
 964                       R600_VS_PRIO(1) |
 965                       R600_GS_PRIO(2) |
 966                       R600_ES_PRIO(3));
 967
 968         if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R600) {
 969                 sq_gpr_resource_mgmt_1 = (R600_NUM_PS_GPRS(124) |
 970                                           R600_NUM_VS_GPRS(124) |
 971                                           R600_NUM_CLAUSE_TEMP_GPRS(4));
 972                 sq_gpr_resource_mgmt_2 = (R600_NUM_GS_GPRS(0) |
 973                                           R600_NUM_ES_GPRS(0));
 974                 sq_thread_resource_mgmt = (R600_NUM_PS_THREADS(136) |
 975                                            R600_NUM_VS_THREADS(48) |
 976                                            R600_NUM_GS_THREADS(4) |
 977                                            R600_NUM_ES_THREADS(4));
 978                 sq_stack_resource_mgmt_1 = (R600_NUM_PS_STACK_ENTRIES(128) |
 979                                             R600_NUM_VS_STACK_ENTRIES(128));
 980                 sq_stack_resource_mgmt_2 = (R600_NUM_GS_STACK_ENTRIES(0) |
 981                                             R600_NUM_ES_STACK_ENTRIES(0));
 982         } else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV610) ||
 983                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV620) ||
 984                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS780)) {
 985                 /* no vertex cache */
 986                 sq_config &= ~R600_VC_ENABLE;
 987
 988                 sq_gpr_resource_mgmt_1 = (R600_NUM_PS_GPRS(44) |
 989                                           R600_NUM_VS_GPRS(44) |
 990                                           R600_NUM_CLAUSE_TEMP_GPRS(2));
 991                 sq_gpr_resource_mgmt_2 = (R600_NUM_GS_GPRS(17) |
 992                                           R600_NUM_ES_GPRS(17));
 993                 sq_thread_resource_mgmt = (R600_NUM_PS_THREADS(79) |
 994                                            R600_NUM_VS_THREADS(78) |
 995                                            R600_NUM_GS_THREADS(4) |
 996                                            R600_NUM_ES_THREADS(31));
 997                 sq_stack_resource_mgmt_1 = (R600_NUM_PS_STACK_ENTRIES(40) |
 998                                             R600_NUM_VS_STACK_ENTRIES(40));
 999                 sq_stack_resource_mgmt_2 = (R600_NUM_GS_STACK_ENTRIES(32) |
1000                                             R600_NUM_ES_STACK_ENTRIES(16));
1001         } else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV630) ||
1002                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV635)) {
1003                 sq_gpr_resource_mgmt_1 = (R600_NUM_PS_GPRS(44) |
1004                                           R600_NUM_VS_GPRS(44) |
1005                                           R600_NUM_CLAUSE_TEMP_GPRS(2));
1006                 sq_gpr_resource_mgmt_2 = (R600_NUM_GS_GPRS(18) |
1007                                           R600_NUM_ES_GPRS(18));
1008                 sq_thread_resource_mgmt = (R600_NUM_PS_THREADS(79) |
1009                                            R600_NUM_VS_THREADS(78) |
1010                                            R600_NUM_GS_THREADS(4) |
1011                                            R600_NUM_ES_THREADS(31));
1012                 sq_stack_resource_mgmt_1 = (R600_NUM_PS_STACK_ENTRIES(40) |
1013                                             R600_NUM_VS_STACK_ENTRIES(40));
1014                 sq_stack_resource_mgmt_2 = (R600_NUM_GS_STACK_ENTRIES(32) |
1015                                             R600_NUM_ES_STACK_ENTRIES(16));
1016         } else if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV670) {
1017                 sq_gpr_resource_mgmt_1 = (R600_NUM_PS_GPRS(44) |
1018                                           R600_NUM_VS_GPRS(44) |
1019                                           R600_NUM_CLAUSE_TEMP_GPRS(2));
1020                 sq_gpr_resource_mgmt_2 = (R600_NUM_GS_GPRS(17) |
1021                                           R600_NUM_ES_GPRS(17));
1022                 sq_thread_resource_mgmt = (R600_NUM_PS_THREADS(79) |
1023                                            R600_NUM_VS_THREADS(78) |
1024                                            R600_NUM_GS_THREADS(4) |
1025                                            R600_NUM_ES_THREADS(31));
1026                 sq_stack_resource_mgmt_1 = (R600_NUM_PS_STACK_ENTRIES(64) |
1027                                             R600_NUM_VS_STACK_ENTRIES(64));
1028                 sq_stack_resource_mgmt_2 = (R600_NUM_GS_STACK_ENTRIES(64) |
1029                                             R600_NUM_ES_STACK_ENTRIES(64));
1030         }
1031
1032         RADEON_WRITE(R600_SQ_CONFIG, sq_config);
1033         RADEON_WRITE(R600_SQ_GPR_RESOURCE_MGMT_1,  sq_gpr_resource_mgmt_1);
1034         RADEON_WRITE(R600_SQ_GPR_RESOURCE_MGMT_2,  sq_gpr_resource_mgmt_2);
1035         RADEON_WRITE(R600_SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
1036         RADEON_WRITE(R600_SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
1037         RADEON_WRITE(R600_SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
1038
1039         if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV610) ||
1040             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV620) ||
1041             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS780))
1042                 RADEON_WRITE(R600_VGT_CACHE_INVALIDATION, R600_CACHE_INVALIDATION(R600_TC_ONLY));
1043         else
1044                 RADEON_WRITE(R600_VGT_CACHE_INVALIDATION, R600_CACHE_INVALIDATION(R600_VC_AND_TC));
1045
1046         RADEON_WRITE(R600_PA_SC_AA_SAMPLE_LOCS_2S, (R600_S0_X(0xc) |
1047                                                     R600_S0_Y(0x4) |
1048                                                     R600_S1_X(0x4) |
1049                                                     R600_S1_Y(0xc)));
1050         RADEON_WRITE(R600_PA_SC_AA_SAMPLE_LOCS_4S, (R600_S0_X(0xe) |
1051                                                     R600_S0_Y(0xe) |
1052                                                     R600_S1_X(0x2) |
1053                                                     R600_S1_Y(0x2) |
1054                                                     R600_S2_X(0xa) |
1055                                                     R600_S2_Y(0x6) |
1056                                                     R600_S3_X(0x6) |
1057                                                     R600_S3_Y(0xa)));
1058         RADEON_WRITE(R600_PA_SC_AA_SAMPLE_LOCS_8S_WD0, (R600_S0_X(0xe) |
1059                                                         R600_S0_Y(0xb) |
1060                                                         R600_S1_X(0x4) |
1061                                                         R600_S1_Y(0xc) |
1062                                                         R600_S2_X(0x1) |
1063                                                         R600_S2_Y(0x6) |
1064                                                         R600_S3_X(0xa) |
1065                                                         R600_S3_Y(0xe)));
1066         RADEON_WRITE(R600_PA_SC_AA_SAMPLE_LOCS_8S_WD1, (R600_S4_X(0x6) |
1067                                                         R600_S4_Y(0x1) |
1068                                                         R600_S5_X(0x0) |
1069                                                         R600_S5_Y(0x0) |
1070                                                         R600_S6_X(0xb) |
1071                                                         R600_S6_Y(0x4) |
1072                                                         R600_S7_X(0x7) |
1073                                                         R600_S7_Y(0x8)));
1074
1075
1076         switch (dev_priv->flags & RADEON_FAMILY_MASK) {
1077         case CHIP_R600:
1078         case CHIP_RV630:
1079         case CHIP_RV635:
1080                 gs_prim_buffer_depth = 0;
1081                 break;
1082         case CHIP_RV610:
1083         case CHIP_RS780:
1084         case CHIP_RV620:
1085                 gs_prim_buffer_depth = 32;
1086                 break;
1087         case CHIP_RV670:
1088                 gs_prim_buffer_depth = 128;
1089                 break;
1090         default:
1091                 break;
1092         }
1093
1094         num_gs_verts_per_thread = dev_priv->r600_max_pipes * 16;
1095         vgt_gs_per_es = gs_prim_buffer_depth + num_gs_verts_per_thread;
1096         /* Max value for this is 256 */
1097         if (vgt_gs_per_es > 256)
1098                 vgt_gs_per_es = 256;
1099
1100         RADEON_WRITE(R600_VGT_ES_PER_GS, 128);
1101         RADEON_WRITE(R600_VGT_GS_PER_ES, vgt_gs_per_es);
1102         RADEON_WRITE(R600_VGT_GS_PER_VS, 2);
1103         RADEON_WRITE(R600_VGT_GS_VERTEX_REUSE, 16);
1104
1105         /* more default values. 2D/3D driver should adjust as needed */
1106         RADEON_WRITE(R600_PA_SC_LINE_STIPPLE_STATE, 0);
1107         RADEON_WRITE(R600_VGT_STRMOUT_EN, 0);
1108         RADEON_WRITE(R600_SX_MISC, 0);
1109         RADEON_WRITE(R600_PA_SC_MODE_CNTL, 0);
1110         RADEON_WRITE(R600_PA_SC_AA_CONFIG, 0);
1111         RADEON_WRITE(R600_PA_SC_LINE_STIPPLE, 0);
1112         RADEON_WRITE(R600_SPI_INPUT_Z, 0);
1113         RADEON_WRITE(R600_SPI_PS_IN_CONTROL_0, R600_NUM_INTERP(2));
1114         RADEON_WRITE(R600_CB_COLOR7_FRAG, 0);
1115
1116         /* clear render buffer base addresses */
1117         RADEON_WRITE(R600_CB_COLOR0_BASE, 0);
1118         RADEON_WRITE(R600_CB_COLOR1_BASE, 0);
1119         RADEON_WRITE(R600_CB_COLOR2_BASE, 0);
1120         RADEON_WRITE(R600_CB_COLOR3_BASE, 0);
1121         RADEON_WRITE(R600_CB_COLOR4_BASE, 0);
1122         RADEON_WRITE(R600_CB_COLOR5_BASE, 0);
1123         RADEON_WRITE(R600_CB_COLOR6_BASE, 0);
1124         RADEON_WRITE(R600_CB_COLOR7_BASE, 0);
1125
1126         switch (dev_priv->flags & RADEON_FAMILY_MASK) {
1127         case CHIP_RV610:
1128         case CHIP_RS780:
1129         case CHIP_RV620:
1130                 tc_cntl = R600_TC_L2_SIZE(8);
1131                 break;
1132         case CHIP_RV630:
1133         case CHIP_RV635:
1134                 tc_cntl = R600_TC_L2_SIZE(4);
1135                 break;
1136         case CHIP_R600:
1137                 tc_cntl = R600_TC_L2_SIZE(0) | R600_L2_DISABLE_LATE_HIT;
1138                 break;
1139         default:
1140                 tc_cntl = R600_TC_L2_SIZE(0);
1141                 break;
1142         }
1143
1144         RADEON_WRITE(R600_TC_CNTL, tc_cntl);
1145
1146         hdp_host_path_cntl = RADEON_READ(R600_HDP_HOST_PATH_CNTL);
1147         RADEON_WRITE(R600_HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
1148
1149         arb_pop = RADEON_READ(R600_ARB_POP);
1150         arb_pop |= R600_ENABLE_TC128;
1151         RADEON_WRITE(R600_ARB_POP, arb_pop);
1152
1153         RADEON_WRITE(R600_PA_SC_MULTI_CHIP_CNTL, 0);
1154         RADEON_WRITE(R600_PA_CL_ENHANCE, (R600_CLIP_VTX_REORDER_ENA |
1155                                           R600_NUM_CLIP_SEQ(3)));
1156         RADEON_WRITE(R600_PA_SC_ENHANCE, R600_FORCE_EOV_MAX_CLK_CNT(4095));
1157
1158 }
1159
1160 static u32 r700_get_tile_pipe_to_backend_map(u32 num_tile_pipes,
1161                                              u32 num_backends,
1162                                              u32 backend_disable_mask)
1163 {
1164         u32 backend_map = 0;
1165         u32 enabled_backends_mask;
1166         u32 enabled_backends_count;
1167         u32 cur_pipe;
1168         u32 swizzle_pipe[R7XX_MAX_PIPES];
1169         u32 cur_backend;
1170         u32 i;
1171
1172         if (num_tile_pipes > R7XX_MAX_PIPES)
1173                 num_tile_pipes = R7XX_MAX_PIPES;
1174         if (num_tile_pipes < 1)
1175                 num_tile_pipes = 1;
1176         if (num_backends > R7XX_MAX_BACKENDS)
1177                 num_backends = R7XX_MAX_BACKENDS;
1178         if (num_backends < 1)
1179                 num_backends = 1;
1180
1181         enabled_backends_mask = 0;
1182         enabled_backends_count = 0;
1183         for (i = 0; i < R7XX_MAX_BACKENDS; ++i) {
1184                 if (((backend_disable_mask >> i) & 1) == 0) {
1185                         enabled_backends_mask |= (1 << i);
1186                         ++enabled_backends_count;
1187                 }
1188                 if (enabled_backends_count == num_backends)
1189                         break;
1190         }
1191
1192         if (enabled_backends_count == 0) {
1193                 enabled_backends_mask = 1;
1194                 enabled_backends_count = 1;
1195         }
1196
1197         if (enabled_backends_count != num_backends)
1198                 num_backends = enabled_backends_count;
1199
1200         memset((uint8_t *)&swizzle_pipe[0], 0, sizeof(u32) * R7XX_MAX_PIPES);
1201         switch (num_tile_pipes) {
1202         case 1:
1203                 swizzle_pipe[0] = 0;
1204                 break;
1205         case 2:
1206                 swizzle_pipe[0] = 0;
1207                 swizzle_pipe[1] = 1;
1208                 break;
1209         case 3:
1210                 swizzle_pipe[0] = 0;
1211                 swizzle_pipe[1] = 2;
1212                 swizzle_pipe[2] = 1;
1213                 break;
1214         case 4:
1215                 swizzle_pipe[0] = 0;
1216                 swizzle_pipe[1] = 2;
1217                 swizzle_pipe[2] = 3;
1218                 swizzle_pipe[3] = 1;
1219                 break;
1220         case 5:
1221                 swizzle_pipe[0] = 0;
1222                 swizzle_pipe[1] = 2;
1223                 swizzle_pipe[2] = 4;
1224                 swizzle_pipe[3] = 1;
1225                 swizzle_pipe[4] = 3;
1226                 break;
1227         case 6:
1228                 swizzle_pipe[0] = 0;
1229                 swizzle_pipe[1] = 2;
1230                 swizzle_pipe[2] = 4;
1231                 swizzle_pipe[3] = 5;
1232                 swizzle_pipe[4] = 3;
1233                 swizzle_pipe[5] = 1;
1234                 break;
1235         case 7:
1236                 swizzle_pipe[0] = 0;
1237                 swizzle_pipe[1] = 2;
1238                 swizzle_pipe[2] = 4;
1239                 swizzle_pipe[3] = 6;
1240                 swizzle_pipe[4] = 3;
1241                 swizzle_pipe[5] = 1;
1242                 swizzle_pipe[6] = 5;
1243                 break;
1244         case 8:
1245                 swizzle_pipe[0] = 0;
1246                 swizzle_pipe[1] = 2;
1247                 swizzle_pipe[2] = 4;
1248                 swizzle_pipe[3] = 6;
1249                 swizzle_pipe[4] = 3;
1250                 swizzle_pipe[5] = 1;
1251                 swizzle_pipe[6] = 7;
1252                 swizzle_pipe[7] = 5;
1253                 break;
1254         }
1255
1256         cur_backend = 0;
1257         for (cur_pipe = 0; cur_pipe < num_tile_pipes; ++cur_pipe) {
1258                 while (((1 << cur_backend) & enabled_backends_mask) == 0)
1259                         cur_backend = (cur_backend + 1) % R7XX_MAX_BACKENDS;
1260
1261                 backend_map |= (u32)(((cur_backend & 3) << (swizzle_pipe[cur_pipe] * 2)));
1262
1263                 cur_backend = (cur_backend + 1) % R7XX_MAX_BACKENDS;
1264         }
1265
1266         return backend_map;
1267 }
1268
1269 static void r700_gfx_init(struct drm_device *dev,
1270                           drm_radeon_private_t *dev_priv)
1271 {
1272         int i, j, num_qd_pipes;
1273         u32 sx_debug_1;
1274         u32 smx_dc_ctl0;
1275         u32 num_gs_verts_per_thread;
1276         u32 vgt_gs_per_es;
1277         u32 gs_prim_buffer_depth = 0;
1278         u32 sq_ms_fifo_sizes;
1279         u32 sq_config;
1280         u32 sq_thread_resource_mgmt;
1281         u32 hdp_host_path_cntl;
1282         u32 sq_dyn_gpr_size_simd_ab_0;
1283         u32 backend_map;
1284         u32 gb_tiling_config = 0;
1285         u32 cc_rb_backend_disable = 0;
1286         u32 cc_gc_shader_pipe_config = 0;
1287         u32 mc_arb_ramcfg;
1288         u32 db_debug4;
1289
1290         /* setup chip specs */
1291         switch (dev_priv->flags & RADEON_FAMILY_MASK) {
1292         case CHIP_RV770:
1293                 dev_priv->r600_max_pipes = 4;
1294                 dev_priv->r600_max_tile_pipes = 8;
1295                 dev_priv->r600_max_simds = 10;
1296                 dev_priv->r600_max_backends = 4;
1297                 dev_priv->r600_max_gprs = 256;
1298                 dev_priv->r600_max_threads = 248;
1299                 dev_priv->r600_max_stack_entries = 512;
1300                 dev_priv->r600_max_hw_contexts = 8;
1301                 dev_priv->r600_max_gs_threads = 16 * 2;
1302                 dev_priv->r600_sx_max_export_size = 128;
1303                 dev_priv->r600_sx_max_export_pos_size = 16;
1304                 dev_priv->r600_sx_max_export_smx_size = 112;
1305                 dev_priv->r600_sq_num_cf_insts = 2;
1306
1307                 dev_priv->r700_sx_num_of_sets = 7;
1308                 dev_priv->r700_sc_prim_fifo_size = 0xF9;
1309                 dev_priv->r700_sc_hiz_tile_fifo_size = 0x30;
1310                 dev_priv->r700_sc_earlyz_tile_fifo_fize = 0x130;
1311                 break;
1312         case CHIP_RV730:
1313                 dev_priv->r600_max_pipes = 2;
1314                 dev_priv->r600_max_tile_pipes = 4;
1315                 dev_priv->r600_max_simds = 8;
1316                 dev_priv->r600_max_backends = 2;
1317                 dev_priv->r600_max_gprs = 128;
1318                 dev_priv->r600_max_threads = 248;
1319                 dev_priv->r600_max_stack_entries = 256;
1320                 dev_priv->r600_max_hw_contexts = 8;
1321                 dev_priv->r600_max_gs_threads = 16 * 2;
1322                 dev_priv->r600_sx_max_export_size = 256;
1323                 dev_priv->r600_sx_max_export_pos_size = 32;
1324                 dev_priv->r600_sx_max_export_smx_size = 224;
1325                 dev_priv->r600_sq_num_cf_insts = 2;
1326
1327                 dev_priv->r700_sx_num_of_sets = 7;
1328                 dev_priv->r700_sc_prim_fifo_size = 0xf9;
1329                 dev_priv->r700_sc_hiz_tile_fifo_size = 0x30;
1330                 dev_priv->r700_sc_earlyz_tile_fifo_fize = 0x130;
1331                 break;
1332         case CHIP_RV710:
1333                 dev_priv->r600_max_pipes = 2;
1334                 dev_priv->r600_max_tile_pipes = 2;
1335                 dev_priv->r600_max_simds = 2;
1336                 dev_priv->r600_max_backends = 1;
1337                 dev_priv->r600_max_gprs = 256;
1338                 dev_priv->r600_max_threads = 192;
1339                 dev_priv->r600_max_stack_entries = 256;
1340                 dev_priv->r600_max_hw_contexts = 4;
1341                 dev_priv->r600_max_gs_threads = 8 * 2;
1342                 dev_priv->r600_sx_max_export_size = 128;
1343                 dev_priv->r600_sx_max_export_pos_size = 16;
1344                 dev_priv->r600_sx_max_export_smx_size = 112;
1345                 dev_priv->r600_sq_num_cf_insts = 1;
1346
1347                 dev_priv->r700_sx_num_of_sets = 7;
1348                 dev_priv->r700_sc_prim_fifo_size = 0x40;
1349                 dev_priv->r700_sc_hiz_tile_fifo_size = 0x30;
1350                 dev_priv->r700_sc_earlyz_tile_fifo_fize = 0x130;
1351                 break;
1352         default:
1353                 break;
1354         }
1355
1356         /* Initialize HDP */
1357         j = 0;
1358         for (i = 0; i < 32; i++) {
1359                 RADEON_WRITE((0x2c14 + j), 0x00000000);
1360                 RADEON_WRITE((0x2c18 + j), 0x00000000);
1361                 RADEON_WRITE((0x2c1c + j), 0x00000000);
1362                 RADEON_WRITE((0x2c20 + j), 0x00000000);
1363                 RADEON_WRITE((0x2c24 + j), 0x00000000);
1364                 j += 0x18;
1365         }
1366
1367         RADEON_WRITE(R600_GRBM_CNTL, R600_GRBM_READ_TIMEOUT(0xff));
1368
1369         /* setup tiling, simd, pipe config */
1370         mc_arb_ramcfg = RADEON_READ(R700_MC_ARB_RAMCFG);
1371
1372         switch (dev_priv->r600_max_tile_pipes) {
1373         case 1:
1374                 gb_tiling_config |= R600_PIPE_TILING(0);
1375                 break;
1376         case 2:
1377                 gb_tiling_config |= R600_PIPE_TILING(1);
1378                 break;
1379         case 4:
1380                 gb_tiling_config |= R600_PIPE_TILING(2);
1381                 break;
1382         case 8:
1383                 gb_tiling_config |= R600_PIPE_TILING(3);
1384                 break;
1385         default:
1386                 break;
1387         }
1388
1389         if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV770)
1390                 gb_tiling_config |= R600_BANK_TILING(1);
1391         else
1392                 gb_tiling_config |= R600_BANK_TILING((mc_arb_ramcfg >> R700_NOOFBANK_SHIFT) & R700_NOOFBANK_MASK);
1393
1394         gb_tiling_config |= R600_GROUP_SIZE(0);
1395
1396         if (((mc_arb_ramcfg >> R700_NOOFROWS_SHIFT) & R700_NOOFROWS_MASK) > 3) {
1397                 gb_tiling_config |= R600_ROW_TILING(3);
1398                 gb_tiling_config |= R600_SAMPLE_SPLIT(3);
1399         } else {
1400                 gb_tiling_config |=
1401                         R600_ROW_TILING(((mc_arb_ramcfg >> R700_NOOFROWS_SHIFT) & R700_NOOFROWS_MASK));
1402                 gb_tiling_config |=
1403                         R600_SAMPLE_SPLIT(((mc_arb_ramcfg >> R700_NOOFROWS_SHIFT) & R700_NOOFROWS_MASK));
1404         }
1405
1406         gb_tiling_config |= R600_BANK_SWAPS(1);
1407
1408         backend_map = r700_get_tile_pipe_to_backend_map(dev_priv->r600_max_tile_pipes,
1409                                                         dev_priv->r600_max_backends,
1410                                                         (0xff << dev_priv->r600_max_backends) & 0xff);
1411         gb_tiling_config |= R600_BACKEND_MAP(backend_map);
1412
1413         cc_gc_shader_pipe_config =
1414                 R600_INACTIVE_QD_PIPES((R7XX_MAX_PIPES_MASK << dev_priv->r600_max_pipes) & R7XX_MAX_PIPES_MASK);
1415         cc_gc_shader_pipe_config |=
1416                 R600_INACTIVE_SIMDS((R7XX_MAX_SIMDS_MASK << dev_priv->r600_max_simds) & R7XX_MAX_SIMDS_MASK);
1417
1418         cc_rb_backend_disable =
1419                 R600_BACKEND_DISABLE((R7XX_MAX_BACKENDS_MASK << dev_priv->r600_max_backends) & R7XX_MAX_BACKENDS_MASK);
1420
1421         RADEON_WRITE(R600_GB_TILING_CONFIG,      gb_tiling_config);
1422         RADEON_WRITE(R600_DCP_TILING_CONFIG,    (gb_tiling_config & 0xffff));
1423         RADEON_WRITE(R600_HDP_TILING_CONFIG,    (gb_tiling_config & 0xffff));
1424
1425         RADEON_WRITE(R600_CC_RB_BACKEND_DISABLE,      cc_rb_backend_disable);
1426         RADEON_WRITE(R600_CC_GC_SHADER_PIPE_CONFIG,   cc_gc_shader_pipe_config);
1427         RADEON_WRITE(R600_GC_USER_SHADER_PIPE_CONFIG, cc_gc_shader_pipe_config);
1428
1429         RADEON_WRITE(R700_CC_SYS_RB_BACKEND_DISABLE, cc_rb_backend_disable);
1430         RADEON_WRITE(R700_CGTS_SYS_TCC_DISABLE, 0);
1431         RADEON_WRITE(R700_CGTS_TCC_DISABLE, 0);
1432         RADEON_WRITE(R700_CGTS_USER_SYS_TCC_DISABLE, 0);
1433         RADEON_WRITE(R700_CGTS_USER_TCC_DISABLE, 0);
1434
1435         num_qd_pipes =
1436                 R7XX_MAX_BACKENDS - r600_count_pipe_bits(cc_gc_shader_pipe_config & R600_INACTIVE_QD_PIPES_MASK);
1437         RADEON_WRITE(R600_VGT_OUT_DEALLOC_CNTL, (num_qd_pipes * 4) & R600_DEALLOC_DIST_MASK);
1438         RADEON_WRITE(R600_VGT_VERTEX_REUSE_BLOCK_CNTL, ((num_qd_pipes * 4) - 2) & R600_VTX_REUSE_DEPTH_MASK);
1439
1440         /* set HW defaults for 3D engine */
1441         RADEON_WRITE(R600_CP_QUEUE_THRESHOLDS, (R600_ROQ_IB1_START(0x16) |
1442                                                 R600_ROQ_IB2_START(0x2b)));
1443
1444         RADEON_WRITE(R600_CP_MEQ_THRESHOLDS, R700_STQ_SPLIT(0x30));
1445
1446         RADEON_WRITE(R600_TA_CNTL_AUX, (R600_DISABLE_CUBE_ANISO |
1447                                         R600_SYNC_GRADIENT |
1448                                         R600_SYNC_WALKER |
1449                                         R600_SYNC_ALIGNER));
1450
1451         sx_debug_1 = RADEON_READ(R700_SX_DEBUG_1);
1452         sx_debug_1 |= R700_ENABLE_NEW_SMX_ADDRESS;
1453         RADEON_WRITE(R700_SX_DEBUG_1, sx_debug_1);
1454
1455         smx_dc_ctl0 = RADEON_READ(R600_SMX_DC_CTL0);
1456         smx_dc_ctl0 &= ~R700_CACHE_DEPTH(0x1ff);
1457         smx_dc_ctl0 |= R700_CACHE_DEPTH((dev_priv->r700_sx_num_of_sets * 64) - 1);
1458         RADEON_WRITE(R600_SMX_DC_CTL0, smx_dc_ctl0);
1459
1460         RADEON_WRITE(R700_SMX_EVENT_CTL, (R700_ES_FLUSH_CTL(4) |
1461                                           R700_GS_FLUSH_CTL(4) |
1462                                           R700_ACK_FLUSH_CTL(3) |
1463                                           R700_SYNC_FLUSH_CTL));
1464
1465         if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV770)
1466                 RADEON_WRITE(R700_DB_DEBUG3, R700_DB_CLK_OFF_DELAY(0x1f));
1467         else {
1468                 db_debug4 = RADEON_READ(RV700_DB_DEBUG4);
1469                 db_debug4 |= RV700_DISABLE_TILE_COVERED_FOR_PS_ITER;
1470                 RADEON_WRITE(RV700_DB_DEBUG4, db_debug4);
1471         }
1472
1473         RADEON_WRITE(R600_SX_EXPORT_BUFFER_SIZES, (R600_COLOR_BUFFER_SIZE((dev_priv->r600_sx_max_export_size / 4) - 1) |
1474                                                    R600_POSITION_BUFFER_SIZE((dev_priv->r600_sx_max_export_pos_size / 4) - 1) |
1475                                                    R600_SMX_BUFFER_SIZE((dev_priv->r600_sx_max_export_smx_size / 4) - 1)));
1476
1477         RADEON_WRITE(R700_PA_SC_FIFO_SIZE_R7XX, (R700_SC_PRIM_FIFO_SIZE(dev_priv->r700_sc_prim_fifo_size) |
1478                                                  R700_SC_HIZ_TILE_FIFO_SIZE(dev_priv->r700_sc_hiz_tile_fifo_size) |
1479                                                  R700_SC_EARLYZ_TILE_FIFO_SIZE(dev_priv->r700_sc_earlyz_tile_fifo_fize)));
1480
1481         RADEON_WRITE(R600_PA_SC_MULTI_CHIP_CNTL, 0);
1482
1483         RADEON_WRITE(R600_VGT_NUM_INSTANCES, 1);
1484
1485         RADEON_WRITE(R600_SPI_CONFIG_CNTL, R600_GPR_WRITE_PRIORITY(0));
1486
1487         RADEON_WRITE(R600_SPI_CONFIG_CNTL_1, R600_VTX_DONE_DELAY(4));
1488
1489         RADEON_WRITE(R600_CP_PERFMON_CNTL, 0);
1490
1491         sq_ms_fifo_sizes = (R600_CACHE_FIFO_SIZE(16 * dev_priv->r600_sq_num_cf_insts) |
1492                             R600_DONE_FIFO_HIWATER(0xe0) |
1493                             R600_ALU_UPDATE_FIFO_HIWATER(0x8));
1494         switch (dev_priv->flags & RADEON_FAMILY_MASK) {
1495         case CHIP_RV770:
1496                 sq_ms_fifo_sizes |= R600_FETCH_FIFO_HIWATER(0x1);
1497                 break;
1498         case CHIP_RV730:
1499         case CHIP_RV710:
1500         default:
1501                 sq_ms_fifo_sizes |= R600_FETCH_FIFO_HIWATER(0x4);
1502                 break;
1503         }
1504         RADEON_WRITE(R600_SQ_MS_FIFO_SIZES, sq_ms_fifo_sizes);
1505
1506         /* SQ_CONFIG, SQ_GPR_RESOURCE_MGMT, SQ_THREAD_RESOURCE_MGMT, SQ_STACK_RESOURCE_MGMT
1507          * should be adjusted as needed by the 2D/3D drivers.  This just sets default values
1508          */
1509         sq_config = RADEON_READ(R600_SQ_CONFIG);
1510         sq_config &= ~(R600_PS_PRIO(3) |
1511                        R600_VS_PRIO(3) |
1512                        R600_GS_PRIO(3) |
1513                        R600_ES_PRIO(3));
1514         sq_config |= (R600_DX9_CONSTS |
1515                       R600_VC_ENABLE |
1516                       R600_EXPORT_SRC_C |
1517                       R600_PS_PRIO(0) |
1518                       R600_VS_PRIO(1) |
1519                       R600_GS_PRIO(2) |
1520                       R600_ES_PRIO(3));
1521         if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV710)
1522                 /* no vertex cache */
1523                 sq_config &= ~R600_VC_ENABLE;
1524
1525         RADEON_WRITE(R600_SQ_CONFIG, sq_config);
1526
1527         RADEON_WRITE(R600_SQ_GPR_RESOURCE_MGMT_1,  (R600_NUM_PS_GPRS((dev_priv->r600_max_gprs * 24)/64) |
1528                                                     R600_NUM_VS_GPRS((dev_priv->r600_max_gprs * 24)/64) |
1529                                                     R600_NUM_CLAUSE_TEMP_GPRS(((dev_priv->r600_max_gprs * 24)/64)/2)));
1530
1531         RADEON_WRITE(R600_SQ_GPR_RESOURCE_MGMT_2,  (R600_NUM_GS_GPRS((dev_priv->r600_max_gprs * 7)/64) |
1532                                                     R600_NUM_ES_GPRS((dev_priv->r600_max_gprs * 7)/64)));
1533
1534         sq_thread_resource_mgmt = (R600_NUM_PS_THREADS((dev_priv->r600_max_threads * 4)/8) |
1535                                    R600_NUM_VS_THREADS((dev_priv->r600_max_threads * 2)/8) |
1536                                    R600_NUM_ES_THREADS((dev_priv->r600_max_threads * 1)/8));
1537         if (((dev_priv->r600_max_threads * 1) / 8) > dev_priv->r600_max_gs_threads)
1538                 sq_thread_resource_mgmt |= R600_NUM_GS_THREADS(dev_priv->r600_max_gs_threads);
1539         else
1540                 sq_thread_resource_mgmt |= R600_NUM_GS_THREADS((dev_priv->r600_max_gs_threads * 1)/8);
1541         RADEON_WRITE(R600_SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
1542
1543         RADEON_WRITE(R600_SQ_STACK_RESOURCE_MGMT_1, (R600_NUM_PS_STACK_ENTRIES((dev_priv->r600_max_stack_entries * 1)/4) |
1544                                                      R600_NUM_VS_STACK_ENTRIES((dev_priv->r600_max_stack_entries * 1)/4)));
1545
1546         RADEON_WRITE(R600_SQ_STACK_RESOURCE_MGMT_2, (R600_NUM_GS_STACK_ENTRIES((dev_priv->r600_max_stack_entries * 1)/4) |
1547                                                      R600_NUM_ES_STACK_ENTRIES((dev_priv->r600_max_stack_entries * 1)/4)));
1548
1549         sq_dyn_gpr_size_simd_ab_0 = (R700_SIMDA_RING0((dev_priv->r600_max_gprs * 38)/64) |
1550                                      R700_SIMDA_RING1((dev_priv->r600_max_gprs * 38)/64) |
1551                                      R700_SIMDB_RING0((dev_priv->r600_max_gprs * 38)/64) |
1552                                      R700_SIMDB_RING1((dev_priv->r600_max_gprs * 38)/64));
1553
1554         RADEON_WRITE(R700_SQ_DYN_GPR_SIZE_SIMD_AB_0, sq_dyn_gpr_size_simd_ab_0);
1555         RADEON_WRITE(R700_SQ_DYN_GPR_SIZE_SIMD_AB_1, sq_dyn_gpr_size_simd_ab_0);
1556         RADEON_WRITE(R700_SQ_DYN_GPR_SIZE_SIMD_AB_2, sq_dyn_gpr_size_simd_ab_0);
1557         RADEON_WRITE(R700_SQ_DYN_GPR_SIZE_SIMD_AB_3, sq_dyn_gpr_size_simd_ab_0);
1558         RADEON_WRITE(R700_SQ_DYN_GPR_SIZE_SIMD_AB_4, sq_dyn_gpr_size_simd_ab_0);
1559         RADEON_WRITE(R700_SQ_DYN_GPR_SIZE_SIMD_AB_5, sq_dyn_gpr_size_simd_ab_0);
1560         RADEON_WRITE(R700_SQ_DYN_GPR_SIZE_SIMD_AB_6, sq_dyn_gpr_size_simd_ab_0);
1561         RADEON_WRITE(R700_SQ_DYN_GPR_SIZE_SIMD_AB_7, sq_dyn_gpr_size_simd_ab_0);
1562
1563         RADEON_WRITE(R700_PA_SC_FORCE_EOV_MAX_CNTS, (R700_FORCE_EOV_MAX_CLK_CNT(4095) |
1564                                                      R700_FORCE_EOV_MAX_REZ_CNT(255)));
1565
1566         if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV710)
1567                 RADEON_WRITE(R600_VGT_CACHE_INVALIDATION, (R600_CACHE_INVALIDATION(R600_TC_ONLY) |
1568                                                            R700_AUTO_INVLD_EN(R700_ES_AND_GS_AUTO)));
1569         else
1570                 RADEON_WRITE(R600_VGT_CACHE_INVALIDATION, (R600_CACHE_INVALIDATION(R600_VC_AND_TC) |
1571                                                            R700_AUTO_INVLD_EN(R700_ES_AND_GS_AUTO)));
1572
1573         switch (dev_priv->flags & RADEON_FAMILY_MASK) {
1574         case CHIP_RV770:
1575         case CHIP_RV730:
1576                 gs_prim_buffer_depth = 384;
1577                 break;
1578         case CHIP_RV710:
1579                 gs_prim_buffer_depth = 128;
1580                 break;
1581         default:
1582                 break;
1583         }
1584
1585         num_gs_verts_per_thread = dev_priv->r600_max_pipes * 16;
1586         vgt_gs_per_es = gs_prim_buffer_depth + num_gs_verts_per_thread;
1587         /* Max value for this is 256 */
1588         if (vgt_gs_per_es > 256)
1589                 vgt_gs_per_es = 256;
1590
1591         RADEON_WRITE(R600_VGT_ES_PER_GS, 128);
1592         RADEON_WRITE(R600_VGT_GS_PER_ES, vgt_gs_per_es);
1593         RADEON_WRITE(R600_VGT_GS_PER_VS, 2);
1594
1595         /* more default values. 2D/3D driver should adjust as needed */
1596         RADEON_WRITE(R600_VGT_GS_VERTEX_REUSE, 16);
1597         RADEON_WRITE(R600_PA_SC_LINE_STIPPLE_STATE, 0);
1598         RADEON_WRITE(R600_VGT_STRMOUT_EN, 0);
1599         RADEON_WRITE(R600_SX_MISC, 0);
1600         RADEON_WRITE(R600_PA_SC_MODE_CNTL, 0);
1601         RADEON_WRITE(R700_PA_SC_EDGERULE, 0xaaaaaaaa);
1602         RADEON_WRITE(R600_PA_SC_AA_CONFIG, 0);
1603         RADEON_WRITE(R600_PA_SC_CLIPRECT_RULE, 0xffff);
1604         RADEON_WRITE(R600_PA_SC_LINE_STIPPLE, 0);
1605         RADEON_WRITE(R600_SPI_INPUT_Z, 0);
1606         RADEON_WRITE(R600_SPI_PS_IN_CONTROL_0, R600_NUM_INTERP(2));
1607         RADEON_WRITE(R600_CB_COLOR7_FRAG, 0);
1608
1609         /* clear render buffer base addresses */
1610         RADEON_WRITE(R600_CB_COLOR0_BASE, 0);
1611         RADEON_WRITE(R600_CB_COLOR1_BASE, 0);
1612         RADEON_WRITE(R600_CB_COLOR2_BASE, 0);
1613         RADEON_WRITE(R600_CB_COLOR3_BASE, 0);
1614         RADEON_WRITE(R600_CB_COLOR4_BASE, 0);
1615         RADEON_WRITE(R600_CB_COLOR5_BASE, 0);
1616         RADEON_WRITE(R600_CB_COLOR6_BASE, 0);
1617         RADEON_WRITE(R600_CB_COLOR7_BASE, 0);
1618
1619         RADEON_WRITE(R700_TCP_CNTL, 0);
1620
1621         hdp_host_path_cntl = RADEON_READ(R600_HDP_HOST_PATH_CNTL);
1622         RADEON_WRITE(R600_HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
1623
1624         RADEON_WRITE(R600_PA_SC_MULTI_CHIP_CNTL, 0);
1625
1626         RADEON_WRITE(R600_PA_CL_ENHANCE, (R600_CLIP_VTX_REORDER_ENA |
1627                                           R600_NUM_CLIP_SEQ(3)));
1628
1629 }
1630
1631 static void r600_cp_init_ring_buffer(struct drm_device *dev,
1632                                        drm_radeon_private_t *dev_priv,
1633                                        struct drm_file *file_priv)
1634 {
1635         u32 ring_start;
1636         u64 rptr_addr;
1637
1638         if (((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV770))
1639                 r700_gfx_init(dev, dev_priv);
1640         else
1641                 r600_gfx_init(dev, dev_priv);
1642
1643         RADEON_WRITE(R600_GRBM_SOFT_RESET, R600_SOFT_RESET_CP);
1644         RADEON_READ(R600_GRBM_SOFT_RESET);
1645         DRM_UDELAY(15000);
1646         RADEON_WRITE(R600_GRBM_SOFT_RESET, 0);
1647
1648
1649         /* Set ring buffer size */
1650 #ifdef __BIG_ENDIAN
1651         RADEON_WRITE(R600_CP_RB_CNTL,
1652                      RADEON_BUF_SWAP_32BIT |
1653                      RADEON_RB_NO_UPDATE |
1654                      (dev_priv->ring.rptr_update_l2qw << 8) |
1655                      dev_priv->ring.size_l2qw);
1656 #else
1657         RADEON_WRITE(R600_CP_RB_CNTL,
1658                      RADEON_RB_NO_UPDATE |
1659                      (dev_priv->ring.rptr_update_l2qw << 8) |
1660                      dev_priv->ring.size_l2qw);
1661 #endif
1662
1663         RADEON_WRITE(R600_CP_SEM_WAIT_TIMER, 0x4);
1664
1665         /* Set the write pointer delay */
1666         RADEON_WRITE(R600_CP_RB_WPTR_DELAY, 0);
1667
1668 #ifdef __BIG_ENDIAN
1669         RADEON_WRITE(R600_CP_RB_CNTL,
1670                      RADEON_BUF_SWAP_32BIT |
1671                      RADEON_RB_NO_UPDATE |
1672                      RADEON_RB_RPTR_WR_ENA |
1673                      (dev_priv->ring.rptr_update_l2qw << 8) |
1674                      dev_priv->ring.size_l2qw);
1675 #else
1676         RADEON_WRITE(R600_CP_RB_CNTL,
1677                      RADEON_RB_NO_UPDATE |
1678                      RADEON_RB_RPTR_WR_ENA |
1679                      (dev_priv->ring.rptr_update_l2qw << 8) |
1680                      dev_priv->ring.size_l2qw);
1681 #endif
1682
1683         /* Initialize the ring buffer's read and write pointers */
1684         RADEON_WRITE(R600_CP_RB_RPTR_WR, 0);
1685         RADEON_WRITE(R600_CP_RB_WPTR, 0);
1686         SET_RING_HEAD(dev_priv, 0);
1687         dev_priv->ring.tail = 0;
1688
1689 #if __OS_HAS_AGP
1690         if (dev_priv->flags & RADEON_IS_AGP) {
1691                 rptr_addr = dev_priv->ring_rptr->offset
1692                         - dev->agp->base +
1693                         dev_priv->gart_vm_start;
1694         } else
1695 #endif
1696         {
1697                 rptr_addr = dev_priv->ring_rptr->offset
1698                         - ((unsigned long) dev->sg->virtual)
1699                         + dev_priv->gart_vm_start;
1700         }
1701         RADEON_WRITE(R600_CP_RB_RPTR_ADDR,
1702                      rptr_addr & 0xffffffff);
1703         RADEON_WRITE(R600_CP_RB_RPTR_ADDR_HI,
1704                      upper_32_bits(rptr_addr));
1705
1706 #ifdef __BIG_ENDIAN
1707         RADEON_WRITE(R600_CP_RB_CNTL,
1708                      RADEON_BUF_SWAP_32BIT |
1709                      (dev_priv->ring.rptr_update_l2qw << 8) |
1710                      dev_priv->ring.size_l2qw);
1711 #else
1712         RADEON_WRITE(R600_CP_RB_CNTL,
1713                      (dev_priv->ring.rptr_update_l2qw << 8) |
1714                      dev_priv->ring.size_l2qw);
1715 #endif
1716
1717 #if __OS_HAS_AGP
1718         if (dev_priv->flags & RADEON_IS_AGP) {
1719                 /* XXX */
1720                 radeon_write_agp_base(dev_priv, dev->agp->base);
1721
1722                 /* XXX */
1723                 radeon_write_agp_location(dev_priv,
1724                              (((dev_priv->gart_vm_start - 1 +
1725                                 dev_priv->gart_size) & 0xffff0000) |
1726                               (dev_priv->gart_vm_start >> 16)));
1727
1728                 ring_start = (dev_priv->cp_ring->offset
1729                               - dev->agp->base
1730                               + dev_priv->gart_vm_start);
1731         } else
1732 #endif
1733                 ring_start = (dev_priv->cp_ring->offset
1734                               - (unsigned long)dev->sg->virtual
1735                               + dev_priv->gart_vm_start);
1736
1737         RADEON_WRITE(R600_CP_RB_BASE, ring_start >> 8);
1738
1739         RADEON_WRITE(R600_CP_ME_CNTL, 0xff);
1740
1741         RADEON_WRITE(R600_CP_DEBUG, (1 << 27) | (1 << 28));
1742
1743         /* Initialize the scratch register pointer.  This will cause
1744          * the scratch register values to be written out to memory
1745          * whenever they are updated.
1746          *
1747          * We simply put this behind the ring read pointer, this works
1748          * with PCI GART as well as (whatever kind of) AGP GART
1749          */
1750         {
1751                 u64 scratch_addr;
1752
1753                 scratch_addr = RADEON_READ(R600_CP_RB_RPTR_ADDR);
1754                 scratch_addr |= ((u64)RADEON_READ(R600_CP_RB_RPTR_ADDR_HI)) << 32;
1755                 scratch_addr += R600_SCRATCH_REG_OFFSET;
1756                 scratch_addr >>= 8;
1757                 scratch_addr &= 0xffffffff;
1758
1759                 RADEON_WRITE(R600_SCRATCH_ADDR, (uint32_t)scratch_addr);
1760         }
1761
1762         RADEON_WRITE(R600_SCRATCH_UMSK, 0x7);
1763
1764         /* Turn on bus mastering */
1765         radeon_enable_bm(dev_priv);
1766
1767         radeon_write_ring_rptr(dev_priv, R600_SCRATCHOFF(0), 0);
1768         RADEON_WRITE(R600_LAST_FRAME_REG, 0);
1769
1770         radeon_write_ring_rptr(dev_priv, R600_SCRATCHOFF(1), 0);
1771         RADEON_WRITE(R600_LAST_DISPATCH_REG, 0);
1772
1773         radeon_write_ring_rptr(dev_priv, R600_SCRATCHOFF(2), 0);
1774         RADEON_WRITE(R600_LAST_CLEAR_REG, 0);
1775
1776         /* reset sarea copies of these */
1777         if (dev_priv->sarea_priv) {
1778                 dev_priv->sarea_priv->last_frame = 0;
1779                 dev_priv->sarea_priv->last_dispatch = 0;
1780                 dev_priv->sarea_priv->last_clear = 0;
1781         }
1782
1783         r600_do_wait_for_idle(dev_priv);
1784
1785 }
1786
1787 int r600_do_cleanup_cp(struct drm_device *dev)
1788 {
1789         drm_radeon_private_t *dev_priv = dev->dev_private;
1790         DRM_DEBUG("\n");
1791
1792         /* Make sure interrupts are disabled here because the uninstall ioctl
1793          * may not have been called from userspace and after dev_private
1794          * is freed, it's too late.
1795          */
1796         if (dev->irq_enabled)
1797                 drm_irq_uninstall(dev);
1798
1799 #if __OS_HAS_AGP
1800         if (dev_priv->flags & RADEON_IS_AGP) {
1801                 if (dev_priv->cp_ring != NULL) {
1802                         drm_core_ioremapfree(dev_priv->cp_ring, dev);
1803                         dev_priv->cp_ring = NULL;
1804                 }
1805                 if (dev_priv->ring_rptr != NULL) {
1806                         drm_core_ioremapfree(dev_priv->ring_rptr, dev);
1807                         dev_priv->ring_rptr = NULL;
1808                 }
1809                 if (dev->agp_buffer_map != NULL) {
1810                         drm_core_ioremapfree(dev->agp_buffer_map, dev);
1811                         dev->agp_buffer_map = NULL;
1812                 }
1813         } else
1814 #endif
1815         {
1816
1817                 if (dev_priv->gart_info.bus_addr)
1818                         r600_page_table_cleanup(dev, &dev_priv->gart_info);
1819
1820                 if (dev_priv->gart_info.gart_table_location == DRM_ATI_GART_FB) {
1821                         drm_core_ioremapfree(&dev_priv->gart_info.mapping, dev);
1822                         dev_priv->gart_info.addr = 0;
1823                 }
1824         }
1825         /* only clear to the start of flags */
1826         memset(dev_priv, 0, offsetof(drm_radeon_private_t, flags));
1827
1828         return 0;
1829 }
1830
1831 int r600_do_init_cp(struct drm_device *dev, drm_radeon_init_t *init,
1832                     struct drm_file *file_priv)
1833 {
1834         drm_radeon_private_t *dev_priv = dev->dev_private;
1835
1836         DRM_DEBUG("\n");
1837
1838         /* if we require new memory map but we don't have it fail */
1839         if ((dev_priv->flags & RADEON_NEW_MEMMAP) && !dev_priv->new_memmap) {
1840                 DRM_ERROR("Cannot initialise DRM on this card\nThis card requires a new X.org DDX for 3D\n");
1841                 r600_do_cleanup_cp(dev);
1842                 return -EINVAL;
1843         }
1844
1845         if (init->is_pci && (dev_priv->flags & RADEON_IS_AGP)) {
1846                 DRM_DEBUG("Forcing AGP card to PCI mode\n");
1847                 dev_priv->flags &= ~RADEON_IS_AGP;
1848                 /* The writeback test succeeds, but when writeback is enabled,
1849                  * the ring buffer read ptr update fails after first 128 bytes.
1850                  */
1851                 radeon_no_wb = 1;
1852         } else if (!(dev_priv->flags & (RADEON_IS_AGP | RADEON_IS_PCI | RADEON_IS_PCIE))
1853                  && !init->is_pci) {
1854                 DRM_DEBUG("Restoring AGP flag\n");
1855                 dev_priv->flags |= RADEON_IS_AGP;
1856         }
1857
1858         dev_priv->usec_timeout = init->usec_timeout;
1859         if (dev_priv->usec_timeout < 1 ||
1860             dev_priv->usec_timeout > RADEON_MAX_USEC_TIMEOUT) {
1861                 DRM_DEBUG("TIMEOUT problem!\n");
1862                 r600_do_cleanup_cp(dev);
1863                 return -EINVAL;
1864         }
1865
1866         /* Enable vblank on CRTC1 for older X servers
1867          */
1868         dev_priv->vblank_crtc = DRM_RADEON_VBLANK_CRTC1;
1869
1870         dev_priv->cp_mode = init->cp_mode;
1871
1872         /* We don't support anything other than bus-mastering ring mode,
1873          * but the ring can be in either AGP or PCI space for the ring
1874          * read pointer.
1875          */
1876         if ((init->cp_mode != RADEON_CSQ_PRIBM_INDDIS) &&
1877             (init->cp_mode != RADEON_CSQ_PRIBM_INDBM)) {
1878                 DRM_DEBUG("BAD cp_mode (%x)!\n", init->cp_mode);
1879                 r600_do_cleanup_cp(dev);
1880                 return -EINVAL;
1881         }
1882
1883         switch (init->fb_bpp) {
1884         case 16:
1885                 dev_priv->color_fmt = RADEON_COLOR_FORMAT_RGB565;
1886                 break;
1887         case 32:
1888         default:
1889                 dev_priv->color_fmt = RADEON_COLOR_FORMAT_ARGB8888;
1890                 break;
1891         }
1892         dev_priv->front_offset = init->front_offset;
1893         dev_priv->front_pitch = init->front_pitch;
1894         dev_priv->back_offset = init->back_offset;
1895         dev_priv->back_pitch = init->back_pitch;
1896
1897         dev_priv->ring_offset = init->ring_offset;
1898         dev_priv->ring_rptr_offset = init->ring_rptr_offset;
1899         dev_priv->buffers_offset = init->buffers_offset;
1900         dev_priv->gart_textures_offset = init->gart_textures_offset;
1901
1902         dev_priv->sarea = drm_getsarea(dev);
1903         if (!dev_priv->sarea) {
1904                 DRM_ERROR("could not find sarea!\n");
1905                 r600_do_cleanup_cp(dev);
1906                 return -EINVAL;
1907         }
1908
1909         dev_priv->cp_ring = drm_core_findmap(dev, init->ring_offset);
1910         if (!dev_priv->cp_ring) {
1911                 DRM_ERROR("could not find cp ring region!\n");
1912                 r600_do_cleanup_cp(dev);
1913                 return -EINVAL;
1914         }
1915         dev_priv->ring_rptr = drm_core_findmap(dev, init->ring_rptr_offset);
1916         if (!dev_priv->ring_rptr) {
1917                 DRM_ERROR("could not find ring read pointer!\n");
1918                 r600_do_cleanup_cp(dev);
1919                 return -EINVAL;
1920         }
1921         dev->agp_buffer_token = init->buffers_offset;
1922         dev->agp_buffer_map = drm_core_findmap(dev, init->buffers_offset);
1923         if (!dev->agp_buffer_map) {
1924                 DRM_ERROR("could not find dma buffer region!\n");
1925                 r600_do_cleanup_cp(dev);
1926                 return -EINVAL;
1927         }
1928
1929         if (init->gart_textures_offset) {
1930                 dev_priv->gart_textures =
1931                     drm_core_findmap(dev, init->gart_textures_offset);
1932                 if (!dev_priv->gart_textures) {
1933                         DRM_ERROR("could not find GART texture region!\n");
1934                         r600_do_cleanup_cp(dev);
1935                         return -EINVAL;
1936                 }
1937         }
1938
1939         dev_priv->sarea_priv =
1940             (drm_radeon_sarea_t *) ((u8 *) dev_priv->sarea->handle +
1941                                     init->sarea_priv_offset);
1942
1943 #if __OS_HAS_AGP
1944         /* XXX */
1945         if (dev_priv->flags & RADEON_IS_AGP) {
1946                 drm_core_ioremap_wc(dev_priv->cp_ring, dev);
1947                 drm_core_ioremap_wc(dev_priv->ring_rptr, dev);
1948                 drm_core_ioremap_wc(dev->agp_buffer_map, dev);
1949                 if (!dev_priv->cp_ring->handle ||
1950                     !dev_priv->ring_rptr->handle ||
1951                     !dev->agp_buffer_map->handle) {
1952                         DRM_ERROR("could not find ioremap agp regions!\n");
1953                         r600_do_cleanup_cp(dev);
1954                         return -EINVAL;
1955                 }
1956         } else
1957 #endif
1958         {
1959                 dev_priv->cp_ring->handle = (void *)dev_priv->cp_ring->offset;
1960                 dev_priv->ring_rptr->handle =
1961                     (void *)dev_priv->ring_rptr->offset;
1962                 dev->agp_buffer_map->handle =
1963                     (void *)dev->agp_buffer_map->offset;
1964
1965                 DRM_DEBUG("dev_priv->cp_ring->handle %p\n",
1966                           dev_priv->cp_ring->handle);
1967                 DRM_DEBUG("dev_priv->ring_rptr->handle %p\n",
1968                           dev_priv->ring_rptr->handle);
1969                 DRM_DEBUG("dev->agp_buffer_map->handle %p\n",
1970                           dev->agp_buffer_map->handle);
1971         }
1972
1973         dev_priv->fb_location = (radeon_read_fb_location(dev_priv) & 0xffff) << 24;
1974         dev_priv->fb_size =
1975                 (((radeon_read_fb_location(dev_priv) & 0xffff0000u) << 8) + 0x1000000)
1976                 - dev_priv->fb_location;
1977
1978         dev_priv->front_pitch_offset = (((dev_priv->front_pitch / 64) << 22) |
1979                                         ((dev_priv->front_offset
1980                                           + dev_priv->fb_location) >> 10));
1981
1982         dev_priv->back_pitch_offset = (((dev_priv->back_pitch / 64) << 22) |
1983                                        ((dev_priv->back_offset
1984                                          + dev_priv->fb_location) >> 10));
1985
1986         dev_priv->depth_pitch_offset = (((dev_priv->depth_pitch / 64) << 22) |
1987                                         ((dev_priv->depth_offset
1988                                           + dev_priv->fb_location) >> 10));
1989
1990         dev_priv->gart_size = init->gart_size;
1991
1992         /* New let's set the memory map ... */
1993         if (dev_priv->new_memmap) {
1994                 u32 base = 0;
1995
1996                 DRM_INFO("Setting GART location based on new memory map\n");
1997
1998                 /* If using AGP, try to locate the AGP aperture at the same
1999                  * location in the card and on the bus, though we have to
2000                  * align it down.
2001                  */
2002 #if __OS_HAS_AGP
2003                 /* XXX */
2004                 if (dev_priv->flags & RADEON_IS_AGP) {
2005                         base = dev->agp->base;
2006                         /* Check if valid */
2007                         if ((base + dev_priv->gart_size - 1) >= dev_priv->fb_location &&
2008                             base < (dev_priv->fb_location + dev_priv->fb_size - 1)) {
2009                                 DRM_INFO("Can't use AGP base @0x%08lx, won't fit\n",
2010                                          dev->agp->base);
2011                                 base = 0;
2012                         }
2013                 }
2014 #endif
2015                 /* If not or if AGP is at 0 (Macs), try to put it elsewhere */
2016                 if (base == 0) {
2017                         base = dev_priv->fb_location + dev_priv->fb_size;
2018                         if (base < dev_priv->fb_location ||
2019                             ((base + dev_priv->gart_size) & 0xfffffffful) < base)
2020                                 base = dev_priv->fb_location
2021                                         - dev_priv->gart_size;
2022                 }
2023                 dev_priv->gart_vm_start = base & 0xffc00000u;
2024                 if (dev_priv->gart_vm_start != base)
2025                         DRM_INFO("GART aligned down from 0x%08x to 0x%08x\n",
2026                                  base, dev_priv->gart_vm_start);
2027         }
2028
2029 #if __OS_HAS_AGP
2030         /* XXX */
2031         if (dev_priv->flags & RADEON_IS_AGP)
2032                 dev_priv->gart_buffers_offset = (dev->agp_buffer_map->offset
2033                                                  - dev->agp->base
2034                                                  + dev_priv->gart_vm_start);
2035         else
2036 #endif
2037                 dev_priv->gart_buffers_offset = (dev->agp_buffer_map->offset
2038                                                  - (unsigned long)dev->sg->virtual
2039                                                  + dev_priv->gart_vm_start);
2040
2041         DRM_DEBUG("fb 0x%08x size %d\n",
2042                   (unsigned int) dev_priv->fb_location,
2043                   (unsigned int) dev_priv->fb_size);
2044         DRM_DEBUG("dev_priv->gart_size %d\n", dev_priv->gart_size);
2045         DRM_DEBUG("dev_priv->gart_vm_start 0x%08x\n",
2046                   (unsigned int) dev_priv->gart_vm_start);
2047         DRM_DEBUG("dev_priv->gart_buffers_offset 0x%08lx\n",
2048                   dev_priv->gart_buffers_offset);
2049
2050         dev_priv->ring.start = (u32 *) dev_priv->cp_ring->handle;
2051         dev_priv->ring.end = ((u32 *) dev_priv->cp_ring->handle
2052                               + init->ring_size / sizeof(u32));
2053         dev_priv->ring.size = init->ring_size;
2054         dev_priv->ring.size_l2qw = drm_order(init->ring_size / 8);
2055
2056         dev_priv->ring.rptr_update = /* init->rptr_update */ 4096;
2057         dev_priv->ring.rptr_update_l2qw = drm_order(/* init->rptr_update */ 4096 / 8);
2058
2059         dev_priv->ring.fetch_size = /* init->fetch_size */ 32;
2060         dev_priv->ring.fetch_size_l2ow = drm_order(/* init->fetch_size */ 32 / 16);
2061
2062         dev_priv->ring.tail_mask = (dev_priv->ring.size / sizeof(u32)) - 1;
2063
2064         dev_priv->ring.high_mark = RADEON_RING_HIGH_MARK;
2065
2066 #if __OS_HAS_AGP
2067         if (dev_priv->flags & RADEON_IS_AGP) {
2068                 /* XXX turn off pcie gart */
2069         } else
2070 #endif
2071         {
2072                 dev_priv->gart_info.table_mask = DMA_BIT_MASK(32);
2073                 /* if we have an offset set from userspace */
2074                 if (!dev_priv->pcigart_offset_set) {
2075                         DRM_ERROR("Need gart offset from userspace\n");
2076                         r600_do_cleanup_cp(dev);
2077                         return -EINVAL;
2078                 }
2079
2080                 DRM_DEBUG("Using gart offset 0x%08lx\n", dev_priv->pcigart_offset);
2081
2082                 dev_priv->gart_info.bus_addr =
2083                         dev_priv->pcigart_offset + dev_priv->fb_location;
2084                 dev_priv->gart_info.mapping.offset =
2085                         dev_priv->pcigart_offset + dev_priv->fb_aper_offset;
2086                 dev_priv->gart_info.mapping.size =
2087                         dev_priv->gart_info.table_size;
2088
2089                 drm_core_ioremap_wc(&dev_priv->gart_info.mapping, dev);
2090                 if (!dev_priv->gart_info.mapping.handle) {
2091                         DRM_ERROR("ioremap failed.\n");
2092                         r600_do_cleanup_cp(dev);
2093                         return -EINVAL;
2094                 }
2095
2096                 dev_priv->gart_info.addr =
2097                         dev_priv->gart_info.mapping.handle;
2098
2099                 DRM_DEBUG("Setting phys_pci_gart to %p %08lX\n",
2100                           dev_priv->gart_info.addr,
2101                           dev_priv->pcigart_offset);
2102
2103                 if (!r600_page_table_init(dev)) {
2104                         DRM_ERROR("Failed to init GART table\n");
2105                         r600_do_cleanup_cp(dev);
2106                         return -EINVAL;
2107                 }
2108
2109                 if (((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV770))
2110                         r700_vm_init(dev);
2111                 else
2112                         r600_vm_init(dev);
2113         }
2114
2115         if (((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV770))
2116                 r700_cp_load_microcode(dev_priv);
2117         else
2118                 r600_cp_load_microcode(dev_priv);
2119
2120         r600_cp_init_ring_buffer(dev, dev_priv, file_priv);
2121
2122         dev_priv->last_buf = 0;
2123
2124         r600_do_engine_reset(dev);
2125         r600_test_writeback(dev_priv);
2126
2127         return 0;
2128 }
2129
2130 int r600_do_resume_cp(struct drm_device *dev, struct drm_file *file_priv)
2131 {
2132         drm_radeon_private_t *dev_priv = dev->dev_private;
2133
2134         DRM_DEBUG("\n");
2135         if (((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV770)) {
2136                 r700_vm_init(dev);
2137                 r700_cp_load_microcode(dev_priv);
2138         } else {
2139                 r600_vm_init(dev);
2140                 r600_cp_load_microcode(dev_priv);
2141         }
2142         r600_cp_init_ring_buffer(dev, dev_priv, file_priv);
2143         r600_do_engine_reset(dev);
2144
2145         return 0;
2146 }
2147
2148 /* Wait for the CP to go idle.
2149  */
2150 int r600_do_cp_idle(drm_radeon_private_t *dev_priv)
2151 {
2152         RING_LOCALS;
2153         DRM_DEBUG("\n");
2154
2155         BEGIN_RING(5);
2156         OUT_RING(CP_PACKET3(R600_IT_EVENT_WRITE, 0));
2157         OUT_RING(R600_CACHE_FLUSH_AND_INV_EVENT);
2158         /* wait for 3D idle clean */
2159         OUT_RING(CP_PACKET3(R600_IT_SET_CONFIG_REG, 1));
2160         OUT_RING((R600_WAIT_UNTIL - R600_SET_CONFIG_REG_OFFSET) >> 2);
2161         OUT_RING(RADEON_WAIT_3D_IDLE | RADEON_WAIT_3D_IDLECLEAN);
2162
2163         ADVANCE_RING();
2164         COMMIT_RING();
2165
2166         return r600_do_wait_for_idle(dev_priv);
2167 }
2168
2169 /* Start the Command Processor.
2170  */
2171 void r600_do_cp_start(drm_radeon_private_t *dev_priv)
2172 {
2173         u32 cp_me;
2174         RING_LOCALS;
2175         DRM_DEBUG("\n");
2176
2177         BEGIN_RING(7);
2178         OUT_RING(CP_PACKET3(R600_IT_ME_INITIALIZE, 5));
2179         OUT_RING(0x00000001);
2180         if (((dev_priv->flags & RADEON_FAMILY_MASK) < CHIP_RV770))
2181                 OUT_RING(0x00000003);
2182         else
2183                 OUT_RING(0x00000000);
2184         OUT_RING((dev_priv->r600_max_hw_contexts - 1));
2185         OUT_RING(R600_ME_INITIALIZE_DEVICE_ID(1));
2186         OUT_RING(0x00000000);
2187         OUT_RING(0x00000000);
2188         ADVANCE_RING();
2189         COMMIT_RING();
2190
2191         /* set the mux and reset the halt bit */
2192         cp_me = 0xff;
2193         RADEON_WRITE(R600_CP_ME_CNTL, cp_me);
2194
2195         dev_priv->cp_running = 1;
2196
2197 }
2198
2199 void r600_do_cp_reset(drm_radeon_private_t *dev_priv)
2200 {
2201         u32 cur_read_ptr;
2202         DRM_DEBUG("\n");
2203
2204         cur_read_ptr = RADEON_READ(R600_CP_RB_RPTR);
2205         RADEON_WRITE(R600_CP_RB_WPTR, cur_read_ptr);
2206         SET_RING_HEAD(dev_priv, cur_read_ptr);
2207         dev_priv->ring.tail = cur_read_ptr;
2208 }
2209
2210 void r600_do_cp_stop(drm_radeon_private_t *dev_priv)
2211 {
2212         uint32_t cp_me;
2213
2214         DRM_DEBUG("\n");
2215
2216         cp_me = 0xff | R600_CP_ME_HALT;
2217
2218         RADEON_WRITE(R600_CP_ME_CNTL, cp_me);
2219
2220         dev_priv->cp_running = 0;
2221 }
2222
2223 int r600_cp_dispatch_indirect(struct drm_device *dev,
2224                               struct drm_buf *buf, int start, int end)
2225 {
2226         drm_radeon_private_t *dev_priv = dev->dev_private;
2227         RING_LOCALS;
2228
2229         if (start != end) {
2230                 unsigned long offset = (dev_priv->gart_buffers_offset
2231                                         + buf->offset + start);
2232                 int dwords = (end - start + 3) / sizeof(u32);
2233
2234                 DRM_DEBUG("dwords:%d\n", dwords);
2235                 DRM_DEBUG("offset 0x%lx\n", offset);
2236
2237
2238                 /* Indirect buffer data must be a multiple of 16 dwords.
2239                  * pad the data with a Type-2 CP packet.
2240                  */
2241                 while (dwords & 0xf) {
2242                         u32 *data = (u32 *)
2243                             ((char *)dev->agp_buffer_map->handle
2244                              + buf->offset + start);
2245                         data[dwords++] = RADEON_CP_PACKET2;
2246                 }
2247
2248                 /* Fire off the indirect buffer */
2249                 BEGIN_RING(4);
2250                 OUT_RING(CP_PACKET3(R600_IT_INDIRECT_BUFFER, 2));
2251                 OUT_RING((offset & 0xfffffffc));
2252                 OUT_RING((upper_32_bits(offset) & 0xff));
2253                 OUT_RING(dwords);
2254                 ADVANCE_RING();
2255         }
2256
2257         return 0;
2258 }