sys/dev/drm2/radeon/r600_cp.c

   1 /*
   2  * Copyright 2008-2009 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  22  * DEALINGS IN THE SOFTWARE.
  23  *
  24  * Authors:
  25  *     Dave Airlie <airlied@redhat.com>
  26  *     Alex Deucher <alexander.deucher@amd.com>
  27  */
  28
  29 #include <sys/cdefs.h>
  30 __FBSDID("$FreeBSD$");
  31
  32 #include <sys/param.h>
  33 #include <sys/systm.h>
  34 #include <sys/linker.h>
  35 #include <sys/firmware.h>
  36
  37 #include <dev/drm2/drmP.h>
  38 #include <dev/drm2/radeon/radeon_drm.h>
  39 #include "radeon_drv.h"
  40 #include "r600_cp.h"
  41
  42 #define PFP_UCODE_SIZE 576
  43 #define PM4_UCODE_SIZE 1792
  44 #define R700_PFP_UCODE_SIZE 848
  45 #define R700_PM4_UCODE_SIZE 1360
  46
  47 # define ATI_PCIGART_PAGE_SIZE          4096    /**< PCI GART page size */
  48 # define ATI_PCIGART_PAGE_MASK          (~(ATI_PCIGART_PAGE_SIZE-1))
  49
  50 #define R600_PTE_VALID     (1 << 0)
  51 #define R600_PTE_SYSTEM    (1 << 1)
  52 #define R600_PTE_SNOOPED   (1 << 2)
  53 #define R600_PTE_READABLE  (1 << 5)
  54 #define R600_PTE_WRITEABLE (1 << 6)
  55
  56 /* MAX values used for gfx init */
  57 #define R6XX_MAX_SH_GPRS           256
  58 #define R6XX_MAX_TEMP_GPRS         16
  59 #define R6XX_MAX_SH_THREADS        256
  60 #define R6XX_MAX_SH_STACK_ENTRIES  4096
  61 #define R6XX_MAX_BACKENDS          8
  62 #define R6XX_MAX_BACKENDS_MASK     0xff
  63 #define R6XX_MAX_SIMDS             8
  64 #define R6XX_MAX_SIMDS_MASK        0xff
  65 #define R6XX_MAX_PIPES             8
  66 #define R6XX_MAX_PIPES_MASK        0xff
  67
  68 #define R7XX_MAX_SH_GPRS           256
  69 #define R7XX_MAX_TEMP_GPRS         16
  70 #define R7XX_MAX_SH_THREADS        256
  71 #define R7XX_MAX_SH_STACK_ENTRIES  4096
  72 #define R7XX_MAX_BACKENDS          8
  73 #define R7XX_MAX_BACKENDS_MASK     0xff
  74 #define R7XX_MAX_SIMDS             16
  75 #define R7XX_MAX_SIMDS_MASK        0xffff
  76 #define R7XX_MAX_PIPES             8
  77 #define R7XX_MAX_PIPES_MASK        0xff
  78
  79 static int r600_do_wait_for_fifo(drm_radeon_private_t *dev_priv, int entries)
  80 {
  81         int i;
  82
  83         dev_priv->stats.boxes |= RADEON_BOX_WAIT_IDLE;
  84
  85         for (i = 0; i < dev_priv->usec_timeout; i++) {
  86                 int slots;
  87                 if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV770)
  88                         slots = (RADEON_READ(R600_GRBM_STATUS)
  89                                  & R700_CMDFIFO_AVAIL_MASK);
  90                 else
  91                         slots = (RADEON_READ(R600_GRBM_STATUS)
  92                                  & R600_CMDFIFO_AVAIL_MASK);
  93                 if (slots >= entries)
  94                         return 0;
  95                 DRM_UDELAY(1);
  96         }
  97         DRM_INFO("wait for fifo failed status : 0x%08X 0x%08X\n",
  98                  RADEON_READ(R600_GRBM_STATUS),
  99                  RADEON_READ(R600_GRBM_STATUS2));
 100
 101         return -EBUSY;
 102 }
 103
 104 static int r600_do_wait_for_idle(drm_radeon_private_t *dev_priv)
 105 {
 106         int i, ret;
 107
 108         dev_priv->stats.boxes |= RADEON_BOX_WAIT_IDLE;
 109
 110         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV770)
 111                 ret = r600_do_wait_for_fifo(dev_priv, 8);
 112         else
 113                 ret = r600_do_wait_for_fifo(dev_priv, 16);
 114         if (ret)
 115                 return ret;
 116         for (i = 0; i < dev_priv->usec_timeout; i++) {
 117                 if (!(RADEON_READ(R600_GRBM_STATUS) & R600_GUI_ACTIVE))
 118                         return 0;
 119                 DRM_UDELAY(1);
 120         }
 121         DRM_INFO("wait idle failed status : 0x%08X 0x%08X\n",
 122                  RADEON_READ(R600_GRBM_STATUS),
 123                  RADEON_READ(R600_GRBM_STATUS2));
 124
 125         return -EBUSY;
 126 }
 127
 128 void r600_page_table_cleanup(struct drm_device *dev, struct drm_ati_pcigart_info *gart_info)
 129 {
 130         struct drm_sg_mem *entry = dev->sg;
 131 #ifdef __linux__
 132         int max_pages;
 133         int pages;
 134         int i;
 135 #endif
 136
 137         if (!entry)
 138                 return;
 139
 140         if (gart_info->bus_addr) {
 141 #ifdef __linux__
 142                 max_pages = (gart_info->table_size / sizeof(u64));
 143                 pages = (entry->pages <= max_pages)
 144                   ? entry->pages : max_pages;
 145
 146                 for (i = 0; i < pages; i++) {
 147                         if (!entry->busaddr[i])
 148                                 break;
 149                         pci_unmap_page(dev->pdev, entry->busaddr[i],
 150                                        PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
 151                 }
 152 #endif
 153                 if (gart_info->gart_table_location == DRM_ATI_GART_MAIN)
 154                         gart_info->bus_addr = 0;
 155         }
 156 }
 157
 158 /* R600 has page table setup */
 159 int r600_page_table_init(struct drm_device *dev)
 160 {
 161         drm_radeon_private_t *dev_priv = dev->dev_private;
 162         struct drm_ati_pcigart_info *gart_info = &dev_priv->gart_info;
 163         struct drm_local_map *map = &gart_info->mapping;
 164         struct drm_sg_mem *entry = dev->sg;
 165         int ret = 0;
 166         int i, j;
 167         int pages;
 168         u64 page_base;
 169         dma_addr_t entry_addr;
 170         int max_ati_pages, max_real_pages, gart_idx;
 171
 172         /* okay page table is available - lets rock */
 173         max_ati_pages = (gart_info->table_size / sizeof(u64));
 174         max_real_pages = max_ati_pages / (PAGE_SIZE / ATI_PCIGART_PAGE_SIZE);
 175
 176         pages = (entry->pages <= max_real_pages) ?
 177                 entry->pages : max_real_pages;
 178
 179         memset_io((void __iomem *)map->handle, 0, max_ati_pages * sizeof(u64));
 180
 181         gart_idx = 0;
 182         for (i = 0; i < pages; i++) {
 183 #ifdef __linux__
 184                 entry->busaddr[i] = pci_map_page(dev->pdev,
 185                                                  entry->pagelist[i], 0,
 186                                                  PAGE_SIZE,
 187                                                  PCI_DMA_BIDIRECTIONAL);
 188                 if (pci_dma_mapping_error(dev->pdev, entry->busaddr[i])) {
 189                         DRM_ERROR("unable to map PCIGART pages!\n");
 190                         r600_page_table_cleanup(dev, gart_info);
 191                         goto done;
 192                 }
 193 #endif
 194                 entry_addr = entry->busaddr[i];
 195                 for (j = 0; j < (PAGE_SIZE / ATI_PCIGART_PAGE_SIZE); j++) {
 196                         page_base = (u64) entry_addr & ATI_PCIGART_PAGE_MASK;
 197                         page_base |= R600_PTE_VALID | R600_PTE_SYSTEM | R600_PTE_SNOOPED;
 198                         page_base |= R600_PTE_READABLE | R600_PTE_WRITEABLE;
 199
 200                         DRM_WRITE64(map, gart_idx * sizeof(u64), page_base);
 201
 202                         gart_idx++;
 203
 204                         if ((i % 128) == 0)
 205                                 DRM_DEBUG("page entry %d: 0x%016llx\n",
 206                                     i, (unsigned long long)page_base);
 207                         entry_addr += ATI_PCIGART_PAGE_SIZE;
 208                 }
 209         }
 210         ret = 1;
 211 #ifdef __linux__
 212 done:
 213 #endif
 214         return ret;
 215 }
 216
 217 static void r600_vm_flush_gart_range(struct drm_device *dev)
 218 {
 219         drm_radeon_private_t *dev_priv = dev->dev_private;
 220         u32 resp, countdown = 1000;
 221         RADEON_WRITE(R600_VM_CONTEXT0_INVALIDATION_LOW_ADDR, dev_priv->gart_vm_start >> 12);
 222         RADEON_WRITE(R600_VM_CONTEXT0_INVALIDATION_HIGH_ADDR, (dev_priv->gart_vm_start + dev_priv->gart_size - 1) >> 12);
 223         RADEON_WRITE(R600_VM_CONTEXT0_REQUEST_RESPONSE, 2);
 224
 225         do {
 226                 resp = RADEON_READ(R600_VM_CONTEXT0_REQUEST_RESPONSE);
 227                 countdown--;
 228                 DRM_UDELAY(1);
 229         } while (((resp & 0xf0) == 0) && countdown);
 230 }
 231
 232 static void r600_vm_init(struct drm_device *dev)
 233 {
 234         drm_radeon_private_t *dev_priv = dev->dev_private;
 235         /* initialise the VM to use the page table we constructed up there */
 236         u32 vm_c0, i;
 237         u32 mc_rd_a;
 238         u32 vm_l2_cntl, vm_l2_cntl3;
 239         /* okay set up the PCIE aperture type thingo */
 240         RADEON_WRITE(R600_MC_VM_SYSTEM_APERTURE_LOW_ADDR, dev_priv->gart_vm_start >> 12);
 241         RADEON_WRITE(R600_MC_VM_SYSTEM_APERTURE_HIGH_ADDR, (dev_priv->gart_vm_start + dev_priv->gart_size - 1) >> 12);
 242         RADEON_WRITE(R600_MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, 0);
 243
 244         /* setup MC RD a */
 245         mc_rd_a = R600_MCD_L1_TLB | R600_MCD_L1_FRAG_PROC | R600_MCD_SYSTEM_ACCESS_MODE_IN_SYS |
 246                 R600_MCD_SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU | R600_MCD_EFFECTIVE_L1_TLB_SIZE(5) |
 247                 R600_MCD_EFFECTIVE_L1_QUEUE_SIZE(5) | R600_MCD_WAIT_L2_QUERY;
 248
 249         RADEON_WRITE(R600_MCD_RD_A_CNTL, mc_rd_a);
 250         RADEON_WRITE(R600_MCD_RD_B_CNTL, mc_rd_a);
 251
 252         RADEON_WRITE(R600_MCD_WR_A_CNTL, mc_rd_a);
 253         RADEON_WRITE(R600_MCD_WR_B_CNTL, mc_rd_a);
 254
 255         RADEON_WRITE(R600_MCD_RD_GFX_CNTL, mc_rd_a);
 256         RADEON_WRITE(R600_MCD_WR_GFX_CNTL, mc_rd_a);
 257
 258         RADEON_WRITE(R600_MCD_RD_SYS_CNTL, mc_rd_a);
 259         RADEON_WRITE(R600_MCD_WR_SYS_CNTL, mc_rd_a);
 260
 261         RADEON_WRITE(R600_MCD_RD_HDP_CNTL, mc_rd_a | R600_MCD_L1_STRICT_ORDERING);
 262         RADEON_WRITE(R600_MCD_WR_HDP_CNTL, mc_rd_a /*| R600_MCD_L1_STRICT_ORDERING*/);
 263
 264         RADEON_WRITE(R600_MCD_RD_PDMA_CNTL, mc_rd_a);
 265         RADEON_WRITE(R600_MCD_WR_PDMA_CNTL, mc_rd_a);
 266
 267         RADEON_WRITE(R600_MCD_RD_SEM_CNTL, mc_rd_a | R600_MCD_SEMAPHORE_MODE);
 268         RADEON_WRITE(R600_MCD_WR_SEM_CNTL, mc_rd_a);
 269
 270         vm_l2_cntl = R600_VM_L2_CACHE_EN | R600_VM_L2_FRAG_PROC | R600_VM_ENABLE_PTE_CACHE_LRU_W;
 271         vm_l2_cntl |= R600_VM_L2_CNTL_QUEUE_SIZE(7);
 272         RADEON_WRITE(R600_VM_L2_CNTL, vm_l2_cntl);
 273
 274         RADEON_WRITE(R600_VM_L2_CNTL2, 0);
 275         vm_l2_cntl3 = (R600_VM_L2_CNTL3_BANK_SELECT_0(0) |
 276                        R600_VM_L2_CNTL3_BANK_SELECT_1(1) |
 277                        R600_VM_L2_CNTL3_CACHE_UPDATE_MODE(2));
 278         RADEON_WRITE(R600_VM_L2_CNTL3, vm_l2_cntl3);
 279
 280         vm_c0 = R600_VM_ENABLE_CONTEXT | R600_VM_PAGE_TABLE_DEPTH_FLAT;
 281
 282         RADEON_WRITE(R600_VM_CONTEXT0_CNTL, vm_c0);
 283
 284         vm_c0 &= ~R600_VM_ENABLE_CONTEXT;
 285
 286         /* disable all other contexts */
 287         for (i = 1; i < 8; i++)
 288                 RADEON_WRITE(R600_VM_CONTEXT0_CNTL + (i * 4), vm_c0);
 289
 290         RADEON_WRITE(R600_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, dev_priv->gart_info.bus_addr >> 12);
 291         RADEON_WRITE(R600_VM_CONTEXT0_PAGE_TABLE_START_ADDR, dev_priv->gart_vm_start >> 12);
 292         RADEON_WRITE(R600_VM_CONTEXT0_PAGE_TABLE_END_ADDR, (dev_priv->gart_vm_start + dev_priv->gart_size - 1) >> 12);
 293
 294         r600_vm_flush_gart_range(dev);
 295 }
 296
 297 static int r600_cp_init_microcode(drm_radeon_private_t *dev_priv)
 298 {
 299         const char *chip_name;
 300         size_t pfp_req_size, me_req_size;
 301         char fw_name[30];
 302         int err;
 303
 304         switch (dev_priv->flags & RADEON_FAMILY_MASK) {
 305         case CHIP_R600:  chip_name = "R600";  break;
 306         case CHIP_RV610: chip_name = "RV610"; break;
 307         case CHIP_RV630: chip_name = "RV630"; break;
 308         case CHIP_RV620: chip_name = "RV620"; break;
 309         case CHIP_RV635: chip_name = "RV635"; break;
 310         case CHIP_RV670: chip_name = "RV670"; break;
 311         case CHIP_RS780:
 312         case CHIP_RS880: chip_name = "RS780"; break;
 313         case CHIP_RV770: chip_name = "RV770"; break;
 314         case CHIP_RV730:
 315         case CHIP_RV740: chip_name = "RV730"; break;
 316         case CHIP_RV710: chip_name = "RV710"; break;
 317         default:         panic("%s: Unsupported family %d", __func__, dev_priv->flags & RADEON_FAMILY_MASK);
 318         }
 319
 320         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV770) {
 321                 pfp_req_size = R700_PFP_UCODE_SIZE * 4;
 322                 me_req_size = R700_PM4_UCODE_SIZE * 4;
 323         } else {
 324                 pfp_req_size = PFP_UCODE_SIZE * 4;
 325                 me_req_size = PM4_UCODE_SIZE * 12;
 326         }
 327
 328         DRM_INFO("Loading %s CP Microcode\n", chip_name);
 329         err = 0;
 330
 331         snprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_pfp", chip_name);
 332         dev_priv->pfp_fw = firmware_get(fw_name);
 333         if (dev_priv->pfp_fw == NULL) {
 334                 err = -ENOENT;
 335                 goto out;
 336         }
 337         if (dev_priv->pfp_fw->datasize != pfp_req_size) {
 338                 DRM_ERROR(
 339                        "r600_cp: Bogus length %zu in firmware \"%s\"\n",
 340                        dev_priv->pfp_fw->datasize, fw_name);
 341                 err = -EINVAL;
 342                 goto out;
 343         }
 344
 345         snprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_me", chip_name);
 346         dev_priv->me_fw = firmware_get(fw_name);
 347         if (dev_priv->me_fw == NULL) {
 348                 err = -ENOENT;
 349                 goto out;
 350         }
 351         if (dev_priv->me_fw->datasize != me_req_size) {
 352                 DRM_ERROR(
 353                        "r600_cp: Bogus length %zu in firmware \"%s\"\n",
 354                        dev_priv->me_fw->datasize, fw_name);
 355                 err = -EINVAL;
 356         }
 357 out:
 358         if (err) {
 359                 if (err != -EINVAL)
 360                         DRM_ERROR(
 361                                "r600_cp: Failed to load firmware \"%s\"\n",
 362                                fw_name);
 363                 if (dev_priv->pfp_fw != NULL) {
 364                         firmware_put(dev_priv->pfp_fw, FIRMWARE_UNLOAD);
 365                         dev_priv->pfp_fw = NULL;
 366                 }
 367                 if (dev_priv->me_fw != NULL) {
 368                         firmware_put(dev_priv->me_fw, FIRMWARE_UNLOAD);
 369                         dev_priv->me_fw = NULL;
 370                 }
 371         }
 372         return err;
 373 }
 374
 375 static void r600_cp_load_microcode(drm_radeon_private_t *dev_priv)
 376 {
 377         const __be32 *fw_data;
 378         int i;
 379
 380         if (!dev_priv->me_fw || !dev_priv->pfp_fw)
 381                 return;
 382
 383         r600_do_cp_stop(dev_priv);
 384
 385         RADEON_WRITE(R600_CP_RB_CNTL,
 386 #ifdef __BIG_ENDIAN
 387                      R600_BUF_SWAP_32BIT |
 388 #endif
 389                      R600_RB_NO_UPDATE |
 390                      R600_RB_BLKSZ(15) |
 391                      R600_RB_BUFSZ(3));
 392
 393         RADEON_WRITE(R600_GRBM_SOFT_RESET, R600_SOFT_RESET_CP);
 394         RADEON_READ(R600_GRBM_SOFT_RESET);
 395         DRM_MDELAY(15);
 396         RADEON_WRITE(R600_GRBM_SOFT_RESET, 0);
 397
 398         fw_data = (const __be32 *)dev_priv->me_fw->data;
 399         RADEON_WRITE(R600_CP_ME_RAM_WADDR, 0);
 400         for (i = 0; i < PM4_UCODE_SIZE * 3; i++)
 401                 RADEON_WRITE(R600_CP_ME_RAM_DATA,
 402                              be32_to_cpup(fw_data++));
 403
 404         fw_data = (const __be32 *)dev_priv->pfp_fw->data;
 405         RADEON_WRITE(R600_CP_PFP_UCODE_ADDR, 0);
 406         for (i = 0; i < PFP_UCODE_SIZE; i++)
 407                 RADEON_WRITE(R600_CP_PFP_UCODE_DATA,
 408                              be32_to_cpup(fw_data++));
 409
 410         RADEON_WRITE(R600_CP_PFP_UCODE_ADDR, 0);
 411         RADEON_WRITE(R600_CP_ME_RAM_WADDR, 0);
 412         RADEON_WRITE(R600_CP_ME_RAM_RADDR, 0);
 413
 414 }
 415
 416 static void r700_vm_init(struct drm_device *dev)
 417 {
 418         drm_radeon_private_t *dev_priv = dev->dev_private;
 419         /* initialise the VM to use the page table we constructed up there */
 420         u32 vm_c0, i;
 421         u32 mc_vm_md_l1;
 422         u32 vm_l2_cntl, vm_l2_cntl3;
 423         /* okay set up the PCIE aperture type thingo */
 424         RADEON_WRITE(R700_MC_VM_SYSTEM_APERTURE_LOW_ADDR, dev_priv->gart_vm_start >> 12);
 425         RADEON_WRITE(R700_MC_VM_SYSTEM_APERTURE_HIGH_ADDR, (dev_priv->gart_vm_start + dev_priv->gart_size - 1) >> 12);
 426         RADEON_WRITE(R700_MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, 0);
 427
 428         mc_vm_md_l1 = R700_ENABLE_L1_TLB |
 429             R700_ENABLE_L1_FRAGMENT_PROCESSING |
 430             R700_SYSTEM_ACCESS_MODE_IN_SYS |
 431             R700_SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
 432             R700_EFFECTIVE_L1_TLB_SIZE(5) |
 433             R700_EFFECTIVE_L1_QUEUE_SIZE(5);
 434
 435         RADEON_WRITE(R700_MC_VM_MD_L1_TLB0_CNTL, mc_vm_md_l1);
 436         RADEON_WRITE(R700_MC_VM_MD_L1_TLB1_CNTL, mc_vm_md_l1);
 437         RADEON_WRITE(R700_MC_VM_MD_L1_TLB2_CNTL, mc_vm_md_l1);
 438         RADEON_WRITE(R700_MC_VM_MB_L1_TLB0_CNTL, mc_vm_md_l1);
 439         RADEON_WRITE(R700_MC_VM_MB_L1_TLB1_CNTL, mc_vm_md_l1);
 440         RADEON_WRITE(R700_MC_VM_MB_L1_TLB2_CNTL, mc_vm_md_l1);
 441         RADEON_WRITE(R700_MC_VM_MB_L1_TLB3_CNTL, mc_vm_md_l1);
 442
 443         vm_l2_cntl = R600_VM_L2_CACHE_EN | R600_VM_L2_FRAG_PROC | R600_VM_ENABLE_PTE_CACHE_LRU_W;
 444         vm_l2_cntl |= R700_VM_L2_CNTL_QUEUE_SIZE(7);
 445         RADEON_WRITE(R600_VM_L2_CNTL, vm_l2_cntl);
 446
 447         RADEON_WRITE(R600_VM_L2_CNTL2, 0);
 448         vm_l2_cntl3 = R700_VM_L2_CNTL3_BANK_SELECT(0) | R700_VM_L2_CNTL3_CACHE_UPDATE_MODE(2);
 449         RADEON_WRITE(R600_VM_L2_CNTL3, vm_l2_cntl3);
 450
 451         vm_c0 = R600_VM_ENABLE_CONTEXT | R600_VM_PAGE_TABLE_DEPTH_FLAT;
 452
 453         RADEON_WRITE(R600_VM_CONTEXT0_CNTL, vm_c0);
 454
 455         vm_c0 &= ~R600_VM_ENABLE_CONTEXT;
 456
 457         /* disable all other contexts */
 458         for (i = 1; i < 8; i++)
 459                 RADEON_WRITE(R600_VM_CONTEXT0_CNTL + (i * 4), vm_c0);
 460
 461         RADEON_WRITE(R700_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, dev_priv->gart_info.bus_addr >> 12);
 462         RADEON_WRITE(R700_VM_CONTEXT0_PAGE_TABLE_START_ADDR, dev_priv->gart_vm_start >> 12);
 463         RADEON_WRITE(R700_VM_CONTEXT0_PAGE_TABLE_END_ADDR, (dev_priv->gart_vm_start + dev_priv->gart_size - 1) >> 12);
 464
 465         r600_vm_flush_gart_range(dev);
 466 }
 467
 468 static void r700_cp_load_microcode(drm_radeon_private_t *dev_priv)
 469 {
 470         const __be32 *fw_data;
 471         int i;
 472
 473         if (!dev_priv->me_fw || !dev_priv->pfp_fw)
 474                 return;
 475
 476         r600_do_cp_stop(dev_priv);
 477
 478         RADEON_WRITE(R600_CP_RB_CNTL,
 479 #ifdef __BIG_ENDIAN
 480                      R600_BUF_SWAP_32BIT |
 481 #endif
 482                      R600_RB_NO_UPDATE |
 483                      R600_RB_BLKSZ(15) |
 484                      R600_RB_BUFSZ(3));
 485
 486         RADEON_WRITE(R600_GRBM_SOFT_RESET, R600_SOFT_RESET_CP);
 487         RADEON_READ(R600_GRBM_SOFT_RESET);
 488         DRM_MDELAY(15);
 489         RADEON_WRITE(R600_GRBM_SOFT_RESET, 0);
 490
 491         fw_data = (const __be32 *)dev_priv->pfp_fw->data;
 492         RADEON_WRITE(R600_CP_PFP_UCODE_ADDR, 0);
 493         for (i = 0; i < R700_PFP_UCODE_SIZE; i++)
 494                 RADEON_WRITE(R600_CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
 495         RADEON_WRITE(R600_CP_PFP_UCODE_ADDR, 0);
 496
 497         fw_data = (const __be32 *)dev_priv->me_fw->data;
 498         RADEON_WRITE(R600_CP_ME_RAM_WADDR, 0);
 499         for (i = 0; i < R700_PM4_UCODE_SIZE; i++)
 500                 RADEON_WRITE(R600_CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
 501         RADEON_WRITE(R600_CP_ME_RAM_WADDR, 0);
 502
 503         RADEON_WRITE(R600_CP_PFP_UCODE_ADDR, 0);
 504         RADEON_WRITE(R600_CP_ME_RAM_WADDR, 0);
 505         RADEON_WRITE(R600_CP_ME_RAM_RADDR, 0);
 506
 507 }
 508
 509 static void r600_test_writeback(drm_radeon_private_t *dev_priv)
 510 {
 511         u32 tmp;
 512
 513         /* Start with assuming that writeback doesn't work */
 514         dev_priv->writeback_works = 0;
 515
 516         /* Writeback doesn't seem to work everywhere, test it here and possibly
 517          * enable it if it appears to work
 518          */
 519         radeon_write_ring_rptr(dev_priv, R600_SCRATCHOFF(1), 0);
 520
 521         RADEON_WRITE(R600_SCRATCH_REG1, 0xdeadbeef);
 522
 523         for (tmp = 0; tmp < dev_priv->usec_timeout; tmp++) {
 524                 u32 val;
 525
 526                 val = radeon_read_ring_rptr(dev_priv, R600_SCRATCHOFF(1));
 527                 if (val == 0xdeadbeef)
 528                         break;
 529                 DRM_UDELAY(1);
 530         }
 531
 532         if (tmp < dev_priv->usec_timeout) {
 533                 dev_priv->writeback_works = 1;
 534                 DRM_INFO("writeback test succeeded in %d usecs\n", tmp);
 535         } else {
 536                 dev_priv->writeback_works = 0;
 537                 DRM_INFO("writeback test failed\n");
 538         }
 539         if (radeon_no_wb == 1) {
 540                 dev_priv->writeback_works = 0;
 541                 DRM_INFO("writeback forced off\n");
 542         }
 543
 544         if (!dev_priv->writeback_works) {
 545                 /* Disable writeback to avoid unnecessary bus master transfer */
 546                 RADEON_WRITE(R600_CP_RB_CNTL,
 547 #ifdef __BIG_ENDIAN
 548                              R600_BUF_SWAP_32BIT |
 549 #endif
 550                              RADEON_READ(R600_CP_RB_CNTL) |
 551                              R600_RB_NO_UPDATE);
 552                 RADEON_WRITE(R600_SCRATCH_UMSK, 0);
 553         }
 554 }
 555
 556 int r600_do_engine_reset(struct drm_device *dev)
 557 {
 558         drm_radeon_private_t *dev_priv = dev->dev_private;
 559         u32 cp_ptr, cp_me_cntl, cp_rb_cntl;
 560
 561         DRM_INFO("Resetting GPU\n");
 562
 563         cp_ptr = RADEON_READ(R600_CP_RB_WPTR);
 564         cp_me_cntl = RADEON_READ(R600_CP_ME_CNTL);
 565         RADEON_WRITE(R600_CP_ME_CNTL, R600_CP_ME_HALT);
 566
 567         RADEON_WRITE(R600_GRBM_SOFT_RESET, 0x7fff);
 568         RADEON_READ(R600_GRBM_SOFT_RESET);
 569         DRM_UDELAY(50);
 570         RADEON_WRITE(R600_GRBM_SOFT_RESET, 0);
 571         RADEON_READ(R600_GRBM_SOFT_RESET);
 572
 573         RADEON_WRITE(R600_CP_RB_WPTR_DELAY, 0);
 574         cp_rb_cntl = RADEON_READ(R600_CP_RB_CNTL);
 575         RADEON_WRITE(R600_CP_RB_CNTL,
 576 #ifdef __BIG_ENDIAN
 577                      R600_BUF_SWAP_32BIT |
 578 #endif
 579                      R600_RB_RPTR_WR_ENA);
 580
 581         RADEON_WRITE(R600_CP_RB_RPTR_WR, cp_ptr);
 582         RADEON_WRITE(R600_CP_RB_WPTR, cp_ptr);
 583         RADEON_WRITE(R600_CP_RB_CNTL, cp_rb_cntl);
 584         RADEON_WRITE(R600_CP_ME_CNTL, cp_me_cntl);
 585
 586         /* Reset the CP ring */
 587         r600_do_cp_reset(dev_priv);
 588
 589         /* The CP is no longer running after an engine reset */
 590         dev_priv->cp_running = 0;
 591
 592         /* Reset any pending vertex, indirect buffers */
 593         radeon_freelist_reset(dev);
 594
 595         return 0;
 596
 597 }
 598
 599 static u32 r600_get_tile_pipe_to_backend_map(u32 num_tile_pipes,
 600                                              u32 num_backends,
 601                                              u32 backend_disable_mask)
 602 {
 603         u32 backend_map = 0;
 604         u32 enabled_backends_mask;
 605         u32 enabled_backends_count;
 606         u32 cur_pipe;
 607         u32 swizzle_pipe[R6XX_MAX_PIPES];
 608         u32 cur_backend;
 609         u32 i;
 610
 611         if (num_tile_pipes > R6XX_MAX_PIPES)
 612                 num_tile_pipes = R6XX_MAX_PIPES;
 613         if (num_tile_pipes < 1)
 614                 num_tile_pipes = 1;
 615         if (num_backends > R6XX_MAX_BACKENDS)
 616                 num_backends = R6XX_MAX_BACKENDS;
 617         if (num_backends < 1)
 618                 num_backends = 1;
 619
 620         enabled_backends_mask = 0;
 621         enabled_backends_count = 0;
 622         for (i = 0; i < R6XX_MAX_BACKENDS; ++i) {
 623                 if (((backend_disable_mask >> i) & 1) == 0) {
 624                         enabled_backends_mask |= (1 << i);
 625                         ++enabled_backends_count;
 626                 }
 627                 if (enabled_backends_count == num_backends)
 628                         break;
 629         }
 630
 631         if (enabled_backends_count == 0) {
 632                 enabled_backends_mask = 1;
 633                 enabled_backends_count = 1;
 634         }
 635
 636         if (enabled_backends_count != num_backends)
 637                 num_backends = enabled_backends_count;
 638
 639         memset((uint8_t *)&swizzle_pipe[0], 0, sizeof(u32) * R6XX_MAX_PIPES);
 640         switch (num_tile_pipes) {
 641         case 1:
 642                 swizzle_pipe[0] = 0;
 643                 break;
 644         case 2:
 645                 swizzle_pipe[0] = 0;
 646                 swizzle_pipe[1] = 1;
 647                 break;
 648         case 3:
 649                 swizzle_pipe[0] = 0;
 650                 swizzle_pipe[1] = 1;
 651                 swizzle_pipe[2] = 2;
 652                 break;
 653         case 4:
 654                 swizzle_pipe[0] = 0;
 655                 swizzle_pipe[1] = 1;
 656                 swizzle_pipe[2] = 2;
 657                 swizzle_pipe[3] = 3;
 658                 break;
 659         case 5:
 660                 swizzle_pipe[0] = 0;
 661                 swizzle_pipe[1] = 1;
 662                 swizzle_pipe[2] = 2;
 663                 swizzle_pipe[3] = 3;
 664                 swizzle_pipe[4] = 4;
 665                 break;
 666         case 6:
 667                 swizzle_pipe[0] = 0;
 668                 swizzle_pipe[1] = 2;
 669                 swizzle_pipe[2] = 4;
 670                 swizzle_pipe[3] = 5;
 671                 swizzle_pipe[4] = 1;
 672                 swizzle_pipe[5] = 3;
 673                 break;
 674         case 7:
 675                 swizzle_pipe[0] = 0;
 676                 swizzle_pipe[1] = 2;
 677                 swizzle_pipe[2] = 4;
 678                 swizzle_pipe[3] = 6;
 679                 swizzle_pipe[4] = 1;
 680                 swizzle_pipe[5] = 3;
 681                 swizzle_pipe[6] = 5;
 682                 break;
 683         case 8:
 684                 swizzle_pipe[0] = 0;
 685                 swizzle_pipe[1] = 2;
 686                 swizzle_pipe[2] = 4;
 687                 swizzle_pipe[3] = 6;
 688                 swizzle_pipe[4] = 1;
 689                 swizzle_pipe[5] = 3;
 690                 swizzle_pipe[6] = 5;
 691                 swizzle_pipe[7] = 7;
 692                 break;
 693         }
 694
 695         cur_backend = 0;
 696         for (cur_pipe = 0; cur_pipe < num_tile_pipes; ++cur_pipe) {
 697                 while (((1 << cur_backend) & enabled_backends_mask) == 0)
 698                         cur_backend = (cur_backend + 1) % R6XX_MAX_BACKENDS;
 699
 700                 backend_map |= (u32)(((cur_backend & 3) << (swizzle_pipe[cur_pipe] * 2)));
 701
 702                 cur_backend = (cur_backend + 1) % R6XX_MAX_BACKENDS;
 703         }
 704
 705         return backend_map;
 706 }
 707
 708 static int r600_count_pipe_bits(uint32_t val)
 709 {
 710         return hweight32(val);
 711 }
 712
 713 static void r600_gfx_init(struct drm_device *dev,
 714                           drm_radeon_private_t *dev_priv)
 715 {
 716         int i, j, num_qd_pipes;
 717         u32 sx_debug_1;
 718         u32 tc_cntl;
 719         u32 arb_pop;
 720         u32 num_gs_verts_per_thread;
 721         u32 vgt_gs_per_es;
 722         u32 gs_prim_buffer_depth = 0;
 723         u32 sq_ms_fifo_sizes;
 724         u32 sq_config;
 725         u32 sq_gpr_resource_mgmt_1 = 0;
 726         u32 sq_gpr_resource_mgmt_2 = 0;
 727         u32 sq_thread_resource_mgmt = 0;
 728         u32 sq_stack_resource_mgmt_1 = 0;
 729         u32 sq_stack_resource_mgmt_2 = 0;
 730         u32 hdp_host_path_cntl;
 731         u32 backend_map;
 732         u32 gb_tiling_config = 0;
 733         u32 cc_rb_backend_disable;
 734         u32 cc_gc_shader_pipe_config;
 735         u32 ramcfg;
 736
 737         /* setup chip specs */
 738         switch (dev_priv->flags & RADEON_FAMILY_MASK) {
 739         case CHIP_R600:
 740                 dev_priv->r600_max_pipes = 4;
 741                 dev_priv->r600_max_tile_pipes = 8;
 742                 dev_priv->r600_max_simds = 4;
 743                 dev_priv->r600_max_backends = 4;
 744                 dev_priv->r600_max_gprs = 256;
 745                 dev_priv->r600_max_threads = 192;
 746                 dev_priv->r600_max_stack_entries = 256;
 747                 dev_priv->r600_max_hw_contexts = 8;
 748                 dev_priv->r600_max_gs_threads = 16;
 749                 dev_priv->r600_sx_max_export_size = 128;
 750                 dev_priv->r600_sx_max_export_pos_size = 16;
 751                 dev_priv->r600_sx_max_export_smx_size = 128;
 752                 dev_priv->r600_sq_num_cf_insts = 2;
 753                 break;
 754         case CHIP_RV630:
 755         case CHIP_RV635:
 756                 dev_priv->r600_max_pipes = 2;
 757                 dev_priv->r600_max_tile_pipes = 2;
 758                 dev_priv->r600_max_simds = 3;
 759                 dev_priv->r600_max_backends = 1;
 760                 dev_priv->r600_max_gprs = 128;
 761                 dev_priv->r600_max_threads = 192;
 762                 dev_priv->r600_max_stack_entries = 128;
 763                 dev_priv->r600_max_hw_contexts = 8;
 764                 dev_priv->r600_max_gs_threads = 4;
 765                 dev_priv->r600_sx_max_export_size = 128;
 766                 dev_priv->r600_sx_max_export_pos_size = 16;
 767                 dev_priv->r600_sx_max_export_smx_size = 128;
 768                 dev_priv->r600_sq_num_cf_insts = 2;
 769                 break;
 770         case CHIP_RV610:
 771         case CHIP_RS780:
 772         case CHIP_RS880:
 773         case CHIP_RV620:
 774                 dev_priv->r600_max_pipes = 1;
 775                 dev_priv->r600_max_tile_pipes = 1;
 776                 dev_priv->r600_max_simds = 2;
 777                 dev_priv->r600_max_backends = 1;
 778                 dev_priv->r600_max_gprs = 128;
 779                 dev_priv->r600_max_threads = 192;
 780                 dev_priv->r600_max_stack_entries = 128;
 781                 dev_priv->r600_max_hw_contexts = 4;
 782                 dev_priv->r600_max_gs_threads = 4;
 783                 dev_priv->r600_sx_max_export_size = 128;
 784                 dev_priv->r600_sx_max_export_pos_size = 16;
 785                 dev_priv->r600_sx_max_export_smx_size = 128;
 786                 dev_priv->r600_sq_num_cf_insts = 1;
 787                 break;
 788         case CHIP_RV670:
 789                 dev_priv->r600_max_pipes = 4;
 790                 dev_priv->r600_max_tile_pipes = 4;
 791                 dev_priv->r600_max_simds = 4;
 792                 dev_priv->r600_max_backends = 4;
 793                 dev_priv->r600_max_gprs = 192;
 794                 dev_priv->r600_max_threads = 192;
 795                 dev_priv->r600_max_stack_entries = 256;
 796                 dev_priv->r600_max_hw_contexts = 8;
 797                 dev_priv->r600_max_gs_threads = 16;
 798                 dev_priv->r600_sx_max_export_size = 128;
 799                 dev_priv->r600_sx_max_export_pos_size = 16;
 800                 dev_priv->r600_sx_max_export_smx_size = 128;
 801                 dev_priv->r600_sq_num_cf_insts = 2;
 802                 break;
 803         default:
 804                 break;
 805         }
 806
 807         /* Initialize HDP */
 808         j = 0;
 809         for (i = 0; i < 32; i++) {
 810                 RADEON_WRITE((0x2c14 + j), 0x00000000);
 811                 RADEON_WRITE((0x2c18 + j), 0x00000000);
 812                 RADEON_WRITE((0x2c1c + j), 0x00000000);
 813                 RADEON_WRITE((0x2c20 + j), 0x00000000);
 814                 RADEON_WRITE((0x2c24 + j), 0x00000000);
 815                 j += 0x18;
 816         }
 817
 818         RADEON_WRITE(R600_GRBM_CNTL, R600_GRBM_READ_TIMEOUT(0xff));
 819
 820         /* setup tiling, simd, pipe config */
 821         ramcfg = RADEON_READ(R600_RAMCFG);
 822
 823         switch (dev_priv->r600_max_tile_pipes) {
 824         case 1:
 825                 gb_tiling_config |= R600_PIPE_TILING(0);
 826                 break;
 827         case 2:
 828                 gb_tiling_config |= R600_PIPE_TILING(1);
 829                 break;
 830         case 4:
 831                 gb_tiling_config |= R600_PIPE_TILING(2);
 832                 break;
 833         case 8:
 834                 gb_tiling_config |= R600_PIPE_TILING(3);
 835                 break;
 836         default:
 837                 break;
 838         }
 839
 840         gb_tiling_config |= R600_BANK_TILING((ramcfg >> R600_NOOFBANK_SHIFT) & R600_NOOFBANK_MASK);
 841
 842         gb_tiling_config |= R600_GROUP_SIZE(0);
 843
 844         if (((ramcfg >> R600_NOOFROWS_SHIFT) & R600_NOOFROWS_MASK) > 3) {
 845                 gb_tiling_config |= R600_ROW_TILING(3);
 846                 gb_tiling_config |= R600_SAMPLE_SPLIT(3);
 847         } else {
 848                 gb_tiling_config |=
 849                         R600_ROW_TILING(((ramcfg >> R600_NOOFROWS_SHIFT) & R600_NOOFROWS_MASK));
 850                 gb_tiling_config |=
 851                         R600_SAMPLE_SPLIT(((ramcfg >> R600_NOOFROWS_SHIFT) & R600_NOOFROWS_MASK));
 852         }
 853
 854         gb_tiling_config |= R600_BANK_SWAPS(1);
 855
 856         cc_rb_backend_disable = RADEON_READ(R600_CC_RB_BACKEND_DISABLE) & 0x00ff0000;
 857         cc_rb_backend_disable |=
 858                 R600_BACKEND_DISABLE((R6XX_MAX_BACKENDS_MASK << dev_priv->r600_max_backends) & R6XX_MAX_BACKENDS_MASK);
 859
 860         cc_gc_shader_pipe_config = RADEON_READ(R600_CC_GC_SHADER_PIPE_CONFIG) & 0xffffff00;
 861         cc_gc_shader_pipe_config |=
 862                 R600_INACTIVE_QD_PIPES((R6XX_MAX_PIPES_MASK << dev_priv->r600_max_pipes) & R6XX_MAX_PIPES_MASK);
 863         cc_gc_shader_pipe_config |=
 864                 R600_INACTIVE_SIMDS((R6XX_MAX_SIMDS_MASK << dev_priv->r600_max_simds) & R6XX_MAX_SIMDS_MASK);
 865
 866         backend_map = r600_get_tile_pipe_to_backend_map(dev_priv->r600_max_tile_pipes,
 867                                                         (R6XX_MAX_BACKENDS -
 868                                                          r600_count_pipe_bits((cc_rb_backend_disable &
 869                                                                                R6XX_MAX_BACKENDS_MASK) >> 16)),
 870                                                         (cc_rb_backend_disable >> 16));
 871         gb_tiling_config |= R600_BACKEND_MAP(backend_map);
 872
 873         RADEON_WRITE(R600_GB_TILING_CONFIG,      gb_tiling_config);
 874         RADEON_WRITE(R600_DCP_TILING_CONFIG,    (gb_tiling_config & 0xffff));
 875         RADEON_WRITE(R600_HDP_TILING_CONFIG,    (gb_tiling_config & 0xffff));
 876         if (gb_tiling_config & 0xc0) {
 877                 dev_priv->r600_group_size = 512;
 878         } else {
 879                 dev_priv->r600_group_size = 256;
 880         }
 881         dev_priv->r600_npipes = 1 << ((gb_tiling_config >> 1) & 0x7);
 882         if (gb_tiling_config & 0x30) {
 883                 dev_priv->r600_nbanks = 8;
 884         } else {
 885                 dev_priv->r600_nbanks = 4;
 886         }
 887
 888         RADEON_WRITE(R600_CC_RB_BACKEND_DISABLE,      cc_rb_backend_disable);
 889         RADEON_WRITE(R600_CC_GC_SHADER_PIPE_CONFIG,   cc_gc_shader_pipe_config);
 890         RADEON_WRITE(R600_GC_USER_SHADER_PIPE_CONFIG, cc_gc_shader_pipe_config);
 891
 892         num_qd_pipes =
 893                 R6XX_MAX_PIPES - r600_count_pipe_bits((cc_gc_shader_pipe_config & R600_INACTIVE_QD_PIPES_MASK) >> 8);
 894         RADEON_WRITE(R600_VGT_OUT_DEALLOC_CNTL, (num_qd_pipes * 4) & R600_DEALLOC_DIST_MASK);
 895         RADEON_WRITE(R600_VGT_VERTEX_REUSE_BLOCK_CNTL, ((num_qd_pipes * 4) - 2) & R600_VTX_REUSE_DEPTH_MASK);
 896
 897         /* set HW defaults for 3D engine */
 898         RADEON_WRITE(R600_CP_QUEUE_THRESHOLDS, (R600_ROQ_IB1_START(0x16) |
 899                                                 R600_ROQ_IB2_START(0x2b)));
 900
 901         RADEON_WRITE(R600_CP_MEQ_THRESHOLDS, (R600_MEQ_END(0x40) |
 902                                               R600_ROQ_END(0x40)));
 903
 904         RADEON_WRITE(R600_TA_CNTL_AUX, (R600_DISABLE_CUBE_ANISO |
 905                                         R600_SYNC_GRADIENT |
 906                                         R600_SYNC_WALKER |
 907                                         R600_SYNC_ALIGNER));
 908
 909         if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV670)
 910                 RADEON_WRITE(R600_ARB_GDEC_RD_CNTL, 0x00000021);
 911
 912         sx_debug_1 = RADEON_READ(R600_SX_DEBUG_1);
 913         sx_debug_1 |= R600_SMX_EVENT_RELEASE;
 914         if (((dev_priv->flags & RADEON_FAMILY_MASK) > CHIP_R600))
 915                 sx_debug_1 |= R600_ENABLE_NEW_SMX_ADDRESS;
 916         RADEON_WRITE(R600_SX_DEBUG_1, sx_debug_1);
 917
 918         if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R600) ||
 919             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV630) ||
 920             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV610) ||
 921             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV620) ||
 922             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS780) ||
 923             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS880))
 924                 RADEON_WRITE(R600_DB_DEBUG, R600_PREZ_MUST_WAIT_FOR_POSTZ_DONE);
 925         else
 926                 RADEON_WRITE(R600_DB_DEBUG, 0);
 927
 928         RADEON_WRITE(R600_DB_WATERMARKS, (R600_DEPTH_FREE(4) |
 929                                           R600_DEPTH_FLUSH(16) |
 930                                           R600_DEPTH_PENDING_FREE(4) |
 931                                           R600_DEPTH_CACHELINE_FREE(16)));
 932         RADEON_WRITE(R600_PA_SC_MULTI_CHIP_CNTL, 0);
 933         RADEON_WRITE(R600_VGT_NUM_INSTANCES, 0);
 934
 935         RADEON_WRITE(R600_SPI_CONFIG_CNTL, R600_GPR_WRITE_PRIORITY(0));
 936         RADEON_WRITE(R600_SPI_CONFIG_CNTL_1, R600_VTX_DONE_DELAY(0));
 937
 938         sq_ms_fifo_sizes = RADEON_READ(R600_SQ_MS_FIFO_SIZES);
 939         if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV610) ||
 940             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV620) ||
 941             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS780) ||
 942             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS880)) {
 943                 sq_ms_fifo_sizes = (R600_CACHE_FIFO_SIZE(0xa) |
 944                                     R600_FETCH_FIFO_HIWATER(0xa) |
 945                                     R600_DONE_FIFO_HIWATER(0xe0) |
 946                                     R600_ALU_UPDATE_FIFO_HIWATER(0x8));
 947         } else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R600) ||
 948                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV630)) {
 949                 sq_ms_fifo_sizes &= ~R600_DONE_FIFO_HIWATER(0xff);
 950                 sq_ms_fifo_sizes |= R600_DONE_FIFO_HIWATER(0x4);
 951         }
 952         RADEON_WRITE(R600_SQ_MS_FIFO_SIZES, sq_ms_fifo_sizes);
 953
 954         /* SQ_CONFIG, SQ_GPR_RESOURCE_MGMT, SQ_THREAD_RESOURCE_MGMT, SQ_STACK_RESOURCE_MGMT
 955          * should be adjusted as needed by the 2D/3D drivers.  This just sets default values
 956          */
 957         sq_config = RADEON_READ(R600_SQ_CONFIG);
 958         sq_config &= ~(R600_PS_PRIO(3) |
 959                        R600_VS_PRIO(3) |
 960                        R600_GS_PRIO(3) |
 961                        R600_ES_PRIO(3));
 962         sq_config |= (R600_DX9_CONSTS |
 963                       R600_VC_ENABLE |
 964                       R600_PS_PRIO(0) |
 965                       R600_VS_PRIO(1) |
 966                       R600_GS_PRIO(2) |
 967                       R600_ES_PRIO(3));
 968
 969         if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R600) {
 970                 sq_gpr_resource_mgmt_1 = (R600_NUM_PS_GPRS(124) |
 971                                           R600_NUM_VS_GPRS(124) |
 972                                           R600_NUM_CLAUSE_TEMP_GPRS(4));
 973                 sq_gpr_resource_mgmt_2 = (R600_NUM_GS_GPRS(0) |
 974                                           R600_NUM_ES_GPRS(0));
 975                 sq_thread_resource_mgmt = (R600_NUM_PS_THREADS(136) |
 976                                            R600_NUM_VS_THREADS(48) |
 977                                            R600_NUM_GS_THREADS(4) |
 978                                            R600_NUM_ES_THREADS(4));
 979                 sq_stack_resource_mgmt_1 = (R600_NUM_PS_STACK_ENTRIES(128) |
 980                                             R600_NUM_VS_STACK_ENTRIES(128));
 981                 sq_stack_resource_mgmt_2 = (R600_NUM_GS_STACK_ENTRIES(0) |
 982                                             R600_NUM_ES_STACK_ENTRIES(0));
 983         } else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV610) ||
 984                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV620) ||
 985                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS780) ||
 986                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS880)) {
 987                 /* no vertex cache */
 988                 sq_config &= ~R600_VC_ENABLE;
 989
 990                 sq_gpr_resource_mgmt_1 = (R600_NUM_PS_GPRS(44) |
 991                                           R600_NUM_VS_GPRS(44) |
 992                                           R600_NUM_CLAUSE_TEMP_GPRS(2));
 993                 sq_gpr_resource_mgmt_2 = (R600_NUM_GS_GPRS(17) |
 994                                           R600_NUM_ES_GPRS(17));
 995                 sq_thread_resource_mgmt = (R600_NUM_PS_THREADS(79) |
 996                                            R600_NUM_VS_THREADS(78) |
 997                                            R600_NUM_GS_THREADS(4) |
 998                                            R600_NUM_ES_THREADS(31));
 999                 sq_stack_resource_mgmt_1 = (R600_NUM_PS_STACK_ENTRIES(40) |
1000                                             R600_NUM_VS_STACK_ENTRIES(40));
1001                 sq_stack_resource_mgmt_2 = (R600_NUM_GS_STACK_ENTRIES(32) |
1002                                             R600_NUM_ES_STACK_ENTRIES(16));
1003         } else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV630) ||
1004                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV635)) {
1005                 sq_gpr_resource_mgmt_1 = (R600_NUM_PS_GPRS(44) |
1006                                           R600_NUM_VS_GPRS(44) |
1007                                           R600_NUM_CLAUSE_TEMP_GPRS(2));
1008                 sq_gpr_resource_mgmt_2 = (R600_NUM_GS_GPRS(18) |
1009                                           R600_NUM_ES_GPRS(18));
1010                 sq_thread_resource_mgmt = (R600_NUM_PS_THREADS(79) |
1011                                            R600_NUM_VS_THREADS(78) |
1012                                            R600_NUM_GS_THREADS(4) |
1013                                            R600_NUM_ES_THREADS(31));
1014                 sq_stack_resource_mgmt_1 = (R600_NUM_PS_STACK_ENTRIES(40) |
1015                                             R600_NUM_VS_STACK_ENTRIES(40));
1016                 sq_stack_resource_mgmt_2 = (R600_NUM_GS_STACK_ENTRIES(32) |
1017                                             R600_NUM_ES_STACK_ENTRIES(16));
1018         } else if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV670) {
1019                 sq_gpr_resource_mgmt_1 = (R600_NUM_PS_GPRS(44) |
1020                                           R600_NUM_VS_GPRS(44) |
1021                                           R600_NUM_CLAUSE_TEMP_GPRS(2));
1022                 sq_gpr_resource_mgmt_2 = (R600_NUM_GS_GPRS(17) |
1023                                           R600_NUM_ES_GPRS(17));
1024                 sq_thread_resource_mgmt = (R600_NUM_PS_THREADS(79) |
1025                                            R600_NUM_VS_THREADS(78) |
1026                                            R600_NUM_GS_THREADS(4) |
1027                                            R600_NUM_ES_THREADS(31));
1028                 sq_stack_resource_mgmt_1 = (R600_NUM_PS_STACK_ENTRIES(64) |
1029                                             R600_NUM_VS_STACK_ENTRIES(64));
1030                 sq_stack_resource_mgmt_2 = (R600_NUM_GS_STACK_ENTRIES(64) |
1031                                             R600_NUM_ES_STACK_ENTRIES(64));
1032         }
1033
1034         RADEON_WRITE(R600_SQ_CONFIG, sq_config);
1035         RADEON_WRITE(R600_SQ_GPR_RESOURCE_MGMT_1,  sq_gpr_resource_mgmt_1);
1036         RADEON_WRITE(R600_SQ_GPR_RESOURCE_MGMT_2,  sq_gpr_resource_mgmt_2);
1037         RADEON_WRITE(R600_SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
1038         RADEON_WRITE(R600_SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
1039         RADEON_WRITE(R600_SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
1040
1041         if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV610) ||
1042             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV620) ||
1043             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS780) ||
1044             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS880))
1045                 RADEON_WRITE(R600_VGT_CACHE_INVALIDATION, R600_CACHE_INVALIDATION(R600_TC_ONLY));
1046         else
1047                 RADEON_WRITE(R600_VGT_CACHE_INVALIDATION, R600_CACHE_INVALIDATION(R600_VC_AND_TC));
1048
1049         RADEON_WRITE(R600_PA_SC_AA_SAMPLE_LOCS_2S, (R600_S0_X(0xc) |
1050                                                     R600_S0_Y(0x4) |
1051                                                     R600_S1_X(0x4) |
1052                                                     R600_S1_Y(0xc)));
1053         RADEON_WRITE(R600_PA_SC_AA_SAMPLE_LOCS_4S, (R600_S0_X(0xe) |
1054                                                     R600_S0_Y(0xe) |
1055                                                     R600_S1_X(0x2) |
1056                                                     R600_S1_Y(0x2) |
1057                                                     R600_S2_X(0xa) |
1058                                                     R600_S2_Y(0x6) |
1059                                                     R600_S3_X(0x6) |
1060                                                     R600_S3_Y(0xa)));
1061         RADEON_WRITE(R600_PA_SC_AA_SAMPLE_LOCS_8S_WD0, (R600_S0_X(0xe) |
1062                                                         R600_S0_Y(0xb) |
1063                                                         R600_S1_X(0x4) |
1064                                                         R600_S1_Y(0xc) |
1065                                                         R600_S2_X(0x1) |
1066                                                         R600_S2_Y(0x6) |
1067                                                         R600_S3_X(0xa) |
1068                                                         R600_S3_Y(0xe)));
1069         RADEON_WRITE(R600_PA_SC_AA_SAMPLE_LOCS_8S_WD1, (R600_S4_X(0x6) |
1070                                                         R600_S4_Y(0x1) |
1071                                                         R600_S5_X(0x0) |
1072                                                         R600_S5_Y(0x0) |
1073                                                         R600_S6_X(0xb) |
1074                                                         R600_S6_Y(0x4) |
1075                                                         R600_S7_X(0x7) |
1076                                                         R600_S7_Y(0x8)));
1077
1078
1079         switch (dev_priv->flags & RADEON_FAMILY_MASK) {
1080         case CHIP_R600:
1081         case CHIP_RV630:
1082         case CHIP_RV635:
1083                 gs_prim_buffer_depth = 0;
1084                 break;
1085         case CHIP_RV610:
1086         case CHIP_RS780:
1087         case CHIP_RS880:
1088         case CHIP_RV620:
1089                 gs_prim_buffer_depth = 32;
1090                 break;
1091         case CHIP_RV670:
1092                 gs_prim_buffer_depth = 128;
1093                 break;
1094         default:
1095                 break;
1096         }
1097
1098         num_gs_verts_per_thread = dev_priv->r600_max_pipes * 16;
1099         vgt_gs_per_es = gs_prim_buffer_depth + num_gs_verts_per_thread;
1100         /* Max value for this is 256 */
1101         if (vgt_gs_per_es > 256)
1102                 vgt_gs_per_es = 256;
1103
1104         RADEON_WRITE(R600_VGT_ES_PER_GS, 128);
1105         RADEON_WRITE(R600_VGT_GS_PER_ES, vgt_gs_per_es);
1106         RADEON_WRITE(R600_VGT_GS_PER_VS, 2);
1107         RADEON_WRITE(R600_VGT_GS_VERTEX_REUSE, 16);
1108
1109         /* more default values. 2D/3D driver should adjust as needed */
1110         RADEON_WRITE(R600_PA_SC_LINE_STIPPLE_STATE, 0);
1111         RADEON_WRITE(R600_VGT_STRMOUT_EN, 0);
1112         RADEON_WRITE(R600_SX_MISC, 0);
1113         RADEON_WRITE(R600_PA_SC_MODE_CNTL, 0);
1114         RADEON_WRITE(R600_PA_SC_AA_CONFIG, 0);
1115         RADEON_WRITE(R600_PA_SC_LINE_STIPPLE, 0);
1116         RADEON_WRITE(R600_SPI_INPUT_Z, 0);
1117         RADEON_WRITE(R600_SPI_PS_IN_CONTROL_0, R600_NUM_INTERP(2));
1118         RADEON_WRITE(R600_CB_COLOR7_FRAG, 0);
1119
1120         /* clear render buffer base addresses */
1121         RADEON_WRITE(R600_CB_COLOR0_BASE, 0);
1122         RADEON_WRITE(R600_CB_COLOR1_BASE, 0);
1123         RADEON_WRITE(R600_CB_COLOR2_BASE, 0);
1124         RADEON_WRITE(R600_CB_COLOR3_BASE, 0);
1125         RADEON_WRITE(R600_CB_COLOR4_BASE, 0);
1126         RADEON_WRITE(R600_CB_COLOR5_BASE, 0);
1127         RADEON_WRITE(R600_CB_COLOR6_BASE, 0);
1128         RADEON_WRITE(R600_CB_COLOR7_BASE, 0);
1129
1130         switch (dev_priv->flags & RADEON_FAMILY_MASK) {
1131         case CHIP_RV610:
1132         case CHIP_RS780:
1133         case CHIP_RS880:
1134         case CHIP_RV620:
1135                 tc_cntl = R600_TC_L2_SIZE(8);
1136                 break;
1137         case CHIP_RV630:
1138         case CHIP_RV635:
1139                 tc_cntl = R600_TC_L2_SIZE(4);
1140                 break;
1141         case CHIP_R600:
1142                 tc_cntl = R600_TC_L2_SIZE(0) | R600_L2_DISABLE_LATE_HIT;
1143                 break;
1144         default:
1145                 tc_cntl = R600_TC_L2_SIZE(0);
1146                 break;
1147         }
1148
1149         RADEON_WRITE(R600_TC_CNTL, tc_cntl);
1150
1151         hdp_host_path_cntl = RADEON_READ(R600_HDP_HOST_PATH_CNTL);
1152         RADEON_WRITE(R600_HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
1153
1154         arb_pop = RADEON_READ(R600_ARB_POP);
1155         arb_pop |= R600_ENABLE_TC128;
1156         RADEON_WRITE(R600_ARB_POP, arb_pop);
1157
1158         RADEON_WRITE(R600_PA_SC_MULTI_CHIP_CNTL, 0);
1159         RADEON_WRITE(R600_PA_CL_ENHANCE, (R600_CLIP_VTX_REORDER_ENA |
1160                                           R600_NUM_CLIP_SEQ(3)));
1161         RADEON_WRITE(R600_PA_SC_ENHANCE, R600_FORCE_EOV_MAX_CLK_CNT(4095));
1162
1163 }
1164
1165 static u32 r700_get_tile_pipe_to_backend_map(drm_radeon_private_t *dev_priv,
1166                                              u32 num_tile_pipes,
1167                                              u32 num_backends,
1168                                              u32 backend_disable_mask)
1169 {
1170         u32 backend_map = 0;
1171         u32 enabled_backends_mask;
1172         u32 enabled_backends_count;
1173         u32 cur_pipe;
1174         u32 swizzle_pipe[R7XX_MAX_PIPES];
1175         u32 cur_backend;
1176         u32 i;
1177         bool force_no_swizzle;
1178
1179         if (num_tile_pipes > R7XX_MAX_PIPES)
1180                 num_tile_pipes = R7XX_MAX_PIPES;
1181         if (num_tile_pipes < 1)
1182                 num_tile_pipes = 1;
1183         if (num_backends > R7XX_MAX_BACKENDS)
1184                 num_backends = R7XX_MAX_BACKENDS;
1185         if (num_backends < 1)
1186                 num_backends = 1;
1187
1188         enabled_backends_mask = 0;
1189         enabled_backends_count = 0;
1190         for (i = 0; i < R7XX_MAX_BACKENDS; ++i) {
1191                 if (((backend_disable_mask >> i) & 1) == 0) {
1192                         enabled_backends_mask |= (1 << i);
1193                         ++enabled_backends_count;
1194                 }
1195                 if (enabled_backends_count == num_backends)
1196                         break;
1197         }
1198
1199         if (enabled_backends_count == 0) {
1200                 enabled_backends_mask = 1;
1201                 enabled_backends_count = 1;
1202         }
1203
1204         if (enabled_backends_count != num_backends)
1205                 num_backends = enabled_backends_count;
1206
1207         switch (dev_priv->flags & RADEON_FAMILY_MASK) {
1208         case CHIP_RV770:
1209         case CHIP_RV730:
1210                 force_no_swizzle = false;
1211                 break;
1212         case CHIP_RV710:
1213         case CHIP_RV740:
1214         default:
1215                 force_no_swizzle = true;
1216                 break;
1217         }
1218
1219         memset((uint8_t *)&swizzle_pipe[0], 0, sizeof(u32) * R7XX_MAX_PIPES);
1220         switch (num_tile_pipes) {
1221         case 1:
1222                 swizzle_pipe[0] = 0;
1223                 break;
1224         case 2:
1225                 swizzle_pipe[0] = 0;
1226                 swizzle_pipe[1] = 1;
1227                 break;
1228         case 3:
1229                 if (force_no_swizzle) {
1230                         swizzle_pipe[0] = 0;
1231                         swizzle_pipe[1] = 1;
1232                         swizzle_pipe[2] = 2;
1233                 } else {
1234                         swizzle_pipe[0] = 0;
1235                         swizzle_pipe[1] = 2;
1236                         swizzle_pipe[2] = 1;
1237                 }
1238                 break;
1239         case 4:
1240                 if (force_no_swizzle) {
1241                         swizzle_pipe[0] = 0;
1242                         swizzle_pipe[1] = 1;
1243                         swizzle_pipe[2] = 2;
1244                         swizzle_pipe[3] = 3;
1245                 } else {
1246                         swizzle_pipe[0] = 0;
1247                         swizzle_pipe[1] = 2;
1248                         swizzle_pipe[2] = 3;
1249                         swizzle_pipe[3] = 1;
1250                 }
1251                 break;
1252         case 5:
1253                 if (force_no_swizzle) {
1254                         swizzle_pipe[0] = 0;
1255                         swizzle_pipe[1] = 1;
1256                         swizzle_pipe[2] = 2;
1257                         swizzle_pipe[3] = 3;
1258                         swizzle_pipe[4] = 4;
1259                 } else {
1260                         swizzle_pipe[0] = 0;
1261                         swizzle_pipe[1] = 2;
1262                         swizzle_pipe[2] = 4;
1263                         swizzle_pipe[3] = 1;
1264                         swizzle_pipe[4] = 3;
1265                 }
1266                 break;
1267         case 6:
1268                 if (force_no_swizzle) {
1269                         swizzle_pipe[0] = 0;
1270                         swizzle_pipe[1] = 1;
1271                         swizzle_pipe[2] = 2;
1272                         swizzle_pipe[3] = 3;
1273                         swizzle_pipe[4] = 4;
1274                         swizzle_pipe[5] = 5;
1275                 } else {
1276                         swizzle_pipe[0] = 0;
1277                         swizzle_pipe[1] = 2;
1278                         swizzle_pipe[2] = 4;
1279                         swizzle_pipe[3] = 5;
1280                         swizzle_pipe[4] = 3;
1281                         swizzle_pipe[5] = 1;
1282                 }
1283                 break;
1284         case 7:
1285                 if (force_no_swizzle) {
1286                         swizzle_pipe[0] = 0;
1287                         swizzle_pipe[1] = 1;
1288                         swizzle_pipe[2] = 2;
1289                         swizzle_pipe[3] = 3;
1290                         swizzle_pipe[4] = 4;
1291                         swizzle_pipe[5] = 5;
1292                         swizzle_pipe[6] = 6;
1293                 } else {
1294                         swizzle_pipe[0] = 0;
1295                         swizzle_pipe[1] = 2;
1296                         swizzle_pipe[2] = 4;
1297                         swizzle_pipe[3] = 6;
1298                         swizzle_pipe[4] = 3;
1299                         swizzle_pipe[5] = 1;
1300                         swizzle_pipe[6] = 5;
1301                 }
1302                 break;
1303         case 8:
1304                 if (force_no_swizzle) {
1305                         swizzle_pipe[0] = 0;
1306                         swizzle_pipe[1] = 1;
1307                         swizzle_pipe[2] = 2;
1308                         swizzle_pipe[3] = 3;
1309                         swizzle_pipe[4] = 4;
1310                         swizzle_pipe[5] = 5;
1311                         swizzle_pipe[6] = 6;
1312                         swizzle_pipe[7] = 7;
1313                 } else {
1314                         swizzle_pipe[0] = 0;
1315                         swizzle_pipe[1] = 2;
1316                         swizzle_pipe[2] = 4;
1317                         swizzle_pipe[3] = 6;
1318                         swizzle_pipe[4] = 3;
1319                         swizzle_pipe[5] = 1;
1320                         swizzle_pipe[6] = 7;
1321                         swizzle_pipe[7] = 5;
1322                 }
1323                 break;
1324         }
1325
1326         cur_backend = 0;
1327         for (cur_pipe = 0; cur_pipe < num_tile_pipes; ++cur_pipe) {
1328                 while (((1 << cur_backend) & enabled_backends_mask) == 0)
1329                         cur_backend = (cur_backend + 1) % R7XX_MAX_BACKENDS;
1330
1331                 backend_map |= (u32)(((cur_backend & 3) << (swizzle_pipe[cur_pipe] * 2)));
1332
1333                 cur_backend = (cur_backend + 1) % R7XX_MAX_BACKENDS;
1334         }
1335
1336         return backend_map;
1337 }
1338
1339 static void r700_gfx_init(struct drm_device *dev,
1340                           drm_radeon_private_t *dev_priv)
1341 {
1342         int i, j, num_qd_pipes;
1343         u32 ta_aux_cntl;
1344         u32 sx_debug_1;
1345         u32 smx_dc_ctl0;
1346         u32 db_debug3;
1347         u32 num_gs_verts_per_thread;
1348         u32 vgt_gs_per_es;
1349         u32 gs_prim_buffer_depth = 0;
1350         u32 sq_ms_fifo_sizes;
1351         u32 sq_config;
1352         u32 sq_thread_resource_mgmt;
1353         u32 hdp_host_path_cntl;
1354         u32 sq_dyn_gpr_size_simd_ab_0;
1355         u32 backend_map;
1356         u32 gb_tiling_config = 0;
1357         u32 cc_rb_backend_disable;
1358         u32 cc_gc_shader_pipe_config;
1359         u32 mc_arb_ramcfg;
1360         u32 db_debug4;
1361
1362         /* setup chip specs */
1363         switch (dev_priv->flags & RADEON_FAMILY_MASK) {
1364         case CHIP_RV770:
1365                 dev_priv->r600_max_pipes = 4;
1366                 dev_priv->r600_max_tile_pipes = 8;
1367                 dev_priv->r600_max_simds = 10;
1368                 dev_priv->r600_max_backends = 4;
1369                 dev_priv->r600_max_gprs = 256;
1370                 dev_priv->r600_max_threads = 248;
1371                 dev_priv->r600_max_stack_entries = 512;
1372                 dev_priv->r600_max_hw_contexts = 8;
1373                 dev_priv->r600_max_gs_threads = 16 * 2;
1374                 dev_priv->r600_sx_max_export_size = 128;
1375                 dev_priv->r600_sx_max_export_pos_size = 16;
1376                 dev_priv->r600_sx_max_export_smx_size = 112;
1377                 dev_priv->r600_sq_num_cf_insts = 2;
1378
1379                 dev_priv->r700_sx_num_of_sets = 7;
1380                 dev_priv->r700_sc_prim_fifo_size = 0xF9;
1381                 dev_priv->r700_sc_hiz_tile_fifo_size = 0x30;
1382                 dev_priv->r700_sc_earlyz_tile_fifo_fize = 0x130;
1383                 break;
1384         case CHIP_RV730:
1385                 dev_priv->r600_max_pipes = 2;
1386                 dev_priv->r600_max_tile_pipes = 4;
1387                 dev_priv->r600_max_simds = 8;
1388                 dev_priv->r600_max_backends = 2;
1389                 dev_priv->r600_max_gprs = 128;
1390                 dev_priv->r600_max_threads = 248;
1391                 dev_priv->r600_max_stack_entries = 256;
1392                 dev_priv->r600_max_hw_contexts = 8;
1393                 dev_priv->r600_max_gs_threads = 16 * 2;
1394                 dev_priv->r600_sx_max_export_size = 256;
1395                 dev_priv->r600_sx_max_export_pos_size = 32;
1396                 dev_priv->r600_sx_max_export_smx_size = 224;
1397                 dev_priv->r600_sq_num_cf_insts = 2;
1398
1399                 dev_priv->r700_sx_num_of_sets = 7;
1400                 dev_priv->r700_sc_prim_fifo_size = 0xf9;
1401                 dev_priv->r700_sc_hiz_tile_fifo_size = 0x30;
1402                 dev_priv->r700_sc_earlyz_tile_fifo_fize = 0x130;
1403                 if (dev_priv->r600_sx_max_export_pos_size > 16) {
1404                         dev_priv->r600_sx_max_export_pos_size -= 16;
1405                         dev_priv->r600_sx_max_export_smx_size += 16;
1406                 }
1407                 break;
1408         case CHIP_RV710:
1409                 dev_priv->r600_max_pipes = 2;
1410                 dev_priv->r600_max_tile_pipes = 2;
1411                 dev_priv->r600_max_simds = 2;
1412                 dev_priv->r600_max_backends = 1;
1413                 dev_priv->r600_max_gprs = 256;
1414                 dev_priv->r600_max_threads = 192;
1415                 dev_priv->r600_max_stack_entries = 256;
1416                 dev_priv->r600_max_hw_contexts = 4;
1417                 dev_priv->r600_max_gs_threads = 8 * 2;
1418                 dev_priv->r600_sx_max_export_size = 128;
1419                 dev_priv->r600_sx_max_export_pos_size = 16;
1420                 dev_priv->r600_sx_max_export_smx_size = 112;
1421                 dev_priv->r600_sq_num_cf_insts = 1;
1422
1423                 dev_priv->r700_sx_num_of_sets = 7;
1424                 dev_priv->r700_sc_prim_fifo_size = 0x40;
1425                 dev_priv->r700_sc_hiz_tile_fifo_size = 0x30;
1426                 dev_priv->r700_sc_earlyz_tile_fifo_fize = 0x130;
1427                 break;
1428         case CHIP_RV740:
1429                 dev_priv->r600_max_pipes = 4;
1430                 dev_priv->r600_max_tile_pipes = 4;
1431                 dev_priv->r600_max_simds = 8;
1432                 dev_priv->r600_max_backends = 4;
1433                 dev_priv->r600_max_gprs = 256;
1434                 dev_priv->r600_max_threads = 248;
1435                 dev_priv->r600_max_stack_entries = 512;
1436                 dev_priv->r600_max_hw_contexts = 8;
1437                 dev_priv->r600_max_gs_threads = 16 * 2;
1438                 dev_priv->r600_sx_max_export_size = 256;
1439                 dev_priv->r600_sx_max_export_pos_size = 32;
1440                 dev_priv->r600_sx_max_export_smx_size = 224;
1441                 dev_priv->r600_sq_num_cf_insts = 2;
1442
1443                 dev_priv->r700_sx_num_of_sets = 7;
1444                 dev_priv->r700_sc_prim_fifo_size = 0x100;
1445                 dev_priv->r700_sc_hiz_tile_fifo_size = 0x30;
1446                 dev_priv->r700_sc_earlyz_tile_fifo_fize = 0x130;
1447
1448                 if (dev_priv->r600_sx_max_export_pos_size > 16) {
1449                         dev_priv->r600_sx_max_export_pos_size -= 16;
1450                         dev_priv->r600_sx_max_export_smx_size += 16;
1451                 }
1452                 break;
1453         default:
1454                 break;
1455         }
1456
1457         /* Initialize HDP */
1458         j = 0;
1459         for (i = 0; i < 32; i++) {
1460                 RADEON_WRITE((0x2c14 + j), 0x00000000);
1461                 RADEON_WRITE((0x2c18 + j), 0x00000000);
1462                 RADEON_WRITE((0x2c1c + j), 0x00000000);
1463                 RADEON_WRITE((0x2c20 + j), 0x00000000);
1464                 RADEON_WRITE((0x2c24 + j), 0x00000000);
1465                 j += 0x18;
1466         }
1467
1468         RADEON_WRITE(R600_GRBM_CNTL, R600_GRBM_READ_TIMEOUT(0xff));
1469
1470         /* setup tiling, simd, pipe config */
1471         mc_arb_ramcfg = RADEON_READ(R700_MC_ARB_RAMCFG);
1472
1473         switch (dev_priv->r600_max_tile_pipes) {
1474         case 1:
1475                 gb_tiling_config |= R600_PIPE_TILING(0);
1476                 break;
1477         case 2:
1478                 gb_tiling_config |= R600_PIPE_TILING(1);
1479                 break;
1480         case 4:
1481                 gb_tiling_config |= R600_PIPE_TILING(2);
1482                 break;
1483         case 8:
1484                 gb_tiling_config |= R600_PIPE_TILING(3);
1485                 break;
1486         default:
1487                 break;
1488         }
1489
1490         if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV770)
1491                 gb_tiling_config |= R600_BANK_TILING(1);
1492         else
1493                 gb_tiling_config |= R600_BANK_TILING((mc_arb_ramcfg >> R700_NOOFBANK_SHIFT) & R700_NOOFBANK_MASK);
1494
1495         gb_tiling_config |= R600_GROUP_SIZE(0);
1496
1497         if (((mc_arb_ramcfg >> R700_NOOFROWS_SHIFT) & R700_NOOFROWS_MASK) > 3) {
1498                 gb_tiling_config |= R600_ROW_TILING(3);
1499                 gb_tiling_config |= R600_SAMPLE_SPLIT(3);
1500         } else {
1501                 gb_tiling_config |=
1502                         R600_ROW_TILING(((mc_arb_ramcfg >> R700_NOOFROWS_SHIFT) & R700_NOOFROWS_MASK));
1503                 gb_tiling_config |=
1504                         R600_SAMPLE_SPLIT(((mc_arb_ramcfg >> R700_NOOFROWS_SHIFT) & R700_NOOFROWS_MASK));
1505         }
1506
1507         gb_tiling_config |= R600_BANK_SWAPS(1);
1508
1509         cc_rb_backend_disable = RADEON_READ(R600_CC_RB_BACKEND_DISABLE) & 0x00ff0000;
1510         cc_rb_backend_disable |=
1511                 R600_BACKEND_DISABLE((R7XX_MAX_BACKENDS_MASK << dev_priv->r600_max_backends) & R7XX_MAX_BACKENDS_MASK);
1512
1513         cc_gc_shader_pipe_config = RADEON_READ(R600_CC_GC_SHADER_PIPE_CONFIG) & 0xffffff00;
1514         cc_gc_shader_pipe_config |=
1515                 R600_INACTIVE_QD_PIPES((R7XX_MAX_PIPES_MASK << dev_priv->r600_max_pipes) & R7XX_MAX_PIPES_MASK);
1516         cc_gc_shader_pipe_config |=
1517                 R600_INACTIVE_SIMDS((R7XX_MAX_SIMDS_MASK << dev_priv->r600_max_simds) & R7XX_MAX_SIMDS_MASK);
1518
1519         if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV740)
1520                 backend_map = 0x28;
1521         else
1522                 backend_map = r700_get_tile_pipe_to_backend_map(dev_priv,
1523                                                                 dev_priv->r600_max_tile_pipes,
1524                                                                 (R7XX_MAX_BACKENDS -
1525                                                                  r600_count_pipe_bits((cc_rb_backend_disable &
1526                                                                                        R7XX_MAX_BACKENDS_MASK) >> 16)),
1527                                                                 (cc_rb_backend_disable >> 16));
1528         gb_tiling_config |= R600_BACKEND_MAP(backend_map);
1529
1530         RADEON_WRITE(R600_GB_TILING_CONFIG,      gb_tiling_config);
1531         RADEON_WRITE(R600_DCP_TILING_CONFIG,    (gb_tiling_config & 0xffff));
1532         RADEON_WRITE(R600_HDP_TILING_CONFIG,    (gb_tiling_config & 0xffff));
1533         if (gb_tiling_config & 0xc0) {
1534                 dev_priv->r600_group_size = 512;
1535         } else {
1536                 dev_priv->r600_group_size = 256;
1537         }
1538         dev_priv->r600_npipes = 1 << ((gb_tiling_config >> 1) & 0x7);
1539         if (gb_tiling_config & 0x30) {
1540                 dev_priv->r600_nbanks = 8;
1541         } else {
1542                 dev_priv->r600_nbanks = 4;
1543         }
1544
1545         RADEON_WRITE(R600_CC_RB_BACKEND_DISABLE,      cc_rb_backend_disable);
1546         RADEON_WRITE(R600_CC_GC_SHADER_PIPE_CONFIG,   cc_gc_shader_pipe_config);
1547         RADEON_WRITE(R600_GC_USER_SHADER_PIPE_CONFIG, cc_gc_shader_pipe_config);
1548
1549         RADEON_WRITE(R700_CC_SYS_RB_BACKEND_DISABLE, cc_rb_backend_disable);
1550         RADEON_WRITE(R700_CGTS_SYS_TCC_DISABLE, 0);
1551         RADEON_WRITE(R700_CGTS_TCC_DISABLE, 0);
1552         RADEON_WRITE(R700_CGTS_USER_SYS_TCC_DISABLE, 0);
1553         RADEON_WRITE(R700_CGTS_USER_TCC_DISABLE, 0);
1554
1555         num_qd_pipes =
1556                 R7XX_MAX_PIPES - r600_count_pipe_bits((cc_gc_shader_pipe_config & R600_INACTIVE_QD_PIPES_MASK) >> 8);
1557         RADEON_WRITE(R600_VGT_OUT_DEALLOC_CNTL, (num_qd_pipes * 4) & R600_DEALLOC_DIST_MASK);
1558         RADEON_WRITE(R600_VGT_VERTEX_REUSE_BLOCK_CNTL, ((num_qd_pipes * 4) - 2) & R600_VTX_REUSE_DEPTH_MASK);
1559
1560         /* set HW defaults for 3D engine */
1561         RADEON_WRITE(R600_CP_QUEUE_THRESHOLDS, (R600_ROQ_IB1_START(0x16) |
1562                                                 R600_ROQ_IB2_START(0x2b)));
1563
1564         RADEON_WRITE(R600_CP_MEQ_THRESHOLDS, R700_STQ_SPLIT(0x30));
1565
1566         ta_aux_cntl = RADEON_READ(R600_TA_CNTL_AUX);
1567         RADEON_WRITE(R600_TA_CNTL_AUX, ta_aux_cntl | R600_DISABLE_CUBE_ANISO);
1568
1569         sx_debug_1 = RADEON_READ(R700_SX_DEBUG_1);
1570         sx_debug_1 |= R700_ENABLE_NEW_SMX_ADDRESS;
1571         RADEON_WRITE(R700_SX_DEBUG_1, sx_debug_1);
1572
1573         smx_dc_ctl0 = RADEON_READ(R600_SMX_DC_CTL0);
1574         smx_dc_ctl0 &= ~R700_CACHE_DEPTH(0x1ff);
1575         smx_dc_ctl0 |= R700_CACHE_DEPTH((dev_priv->r700_sx_num_of_sets * 64) - 1);
1576         RADEON_WRITE(R600_SMX_DC_CTL0, smx_dc_ctl0);
1577
1578         if ((dev_priv->flags & RADEON_FAMILY_MASK) != CHIP_RV740)
1579                 RADEON_WRITE(R700_SMX_EVENT_CTL, (R700_ES_FLUSH_CTL(4) |
1580                                                   R700_GS_FLUSH_CTL(4) |
1581                                                   R700_ACK_FLUSH_CTL(3) |
1582                                                   R700_SYNC_FLUSH_CTL));
1583
1584         db_debug3 = RADEON_READ(R700_DB_DEBUG3);
1585         db_debug3 &= ~R700_DB_CLK_OFF_DELAY(0x1f);
1586         switch (dev_priv->flags & RADEON_FAMILY_MASK) {
1587         case CHIP_RV770:
1588         case CHIP_RV740:
1589                 db_debug3 |= R700_DB_CLK_OFF_DELAY(0x1f);
1590                 break;
1591         case CHIP_RV710:
1592         case CHIP_RV730:
1593         default:
1594                 db_debug3 |= R700_DB_CLK_OFF_DELAY(2);
1595                 break;
1596         }
1597         RADEON_WRITE(R700_DB_DEBUG3, db_debug3);
1598
1599         if ((dev_priv->flags & RADEON_FAMILY_MASK) != CHIP_RV770) {
1600                 db_debug4 = RADEON_READ(RV700_DB_DEBUG4);
1601                 db_debug4 |= RV700_DISABLE_TILE_COVERED_FOR_PS_ITER;
1602                 RADEON_WRITE(RV700_DB_DEBUG4, db_debug4);
1603         }
1604
1605         RADEON_WRITE(R600_SX_EXPORT_BUFFER_SIZES, (R600_COLOR_BUFFER_SIZE((dev_priv->r600_sx_max_export_size / 4) - 1) |
1606                                                    R600_POSITION_BUFFER_SIZE((dev_priv->r600_sx_max_export_pos_size / 4) - 1) |
1607                                                    R600_SMX_BUFFER_SIZE((dev_priv->r600_sx_max_export_smx_size / 4) - 1)));
1608
1609         RADEON_WRITE(R700_PA_SC_FIFO_SIZE_R7XX, (R700_SC_PRIM_FIFO_SIZE(dev_priv->r700_sc_prim_fifo_size) |
1610                                                  R700_SC_HIZ_TILE_FIFO_SIZE(dev_priv->r700_sc_hiz_tile_fifo_size) |
1611                                                  R700_SC_EARLYZ_TILE_FIFO_SIZE(dev_priv->r700_sc_earlyz_tile_fifo_fize)));
1612
1613         RADEON_WRITE(R600_PA_SC_MULTI_CHIP_CNTL, 0);
1614
1615         RADEON_WRITE(R600_VGT_NUM_INSTANCES, 1);
1616
1617         RADEON_WRITE(R600_SPI_CONFIG_CNTL, R600_GPR_WRITE_PRIORITY(0));
1618
1619         RADEON_WRITE(R600_SPI_CONFIG_CNTL_1, R600_VTX_DONE_DELAY(4));
1620
1621         RADEON_WRITE(R600_CP_PERFMON_CNTL, 0);
1622
1623         sq_ms_fifo_sizes = (R600_CACHE_FIFO_SIZE(16 * dev_priv->r600_sq_num_cf_insts) |
1624                             R600_DONE_FIFO_HIWATER(0xe0) |
1625                             R600_ALU_UPDATE_FIFO_HIWATER(0x8));
1626         switch (dev_priv->flags & RADEON_FAMILY_MASK) {
1627         case CHIP_RV770:
1628         case CHIP_RV730:
1629         case CHIP_RV710:
1630                 sq_ms_fifo_sizes |= R600_FETCH_FIFO_HIWATER(0x1);
1631                 break;
1632         case CHIP_RV740:
1633         default:
1634                 sq_ms_fifo_sizes |= R600_FETCH_FIFO_HIWATER(0x4);
1635                 break;
1636         }
1637         RADEON_WRITE(R600_SQ_MS_FIFO_SIZES, sq_ms_fifo_sizes);
1638
1639         /* SQ_CONFIG, SQ_GPR_RESOURCE_MGMT, SQ_THREAD_RESOURCE_MGMT, SQ_STACK_RESOURCE_MGMT
1640          * should be adjusted as needed by the 2D/3D drivers.  This just sets default values
1641          */
1642         sq_config = RADEON_READ(R600_SQ_CONFIG);
1643         sq_config &= ~(R600_PS_PRIO(3) |
1644                        R600_VS_PRIO(3) |
1645                        R600_GS_PRIO(3) |
1646                        R600_ES_PRIO(3));
1647         sq_config |= (R600_DX9_CONSTS |
1648                       R600_VC_ENABLE |
1649                       R600_EXPORT_SRC_C |
1650                       R600_PS_PRIO(0) |
1651                       R600_VS_PRIO(1) |
1652                       R600_GS_PRIO(2) |
1653                       R600_ES_PRIO(3));
1654         if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV710)
1655                 /* no vertex cache */
1656                 sq_config &= ~R600_VC_ENABLE;
1657
1658         RADEON_WRITE(R600_SQ_CONFIG, sq_config);
1659
1660         RADEON_WRITE(R600_SQ_GPR_RESOURCE_MGMT_1,  (R600_NUM_PS_GPRS((dev_priv->r600_max_gprs * 24)/64) |
1661                                                     R600_NUM_VS_GPRS((dev_priv->r600_max_gprs * 24)/64) |
1662                                                     R600_NUM_CLAUSE_TEMP_GPRS(((dev_priv->r600_max_gprs * 24)/64)/2)));
1663
1664         RADEON_WRITE(R600_SQ_GPR_RESOURCE_MGMT_2,  (R600_NUM_GS_GPRS((dev_priv->r600_max_gprs * 7)/64) |
1665                                                     R600_NUM_ES_GPRS((dev_priv->r600_max_gprs * 7)/64)));
1666
1667         sq_thread_resource_mgmt = (R600_NUM_PS_THREADS((dev_priv->r600_max_threads * 4)/8) |
1668                                    R600_NUM_VS_THREADS((dev_priv->r600_max_threads * 2)/8) |
1669                                    R600_NUM_ES_THREADS((dev_priv->r600_max_threads * 1)/8));
1670         if (((dev_priv->r600_max_threads * 1) / 8) > dev_priv->r600_max_gs_threads)
1671                 sq_thread_resource_mgmt |= R600_NUM_GS_THREADS(dev_priv->r600_max_gs_threads);
1672         else
1673                 sq_thread_resource_mgmt |= R600_NUM_GS_THREADS((dev_priv->r600_max_gs_threads * 1)/8);
1674         RADEON_WRITE(R600_SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
1675
1676         RADEON_WRITE(R600_SQ_STACK_RESOURCE_MGMT_1, (R600_NUM_PS_STACK_ENTRIES((dev_priv->r600_max_stack_entries * 1)/4) |
1677                                                      R600_NUM_VS_STACK_ENTRIES((dev_priv->r600_max_stack_entries * 1)/4)));
1678
1679         RADEON_WRITE(R600_SQ_STACK_RESOURCE_MGMT_2, (R600_NUM_GS_STACK_ENTRIES((dev_priv->r600_max_stack_entries * 1)/4) |
1680                                                      R600_NUM_ES_STACK_ENTRIES((dev_priv->r600_max_stack_entries * 1)/4)));
1681
1682         sq_dyn_gpr_size_simd_ab_0 = (R700_SIMDA_RING0((dev_priv->r600_max_gprs * 38)/64) |
1683                                      R700_SIMDA_RING1((dev_priv->r600_max_gprs * 38)/64) |
1684                                      R700_SIMDB_RING0((dev_priv->r600_max_gprs * 38)/64) |
1685                                      R700_SIMDB_RING1((dev_priv->r600_max_gprs * 38)/64));
1686
1687         RADEON_WRITE(R700_SQ_DYN_GPR_SIZE_SIMD_AB_0, sq_dyn_gpr_size_simd_ab_0);
1688         RADEON_WRITE(R700_SQ_DYN_GPR_SIZE_SIMD_AB_1, sq_dyn_gpr_size_simd_ab_0);
1689         RADEON_WRITE(R700_SQ_DYN_GPR_SIZE_SIMD_AB_2, sq_dyn_gpr_size_simd_ab_0);
1690         RADEON_WRITE(R700_SQ_DYN_GPR_SIZE_SIMD_AB_3, sq_dyn_gpr_size_simd_ab_0);
1691         RADEON_WRITE(R700_SQ_DYN_GPR_SIZE_SIMD_AB_4, sq_dyn_gpr_size_simd_ab_0);
1692         RADEON_WRITE(R700_SQ_DYN_GPR_SIZE_SIMD_AB_5, sq_dyn_gpr_size_simd_ab_0);
1693         RADEON_WRITE(R700_SQ_DYN_GPR_SIZE_SIMD_AB_6, sq_dyn_gpr_size_simd_ab_0);
1694         RADEON_WRITE(R700_SQ_DYN_GPR_SIZE_SIMD_AB_7, sq_dyn_gpr_size_simd_ab_0);
1695
1696         RADEON_WRITE(R700_PA_SC_FORCE_EOV_MAX_CNTS, (R700_FORCE_EOV_MAX_CLK_CNT(4095) |
1697                                                      R700_FORCE_EOV_MAX_REZ_CNT(255)));
1698
1699         if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV710)
1700                 RADEON_WRITE(R600_VGT_CACHE_INVALIDATION, (R600_CACHE_INVALIDATION(R600_TC_ONLY) |
1701                                                            R700_AUTO_INVLD_EN(R700_ES_AND_GS_AUTO)));
1702         else
1703                 RADEON_WRITE(R600_VGT_CACHE_INVALIDATION, (R600_CACHE_INVALIDATION(R600_VC_AND_TC) |
1704                                                            R700_AUTO_INVLD_EN(R700_ES_AND_GS_AUTO)));
1705
1706         switch (dev_priv->flags & RADEON_FAMILY_MASK) {
1707         case CHIP_RV770:
1708         case CHIP_RV730:
1709         case CHIP_RV740:
1710                 gs_prim_buffer_depth = 384;
1711                 break;
1712         case CHIP_RV710:
1713                 gs_prim_buffer_depth = 128;
1714                 break;
1715         default:
1716                 break;
1717         }
1718
1719         num_gs_verts_per_thread = dev_priv->r600_max_pipes * 16;
1720         vgt_gs_per_es = gs_prim_buffer_depth + num_gs_verts_per_thread;
1721         /* Max value for this is 256 */
1722         if (vgt_gs_per_es > 256)
1723                 vgt_gs_per_es = 256;
1724
1725         RADEON_WRITE(R600_VGT_ES_PER_GS, 128);
1726         RADEON_WRITE(R600_VGT_GS_PER_ES, vgt_gs_per_es);
1727         RADEON_WRITE(R600_VGT_GS_PER_VS, 2);
1728
1729         /* more default values. 2D/3D driver should adjust as needed */
1730         RADEON_WRITE(R600_VGT_GS_VERTEX_REUSE, 16);
1731         RADEON_WRITE(R600_PA_SC_LINE_STIPPLE_STATE, 0);
1732         RADEON_WRITE(R600_VGT_STRMOUT_EN, 0);
1733         RADEON_WRITE(R600_SX_MISC, 0);
1734         RADEON_WRITE(R600_PA_SC_MODE_CNTL, 0);
1735         RADEON_WRITE(R700_PA_SC_EDGERULE, 0xaaaaaaaa);
1736         RADEON_WRITE(R600_PA_SC_AA_CONFIG, 0);
1737         RADEON_WRITE(R600_PA_SC_CLIPRECT_RULE, 0xffff);
1738         RADEON_WRITE(R600_PA_SC_LINE_STIPPLE, 0);
1739         RADEON_WRITE(R600_SPI_INPUT_Z, 0);
1740         RADEON_WRITE(R600_SPI_PS_IN_CONTROL_0, R600_NUM_INTERP(2));
1741         RADEON_WRITE(R600_CB_COLOR7_FRAG, 0);
1742
1743         /* clear render buffer base addresses */
1744         RADEON_WRITE(R600_CB_COLOR0_BASE, 0);
1745         RADEON_WRITE(R600_CB_COLOR1_BASE, 0);
1746         RADEON_WRITE(R600_CB_COLOR2_BASE, 0);
1747         RADEON_WRITE(R600_CB_COLOR3_BASE, 0);
1748         RADEON_WRITE(R600_CB_COLOR4_BASE, 0);
1749         RADEON_WRITE(R600_CB_COLOR5_BASE, 0);
1750         RADEON_WRITE(R600_CB_COLOR6_BASE, 0);
1751         RADEON_WRITE(R600_CB_COLOR7_BASE, 0);
1752
1753         RADEON_WRITE(R700_TCP_CNTL, 0);
1754
1755         hdp_host_path_cntl = RADEON_READ(R600_HDP_HOST_PATH_CNTL);
1756         RADEON_WRITE(R600_HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
1757
1758         RADEON_WRITE(R600_PA_SC_MULTI_CHIP_CNTL, 0);
1759
1760         RADEON_WRITE(R600_PA_CL_ENHANCE, (R600_CLIP_VTX_REORDER_ENA |
1761                                           R600_NUM_CLIP_SEQ(3)));
1762
1763 }
1764
1765 static void r600_cp_init_ring_buffer(struct drm_device *dev,
1766                                        drm_radeon_private_t *dev_priv,
1767                                        struct drm_file *file_priv)
1768 {
1769         struct drm_radeon_master_private *master_priv;
1770         u32 ring_start;
1771         u64 rptr_addr;
1772
1773         if (((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV770))
1774                 r700_gfx_init(dev, dev_priv);
1775         else
1776                 r600_gfx_init(dev, dev_priv);
1777
1778         RADEON_WRITE(R600_GRBM_SOFT_RESET, R600_SOFT_RESET_CP);
1779         RADEON_READ(R600_GRBM_SOFT_RESET);
1780         DRM_MDELAY(15);
1781         RADEON_WRITE(R600_GRBM_SOFT_RESET, 0);
1782
1783
1784         /* Set ring buffer size */
1785 #ifdef __BIG_ENDIAN
1786         RADEON_WRITE(R600_CP_RB_CNTL,
1787                      R600_BUF_SWAP_32BIT |
1788                      R600_RB_NO_UPDATE |
1789                      (dev_priv->ring.rptr_update_l2qw << 8) |
1790                      dev_priv->ring.size_l2qw);
1791 #else
1792         RADEON_WRITE(R600_CP_RB_CNTL,
1793                      RADEON_RB_NO_UPDATE |
1794                      (dev_priv->ring.rptr_update_l2qw << 8) |
1795                      dev_priv->ring.size_l2qw);
1796 #endif
1797
1798         RADEON_WRITE(R600_CP_SEM_WAIT_TIMER, 0x0);
1799
1800         /* Set the write pointer delay */
1801         RADEON_WRITE(R600_CP_RB_WPTR_DELAY, 0);
1802
1803 #ifdef __BIG_ENDIAN
1804         RADEON_WRITE(R600_CP_RB_CNTL,
1805                      R600_BUF_SWAP_32BIT |
1806                      R600_RB_NO_UPDATE |
1807                      R600_RB_RPTR_WR_ENA |
1808                      (dev_priv->ring.rptr_update_l2qw << 8) |
1809                      dev_priv->ring.size_l2qw);
1810 #else
1811         RADEON_WRITE(R600_CP_RB_CNTL,
1812                      R600_RB_NO_UPDATE |
1813                      R600_RB_RPTR_WR_ENA |
1814                      (dev_priv->ring.rptr_update_l2qw << 8) |
1815                      dev_priv->ring.size_l2qw);
1816 #endif
1817
1818         /* Initialize the ring buffer's read and write pointers */
1819         RADEON_WRITE(R600_CP_RB_RPTR_WR, 0);
1820         RADEON_WRITE(R600_CP_RB_WPTR, 0);
1821         SET_RING_HEAD(dev_priv, 0);
1822         dev_priv->ring.tail = 0;
1823
1824 #if __OS_HAS_AGP
1825         if (dev_priv->flags & RADEON_IS_AGP) {
1826                 rptr_addr = dev_priv->ring_rptr->offset
1827                         - dev->agp->base +
1828                         dev_priv->gart_vm_start;
1829         } else
1830 #endif
1831         {
1832                 rptr_addr = dev_priv->ring_rptr->offset
1833                         - ((unsigned long) dev->sg->vaddr)
1834                         + dev_priv->gart_vm_start;
1835         }
1836         RADEON_WRITE(R600_CP_RB_RPTR_ADDR, (rptr_addr & 0xfffffffc));
1837         RADEON_WRITE(R600_CP_RB_RPTR_ADDR_HI, upper_32_bits(rptr_addr));
1838
1839 #ifdef __BIG_ENDIAN
1840         RADEON_WRITE(R600_CP_RB_CNTL,
1841                      RADEON_BUF_SWAP_32BIT |
1842                      (dev_priv->ring.rptr_update_l2qw << 8) |
1843                      dev_priv->ring.size_l2qw);
1844 #else
1845         RADEON_WRITE(R600_CP_RB_CNTL,
1846                      (dev_priv->ring.rptr_update_l2qw << 8) |
1847                      dev_priv->ring.size_l2qw);
1848 #endif
1849
1850 #if __OS_HAS_AGP
1851         if (dev_priv->flags & RADEON_IS_AGP) {
1852                 /* XXX */
1853                 radeon_write_agp_base(dev_priv, dev->agp->base);
1854
1855                 /* XXX */
1856                 radeon_write_agp_location(dev_priv,
1857                              (((dev_priv->gart_vm_start - 1 +
1858                                 dev_priv->gart_size) & 0xffff0000) |
1859                               (dev_priv->gart_vm_start >> 16)));
1860
1861                 ring_start = (dev_priv->cp_ring->offset
1862                               - dev->agp->base
1863                               + dev_priv->gart_vm_start);
1864         } else
1865 #endif
1866                 ring_start = (dev_priv->cp_ring->offset
1867                               - (unsigned long)dev->sg->vaddr>
1868                               + dev_priv->gart_vm_start);
1869
1870         RADEON_WRITE(R600_CP_RB_BASE, ring_start >> 8);
1871
1872         RADEON_WRITE(R600_CP_ME_CNTL, 0xff);
1873
1874         RADEON_WRITE(R600_CP_DEBUG, (1 << 27) | (1 << 28));
1875
1876         /* Initialize the scratch register pointer.  This will cause
1877          * the scratch register values to be written out to memory
1878          * whenever they are updated.
1879          *
1880          * We simply put this behind the ring read pointer, this works
1881          * with PCI GART as well as (whatever kind of) AGP GART
1882          */
1883         {
1884                 u64 scratch_addr;
1885
1886                 scratch_addr = RADEON_READ(R600_CP_RB_RPTR_ADDR) & 0xFFFFFFFC;
1887                 scratch_addr |= ((u64)RADEON_READ(R600_CP_RB_RPTR_ADDR_HI)) << 32;
1888                 scratch_addr += R600_SCRATCH_REG_OFFSET;
1889                 scratch_addr >>= 8;
1890                 scratch_addr &= 0xffffffff;
1891
1892                 RADEON_WRITE(R600_SCRATCH_ADDR, (uint32_t)scratch_addr);
1893         }
1894
1895         RADEON_WRITE(R600_SCRATCH_UMSK, 0x7);
1896
1897         /* Turn on bus mastering */
1898         radeon_enable_bm(dev_priv);
1899
1900         radeon_write_ring_rptr(dev_priv, R600_SCRATCHOFF(0), 0);
1901         RADEON_WRITE(R600_LAST_FRAME_REG, 0);
1902
1903         radeon_write_ring_rptr(dev_priv, R600_SCRATCHOFF(1), 0);
1904         RADEON_WRITE(R600_LAST_DISPATCH_REG, 0);
1905
1906         radeon_write_ring_rptr(dev_priv, R600_SCRATCHOFF(2), 0);
1907         RADEON_WRITE(R600_LAST_CLEAR_REG, 0);
1908
1909         /* reset sarea copies of these */
1910         master_priv = file_priv->masterp->driver_priv;
1911         if (master_priv->sarea_priv) {
1912                 master_priv->sarea_priv->last_frame = 0;
1913                 master_priv->sarea_priv->last_dispatch = 0;
1914                 master_priv->sarea_priv->last_clear = 0;
1915         }
1916
1917         r600_do_wait_for_idle(dev_priv);
1918
1919 }
1920
1921 int r600_do_cleanup_cp(struct drm_device *dev)
1922 {
1923         drm_radeon_private_t *dev_priv = dev->dev_private;
1924         DRM_DEBUG("\n");
1925
1926         /* Make sure interrupts are disabled here because the uninstall ioctl
1927          * may not have been called from userspace and after dev_private
1928          * is freed, it's too late.
1929          */
1930         if (dev->irq_enabled)
1931                 drm_irq_uninstall(dev);
1932
1933 #if __OS_HAS_AGP
1934         if (dev_priv->flags & RADEON_IS_AGP) {
1935                 if (dev_priv->cp_ring != NULL) {
1936                         drm_core_ioremapfree(dev_priv->cp_ring, dev);
1937                         dev_priv->cp_ring = NULL;
1938                 }
1939                 if (dev_priv->ring_rptr != NULL) {
1940                         drm_core_ioremapfree(dev_priv->ring_rptr, dev);
1941                         dev_priv->ring_rptr = NULL;
1942                 }
1943                 if (dev->agp_buffer_map != NULL) {
1944                         drm_core_ioremapfree(dev->agp_buffer_map, dev);
1945                         dev->agp_buffer_map = NULL;
1946                 }
1947         } else
1948 #endif
1949         {
1950
1951                 if (dev_priv->gart_info.bus_addr)
1952                         r600_page_table_cleanup(dev, &dev_priv->gart_info);
1953
1954                 if (dev_priv->gart_info.gart_table_location == DRM_ATI_GART_FB) {
1955                         drm_core_ioremapfree(&dev_priv->gart_info.mapping, dev);
1956                         dev_priv->gart_info.addr = NULL;
1957                 }
1958         }
1959         /* only clear to the start of flags */
1960         memset(dev_priv, 0, offsetof(drm_radeon_private_t, flags));
1961
1962         return 0;
1963 }
1964
1965 int r600_do_init_cp(struct drm_device *dev, drm_radeon_init_t *init,
1966                     struct drm_file *file_priv)
1967 {
1968         drm_radeon_private_t *dev_priv = dev->dev_private;
1969         struct drm_radeon_master_private *master_priv = file_priv->masterp->driver_priv;
1970
1971         DRM_DEBUG("\n");
1972
1973         sx_init(&dev_priv->cs_mutex, "drm__radeon_private__cs_mutex");
1974         r600_cs_legacy_init();
1975         /* if we require new memory map but we don't have it fail */
1976         if ((dev_priv->flags & RADEON_NEW_MEMMAP) && !dev_priv->new_memmap) {
1977                 DRM_ERROR("Cannot initialise DRM on this card\nThis card requires a new X.org DDX for 3D\n");
1978                 r600_do_cleanup_cp(dev);
1979                 return -EINVAL;
1980         }
1981
1982         if (init->is_pci && (dev_priv->flags & RADEON_IS_AGP)) {
1983                 DRM_DEBUG("Forcing AGP card to PCI mode\n");
1984                 dev_priv->flags &= ~RADEON_IS_AGP;
1985                 /* The writeback test succeeds, but when writeback is enabled,
1986                  * the ring buffer read ptr update fails after first 128 bytes.
1987                  */
1988                 radeon_no_wb = 1;
1989         } else if (!(dev_priv->flags & (RADEON_IS_AGP | RADEON_IS_PCI | RADEON_IS_PCIE))
1990                  && !init->is_pci) {
1991                 DRM_DEBUG("Restoring AGP flag\n");
1992                 dev_priv->flags |= RADEON_IS_AGP;
1993         }
1994
1995         dev_priv->usec_timeout = init->usec_timeout;
1996         if (dev_priv->usec_timeout < 1 ||
1997             dev_priv->usec_timeout > RADEON_MAX_USEC_TIMEOUT) {
1998                 DRM_DEBUG("TIMEOUT problem!\n");
1999                 r600_do_cleanup_cp(dev);
2000                 return -EINVAL;
2001         }
2002
2003         /* Enable vblank on CRTC1 for older X servers
2004          */
2005         dev_priv->vblank_crtc = DRM_RADEON_VBLANK_CRTC1;
2006         dev_priv->do_boxes = 0;
2007         dev_priv->cp_mode = init->cp_mode;
2008
2009         /* We don't support anything other than bus-mastering ring mode,
2010          * but the ring can be in either AGP or PCI space for the ring
2011          * read pointer.
2012          */
2013         if ((init->cp_mode != RADEON_CSQ_PRIBM_INDDIS) &&
2014             (init->cp_mode != RADEON_CSQ_PRIBM_INDBM)) {
2015                 DRM_DEBUG("BAD cp_mode (%x)!\n", init->cp_mode);
2016                 r600_do_cleanup_cp(dev);
2017                 return -EINVAL;
2018         }
2019
2020         switch (init->fb_bpp) {
2021         case 16:
2022                 dev_priv->color_fmt = RADEON_COLOR_FORMAT_RGB565;
2023                 break;
2024         case 32:
2025         default:
2026                 dev_priv->color_fmt = RADEON_COLOR_FORMAT_ARGB8888;
2027                 break;
2028         }
2029         dev_priv->front_offset = init->front_offset;
2030         dev_priv->front_pitch = init->front_pitch;
2031         dev_priv->back_offset = init->back_offset;
2032         dev_priv->back_pitch = init->back_pitch;
2033
2034         dev_priv->ring_offset = init->ring_offset;
2035         dev_priv->ring_rptr_offset = init->ring_rptr_offset;
2036         dev_priv->buffers_offset = init->buffers_offset;
2037         dev_priv->gart_textures_offset = init->gart_textures_offset;
2038
2039         master_priv->sarea = drm_getsarea(dev);
2040         if (!master_priv->sarea) {
2041                 DRM_ERROR("could not find sarea!\n");
2042                 r600_do_cleanup_cp(dev);
2043                 return -EINVAL;
2044         }
2045
2046         dev_priv->cp_ring = drm_core_findmap(dev, init->ring_offset);
2047         if (!dev_priv->cp_ring) {
2048                 DRM_ERROR("could not find cp ring region!\n");
2049                 r600_do_cleanup_cp(dev);
2050                 return -EINVAL;
2051         }
2052         dev_priv->ring_rptr = drm_core_findmap(dev, init->ring_rptr_offset);
2053         if (!dev_priv->ring_rptr) {
2054                 DRM_ERROR("could not find ring read pointer!\n");
2055                 r600_do_cleanup_cp(dev);
2056                 return -EINVAL;
2057         }
2058         dev->agp_buffer_token = init->buffers_offset;
2059         dev->agp_buffer_map = drm_core_findmap(dev, init->buffers_offset);
2060         if (!dev->agp_buffer_map) {
2061                 DRM_ERROR("could not find dma buffer region!\n");
2062                 r600_do_cleanup_cp(dev);
2063                 return -EINVAL;
2064         }
2065
2066         if (init->gart_textures_offset) {
2067                 dev_priv->gart_textures =
2068                     drm_core_findmap(dev, init->gart_textures_offset);
2069                 if (!dev_priv->gart_textures) {
2070                         DRM_ERROR("could not find GART texture region!\n");
2071                         r600_do_cleanup_cp(dev);
2072                         return -EINVAL;
2073                 }
2074         }
2075
2076 #if __OS_HAS_AGP
2077         /* XXX */
2078         if (dev_priv->flags & RADEON_IS_AGP) {
2079                 drm_core_ioremap_wc(dev_priv->cp_ring, dev);
2080                 drm_core_ioremap_wc(dev_priv->ring_rptr, dev);
2081                 drm_core_ioremap_wc(dev->agp_buffer_map, dev);
2082                 if (!dev_priv->cp_ring->handle ||
2083                     !dev_priv->ring_rptr->handle ||
2084                     !dev->agp_buffer_map->handle) {
2085                         DRM_ERROR("could not find ioremap agp regions!\n");
2086                         r600_do_cleanup_cp(dev);
2087                         return -EINVAL;
2088                 }
2089         } else
2090 #endif
2091         {
2092                 dev_priv->cp_ring->handle = (void *)(unsigned long)dev_priv->cp_ring->offset;
2093                 dev_priv->ring_rptr->handle =
2094                         (void *)(unsigned long)dev_priv->ring_rptr->offset;
2095                 dev->agp_buffer_map->handle =
2096                         (void *)(unsigned long)dev->agp_buffer_map->offset;
2097
2098                 DRM_DEBUG("dev_priv->cp_ring->handle %p\n",
2099                           dev_priv->cp_ring->handle);
2100                 DRM_DEBUG("dev_priv->ring_rptr->handle %p\n",
2101                           dev_priv->ring_rptr->handle);
2102                 DRM_DEBUG("dev->agp_buffer_map->handle %p\n",
2103                           dev->agp_buffer_map->handle);
2104         }
2105
2106         dev_priv->fb_location = (radeon_read_fb_location(dev_priv) & 0xffff) << 24;
2107         dev_priv->fb_size =
2108                 (((radeon_read_fb_location(dev_priv) & 0xffff0000u) << 8) + 0x1000000)
2109                 - dev_priv->fb_location;
2110
2111         dev_priv->front_pitch_offset = (((dev_priv->front_pitch / 64) << 22) |
2112                                         ((dev_priv->front_offset
2113                                           + dev_priv->fb_location) >> 10));
2114
2115         dev_priv->back_pitch_offset = (((dev_priv->back_pitch / 64) << 22) |
2116                                        ((dev_priv->back_offset
2117                                          + dev_priv->fb_location) >> 10));
2118
2119         dev_priv->depth_pitch_offset = (((dev_priv->depth_pitch / 64) << 22) |
2120                                         ((dev_priv->depth_offset
2121                                           + dev_priv->fb_location) >> 10));
2122
2123         dev_priv->gart_size = init->gart_size;
2124
2125         /* New let's set the memory map ... */
2126         if (dev_priv->new_memmap) {
2127                 u32 base = 0;
2128
2129                 DRM_INFO("Setting GART location based on new memory map\n");
2130
2131                 /* If using AGP, try to locate the AGP aperture at the same
2132                  * location in the card and on the bus, though we have to
2133                  * align it down.
2134                  */
2135 #if __OS_HAS_AGP
2136                 /* XXX */
2137                 if (dev_priv->flags & RADEON_IS_AGP) {
2138                         base = dev->agp->base;
2139                         /* Check if valid */
2140                         if ((base + dev_priv->gart_size - 1) >= dev_priv->fb_location &&
2141                             base < (dev_priv->fb_location + dev_priv->fb_size - 1)) {
2142                                 DRM_INFO("Can't use AGP base @0x%08lx, won't fit\n",
2143                                          dev->agp->base);
2144                                 base = 0;
2145                         }
2146                 }
2147 #endif
2148                 /* If not or if AGP is at 0 (Macs), try to put it elsewhere */
2149                 if (base == 0) {
2150                         base = dev_priv->fb_location + dev_priv->fb_size;
2151                         if (base < dev_priv->fb_location ||
2152                             ((base + dev_priv->gart_size) & 0xfffffffful) < base)
2153                                 base = dev_priv->fb_location
2154                                         - dev_priv->gart_size;
2155                 }
2156                 dev_priv->gart_vm_start = base & 0xffc00000u;
2157                 if (dev_priv->gart_vm_start != base)
2158                         DRM_INFO("GART aligned down from 0x%08x to 0x%08x\n",
2159                                  base, dev_priv->gart_vm_start);
2160         }
2161
2162 #if __OS_HAS_AGP
2163         /* XXX */
2164         if (dev_priv->flags & RADEON_IS_AGP)
2165                 dev_priv->gart_buffers_offset = (dev->agp_buffer_map->offset
2166                                                  - dev->agp->base
2167                                                  + dev_priv->gart_vm_start);
2168         else
2169 #endif
2170                 dev_priv->gart_buffers_offset = (dev->agp_buffer_map->offset
2171                                                  - (unsigned long)dev->sg->vaddr
2172                                                  + dev_priv->gart_vm_start);
2173
2174         DRM_DEBUG("fb 0x%08x size %d\n",
2175                   (unsigned int) dev_priv->fb_location,
2176                   (unsigned int) dev_priv->fb_size);
2177         DRM_DEBUG("dev_priv->gart_size %d\n", dev_priv->gart_size);
2178         DRM_DEBUG("dev_priv->gart_vm_start 0x%08x\n",
2179                   (unsigned int) dev_priv->gart_vm_start);
2180         DRM_DEBUG("dev_priv->gart_buffers_offset 0x%08lx\n",
2181                   dev_priv->gart_buffers_offset);
2182
2183         dev_priv->ring.start = (u32 *) dev_priv->cp_ring->handle;
2184         dev_priv->ring.end = ((u32 *) dev_priv->cp_ring->handle
2185                               + init->ring_size / sizeof(u32));
2186         dev_priv->ring.size = init->ring_size;
2187         dev_priv->ring.size_l2qw = drm_order(init->ring_size / 8);
2188
2189         dev_priv->ring.rptr_update = /* init->rptr_update */ 4096;
2190         dev_priv->ring.rptr_update_l2qw = drm_order(/* init->rptr_update */ 4096 / 8);
2191
2192         dev_priv->ring.fetch_size = /* init->fetch_size */ 32;
2193         dev_priv->ring.fetch_size_l2ow = drm_order(/* init->fetch_size */ 32 / 16);
2194
2195         dev_priv->ring.tail_mask = (dev_priv->ring.size / sizeof(u32)) - 1;
2196
2197         dev_priv->ring.high_mark = RADEON_RING_HIGH_MARK;
2198
2199 #if __OS_HAS_AGP
2200         if (dev_priv->flags & RADEON_IS_AGP) {
2201                 /* XXX turn off pcie gart */
2202         } else
2203 #endif
2204         {
2205                 dev_priv->gart_info.table_mask = DMA_BIT_MASK(32);
2206                 /* if we have an offset set from userspace */
2207                 if (!dev_priv->pcigart_offset_set) {
2208                         DRM_ERROR("Need gart offset from userspace\n");
2209                         r600_do_cleanup_cp(dev);
2210                         return -EINVAL;
2211                 }
2212
2213                 DRM_DEBUG("Using gart offset 0x%08lx\n", dev_priv->pcigart_offset);
2214
2215                 dev_priv->gart_info.bus_addr =
2216                         dev_priv->pcigart_offset + dev_priv->fb_location;
2217                 dev_priv->gart_info.mapping.offset =
2218                         dev_priv->pcigart_offset + dev_priv->fb_aper_offset;
2219                 dev_priv->gart_info.mapping.size =
2220                         dev_priv->gart_info.table_size;
2221
2222                 drm_core_ioremap_wc(&dev_priv->gart_info.mapping, dev);
2223                 if (!dev_priv->gart_info.mapping.handle) {
2224                         DRM_ERROR("ioremap failed.\n");
2225                         r600_do_cleanup_cp(dev);
2226                         return -EINVAL;
2227                 }
2228
2229                 dev_priv->gart_info.addr =
2230                         dev_priv->gart_info.mapping.handle;
2231
2232                 DRM_DEBUG("Setting phys_pci_gart to %p %08lX\n",
2233                           dev_priv->gart_info.addr,
2234                           dev_priv->pcigart_offset);
2235
2236                 if (!r600_page_table_init(dev)) {
2237                         DRM_ERROR("Failed to init GART table\n");
2238                         r600_do_cleanup_cp(dev);
2239                         return -EINVAL;
2240                 }
2241
2242                 if (((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV770))
2243                         r700_vm_init(dev);
2244                 else
2245                         r600_vm_init(dev);
2246         }
2247
2248         if (!dev_priv->me_fw || !dev_priv->pfp_fw) {
2249                 int err = r600_cp_init_microcode(dev_priv);
2250                 if (err) {
2251                         DRM_ERROR("Failed to load firmware!\n");
2252                         r600_do_cleanup_cp(dev);
2253                         return err;
2254                 }
2255         }
2256         if (((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV770))
2257                 r700_cp_load_microcode(dev_priv);
2258         else
2259                 r600_cp_load_microcode(dev_priv);
2260
2261         r600_cp_init_ring_buffer(dev, dev_priv, file_priv);
2262
2263         dev_priv->last_buf = 0;
2264
2265         r600_do_engine_reset(dev);
2266         r600_test_writeback(dev_priv);
2267
2268         return 0;
2269 }
2270
2271 int r600_do_resume_cp(struct drm_device *dev, struct drm_file *file_priv)
2272 {
2273         drm_radeon_private_t *dev_priv = dev->dev_private;
2274
2275         DRM_DEBUG("\n");
2276         if (((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV770)) {
2277                 r700_vm_init(dev);
2278                 r700_cp_load_microcode(dev_priv);
2279         } else {
2280                 r600_vm_init(dev);
2281                 r600_cp_load_microcode(dev_priv);
2282         }
2283         r600_cp_init_ring_buffer(dev, dev_priv, file_priv);
2284         r600_do_engine_reset(dev);
2285
2286         return 0;
2287 }
2288
2289 /* Wait for the CP to go idle.
2290  */
2291 int r600_do_cp_idle(drm_radeon_private_t *dev_priv)
2292 {
2293         RING_LOCALS;
2294         DRM_DEBUG("\n");
2295
2296         BEGIN_RING(5);
2297         OUT_RING(CP_PACKET3(R600_IT_EVENT_WRITE, 0));
2298         OUT_RING(R600_CACHE_FLUSH_AND_INV_EVENT);
2299         /* wait for 3D idle clean */
2300         OUT_RING(CP_PACKET3(R600_IT_SET_CONFIG_REG, 1));
2301         OUT_RING((R600_WAIT_UNTIL - R600_SET_CONFIG_REG_OFFSET) >> 2);
2302         OUT_RING(RADEON_WAIT_3D_IDLE | RADEON_WAIT_3D_IDLECLEAN);
2303
2304         ADVANCE_RING();
2305         COMMIT_RING();
2306
2307         return r600_do_wait_for_idle(dev_priv);
2308 }
2309
2310 /* Start the Command Processor.
2311  */
2312 void r600_do_cp_start(drm_radeon_private_t *dev_priv)
2313 {
2314         u32 cp_me;
2315         RING_LOCALS;
2316         DRM_DEBUG("\n");
2317
2318         BEGIN_RING(7);
2319         OUT_RING(CP_PACKET3(R600_IT_ME_INITIALIZE, 5));
2320         OUT_RING(0x00000001);
2321         if (((dev_priv->flags & RADEON_FAMILY_MASK) < CHIP_RV770))
2322                 OUT_RING(0x00000003);
2323         else
2324                 OUT_RING(0x00000000);
2325         OUT_RING((dev_priv->r600_max_hw_contexts - 1));
2326         OUT_RING(R600_ME_INITIALIZE_DEVICE_ID(1));
2327         OUT_RING(0x00000000);
2328         OUT_RING(0x00000000);
2329         ADVANCE_RING();
2330         COMMIT_RING();
2331
2332         /* set the mux and reset the halt bit */
2333         cp_me = 0xff;
2334         RADEON_WRITE(R600_CP_ME_CNTL, cp_me);
2335
2336         dev_priv->cp_running = 1;
2337
2338 }
2339
2340 void r600_do_cp_reset(drm_radeon_private_t *dev_priv)
2341 {
2342         u32 cur_read_ptr;
2343         DRM_DEBUG("\n");
2344
2345         cur_read_ptr = RADEON_READ(R600_CP_RB_RPTR);
2346         RADEON_WRITE(R600_CP_RB_WPTR, cur_read_ptr);
2347         SET_RING_HEAD(dev_priv, cur_read_ptr);
2348         dev_priv->ring.tail = cur_read_ptr;
2349 }
2350
2351 void r600_do_cp_stop(drm_radeon_private_t *dev_priv)
2352 {
2353         uint32_t cp_me;
2354
2355         DRM_DEBUG("\n");
2356
2357         cp_me = 0xff | R600_CP_ME_HALT;
2358
2359         RADEON_WRITE(R600_CP_ME_CNTL, cp_me);
2360
2361         dev_priv->cp_running = 0;
2362 }
2363
2364 int r600_cp_dispatch_indirect(struct drm_device *dev,
2365                               struct drm_buf *buf, int start, int end)
2366 {
2367         drm_radeon_private_t *dev_priv = dev->dev_private;
2368         RING_LOCALS;
2369
2370         if (start != end) {
2371                 unsigned long offset = (dev_priv->gart_buffers_offset
2372                                         + buf->offset + start);
2373                 int dwords = (end - start + 3) / sizeof(u32);
2374
2375                 DRM_DEBUG("dwords:%d\n", dwords);
2376                 DRM_DEBUG("offset 0x%lx\n", offset);
2377
2378
2379                 /* Indirect buffer data must be a multiple of 16 dwords.
2380                  * pad the data with a Type-2 CP packet.
2381                  */
2382                 while (dwords & 0xf) {
2383                         u32 *data = (u32 *)
2384                             ((char *)dev->agp_buffer_map->handle
2385                              + buf->offset + start);
2386                         data[dwords++] = RADEON_CP_PACKET2;
2387                 }
2388
2389                 /* Fire off the indirect buffer */
2390                 BEGIN_RING(4);
2391                 OUT_RING(CP_PACKET3(R600_IT_INDIRECT_BUFFER, 2));
2392                 OUT_RING((offset & 0xfffffffc));
2393                 OUT_RING((upper_32_bits(offset) & 0xff));
2394                 OUT_RING(dwords);
2395                 ADVANCE_RING();
2396         }
2397
2398         return 0;
2399 }
2400
2401 void r600_cp_dispatch_swap(struct drm_device *dev, struct drm_file *file_priv)
2402 {
2403         drm_radeon_private_t *dev_priv = dev->dev_private;
2404         struct drm_master *master = file_priv->masterp;
2405         struct drm_radeon_master_private *master_priv = master->driver_priv;
2406         drm_radeon_sarea_t *sarea_priv = master_priv->sarea_priv;
2407         int nbox = sarea_priv->nbox;
2408         struct drm_clip_rect *pbox = sarea_priv->boxes;
2409         int i, cpp, src_pitch, dst_pitch;
2410         uint64_t src, dst;
2411         RING_LOCALS;
2412         DRM_DEBUG("\n");
2413
2414         if (dev_priv->color_fmt == RADEON_COLOR_FORMAT_ARGB8888)
2415                 cpp = 4;
2416         else
2417                 cpp = 2;
2418
2419         if (sarea_priv->pfCurrentPage == 0) {
2420                 src_pitch = dev_priv->back_pitch;
2421                 dst_pitch = dev_priv->front_pitch;
2422                 src = dev_priv->back_offset + dev_priv->fb_location;
2423                 dst = dev_priv->front_offset + dev_priv->fb_location;
2424         } else {
2425                 src_pitch = dev_priv->front_pitch;
2426                 dst_pitch = dev_priv->back_pitch;
2427                 src = dev_priv->front_offset + dev_priv->fb_location;
2428                 dst = dev_priv->back_offset + dev_priv->fb_location;
2429         }
2430
2431         if (r600_prepare_blit_copy(dev, file_priv)) {
2432                 DRM_ERROR("unable to allocate vertex buffer for swap buffer\n");
2433                 return;
2434         }
2435         for (i = 0; i < nbox; i++) {
2436                 int x = pbox[i].x1;
2437                 int y = pbox[i].y1;
2438                 int w = pbox[i].x2 - x;
2439                 int h = pbox[i].y2 - y;
2440
2441                 DRM_DEBUG("%d,%d-%d,%d\n", x, y, w, h);
2442
2443                 r600_blit_swap(dev,
2444                                src, dst,
2445                                x, y, x, y, w, h,
2446                                src_pitch, dst_pitch, cpp);
2447         }
2448         r600_done_blit_copy(dev);
2449
2450         /* Increment the frame counter.  The client-side 3D driver must
2451          * throttle the framerate by waiting for this value before
2452          * performing the swapbuffer ioctl.
2453          */
2454         sarea_priv->last_frame++;
2455
2456         BEGIN_RING(3);
2457         R600_FRAME_AGE(sarea_priv->last_frame);
2458         ADVANCE_RING();
2459 }
2460
2461 int r600_cp_dispatch_texture(struct drm_device *dev,
2462                              struct drm_file *file_priv,
2463                              drm_radeon_texture_t *tex,
2464                              drm_radeon_tex_image_t *image)
2465 {
2466         drm_radeon_private_t *dev_priv = dev->dev_private;
2467         struct drm_buf *buf;
2468         u32 *buffer;
2469         const u8 __user *data;
2470         int size, pass_size;
2471         u64 src_offset, dst_offset;
2472
2473         if (!radeon_check_offset(dev_priv, tex->offset)) {
2474                 DRM_ERROR("Invalid destination offset\n");
2475                 return -EINVAL;
2476         }
2477
2478         /* this might fail for zero-sized uploads - are those illegal? */
2479         if (!radeon_check_offset(dev_priv, tex->offset + tex->height * tex->pitch - 1)) {
2480                 DRM_ERROR("Invalid final destination offset\n");
2481                 return -EINVAL;
2482         }
2483
2484         size = tex->height * tex->pitch;
2485
2486         if (size == 0)
2487                 return 0;
2488
2489         dst_offset = tex->offset;
2490
2491         if (r600_prepare_blit_copy(dev, file_priv)) {
2492                 DRM_ERROR("unable to allocate vertex buffer for swap buffer\n");
2493                 return -EAGAIN;
2494         }
2495         do {
2496                 data = (const u8 __user *)image->data;
2497                 pass_size = size;
2498
2499                 buf = radeon_freelist_get(dev);
2500                 if (!buf) {
2501                         DRM_DEBUG("EAGAIN\n");
2502                         if (DRM_COPY_TO_USER(tex->image, image, sizeof(*image)))
2503                                 return -EFAULT;
2504                         return -EAGAIN;
2505                 }
2506
2507                 if (pass_size > buf->total)
2508                         pass_size = buf->total;
2509
2510                 /* Dispatch the indirect buffer.
2511                  */
2512                 buffer =
2513                     (u32 *) ((char *)dev->agp_buffer_map->handle + buf->offset);
2514
2515                 if (DRM_COPY_FROM_USER(buffer, data, pass_size)) {
2516                         DRM_ERROR("EFAULT on pad, %d bytes\n", pass_size);
2517                         return -EFAULT;
2518                 }
2519
2520                 buf->file_priv = file_priv;
2521                 buf->used = pass_size;
2522                 src_offset = dev_priv->gart_buffers_offset + buf->offset;
2523
2524                 r600_blit_copy(dev, src_offset, dst_offset, pass_size);
2525
2526                 radeon_cp_discard_buffer(dev, file_priv->masterp, buf);
2527
2528                 /* Update the input parameters for next time */
2529                 image->data = (const u8 __user *)image->data + pass_size;
2530                 dst_offset += pass_size;
2531                 size -= pass_size;
2532         } while (size > 0);
2533         r600_done_blit_copy(dev);
2534
2535         return 0;
2536 }
2537
2538 /*
2539  * Legacy cs ioctl
2540  */
2541 static u32 radeon_cs_id_get(struct drm_radeon_private *radeon)
2542 {
2543         /* FIXME: check if wrap affect last reported wrap & sequence */
2544         radeon->cs_id_scnt = (radeon->cs_id_scnt + 1) & 0x00FFFFFF;
2545         if (!radeon->cs_id_scnt) {
2546                 /* increment wrap counter */
2547                 radeon->cs_id_wcnt += 0x01000000;
2548                 /* valid sequence counter start at 1 */
2549                 radeon->cs_id_scnt = 1;
2550         }
2551         return (radeon->cs_id_scnt | radeon->cs_id_wcnt);
2552 }
2553
2554 static void r600_cs_id_emit(drm_radeon_private_t *dev_priv, u32 *id)
2555 {
2556         RING_LOCALS;
2557
2558         *id = radeon_cs_id_get(dev_priv);
2559
2560         /* SCRATCH 2 */
2561         BEGIN_RING(3);
2562         R600_CLEAR_AGE(*id);
2563         ADVANCE_RING();
2564         COMMIT_RING();
2565 }
2566
2567 static int r600_ib_get(struct drm_device *dev,
2568                         struct drm_file *fpriv,
2569                         struct drm_buf **buffer)
2570 {
2571         struct drm_buf *buf;
2572
2573         *buffer = NULL;
2574         buf = radeon_freelist_get(dev);
2575         if (!buf) {
2576                 return -EBUSY;
2577         }
2578         buf->file_priv = fpriv;
2579         *buffer = buf;
2580         return 0;
2581 }
2582
2583 static void r600_ib_free(struct drm_device *dev, struct drm_buf *buf,
2584                         struct drm_file *fpriv, int l, int r)
2585 {
2586         drm_radeon_private_t *dev_priv = dev->dev_private;
2587
2588         if (buf) {
2589                 if (!r)
2590                         r600_cp_dispatch_indirect(dev, buf, 0, l * 4);
2591                 radeon_cp_discard_buffer(dev, fpriv->masterp, buf);
2592                 COMMIT_RING();
2593         }
2594 }
2595
2596 int r600_cs_legacy_ioctl(struct drm_device *dev, void *data, struct drm_file *fpriv)
2597 {
2598         struct drm_radeon_private *dev_priv = dev->dev_private;
2599         struct drm_radeon_cs *cs = data;
2600         struct drm_buf *buf;
2601         unsigned family;
2602         int l, r = 0;
2603         u32 *ib, cs_id = 0;
2604
2605         if (dev_priv == NULL) {
2606                 DRM_ERROR("called with no initialization\n");
2607                 return -EINVAL;
2608         }
2609         family = dev_priv->flags & RADEON_FAMILY_MASK;
2610         if (family < CHIP_R600) {
2611                 DRM_ERROR("cs ioctl valid only for R6XX & R7XX in legacy mode\n");
2612                 return -EINVAL;
2613         }
2614         sx_xlock(&dev_priv->cs_mutex);
2615         /* get ib */
2616         l = 0;
2617         r = r600_ib_get(dev, fpriv, &buf);
2618         if (r) {
2619                 DRM_ERROR("ib_get failed\n");
2620                 goto out;
2621         }
2622         ib = (u32 *)((uintptr_t)dev->agp_buffer_map->handle + buf->offset);
2623         /* now parse command stream */
2624         r = r600_cs_legacy(dev, data,  fpriv, family, ib, &l);
2625         if (r) {
2626                 goto out;
2627         }
2628
2629 out:
2630         r600_ib_free(dev, buf, fpriv, l, r);
2631         /* emit cs id sequence */
2632         r600_cs_id_emit(dev_priv, &cs_id);
2633         cs->cs_id = cs_id;
2634         sx_xunlock(&dev_priv->cs_mutex);
2635         return r;
2636 }
2637
2638 void r600_cs_legacy_get_tiling_conf(struct drm_device *dev, u32 *npipes, u32 *nbanks, u32 *group_size)
2639 {
2640         struct drm_radeon_private *dev_priv = dev->dev_private;
2641
2642         *npipes = dev_priv->r600_npipes;
2643         *nbanks = dev_priv->r600_nbanks;
2644         *group_size = dev_priv->r600_group_size;
2645 }