sys/dev/drm/radeon_cp.c

   1 /*-
   2  * Copyright 2000 Precision Insight, Inc., Cedar Park, Texas.
   3  * Copyright 2000 VA Linux Systems, Inc., Fremont, California.
   4  * Copyright 2007 Advanced Micro Devices, Inc.
   5  * All Rights Reserved.
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining a
   8  * copy of this software and associated documentation files (the "Software"),
   9  * to deal in the Software without restriction, including without limitation
  10  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  11  * and/or sell copies of the Software, and to permit persons to whom the
  12  * Software is furnished to do so, subject to the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the next
  15  * paragraph) shall be included in all copies or substantial portions of the
  16  * Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  21  * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  22  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  23  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  24  * DEALINGS IN THE SOFTWARE.
  25  *
  26  * Authors:
  27  *    Kevin E. Martin <martin@valinux.com>
  28  *    Gareth Hughes <gareth@valinux.com>
  29  */
  30
  31 #include <sys/cdefs.h>
  32 __FBSDID("$FreeBSD$");
  33
  34 #include "dev/drm/drmP.h"
  35 #include "dev/drm/drm.h"
  36 #include "dev/drm/drm_sarea.h"
  37 #include "dev/drm/radeon_drm.h"
  38 #include "dev/drm/radeon_drv.h"
  39 #include "dev/drm/r300_reg.h"
  40
  41 #include "dev/drm/radeon_microcode.h"
  42
  43 #define RADEON_FIFO_DEBUG       0
  44
  45 static int radeon_do_cleanup_cp(struct drm_device * dev);
  46 static void radeon_do_cp_start(drm_radeon_private_t * dev_priv);
  47
  48 u32 radeon_read_ring_rptr(drm_radeon_private_t *dev_priv, u32 off)
  49 {
  50         u32 val;
  51
  52         if (dev_priv->flags & RADEON_IS_AGP) {
  53                 val = DRM_READ32(dev_priv->ring_rptr, off);
  54         } else {
  55                 val = *(((volatile u32 *)
  56                          dev_priv->ring_rptr->handle) +
  57                         (off / sizeof(u32)));
  58                 val = le32_to_cpu(val);
  59         }
  60         return val;
  61 }
  62
  63 u32 radeon_get_ring_head(drm_radeon_private_t *dev_priv)
  64 {
  65         if (dev_priv->writeback_works)
  66                 return radeon_read_ring_rptr(dev_priv, 0);
  67         else {
  68                 if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
  69                         return RADEON_READ(R600_CP_RB_RPTR);
  70                 else
  71                         return RADEON_READ(RADEON_CP_RB_RPTR);
  72         }
  73 }
  74
  75 void radeon_write_ring_rptr(drm_radeon_private_t *dev_priv, u32 off, u32 val)
  76 {
  77         if (dev_priv->flags & RADEON_IS_AGP)
  78                 DRM_WRITE32(dev_priv->ring_rptr, off, val);
  79         else
  80                 *(((volatile u32 *) dev_priv->ring_rptr->handle) +
  81                   (off / sizeof(u32))) = cpu_to_le32(val);
  82 }
  83
  84 void radeon_set_ring_head(drm_radeon_private_t *dev_priv, u32 val)
  85 {
  86         radeon_write_ring_rptr(dev_priv, 0, val);
  87 }
  88
  89 u32 radeon_get_scratch(drm_radeon_private_t *dev_priv, int index)
  90 {
  91         if (dev_priv->writeback_works) {
  92                 if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
  93                         return radeon_read_ring_rptr(dev_priv,
  94                                                      R600_SCRATCHOFF(index));
  95                 else
  96                         return radeon_read_ring_rptr(dev_priv,
  97                                                      RADEON_SCRATCHOFF(index));
  98         } else {
  99                 if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
 100                         return RADEON_READ(R600_SCRATCH_REG0 + 4*index);
 101                 else
 102                         return RADEON_READ(RADEON_SCRATCH_REG0 + 4*index);
 103         }
 104 }
 105
 106 u32 RADEON_READ_MM(drm_radeon_private_t *dev_priv, int addr)
 107 {
 108         u32 ret;
 109
 110         if (addr < 0x10000)
 111                 ret = DRM_READ32(dev_priv->mmio, addr);
 112         else {
 113                 DRM_WRITE32(dev_priv->mmio, RADEON_MM_INDEX, addr);
 114                 ret = DRM_READ32(dev_priv->mmio, RADEON_MM_DATA);
 115         }
 116
 117         return ret;
 118 }
 119
 120 static u32 R500_READ_MCIND(drm_radeon_private_t *dev_priv, int addr)
 121 {
 122         u32 ret;
 123         RADEON_WRITE(R520_MC_IND_INDEX, 0x7f0000 | (addr & 0xff));
 124         ret = RADEON_READ(R520_MC_IND_DATA);
 125         RADEON_WRITE(R520_MC_IND_INDEX, 0);
 126         return ret;
 127 }
 128
 129 static u32 RS480_READ_MCIND(drm_radeon_private_t *dev_priv, int addr)
 130 {
 131         u32 ret;
 132         RADEON_WRITE(RS480_NB_MC_INDEX, addr & 0xff);
 133         ret = RADEON_READ(RS480_NB_MC_DATA);
 134         RADEON_WRITE(RS480_NB_MC_INDEX, 0xff);
 135         return ret;
 136 }
 137
 138 static u32 RS690_READ_MCIND(drm_radeon_private_t *dev_priv, int addr)
 139 {
 140         u32 ret;
 141         RADEON_WRITE(RS690_MC_INDEX, (addr & RS690_MC_INDEX_MASK));
 142         ret = RADEON_READ(RS690_MC_DATA);
 143         RADEON_WRITE(RS690_MC_INDEX, RS690_MC_INDEX_MASK);
 144         return ret;
 145 }
 146
 147 static u32 RS600_READ_MCIND(drm_radeon_private_t *dev_priv, int addr)
 148 {
 149         u32 ret;
 150         RADEON_WRITE(RS600_MC_INDEX, ((addr & RS600_MC_ADDR_MASK) |
 151                                       RS600_MC_IND_CITF_ARB0));
 152         ret = RADEON_READ(RS600_MC_DATA);
 153         return ret;
 154 }
 155
 156 static u32 IGP_READ_MCIND(drm_radeon_private_t *dev_priv, int addr)
 157 {
 158         if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690) ||
 159             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS740))
 160                 return RS690_READ_MCIND(dev_priv, addr);
 161         else if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS600)
 162                 return RS600_READ_MCIND(dev_priv, addr);
 163         else
 164                 return RS480_READ_MCIND(dev_priv, addr);
 165 }
 166
 167 u32 radeon_read_fb_location(drm_radeon_private_t *dev_priv)
 168 {
 169
 170         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV770)
 171                 return RADEON_READ(R700_MC_VM_FB_LOCATION);
 172         else if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
 173                 return RADEON_READ(R600_MC_VM_FB_LOCATION);
 174         else if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV515)
 175                 return R500_READ_MCIND(dev_priv, RV515_MC_FB_LOCATION);
 176         else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690) ||
 177                  ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS740))
 178                 return RS690_READ_MCIND(dev_priv, RS690_MC_FB_LOCATION);
 179         else if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS600)
 180                 return RS600_READ_MCIND(dev_priv, RS600_MC_FB_LOCATION);
 181         else if ((dev_priv->flags & RADEON_FAMILY_MASK) > CHIP_RV515)
 182                 return R500_READ_MCIND(dev_priv, R520_MC_FB_LOCATION);
 183         else
 184                 return RADEON_READ(RADEON_MC_FB_LOCATION);
 185 }
 186
 187 static void radeon_write_fb_location(drm_radeon_private_t *dev_priv, u32 fb_loc)
 188 {
 189         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV770)
 190                 RADEON_WRITE(R700_MC_VM_FB_LOCATION, fb_loc);
 191         else if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
 192                 RADEON_WRITE(R600_MC_VM_FB_LOCATION, fb_loc);
 193         else if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV515)
 194                 R500_WRITE_MCIND(RV515_MC_FB_LOCATION, fb_loc);
 195         else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690) ||
 196                  ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS740))
 197                 RS690_WRITE_MCIND(RS690_MC_FB_LOCATION, fb_loc);
 198         else if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS600)
 199                 RS600_WRITE_MCIND(RS600_MC_FB_LOCATION, fb_loc);
 200         else if ((dev_priv->flags & RADEON_FAMILY_MASK) > CHIP_RV515)
 201                 R500_WRITE_MCIND(R520_MC_FB_LOCATION, fb_loc);
 202         else
 203                 RADEON_WRITE(RADEON_MC_FB_LOCATION, fb_loc);
 204 }
 205
 206 void radeon_write_agp_location(drm_radeon_private_t *dev_priv, u32 agp_loc)
 207 {
 208         /*R6xx/R7xx: AGP_TOP and BOT are actually 18 bits each */
 209         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV770) {
 210                 RADEON_WRITE(R700_MC_VM_AGP_BOT, agp_loc & 0xffff); /* FIX ME */
 211                 RADEON_WRITE(R700_MC_VM_AGP_TOP, (agp_loc >> 16) & 0xffff);
 212         } else if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600) {
 213                 RADEON_WRITE(R600_MC_VM_AGP_BOT, agp_loc & 0xffff); /* FIX ME */
 214                 RADEON_WRITE(R600_MC_VM_AGP_TOP, (agp_loc >> 16) & 0xffff);
 215         } else if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV515)
 216                 R500_WRITE_MCIND(RV515_MC_AGP_LOCATION, agp_loc);
 217         else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690) ||
 218                  ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS740))
 219                 RS690_WRITE_MCIND(RS690_MC_AGP_LOCATION, agp_loc);
 220         else if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS600)
 221                 RS600_WRITE_MCIND(RS600_MC_AGP_LOCATION, agp_loc);
 222         else if ((dev_priv->flags & RADEON_FAMILY_MASK) > CHIP_RV515)
 223                 R500_WRITE_MCIND(R520_MC_AGP_LOCATION, agp_loc);
 224         else
 225                 RADEON_WRITE(RADEON_MC_AGP_LOCATION, agp_loc);
 226 }
 227
 228 void radeon_write_agp_base(drm_radeon_private_t *dev_priv, u64 agp_base)
 229 {
 230         u32 agp_base_hi = upper_32_bits(agp_base);
 231         u32 agp_base_lo = agp_base & 0xffffffff;
 232         u32 r6xx_agp_base = (agp_base >> 22) & 0x3ffff;
 233
 234         /* R6xx/R7xx must be aligned to a 4MB boundry */
 235         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV770)
 236                 RADEON_WRITE(R700_MC_VM_AGP_BASE, r6xx_agp_base);
 237         else if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
 238                 RADEON_WRITE(R600_MC_VM_AGP_BASE, r6xx_agp_base);
 239         else if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV515) {
 240                 R500_WRITE_MCIND(RV515_MC_AGP_BASE, agp_base_lo);
 241                 R500_WRITE_MCIND(RV515_MC_AGP_BASE_2, agp_base_hi);
 242         } else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690) ||
 243                  ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS740)) {
 244                 RS690_WRITE_MCIND(RS690_MC_AGP_BASE, agp_base_lo);
 245                 RS690_WRITE_MCIND(RS690_MC_AGP_BASE_2, agp_base_hi);
 246         } else if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS600) {
 247                 RS600_WRITE_MCIND(RS600_AGP_BASE, agp_base_lo);
 248                 RS600_WRITE_MCIND(RS600_AGP_BASE_2, agp_base_hi);
 249         } else if ((dev_priv->flags & RADEON_FAMILY_MASK) > CHIP_RV515) {
 250                 R500_WRITE_MCIND(R520_MC_AGP_BASE, agp_base_lo);
 251                 R500_WRITE_MCIND(R520_MC_AGP_BASE_2, agp_base_hi);
 252         } else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS400) ||
 253                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS480)) {
 254                 RADEON_WRITE(RADEON_AGP_BASE, agp_base_lo);
 255                 RADEON_WRITE(RS480_AGP_BASE_2, agp_base_hi);
 256         } else {
 257                 RADEON_WRITE(RADEON_AGP_BASE, agp_base_lo);
 258                 if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R200)
 259                         RADEON_WRITE(RADEON_AGP_BASE_2, agp_base_hi);
 260         }
 261 }
 262
 263 void radeon_enable_bm(struct drm_radeon_private *dev_priv)
 264 {
 265         u32 tmp;
 266         /* Turn on bus mastering */
 267         if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690) ||
 268             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS740)) {
 269                 /* rs600/rs690/rs740 */
 270                 tmp = RADEON_READ(RADEON_BUS_CNTL) & ~RS600_BUS_MASTER_DIS;
 271                 RADEON_WRITE(RADEON_BUS_CNTL, tmp);
 272         } else if (((dev_priv->flags & RADEON_FAMILY_MASK) <= CHIP_RV350) ||
 273                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R420) ||
 274                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS400) ||
 275                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS480)) {
 276                 /* r1xx, r2xx, r300, r(v)350, r420/r481, rs400/rs480 */
 277                 tmp = RADEON_READ(RADEON_BUS_CNTL) & ~RADEON_BUS_MASTER_DIS;
 278                 RADEON_WRITE(RADEON_BUS_CNTL, tmp);
 279         } /* PCIE cards appears to not need this */
 280 }
 281
 282 static int RADEON_READ_PLL(struct drm_device * dev, int addr)
 283 {
 284         drm_radeon_private_t *dev_priv = dev->dev_private;
 285
 286         RADEON_WRITE8(RADEON_CLOCK_CNTL_INDEX, addr & 0x1f);
 287         return RADEON_READ(RADEON_CLOCK_CNTL_DATA);
 288 }
 289
 290 static u32 RADEON_READ_PCIE(drm_radeon_private_t *dev_priv, int addr)
 291 {
 292         RADEON_WRITE8(RADEON_PCIE_INDEX, addr & 0xff);
 293         return RADEON_READ(RADEON_PCIE_DATA);
 294 }
 295
 296 #if RADEON_FIFO_DEBUG
 297 static void radeon_status(drm_radeon_private_t * dev_priv)
 298 {
 299         printk("%s:\n", __func__);
 300         printk("RBBM_STATUS = 0x%08x\n",
 301                (unsigned int)RADEON_READ(RADEON_RBBM_STATUS));
 302         printk("CP_RB_RTPR = 0x%08x\n",
 303                (unsigned int)RADEON_READ(RADEON_CP_RB_RPTR));
 304         printk("CP_RB_WTPR = 0x%08x\n",
 305                (unsigned int)RADEON_READ(RADEON_CP_RB_WPTR));
 306         printk("AIC_CNTL = 0x%08x\n",
 307                (unsigned int)RADEON_READ(RADEON_AIC_CNTL));
 308         printk("AIC_STAT = 0x%08x\n",
 309                (unsigned int)RADEON_READ(RADEON_AIC_STAT));
 310         printk("AIC_PT_BASE = 0x%08x\n",
 311                (unsigned int)RADEON_READ(RADEON_AIC_PT_BASE));
 312         printk("TLB_ADDR = 0x%08x\n",
 313                (unsigned int)RADEON_READ(RADEON_AIC_TLB_ADDR));
 314         printk("TLB_DATA = 0x%08x\n",
 315                (unsigned int)RADEON_READ(RADEON_AIC_TLB_DATA));
 316 }
 317 #endif
 318
 319 /* ================================================================
 320  * Engine, FIFO control
 321  */
 322
 323 static int radeon_do_pixcache_flush(drm_radeon_private_t * dev_priv)
 324 {
 325         u32 tmp;
 326         int i;
 327
 328         dev_priv->stats.boxes |= RADEON_BOX_WAIT_IDLE;
 329
 330         if ((dev_priv->flags & RADEON_FAMILY_MASK) <= CHIP_RV280) {
 331                 tmp = RADEON_READ(RADEON_RB3D_DSTCACHE_CTLSTAT);
 332                 tmp |= RADEON_RB3D_DC_FLUSH_ALL;
 333                 RADEON_WRITE(RADEON_RB3D_DSTCACHE_CTLSTAT, tmp);
 334
 335                 for (i = 0; i < dev_priv->usec_timeout; i++) {
 336                         if (!(RADEON_READ(RADEON_RB3D_DSTCACHE_CTLSTAT)
 337                               & RADEON_RB3D_DC_BUSY)) {
 338                                 return 0;
 339                         }
 340                         DRM_UDELAY(1);
 341                 }
 342         } else {
 343                 /* don't flush or purge cache here or lockup */
 344                 return 0;
 345         }
 346
 347 #if RADEON_FIFO_DEBUG
 348         DRM_ERROR("failed!\n");
 349         radeon_status(dev_priv);
 350 #endif
 351         return -EBUSY;
 352 }
 353
 354 static int radeon_do_wait_for_fifo(drm_radeon_private_t * dev_priv, int entries)
 355 {
 356         int i;
 357
 358         dev_priv->stats.boxes |= RADEON_BOX_WAIT_IDLE;
 359
 360         for (i = 0; i < dev_priv->usec_timeout; i++) {
 361                 int slots = (RADEON_READ(RADEON_RBBM_STATUS)
 362                              & RADEON_RBBM_FIFOCNT_MASK);
 363                 if (slots >= entries)
 364                         return 0;
 365                 DRM_UDELAY(1);
 366         }
 367         DRM_DEBUG("wait for fifo failed status : 0x%08X 0x%08X\n",
 368                  RADEON_READ(RADEON_RBBM_STATUS),
 369                  RADEON_READ(R300_VAP_CNTL_STATUS));
 370
 371 #if RADEON_FIFO_DEBUG
 372         DRM_ERROR("failed!\n");
 373         radeon_status(dev_priv);
 374 #endif
 375         return -EBUSY;
 376 }
 377
 378 static int radeon_do_wait_for_idle(drm_radeon_private_t * dev_priv)
 379 {
 380         int i, ret;
 381
 382         dev_priv->stats.boxes |= RADEON_BOX_WAIT_IDLE;
 383
 384         ret = radeon_do_wait_for_fifo(dev_priv, 64);
 385         if (ret)
 386                 return ret;
 387
 388         for (i = 0; i < dev_priv->usec_timeout; i++) {
 389                 if (!(RADEON_READ(RADEON_RBBM_STATUS)
 390                       & RADEON_RBBM_ACTIVE)) {
 391                         radeon_do_pixcache_flush(dev_priv);
 392                         return 0;
 393                 }
 394                 DRM_UDELAY(1);
 395         }
 396         DRM_DEBUG("wait idle failed status : 0x%08X 0x%08X\n",
 397                  RADEON_READ(RADEON_RBBM_STATUS),
 398                  RADEON_READ(R300_VAP_CNTL_STATUS));
 399
 400 #if RADEON_FIFO_DEBUG
 401         DRM_ERROR("failed!\n");
 402         radeon_status(dev_priv);
 403 #endif
 404         return -EBUSY;
 405 }
 406
 407 static void radeon_init_pipes(drm_radeon_private_t *dev_priv)
 408 {
 409         uint32_t gb_tile_config, gb_pipe_sel = 0;
 410
 411         /* RS4xx/RS6xx/R4xx/R5xx */
 412         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R420) {
 413                 gb_pipe_sel = RADEON_READ(R400_GB_PIPE_SELECT);
 414                 dev_priv->num_gb_pipes = ((gb_pipe_sel >> 12) & 0x3) + 1;
 415         } else {
 416                 /* R3xx */
 417                 if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R300) ||
 418                     ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R350)) {
 419                         dev_priv->num_gb_pipes = 2;
 420                 } else {
 421                         /* R3Vxx */
 422                         dev_priv->num_gb_pipes = 1;
 423                 }
 424         }
 425         DRM_INFO("Num pipes: %d\n", dev_priv->num_gb_pipes);
 426
 427         gb_tile_config = (R300_ENABLE_TILING | R300_TILE_SIZE_16 /*| R300_SUBPIXEL_1_16*/);
 428
 429         switch (dev_priv->num_gb_pipes) {
 430         case 2: gb_tile_config |= R300_PIPE_COUNT_R300; break;
 431         case 3: gb_tile_config |= R300_PIPE_COUNT_R420_3P; break;
 432         case 4: gb_tile_config |= R300_PIPE_COUNT_R420; break;
 433         default:
 434         case 1: gb_tile_config |= R300_PIPE_COUNT_RV350; break;
 435         }
 436
 437         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV515) {
 438                 RADEON_WRITE_PLL(R500_DYN_SCLK_PWMEM_PIPE, (1 | ((gb_pipe_sel >> 8) & 0xf) << 4));
 439                 RADEON_WRITE(R500_SU_REG_DEST, ((1 << dev_priv->num_gb_pipes) - 1));
 440         }
 441         RADEON_WRITE(R300_GB_TILE_CONFIG, gb_tile_config);
 442         radeon_do_wait_for_idle(dev_priv);
 443         RADEON_WRITE(R300_DST_PIPE_CONFIG, RADEON_READ(R300_DST_PIPE_CONFIG) | R300_PIPE_AUTO_CONFIG);
 444         RADEON_WRITE(R300_RB2D_DSTCACHE_MODE, (RADEON_READ(R300_RB2D_DSTCACHE_MODE) |
 445                                                R300_DC_AUTOFLUSH_ENABLE |
 446                                                R300_DC_DC_DISABLE_IGNORE_PE));
 447
 448
 449 }
 450
 451 /* ================================================================
 452  * CP control, initialization
 453  */
 454
 455 /* Load the microcode for the CP */
 456 static void radeon_cp_load_microcode(drm_radeon_private_t * dev_priv)
 457 {
 458         int i;
 459         DRM_DEBUG("\n");
 460
 461         radeon_do_wait_for_idle(dev_priv);
 462
 463         RADEON_WRITE(RADEON_CP_ME_RAM_ADDR, 0);
 464         if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R100) ||
 465             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV100) ||
 466             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV200) ||
 467             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS100) ||
 468             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS200)) {
 469                 DRM_INFO("Loading R100 Microcode\n");
 470                 for (i = 0; i < 256; i++) {
 471                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAH,
 472                                      R100_cp_microcode[i][1]);
 473                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAL,
 474                                      R100_cp_microcode[i][0]);
 475                 }
 476         } else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R200) ||
 477                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV250) ||
 478                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV280) ||
 479                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS300)) {
 480                 DRM_INFO("Loading R200 Microcode\n");
 481                 for (i = 0; i < 256; i++) {
 482                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAH,
 483                                      R200_cp_microcode[i][1]);
 484                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAL,
 485                                      R200_cp_microcode[i][0]);
 486                 }
 487         } else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R300) ||
 488                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R350) ||
 489                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV350) ||
 490                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV380) ||
 491                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS400) ||
 492                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS480)) {
 493                 DRM_INFO("Loading R300 Microcode\n");
 494                 for (i = 0; i < 256; i++) {
 495                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAH,
 496                                      R300_cp_microcode[i][1]);
 497                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAL,
 498                                      R300_cp_microcode[i][0]);
 499                 }
 500         } else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R420) ||
 501                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R423) ||
 502                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV410)) {
 503                 DRM_INFO("Loading R400 Microcode\n");
 504                 for (i = 0; i < 256; i++) {
 505                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAH,
 506                                      R420_cp_microcode[i][1]);
 507                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAL,
 508                                      R420_cp_microcode[i][0]);
 509                 }
 510         } else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690) ||
 511                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS740)) {
 512                 DRM_INFO("Loading RS690/RS740 Microcode\n");
 513                 for (i = 0; i < 256; i++) {
 514                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAH,
 515                                      RS690_cp_microcode[i][1]);
 516                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAL,
 517                                      RS690_cp_microcode[i][0]);
 518                 }
 519         } else if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS600) {
 520                 DRM_INFO("Loading RS600 Microcode\n");
 521                 for (i = 0; i < 256; i++) {
 522                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAH,
 523                                      RS600_cp_microcode[i][1]);
 524                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAL,
 525                                      RS600_cp_microcode[i][0]);
 526                 }
 527         } else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV515) ||
 528                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R520) ||
 529                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV530) ||
 530                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R580) ||
 531                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV560) ||
 532                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV570)) {
 533                 DRM_INFO("Loading R500 Microcode\n");
 534                 for (i = 0; i < 256; i++) {
 535                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAH,
 536                                      R520_cp_microcode[i][1]);
 537                         RADEON_WRITE(RADEON_CP_ME_RAM_DATAL,
 538                                      R520_cp_microcode[i][0]);
 539                 }
 540         }
 541 }
 542
 543 /* Flush any pending commands to the CP.  This should only be used just
 544  * prior to a wait for idle, as it informs the engine that the command
 545  * stream is ending.
 546  */
 547 static void radeon_do_cp_flush(drm_radeon_private_t * dev_priv)
 548 {
 549         DRM_DEBUG("\n");
 550 #if 0
 551         u32 tmp;
 552
 553         tmp = RADEON_READ(RADEON_CP_RB_WPTR) | (1 << 31);
 554         RADEON_WRITE(RADEON_CP_RB_WPTR, tmp);
 555 #endif
 556 }
 557
 558 /* Wait for the CP to go idle.
 559  */
 560 int radeon_do_cp_idle(drm_radeon_private_t * dev_priv)
 561 {
 562         RING_LOCALS;
 563         DRM_DEBUG("\n");
 564
 565         BEGIN_RING(6);
 566
 567         RADEON_PURGE_CACHE();
 568         RADEON_PURGE_ZCACHE();
 569         RADEON_WAIT_UNTIL_IDLE();
 570
 571         ADVANCE_RING();
 572         COMMIT_RING();
 573
 574         return radeon_do_wait_for_idle(dev_priv);
 575 }
 576
 577 /* Start the Command Processor.
 578  */
 579 static void radeon_do_cp_start(drm_radeon_private_t * dev_priv)
 580 {
 581         RING_LOCALS;
 582         DRM_DEBUG("\n");
 583
 584         radeon_do_wait_for_idle(dev_priv);
 585
 586         RADEON_WRITE(RADEON_CP_CSQ_CNTL, dev_priv->cp_mode);
 587
 588         dev_priv->cp_running = 1;
 589
 590         BEGIN_RING(8);
 591         /* isync can only be written through cp on r5xx write it here */
 592         OUT_RING(CP_PACKET0(RADEON_ISYNC_CNTL, 0));
 593         OUT_RING(RADEON_ISYNC_ANY2D_IDLE3D |
 594                  RADEON_ISYNC_ANY3D_IDLE2D |
 595                  RADEON_ISYNC_WAIT_IDLEGUI |
 596                  RADEON_ISYNC_CPSCRATCH_IDLEGUI);
 597         RADEON_PURGE_CACHE();
 598         RADEON_PURGE_ZCACHE();
 599         RADEON_WAIT_UNTIL_IDLE();
 600         ADVANCE_RING();
 601         COMMIT_RING();
 602
 603         dev_priv->track_flush |= RADEON_FLUSH_EMITED | RADEON_PURGE_EMITED;
 604 }
 605
 606 /* Reset the Command Processor.  This will not flush any pending
 607  * commands, so you must wait for the CP command stream to complete
 608  * before calling this routine.
 609  */
 610 static void radeon_do_cp_reset(drm_radeon_private_t * dev_priv)
 611 {
 612         u32 cur_read_ptr;
 613         DRM_DEBUG("\n");
 614
 615         cur_read_ptr = RADEON_READ(RADEON_CP_RB_RPTR);
 616         RADEON_WRITE(RADEON_CP_RB_WPTR, cur_read_ptr);
 617         SET_RING_HEAD(dev_priv, cur_read_ptr);
 618         dev_priv->ring.tail = cur_read_ptr;
 619 }
 620
 621 /* Stop the Command Processor.  This will not flush any pending
 622  * commands, so you must flush the command stream and wait for the CP
 623  * to go idle before calling this routine.
 624  */
 625 static void radeon_do_cp_stop(drm_radeon_private_t * dev_priv)
 626 {
 627         DRM_DEBUG("\n");
 628
 629         RADEON_WRITE(RADEON_CP_CSQ_CNTL, RADEON_CSQ_PRIDIS_INDDIS);
 630
 631         dev_priv->cp_running = 0;
 632 }
 633
 634 /* Reset the engine.  This will stop the CP if it is running.
 635  */
 636 static int radeon_do_engine_reset(struct drm_device * dev)
 637 {
 638         drm_radeon_private_t *dev_priv = dev->dev_private;
 639         u32 clock_cntl_index = 0, mclk_cntl = 0, rbbm_soft_reset;
 640         DRM_DEBUG("\n");
 641
 642         radeon_do_pixcache_flush(dev_priv);
 643
 644         if ((dev_priv->flags & RADEON_FAMILY_MASK) <= CHIP_RV410) {
 645                 /* may need something similar for newer chips */
 646                 clock_cntl_index = RADEON_READ(RADEON_CLOCK_CNTL_INDEX);
 647                 mclk_cntl = RADEON_READ_PLL(dev, RADEON_MCLK_CNTL);
 648
 649                 RADEON_WRITE_PLL(RADEON_MCLK_CNTL, (mclk_cntl |
 650                                                     RADEON_FORCEON_MCLKA |
 651                                                     RADEON_FORCEON_MCLKB |
 652                                                     RADEON_FORCEON_YCLKA |
 653                                                     RADEON_FORCEON_YCLKB |
 654                                                     RADEON_FORCEON_MC |
 655                                                     RADEON_FORCEON_AIC));
 656         }
 657
 658         rbbm_soft_reset = RADEON_READ(RADEON_RBBM_SOFT_RESET);
 659
 660         RADEON_WRITE(RADEON_RBBM_SOFT_RESET, (rbbm_soft_reset |
 661                                               RADEON_SOFT_RESET_CP |
 662                                               RADEON_SOFT_RESET_HI |
 663                                               RADEON_SOFT_RESET_SE |
 664                                               RADEON_SOFT_RESET_RE |
 665                                               RADEON_SOFT_RESET_PP |
 666                                               RADEON_SOFT_RESET_E2 |
 667                                               RADEON_SOFT_RESET_RB));
 668         RADEON_READ(RADEON_RBBM_SOFT_RESET);
 669         RADEON_WRITE(RADEON_RBBM_SOFT_RESET, (rbbm_soft_reset &
 670                                               ~(RADEON_SOFT_RESET_CP |
 671                                                 RADEON_SOFT_RESET_HI |
 672                                                 RADEON_SOFT_RESET_SE |
 673                                                 RADEON_SOFT_RESET_RE |
 674                                                 RADEON_SOFT_RESET_PP |
 675                                                 RADEON_SOFT_RESET_E2 |
 676                                                 RADEON_SOFT_RESET_RB)));
 677         RADEON_READ(RADEON_RBBM_SOFT_RESET);
 678
 679         if ((dev_priv->flags & RADEON_FAMILY_MASK) <= CHIP_RV410) {
 680                 RADEON_WRITE_PLL(RADEON_MCLK_CNTL, mclk_cntl);
 681                 RADEON_WRITE(RADEON_CLOCK_CNTL_INDEX, clock_cntl_index);
 682                 RADEON_WRITE(RADEON_RBBM_SOFT_RESET, rbbm_soft_reset);
 683         }
 684
 685         /* setup the raster pipes */
 686         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R300)
 687             radeon_init_pipes(dev_priv);
 688
 689         /* Reset the CP ring */
 690         radeon_do_cp_reset(dev_priv);
 691
 692         /* The CP is no longer running after an engine reset */
 693         dev_priv->cp_running = 0;
 694
 695         /* Reset any pending vertex, indirect buffers */
 696         radeon_freelist_reset(dev);
 697
 698         return 0;
 699 }
 700
 701 static void radeon_cp_init_ring_buffer(struct drm_device * dev,
 702                                        drm_radeon_private_t *dev_priv,
 703                                        struct drm_file *file_priv)
 704 {
 705         u32 ring_start, cur_read_ptr;
 706
 707         /* Initialize the memory controller. With new memory map, the fb location
 708          * is not changed, it should have been properly initialized already. Part
 709          * of the problem is that the code below is bogus, assuming the GART is
 710          * always appended to the fb which is not necessarily the case
 711          */
 712         if (!dev_priv->new_memmap)
 713                 radeon_write_fb_location(dev_priv,
 714                              ((dev_priv->gart_vm_start - 1) & 0xffff0000)
 715                              | (dev_priv->fb_location >> 16));
 716
 717 #if __OS_HAS_AGP
 718         if (dev_priv->flags & RADEON_IS_AGP) {
 719                 radeon_write_agp_base(dev_priv, dev->agp->base);
 720
 721                 radeon_write_agp_location(dev_priv,
 722                              (((dev_priv->gart_vm_start - 1 +
 723                                 dev_priv->gart_size) & 0xffff0000) |
 724                               (dev_priv->gart_vm_start >> 16)));
 725
 726                 ring_start = (dev_priv->cp_ring->offset
 727                               - dev->agp->base
 728                               + dev_priv->gart_vm_start);
 729         } else
 730 #endif
 731                 ring_start = (dev_priv->cp_ring->offset
 732                               - (unsigned long)dev->sg->virtual
 733                               + dev_priv->gart_vm_start);
 734
 735         RADEON_WRITE(RADEON_CP_RB_BASE, ring_start);
 736
 737         /* Set the write pointer delay */
 738         RADEON_WRITE(RADEON_CP_RB_WPTR_DELAY, 0);
 739
 740         /* Initialize the ring buffer's read and write pointers */
 741         cur_read_ptr = RADEON_READ(RADEON_CP_RB_RPTR);
 742         RADEON_WRITE(RADEON_CP_RB_WPTR, cur_read_ptr);
 743         SET_RING_HEAD(dev_priv, cur_read_ptr);
 744         dev_priv->ring.tail = cur_read_ptr;
 745
 746 #if __OS_HAS_AGP
 747         if (dev_priv->flags & RADEON_IS_AGP) {
 748                 RADEON_WRITE(RADEON_CP_RB_RPTR_ADDR,
 749                              dev_priv->ring_rptr->offset
 750                              - dev->agp->base + dev_priv->gart_vm_start);
 751         } else
 752 #endif
 753         {
 754                 RADEON_WRITE(RADEON_CP_RB_RPTR_ADDR,
 755                              dev_priv->ring_rptr->offset
 756                              - ((unsigned long) dev->sg->virtual)
 757                              + dev_priv->gart_vm_start);
 758         }
 759
 760         /* Set ring buffer size */
 761 #ifdef __BIG_ENDIAN
 762         RADEON_WRITE(RADEON_CP_RB_CNTL,
 763                      RADEON_BUF_SWAP_32BIT |
 764                      (dev_priv->ring.fetch_size_l2ow << 18) |
 765                      (dev_priv->ring.rptr_update_l2qw << 8) |
 766                      dev_priv->ring.size_l2qw);
 767 #else
 768         RADEON_WRITE(RADEON_CP_RB_CNTL,
 769                      (dev_priv->ring.fetch_size_l2ow << 18) |
 770                      (dev_priv->ring.rptr_update_l2qw << 8) |
 771                      dev_priv->ring.size_l2qw);
 772 #endif
 773
 774
 775         /* Initialize the scratch register pointer.  This will cause
 776          * the scratch register values to be written out to memory
 777          * whenever they are updated.
 778          *
 779          * We simply put this behind the ring read pointer, this works
 780          * with PCI GART as well as (whatever kind of) AGP GART
 781          */
 782         RADEON_WRITE(RADEON_SCRATCH_ADDR, RADEON_READ(RADEON_CP_RB_RPTR_ADDR)
 783                      + RADEON_SCRATCH_REG_OFFSET);
 784
 785         RADEON_WRITE(RADEON_SCRATCH_UMSK, 0x7);
 786
 787         radeon_enable_bm(dev_priv);
 788
 789         radeon_write_ring_rptr(dev_priv, RADEON_SCRATCHOFF(0), 0);
 790         RADEON_WRITE(RADEON_LAST_FRAME_REG, 0);
 791
 792         radeon_write_ring_rptr(dev_priv, RADEON_SCRATCHOFF(1), 0);
 793         RADEON_WRITE(RADEON_LAST_DISPATCH_REG, 0);
 794
 795         radeon_write_ring_rptr(dev_priv, RADEON_SCRATCHOFF(2), 0);
 796         RADEON_WRITE(RADEON_LAST_CLEAR_REG, 0);
 797
 798         /* reset sarea copies of these */
 799         if (dev_priv->sarea_priv) {
 800                 dev_priv->sarea_priv->last_frame = 0;
 801                 dev_priv->sarea_priv->last_dispatch = 0;
 802                 dev_priv->sarea_priv->last_clear = 0;
 803         }
 804
 805         radeon_do_wait_for_idle(dev_priv);
 806
 807         /* Sync everything up */
 808         RADEON_WRITE(RADEON_ISYNC_CNTL,
 809                      (RADEON_ISYNC_ANY2D_IDLE3D |
 810                       RADEON_ISYNC_ANY3D_IDLE2D |
 811                       RADEON_ISYNC_WAIT_IDLEGUI |
 812                       RADEON_ISYNC_CPSCRATCH_IDLEGUI));
 813
 814 }
 815
 816 static void radeon_test_writeback(drm_radeon_private_t * dev_priv)
 817 {
 818         u32 tmp;
 819
 820         /* Start with assuming that writeback doesn't work */
 821         dev_priv->writeback_works = 0;
 822
 823         /* Writeback doesn't seem to work everywhere, test it here and possibly
 824          * enable it if it appears to work
 825          */
 826         radeon_write_ring_rptr(dev_priv, RADEON_SCRATCHOFF(1), 0);
 827
 828         RADEON_WRITE(RADEON_SCRATCH_REG1, 0xdeadbeef);
 829
 830         for (tmp = 0; tmp < dev_priv->usec_timeout; tmp++) {
 831                 u32 val;
 832
 833                 val = radeon_read_ring_rptr(dev_priv, RADEON_SCRATCHOFF(1));
 834                 if (val == 0xdeadbeef)
 835                         break;
 836                 DRM_UDELAY(1);
 837         }
 838
 839         if (tmp < dev_priv->usec_timeout) {
 840                 dev_priv->writeback_works = 1;
 841                 DRM_INFO("writeback test succeeded in %d usecs\n", tmp);
 842         } else {
 843                 dev_priv->writeback_works = 0;
 844                 DRM_INFO("writeback test failed\n");
 845         }
 846         if (radeon_no_wb == 1) {
 847                 dev_priv->writeback_works = 0;
 848                 DRM_INFO("writeback forced off\n");
 849         }
 850
 851         if (!dev_priv->writeback_works) {
 852                 /* Disable writeback to avoid unnecessary bus master transfer */
 853                 RADEON_WRITE(RADEON_CP_RB_CNTL, RADEON_READ(RADEON_CP_RB_CNTL) |
 854                              RADEON_RB_NO_UPDATE);
 855                 RADEON_WRITE(RADEON_SCRATCH_UMSK, 0);
 856         }
 857 }
 858
 859 /* Enable or disable IGP GART on the chip */
 860 static void radeon_set_igpgart(drm_radeon_private_t * dev_priv, int on)
 861 {
 862         u32 temp;
 863
 864         if (on) {
 865                 DRM_DEBUG("programming igp gart %08X %08lX %08X\n",
 866                           dev_priv->gart_vm_start,
 867                           (long)dev_priv->gart_info.bus_addr,
 868                           dev_priv->gart_size);
 869
 870                 temp = IGP_READ_MCIND(dev_priv, RS480_MC_MISC_CNTL);
 871                 if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690) ||
 872                     ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS740))
 873                         IGP_WRITE_MCIND(RS480_MC_MISC_CNTL, (RS480_GART_INDEX_REG_EN |
 874                                                              RS690_BLOCK_GFX_D3_EN));
 875                 else
 876                         IGP_WRITE_MCIND(RS480_MC_MISC_CNTL, RS480_GART_INDEX_REG_EN);
 877
 878                 IGP_WRITE_MCIND(RS480_AGP_ADDRESS_SPACE_SIZE, (RS480_GART_EN |
 879                                                                RS480_VA_SIZE_32MB));
 880
 881                 temp = IGP_READ_MCIND(dev_priv, RS480_GART_FEATURE_ID);
 882                 IGP_WRITE_MCIND(RS480_GART_FEATURE_ID, (RS480_HANG_EN |
 883                                                         RS480_TLB_ENABLE |
 884                                                         RS480_GTW_LAC_EN |
 885                                                         RS480_1LEVEL_GART));
 886
 887                 temp = dev_priv->gart_info.bus_addr & 0xfffff000;
 888                 temp |= (upper_32_bits(dev_priv->gart_info.bus_addr) & 0xff) << 4;
 889                 IGP_WRITE_MCIND(RS480_GART_BASE, temp);
 890
 891                 temp = IGP_READ_MCIND(dev_priv, RS480_AGP_MODE_CNTL);
 892                 IGP_WRITE_MCIND(RS480_AGP_MODE_CNTL, ((1 << RS480_REQ_TYPE_SNOOP_SHIFT) |
 893                                                       RS480_REQ_TYPE_SNOOP_DIS));
 894
 895                 radeon_write_agp_base(dev_priv, dev_priv->gart_vm_start);
 896
 897                 dev_priv->gart_size = 32*1024*1024;
 898                 temp = (((dev_priv->gart_vm_start - 1 + dev_priv->gart_size) &
 899                          0xffff0000) | (dev_priv->gart_vm_start >> 16));
 900
 901                 radeon_write_agp_location(dev_priv, temp);
 902
 903                 temp = IGP_READ_MCIND(dev_priv, RS480_AGP_ADDRESS_SPACE_SIZE);
 904                 IGP_WRITE_MCIND(RS480_AGP_ADDRESS_SPACE_SIZE, (RS480_GART_EN |
 905                                                                RS480_VA_SIZE_32MB));
 906
 907                 do {
 908                         temp = IGP_READ_MCIND(dev_priv, RS480_GART_CACHE_CNTRL);
 909                         if ((temp & RS480_GART_CACHE_INVALIDATE) == 0)
 910                                 break;
 911                         DRM_UDELAY(1);
 912                 } while (1);
 913
 914                 IGP_WRITE_MCIND(RS480_GART_CACHE_CNTRL,
 915                                 RS480_GART_CACHE_INVALIDATE);
 916
 917                 do {
 918                         temp = IGP_READ_MCIND(dev_priv, RS480_GART_CACHE_CNTRL);
 919                         if ((temp & RS480_GART_CACHE_INVALIDATE) == 0)
 920                                 break;
 921                         DRM_UDELAY(1);
 922                 } while (1);
 923
 924                 IGP_WRITE_MCIND(RS480_GART_CACHE_CNTRL, 0);
 925         } else {
 926                 IGP_WRITE_MCIND(RS480_AGP_ADDRESS_SPACE_SIZE, 0);
 927         }
 928 }
 929
 930 /* Enable or disable IGP GART on the chip */
 931 static void rs600_set_igpgart(drm_radeon_private_t *dev_priv, int on)
 932 {
 933         u32 temp;
 934         int i;
 935
 936         if (on) {
 937                 DRM_DEBUG("programming igp gart %08X %08lX %08X\n",
 938                          dev_priv->gart_vm_start,
 939                          (long)dev_priv->gart_info.bus_addr,
 940                          dev_priv->gart_size);
 941
 942                 IGP_WRITE_MCIND(RS600_MC_PT0_CNTL, (RS600_EFFECTIVE_L2_CACHE_SIZE(6) |
 943                                                     RS600_EFFECTIVE_L2_QUEUE_SIZE(6)));
 944
 945                 for (i = 0; i < 19; i++)
 946                         IGP_WRITE_MCIND(RS600_MC_PT0_CLIENT0_CNTL + i,
 947                                         (RS600_ENABLE_TRANSLATION_MODE_OVERRIDE |
 948                                          RS600_SYSTEM_ACCESS_MODE_IN_SYS |
 949                                          RS600_SYSTEM_APERTURE_UNMAPPED_ACCESS_PASSTHROUGH |
 950                                          RS600_EFFECTIVE_L1_CACHE_SIZE(3) |
 951                                          RS600_ENABLE_FRAGMENT_PROCESSING |
 952                                          RS600_EFFECTIVE_L1_QUEUE_SIZE(3)));
 953
 954                 IGP_WRITE_MCIND(RS600_MC_PT0_CONTEXT0_CNTL, (RS600_ENABLE_PAGE_TABLE |
 955                                                              RS600_PAGE_TABLE_TYPE_FLAT));
 956
 957                 /* disable all other contexts */
 958                 for (i = 1; i < 8; i++)
 959                         IGP_WRITE_MCIND(RS600_MC_PT0_CONTEXT0_CNTL + i, 0);
 960
 961                 /* setup the page table aperture */
 962                 IGP_WRITE_MCIND(RS600_MC_PT0_CONTEXT0_FLAT_BASE_ADDR,
 963                                 dev_priv->gart_info.bus_addr);
 964                 IGP_WRITE_MCIND(RS600_MC_PT0_CONTEXT0_FLAT_START_ADDR,
 965                                 dev_priv->gart_vm_start);
 966                 IGP_WRITE_MCIND(RS600_MC_PT0_CONTEXT0_FLAT_END_ADDR,
 967                                 (dev_priv->gart_vm_start + dev_priv->gart_size - 1));
 968                 IGP_WRITE_MCIND(RS600_MC_PT0_CONTEXT0_DEFAULT_READ_ADDR, 0);
 969
 970                 /* setup the system aperture */
 971                 IGP_WRITE_MCIND(RS600_MC_PT0_SYSTEM_APERTURE_LOW_ADDR,
 972                                 dev_priv->gart_vm_start);
 973                 IGP_WRITE_MCIND(RS600_MC_PT0_SYSTEM_APERTURE_HIGH_ADDR,
 974                                 (dev_priv->gart_vm_start + dev_priv->gart_size - 1));
 975
 976                 /* enable page tables */
 977                 temp = IGP_READ_MCIND(dev_priv, RS600_MC_PT0_CNTL);
 978                 IGP_WRITE_MCIND(RS600_MC_PT0_CNTL, (temp | RS600_ENABLE_PT));
 979
 980                 temp = IGP_READ_MCIND(dev_priv, RS600_MC_CNTL1);
 981                 IGP_WRITE_MCIND(RS600_MC_CNTL1, (temp | RS600_ENABLE_PAGE_TABLES));
 982
 983                 /* invalidate the cache */
 984                 temp = IGP_READ_MCIND(dev_priv, RS600_MC_PT0_CNTL);
 985
 986                 temp &= ~(RS600_INVALIDATE_ALL_L1_TLBS | RS600_INVALIDATE_L2_CACHE);
 987                 IGP_WRITE_MCIND(RS600_MC_PT0_CNTL, temp);
 988                 temp = IGP_READ_MCIND(dev_priv, RS600_MC_PT0_CNTL);
 989
 990                 temp |= RS600_INVALIDATE_ALL_L1_TLBS | RS600_INVALIDATE_L2_CACHE;
 991                 IGP_WRITE_MCIND(RS600_MC_PT0_CNTL, temp);
 992                 temp = IGP_READ_MCIND(dev_priv, RS600_MC_PT0_CNTL);
 993
 994                 temp &= ~(RS600_INVALIDATE_ALL_L1_TLBS | RS600_INVALIDATE_L2_CACHE);
 995                 IGP_WRITE_MCIND(RS600_MC_PT0_CNTL, temp);
 996                 temp = IGP_READ_MCIND(dev_priv, RS600_MC_PT0_CNTL);
 997
 998         } else {
 999                 IGP_WRITE_MCIND(RS600_MC_PT0_CNTL, 0);
1000                 temp = IGP_READ_MCIND(dev_priv, RS600_MC_CNTL1);
1001                 temp &= ~RS600_ENABLE_PAGE_TABLES;
1002                 IGP_WRITE_MCIND(RS600_MC_CNTL1, temp);
1003         }
1004 }
1005
1006 static void radeon_set_pciegart(drm_radeon_private_t * dev_priv, int on)
1007 {
1008         u32 tmp = RADEON_READ_PCIE(dev_priv, RADEON_PCIE_TX_GART_CNTL);
1009         if (on) {
1010
1011                 DRM_DEBUG("programming pcie %08X %08lX %08X\n",
1012                           dev_priv->gart_vm_start,
1013                           (long)dev_priv->gart_info.bus_addr,
1014                           dev_priv->gart_size);
1015                 RADEON_WRITE_PCIE(RADEON_PCIE_TX_DISCARD_RD_ADDR_LO,
1016                                   dev_priv->gart_vm_start);
1017                 RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_BASE,
1018                                   dev_priv->gart_info.bus_addr);
1019                 RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_START_LO,
1020                                   dev_priv->gart_vm_start);
1021                 RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_END_LO,
1022                                   dev_priv->gart_vm_start +
1023                                   dev_priv->gart_size - 1);
1024
1025                 radeon_write_agp_location(dev_priv, 0xffffffc0); /* ?? */
1026
1027                 RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_CNTL,
1028                                   RADEON_PCIE_TX_GART_EN);
1029         } else {
1030                 RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_CNTL,
1031                                   tmp & ~RADEON_PCIE_TX_GART_EN);
1032         }
1033 }
1034
1035 /* Enable or disable PCI GART on the chip */
1036 static void radeon_set_pcigart(drm_radeon_private_t * dev_priv, int on)
1037 {
1038         u32 tmp;
1039
1040         if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690) ||
1041             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS740) ||
1042             (dev_priv->flags & RADEON_IS_IGPGART)) {
1043                 radeon_set_igpgart(dev_priv, on);
1044                 return;
1045         }
1046
1047         if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS600) {
1048                 rs600_set_igpgart(dev_priv, on);
1049                 return;
1050         }
1051
1052         if (dev_priv->flags & RADEON_IS_PCIE) {
1053                 radeon_set_pciegart(dev_priv, on);
1054                 return;
1055         }
1056
1057         tmp = RADEON_READ(RADEON_AIC_CNTL);
1058
1059         if (on) {
1060                 RADEON_WRITE(RADEON_AIC_CNTL,
1061                              tmp | RADEON_PCIGART_TRANSLATE_EN);
1062
1063                 /* set PCI GART page-table base address
1064                  */
1065                 RADEON_WRITE(RADEON_AIC_PT_BASE, dev_priv->gart_info.bus_addr);
1066
1067                 /* set address range for PCI address translate
1068                  */
1069                 RADEON_WRITE(RADEON_AIC_LO_ADDR, dev_priv->gart_vm_start);
1070                 RADEON_WRITE(RADEON_AIC_HI_ADDR, dev_priv->gart_vm_start
1071                              + dev_priv->gart_size - 1);
1072
1073                 /* Turn off AGP aperture -- is this required for PCI GART?
1074                  */
1075                 radeon_write_agp_location(dev_priv, 0xffffffc0);
1076                 RADEON_WRITE(RADEON_AGP_COMMAND, 0);    /* clear AGP_COMMAND */
1077         } else {
1078                 RADEON_WRITE(RADEON_AIC_CNTL,
1079                              tmp & ~RADEON_PCIGART_TRANSLATE_EN);
1080         }
1081 }
1082
1083 static int radeon_setup_pcigart_surface(drm_radeon_private_t *dev_priv)
1084 {
1085         struct drm_ati_pcigart_info *gart_info = &dev_priv->gart_info;
1086         struct radeon_virt_surface *vp;
1087         int i;
1088
1089         for (i = 0; i < RADEON_MAX_SURFACES * 2; i++) {
1090                 if (!dev_priv->virt_surfaces[i].file_priv ||
1091                     dev_priv->virt_surfaces[i].file_priv == PCIGART_FILE_PRIV)
1092                         break;
1093         }
1094         if (i >= 2 * RADEON_MAX_SURFACES)
1095                 return -ENOMEM;
1096         vp = &dev_priv->virt_surfaces[i];
1097
1098         for (i = 0; i < RADEON_MAX_SURFACES; i++) {
1099                 struct radeon_surface *sp = &dev_priv->surfaces[i];
1100                 if (sp->refcount)
1101                         continue;
1102
1103                 vp->surface_index = i;
1104                 vp->lower = gart_info->bus_addr;
1105                 vp->upper = vp->lower + gart_info->table_size;
1106                 vp->flags = 0;
1107                 vp->file_priv = PCIGART_FILE_PRIV;
1108
1109                 sp->refcount = 1;
1110                 sp->lower = vp->lower;
1111                 sp->upper = vp->upper;
1112                 sp->flags = 0;
1113
1114                 RADEON_WRITE(RADEON_SURFACE0_INFO + 16 * i, sp->flags);
1115                 RADEON_WRITE(RADEON_SURFACE0_LOWER_BOUND + 16 * i, sp->lower);
1116                 RADEON_WRITE(RADEON_SURFACE0_UPPER_BOUND + 16 * i, sp->upper);
1117                 return 0;
1118         }
1119
1120         return -ENOMEM;
1121 }
1122
1123 static int radeon_do_init_cp(struct drm_device *dev, drm_radeon_init_t *init,
1124                              struct drm_file *file_priv)
1125 {
1126         drm_radeon_private_t *dev_priv = dev->dev_private;
1127
1128         DRM_DEBUG("\n");
1129
1130         /* if we require new memory map but we don't have it fail */
1131         if ((dev_priv->flags & RADEON_NEW_MEMMAP) && !dev_priv->new_memmap) {
1132                 DRM_ERROR("Cannot initialise DRM on this card\nThis card requires a new X.org DDX for 3D\n");
1133                 radeon_do_cleanup_cp(dev);
1134                 return -EINVAL;
1135         }
1136
1137         if (init->is_pci && (dev_priv->flags & RADEON_IS_AGP)) {
1138                 DRM_DEBUG("Forcing AGP card to PCI mode\n");
1139                 dev_priv->flags &= ~RADEON_IS_AGP;
1140         } else if (!(dev_priv->flags & (RADEON_IS_AGP | RADEON_IS_PCI | RADEON_IS_PCIE))
1141                    && !init->is_pci) {
1142                 DRM_DEBUG("Restoring AGP flag\n");
1143                 dev_priv->flags |= RADEON_IS_AGP;
1144         }
1145
1146         if ((!(dev_priv->flags & RADEON_IS_AGP)) && !dev->sg) {
1147                 DRM_ERROR("PCI GART memory not allocated!\n");
1148                 radeon_do_cleanup_cp(dev);
1149                 return -EINVAL;
1150         }
1151
1152         dev_priv->usec_timeout = init->usec_timeout;
1153         if (dev_priv->usec_timeout < 1 ||
1154             dev_priv->usec_timeout > RADEON_MAX_USEC_TIMEOUT) {
1155                 DRM_DEBUG("TIMEOUT problem!\n");
1156                 radeon_do_cleanup_cp(dev);
1157                 return -EINVAL;
1158         }
1159
1160         /* Enable vblank on CRTC1 for older X servers
1161          */
1162         dev_priv->vblank_crtc = DRM_RADEON_VBLANK_CRTC1;
1163
1164         switch(init->func) {
1165         case RADEON_INIT_R200_CP:
1166                 dev_priv->microcode_version = UCODE_R200;
1167                 break;
1168         case RADEON_INIT_R300_CP:
1169                 dev_priv->microcode_version = UCODE_R300;
1170                 break;
1171         default:
1172                 dev_priv->microcode_version = UCODE_R100;
1173         }
1174
1175         dev_priv->do_boxes = 0;
1176         dev_priv->cp_mode = init->cp_mode;
1177
1178         /* We don't support anything other than bus-mastering ring mode,
1179          * but the ring can be in either AGP or PCI space for the ring
1180          * read pointer.
1181          */
1182         if ((init->cp_mode != RADEON_CSQ_PRIBM_INDDIS) &&
1183             (init->cp_mode != RADEON_CSQ_PRIBM_INDBM)) {
1184                 DRM_DEBUG("BAD cp_mode (%x)!\n", init->cp_mode);
1185                 radeon_do_cleanup_cp(dev);
1186                 return -EINVAL;
1187         }
1188
1189         switch (init->fb_bpp) {
1190         case 16:
1191                 dev_priv->color_fmt = RADEON_COLOR_FORMAT_RGB565;
1192                 break;
1193         case 32:
1194         default:
1195                 dev_priv->color_fmt = RADEON_COLOR_FORMAT_ARGB8888;
1196                 break;
1197         }
1198         dev_priv->front_offset = init->front_offset;
1199         dev_priv->front_pitch = init->front_pitch;
1200         dev_priv->back_offset = init->back_offset;
1201         dev_priv->back_pitch = init->back_pitch;
1202
1203         switch (init->depth_bpp) {
1204         case 16:
1205                 dev_priv->depth_fmt = RADEON_DEPTH_FORMAT_16BIT_INT_Z;
1206                 break;
1207         case 32:
1208         default:
1209                 dev_priv->depth_fmt = RADEON_DEPTH_FORMAT_24BIT_INT_Z;
1210                 break;
1211         }
1212         dev_priv->depth_offset = init->depth_offset;
1213         dev_priv->depth_pitch = init->depth_pitch;
1214
1215         /* Hardware state for depth clears.  Remove this if/when we no
1216          * longer clear the depth buffer with a 3D rectangle.  Hard-code
1217          * all values to prevent unwanted 3D state from slipping through
1218          * and screwing with the clear operation.
1219          */
1220         dev_priv->depth_clear.rb3d_cntl = (RADEON_PLANE_MASK_ENABLE |
1221                                            (dev_priv->color_fmt << 10) |
1222                                            (dev_priv->microcode_version ==
1223                                             UCODE_R100 ? RADEON_ZBLOCK16 : 0));
1224
1225         dev_priv->depth_clear.rb3d_zstencilcntl =
1226             (dev_priv->depth_fmt |
1227              RADEON_Z_TEST_ALWAYS |
1228              RADEON_STENCIL_TEST_ALWAYS |
1229              RADEON_STENCIL_S_FAIL_REPLACE |
1230              RADEON_STENCIL_ZPASS_REPLACE |
1231              RADEON_STENCIL_ZFAIL_REPLACE | RADEON_Z_WRITE_ENABLE);
1232
1233         dev_priv->depth_clear.se_cntl = (RADEON_FFACE_CULL_CW |
1234                                          RADEON_BFACE_SOLID |
1235                                          RADEON_FFACE_SOLID |
1236                                          RADEON_FLAT_SHADE_VTX_LAST |
1237                                          RADEON_DIFFUSE_SHADE_FLAT |
1238                                          RADEON_ALPHA_SHADE_FLAT |
1239                                          RADEON_SPECULAR_SHADE_FLAT |
1240                                          RADEON_FOG_SHADE_FLAT |
1241                                          RADEON_VTX_PIX_CENTER_OGL |
1242                                          RADEON_ROUND_MODE_TRUNC |
1243                                          RADEON_ROUND_PREC_8TH_PIX);
1244
1245
1246         dev_priv->ring_offset = init->ring_offset;
1247         dev_priv->ring_rptr_offset = init->ring_rptr_offset;
1248         dev_priv->buffers_offset = init->buffers_offset;
1249         dev_priv->gart_textures_offset = init->gart_textures_offset;
1250
1251         dev_priv->sarea = drm_getsarea(dev);
1252         if (!dev_priv->sarea) {
1253                 DRM_ERROR("could not find sarea!\n");
1254                 radeon_do_cleanup_cp(dev);
1255                 return -EINVAL;
1256         }
1257
1258         dev_priv->cp_ring = drm_core_findmap(dev, init->ring_offset);
1259         if (!dev_priv->cp_ring) {
1260                 DRM_ERROR("could not find cp ring region!\n");
1261                 radeon_do_cleanup_cp(dev);
1262                 return -EINVAL;
1263         }
1264         dev_priv->ring_rptr = drm_core_findmap(dev, init->ring_rptr_offset);
1265         if (!dev_priv->ring_rptr) {
1266                 DRM_ERROR("could not find ring read pointer!\n");
1267                 radeon_do_cleanup_cp(dev);
1268                 return -EINVAL;
1269         }
1270         dev->agp_buffer_token = init->buffers_offset;
1271         dev->agp_buffer_map = drm_core_findmap(dev, init->buffers_offset);
1272         if (!dev->agp_buffer_map) {
1273                 DRM_ERROR("could not find dma buffer region!\n");
1274                 radeon_do_cleanup_cp(dev);
1275                 return -EINVAL;
1276         }
1277
1278         if (init->gart_textures_offset) {
1279                 dev_priv->gart_textures =
1280                     drm_core_findmap(dev, init->gart_textures_offset);
1281                 if (!dev_priv->gart_textures) {
1282                         DRM_ERROR("could not find GART texture region!\n");
1283                         radeon_do_cleanup_cp(dev);
1284                         return -EINVAL;
1285                 }
1286         }
1287
1288         dev_priv->sarea_priv =
1289             (drm_radeon_sarea_t *) ((u8 *) dev_priv->sarea->handle +
1290                                     init->sarea_priv_offset);
1291
1292 #if __OS_HAS_AGP
1293         if (dev_priv->flags & RADEON_IS_AGP) {
1294                 drm_core_ioremap_wc(dev_priv->cp_ring, dev);
1295                 drm_core_ioremap_wc(dev_priv->ring_rptr, dev);
1296                 drm_core_ioremap_wc(dev->agp_buffer_map, dev);
1297                 if (!dev_priv->cp_ring->handle ||
1298                     !dev_priv->ring_rptr->handle ||
1299                     !dev->agp_buffer_map->handle) {
1300                         DRM_ERROR("could not find ioremap agp regions!\n");
1301                         radeon_do_cleanup_cp(dev);
1302                         return -EINVAL;
1303                 }
1304         } else
1305 #endif
1306         {
1307                 dev_priv->cp_ring->handle =
1308                         (void *)(unsigned long)dev_priv->cp_ring->offset;
1309                 dev_priv->ring_rptr->handle =
1310                         (void *)(unsigned long)dev_priv->ring_rptr->offset;
1311                 dev->agp_buffer_map->handle =
1312                         (void *)(unsigned long)dev->agp_buffer_map->offset;
1313
1314                 DRM_DEBUG("dev_priv->cp_ring->handle %p\n",
1315                           dev_priv->cp_ring->handle);
1316                 DRM_DEBUG("dev_priv->ring_rptr->handle %p\n",
1317                           dev_priv->ring_rptr->handle);
1318                 DRM_DEBUG("dev->agp_buffer_map->handle %p\n",
1319                           dev->agp_buffer_map->handle);
1320         }
1321
1322         dev_priv->fb_location = (radeon_read_fb_location(dev_priv) & 0xffff) << 16;
1323         dev_priv->fb_size =
1324                 ((radeon_read_fb_location(dev_priv) & 0xffff0000u) + 0x10000)
1325                 - dev_priv->fb_location;
1326
1327         dev_priv->front_pitch_offset = (((dev_priv->front_pitch / 64) << 22) |
1328                                         ((dev_priv->front_offset
1329                                           + dev_priv->fb_location) >> 10));
1330
1331         dev_priv->back_pitch_offset = (((dev_priv->back_pitch / 64) << 22) |
1332                                        ((dev_priv->back_offset
1333                                          + dev_priv->fb_location) >> 10));
1334
1335         dev_priv->depth_pitch_offset = (((dev_priv->depth_pitch / 64) << 22) |
1336                                         ((dev_priv->depth_offset
1337                                           + dev_priv->fb_location) >> 10));
1338
1339         dev_priv->gart_size = init->gart_size;
1340
1341         /* New let's set the memory map ... */
1342         if (dev_priv->new_memmap) {
1343                 u32 base = 0;
1344
1345                 DRM_INFO("Setting GART location based on new memory map\n");
1346
1347                 /* If using AGP, try to locate the AGP aperture at the same
1348                  * location in the card and on the bus, though we have to
1349                  * align it down.
1350                  */
1351 #if __OS_HAS_AGP
1352                 if (dev_priv->flags & RADEON_IS_AGP) {
1353                         base = dev->agp->base;
1354                         /* Check if valid */
1355                         if ((base + dev_priv->gart_size - 1) >= dev_priv->fb_location &&
1356                             base < (dev_priv->fb_location + dev_priv->fb_size - 1)) {
1357                                 DRM_INFO("Can't use AGP base @0x%08lx, won't fit\n",
1358                                          dev->agp->base);
1359                                 base = 0;
1360                         }
1361                 }
1362 #endif
1363                 /* If not or if AGP is at 0 (Macs), try to put it elsewhere */
1364                 if (base == 0) {
1365                         base = dev_priv->fb_location + dev_priv->fb_size;
1366                         if (base < dev_priv->fb_location ||
1367                             ((base + dev_priv->gart_size) & 0xfffffffful) < base)
1368                                 base = dev_priv->fb_location
1369                                         - dev_priv->gart_size;
1370                 }
1371                 dev_priv->gart_vm_start = base & 0xffc00000u;
1372                 if (dev_priv->gart_vm_start != base)
1373                         DRM_INFO("GART aligned down from 0x%08x to 0x%08x\n",
1374                                  base, dev_priv->gart_vm_start);
1375         } else {
1376                 DRM_INFO("Setting GART location based on old memory map\n");
1377                 dev_priv->gart_vm_start = dev_priv->fb_location +
1378                         RADEON_READ(RADEON_CONFIG_APER_SIZE);
1379         }
1380
1381 #if __OS_HAS_AGP
1382         if (dev_priv->flags & RADEON_IS_AGP)
1383                 dev_priv->gart_buffers_offset = (dev->agp_buffer_map->offset
1384                                                  - dev->agp->base
1385                                                  + dev_priv->gart_vm_start);
1386         else
1387 #endif
1388                 dev_priv->gart_buffers_offset = (dev->agp_buffer_map->offset
1389                                         - (unsigned long)dev->sg->virtual
1390                                         + dev_priv->gart_vm_start);
1391
1392         DRM_DEBUG("dev_priv->gart_size %d\n", dev_priv->gart_size);
1393         DRM_DEBUG("dev_priv->gart_vm_start 0x%x\n", dev_priv->gart_vm_start);
1394         DRM_DEBUG("dev_priv->gart_buffers_offset 0x%lx\n",
1395                   dev_priv->gart_buffers_offset);
1396
1397         dev_priv->ring.start = (u32 *) dev_priv->cp_ring->handle;
1398         dev_priv->ring.end = ((u32 *) dev_priv->cp_ring->handle
1399                               + init->ring_size / sizeof(u32));
1400         dev_priv->ring.size = init->ring_size;
1401         dev_priv->ring.size_l2qw = drm_order(init->ring_size / 8);
1402
1403         dev_priv->ring.rptr_update = /* init->rptr_update */ 4096;
1404         dev_priv->ring.rptr_update_l2qw = drm_order( /* init->rptr_update */ 4096 / 8);
1405
1406         dev_priv->ring.fetch_size = /* init->fetch_size */ 32;
1407         dev_priv->ring.fetch_size_l2ow = drm_order( /* init->fetch_size */ 32 / 16);
1408         dev_priv->ring.tail_mask = (dev_priv->ring.size / sizeof(u32)) - 1;
1409
1410         dev_priv->ring.high_mark = RADEON_RING_HIGH_MARK;
1411
1412 #if __OS_HAS_AGP
1413         if (dev_priv->flags & RADEON_IS_AGP) {
1414                 /* Turn off PCI GART */
1415                 radeon_set_pcigart(dev_priv, 0);
1416         } else
1417 #endif
1418         {
1419                 u32 sctrl;
1420                 int ret;
1421
1422                 dev_priv->gart_info.table_mask = DMA_BIT_MASK(32);
1423                 /* if we have an offset set from userspace */
1424                 if (dev_priv->pcigart_offset_set) {
1425                         dev_priv->gart_info.bus_addr =
1426                             dev_priv->pcigart_offset + dev_priv->fb_location;
1427                         dev_priv->gart_info.mapping.offset =
1428                             dev_priv->pcigart_offset + dev_priv->fb_aper_offset;
1429                         dev_priv->gart_info.mapping.size =
1430                             dev_priv->gart_info.table_size;
1431
1432                         drm_core_ioremap_wc(&dev_priv->gart_info.mapping, dev);
1433                         dev_priv->gart_info.addr =
1434                             dev_priv->gart_info.mapping.handle;
1435
1436                         if (dev_priv->flags & RADEON_IS_PCIE)
1437                                 dev_priv->gart_info.gart_reg_if = DRM_ATI_GART_PCIE;
1438                         else
1439                                 dev_priv->gart_info.gart_reg_if = DRM_ATI_GART_PCI;
1440                         dev_priv->gart_info.gart_table_location =
1441                             DRM_ATI_GART_FB;
1442
1443                         DRM_DEBUG("Setting phys_pci_gart to %p %08lX\n",
1444                                   dev_priv->gart_info.addr,
1445                                   dev_priv->pcigart_offset);
1446                 } else {
1447                         if (dev_priv->flags & RADEON_IS_IGPGART)
1448                                 dev_priv->gart_info.gart_reg_if = DRM_ATI_GART_IGP;
1449                         else
1450                                 dev_priv->gart_info.gart_reg_if = DRM_ATI_GART_PCI;
1451                         dev_priv->gart_info.gart_table_location =
1452                             DRM_ATI_GART_MAIN;
1453                         dev_priv->gart_info.addr = NULL;
1454                         dev_priv->gart_info.bus_addr = 0;
1455                         if (dev_priv->flags & RADEON_IS_PCIE) {
1456                                 DRM_ERROR
1457                                     ("Cannot use PCI Express without GART in FB memory\n");
1458                                 radeon_do_cleanup_cp(dev);
1459                                 return -EINVAL;
1460                         }
1461                 }
1462
1463                 sctrl = RADEON_READ(RADEON_SURFACE_CNTL);
1464                 RADEON_WRITE(RADEON_SURFACE_CNTL, 0);
1465                 if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS600)
1466                         ret = r600_page_table_init(dev);
1467                 else
1468                         ret = drm_ati_pcigart_init(dev, &dev_priv->gart_info);
1469                 RADEON_WRITE(RADEON_SURFACE_CNTL, sctrl);
1470
1471                 if (!ret) {
1472                         DRM_ERROR("failed to init PCI GART!\n");
1473                         radeon_do_cleanup_cp(dev);
1474                         return -ENOMEM;
1475                 }
1476
1477                 ret = radeon_setup_pcigart_surface(dev_priv);
1478                 if (ret) {
1479                         DRM_ERROR("failed to setup GART surface!\n");
1480                         if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS600)
1481                                 r600_page_table_cleanup(dev, &dev_priv->gart_info);
1482                         else
1483                                 drm_ati_pcigart_cleanup(dev, &dev_priv->gart_info);
1484                         radeon_do_cleanup_cp(dev);
1485                         return ret;
1486                 }
1487
1488                 /* Turn on PCI GART */
1489                 radeon_set_pcigart(dev_priv, 1);
1490         }
1491
1492         radeon_cp_load_microcode(dev_priv);
1493         radeon_cp_init_ring_buffer(dev, dev_priv, file_priv);
1494
1495         dev_priv->last_buf = 0;
1496
1497         radeon_do_engine_reset(dev);
1498         radeon_test_writeback(dev_priv);
1499
1500         return 0;
1501 }
1502
1503 static int radeon_do_cleanup_cp(struct drm_device * dev)
1504 {
1505         drm_radeon_private_t *dev_priv = dev->dev_private;
1506         DRM_DEBUG("\n");
1507
1508         /* Make sure interrupts are disabled here because the uninstall ioctl
1509          * may not have been called from userspace and after dev_private
1510          * is freed, it's too late.
1511          */
1512         if (dev->irq_enabled)
1513                 drm_irq_uninstall(dev);
1514
1515 #if __OS_HAS_AGP
1516         if (dev_priv->flags & RADEON_IS_AGP) {
1517                 if (dev_priv->cp_ring != NULL) {
1518                         drm_core_ioremapfree(dev_priv->cp_ring, dev);
1519                         dev_priv->cp_ring = NULL;
1520                 }
1521                 if (dev_priv->ring_rptr != NULL) {
1522                         drm_core_ioremapfree(dev_priv->ring_rptr, dev);
1523                         dev_priv->ring_rptr = NULL;
1524                 }
1525                 if (dev->agp_buffer_map != NULL) {
1526                         drm_core_ioremapfree(dev->agp_buffer_map, dev);
1527                         dev->agp_buffer_map = NULL;
1528                 }
1529         } else
1530 #endif
1531         {
1532
1533                 if (dev_priv->gart_info.bus_addr) {
1534                         /* Turn off PCI GART */
1535                         radeon_set_pcigart(dev_priv, 0);
1536                         if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS600)
1537                                 r600_page_table_cleanup(dev, &dev_priv->gart_info);
1538                         else {
1539                                 if (!drm_ati_pcigart_cleanup(dev, &dev_priv->gart_info))
1540                                         DRM_ERROR("failed to cleanup PCI GART!\n");
1541                         }
1542                 }
1543
1544                 if (dev_priv->gart_info.gart_table_location == DRM_ATI_GART_FB)
1545                 {
1546                         drm_core_ioremapfree(&dev_priv->gart_info.mapping, dev);
1547                         dev_priv->gart_info.addr = 0;
1548                 }
1549         }
1550         /* only clear to the start of flags */
1551         memset(dev_priv, 0, offsetof(drm_radeon_private_t, flags));
1552
1553         return 0;
1554 }
1555
1556 /* This code will reinit the Radeon CP hardware after a resume from disc.
1557  * AFAIK, it would be very difficult to pickle the state at suspend time, so
1558  * here we make sure that all Radeon hardware initialisation is re-done without
1559  * affecting running applications.
1560  *
1561  * Charl P. Botha <http://cpbotha.net>
1562  */
1563 static int radeon_do_resume_cp(struct drm_device *dev, struct drm_file *file_priv)
1564 {
1565         drm_radeon_private_t *dev_priv = dev->dev_private;
1566
1567         if (!dev_priv) {
1568                 DRM_ERROR("Called with no initialization\n");
1569                 return -EINVAL;
1570         }
1571
1572         DRM_DEBUG("Starting radeon_do_resume_cp()\n");
1573
1574 #if __OS_HAS_AGP
1575         if (dev_priv->flags & RADEON_IS_AGP) {
1576                 /* Turn off PCI GART */
1577                 radeon_set_pcigart(dev_priv, 0);
1578         } else
1579 #endif
1580         {
1581                 /* Turn on PCI GART */
1582                 radeon_set_pcigart(dev_priv, 1);
1583         }
1584
1585         radeon_cp_load_microcode(dev_priv);
1586         radeon_cp_init_ring_buffer(dev, dev_priv, file_priv);
1587
1588         radeon_do_engine_reset(dev);
1589         radeon_irq_set_state(dev, RADEON_SW_INT_ENABLE, 1);
1590
1591         DRM_DEBUG("radeon_do_resume_cp() complete\n");
1592
1593         return 0;
1594 }
1595
1596 int radeon_cp_init(struct drm_device *dev, void *data, struct drm_file *file_priv)
1597 {
1598         drm_radeon_private_t *dev_priv = dev->dev_private;
1599         drm_radeon_init_t *init = data;
1600
1601         LOCK_TEST_WITH_RETURN(dev, file_priv);
1602
1603         if (init->func == RADEON_INIT_R300_CP)
1604                 r300_init_reg_flags(dev);
1605
1606         switch (init->func) {
1607         case RADEON_INIT_CP:
1608         case RADEON_INIT_R200_CP:
1609         case RADEON_INIT_R300_CP:
1610                 return radeon_do_init_cp(dev, init, file_priv);
1611         case RADEON_INIT_R600_CP:
1612                 return r600_do_init_cp(dev, init, file_priv);
1613         case RADEON_CLEANUP_CP:
1614                 if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
1615                         return r600_do_cleanup_cp(dev);
1616                 else
1617                         return radeon_do_cleanup_cp(dev);
1618         }
1619
1620         return -EINVAL;
1621 }
1622
1623 int radeon_cp_start(struct drm_device *dev, void *data, struct drm_file *file_priv)
1624 {
1625         drm_radeon_private_t *dev_priv = dev->dev_private;
1626         DRM_DEBUG("\n");
1627
1628         LOCK_TEST_WITH_RETURN(dev, file_priv);
1629
1630         if (dev_priv->cp_running) {
1631                 DRM_DEBUG("while CP running\n");
1632                 return 0;
1633         }
1634         if (dev_priv->cp_mode == RADEON_CSQ_PRIDIS_INDDIS) {
1635                 DRM_DEBUG("called with bogus CP mode (%d)\n",
1636                           dev_priv->cp_mode);
1637                 return 0;
1638         }
1639
1640         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
1641                 r600_do_cp_start(dev_priv);
1642         else
1643                 radeon_do_cp_start(dev_priv);
1644
1645         return 0;
1646 }
1647
1648 /* Stop the CP.  The engine must have been idled before calling this
1649  * routine.
1650  */
1651 int radeon_cp_stop(struct drm_device *dev, void *data, struct drm_file *file_priv)
1652 {
1653         drm_radeon_private_t *dev_priv = dev->dev_private;
1654         drm_radeon_cp_stop_t *stop = data;
1655         int ret;
1656         DRM_DEBUG("\n");
1657
1658         LOCK_TEST_WITH_RETURN(dev, file_priv);
1659
1660         if (!dev_priv->cp_running)
1661                 return 0;
1662
1663         /* Flush any pending CP commands.  This ensures any outstanding
1664          * commands are exectuted by the engine before we turn it off.
1665          */
1666         if (stop->flush) {
1667                 radeon_do_cp_flush(dev_priv);
1668         }
1669
1670         /* If we fail to make the engine go idle, we return an error
1671          * code so that the DRM ioctl wrapper can try again.
1672          */
1673         if (stop->idle) {
1674                 if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
1675                         ret = r600_do_cp_idle(dev_priv);
1676                 else
1677                         ret = radeon_do_cp_idle(dev_priv);
1678                 if (ret)
1679                         return ret;
1680         }
1681
1682         /* Finally, we can turn off the CP.  If the engine isn't idle,
1683          * we will get some dropped triangles as they won't be fully
1684          * rendered before the CP is shut down.
1685          */
1686         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
1687                 r600_do_cp_stop(dev_priv);
1688         else
1689                 radeon_do_cp_stop(dev_priv);
1690
1691         /* Reset the engine */
1692         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
1693                 r600_do_engine_reset(dev);
1694         else
1695                 radeon_do_engine_reset(dev);
1696
1697         return 0;
1698 }
1699
1700 void radeon_do_release(struct drm_device * dev)
1701 {
1702         drm_radeon_private_t *dev_priv = dev->dev_private;
1703         int i, ret;
1704
1705         if (dev_priv) {
1706                 if (dev_priv->cp_running) {
1707                         /* Stop the cp */
1708                         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600) {
1709                                 while ((ret = r600_do_cp_idle(dev_priv)) != 0) {
1710                                         DRM_DEBUG("radeon_do_cp_idle %d\n", ret);
1711                                         mtx_sleep(&ret, &dev->dev_lock, 0,
1712                                             "rdnrel", 1);
1713                                 }
1714                         } else {
1715                                 while ((ret = radeon_do_cp_idle(dev_priv)) != 0) {
1716                                         DRM_DEBUG("radeon_do_cp_idle %d\n", ret);
1717                                         mtx_sleep(&ret, &dev->dev_lock, 0,
1718                                             "rdnrel", 1);
1719                                 }
1720                         }
1721                         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600) {
1722                                 r600_do_cp_stop(dev_priv);
1723                                 r600_do_engine_reset(dev);
1724                         } else {
1725                                 radeon_do_cp_stop(dev_priv);
1726                                 radeon_do_engine_reset(dev);
1727                         }
1728                 }
1729
1730                 if ((dev_priv->flags & RADEON_FAMILY_MASK) < CHIP_R600) {
1731                         /* Disable *all* interrupts */
1732                         if (dev_priv->mmio)     /* remove this after permanent addmaps */
1733                                 RADEON_WRITE(RADEON_GEN_INT_CNTL, 0);
1734
1735                         if (dev_priv->mmio) {   /* remove all surfaces */
1736                                 for (i = 0; i < RADEON_MAX_SURFACES; i++) {
1737                                         RADEON_WRITE(RADEON_SURFACE0_INFO + 16 * i, 0);
1738                                         RADEON_WRITE(RADEON_SURFACE0_LOWER_BOUND +
1739                                                      16 * i, 0);
1740                                         RADEON_WRITE(RADEON_SURFACE0_UPPER_BOUND +
1741                                                      16 * i, 0);
1742                                 }
1743                         }
1744                 }
1745
1746                 /* Free memory heap structures */
1747                 radeon_mem_takedown(&(dev_priv->gart_heap));
1748                 radeon_mem_takedown(&(dev_priv->fb_heap));
1749
1750                 /* deallocate kernel resources */
1751                 if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
1752                         r600_do_cleanup_cp(dev);
1753                 else
1754                         radeon_do_cleanup_cp(dev);
1755         }
1756 }
1757
1758 /* Just reset the CP ring.  Called as part of an X Server engine reset.
1759  */
1760 int radeon_cp_reset(struct drm_device *dev, void *data, struct drm_file *file_priv)
1761 {
1762         drm_radeon_private_t *dev_priv = dev->dev_private;
1763         DRM_DEBUG("\n");
1764
1765         LOCK_TEST_WITH_RETURN(dev, file_priv);
1766
1767         if (!dev_priv) {
1768                 DRM_DEBUG("called before init done\n");
1769                 return -EINVAL;
1770         }
1771
1772         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
1773                 r600_do_cp_reset(dev_priv);
1774         else
1775                 radeon_do_cp_reset(dev_priv);
1776
1777         /* The CP is no longer running after an engine reset */
1778         dev_priv->cp_running = 0;
1779
1780         return 0;
1781 }
1782
1783 int radeon_cp_idle(struct drm_device *dev, void *data, struct drm_file *file_priv)
1784 {
1785         drm_radeon_private_t *dev_priv = dev->dev_private;
1786         DRM_DEBUG("\n");
1787
1788         LOCK_TEST_WITH_RETURN(dev, file_priv);
1789
1790         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
1791                 return r600_do_cp_idle(dev_priv);
1792         else
1793                 return radeon_do_cp_idle(dev_priv);
1794 }
1795
1796 /* Added by Charl P. Botha to call radeon_do_resume_cp().
1797  */
1798 int radeon_cp_resume(struct drm_device *dev, void *data, struct drm_file *file_priv)
1799 {
1800         drm_radeon_private_t *dev_priv = dev->dev_private;
1801         DRM_DEBUG("\n");
1802
1803         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
1804                 return r600_do_resume_cp(dev, file_priv);
1805         else
1806                 return radeon_do_resume_cp(dev, file_priv);
1807 }
1808
1809 int radeon_engine_reset(struct drm_device *dev, void *data, struct drm_file *file_priv)
1810 {
1811         drm_radeon_private_t *dev_priv = dev->dev_private;
1812         DRM_DEBUG("\n");
1813
1814         LOCK_TEST_WITH_RETURN(dev, file_priv);
1815
1816         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
1817                 return r600_do_engine_reset(dev);
1818         else
1819                 return radeon_do_engine_reset(dev);
1820 }
1821
1822 /* ================================================================
1823  * Fullscreen mode
1824  */
1825
1826 /* KW: Deprecated to say the least:
1827  */
1828 int radeon_fullscreen(struct drm_device *dev, void *data, struct drm_file *file_priv)
1829 {
1830         return 0;
1831 }
1832
1833 /* ================================================================
1834  * Freelist management
1835  */
1836
1837 /* Original comment: FIXME: ROTATE_BUFS is a hack to cycle through
1838  *   bufs until freelist code is used.  Note this hides a problem with
1839  *   the scratch register * (used to keep track of last buffer
1840  *   completed) being written to before * the last buffer has actually
1841  *   completed rendering.
1842  *
1843  * KW:  It's also a good way to find free buffers quickly.
1844  *
1845  * KW: Ideally this loop wouldn't exist, and freelist_get wouldn't
1846  * sleep.  However, bugs in older versions of radeon_accel.c mean that
1847  * we essentially have to do this, else old clients will break.
1848  *
1849  * However, it does leave open a potential deadlock where all the
1850  * buffers are held by other clients, which can't release them because
1851  * they can't get the lock.
1852  */
1853
1854 struct drm_buf *radeon_freelist_get(struct drm_device * dev)
1855 {
1856         struct drm_device_dma *dma = dev->dma;
1857         drm_radeon_private_t *dev_priv = dev->dev_private;
1858         drm_radeon_buf_priv_t *buf_priv;
1859         struct drm_buf *buf;
1860         int i, t;
1861         int start;
1862
1863         if (++dev_priv->last_buf >= dma->buf_count)
1864                 dev_priv->last_buf = 0;
1865
1866         start = dev_priv->last_buf;
1867
1868         for (t = 0; t < dev_priv->usec_timeout; t++) {
1869                 u32 done_age = GET_SCRATCH(dev_priv, 1);
1870                 DRM_DEBUG("done_age = %d\n", done_age);
1871                 for (i = start; i < dma->buf_count; i++) {
1872                         buf = dma->buflist[i];
1873                         buf_priv = buf->dev_private;
1874                         if (buf->file_priv == NULL || (buf->pending &&
1875                                                        buf_priv->age <=
1876                                                        done_age)) {
1877                                 dev_priv->stats.requested_bufs++;
1878                                 buf->pending = 0;
1879                                 return buf;
1880                         }
1881                         start = 0;
1882                 }
1883
1884                 if (t) {
1885                         DRM_UDELAY(1);
1886                         dev_priv->stats.freelist_loops++;
1887                 }
1888         }
1889
1890         DRM_DEBUG("returning NULL!\n");
1891         return NULL;
1892 }
1893
1894 #if 0
1895 struct drm_buf *radeon_freelist_get(struct drm_device * dev)
1896 {
1897         struct drm_device_dma *dma = dev->dma;
1898         drm_radeon_private_t *dev_priv = dev->dev_private;
1899         drm_radeon_buf_priv_t *buf_priv;
1900         struct drm_buf *buf;
1901         int i, t;
1902         int start;
1903         u32 done_age;
1904
1905         done_age = radeon_read_ring_rptr(dev_priv, RADEON_SCRATCHOFF(1));
1906         if (++dev_priv->last_buf >= dma->buf_count)
1907                 dev_priv->last_buf = 0;
1908
1909         start = dev_priv->last_buf;
1910         dev_priv->stats.freelist_loops++;
1911
1912         for (t = 0; t < 2; t++) {
1913                 for (i = start; i < dma->buf_count; i++) {
1914                         buf = dma->buflist[i];
1915                         buf_priv = buf->dev_private;
1916                         if (buf->file_priv == 0 || (buf->pending &&
1917                                                     buf_priv->age <=
1918                                                     done_age)) {
1919                                 dev_priv->stats.requested_bufs++;
1920                                 buf->pending = 0;
1921                                 return buf;
1922                         }
1923                 }
1924                 start = 0;
1925         }
1926
1927         return NULL;
1928 }
1929 #endif
1930
1931 void radeon_freelist_reset(struct drm_device * dev)
1932 {
1933         struct drm_device_dma *dma = dev->dma;
1934         drm_radeon_private_t *dev_priv = dev->dev_private;
1935         int i;
1936
1937         dev_priv->last_buf = 0;
1938         for (i = 0; i < dma->buf_count; i++) {
1939                 struct drm_buf *buf = dma->buflist[i];
1940                 drm_radeon_buf_priv_t *buf_priv = buf->dev_private;
1941                 buf_priv->age = 0;
1942         }
1943 }
1944
1945 /* ================================================================
1946  * CP command submission
1947  */
1948
1949 int radeon_wait_ring(drm_radeon_private_t * dev_priv, int n)
1950 {
1951         drm_radeon_ring_buffer_t *ring = &dev_priv->ring;
1952         int i;
1953         u32 last_head = GET_RING_HEAD(dev_priv);
1954
1955         for (i = 0; i < dev_priv->usec_timeout; i++) {
1956                 u32 head = GET_RING_HEAD(dev_priv);
1957
1958                 ring->space = (head - ring->tail) * sizeof(u32);
1959                 if (ring->space <= 0)
1960                         ring->space += ring->size;
1961                 if (ring->space > n)
1962                         return 0;
1963
1964                 dev_priv->stats.boxes |= RADEON_BOX_WAIT_IDLE;
1965
1966                 if (head != last_head)
1967                         i = 0;
1968                 last_head = head;
1969
1970                 DRM_UDELAY(1);
1971         }
1972
1973         /* FIXME: This return value is ignored in the BEGIN_RING macro! */
1974 #if RADEON_FIFO_DEBUG
1975         radeon_status(dev_priv);
1976         DRM_ERROR("failed!\n");
1977 #endif
1978         return -EBUSY;
1979 }
1980
1981 static int radeon_cp_get_buffers(struct drm_device *dev,
1982                                  struct drm_file *file_priv,
1983                                  struct drm_dma * d)
1984 {
1985         int i;
1986         struct drm_buf *buf;
1987
1988         for (i = d->granted_count; i < d->request_count; i++) {
1989                 buf = radeon_freelist_get(dev);
1990                 if (!buf)
1991                         return -EBUSY;  /* NOTE: broken client */
1992
1993                 buf->file_priv = file_priv;
1994
1995                 if (DRM_COPY_TO_USER(&d->request_indices[i], &buf->idx,
1996                                      sizeof(buf->idx)))
1997                         return -EFAULT;
1998                 if (DRM_COPY_TO_USER(&d->request_sizes[i], &buf->total,
1999                                      sizeof(buf->total)))
2000                         return -EFAULT;
2001
2002                 d->granted_count++;
2003         }
2004         return 0;
2005 }
2006
2007 int radeon_cp_buffers(struct drm_device *dev, void *data, struct drm_file *file_priv)
2008 {
2009         struct drm_device_dma *dma = dev->dma;
2010         int ret = 0;
2011         struct drm_dma *d = data;
2012
2013         LOCK_TEST_WITH_RETURN(dev, file_priv);
2014
2015         /* Please don't send us buffers.
2016          */
2017         if (d->send_count != 0) {
2018                 DRM_ERROR("Process %d trying to send %d buffers via drmDMA\n",
2019                           DRM_CURRENTPID, d->send_count);
2020                 return -EINVAL;
2021         }
2022
2023         /* We'll send you buffers.
2024          */
2025         if (d->request_count < 0 || d->request_count > dma->buf_count) {
2026                 DRM_ERROR("Process %d trying to get %d buffers (of %d max)\n",
2027                           DRM_CURRENTPID, d->request_count, dma->buf_count);
2028                 return -EINVAL;
2029         }
2030
2031         d->granted_count = 0;
2032
2033         if (d->request_count) {
2034                 ret = radeon_cp_get_buffers(dev, file_priv, d);
2035         }
2036
2037         return ret;
2038 }
2039
2040 int radeon_driver_load(struct drm_device *dev, unsigned long flags)
2041 {
2042         drm_radeon_private_t *dev_priv;
2043         int ret = 0;
2044
2045         dev_priv = drm_alloc(sizeof(drm_radeon_private_t), DRM_MEM_DRIVER);
2046         if (dev_priv == NULL)
2047                 return -ENOMEM;
2048
2049         memset(dev_priv, 0, sizeof(drm_radeon_private_t));
2050         dev->dev_private = (void *)dev_priv;
2051         dev_priv->flags = flags;
2052
2053         switch (flags & RADEON_FAMILY_MASK) {
2054         case CHIP_R100:
2055         case CHIP_RV200:
2056         case CHIP_R200:
2057         case CHIP_R300:
2058         case CHIP_R350:
2059         case CHIP_R420:
2060         case CHIP_R423:
2061         case CHIP_RV410:
2062         case CHIP_RV515:
2063         case CHIP_R520:
2064         case CHIP_RV570:
2065         case CHIP_R580:
2066                 dev_priv->flags |= RADEON_HAS_HIERZ;
2067                 break;
2068         default:
2069                 /* all other chips have no hierarchical z buffer */
2070                 break;
2071         }
2072
2073         if (drm_device_is_agp(dev))
2074                 dev_priv->flags |= RADEON_IS_AGP;
2075         else if (drm_device_is_pcie(dev))
2076                 dev_priv->flags |= RADEON_IS_PCIE;
2077         else
2078                 dev_priv->flags |= RADEON_IS_PCI;
2079
2080         ret = drm_addmap(dev, drm_get_resource_start(dev, 2),
2081                          drm_get_resource_len(dev, 2), _DRM_REGISTERS,
2082                          _DRM_READ_ONLY | _DRM_DRIVER, &dev_priv->mmio);
2083         if (ret != 0)
2084                 return ret;
2085
2086         ret = drm_vblank_init(dev, 2);
2087         if (ret) {
2088                 radeon_driver_unload(dev);
2089                 return ret;
2090         }
2091
2092         DRM_DEBUG("%s card detected\n",
2093                   ((dev_priv->flags & RADEON_IS_AGP) ? "AGP" : (((dev_priv->flags & RADEON_IS_PCIE) ? "PCIE" : "PCI"))));
2094         return ret;
2095 }
2096
2097 /* Create mappings for registers and framebuffer so userland doesn't necessarily
2098  * have to find them.
2099  */
2100 int radeon_driver_firstopen(struct drm_device *dev)
2101 {
2102         int ret;
2103         drm_local_map_t *map;
2104         drm_radeon_private_t *dev_priv = dev->dev_private;
2105
2106         dev_priv->gart_info.table_size = RADEON_PCIGART_TABLE_SIZE;
2107
2108         dev_priv->fb_aper_offset = drm_get_resource_start(dev, 0);
2109         ret = drm_addmap(dev, dev_priv->fb_aper_offset,
2110                          drm_get_resource_len(dev, 0), _DRM_FRAME_BUFFER,
2111                          _DRM_WRITE_COMBINING, &map);
2112         if (ret != 0)
2113                 return ret;
2114
2115         return 0;
2116 }
2117
2118 int radeon_driver_unload(struct drm_device *dev)
2119 {
2120         drm_radeon_private_t *dev_priv = dev->dev_private;
2121
2122         DRM_DEBUG("\n");
2123
2124         drm_rmmap(dev, dev_priv->mmio);
2125
2126         drm_free(dev_priv, sizeof(*dev_priv), DRM_MEM_DRIVER);
2127
2128         dev->dev_private = NULL;
2129         return 0;
2130 }
2131
2132 void radeon_commit_ring(drm_radeon_private_t *dev_priv)
2133 {
2134         int i;
2135         u32 *ring;
2136         int tail_aligned;
2137
2138         /* check if the ring is padded out to 16-dword alignment */
2139
2140         tail_aligned = dev_priv->ring.tail & 0xf;
2141         if (tail_aligned) {
2142                 int num_p2 = 16 - tail_aligned;
2143
2144                 ring = dev_priv->ring.start;
2145                 /* pad with some CP_PACKET2 */
2146                 for (i = 0; i < num_p2; i++)
2147                         ring[dev_priv->ring.tail + i] = CP_PACKET2();
2148
2149                 dev_priv->ring.tail += i;
2150
2151                 dev_priv->ring.space -= num_p2 * sizeof(u32);
2152         }
2153
2154         dev_priv->ring.tail &= dev_priv->ring.tail_mask;
2155
2156         DRM_MEMORYBARRIER();
2157         GET_RING_HEAD( dev_priv );
2158
2159         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600) {
2160                 RADEON_WRITE(R600_CP_RB_WPTR, dev_priv->ring.tail);
2161                 /* read from PCI bus to ensure correct posting */
2162                 RADEON_READ(R600_CP_RB_RPTR);
2163         } else {
2164                 RADEON_WRITE(RADEON_CP_RB_WPTR, dev_priv->ring.tail);
2165                 /* read from PCI bus to ensure correct posting */
2166                 RADEON_READ(RADEON_CP_RB_RPTR);
2167         }
2168 }