sys/dev/drm/radeon_cp.c

   1 /* radeon_cp.c -- CP support for Radeon -*- linux-c -*-
   2  *
   3  * Copyright 2000 Precision Insight, Inc., Cedar Park, Texas.
   4  * Copyright 2000 VA Linux Systems, Inc., Fremont, California.
   5  * All Rights Reserved.
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining a
   8  * copy of this software and associated documentation files (the "Software"),
   9  * to deal in the Software without restriction, including without limitation
  10  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  11  * and/or sell copies of the Software, and to permit persons to whom the
  12  * Software is furnished to do so, subject to the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the next
  15  * paragraph) shall be included in all copies or substantial portions of the
  16  * Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  21  * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  22  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  23  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  24  * DEALINGS IN THE SOFTWARE.
  25  *
  26  * Authors:
  27  *    Kevin E. Martin <martin@valinux.com>
  28  *    Gareth Hughes <gareth@valinux.com>
  29  *
  30  * $FreeBSD$
  31  */
  32
  33 #include "dev/drm/radeon.h"
  34 #include "dev/drm/drmP.h"
  35 #include "dev/drm/radeon_drm.h"
  36 #include "dev/drm/radeon_drv.h"
  37
  38 #ifdef __linux__
  39 #define __NO_VERSION__
  40 #include <linux/interrupt.h>    /* For task queue support */
  41 #include <linux/delay.h>
  42 #endif /* __linux__ */
  43 #ifdef __FreeBSD__
  44 #include <vm/vm.h>
  45 #include <vm/pmap.h>
  46 #endif /* __FreeBSD__ */
  47
  48 #define RADEON_FIFO_DEBUG       0
  49
  50 #if defined(__alpha__)
  51 # define PCIGART_ENABLED
  52 #else
  53 # undef PCIGART_ENABLED
  54 #endif
  55
  56
  57 /* CP microcode (from ATI) */
  58 static u32 radeon_cp_microcode[][2] = {
  59         { 0x21007000, 0000000000 },
  60         { 0x20007000, 0000000000 },
  61         { 0x000000b4, 0x00000004 },
  62         { 0x000000b8, 0x00000004 },
  63         { 0x6f5b4d4c, 0000000000 },
  64         { 0x4c4c427f, 0000000000 },
  65         { 0x5b568a92, 0000000000 },
  66         { 0x4ca09c6d, 0000000000 },
  67         { 0xad4c4c4c, 0000000000 },
  68         { 0x4ce1af3d, 0000000000 },
  69         { 0xd8afafaf, 0000000000 },
  70         { 0xd64c4cdc, 0000000000 },
  71         { 0x4cd10d10, 0000000000 },
  72         { 0x000f0000, 0x00000016 },
  73         { 0x362f242d, 0000000000 },
  74         { 0x00000012, 0x00000004 },
  75         { 0x000f0000, 0x00000016 },
  76         { 0x362f282d, 0000000000 },
  77         { 0x000380e7, 0x00000002 },
  78         { 0x04002c97, 0x00000002 },
  79         { 0x000f0001, 0x00000016 },
  80         { 0x333a3730, 0000000000 },
  81         { 0x000077ef, 0x00000002 },
  82         { 0x00061000, 0x00000002 },
  83         { 0x00000021, 0x0000001a },
  84         { 0x00004000, 0x0000001e },
  85         { 0x00061000, 0x00000002 },
  86         { 0x00000021, 0x0000001a },
  87         { 0x00004000, 0x0000001e },
  88         { 0x00061000, 0x00000002 },
  89         { 0x00000021, 0x0000001a },
  90         { 0x00004000, 0x0000001e },
  91         { 0x00000017, 0x00000004 },
  92         { 0x0003802b, 0x00000002 },
  93         { 0x040067e0, 0x00000002 },
  94         { 0x00000017, 0x00000004 },
  95         { 0x000077e0, 0x00000002 },
  96         { 0x00065000, 0x00000002 },
  97         { 0x000037e1, 0x00000002 },
  98         { 0x040067e1, 0x00000006 },
  99         { 0x000077e0, 0x00000002 },
 100         { 0x000077e1, 0x00000002 },
 101         { 0x000077e1, 0x00000006 },
 102         { 0xffffffff, 0000000000 },
 103         { 0x10000000, 0000000000 },
 104         { 0x0003802b, 0x00000002 },
 105         { 0x040067e0, 0x00000006 },
 106         { 0x00007675, 0x00000002 },
 107         { 0x00007676, 0x00000002 },
 108         { 0x00007677, 0x00000002 },
 109         { 0x00007678, 0x00000006 },
 110         { 0x0003802c, 0x00000002 },
 111         { 0x04002676, 0x00000002 },
 112         { 0x00007677, 0x00000002 },
 113         { 0x00007678, 0x00000006 },
 114         { 0x0000002f, 0x00000018 },
 115         { 0x0000002f, 0x00000018 },
 116         { 0000000000, 0x00000006 },
 117         { 0x00000030, 0x00000018 },
 118         { 0x00000030, 0x00000018 },
 119         { 0000000000, 0x00000006 },
 120         { 0x01605000, 0x00000002 },
 121         { 0x00065000, 0x00000002 },
 122         { 0x00098000, 0x00000002 },
 123         { 0x00061000, 0x00000002 },
 124         { 0x64c0603e, 0x00000004 },
 125         { 0x000380e6, 0x00000002 },
 126         { 0x040025c5, 0x00000002 },
 127         { 0x00080000, 0x00000016 },
 128         { 0000000000, 0000000000 },
 129         { 0x0400251d, 0x00000002 },
 130         { 0x00007580, 0x00000002 },
 131         { 0x00067581, 0x00000002 },
 132         { 0x04002580, 0x00000002 },
 133         { 0x00067581, 0x00000002 },
 134         { 0x00000049, 0x00000004 },
 135         { 0x00005000, 0000000000 },
 136         { 0x000380e6, 0x00000002 },
 137         { 0x040025c5, 0x00000002 },
 138         { 0x00061000, 0x00000002 },
 139         { 0x0000750e, 0x00000002 },
 140         { 0x00019000, 0x00000002 },
 141         { 0x00011055, 0x00000014 },
 142         { 0x00000055, 0x00000012 },
 143         { 0x0400250f, 0x00000002 },
 144         { 0x0000504f, 0x00000004 },
 145         { 0x000380e6, 0x00000002 },
 146         { 0x040025c5, 0x00000002 },
 147         { 0x00007565, 0x00000002 },
 148         { 0x00007566, 0x00000002 },
 149         { 0x00000058, 0x00000004 },
 150         { 0x000380e6, 0x00000002 },
 151         { 0x040025c5, 0x00000002 },
 152         { 0x01e655b4, 0x00000002 },
 153         { 0x4401b0e4, 0x00000002 },
 154         { 0x01c110e4, 0x00000002 },
 155         { 0x26667066, 0x00000018 },
 156         { 0x040c2565, 0x00000002 },
 157         { 0x00000066, 0x00000018 },
 158         { 0x04002564, 0x00000002 },
 159         { 0x00007566, 0x00000002 },
 160         { 0x0000005d, 0x00000004 },
 161         { 0x00401069, 0x00000008 },
 162         { 0x00101000, 0x00000002 },
 163         { 0x000d80ff, 0x00000002 },
 164         { 0x0080006c, 0x00000008 },
 165         { 0x000f9000, 0x00000002 },
 166         { 0x000e00ff, 0x00000002 },
 167         { 0000000000, 0x00000006 },
 168         { 0x0000008f, 0x00000018 },
 169         { 0x0000005b, 0x00000004 },
 170         { 0x000380e6, 0x00000002 },
 171         { 0x040025c5, 0x00000002 },
 172         { 0x00007576, 0x00000002 },
 173         { 0x00065000, 0x00000002 },
 174         { 0x00009000, 0x00000002 },
 175         { 0x00041000, 0x00000002 },
 176         { 0x0c00350e, 0x00000002 },
 177         { 0x00049000, 0x00000002 },
 178         { 0x00051000, 0x00000002 },
 179         { 0x01e785f8, 0x00000002 },
 180         { 0x00200000, 0x00000002 },
 181         { 0x0060007e, 0x0000000c },
 182         { 0x00007563, 0x00000002 },
 183         { 0x006075f0, 0x00000021 },
 184         { 0x20007073, 0x00000004 },
 185         { 0x00005073, 0x00000004 },
 186         { 0x000380e6, 0x00000002 },
 187         { 0x040025c5, 0x00000002 },
 188         { 0x00007576, 0x00000002 },
 189         { 0x00007577, 0x00000002 },
 190         { 0x0000750e, 0x00000002 },
 191         { 0x0000750f, 0x00000002 },
 192         { 0x00a05000, 0x00000002 },
 193         { 0x00600083, 0x0000000c },
 194         { 0x006075f0, 0x00000021 },
 195         { 0x000075f8, 0x00000002 },
 196         { 0x00000083, 0x00000004 },
 197         { 0x000a750e, 0x00000002 },
 198         { 0x000380e6, 0x00000002 },
 199         { 0x040025c5, 0x00000002 },
 200         { 0x0020750f, 0x00000002 },
 201         { 0x00600086, 0x00000004 },
 202         { 0x00007570, 0x00000002 },
 203         { 0x00007571, 0x00000002 },
 204         { 0x00007572, 0x00000006 },
 205         { 0x000380e6, 0x00000002 },
 206         { 0x040025c5, 0x00000002 },
 207         { 0x00005000, 0x00000002 },
 208         { 0x00a05000, 0x00000002 },
 209         { 0x00007568, 0x00000002 },
 210         { 0x00061000, 0x00000002 },
 211         { 0x00000095, 0x0000000c },
 212         { 0x00058000, 0x00000002 },
 213         { 0x0c607562, 0x00000002 },
 214         { 0x00000097, 0x00000004 },
 215         { 0x000380e6, 0x00000002 },
 216         { 0x040025c5, 0x00000002 },
 217         { 0x00600096, 0x00000004 },
 218         { 0x400070e5, 0000000000 },
 219         { 0x000380e6, 0x00000002 },
 220         { 0x040025c5, 0x00000002 },
 221         { 0x000380e5, 0x00000002 },
 222         { 0x000000a8, 0x0000001c },
 223         { 0x000650aa, 0x00000018 },
 224         { 0x040025bb, 0x00000002 },
 225         { 0x000610ab, 0x00000018 },
 226         { 0x040075bc, 0000000000 },
 227         { 0x000075bb, 0x00000002 },
 228         { 0x000075bc, 0000000000 },
 229         { 0x00090000, 0x00000006 },
 230         { 0x00090000, 0x00000002 },
 231         { 0x000d8002, 0x00000006 },
 232         { 0x00007832, 0x00000002 },
 233         { 0x00005000, 0x00000002 },
 234         { 0x000380e7, 0x00000002 },
 235         { 0x04002c97, 0x00000002 },
 236         { 0x00007820, 0x00000002 },
 237         { 0x00007821, 0x00000002 },
 238         { 0x00007800, 0000000000 },
 239         { 0x01200000, 0x00000002 },
 240         { 0x20077000, 0x00000002 },
 241         { 0x01200000, 0x00000002 },
 242         { 0x20007000, 0x00000002 },
 243         { 0x00061000, 0x00000002 },
 244         { 0x0120751b, 0x00000002 },
 245         { 0x8040750a, 0x00000002 },
 246         { 0x8040750b, 0x00000002 },
 247         { 0x00110000, 0x00000002 },
 248         { 0x000380e5, 0x00000002 },
 249         { 0x000000c6, 0x0000001c },
 250         { 0x000610ab, 0x00000018 },
 251         { 0x844075bd, 0x00000002 },
 252         { 0x000610aa, 0x00000018 },
 253         { 0x840075bb, 0x00000002 },
 254         { 0x000610ab, 0x00000018 },
 255         { 0x844075bc, 0x00000002 },
 256         { 0x000000c9, 0x00000004 },
 257         { 0x804075bd, 0x00000002 },
 258         { 0x800075bb, 0x00000002 },
 259         { 0x804075bc, 0x00000002 },
 260         { 0x00108000, 0x00000002 },
 261         { 0x01400000, 0x00000002 },
 262         { 0x006000cd, 0x0000000c },
 263         { 0x20c07000, 0x00000020 },
 264         { 0x000000cf, 0x00000012 },
 265         { 0x00800000, 0x00000006 },
 266         { 0x0080751d, 0x00000006 },
 267         { 0000000000, 0000000000 },
 268         { 0x0000775c, 0x00000002 },
 269         { 0x00a05000, 0x00000002 },
 270         { 0x00661000, 0x00000002 },
 271         { 0x0460275d, 0x00000020 },
 272         { 0x00004000, 0000000000 },
 273         { 0x01e00830, 0x00000002 },
 274         { 0x21007000, 0000000000 },
 275         { 0x6464614d, 0000000000 },
 276         { 0x69687420, 0000000000 },
 277         { 0x00000073, 0000000000 },
 278         { 0000000000, 0000000000 },
 279         { 0x00005000, 0x00000002 },
 280         { 0x000380d0, 0x00000002 },
 281         { 0x040025e0, 0x00000002 },
 282         { 0x000075e1, 0000000000 },
 283         { 0x00000001, 0000000000 },
 284         { 0x000380e0, 0x00000002 },
 285         { 0x04002394, 0x00000002 },
 286         { 0x00005000, 0000000000 },
 287         { 0000000000, 0000000000 },
 288         { 0000000000, 0000000000 },
 289         { 0x00000008, 0000000000 },
 290         { 0x00000004, 0000000000 },
 291         { 0000000000, 0000000000 },
 292         { 0000000000, 0000000000 },
 293         { 0000000000, 0000000000 },
 294         { 0000000000, 0000000000 },
 295         { 0000000000, 0000000000 },
 296         { 0000000000, 0000000000 },
 297         { 0000000000, 0000000000 },
 298         { 0000000000, 0000000000 },
 299         { 0000000000, 0000000000 },
 300         { 0000000000, 0000000000 },
 301         { 0000000000, 0000000000 },
 302         { 0000000000, 0000000000 },
 303         { 0000000000, 0000000000 },
 304         { 0000000000, 0000000000 },
 305         { 0000000000, 0000000000 },
 306         { 0000000000, 0000000000 },
 307         { 0000000000, 0000000000 },
 308         { 0000000000, 0000000000 },
 309         { 0000000000, 0000000000 },
 310         { 0000000000, 0000000000 },
 311         { 0000000000, 0000000000 },
 312         { 0000000000, 0000000000 },
 313         { 0000000000, 0000000000 },
 314         { 0000000000, 0000000000 },
 315 };
 316
 317
 318 int RADEON_READ_PLL(drm_device_t *dev, int addr)
 319 {
 320         drm_radeon_private_t *dev_priv = dev->dev_private;
 321
 322         RADEON_WRITE8(RADEON_CLOCK_CNTL_INDEX, addr & 0x1f);
 323         return RADEON_READ(RADEON_CLOCK_CNTL_DATA);
 324 }
 325
 326 #if RADEON_FIFO_DEBUG
 327 static void radeon_status( drm_radeon_private_t *dev_priv )
 328 {
 329         printk( "%s:\n", __func__ );
 330         printk( "RBBM_STATUS = 0x%08x\n",
 331                 (unsigned int)RADEON_READ( RADEON_RBBM_STATUS ) );
 332         printk( "CP_RB_RTPR = 0x%08x\n",
 333                 (unsigned int)RADEON_READ( RADEON_CP_RB_RPTR ) );
 334         printk( "CP_RB_WTPR = 0x%08x\n",
 335                 (unsigned int)RADEON_READ( RADEON_CP_RB_WPTR ) );
 336         printk( "AIC_CNTL = 0x%08x\n",
 337                 (unsigned int)RADEON_READ( RADEON_AIC_CNTL ) );
 338         printk( "AIC_STAT = 0x%08x\n",
 339                 (unsigned int)RADEON_READ( RADEON_AIC_STAT ) );
 340         printk( "AIC_PT_BASE = 0x%08x\n",
 341                 (unsigned int)RADEON_READ( RADEON_AIC_PT_BASE ) );
 342         printk( "TLB_ADDR = 0x%08x\n",
 343                 (unsigned int)RADEON_READ( RADEON_AIC_TLB_ADDR ) );
 344         printk( "TLB_DATA = 0x%08x\n",
 345                 (unsigned int)RADEON_READ( RADEON_AIC_TLB_DATA ) );
 346 }
 347 #endif
 348
 349
 350 /* ================================================================
 351  * Engine, FIFO control
 352  */
 353
 354 static int radeon_do_pixcache_flush( drm_radeon_private_t *dev_priv )
 355 {
 356         u32 tmp;
 357         int i;
 358
 359         tmp  = RADEON_READ( RADEON_RB2D_DSTCACHE_CTLSTAT );
 360         tmp |= RADEON_RB2D_DC_FLUSH_ALL;
 361         RADEON_WRITE( RADEON_RB2D_DSTCACHE_CTLSTAT, tmp );
 362
 363         for ( i = 0 ; i < dev_priv->usec_timeout ; i++ ) {
 364                 if ( !(RADEON_READ( RADEON_RB2D_DSTCACHE_CTLSTAT )
 365                        & RADEON_RB2D_DC_BUSY) ) {
 366                         return 0;
 367                 }
 368                 DRM_OS_DELAY( 1 );
 369         }
 370
 371 #if RADEON_FIFO_DEBUG
 372         DRM_ERROR( "failed!\n" );
 373         radeon_status( dev_priv );
 374 #endif
 375         return DRM_OS_ERR(EBUSY);
 376 }
 377
 378 static int radeon_do_wait_for_fifo( drm_radeon_private_t *dev_priv,
 379                                     int entries )
 380 {
 381         int i;
 382
 383         for ( i = 0 ; i < dev_priv->usec_timeout ; i++ ) {
 384                 int slots = ( RADEON_READ( RADEON_RBBM_STATUS )
 385                               & RADEON_RBBM_FIFOCNT_MASK );
 386                 if ( slots >= entries ) return 0;
 387                 DRM_OS_DELAY( 1 );
 388         }
 389
 390 #if RADEON_FIFO_DEBUG
 391         DRM_ERROR( "failed!\n" );
 392         radeon_status( dev_priv );
 393 #endif
 394         return DRM_OS_ERR(EBUSY);
 395 }
 396
 397 static int radeon_do_wait_for_idle( drm_radeon_private_t *dev_priv )
 398 {
 399         int i, ret;
 400
 401         ret = radeon_do_wait_for_fifo( dev_priv, 64 );
 402         if ( ret ) return ret;
 403         for ( i = 0 ; i < dev_priv->usec_timeout ; i++ ) {
 404                 if ( !(RADEON_READ( RADEON_RBBM_STATUS )
 405                        & RADEON_RBBM_ACTIVE) ) {
 406                         radeon_do_pixcache_flush( dev_priv );
 407                         return 0;
 408                 }
 409                 DRM_OS_DELAY( 1 );
 410         }
 411
 412 #if RADEON_FIFO_DEBUG
 413         DRM_ERROR( "failed!\n" );
 414         radeon_status( dev_priv );
 415 #endif
 416         return DRM_OS_ERR(EBUSY);
 417 }
 418
 419
 420 /* ================================================================
 421  * CP control, initialization
 422  */
 423
 424 /* Load the microcode for the CP */
 425 static void radeon_cp_load_microcode( drm_radeon_private_t *dev_priv )
 426 {
 427         int i;
 428         DRM_DEBUG( "%s\n", __func__ );
 429
 430         radeon_do_wait_for_idle( dev_priv );
 431
 432         RADEON_WRITE( RADEON_CP_ME_RAM_ADDR, 0 );
 433         for ( i = 0 ; i < 256 ; i++ ) {
 434                 RADEON_WRITE( RADEON_CP_ME_RAM_DATAH,
 435                               radeon_cp_microcode[i][1] );
 436                 RADEON_WRITE( RADEON_CP_ME_RAM_DATAL,
 437                               radeon_cp_microcode[i][0] );
 438         }
 439 }
 440
 441 /* Flush any pending commands to the CP.  This should only be used just
 442  * prior to a wait for idle, as it informs the engine that the command
 443  * stream is ending.
 444  */
 445 static void radeon_do_cp_flush( drm_radeon_private_t *dev_priv )
 446 {
 447         DRM_DEBUG( "%s\n", __func__ );
 448 #if 0
 449         u32 tmp;
 450
 451         tmp = RADEON_READ( RADEON_CP_RB_WPTR ) | (1 << 31);
 452         RADEON_WRITE( RADEON_CP_RB_WPTR, tmp );
 453 #endif
 454 }
 455
 456 /* Wait for the CP to go idle.
 457  */
 458 int radeon_do_cp_idle( drm_radeon_private_t *dev_priv )
 459 {
 460         RING_LOCALS;
 461         DRM_DEBUG( "%s\n", __func__ );
 462
 463         BEGIN_RING( 6 );
 464
 465         RADEON_PURGE_CACHE();
 466         RADEON_PURGE_ZCACHE();
 467         RADEON_WAIT_UNTIL_IDLE();
 468
 469         ADVANCE_RING();
 470
 471         return radeon_do_wait_for_idle( dev_priv );
 472 }
 473
 474 /* Start the Command Processor.
 475  */
 476 static void radeon_do_cp_start( drm_radeon_private_t *dev_priv )
 477 {
 478         RING_LOCALS;
 479         DRM_DEBUG( "%s\n", __func__ );
 480
 481         radeon_do_wait_for_idle( dev_priv );
 482
 483         RADEON_WRITE( RADEON_CP_CSQ_CNTL, dev_priv->cp_mode );
 484
 485         dev_priv->cp_running = 1;
 486
 487         BEGIN_RING( 6 );
 488
 489         RADEON_PURGE_CACHE();
 490         RADEON_PURGE_ZCACHE();
 491         RADEON_WAIT_UNTIL_IDLE();
 492
 493         ADVANCE_RING();
 494 }
 495
 496 /* Reset the Command Processor.  This will not flush any pending
 497  * commands, so you must wait for the CP command stream to complete
 498  * before calling this routine.
 499  */
 500 static void radeon_do_cp_reset( drm_radeon_private_t *dev_priv )
 501 {
 502         u32 cur_read_ptr;
 503         DRM_DEBUG( "%s\n", __func__ );
 504
 505         cur_read_ptr = RADEON_READ( RADEON_CP_RB_RPTR );
 506         RADEON_WRITE( RADEON_CP_RB_WPTR, cur_read_ptr );
 507         *dev_priv->ring.head = cur_read_ptr;
 508         dev_priv->ring.tail = cur_read_ptr;
 509 }
 510
 511 /* Stop the Command Processor.  This will not flush any pending
 512  * commands, so you must flush the command stream and wait for the CP
 513  * to go idle before calling this routine.
 514  */
 515 static void radeon_do_cp_stop( drm_radeon_private_t *dev_priv )
 516 {
 517         DRM_DEBUG( "%s\n", __func__ );
 518
 519         RADEON_WRITE( RADEON_CP_CSQ_CNTL, RADEON_CSQ_PRIDIS_INDDIS );
 520
 521         dev_priv->cp_running = 0;
 522 }
 523
 524 /* Reset the engine.  This will stop the CP if it is running.
 525  */
 526 static int radeon_do_engine_reset( drm_device_t *dev )
 527 {
 528         drm_radeon_private_t *dev_priv = dev->dev_private;
 529         u32 clock_cntl_index, mclk_cntl, rbbm_soft_reset;
 530         DRM_DEBUG( "%s\n", __func__ );
 531
 532         radeon_do_pixcache_flush( dev_priv );
 533
 534         clock_cntl_index = RADEON_READ( RADEON_CLOCK_CNTL_INDEX );
 535         mclk_cntl = RADEON_READ_PLL( dev, RADEON_MCLK_CNTL );
 536
 537         RADEON_WRITE_PLL( RADEON_MCLK_CNTL, ( mclk_cntl |
 538                                               RADEON_FORCEON_MCLKA |
 539                                               RADEON_FORCEON_MCLKB |
 540                                               RADEON_FORCEON_YCLKA |
 541                                               RADEON_FORCEON_YCLKB |
 542                                               RADEON_FORCEON_MC |
 543                                               RADEON_FORCEON_AIC ) );
 544
 545         rbbm_soft_reset = RADEON_READ( RADEON_RBBM_SOFT_RESET );
 546
 547         RADEON_WRITE( RADEON_RBBM_SOFT_RESET, ( rbbm_soft_reset |
 548                                                 RADEON_SOFT_RESET_CP |
 549                                                 RADEON_SOFT_RESET_HI |
 550                                                 RADEON_SOFT_RESET_SE |
 551                                                 RADEON_SOFT_RESET_RE |
 552                                                 RADEON_SOFT_RESET_PP |
 553                                                 RADEON_SOFT_RESET_E2 |
 554                                                 RADEON_SOFT_RESET_RB ) );
 555         RADEON_READ( RADEON_RBBM_SOFT_RESET );
 556         RADEON_WRITE( RADEON_RBBM_SOFT_RESET, ( rbbm_soft_reset &
 557                                                 ~( RADEON_SOFT_RESET_CP |
 558                                                    RADEON_SOFT_RESET_HI |
 559                                                    RADEON_SOFT_RESET_SE |
 560                                                    RADEON_SOFT_RESET_RE |
 561                                                    RADEON_SOFT_RESET_PP |
 562                                                    RADEON_SOFT_RESET_E2 |
 563                                                    RADEON_SOFT_RESET_RB ) ) );
 564         RADEON_READ( RADEON_RBBM_SOFT_RESET );
 565
 566
 567         RADEON_WRITE_PLL( RADEON_MCLK_CNTL, mclk_cntl );
 568         RADEON_WRITE( RADEON_CLOCK_CNTL_INDEX, clock_cntl_index );
 569         RADEON_WRITE( RADEON_RBBM_SOFT_RESET,  rbbm_soft_reset );
 570
 571         /* Reset the CP ring */
 572         radeon_do_cp_reset( dev_priv );
 573
 574         /* The CP is no longer running after an engine reset */
 575         dev_priv->cp_running = 0;
 576
 577         /* Reset any pending vertex, indirect buffers */
 578         radeon_freelist_reset( dev );
 579
 580         return 0;
 581 }
 582
 583 static void radeon_cp_init_ring_buffer( drm_device_t *dev,
 584                                         drm_radeon_private_t *dev_priv )
 585 {
 586         u32 ring_start, cur_read_ptr;
 587         u32 tmp;
 588
 589         /* Initialize the memory controller */
 590         RADEON_WRITE( RADEON_MC_FB_LOCATION,
 591                       (dev_priv->agp_vm_start - 1) & 0xffff0000 );
 592
 593         if ( !dev_priv->is_pci ) {
 594                 RADEON_WRITE( RADEON_MC_AGP_LOCATION,
 595                               (((dev_priv->agp_vm_start - 1 +
 596                                  dev_priv->agp_size) & 0xffff0000) |
 597                                (dev_priv->agp_vm_start >> 16)) );
 598         }
 599
 600 #if __REALLY_HAVE_AGP
 601         if ( !dev_priv->is_pci )
 602                 ring_start = (dev_priv->cp_ring->offset
 603                               - dev->agp->base
 604                               + dev_priv->agp_vm_start);
 605        else
 606 #endif
 607                 ring_start = (dev_priv->cp_ring->offset
 608                               - dev->sg->handle
 609                               + dev_priv->agp_vm_start);
 610
 611         RADEON_WRITE( RADEON_CP_RB_BASE, ring_start );
 612
 613         /* Set the write pointer delay */
 614         RADEON_WRITE( RADEON_CP_RB_WPTR_DELAY, 0 );
 615
 616         /* Initialize the ring buffer's read and write pointers */
 617         cur_read_ptr = RADEON_READ( RADEON_CP_RB_RPTR );
 618         RADEON_WRITE( RADEON_CP_RB_WPTR, cur_read_ptr );
 619         *dev_priv->ring.head = cur_read_ptr;
 620         dev_priv->ring.tail = cur_read_ptr;
 621
 622 #if __REALLY_HAVE_SG
 623         if ( !dev_priv->is_pci ) {
 624 #endif
 625                 RADEON_WRITE( RADEON_CP_RB_RPTR_ADDR,
 626                               dev_priv->ring_rptr->offset );
 627 #if __REALLY_HAVE_SG
 628         } else {
 629                 drm_sg_mem_t *entry = dev->sg;
 630                 unsigned long tmp_ofs, page_ofs;
 631
 632                 tmp_ofs = dev_priv->ring_rptr->offset - dev->sg->handle;
 633                 page_ofs = tmp_ofs >> PAGE_SHIFT;
 634
 635                 RADEON_WRITE( RADEON_CP_RB_RPTR_ADDR,
 636                              entry->busaddr[page_ofs]);
 637                 DRM_DEBUG( "ring rptr: offset=0x%08x handle=0x%08lx\n",
 638                            entry->busaddr[page_ofs],
 639                            entry->handle + tmp_ofs );
 640         }
 641 #endif
 642
 643         /* Set ring buffer size */
 644         RADEON_WRITE( RADEON_CP_RB_CNTL, dev_priv->ring.size_l2qw );
 645
 646         radeon_do_wait_for_idle( dev_priv );
 647
 648         /* Turn on bus mastering */
 649         tmp = RADEON_READ( RADEON_BUS_CNTL ) & ~RADEON_BUS_MASTER_DIS;
 650         RADEON_WRITE( RADEON_BUS_CNTL, tmp );
 651
 652         /* Sync everything up */
 653         RADEON_WRITE( RADEON_ISYNC_CNTL,
 654                       (RADEON_ISYNC_ANY2D_IDLE3D |
 655                        RADEON_ISYNC_ANY3D_IDLE2D |
 656                        RADEON_ISYNC_WAIT_IDLEGUI |
 657                        RADEON_ISYNC_CPSCRATCH_IDLEGUI) );
 658 }
 659
 660 static int radeon_do_init_cp( drm_device_t *dev, drm_radeon_init_t *init )
 661 {
 662         drm_radeon_private_t *dev_priv;
 663 #ifdef __linux__
 664         struct list_head *list;
 665 #endif /* __linux__ */
 666 #ifdef __FreeBSD__
 667         drm_map_list_entry_t *listentry;
 668 #endif /* __FreeBSD__ */
 669         u32 tmp;
 670         DRM_DEBUG( "%s\n", __func__ );
 671
 672         dev_priv = DRM(alloc)( sizeof(drm_radeon_private_t), DRM_MEM_DRIVER );
 673         if ( dev_priv == NULL )
 674                 return DRM_OS_ERR(ENOMEM);
 675
 676         memset( dev_priv, 0, sizeof(drm_radeon_private_t) );
 677
 678         dev_priv->is_pci = init->is_pci;
 679
 680 #if !defined(PCIGART_ENABLED)
 681         /* PCI support is not 100% working, so we disable it here.
 682          */
 683         if ( dev_priv->is_pci ) {
 684                 DRM_ERROR( "PCI GART not yet supported for Radeon!\n" );
 685                 dev->dev_private = (void *)dev_priv;
 686                 radeon_do_cleanup_cp(dev);
 687                 return DRM_OS_ERR(EINVAL);
 688         }
 689 #endif
 690
 691         if ( dev_priv->is_pci && !dev->sg ) {
 692                 DRM_ERROR( "PCI GART memory not allocated!\n" );
 693                 dev->dev_private = (void *)dev_priv;
 694                 radeon_do_cleanup_cp(dev);
 695                 return DRM_OS_ERR(EINVAL);
 696         }
 697
 698         dev_priv->usec_timeout = init->usec_timeout;
 699         if ( dev_priv->usec_timeout < 1 ||
 700              dev_priv->usec_timeout > RADEON_MAX_USEC_TIMEOUT ) {
 701                 DRM_DEBUG( "TIMEOUT problem!\n" );
 702                 dev->dev_private = (void *)dev_priv;
 703                 radeon_do_cleanup_cp(dev);
 704                 return DRM_OS_ERR(EINVAL);
 705         }
 706
 707         dev_priv->cp_mode = init->cp_mode;
 708
 709         /* Simple idle check.
 710          */
 711         atomic_set( &dev_priv->idle_count, 0 );
 712
 713         /* We don't support anything other than bus-mastering ring mode,
 714          * but the ring can be in either AGP or PCI space for the ring
 715          * read pointer.
 716          */
 717         if ( ( init->cp_mode != RADEON_CSQ_PRIBM_INDDIS ) &&
 718              ( init->cp_mode != RADEON_CSQ_PRIBM_INDBM ) ) {
 719                 DRM_DEBUG( "BAD cp_mode (%x)!\n", init->cp_mode );
 720                 dev->dev_private = (void *)dev_priv;
 721                 radeon_do_cleanup_cp(dev);
 722                 return DRM_OS_ERR(EINVAL);
 723         }
 724
 725         switch ( init->fb_bpp ) {
 726         case 16:
 727                 dev_priv->color_fmt = RADEON_COLOR_FORMAT_RGB565;
 728                 break;
 729         case 32:
 730         default:
 731                 dev_priv->color_fmt = RADEON_COLOR_FORMAT_ARGB8888;
 732                 break;
 733         }
 734         dev_priv->front_offset  = init->front_offset;
 735         dev_priv->front_pitch   = init->front_pitch;
 736         dev_priv->back_offset   = init->back_offset;
 737         dev_priv->back_pitch    = init->back_pitch;
 738
 739         switch ( init->depth_bpp ) {
 740         case 16:
 741                 dev_priv->depth_fmt = RADEON_DEPTH_FORMAT_16BIT_INT_Z;
 742                 break;
 743         case 32:
 744         default:
 745                 dev_priv->depth_fmt = RADEON_DEPTH_FORMAT_24BIT_INT_Z;
 746                 break;
 747         }
 748         dev_priv->depth_offset  = init->depth_offset;
 749         dev_priv->depth_pitch   = init->depth_pitch;
 750
 751         dev_priv->front_pitch_offset = (((dev_priv->front_pitch/64) << 22) |
 752                                         (dev_priv->front_offset >> 10));
 753         dev_priv->back_pitch_offset = (((dev_priv->back_pitch/64) << 22) |
 754                                        (dev_priv->back_offset >> 10));
 755         dev_priv->depth_pitch_offset = (((dev_priv->depth_pitch/64) << 22) |
 756                                         (dev_priv->depth_offset >> 10));
 757
 758         /* Hardware state for depth clears.  Remove this if/when we no
 759          * longer clear the depth buffer with a 3D rectangle.  Hard-code
 760          * all values to prevent unwanted 3D state from slipping through
 761          * and screwing with the clear operation.
 762          */
 763         dev_priv->depth_clear.rb3d_cntl = (RADEON_PLANE_MASK_ENABLE |
 764                                            RADEON_Z_ENABLE |
 765                                            (dev_priv->color_fmt << 10) |
 766                                            RADEON_ZBLOCK16);
 767
 768         dev_priv->depth_clear.rb3d_zstencilcntl = (dev_priv->depth_fmt |
 769                                                    RADEON_Z_TEST_ALWAYS |
 770                                                    RADEON_STENCIL_TEST_ALWAYS |
 771                                                    RADEON_STENCIL_S_FAIL_KEEP |
 772                                                    RADEON_STENCIL_ZPASS_KEEP |
 773                                                    RADEON_STENCIL_ZFAIL_KEEP |
 774                                                    RADEON_Z_WRITE_ENABLE);
 775
 776         dev_priv->depth_clear.se_cntl = (RADEON_FFACE_CULL_CW |
 777                                          RADEON_BFACE_SOLID |
 778                                          RADEON_FFACE_SOLID |
 779                                          RADEON_FLAT_SHADE_VTX_LAST |
 780                                          RADEON_DIFFUSE_SHADE_FLAT |
 781                                          RADEON_ALPHA_SHADE_FLAT |
 782                                          RADEON_SPECULAR_SHADE_FLAT |
 783                                          RADEON_FOG_SHADE_FLAT |
 784                                          RADEON_VTX_PIX_CENTER_OGL |
 785                                          RADEON_ROUND_MODE_TRUNC |
 786                                          RADEON_ROUND_PREC_8TH_PIX);
 787
 788 #ifdef __linux__
 789         list_for_each(list, &dev->maplist->head) {
 790                 drm_map_list_t *r_list = (drm_map_list_t *)list;
 791                 if( r_list->map &&
 792                     r_list->map->type == _DRM_SHM &&
 793                     r_list->map->flags & _DRM_CONTAINS_LOCK ) {
 794                         dev_priv->sarea = r_list->map;
 795                         break;
 796                 }
 797         }
 798 #endif /* __linux__ */
 799 #ifdef __FreeBSD__
 800         TAILQ_FOREACH(listentry, dev->maplist, link) {
 801                 drm_map_t *map = listentry->map;
 802                 if (map->type == _DRM_SHM &&
 803                         map->flags & _DRM_CONTAINS_LOCK) {
 804                         dev_priv->sarea = map;
 805                         break;
 806                 }
 807         }
 808 #endif /* __FreeBSD__ */
 809
 810         if(!dev_priv->sarea) {
 811                 DRM_ERROR("could not find sarea!\n");
 812                 dev->dev_private = (void *)dev_priv;
 813                 radeon_do_cleanup_cp(dev);
 814                 return DRM_OS_ERR(EINVAL);
 815         }
 816
 817         DRM_FIND_MAP( dev_priv->fb, init->fb_offset );
 818         if(!dev_priv->fb) {
 819                 DRM_ERROR("could not find framebuffer!\n");
 820                 dev->dev_private = (void *)dev_priv;
 821                 radeon_do_cleanup_cp(dev);
 822                 return DRM_OS_ERR(EINVAL);
 823         }
 824         DRM_FIND_MAP( dev_priv->mmio, init->mmio_offset );
 825         if(!dev_priv->mmio) {
 826                 DRM_ERROR("could not find mmio region!\n");
 827                 dev->dev_private = (void *)dev_priv;
 828                 radeon_do_cleanup_cp(dev);
 829                 return DRM_OS_ERR(EINVAL);
 830         }
 831         DRM_FIND_MAP( dev_priv->cp_ring, init->ring_offset );
 832         if(!dev_priv->cp_ring) {
 833                 DRM_ERROR("could not find cp ring region!\n");
 834                 dev->dev_private = (void *)dev_priv;
 835                 radeon_do_cleanup_cp(dev);
 836                 return DRM_OS_ERR(EINVAL);
 837         }
 838         DRM_FIND_MAP( dev_priv->ring_rptr, init->ring_rptr_offset );
 839         if(!dev_priv->ring_rptr) {
 840                 DRM_ERROR("could not find ring read pointer!\n");
 841                 dev->dev_private = (void *)dev_priv;
 842                 radeon_do_cleanup_cp(dev);
 843                 return DRM_OS_ERR(EINVAL);
 844         }
 845         DRM_FIND_MAP( dev_priv->buffers, init->buffers_offset );
 846         if(!dev_priv->buffers) {
 847                 DRM_ERROR("could not find dma buffer region!\n");
 848                 dev->dev_private = (void *)dev_priv;
 849                 radeon_do_cleanup_cp(dev);
 850                 return DRM_OS_ERR(EINVAL);
 851         }
 852
 853         if ( !dev_priv->is_pci ) {
 854                 DRM_FIND_MAP( dev_priv->agp_textures,
 855                               init->agp_textures_offset );
 856                 if(!dev_priv->agp_textures) {
 857                         DRM_ERROR("could not find agp texture region!\n");
 858                         dev->dev_private = (void *)dev_priv;
 859                         radeon_do_cleanup_cp(dev);
 860                         return DRM_OS_ERR(EINVAL);
 861                 }
 862         }
 863
 864         dev_priv->sarea_priv =
 865                 (drm_radeon_sarea_t *)((u8 *)dev_priv->sarea->handle +
 866                                        init->sarea_priv_offset);
 867
 868         if ( !dev_priv->is_pci ) {
 869                 DRM_IOREMAP( dev_priv->cp_ring );
 870                 DRM_IOREMAP( dev_priv->ring_rptr );
 871                 DRM_IOREMAP( dev_priv->buffers );
 872                 if(!dev_priv->cp_ring->handle ||
 873                    !dev_priv->ring_rptr->handle ||
 874                    !dev_priv->buffers->handle) {
 875                         DRM_ERROR("could not find ioremap agp regions!\n");
 876                         dev->dev_private = (void *)dev_priv;
 877                         radeon_do_cleanup_cp(dev);
 878                         return DRM_OS_ERR(EINVAL);
 879                 }
 880         } else {
 881                 dev_priv->cp_ring->handle =
 882                         (void *)dev_priv->cp_ring->offset;
 883                 dev_priv->ring_rptr->handle =
 884                         (void *)dev_priv->ring_rptr->offset;
 885                 dev_priv->buffers->handle = (void *)dev_priv->buffers->offset;
 886
 887                 DRM_DEBUG( "dev_priv->cp_ring->handle %p\n",
 888                            dev_priv->cp_ring->handle );
 889                 DRM_DEBUG( "dev_priv->ring_rptr->handle %p\n",
 890                            dev_priv->ring_rptr->handle );
 891                 DRM_DEBUG( "dev_priv->buffers->handle %p\n",
 892                            dev_priv->buffers->handle );
 893         }
 894
 895
 896         dev_priv->agp_size = init->agp_size;
 897         dev_priv->agp_vm_start = RADEON_READ( RADEON_CONFIG_APER_SIZE );
 898 #if __REALLY_HAVE_AGP
 899         if ( !dev_priv->is_pci )
 900                 dev_priv->agp_buffers_offset = (dev_priv->buffers->offset
 901                                                 - dev->agp->base
 902                                                 + dev_priv->agp_vm_start);
 903         else
 904 #endif
 905                 dev_priv->agp_buffers_offset = (dev_priv->buffers->offset
 906                                                 - dev->sg->handle
 907                                                 + dev_priv->agp_vm_start);
 908
 909         DRM_DEBUG( "dev_priv->agp_size %d\n",
 910                    dev_priv->agp_size );
 911         DRM_DEBUG( "dev_priv->agp_vm_start 0x%x\n",
 912                    dev_priv->agp_vm_start );
 913         DRM_DEBUG( "dev_priv->agp_buffers_offset 0x%lx\n",
 914                    dev_priv->agp_buffers_offset );
 915
 916         dev_priv->ring.head = ((__volatile__ u32 *)
 917                                dev_priv->ring_rptr->handle);
 918
 919         dev_priv->ring.start = (u32 *)dev_priv->cp_ring->handle;
 920         dev_priv->ring.end = ((u32 *)dev_priv->cp_ring->handle
 921                               + init->ring_size / sizeof(u32));
 922         dev_priv->ring.size = init->ring_size;
 923         dev_priv->ring.size_l2qw = DRM(order)( init->ring_size / 8 );
 924
 925         dev_priv->ring.tail_mask =
 926                 (dev_priv->ring.size / sizeof(u32)) - 1;
 927
 928         dev_priv->ring.high_mark = RADEON_RING_HIGH_MARK;
 929
 930 #if 0
 931         /* Initialize the scratch register pointer.  This will cause
 932          * the scratch register values to be written out to memory
 933          * whenever they are updated.
 934          * FIXME: This doesn't quite work yet, so we're disabling it
 935          * for the release.
 936          */
 937         RADEON_WRITE( RADEON_SCRATCH_ADDR, (dev_priv->ring_rptr->offset +
 938                                             RADEON_SCRATCH_REG_OFFSET) );
 939         RADEON_WRITE( RADEON_SCRATCH_UMSK, 0x7 );
 940 #endif
 941
 942         dev_priv->scratch = ((__volatile__ u32 *)
 943                              dev_priv->ring_rptr->handle +
 944                              (RADEON_SCRATCH_REG_OFFSET / sizeof(u32)));
 945
 946         dev_priv->sarea_priv->last_frame = 0;
 947         RADEON_WRITE( RADEON_LAST_FRAME_REG,
 948                       dev_priv->sarea_priv->last_frame );
 949
 950         dev_priv->sarea_priv->last_dispatch = 0;
 951         RADEON_WRITE( RADEON_LAST_DISPATCH_REG,
 952                       dev_priv->sarea_priv->last_dispatch );
 953
 954         dev_priv->sarea_priv->last_clear = 0;
 955         RADEON_WRITE( RADEON_LAST_CLEAR_REG,
 956                       dev_priv->sarea_priv->last_clear );
 957
 958 #if __REALLY_HAVE_SG
 959         if ( dev_priv->is_pci ) {
 960                 if (!DRM(ati_pcigart_init)( dev, &dev_priv->phys_pci_gart,
 961                                             &dev_priv->bus_pci_gart)) {
 962                         DRM_ERROR( "failed to init PCI GART!\n" );
 963                         dev->dev_private = (void *)dev_priv;
 964                         radeon_do_cleanup_cp(dev);
 965                         return DRM_OS_ERR(ENOMEM);
 966                 }
 967                 /* Turn on PCI GART
 968                  */
 969                 tmp = RADEON_READ( RADEON_AIC_CNTL )
 970                       | RADEON_PCIGART_TRANSLATE_EN;
 971                 RADEON_WRITE( RADEON_AIC_CNTL, tmp );
 972
 973                 /* set PCI GART page-table base address
 974                  */
 975                 RADEON_WRITE( RADEON_AIC_PT_BASE, dev_priv->bus_pci_gart );
 976
 977                 /* set address range for PCI address translate
 978                  */
 979                 RADEON_WRITE( RADEON_AIC_LO_ADDR, dev_priv->agp_vm_start );
 980                 RADEON_WRITE( RADEON_AIC_HI_ADDR, dev_priv->agp_vm_start
 981                                                   + dev_priv->agp_size - 1);
 982
 983                 /* Turn off AGP aperture -- is this required for PCIGART?
 984                  */
 985                 RADEON_WRITE( RADEON_MC_AGP_LOCATION, 0xffffffc0 ); /* ?? */
 986                 RADEON_WRITE( RADEON_AGP_COMMAND, 0 ); /* clear AGP_COMMAND */
 987         } else {
 988 #endif
 989                 /* Turn off PCI GART
 990                  */
 991                 tmp = RADEON_READ( RADEON_AIC_CNTL )
 992                       & ~RADEON_PCIGART_TRANSLATE_EN;
 993                 RADEON_WRITE( RADEON_AIC_CNTL, tmp );
 994 #if __REALLY_HAVE_SG
 995         }
 996 #endif
 997
 998         radeon_cp_load_microcode( dev_priv );
 999         radeon_cp_init_ring_buffer( dev, dev_priv );
1000
1001 #if ROTATE_BUFS
1002         dev_priv->last_buf = 0;
1003 #endif
1004
1005         dev->dev_private = (void *)dev_priv;
1006
1007         radeon_do_engine_reset( dev );
1008
1009         return 0;
1010 }
1011
1012 int radeon_do_cleanup_cp( drm_device_t *dev )
1013 {
1014         DRM_DEBUG( "%s\n", __func__ );
1015
1016         if ( dev->dev_private ) {
1017                 drm_radeon_private_t *dev_priv = dev->dev_private;
1018
1019 #if __REALLY_HAVE_SG
1020                 if ( !dev_priv->is_pci ) {
1021 #endif
1022                         DRM_IOREMAPFREE( dev_priv->cp_ring );
1023                         DRM_IOREMAPFREE( dev_priv->ring_rptr );
1024                         DRM_IOREMAPFREE( dev_priv->buffers );
1025 #if __REALLY_HAVE_SG
1026                 } else {
1027                         if (!DRM(ati_pcigart_cleanup)( dev,
1028                                                 dev_priv->phys_pci_gart,
1029                                                 dev_priv->bus_pci_gart ))
1030                                 DRM_ERROR( "failed to cleanup PCI GART!\n" );
1031                 }
1032 #endif
1033
1034                 DRM(free)( dev->dev_private, sizeof(drm_radeon_private_t),
1035                            DRM_MEM_DRIVER );
1036                 dev->dev_private = NULL;
1037         }
1038
1039         return 0;
1040 }
1041
1042 int radeon_cp_init( DRM_OS_IOCTL )
1043 {
1044         DRM_OS_DEVICE;
1045         drm_radeon_init_t init;
1046
1047         DRM_OS_KRNFROMUSR( init, (drm_radeon_init_t *) data, sizeof(init) );
1048
1049         switch ( init.func ) {
1050         case RADEON_INIT_CP:
1051                 return radeon_do_init_cp( dev, &init );
1052         case RADEON_CLEANUP_CP:
1053                 return radeon_do_cleanup_cp( dev );
1054         }
1055
1056         return DRM_OS_ERR(EINVAL);
1057 }
1058
1059 int radeon_cp_start( DRM_OS_IOCTL )
1060 {
1061         DRM_OS_DEVICE;
1062         drm_radeon_private_t *dev_priv = dev->dev_private;
1063         DRM_DEBUG( "%s\n", __func__ );
1064
1065         LOCK_TEST_WITH_RETURN( dev );
1066
1067         if ( dev_priv->cp_running ) {
1068                 DRM_DEBUG( "%s while CP running\n", __func__ );
1069                 return 0;
1070         }
1071         if ( dev_priv->cp_mode == RADEON_CSQ_PRIDIS_INDDIS ) {
1072                 DRM_DEBUG( "%s called with bogus CP mode (%d)\n",
1073                            __func__, dev_priv->cp_mode );
1074                 return 0;
1075         }
1076
1077         radeon_do_cp_start( dev_priv );
1078
1079         return 0;
1080 }
1081
1082 /* Stop the CP.  The engine must have been idled before calling this
1083  * routine.
1084  */
1085 int radeon_cp_stop( DRM_OS_IOCTL )
1086 {
1087         DRM_OS_DEVICE;
1088         drm_radeon_private_t *dev_priv = dev->dev_private;
1089         drm_radeon_cp_stop_t stop;
1090         int ret;
1091         DRM_DEBUG( "%s\n", __func__ );
1092
1093         LOCK_TEST_WITH_RETURN( dev );
1094
1095         DRM_OS_KRNFROMUSR( stop, (drm_radeon_cp_stop_t *) data, sizeof(stop) );
1096
1097         /* Flush any pending CP commands.  This ensures any outstanding
1098          * commands are exectuted by the engine before we turn it off.
1099          */
1100         if ( stop.flush ) {
1101                 radeon_do_cp_flush( dev_priv );
1102         }
1103
1104         /* If we fail to make the engine go idle, we return an error
1105          * code so that the DRM ioctl wrapper can try again.
1106          */
1107         if ( stop.idle ) {
1108                 ret = radeon_do_cp_idle( dev_priv );
1109                 if ( ret ) return ret;
1110         }
1111
1112         /* Finally, we can turn off the CP.  If the engine isn't idle,
1113          * we will get some dropped triangles as they won't be fully
1114          * rendered before the CP is shut down.
1115          */
1116         radeon_do_cp_stop( dev_priv );
1117
1118         /* Reset the engine */
1119         radeon_do_engine_reset( dev );
1120
1121         return 0;
1122 }
1123
1124 /* Just reset the CP ring.  Called as part of an X Server engine reset.
1125  */
1126 int radeon_cp_reset( DRM_OS_IOCTL )
1127 {
1128         DRM_OS_DEVICE;
1129         drm_radeon_private_t *dev_priv = dev->dev_private;
1130         DRM_DEBUG( "%s\n", __func__ );
1131
1132         LOCK_TEST_WITH_RETURN( dev );
1133
1134         if ( !dev_priv ) {
1135                 DRM_DEBUG( "%s called before init done\n", __func__ );
1136                 return DRM_OS_ERR(EINVAL);
1137         }
1138
1139         radeon_do_cp_reset( dev_priv );
1140
1141         /* The CP is no longer running after an engine reset */
1142         dev_priv->cp_running = 0;
1143
1144         return 0;
1145 }
1146
1147 int radeon_cp_idle( DRM_OS_IOCTL )
1148 {
1149         DRM_OS_DEVICE;
1150         drm_radeon_private_t *dev_priv = dev->dev_private;
1151         DRM_DEBUG( "%s\n", __func__ );
1152
1153         LOCK_TEST_WITH_RETURN( dev );
1154
1155         return radeon_do_cp_idle( dev_priv );
1156 }
1157
1158 int radeon_engine_reset( DRM_OS_IOCTL )
1159 {
1160         DRM_OS_DEVICE;
1161         DRM_DEBUG( "%s\n", __func__ );
1162
1163         LOCK_TEST_WITH_RETURN( dev );
1164
1165         return radeon_do_engine_reset( dev );
1166 }
1167
1168
1169 /* ================================================================
1170  * Fullscreen mode
1171  */
1172
1173 static int radeon_do_init_pageflip( drm_device_t *dev )
1174 {
1175         drm_radeon_private_t *dev_priv = dev->dev_private;
1176         DRM_DEBUG( "%s\n", __func__ );
1177
1178         dev_priv->crtc_offset =      RADEON_READ( RADEON_CRTC_OFFSET );
1179         dev_priv->crtc_offset_cntl = RADEON_READ( RADEON_CRTC_OFFSET_CNTL );
1180
1181         RADEON_WRITE( RADEON_CRTC_OFFSET, dev_priv->front_offset );
1182         RADEON_WRITE( RADEON_CRTC_OFFSET_CNTL,
1183                       dev_priv->crtc_offset_cntl |
1184                       RADEON_CRTC_OFFSET_FLIP_CNTL );
1185
1186         dev_priv->page_flipping = 1;
1187         dev_priv->current_page = 0;
1188
1189         return 0;
1190 }
1191
1192 int radeon_do_cleanup_pageflip( drm_device_t *dev )
1193 {
1194         drm_radeon_private_t *dev_priv = dev->dev_private;
1195         DRM_DEBUG( "%s\n", __func__ );
1196
1197         RADEON_WRITE( RADEON_CRTC_OFFSET,      dev_priv->crtc_offset );
1198         RADEON_WRITE( RADEON_CRTC_OFFSET_CNTL, dev_priv->crtc_offset_cntl );
1199
1200         dev_priv->page_flipping = 0;
1201         dev_priv->current_page = 0;
1202
1203         return 0;
1204 }
1205
1206 int radeon_fullscreen( DRM_OS_IOCTL )
1207 {
1208         DRM_OS_DEVICE;
1209         drm_radeon_fullscreen_t fs;
1210
1211         LOCK_TEST_WITH_RETURN( dev );
1212
1213         DRM_OS_KRNFROMUSR( fs, (drm_radeon_fullscreen_t *) data,
1214                              sizeof(fs) );
1215
1216         switch ( fs.func ) {
1217         case RADEON_INIT_FULLSCREEN:
1218                 return radeon_do_init_pageflip( dev );
1219         case RADEON_CLEANUP_FULLSCREEN:
1220                 return radeon_do_cleanup_pageflip( dev );
1221         }
1222
1223         return DRM_OS_ERR(EINVAL);
1224 }
1225
1226
1227 /* ================================================================
1228  * Freelist management
1229  */
1230 #define RADEON_BUFFER_USED      0xffffffff
1231 #define RADEON_BUFFER_FREE      0
1232
1233 #if 0
1234 static int radeon_freelist_init( drm_device_t *dev )
1235 {
1236         drm_device_dma_t *dma = dev->dma;
1237         drm_radeon_private_t *dev_priv = dev->dev_private;
1238         drm_buf_t *buf;
1239         drm_radeon_buf_priv_t *buf_priv;
1240         drm_radeon_freelist_t *entry;
1241         int i;
1242
1243         dev_priv->head = DRM(alloc)( sizeof(drm_radeon_freelist_t),
1244                                      DRM_MEM_DRIVER );
1245         if ( dev_priv->head == NULL )
1246                 return DRM_OS_ERR(ENOMEM);
1247
1248         memset( dev_priv->head, 0, sizeof(drm_radeon_freelist_t) );
1249         dev_priv->head->age = RADEON_BUFFER_USED;
1250
1251         for ( i = 0 ; i < dma->buf_count ; i++ ) {
1252                 buf = dma->buflist[i];
1253                 buf_priv = buf->dev_private;
1254
1255                 entry = DRM(alloc)( sizeof(drm_radeon_freelist_t),
1256                                     DRM_MEM_DRIVER );
1257                 if ( !entry ) return DRM_OS_ERR(ENOMEM);
1258
1259                 entry->age = RADEON_BUFFER_FREE;
1260                 entry->buf = buf;
1261                 entry->prev = dev_priv->head;
1262                 entry->next = dev_priv->head->next;
1263                 if ( !entry->next )
1264                         dev_priv->tail = entry;
1265
1266                 buf_priv->discard = 0;
1267                 buf_priv->dispatched = 0;
1268                 buf_priv->list_entry = entry;
1269
1270                 dev_priv->head->next = entry;
1271
1272                 if ( dev_priv->head->next )
1273                         dev_priv->head->next->prev = entry;
1274         }
1275
1276         return 0;
1277
1278 }
1279 #endif
1280
1281 drm_buf_t *radeon_freelist_get( drm_device_t *dev )
1282 {
1283         drm_device_dma_t *dma = dev->dma;
1284         drm_radeon_private_t *dev_priv = dev->dev_private;
1285         drm_radeon_buf_priv_t *buf_priv;
1286         drm_buf_t *buf;
1287         int i, t;
1288 #if ROTATE_BUFS
1289         int start;
1290 #endif
1291
1292         /* FIXME: Optimize -- use freelist code */
1293
1294         for ( i = 0 ; i < dma->buf_count ; i++ ) {
1295                 buf = dma->buflist[i];
1296                 buf_priv = buf->dev_private;
1297                 if ( buf->pid == 0 ) {
1298                         DRM_DEBUG( "  ret buf=%d last=%d pid=0\n",
1299                                    buf->idx, dev_priv->last_buf );
1300                         return buf;
1301                 }
1302                 DRM_DEBUG( "    skipping buf=%d pid=%d\n",
1303                            buf->idx, buf->pid );
1304         }
1305
1306 #if ROTATE_BUFS
1307         if ( ++dev_priv->last_buf >= dma->buf_count )
1308                 dev_priv->last_buf = 0;
1309         start = dev_priv->last_buf;
1310 #endif
1311         for ( t = 0 ; t < dev_priv->usec_timeout ; t++ ) {
1312 #if 0
1313                 /* FIXME: Disable this for now */
1314                 u32 done_age = dev_priv->scratch[RADEON_LAST_DISPATCH];
1315 #else
1316                 u32 done_age = RADEON_READ( RADEON_LAST_DISPATCH_REG );
1317 #endif
1318 #if ROTATE_BUFS
1319                 for ( i = start ; i < dma->buf_count ; i++ ) {
1320 #else
1321                 for ( i = 0 ; i < dma->buf_count ; i++ ) {
1322 #endif
1323                         buf = dma->buflist[i];
1324                         buf_priv = buf->dev_private;
1325                         if ( buf->pending && buf_priv->age <= done_age ) {
1326                                 /* The buffer has been processed, so it
1327                                  * can now be used.
1328                                  */
1329                                 buf->pending = 0;
1330                                 DRM_DEBUG( "  ret buf=%d last=%d age=%d done=%d\n", buf->idx, dev_priv->last_buf, buf_priv->age, done_age );
1331                                 return buf;
1332                         }
1333                         DRM_DEBUG( "    skipping buf=%d age=%d done=%d\n",
1334                                    buf->idx, buf_priv->age,
1335                                    done_age );
1336 #if ROTATE_BUFS
1337                         start = 0;
1338 #endif
1339                 }
1340                 DRM_OS_DELAY( 1 );
1341         }
1342
1343         DRM_ERROR( "returning NULL!\n" );
1344         return NULL;
1345 }
1346
1347 void radeon_freelist_reset( drm_device_t *dev )
1348 {
1349         drm_device_dma_t *dma = dev->dma;
1350 #if ROTATE_BUFS
1351         drm_radeon_private_t *dev_priv = dev->dev_private;
1352 #endif
1353         int i;
1354
1355 #if ROTATE_BUFS
1356         dev_priv->last_buf = 0;
1357 #endif
1358         for ( i = 0 ; i < dma->buf_count ; i++ ) {
1359                 drm_buf_t *buf = dma->buflist[i];
1360                 drm_radeon_buf_priv_t *buf_priv = buf->dev_private;
1361                 buf_priv->age = 0;
1362         }
1363 }
1364
1365
1366 /* ================================================================
1367  * CP command submission
1368  */
1369
1370 int radeon_wait_ring( drm_radeon_private_t *dev_priv, int n )
1371 {
1372         drm_radeon_ring_buffer_t *ring = &dev_priv->ring;
1373         int i;
1374
1375         for ( i = 0 ; i < dev_priv->usec_timeout ; i++ ) {
1376                 radeon_update_ring_snapshot( ring );
1377                 if ( ring->space > n )
1378                         return 0;
1379                 DRM_OS_DELAY( 1 );
1380         }
1381
1382         /* FIXME: This return value is ignored in the BEGIN_RING macro! */
1383 #if RADEON_FIFO_DEBUG
1384         radeon_status( dev_priv );
1385         DRM_ERROR( "failed!\n" );
1386 #endif
1387         return DRM_OS_ERR(EBUSY);
1388 }
1389
1390 static int radeon_cp_get_buffers( drm_device_t *dev, drm_dma_t *d )
1391 {
1392         int i;
1393         drm_buf_t *buf;
1394
1395         for ( i = d->granted_count ; i < d->request_count ; i++ ) {
1396                 buf = radeon_freelist_get( dev );
1397                 if ( !buf ) return DRM_OS_ERR(EAGAIN);
1398
1399                 buf->pid = DRM_OS_CURRENTPID;
1400
1401                 if (DRM_OS_COPYTOUSR( &d->request_indices[i], &buf->idx,
1402                                    sizeof(buf->idx) ) )
1403                         return DRM_OS_ERR(EFAULT);
1404                 if (DRM_OS_COPYTOUSR( &d->request_sizes[i], &buf->total,
1405                                    sizeof(buf->total) ) )
1406                         return DRM_OS_ERR(EFAULT);
1407
1408                 d->granted_count++;
1409         }
1410         return 0;
1411 }
1412
1413 int radeon_cp_buffers( DRM_OS_IOCTL )
1414 {
1415         DRM_OS_DEVICE;
1416         drm_device_dma_t *dma = dev->dma;
1417         int ret = 0;
1418         drm_dma_t d;
1419
1420         LOCK_TEST_WITH_RETURN( dev );
1421
1422         DRM_OS_KRNFROMUSR( d, (drm_dma_t *) data, sizeof(d) );
1423
1424         /* Please don't send us buffers.
1425          */
1426         if ( d.send_count != 0 ) {
1427                 DRM_ERROR( "Process %d trying to send %d buffers via drmDMA\n",
1428                            DRM_OS_CURRENTPID, d.send_count );
1429                 return DRM_OS_ERR(EINVAL);
1430         }
1431
1432         /* We'll send you buffers.
1433          */
1434         if ( d.request_count < 0 || d.request_count > dma->buf_count ) {
1435                 DRM_ERROR( "Process %d trying to get %d buffers (of %d max)\n",
1436                            DRM_OS_CURRENTPID, d.request_count, dma->buf_count );
1437                 return DRM_OS_ERR(EINVAL);
1438         }
1439
1440         d.granted_count = 0;
1441
1442         if ( d.request_count ) {
1443                 ret = radeon_cp_get_buffers( dev, &d );
1444         }
1445
1446         DRM_OS_KRNTOUSR( (drm_dma_t *) data, d, sizeof(d) );
1447
1448         return ret;
1449 }