sys/dev/drm/radeon_state.c

   1 /* radeon_state.c -- State support for Radeon -*- linux-c -*-
   2  *
   3  * Copyright 2000 VA Linux Systems, Inc., Fremont, California.
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice (including the next
  14  * paragraph) shall be included in all copies or substantial portions of the
  15  * Software.
  16  *
  17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  18  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  19  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  20  * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  21  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  22  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  23  * DEALINGS IN THE SOFTWARE.
  24  *
  25  * Authors:
  26  *    Gareth Hughes <gareth@valinux.com>
  27  *    Kevin E. Martin <martin@valinux.com>
  28  *
  29  * $FreeBSD$
  30  */
  31
  32 #ifdef __linux__
  33 #define __NO_VERSION__
  34 #include <linux/delay.h>
  35 #endif /* __linux__ */
  36 #include "dev/drm/radeon.h"
  37 #include "dev/drm/drmP.h"
  38 #include "dev/drm/radeon_drm.h"
  39 #include "dev/drm/radeon_drv.h"
  40 #include "dev/drm/drm.h"
  41
  42
  43 /* ================================================================
  44  * CP hardware state programming functions
  45  */
  46
  47 static __inline__ void radeon_emit_clip_rect( drm_radeon_private_t *dev_priv,
  48                                           drm_clip_rect_t *box )
  49 {
  50         RING_LOCALS;
  51
  52         DRM_DEBUG( "   box:  x1=%d y1=%d  x2=%d y2=%d\n",
  53                    box->x1, box->y1, box->x2, box->y2 );
  54
  55         BEGIN_RING( 4 );
  56
  57         OUT_RING( CP_PACKET0( RADEON_RE_TOP_LEFT, 0 ) );
  58         OUT_RING( (box->y1 << 16) | box->x1 );
  59
  60         OUT_RING( CP_PACKET0( RADEON_RE_WIDTH_HEIGHT, 0 ) );
  61         OUT_RING( ((box->y2 - 1) << 16) | (box->x2 - 1) );
  62
  63         ADVANCE_RING();
  64 }
  65
  66 static __inline__ void radeon_emit_context( drm_radeon_private_t *dev_priv )
  67 {
  68         drm_radeon_sarea_t *sarea_priv = dev_priv->sarea_priv;
  69         drm_radeon_context_regs_t *ctx = &sarea_priv->context_state;
  70         RING_LOCALS;
  71         DRM_DEBUG( "    %s\n", __func__ );
  72
  73         BEGIN_RING( 14 );
  74
  75         OUT_RING( CP_PACKET0( RADEON_PP_MISC, 6 ) );
  76         OUT_RING( ctx->pp_misc );
  77         OUT_RING( ctx->pp_fog_color );
  78         OUT_RING( ctx->re_solid_color );
  79         OUT_RING( ctx->rb3d_blendcntl );
  80         OUT_RING( ctx->rb3d_depthoffset );
  81         OUT_RING( ctx->rb3d_depthpitch );
  82         OUT_RING( ctx->rb3d_zstencilcntl );
  83
  84         OUT_RING( CP_PACKET0( RADEON_PP_CNTL, 2 ) );
  85         OUT_RING( ctx->pp_cntl );
  86         OUT_RING( ctx->rb3d_cntl );
  87         OUT_RING( ctx->rb3d_coloroffset );
  88
  89         OUT_RING( CP_PACKET0( RADEON_RB3D_COLORPITCH, 0 ) );
  90         OUT_RING( ctx->rb3d_colorpitch );
  91
  92         ADVANCE_RING();
  93 }
  94
  95 static __inline__ void radeon_emit_vertfmt( drm_radeon_private_t *dev_priv )
  96 {
  97         drm_radeon_sarea_t *sarea_priv = dev_priv->sarea_priv;
  98         drm_radeon_context_regs_t *ctx = &sarea_priv->context_state;
  99         RING_LOCALS;
 100         DRM_DEBUG( "    %s\n", __func__ );
 101
 102         BEGIN_RING( 2 );
 103
 104         OUT_RING( CP_PACKET0( RADEON_SE_COORD_FMT, 0 ) );
 105         OUT_RING( ctx->se_coord_fmt );
 106
 107         ADVANCE_RING();
 108 }
 109
 110 static __inline__ void radeon_emit_line( drm_radeon_private_t *dev_priv )
 111 {
 112         drm_radeon_sarea_t *sarea_priv = dev_priv->sarea_priv;
 113         drm_radeon_context_regs_t *ctx = &sarea_priv->context_state;
 114         RING_LOCALS;
 115         DRM_DEBUG( "    %s\n", __func__ );
 116
 117         BEGIN_RING( 5 );
 118
 119         OUT_RING( CP_PACKET0( RADEON_RE_LINE_PATTERN, 1 ) );
 120         OUT_RING( ctx->re_line_pattern );
 121         OUT_RING( ctx->re_line_state );
 122
 123         OUT_RING( CP_PACKET0( RADEON_SE_LINE_WIDTH, 0 ) );
 124         OUT_RING( ctx->se_line_width );
 125
 126         ADVANCE_RING();
 127 }
 128
 129 static __inline__ void radeon_emit_bumpmap( drm_radeon_private_t *dev_priv )
 130 {
 131         drm_radeon_sarea_t *sarea_priv = dev_priv->sarea_priv;
 132         drm_radeon_context_regs_t *ctx = &sarea_priv->context_state;
 133         RING_LOCALS;
 134         DRM_DEBUG( "    %s\n", __func__ );
 135
 136         BEGIN_RING( 5 );
 137
 138         OUT_RING( CP_PACKET0( RADEON_PP_LUM_MATRIX, 0 ) );
 139         OUT_RING( ctx->pp_lum_matrix );
 140
 141         OUT_RING( CP_PACKET0( RADEON_PP_ROT_MATRIX_0, 1 ) );
 142         OUT_RING( ctx->pp_rot_matrix_0 );
 143         OUT_RING( ctx->pp_rot_matrix_1 );
 144
 145         ADVANCE_RING();
 146 }
 147
 148 static __inline__ void radeon_emit_masks( drm_radeon_private_t *dev_priv )
 149 {
 150         drm_radeon_sarea_t *sarea_priv = dev_priv->sarea_priv;
 151         drm_radeon_context_regs_t *ctx = &sarea_priv->context_state;
 152         RING_LOCALS;
 153         DRM_DEBUG( "    %s\n", __func__ );
 154
 155         BEGIN_RING( 4 );
 156
 157         OUT_RING( CP_PACKET0( RADEON_RB3D_STENCILREFMASK, 2 ) );
 158         OUT_RING( ctx->rb3d_stencilrefmask );
 159         OUT_RING( ctx->rb3d_ropcntl );
 160         OUT_RING( ctx->rb3d_planemask );
 161
 162         ADVANCE_RING();
 163 }
 164
 165 static __inline__ void radeon_emit_viewport( drm_radeon_private_t *dev_priv )
 166 {
 167         drm_radeon_sarea_t *sarea_priv = dev_priv->sarea_priv;
 168         drm_radeon_context_regs_t *ctx = &sarea_priv->context_state;
 169         RING_LOCALS;
 170         DRM_DEBUG( "    %s\n", __func__ );
 171
 172         BEGIN_RING( 7 );
 173
 174         OUT_RING( CP_PACKET0( RADEON_SE_VPORT_XSCALE, 5 ) );
 175         OUT_RING( ctx->se_vport_xscale );
 176         OUT_RING( ctx->se_vport_xoffset );
 177         OUT_RING( ctx->se_vport_yscale );
 178         OUT_RING( ctx->se_vport_yoffset );
 179         OUT_RING( ctx->se_vport_zscale );
 180         OUT_RING( ctx->se_vport_zoffset );
 181
 182         ADVANCE_RING();
 183 }
 184
 185 static __inline__ void radeon_emit_setup( drm_radeon_private_t *dev_priv )
 186 {
 187         drm_radeon_sarea_t *sarea_priv = dev_priv->sarea_priv;
 188         drm_radeon_context_regs_t *ctx = &sarea_priv->context_state;
 189         RING_LOCALS;
 190         DRM_DEBUG( "    %s\n", __func__ );
 191
 192         BEGIN_RING( 4 );
 193
 194         OUT_RING( CP_PACKET0( RADEON_SE_CNTL, 0 ) );
 195         OUT_RING( ctx->se_cntl );
 196         OUT_RING( CP_PACKET0( RADEON_SE_CNTL_STATUS, 0 ) );
 197         OUT_RING( ctx->se_cntl_status );
 198
 199         ADVANCE_RING();
 200 }
 201
 202 static __inline__ void radeon_emit_tcl( drm_radeon_private_t *dev_priv )
 203 {
 204 #ifdef TCL_ENABLE
 205         drm_radeon_sarea_t *sarea_priv = dev_priv->sarea_priv;
 206         drm_radeon_context_regs_t *ctx = &sarea_priv->context_state;
 207         RING_LOCALS;
 208         DRM_DEBUG( "    %s\n", __func__ );
 209
 210         BEGIN_RING( 29 );
 211
 212         OUT_RING( CP_PACKET0( RADEON_SE_TCL_MATERIAL_EMMISSIVE_RED, 27 ) );
 213         OUT_RING( ctx->se_tcl_material_emmissive.red );
 214         OUT_RING( ctx->se_tcl_material_emmissive.green );
 215         OUT_RING( ctx->se_tcl_material_emmissive.blue );
 216         OUT_RING( ctx->se_tcl_material_emmissive.alpha );
 217         OUT_RING( ctx->se_tcl_material_ambient.red );
 218         OUT_RING( ctx->se_tcl_material_ambient.green );
 219         OUT_RING( ctx->se_tcl_material_ambient.blue );
 220         OUT_RING( ctx->se_tcl_material_ambient.alpha );
 221         OUT_RING( ctx->se_tcl_material_diffuse.red );
 222         OUT_RING( ctx->se_tcl_material_diffuse.green );
 223         OUT_RING( ctx->se_tcl_material_diffuse.blue );
 224         OUT_RING( ctx->se_tcl_material_diffuse.alpha );
 225         OUT_RING( ctx->se_tcl_material_specular.red );
 226         OUT_RING( ctx->se_tcl_material_specular.green );
 227         OUT_RING( ctx->se_tcl_material_specular.blue );
 228         OUT_RING( ctx->se_tcl_material_specular.alpha );
 229         OUT_RING( ctx->se_tcl_shininess );
 230         OUT_RING( ctx->se_tcl_output_vtx_fmt );
 231         OUT_RING( ctx->se_tcl_output_vtx_sel );
 232         OUT_RING( ctx->se_tcl_matrix_select_0 );
 233         OUT_RING( ctx->se_tcl_matrix_select_1 );
 234         OUT_RING( ctx->se_tcl_ucp_vert_blend_ctl );
 235         OUT_RING( ctx->se_tcl_texture_proc_ctl );
 236         OUT_RING( ctx->se_tcl_light_model_ctl );
 237         for ( i = 0 ; i < 4 ; i++ ) {
 238                 OUT_RING( ctx->se_tcl_per_light_ctl[i] );
 239         }
 240
 241         ADVANCE_RING();
 242 #else
 243         DRM_ERROR( "TCL not enabled!\n" );
 244 #endif
 245 }
 246
 247 static __inline__ void radeon_emit_misc( drm_radeon_private_t *dev_priv )
 248 {
 249         drm_radeon_sarea_t *sarea_priv = dev_priv->sarea_priv;
 250         drm_radeon_context_regs_t *ctx = &sarea_priv->context_state;
 251         RING_LOCALS;
 252         DRM_DEBUG( "    %s\n", __func__ );
 253
 254         BEGIN_RING( 2 );
 255
 256         OUT_RING( CP_PACKET0( RADEON_RE_MISC, 0 ) );
 257         OUT_RING( ctx->re_misc );
 258
 259         ADVANCE_RING();
 260 }
 261
 262 static __inline__ void radeon_emit_tex0( drm_radeon_private_t *dev_priv )
 263 {
 264         drm_radeon_sarea_t *sarea_priv = dev_priv->sarea_priv;
 265         drm_radeon_texture_regs_t *tex = &sarea_priv->tex_state[0];
 266         RING_LOCALS;
 267         DRM_DEBUG( "    %s: offset=0x%x\n", __func__, tex->pp_txoffset );
 268
 269         BEGIN_RING( 9 );
 270
 271         OUT_RING( CP_PACKET0( RADEON_PP_TXFILTER_0, 5 ) );
 272         OUT_RING( tex->pp_txfilter );
 273         OUT_RING( tex->pp_txformat );
 274         OUT_RING( tex->pp_txoffset );
 275         OUT_RING( tex->pp_txcblend );
 276         OUT_RING( tex->pp_txablend );
 277         OUT_RING( tex->pp_tfactor );
 278
 279         OUT_RING( CP_PACKET0( RADEON_PP_BORDER_COLOR_0, 0 ) );
 280         OUT_RING( tex->pp_border_color );
 281
 282         ADVANCE_RING();
 283 }
 284
 285 static __inline__ void radeon_emit_tex1( drm_radeon_private_t *dev_priv )
 286 {
 287         drm_radeon_sarea_t *sarea_priv = dev_priv->sarea_priv;
 288         drm_radeon_texture_regs_t *tex = &sarea_priv->tex_state[1];
 289         RING_LOCALS;
 290         DRM_DEBUG( "    %s: offset=0x%x\n", __func__, tex->pp_txoffset );
 291
 292         BEGIN_RING( 9 );
 293
 294         OUT_RING( CP_PACKET0( RADEON_PP_TXFILTER_1, 5 ) );
 295         OUT_RING( tex->pp_txfilter );
 296         OUT_RING( tex->pp_txformat );
 297         OUT_RING( tex->pp_txoffset );
 298         OUT_RING( tex->pp_txcblend );
 299         OUT_RING( tex->pp_txablend );
 300         OUT_RING( tex->pp_tfactor );
 301
 302         OUT_RING( CP_PACKET0( RADEON_PP_BORDER_COLOR_1, 0 ) );
 303         OUT_RING( tex->pp_border_color );
 304
 305         ADVANCE_RING();
 306 }
 307
 308 static __inline__ void radeon_emit_tex2( drm_radeon_private_t *dev_priv )
 309 {
 310         drm_radeon_sarea_t *sarea_priv = dev_priv->sarea_priv;
 311         drm_radeon_texture_regs_t *tex = &sarea_priv->tex_state[2];
 312         RING_LOCALS;
 313         DRM_DEBUG( "    %s\n", __func__ );
 314
 315         BEGIN_RING( 9 );
 316
 317         OUT_RING( CP_PACKET0( RADEON_PP_TXFILTER_2, 5 ) );
 318         OUT_RING( tex->pp_txfilter );
 319         OUT_RING( tex->pp_txformat );
 320         OUT_RING( tex->pp_txoffset );
 321         OUT_RING( tex->pp_txcblend );
 322         OUT_RING( tex->pp_txablend );
 323         OUT_RING( tex->pp_tfactor );
 324
 325         OUT_RING( CP_PACKET0( RADEON_PP_BORDER_COLOR_2, 0 ) );
 326         OUT_RING( tex->pp_border_color );
 327
 328         ADVANCE_RING();
 329 }
 330
 331 static __inline__ void radeon_emit_state( drm_radeon_private_t *dev_priv )
 332 {
 333         drm_radeon_sarea_t *sarea_priv = dev_priv->sarea_priv;
 334         unsigned int dirty = sarea_priv->dirty;
 335
 336         DRM_DEBUG( "%s: dirty=0x%08x\n", __func__, dirty );
 337
 338         if ( dirty & RADEON_UPLOAD_CONTEXT ) {
 339                 radeon_emit_context( dev_priv );
 340                 sarea_priv->dirty &= ~RADEON_UPLOAD_CONTEXT;
 341         }
 342
 343         if ( dirty & RADEON_UPLOAD_VERTFMT ) {
 344                 radeon_emit_vertfmt( dev_priv );
 345                 sarea_priv->dirty &= ~RADEON_UPLOAD_VERTFMT;
 346         }
 347
 348         if ( dirty & RADEON_UPLOAD_LINE ) {
 349                 radeon_emit_line( dev_priv );
 350                 sarea_priv->dirty &= ~RADEON_UPLOAD_LINE;
 351         }
 352
 353         if ( dirty & RADEON_UPLOAD_BUMPMAP ) {
 354                 radeon_emit_bumpmap( dev_priv );
 355                 sarea_priv->dirty &= ~RADEON_UPLOAD_BUMPMAP;
 356         }
 357
 358         if ( dirty & RADEON_UPLOAD_MASKS ) {
 359                 radeon_emit_masks( dev_priv );
 360                 sarea_priv->dirty &= ~RADEON_UPLOAD_MASKS;
 361         }
 362
 363         if ( dirty & RADEON_UPLOAD_VIEWPORT ) {
 364                 radeon_emit_viewport( dev_priv );
 365                 sarea_priv->dirty &= ~RADEON_UPLOAD_VIEWPORT;
 366         }
 367
 368         if ( dirty & RADEON_UPLOAD_SETUP ) {
 369                 radeon_emit_setup( dev_priv );
 370                 sarea_priv->dirty &= ~RADEON_UPLOAD_SETUP;
 371         }
 372
 373         if ( dirty & RADEON_UPLOAD_TCL ) {
 374 #ifdef TCL_ENABLE
 375                 radeon_emit_tcl( dev_priv );
 376 #endif
 377                 sarea_priv->dirty &= ~RADEON_UPLOAD_TCL;
 378         }
 379
 380         if ( dirty & RADEON_UPLOAD_MISC ) {
 381                 radeon_emit_misc( dev_priv );
 382                 sarea_priv->dirty &= ~RADEON_UPLOAD_MISC;
 383         }
 384
 385         if ( dirty & RADEON_UPLOAD_TEX0 ) {
 386                 radeon_emit_tex0( dev_priv );
 387                 sarea_priv->dirty &= ~RADEON_UPLOAD_TEX0;
 388         }
 389
 390         if ( dirty & RADEON_UPLOAD_TEX1 ) {
 391                 radeon_emit_tex1( dev_priv );
 392                 sarea_priv->dirty &= ~RADEON_UPLOAD_TEX1;
 393         }
 394
 395         if ( dirty & RADEON_UPLOAD_TEX2 ) {
 396 #if 0
 397                 radeon_emit_tex2( dev_priv );
 398 #endif
 399                 sarea_priv->dirty &= ~RADEON_UPLOAD_TEX2;
 400         }
 401
 402         sarea_priv->dirty &= ~(RADEON_UPLOAD_TEX0IMAGES |
 403                                RADEON_UPLOAD_TEX1IMAGES |
 404                                RADEON_UPLOAD_TEX2IMAGES |
 405                                RADEON_REQUIRE_QUIESCENCE);
 406 }
 407
 408
 409 #if RADEON_PERFORMANCE_BOXES
 410 /* ================================================================
 411  * Performance monitoring functions
 412  */
 413
 414 static void radeon_clear_box( drm_radeon_private_t *dev_priv,
 415                               int x, int y, int w, int h,
 416                               int r, int g, int b )
 417 {
 418         u32 pitch, offset;
 419         u32 color;
 420         RING_LOCALS;
 421
 422         switch ( dev_priv->color_fmt ) {
 423         case RADEON_COLOR_FORMAT_RGB565:
 424                 color = (((r & 0xf8) << 8) |
 425                          ((g & 0xfc) << 3) |
 426                          ((b & 0xf8) >> 3));
 427                 break;
 428         case RADEON_COLOR_FORMAT_ARGB8888:
 429         default:
 430                 color = (((0xff) << 24) | (r << 16) | (g <<  8) | b);
 431                 break;
 432         }
 433
 434         offset = dev_priv->back_offset;
 435         pitch = dev_priv->back_pitch >> 3;
 436
 437         BEGIN_RING( 6 );
 438
 439         OUT_RING( CP_PACKET3( RADEON_CNTL_PAINT_MULTI, 4 ) );
 440         OUT_RING( RADEON_GMC_DST_PITCH_OFFSET_CNTL |
 441                   RADEON_GMC_BRUSH_SOLID_COLOR |
 442                   (dev_priv->color_fmt << 8) |
 443                   RADEON_GMC_SRC_DATATYPE_COLOR |
 444                   RADEON_ROP3_P |
 445                   RADEON_GMC_CLR_CMP_CNTL_DIS );
 446
 447         OUT_RING( (pitch << 22) | (offset >> 5) );
 448         OUT_RING( color );
 449
 450         OUT_RING( (x << 16) | y );
 451         OUT_RING( (w << 16) | h );
 452
 453         ADVANCE_RING();
 454 }
 455
 456 static void radeon_cp_performance_boxes( drm_radeon_private_t *dev_priv )
 457 {
 458         if ( atomic_read( &dev_priv->idle_count ) == 0 ) {
 459                 radeon_clear_box( dev_priv, 64, 4, 8, 8, 0, 255, 0 );
 460         } else {
 461                 atomic_set( &dev_priv->idle_count, 0 );
 462         }
 463 }
 464
 465 #endif
 466
 467
 468 /* ================================================================
 469  * CP command dispatch functions
 470  */
 471
 472 static void radeon_print_dirty( const char *msg, unsigned int flags )
 473 {
 474         DRM_DEBUG( "%s: (0x%x) %s%s%s%s%s%s%s%s%s%s%s%s%s%s\n",
 475                    msg,
 476                    flags,
 477                    (flags & RADEON_UPLOAD_CONTEXT)     ? "context, " : "",
 478                    (flags & RADEON_UPLOAD_VERTFMT)     ? "vertfmt, " : "",
 479                    (flags & RADEON_UPLOAD_LINE)        ? "line, " : "",
 480                    (flags & RADEON_UPLOAD_BUMPMAP)     ? "bumpmap, " : "",
 481                    (flags & RADEON_UPLOAD_MASKS)       ? "masks, " : "",
 482                    (flags & RADEON_UPLOAD_VIEWPORT)    ? "viewport, " : "",
 483                    (flags & RADEON_UPLOAD_SETUP)       ? "setup, " : "",
 484                    (flags & RADEON_UPLOAD_TCL)         ? "tcl, " : "",
 485                    (flags & RADEON_UPLOAD_MISC)        ? "misc, " : "",
 486                    (flags & RADEON_UPLOAD_TEX0)        ? "tex0, " : "",
 487                    (flags & RADEON_UPLOAD_TEX1)        ? "tex1, " : "",
 488                    (flags & RADEON_UPLOAD_TEX2)        ? "tex2, " : "",
 489                    (flags & RADEON_UPLOAD_CLIPRECTS)   ? "cliprects, " : "",
 490                    (flags & RADEON_REQUIRE_QUIESCENCE) ? "quiescence, " : "" );
 491 }
 492
 493 static void radeon_cp_dispatch_clear( drm_device_t *dev,
 494                                       drm_radeon_clear_t *clear,
 495                                       drm_radeon_clear_rect_t *depth_boxes )
 496 {
 497         drm_radeon_private_t *dev_priv = dev->dev_private;
 498         drm_radeon_sarea_t *sarea_priv = dev_priv->sarea_priv;
 499         int nbox = sarea_priv->nbox;
 500         drm_clip_rect_t *pbox = sarea_priv->boxes;
 501         unsigned int flags = clear->flags;
 502         int i;
 503         RING_LOCALS;
 504         DRM_DEBUG( "%s\n", __func__ );
 505
 506         if ( dev_priv->page_flipping && dev_priv->current_page == 1 ) {
 507                 unsigned int tmp = flags;
 508
 509                 flags &= ~(RADEON_FRONT | RADEON_BACK);
 510                 if ( tmp & RADEON_FRONT ) flags |= RADEON_BACK;
 511                 if ( tmp & RADEON_BACK )  flags |= RADEON_FRONT;
 512         }
 513
 514         for ( i = 0 ; i < nbox ; i++ ) {
 515                 int x = pbox[i].x1;
 516                 int y = pbox[i].y1;
 517                 int w = pbox[i].x2 - x;
 518                 int h = pbox[i].y2 - y;
 519
 520                 DRM_DEBUG( "dispatch clear %d,%d-%d,%d flags 0x%x\n",
 521                            x, y, w, h, flags );
 522
 523                 if ( flags & (RADEON_FRONT | RADEON_BACK) ) {
 524                         BEGIN_RING( 4 );
 525
 526                         /* Ensure the 3D stream is idle before doing a
 527                          * 2D fill to clear the front or back buffer.
 528                          */
 529                         RADEON_WAIT_UNTIL_3D_IDLE();
 530
 531                         OUT_RING( CP_PACKET0( RADEON_DP_WRITE_MASK, 0 ) );
 532                         OUT_RING( clear->color_mask );
 533
 534                         ADVANCE_RING();
 535
 536                         /* Make sure we restore the 3D state next time.
 537                          */
 538                         dev_priv->sarea_priv->dirty |= (RADEON_UPLOAD_CONTEXT |
 539                                                         RADEON_UPLOAD_MASKS);
 540                 }
 541
 542                 if ( flags & RADEON_FRONT ) {
 543                         BEGIN_RING( 6 );
 544
 545                         OUT_RING( CP_PACKET3( RADEON_CNTL_PAINT_MULTI, 4 ) );
 546                         OUT_RING( RADEON_GMC_DST_PITCH_OFFSET_CNTL |
 547                                   RADEON_GMC_BRUSH_SOLID_COLOR |
 548                                   (dev_priv->color_fmt << 8) |
 549                                   RADEON_GMC_SRC_DATATYPE_COLOR |
 550                                   RADEON_ROP3_P |
 551                                   RADEON_GMC_CLR_CMP_CNTL_DIS );
 552
 553                         OUT_RING( dev_priv->front_pitch_offset );
 554                         OUT_RING( clear->clear_color );
 555
 556                         OUT_RING( (x << 16) | y );
 557                         OUT_RING( (w << 16) | h );
 558
 559                         ADVANCE_RING();
 560                 }
 561
 562                 if ( flags & RADEON_BACK ) {
 563                         BEGIN_RING( 6 );
 564
 565                         OUT_RING( CP_PACKET3( RADEON_CNTL_PAINT_MULTI, 4 ) );
 566                         OUT_RING( RADEON_GMC_DST_PITCH_OFFSET_CNTL |
 567                                   RADEON_GMC_BRUSH_SOLID_COLOR |
 568                                   (dev_priv->color_fmt << 8) |
 569                                   RADEON_GMC_SRC_DATATYPE_COLOR |
 570                                   RADEON_ROP3_P |
 571                                   RADEON_GMC_CLR_CMP_CNTL_DIS );
 572
 573                         OUT_RING( dev_priv->back_pitch_offset );
 574                         OUT_RING( clear->clear_color );
 575
 576                         OUT_RING( (x << 16) | y );
 577                         OUT_RING( (w << 16) | h );
 578
 579                         ADVANCE_RING();
 580
 581                 }
 582
 583                 if ( flags & RADEON_DEPTH ) {
 584                         drm_radeon_depth_clear_t *depth_clear =
 585                            &dev_priv->depth_clear;
 586
 587                         if ( sarea_priv->dirty & ~RADEON_UPLOAD_CLIPRECTS ) {
 588                                 radeon_emit_state( dev_priv );
 589                         }
 590
 591                         /* FIXME: Render a rectangle to clear the depth
 592                          * buffer.  So much for those "fast Z clears"...
 593                          */
 594                         BEGIN_RING( 23 );
 595
 596                         RADEON_WAIT_UNTIL_2D_IDLE();
 597
 598                         OUT_RING( CP_PACKET0( RADEON_PP_CNTL, 1 ) );
 599                         OUT_RING( 0x00000000 );
 600                         OUT_RING( depth_clear->rb3d_cntl );
 601                         OUT_RING( CP_PACKET0( RADEON_RB3D_ZSTENCILCNTL, 0 ) );
 602                         OUT_RING( depth_clear->rb3d_zstencilcntl );
 603                         OUT_RING( CP_PACKET0( RADEON_RB3D_PLANEMASK, 0 ) );
 604                         OUT_RING( 0x00000000 );
 605                         OUT_RING( CP_PACKET0( RADEON_SE_CNTL, 0 ) );
 606                         OUT_RING( depth_clear->se_cntl );
 607
 608                         OUT_RING( CP_PACKET3( RADEON_3D_DRAW_IMMD, 10 ) );
 609                         OUT_RING( RADEON_VTX_Z_PRESENT );
 610                         OUT_RING( (RADEON_PRIM_TYPE_RECT_LIST |
 611                                    RADEON_PRIM_WALK_RING |
 612                                    RADEON_MAOS_ENABLE |
 613                                    RADEON_VTX_FMT_RADEON_MODE |
 614                                    (3 << RADEON_NUM_VERTICES_SHIFT)) );
 615
 616                         OUT_RING( depth_boxes[i].ui[CLEAR_X1] );
 617                         OUT_RING( depth_boxes[i].ui[CLEAR_Y1] );
 618                         OUT_RING( depth_boxes[i].ui[CLEAR_DEPTH] );
 619
 620                         OUT_RING( depth_boxes[i].ui[CLEAR_X1] );
 621                         OUT_RING( depth_boxes[i].ui[CLEAR_Y2] );
 622                         OUT_RING( depth_boxes[i].ui[CLEAR_DEPTH] );
 623
 624                         OUT_RING( depth_boxes[i].ui[CLEAR_X2] );
 625                         OUT_RING( depth_boxes[i].ui[CLEAR_Y2] );
 626                         OUT_RING( depth_boxes[i].ui[CLEAR_DEPTH] );
 627
 628                         ADVANCE_RING();
 629
 630                         /* Make sure we restore the 3D state next time.
 631                          */
 632                         dev_priv->sarea_priv->dirty |= (RADEON_UPLOAD_CONTEXT |
 633                                                         RADEON_UPLOAD_SETUP |
 634                                                         RADEON_UPLOAD_MASKS);
 635                 }
 636         }
 637
 638         /* Increment the clear counter.  The client-side 3D driver must
 639          * wait on this value before performing the clear ioctl.  We
 640          * need this because the card's so damned fast...
 641          */
 642         dev_priv->sarea_priv->last_clear++;
 643
 644         BEGIN_RING( 4 );
 645
 646         RADEON_CLEAR_AGE( dev_priv->sarea_priv->last_clear );
 647         RADEON_WAIT_UNTIL_IDLE();
 648
 649         ADVANCE_RING();
 650 }
 651
 652 static void radeon_cp_dispatch_swap( drm_device_t *dev )
 653 {
 654         drm_radeon_private_t *dev_priv = dev->dev_private;
 655         drm_radeon_sarea_t *sarea_priv = dev_priv->sarea_priv;
 656         int nbox = sarea_priv->nbox;
 657         drm_clip_rect_t *pbox = sarea_priv->boxes;
 658         int i;
 659         RING_LOCALS;
 660         DRM_DEBUG( "%s\n", __func__ );
 661
 662 #if RADEON_PERFORMANCE_BOXES
 663         /* Do some trivial performance monitoring...
 664          */
 665         radeon_cp_performance_boxes( dev_priv );
 666 #endif
 667
 668         /* Wait for the 3D stream to idle before dispatching the bitblt.
 669          * This will prevent data corruption between the two streams.
 670          */
 671         BEGIN_RING( 2 );
 672
 673         RADEON_WAIT_UNTIL_3D_IDLE();
 674
 675         ADVANCE_RING();
 676
 677         for ( i = 0 ; i < nbox ; i++ ) {
 678                 int x = pbox[i].x1;
 679                 int y = pbox[i].y1;
 680                 int w = pbox[i].x2 - x;
 681                 int h = pbox[i].y2 - y;
 682
 683                 DRM_DEBUG( "dispatch swap %d,%d-%d,%d\n",
 684                            x, y, w, h );
 685
 686                 BEGIN_RING( 7 );
 687
 688                 OUT_RING( CP_PACKET3( RADEON_CNTL_BITBLT_MULTI, 5 ) );
 689                 OUT_RING( RADEON_GMC_SRC_PITCH_OFFSET_CNTL |
 690                           RADEON_GMC_DST_PITCH_OFFSET_CNTL |
 691                           RADEON_GMC_BRUSH_NONE |
 692                           (dev_priv->color_fmt << 8) |
 693                           RADEON_GMC_SRC_DATATYPE_COLOR |
 694                           RADEON_ROP3_S |
 695                           RADEON_DP_SRC_SOURCE_MEMORY |
 696                           RADEON_GMC_CLR_CMP_CNTL_DIS |
 697                           RADEON_GMC_WR_MSK_DIS );
 698
 699                 OUT_RING( dev_priv->back_pitch_offset );
 700                 OUT_RING( dev_priv->front_pitch_offset );
 701
 702                 OUT_RING( (x << 16) | y );
 703                 OUT_RING( (x << 16) | y );
 704                 OUT_RING( (w << 16) | h );
 705
 706                 ADVANCE_RING();
 707         }
 708
 709         /* Increment the frame counter.  The client-side 3D driver must
 710          * throttle the framerate by waiting for this value before
 711          * performing the swapbuffer ioctl.
 712          */
 713         dev_priv->sarea_priv->last_frame++;
 714
 715         BEGIN_RING( 4 );
 716
 717         RADEON_FRAME_AGE( dev_priv->sarea_priv->last_frame );
 718         RADEON_WAIT_UNTIL_2D_IDLE();
 719
 720         ADVANCE_RING();
 721 }
 722
 723 static void radeon_cp_dispatch_flip( drm_device_t *dev )
 724 {
 725         drm_radeon_private_t *dev_priv = dev->dev_private;
 726         RING_LOCALS;
 727         DRM_DEBUG( "%s: page=%d\n", __func__, dev_priv->current_page );
 728
 729 #if RADEON_PERFORMANCE_BOXES
 730         /* Do some trivial performance monitoring...
 731          */
 732         radeon_cp_performance_boxes( dev_priv );
 733 #endif
 734
 735         BEGIN_RING( 6 );
 736
 737         RADEON_WAIT_UNTIL_3D_IDLE();
 738         RADEON_WAIT_UNTIL_PAGE_FLIPPED();
 739
 740         OUT_RING( CP_PACKET0( RADEON_CRTC_OFFSET, 0 ) );
 741
 742         if ( dev_priv->current_page == 0 ) {
 743                 OUT_RING( dev_priv->back_offset );
 744                 dev_priv->current_page = 1;
 745         } else {
 746                 OUT_RING( dev_priv->front_offset );
 747                 dev_priv->current_page = 0;
 748         }
 749
 750         ADVANCE_RING();
 751
 752         /* Increment the frame counter.  The client-side 3D driver must
 753          * throttle the framerate by waiting for this value before
 754          * performing the swapbuffer ioctl.
 755          */
 756         dev_priv->sarea_priv->last_frame++;
 757
 758         BEGIN_RING( 2 );
 759
 760         RADEON_FRAME_AGE( dev_priv->sarea_priv->last_frame );
 761
 762         ADVANCE_RING();
 763 }
 764
 765 static void radeon_cp_dispatch_vertex( drm_device_t *dev,
 766                                        drm_buf_t *buf )
 767 {
 768         drm_radeon_private_t *dev_priv = dev->dev_private;
 769         drm_radeon_buf_priv_t *buf_priv = buf->dev_private;
 770         drm_radeon_sarea_t *sarea_priv = dev_priv->sarea_priv;
 771         int format = sarea_priv->vc_format;
 772         int offset = dev_priv->agp_buffers_offset + buf->offset;
 773         int size = buf->used;
 774         int prim = buf_priv->prim;
 775         int i = 0;
 776         RING_LOCALS;
 777         DRM_DEBUG( "%s: nbox=%d\n", __func__, sarea_priv->nbox );
 778
 779         if ( 0 )
 780                 radeon_print_dirty( "dispatch_vertex", sarea_priv->dirty );
 781
 782         if ( buf->used ) {
 783                 buf_priv->dispatched = 1;
 784
 785                 if ( sarea_priv->dirty & ~RADEON_UPLOAD_CLIPRECTS ) {
 786                         radeon_emit_state( dev_priv );
 787                 }
 788
 789                 do {
 790                         /* Emit the next set of up to three cliprects */
 791                         if ( i < sarea_priv->nbox ) {
 792                                 radeon_emit_clip_rect( dev_priv,
 793                                                        &sarea_priv->boxes[i] );
 794                         }
 795
 796                         /* Emit the vertex buffer rendering commands */
 797                         BEGIN_RING( 5 );
 798
 799                         OUT_RING( CP_PACKET3( RADEON_3D_RNDR_GEN_INDX_PRIM, 3 ) );
 800                         OUT_RING( offset );
 801                         OUT_RING( size );
 802                         OUT_RING( format );
 803                         OUT_RING( prim | RADEON_PRIM_WALK_LIST |
 804                                   RADEON_COLOR_ORDER_RGBA |
 805                                   RADEON_VTX_FMT_RADEON_MODE |
 806                                   (size << RADEON_NUM_VERTICES_SHIFT) );
 807
 808                         ADVANCE_RING();
 809
 810                         i++;
 811                 } while ( i < sarea_priv->nbox );
 812         }
 813
 814         if ( buf_priv->discard ) {
 815                 buf_priv->age = dev_priv->sarea_priv->last_dispatch;
 816
 817                 /* Emit the vertex buffer age */
 818                 BEGIN_RING( 2 );
 819                 RADEON_DISPATCH_AGE( buf_priv->age );
 820                 ADVANCE_RING();
 821
 822                 buf->pending = 1;
 823                 buf->used = 0;
 824                 /* FIXME: Check dispatched field */
 825                 buf_priv->dispatched = 0;
 826         }
 827
 828         dev_priv->sarea_priv->last_dispatch++;
 829
 830         sarea_priv->dirty &= ~RADEON_UPLOAD_CLIPRECTS;
 831         sarea_priv->nbox = 0;
 832 }
 833
 834
 835 static void radeon_cp_dispatch_indirect( drm_device_t *dev,
 836                                          drm_buf_t *buf,
 837                                          int start, int end )
 838 {
 839         drm_radeon_private_t *dev_priv = dev->dev_private;
 840         drm_radeon_buf_priv_t *buf_priv = buf->dev_private;
 841         RING_LOCALS;
 842         DRM_DEBUG( "indirect: buf=%d s=0x%x e=0x%x\n",
 843                    buf->idx, start, end );
 844
 845         if ( start != end ) {
 846                 int offset = (dev_priv->agp_buffers_offset
 847                               + buf->offset + start);
 848                 int dwords = (end - start + 3) / sizeof(u32);
 849
 850                 /* Indirect buffer data must be an even number of
 851                  * dwords, so if we've been given an odd number we must
 852                  * pad the data with a Type-2 CP packet.
 853                  */
 854                 if ( dwords & 1 ) {
 855                         u32 *data = (u32 *)
 856                                 ((char *)dev_priv->buffers->handle
 857                                  + buf->offset + start);
 858                         data[dwords++] = RADEON_CP_PACKET2;
 859                 }
 860
 861                 buf_priv->dispatched = 1;
 862
 863                 /* Fire off the indirect buffer */
 864                 BEGIN_RING( 3 );
 865
 866                 OUT_RING( CP_PACKET0( RADEON_CP_IB_BASE, 1 ) );
 867                 OUT_RING( offset );
 868                 OUT_RING( dwords );
 869
 870                 ADVANCE_RING();
 871         }
 872
 873         if ( buf_priv->discard ) {
 874                 buf_priv->age = dev_priv->sarea_priv->last_dispatch;
 875
 876                 /* Emit the indirect buffer age */
 877                 BEGIN_RING( 2 );
 878                 RADEON_DISPATCH_AGE( buf_priv->age );
 879                 ADVANCE_RING();
 880
 881                 buf->pending = 1;
 882                 buf->used = 0;
 883                 /* FIXME: Check dispatched field */
 884                 buf_priv->dispatched = 0;
 885         }
 886
 887         dev_priv->sarea_priv->last_dispatch++;
 888 }
 889
 890 static void radeon_cp_dispatch_indices( drm_device_t *dev,
 891                                         drm_buf_t *buf,
 892                                         int start, int end,
 893                                         int count )
 894 {
 895         drm_radeon_private_t *dev_priv = dev->dev_private;
 896         drm_radeon_buf_priv_t *buf_priv = buf->dev_private;
 897         drm_radeon_sarea_t *sarea_priv = dev_priv->sarea_priv;
 898         int format = sarea_priv->vc_format;
 899         int offset = dev_priv->agp_buffers_offset;
 900         int prim = buf_priv->prim;
 901         u32 *data;
 902         int dwords;
 903         int i = 0;
 904         RING_LOCALS;
 905         DRM_DEBUG( "indices: s=%d e=%d c=%d\n", start, end, count );
 906
 907         if ( 0 )
 908                 radeon_print_dirty( "dispatch_indices", sarea_priv->dirty );
 909
 910         if ( start != end ) {
 911                 buf_priv->dispatched = 1;
 912
 913                 if ( sarea_priv->dirty & ~RADEON_UPLOAD_CLIPRECTS ) {
 914                         radeon_emit_state( dev_priv );
 915                 }
 916
 917                 dwords = (end - start + 3) / sizeof(u32);
 918
 919                 data = (u32 *)((char *)dev_priv->buffers->handle
 920                                + buf->offset + start);
 921
 922                 data[0] = CP_PACKET3( RADEON_3D_RNDR_GEN_INDX_PRIM, dwords-2 );
 923
 924                 data[1] = offset;
 925                 data[2] = RADEON_MAX_VB_VERTS;
 926                 data[3] = format;
 927                 data[4] = (prim | RADEON_PRIM_WALK_IND |
 928                            RADEON_COLOR_ORDER_RGBA |
 929                            RADEON_VTX_FMT_RADEON_MODE |
 930                            (count << RADEON_NUM_VERTICES_SHIFT) );
 931
 932                 if ( count & 0x1 ) {
 933                         data[dwords-1] &= 0x0000ffff;
 934                 }
 935
 936                 do {
 937                         /* Emit the next set of up to three cliprects */
 938                         if ( i < sarea_priv->nbox ) {
 939                                 radeon_emit_clip_rect( dev_priv,
 940                                                        &sarea_priv->boxes[i] );
 941                         }
 942
 943                         radeon_cp_dispatch_indirect( dev, buf, start, end );
 944
 945                         i++;
 946                 } while ( i < sarea_priv->nbox );
 947         }
 948
 949         if ( buf_priv->discard ) {
 950                 buf_priv->age = dev_priv->sarea_priv->last_dispatch;
 951
 952                 /* Emit the vertex buffer age */
 953                 BEGIN_RING( 2 );
 954                 RADEON_DISPATCH_AGE( buf_priv->age );
 955                 ADVANCE_RING();
 956
 957                 buf->pending = 1;
 958                 /* FIXME: Check dispatched field */
 959                 buf_priv->dispatched = 0;
 960         }
 961
 962         dev_priv->sarea_priv->last_dispatch++;
 963
 964         sarea_priv->dirty &= ~RADEON_UPLOAD_CLIPRECTS;
 965         sarea_priv->nbox = 0;
 966 }
 967
 968 #define RADEON_MAX_TEXTURE_SIZE (RADEON_BUFFER_SIZE - 8 * sizeof(u32))
 969
 970 static int radeon_cp_dispatch_texture( drm_device_t *dev,
 971                                        drm_radeon_texture_t *tex,
 972                                        drm_radeon_tex_image_t *image, int pid )
 973 {
 974         drm_radeon_private_t *dev_priv = dev->dev_private;
 975         drm_buf_t *buf;
 976         drm_radeon_buf_priv_t *buf_priv;
 977         u32 format;
 978         u32 *buffer;
 979         const u8 *data;
 980         int size, dwords, tex_width, blit_width;
 981         u32 y, height;
 982         int ret = 0, i;
 983         RING_LOCALS;
 984
 985         /* FIXME: Be smarter about this...
 986          */
 987         buf = radeon_freelist_get( dev );
 988         if ( !buf ) return DRM_OS_ERR(EAGAIN);
 989
 990         DRM_DEBUG( "tex: ofs=0x%x p=%d f=%d x=%hd y=%hd w=%hd h=%hd\n",
 991                    tex->offset >> 10, tex->pitch, tex->format,
 992                    image->x, image->y, image->width, image->height );
 993
 994         buf_priv = buf->dev_private;
 995
 996         /* The compiler won't optimize away a division by a variable,
 997          * even if the only legal values are powers of two.  Thus, we'll
 998          * use a shift instead.
 999          */
1000         switch ( tex->format ) {
1001         case RADEON_TXFORMAT_ARGB8888:
1002         case RADEON_TXFORMAT_RGBA8888:
1003                 format = RADEON_COLOR_FORMAT_ARGB8888;
1004                 tex_width = tex->width * 4;
1005                 blit_width = image->width * 4;
1006                 break;
1007         case RADEON_TXFORMAT_AI88:
1008         case RADEON_TXFORMAT_ARGB1555:
1009         case RADEON_TXFORMAT_RGB565:
1010         case RADEON_TXFORMAT_ARGB4444:
1011                 format = RADEON_COLOR_FORMAT_RGB565;
1012                 tex_width = tex->width * 2;
1013                 blit_width = image->width * 2;
1014                 break;
1015         case RADEON_TXFORMAT_I8:
1016         case RADEON_TXFORMAT_RGB332:
1017                 format = RADEON_COLOR_FORMAT_CI8;
1018                 tex_width = tex->width * 1;
1019                 blit_width = image->width * 1;
1020                 break;
1021         default:
1022                 DRM_ERROR( "invalid texture format %d\n", tex->format );
1023                 return DRM_OS_ERR(EINVAL);
1024         }
1025
1026         DRM_DEBUG( "   tex=%dx%d  blit=%d\n",
1027                    tex_width, tex->height, blit_width );
1028
1029         /* Flush the pixel cache.  This ensures no pixel data gets mixed
1030          * up with the texture data from the host data blit, otherwise
1031          * part of the texture image may be corrupted.
1032          */
1033         BEGIN_RING( 4 );
1034
1035         RADEON_FLUSH_CACHE();
1036         RADEON_WAIT_UNTIL_IDLE();
1037
1038         ADVANCE_RING();
1039
1040         /* Make a copy of the parameters in case we have to update them
1041          * for a multi-pass texture blit.
1042          */
1043         y = image->y;
1044         height = image->height;
1045         data = image->data;
1046
1047         size = height * blit_width;
1048
1049         if ( size > RADEON_MAX_TEXTURE_SIZE ) {
1050                 /* Texture image is too large, do a multipass upload */
1051                 ret = EAGAIN;
1052
1053                 /* Adjust the blit size to fit the indirect buffer */
1054                 height = RADEON_MAX_TEXTURE_SIZE / blit_width;
1055                 size = height * blit_width;
1056
1057                 /* Update the input parameters for next time */
1058                 image->y += height;
1059                 image->height -= height;
1060                 image->data = (const char *)image->data + size;
1061
1062                 if ( DRM_OS_COPYTOUSR( tex->image, image, sizeof(*image) ) ) {
1063                         DRM_ERROR( "EFAULT on tex->image\n" );
1064                         return DRM_OS_ERR(EFAULT);
1065                 }
1066         } else if ( size < 4 && size > 0 ) {
1067                 size = 4;
1068         }
1069
1070         dwords = size / 4;
1071
1072         /* Dispatch the indirect buffer.
1073          */
1074         buffer = (u32 *)((char *)dev_priv->buffers->handle + buf->offset);
1075
1076         buffer[0] = CP_PACKET3( RADEON_CNTL_HOSTDATA_BLT, dwords + 6 );
1077         buffer[1] = (RADEON_GMC_DST_PITCH_OFFSET_CNTL |
1078                      RADEON_GMC_BRUSH_NONE |
1079                      (format << 8) |
1080                      RADEON_GMC_SRC_DATATYPE_COLOR |
1081                      RADEON_ROP3_S |
1082                      RADEON_DP_SRC_SOURCE_HOST_DATA |
1083                      RADEON_GMC_CLR_CMP_CNTL_DIS |
1084                      RADEON_GMC_WR_MSK_DIS);
1085
1086         buffer[2] = (tex->pitch << 22) | (tex->offset >> 10);
1087         buffer[3] = 0xffffffff;
1088         buffer[4] = 0xffffffff;
1089         buffer[5] = (y << 16) | image->x;
1090         buffer[6] = (height << 16) | image->width;
1091         buffer[7] = dwords;
1092
1093         buffer += 8;
1094
1095         if ( tex_width >= 32 ) {
1096                 /* Texture image width is larger than the minimum, so we
1097                  * can upload it directly.
1098                  */
1099                 if ( DRM_OS_COPYFROMUSR( buffer, data, dwords * sizeof(u32) ) ) {
1100                         DRM_ERROR( "EFAULT on data, %d dwords\n", dwords );
1101                         return DRM_OS_ERR(EFAULT);
1102                 }
1103         } else {
1104                 /* Texture image width is less than the minimum, so we
1105                  * need to pad out each image scanline to the minimum
1106                  * width.
1107                  */
1108                 for ( i = 0 ; i < tex->height ; i++ ) {
1109                         if ( DRM_OS_COPYFROMUSR( buffer, data, tex_width ) ) {
1110                                 DRM_ERROR( "EFAULT on pad, %d bytes\n",
1111                                            tex_width );
1112                                 return DRM_OS_ERR(EFAULT);
1113                         }
1114                         buffer += 8;
1115                         data += tex_width;
1116                 }
1117         }
1118
1119         buf->pid = pid;
1120         buf->used = (dwords + 8) * sizeof(u32);
1121         buf_priv->discard = 1;
1122
1123         radeon_cp_dispatch_indirect( dev, buf, 0, buf->used );
1124
1125         /* Flush the pixel cache after the blit completes.  This ensures
1126          * the texture data is written out to memory before rendering
1127          * continues.
1128          */
1129         BEGIN_RING( 4 );
1130
1131         RADEON_FLUSH_CACHE();
1132         RADEON_WAIT_UNTIL_2D_IDLE();
1133
1134         ADVANCE_RING();
1135
1136         return ret;
1137 }
1138
1139 static void radeon_cp_dispatch_stipple( drm_device_t *dev, u32 *stipple )
1140 {
1141         drm_radeon_private_t *dev_priv = dev->dev_private;
1142         int i;
1143         RING_LOCALS;
1144         DRM_DEBUG( "%s\n", __func__ );
1145
1146         BEGIN_RING( 35 );
1147
1148         OUT_RING( CP_PACKET0( RADEON_RE_STIPPLE_ADDR, 0 ) );
1149         OUT_RING( 0x00000000 );
1150
1151         OUT_RING( CP_PACKET0_TABLE( RADEON_RE_STIPPLE_DATA, 31 ) );
1152         for ( i = 0 ; i < 32 ; i++ ) {
1153                 OUT_RING( stipple[i] );
1154         }
1155
1156         ADVANCE_RING();
1157 }
1158
1159
1160 /* ================================================================
1161  * IOCTL functions
1162  */
1163
1164 int radeon_cp_clear( DRM_OS_IOCTL )
1165 {
1166         DRM_OS_DEVICE;
1167         drm_radeon_private_t *dev_priv = dev->dev_private;
1168         drm_radeon_sarea_t *sarea_priv = dev_priv->sarea_priv;
1169         drm_radeon_clear_t clear;
1170         drm_radeon_clear_rect_t depth_boxes[RADEON_NR_SAREA_CLIPRECTS];
1171         DRM_DEBUG( "%s\n", __func__ );
1172
1173         LOCK_TEST_WITH_RETURN( dev );
1174
1175         DRM_OS_KRNFROMUSR( clear, (drm_radeon_clear_t *) data,
1176                              sizeof(clear) );
1177
1178         RING_SPACE_TEST_WITH_RETURN( dev_priv );
1179
1180         if ( sarea_priv->nbox > RADEON_NR_SAREA_CLIPRECTS )
1181                 sarea_priv->nbox = RADEON_NR_SAREA_CLIPRECTS;
1182
1183         if ( DRM_OS_COPYFROMUSR( &depth_boxes, clear.depth_boxes,
1184                              sarea_priv->nbox * sizeof(depth_boxes[0]) ) )
1185                 return DRM_OS_ERR(EFAULT);
1186
1187         radeon_cp_dispatch_clear( dev, &clear, depth_boxes );
1188
1189         return 0;
1190 }
1191
1192 int radeon_cp_swap( DRM_OS_IOCTL )
1193 {
1194         DRM_OS_DEVICE;
1195         drm_radeon_private_t *dev_priv = dev->dev_private;
1196         drm_radeon_sarea_t *sarea_priv = dev_priv->sarea_priv;
1197         DRM_DEBUG( "%s\n", __func__ );
1198
1199         LOCK_TEST_WITH_RETURN( dev );
1200
1201         RING_SPACE_TEST_WITH_RETURN( dev_priv );
1202
1203         if ( sarea_priv->nbox > RADEON_NR_SAREA_CLIPRECTS )
1204                 sarea_priv->nbox = RADEON_NR_SAREA_CLIPRECTS;
1205
1206         if ( !dev_priv->page_flipping ) {
1207                 radeon_cp_dispatch_swap( dev );
1208                 dev_priv->sarea_priv->dirty |= (RADEON_UPLOAD_CONTEXT |
1209                                                 RADEON_UPLOAD_MASKS);
1210         } else {
1211                 radeon_cp_dispatch_flip( dev );
1212         }
1213
1214         return 0;
1215 }
1216
1217 int radeon_cp_vertex( DRM_OS_IOCTL )
1218 {
1219         DRM_OS_DEVICE;
1220         drm_radeon_private_t *dev_priv = dev->dev_private;
1221         drm_device_dma_t *dma = dev->dma;
1222         drm_buf_t *buf;
1223         drm_radeon_buf_priv_t *buf_priv;
1224         drm_radeon_vertex_t vertex;
1225
1226         LOCK_TEST_WITH_RETURN( dev );
1227
1228         if ( !dev_priv ) {
1229                 DRM_ERROR( "%s called with no initialization\n", __func__ );
1230                 return DRM_OS_ERR(EINVAL);
1231         }
1232
1233         DRM_OS_KRNFROMUSR( vertex, (drm_radeon_vertex_t *) data,
1234                              sizeof(vertex) );
1235
1236         DRM_DEBUG( "%s: pid=%d index=%d count=%d discard=%d\n",
1237                    __func__, DRM_OS_CURRENTPID,
1238                    vertex.idx, vertex.count, vertex.discard );
1239
1240         if ( vertex.idx < 0 || vertex.idx >= dma->buf_count ) {
1241                 DRM_ERROR( "buffer index %d (of %d max)\n",
1242                            vertex.idx, dma->buf_count - 1 );
1243                 return DRM_OS_ERR(EINVAL);
1244         }
1245         if ( vertex.prim < 0 ||
1246              vertex.prim > RADEON_PRIM_TYPE_3VRT_LINE_LIST ) {
1247                 DRM_ERROR( "buffer prim %d\n", vertex.prim );
1248                 return DRM_OS_ERR(EINVAL);
1249         }
1250
1251         RING_SPACE_TEST_WITH_RETURN( dev_priv );
1252         VB_AGE_TEST_WITH_RETURN( dev_priv );
1253
1254         buf = dma->buflist[vertex.idx];
1255         buf_priv = buf->dev_private;
1256
1257         if ( buf->pid != DRM_OS_CURRENTPID ) {
1258                 DRM_ERROR( "process %d using buffer owned by %d\n",
1259                            DRM_OS_CURRENTPID, buf->pid );
1260                 return DRM_OS_ERR(EINVAL);
1261         }
1262         if ( buf->pending ) {
1263                 DRM_ERROR( "sending pending buffer %d\n", vertex.idx );
1264                 return DRM_OS_ERR(EINVAL);
1265         }
1266
1267         buf->used = vertex.count;
1268         buf_priv->prim = vertex.prim;
1269         buf_priv->discard = vertex.discard;
1270
1271         radeon_cp_dispatch_vertex( dev, buf );
1272
1273         return 0;
1274 }
1275
1276 int radeon_cp_indices( DRM_OS_IOCTL )
1277 {
1278         DRM_OS_DEVICE;
1279         drm_radeon_private_t *dev_priv = dev->dev_private;
1280         drm_device_dma_t *dma = dev->dma;
1281         drm_buf_t *buf;
1282         drm_radeon_buf_priv_t *buf_priv;
1283         drm_radeon_indices_t elts;
1284         int count;
1285
1286         LOCK_TEST_WITH_RETURN( dev );
1287
1288         if ( !dev_priv ) {
1289                 DRM_ERROR( "%s called with no initialization\n", __func__ );
1290                 return DRM_OS_ERR(EINVAL);
1291         }
1292
1293         DRM_OS_KRNFROMUSR( elts, (drm_radeon_indices_t *) data,
1294                              sizeof(elts) );
1295
1296         DRM_DEBUG( "%s: pid=%d index=%d start=%d end=%d discard=%d\n",
1297                    __func__, DRM_OS_CURRENTPID,
1298                    elts.idx, elts.start, elts.end, elts.discard );
1299
1300         if ( elts.idx < 0 || elts.idx >= dma->buf_count ) {
1301                 DRM_ERROR( "buffer index %d (of %d max)\n",
1302                            elts.idx, dma->buf_count - 1 );
1303                 return DRM_OS_ERR(EINVAL);
1304         }
1305         if ( elts.prim < 0 ||
1306              elts.prim > RADEON_PRIM_TYPE_3VRT_LINE_LIST ) {
1307                 DRM_ERROR( "buffer prim %d\n", elts.prim );
1308                 return DRM_OS_ERR(EINVAL);
1309         }
1310
1311         RING_SPACE_TEST_WITH_RETURN( dev_priv );
1312         VB_AGE_TEST_WITH_RETURN( dev_priv );
1313
1314         buf = dma->buflist[elts.idx];
1315         buf_priv = buf->dev_private;
1316
1317         if ( buf->pid != DRM_OS_CURRENTPID ) {
1318                 DRM_ERROR( "process %d using buffer owned by %d\n",
1319                           DRM_OS_CURRENTPID, buf->pid );
1320                 return DRM_OS_ERR(EINVAL);
1321         }
1322         if ( buf->pending ) {
1323                 DRM_ERROR( "sending pending buffer %d\n", elts.idx );
1324                 return DRM_OS_ERR(EINVAL);
1325         }
1326
1327         count = (elts.end - elts.start) / sizeof(u16);
1328         elts.start -= RADEON_INDEX_PRIM_OFFSET;
1329
1330         if ( elts.start & 0x7 ) {
1331                 DRM_ERROR( "misaligned buffer 0x%x\n", elts.start );
1332                 return DRM_OS_ERR(EINVAL);
1333         }
1334         if ( elts.start < buf->used ) {
1335                 DRM_ERROR( "no header 0x%x - 0x%x\n", elts.start, buf->used );
1336                 return DRM_OS_ERR(EINVAL);
1337         }
1338
1339         buf->used = elts.end;
1340         buf_priv->prim = elts.prim;
1341         buf_priv->discard = elts.discard;
1342
1343         radeon_cp_dispatch_indices( dev, buf, elts.start, elts.end, count );
1344
1345         return 0;
1346 }
1347
1348 int radeon_cp_texture( DRM_OS_IOCTL )
1349 {
1350         DRM_OS_DEVICE;
1351         drm_radeon_private_t *dev_priv = dev->dev_private;
1352         drm_radeon_texture_t tex;
1353         drm_radeon_tex_image_t image;
1354
1355         LOCK_TEST_WITH_RETURN( dev );
1356
1357         DRM_OS_KRNFROMUSR( tex, (drm_radeon_texture_t *) data, sizeof(tex) );
1358
1359         if ( tex.image == NULL ) {
1360                 DRM_ERROR( "null texture image!\n" );
1361                 return DRM_OS_ERR(EINVAL);
1362         }
1363
1364         if ( DRM_OS_COPYFROMUSR( &image,
1365                              (drm_radeon_tex_image_t *)tex.image,
1366                              sizeof(image) ) )
1367                 return DRM_OS_ERR(EFAULT);
1368
1369         RING_SPACE_TEST_WITH_RETURN( dev_priv );
1370         VB_AGE_TEST_WITH_RETURN( dev_priv );
1371
1372         return radeon_cp_dispatch_texture( dev, &tex, &image, DRM_OS_CURRENTPID );
1373 }
1374
1375 int radeon_cp_stipple( DRM_OS_IOCTL )
1376 {
1377         DRM_OS_DEVICE;
1378         drm_radeon_private_t *dev_priv = dev->dev_private;
1379         drm_radeon_stipple_t stipple;
1380         u32 mask[32];
1381
1382         LOCK_TEST_WITH_RETURN( dev );
1383
1384         DRM_OS_KRNFROMUSR( stipple, (drm_radeon_stipple_t *) data,
1385                              sizeof(stipple) );
1386
1387         if ( DRM_OS_COPYFROMUSR( &mask, stipple.mask, 32 * sizeof(u32) ) )
1388                 return DRM_OS_ERR(EFAULT);
1389
1390         RING_SPACE_TEST_WITH_RETURN( dev_priv );
1391
1392         radeon_cp_dispatch_stipple( dev, mask );
1393
1394         return 0;
1395 }
1396
1397 int radeon_cp_indirect( DRM_OS_IOCTL )
1398 {
1399         DRM_OS_DEVICE;
1400         drm_radeon_private_t *dev_priv = dev->dev_private;
1401         drm_device_dma_t *dma = dev->dma;
1402         drm_buf_t *buf;
1403         drm_radeon_buf_priv_t *buf_priv;
1404         drm_radeon_indirect_t indirect;
1405         RING_LOCALS;
1406
1407         LOCK_TEST_WITH_RETURN( dev );
1408
1409         if ( !dev_priv ) {
1410                 DRM_ERROR( "%s called with no initialization\n", __func__ );
1411                 return DRM_OS_ERR(EINVAL);
1412         }
1413
1414         DRM_OS_KRNFROMUSR( indirect, (drm_radeon_indirect_t *) data,
1415                              sizeof(indirect) );
1416
1417         DRM_DEBUG( "indirect: idx=%d s=%d e=%d d=%d\n",
1418                    indirect.idx, indirect.start,
1419                    indirect.end, indirect.discard );
1420
1421         if ( indirect.idx < 0 || indirect.idx >= dma->buf_count ) {
1422                 DRM_ERROR( "buffer index %d (of %d max)\n",
1423                            indirect.idx, dma->buf_count - 1 );
1424                 return DRM_OS_ERR(EINVAL);
1425         }
1426
1427         buf = dma->buflist[indirect.idx];
1428         buf_priv = buf->dev_private;
1429
1430         if ( buf->pid != DRM_OS_CURRENTPID ) {
1431                 DRM_ERROR( "process %d using buffer owned by %d\n",
1432                            DRM_OS_CURRENTPID, buf->pid );
1433                 return DRM_OS_ERR(EINVAL);
1434         }
1435         if ( buf->pending ) {
1436                 DRM_ERROR( "sending pending buffer %d\n", indirect.idx );
1437                 return DRM_OS_ERR(EINVAL);
1438         }
1439
1440         if ( indirect.start < buf->used ) {
1441                 DRM_ERROR( "reusing indirect: start=0x%x actual=0x%x\n",
1442                            indirect.start, buf->used );
1443                 return DRM_OS_ERR(EINVAL);
1444         }
1445
1446         RING_SPACE_TEST_WITH_RETURN( dev_priv );
1447         VB_AGE_TEST_WITH_RETURN( dev_priv );
1448
1449         buf->used = indirect.end;
1450         buf_priv->discard = indirect.discard;
1451
1452         /* Wait for the 3D stream to idle before the indirect buffer
1453          * containing 2D acceleration commands is processed.
1454          */
1455         BEGIN_RING( 2 );
1456
1457         RADEON_WAIT_UNTIL_3D_IDLE();
1458
1459         ADVANCE_RING();
1460
1461         /* Dispatch the indirect buffer full of commands from the
1462          * X server.  This is insecure and is thus only available to
1463          * privileged clients.
1464          */
1465         radeon_cp_dispatch_indirect( dev, buf, indirect.start, indirect.end );
1466
1467         return 0;
1468 }