sys/dev/drm/r300_cmdbuf.c

   1 /* r300_cmdbuf.c -- Command buffer emission for R300 -*- linux-c -*-
   2  *
   3  * Copyright (C) The Weather Channel, Inc.  2002.
   4  * Copyright (C) 2004 Nicolai Haehnle.
   5  * All Rights Reserved.
   6  *
   7  * The Weather Channel (TM) funded Tungsten Graphics to develop the
   8  * initial release of the Radeon 8500 driver under the XFree86 license.
   9  * This notice must be preserved.
  10  *
  11  * Permission is hereby granted, free of charge, to any person obtaining a
  12  * copy of this software and associated documentation files (the "Software"),
  13  * to deal in the Software without restriction, including without limitation
  14  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  15  * and/or sell copies of the Software, and to permit persons to whom the
  16  * Software is furnished to do so, subject to the following conditions:
  17  *
  18  * The above copyright notice and this permission notice (including the next
  19  * paragraph) shall be included in all copies or substantial portions of the
  20  * Software.
  21  *
  22  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  23  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  24  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  25  * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  26  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  27  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  28  * DEALINGS IN THE SOFTWARE.
  29  *
  30  * Authors:
  31  *    Nicolai Haehnle <prefect_@gmx.net>
  32  */
  33
  34 #include <sys/cdefs.h>
  35 __FBSDID("$FreeBSD$");
  36
  37 #include "dev/drm/drmP.h"
  38 #include "dev/drm/drm.h"
  39 #include "dev/drm/radeon_drm.h"
  40 #include "dev/drm/radeon_drv.h"
  41 #include "dev/drm/r300_reg.h"
  42
  43 #define R300_SIMULTANEOUS_CLIPRECTS             4
  44
  45 /* Values for R300_RE_CLIPRECT_CNTL depending on the number of cliprects
  46  */
  47 static const int r300_cliprect_cntl[4] = {
  48         0xAAAA,
  49         0xEEEE,
  50         0xFEFE,
  51         0xFFFE
  52 };
  53
  54 /**
  55  * Emit up to R300_SIMULTANEOUS_CLIPRECTS cliprects from the given command
  56  * buffer, starting with index n.
  57  */
  58 static int r300_emit_cliprects(drm_radeon_private_t *dev_priv,
  59                                drm_radeon_kcmd_buffer_t *cmdbuf, int n)
  60 {
  61         struct drm_clip_rect box;
  62         int nr;
  63         int i;
  64         RING_LOCALS;
  65
  66         nr = cmdbuf->nbox - n;
  67         if (nr > R300_SIMULTANEOUS_CLIPRECTS)
  68                 nr = R300_SIMULTANEOUS_CLIPRECTS;
  69
  70         DRM_DEBUG("%i cliprects\n", nr);
  71
  72         if (nr) {
  73                 BEGIN_RING(6 + nr * 2);
  74                 OUT_RING(CP_PACKET0(R300_RE_CLIPRECT_TL_0, nr * 2 - 1));
  75
  76                 for (i = 0; i < nr; ++i) {
  77                         if (DRM_COPY_FROM_USER_UNCHECKED
  78                             (&box, &cmdbuf->boxes[n + i], sizeof(box))) {
  79                                 DRM_ERROR("copy cliprect faulted\n");
  80                                 return -EFAULT;
  81                         }
  82
  83                         box.x2--; /* Hardware expects inclusive bottom-right corner */
  84                         box.y2--;
  85
  86                         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV515) {
  87                                 box.x1 = (box.x1) &
  88                                         R300_CLIPRECT_MASK;
  89                                 box.y1 = (box.y1) &
  90                                         R300_CLIPRECT_MASK;
  91                                 box.x2 = (box.x2) &
  92                                         R300_CLIPRECT_MASK;
  93                                 box.y2 = (box.y2) &
  94                                         R300_CLIPRECT_MASK;
  95                         } else {
  96                                 box.x1 = (box.x1 + R300_CLIPRECT_OFFSET) &
  97                                         R300_CLIPRECT_MASK;
  98                                 box.y1 = (box.y1 + R300_CLIPRECT_OFFSET) &
  99                                         R300_CLIPRECT_MASK;
 100                                 box.x2 = (box.x2 + R300_CLIPRECT_OFFSET) &
 101                                         R300_CLIPRECT_MASK;
 102                                 box.y2 = (box.y2 + R300_CLIPRECT_OFFSET) &
 103                                         R300_CLIPRECT_MASK;
 104                         }
 105
 106                         OUT_RING((box.x1 << R300_CLIPRECT_X_SHIFT) |
 107                                  (box.y1 << R300_CLIPRECT_Y_SHIFT));
 108                         OUT_RING((box.x2 << R300_CLIPRECT_X_SHIFT) |
 109                                  (box.y2 << R300_CLIPRECT_Y_SHIFT));
 110
 111                 }
 112
 113                 OUT_RING_REG(R300_RE_CLIPRECT_CNTL, r300_cliprect_cntl[nr - 1]);
 114
 115                 /* TODO/SECURITY: Force scissors to a safe value, otherwise the
 116                  * client might be able to trample over memory.
 117                  * The impact should be very limited, but I'd rather be safe than
 118                  * sorry.
 119                  */
 120                 OUT_RING(CP_PACKET0(R300_RE_SCISSORS_TL, 1));
 121                 OUT_RING(0);
 122                 OUT_RING(R300_SCISSORS_X_MASK | R300_SCISSORS_Y_MASK);
 123                 ADVANCE_RING();
 124         } else {
 125                 /* Why we allow zero cliprect rendering:
 126                  * There are some commands in a command buffer that must be submitted
 127                  * even when there are no cliprects, e.g. DMA buffer discard
 128                  * or state setting (though state setting could be avoided by
 129                  * simulating a loss of context).
 130                  *
 131                  * Now since the cmdbuf interface is so chaotic right now (and is
 132                  * bound to remain that way for a bit until things settle down),
 133                  * it is basically impossible to filter out the commands that are
 134                  * necessary and those that aren't.
 135                  *
 136                  * So I choose the safe way and don't do any filtering at all;
 137                  * instead, I simply set up the engine so that all rendering
 138                  * can't produce any fragments.
 139                  */
 140                 BEGIN_RING(2);
 141                 OUT_RING_REG(R300_RE_CLIPRECT_CNTL, 0);
 142                 ADVANCE_RING();
 143         }
 144
 145         /* flus cache and wait idle clean after cliprect change */
 146         BEGIN_RING(2);
 147         OUT_RING(CP_PACKET0(R300_RB3D_DSTCACHE_CTLSTAT, 0));
 148         OUT_RING(R300_RB3D_DC_FLUSH);
 149         ADVANCE_RING();
 150         BEGIN_RING(2);
 151         OUT_RING(CP_PACKET0(RADEON_WAIT_UNTIL, 0));
 152         OUT_RING(RADEON_WAIT_3D_IDLECLEAN);
 153         ADVANCE_RING();
 154         /* set flush flag */
 155         dev_priv->track_flush |= RADEON_FLUSH_EMITED;
 156
 157         return 0;
 158 }
 159
 160 static u8 r300_reg_flags[0x10000 >> 2];
 161
 162 void r300_init_reg_flags(struct drm_device *dev)
 163 {
 164         int i;
 165         drm_radeon_private_t *dev_priv = dev->dev_private;
 166
 167         memset(r300_reg_flags, 0, 0x10000 >> 2);
 168 #define ADD_RANGE_MARK(reg, count,mark) \
 169                 for(i=((reg)>>2);i<((reg)>>2)+(count);i++)\
 170                         r300_reg_flags[i]|=(mark);
 171
 172 #define MARK_SAFE               1
 173 #define MARK_CHECK_OFFSET       2
 174
 175 #define ADD_RANGE(reg, count)   ADD_RANGE_MARK(reg, count, MARK_SAFE)
 176
 177         /* these match cmducs() command in r300_driver/r300/r300_cmdbuf.c */
 178         ADD_RANGE(R300_SE_VPORT_XSCALE, 6);
 179         ADD_RANGE(R300_VAP_CNTL, 1);
 180         ADD_RANGE(R300_SE_VTE_CNTL, 2);
 181         ADD_RANGE(0x2134, 2);
 182         ADD_RANGE(R300_VAP_CNTL_STATUS, 1);
 183         ADD_RANGE(R300_VAP_INPUT_CNTL_0, 2);
 184         ADD_RANGE(0x21DC, 1);
 185         ADD_RANGE(R300_VAP_UNKNOWN_221C, 1);
 186         ADD_RANGE(R300_VAP_CLIP_X_0, 4);
 187         ADD_RANGE(R300_VAP_PVS_STATE_FLUSH_REG, 1);
 188         ADD_RANGE(R300_VAP_UNKNOWN_2288, 1);
 189         ADD_RANGE(R300_VAP_OUTPUT_VTX_FMT_0, 2);
 190         ADD_RANGE(R300_VAP_PVS_CNTL_1, 3);
 191         ADD_RANGE(R300_GB_ENABLE, 1);
 192         ADD_RANGE(R300_GB_MSPOS0, 5);
 193         ADD_RANGE(R300_TX_INVALTAGS, 1);
 194         ADD_RANGE(R300_TX_ENABLE, 1);
 195         ADD_RANGE(0x4200, 4);
 196         ADD_RANGE(0x4214, 1);
 197         ADD_RANGE(R300_RE_POINTSIZE, 1);
 198         ADD_RANGE(0x4230, 3);
 199         ADD_RANGE(R300_RE_LINE_CNT, 1);
 200         ADD_RANGE(R300_RE_UNK4238, 1);
 201         ADD_RANGE(0x4260, 3);
 202         ADD_RANGE(R300_RE_SHADE, 4);
 203         ADD_RANGE(R300_RE_POLYGON_MODE, 5);
 204         ADD_RANGE(R300_RE_ZBIAS_CNTL, 1);
 205         ADD_RANGE(R300_RE_ZBIAS_T_FACTOR, 4);
 206         ADD_RANGE(R300_RE_OCCLUSION_CNTL, 1);
 207         ADD_RANGE(R300_RE_CULL_CNTL, 1);
 208         ADD_RANGE(0x42C0, 2);
 209         ADD_RANGE(R300_RS_CNTL_0, 2);
 210
 211         ADD_RANGE(R300_SU_REG_DEST, 1);
 212         if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV530)
 213                 ADD_RANGE(RV530_FG_ZBREG_DEST, 1);
 214
 215         ADD_RANGE(R300_SC_HYPERZ, 2);
 216         ADD_RANGE(0x43E8, 1);
 217
 218         ADD_RANGE(0x46A4, 5);
 219
 220         ADD_RANGE(R300_RE_FOG_STATE, 1);
 221         ADD_RANGE(R300_FOG_COLOR_R, 3);
 222         ADD_RANGE(R300_PP_ALPHA_TEST, 2);
 223         ADD_RANGE(0x4BD8, 1);
 224         ADD_RANGE(R300_PFS_PARAM_0_X, 64);
 225         ADD_RANGE(0x4E00, 1);
 226         ADD_RANGE(R300_RB3D_CBLEND, 2);
 227         ADD_RANGE(R300_RB3D_COLORMASK, 1);
 228         ADD_RANGE(R300_RB3D_BLEND_COLOR, 3);
 229         ADD_RANGE_MARK(R300_RB3D_COLOROFFSET0, 1, MARK_CHECK_OFFSET);   /* check offset */
 230         ADD_RANGE(R300_RB3D_COLORPITCH0, 1);
 231         ADD_RANGE(0x4E50, 9);
 232         ADD_RANGE(0x4E88, 1);
 233         ADD_RANGE(0x4EA0, 2);
 234         ADD_RANGE(R300_ZB_CNTL, 3);
 235         ADD_RANGE(R300_ZB_FORMAT, 4);
 236         ADD_RANGE_MARK(R300_ZB_DEPTHOFFSET, 1, MARK_CHECK_OFFSET);      /* check offset */
 237         ADD_RANGE(R300_ZB_DEPTHPITCH, 1);
 238         ADD_RANGE(R300_ZB_DEPTHCLEARVALUE, 1);
 239         ADD_RANGE(R300_ZB_ZMASK_OFFSET, 5);
 240         ADD_RANGE(R300_ZB_HIZ_OFFSET, 5);
 241         ADD_RANGE(R300_ZB_ZPASS_DATA, 1);
 242         ADD_RANGE_MARK(R300_ZB_ZPASS_ADDR, 1, MARK_CHECK_OFFSET);       /* check offset */
 243         ADD_RANGE(R300_ZB_DEPTHXY_OFFSET, 1)
 244
 245         ADD_RANGE(R300_TX_FILTER_0, 16);
 246         ADD_RANGE(R300_TX_FILTER1_0, 16);
 247         ADD_RANGE(R300_TX_SIZE_0, 16);
 248         ADD_RANGE(R300_TX_FORMAT_0, 16);
 249         ADD_RANGE(R300_TX_PITCH_0, 16);
 250         /* Texture offset is dangerous and needs more checking */
 251         ADD_RANGE_MARK(R300_TX_OFFSET_0, 16, MARK_CHECK_OFFSET);
 252         ADD_RANGE(R300_TX_CHROMA_KEY_0, 16);
 253         ADD_RANGE(R300_TX_BORDER_COLOR_0, 16);
 254
 255         /* Sporadic registers used as primitives are emitted */
 256         ADD_RANGE(R300_ZB_ZCACHE_CTLSTAT, 1);
 257         ADD_RANGE(R300_RB3D_DSTCACHE_CTLSTAT, 1);
 258         ADD_RANGE(R300_VAP_INPUT_ROUTE_0_0, 8);
 259         ADD_RANGE(R300_VAP_INPUT_ROUTE_1_0, 8);
 260
 261         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV515) {
 262                 ADD_RANGE(R500_VAP_INDEX_OFFSET, 1);
 263                 ADD_RANGE(R500_US_CONFIG, 2);
 264                 ADD_RANGE(R500_US_CODE_ADDR, 3);
 265                 ADD_RANGE(R500_US_FC_CTRL, 1);
 266                 ADD_RANGE(R500_RS_IP_0, 16);
 267                 ADD_RANGE(R500_RS_INST_0, 16);
 268                 ADD_RANGE(R500_RB3D_COLOR_CLEAR_VALUE_AR, 2);
 269                 ADD_RANGE(R500_RB3D_CONSTANT_COLOR_AR, 2);
 270                 ADD_RANGE(R500_ZB_FIFO_SIZE, 2);
 271         } else {
 272                 ADD_RANGE(R300_PFS_CNTL_0, 3);
 273                 ADD_RANGE(R300_PFS_NODE_0, 4);
 274                 ADD_RANGE(R300_PFS_TEXI_0, 64);
 275                 ADD_RANGE(R300_PFS_INSTR0_0, 64);
 276                 ADD_RANGE(R300_PFS_INSTR1_0, 64);
 277                 ADD_RANGE(R300_PFS_INSTR2_0, 64);
 278                 ADD_RANGE(R300_PFS_INSTR3_0, 64);
 279                 ADD_RANGE(R300_RS_INTERP_0, 8);
 280                 ADD_RANGE(R300_RS_ROUTE_0, 8);
 281
 282         }
 283 }
 284
 285 static __inline__ int r300_check_range(unsigned reg, int count)
 286 {
 287         int i;
 288         if (reg & ~0xffff)
 289                 return -1;
 290         for (i = (reg >> 2); i < (reg >> 2) + count; i++)
 291                 if (r300_reg_flags[i] != MARK_SAFE)
 292                         return 1;
 293         return 0;
 294 }
 295
 296 static __inline__ int r300_emit_carefully_checked_packet0(drm_radeon_private_t *
 297                                                           dev_priv,
 298                                                           drm_radeon_kcmd_buffer_t
 299                                                           * cmdbuf,
 300                                                           drm_r300_cmd_header_t
 301                                                           header)
 302 {
 303         int reg;
 304         int sz;
 305         int i;
 306         int values[64];
 307         RING_LOCALS;
 308
 309         sz = header.packet0.count;
 310         reg = (header.packet0.reghi << 8) | header.packet0.reglo;
 311
 312         if ((sz > 64) || (sz < 0)) {
 313                 DRM_ERROR
 314                     ("Cannot emit more than 64 values at a time (reg=%04x sz=%d)\n",
 315                      reg, sz);
 316                 return -EINVAL;
 317         }
 318         for (i = 0; i < sz; i++) {
 319                 values[i] = ((int *)cmdbuf->buf)[i];
 320                 switch (r300_reg_flags[(reg >> 2) + i]) {
 321                 case MARK_SAFE:
 322                         break;
 323                 case MARK_CHECK_OFFSET:
 324                         if (!radeon_check_offset(dev_priv, (u32) values[i])) {
 325                                 DRM_ERROR
 326                                     ("Offset failed range check (reg=%04x sz=%d)\n",
 327                                      reg, sz);
 328                                 return -EINVAL;
 329                         }
 330                         break;
 331                 default:
 332                         DRM_ERROR("Register %04x failed check as flag=%02x\n",
 333                                   reg + i * 4, r300_reg_flags[(reg >> 2) + i]);
 334                         return -EINVAL;
 335                 }
 336         }
 337
 338         BEGIN_RING(1 + sz);
 339         OUT_RING(CP_PACKET0(reg, sz - 1));
 340         OUT_RING_TABLE(values, sz);
 341         ADVANCE_RING();
 342
 343         cmdbuf->buf += sz * 4;
 344         cmdbuf->bufsz -= sz * 4;
 345
 346         return 0;
 347 }
 348
 349 /**
 350  * Emits a packet0 setting arbitrary registers.
 351  * Called by r300_do_cp_cmdbuf.
 352  *
 353  * Note that checks are performed on contents and addresses of the registers
 354  */
 355 static __inline__ int r300_emit_packet0(drm_radeon_private_t *dev_priv,
 356                                         drm_radeon_kcmd_buffer_t *cmdbuf,
 357                                         drm_r300_cmd_header_t header)
 358 {
 359         int reg;
 360         int sz;
 361         RING_LOCALS;
 362
 363         sz = header.packet0.count;
 364         reg = (header.packet0.reghi << 8) | header.packet0.reglo;
 365
 366         DRM_DEBUG("R300_CMD_PACKET0: reg %04x, sz %d\n", reg, sz);
 367         if (!sz)
 368                 return 0;
 369
 370         if (sz * 4 > cmdbuf->bufsz)
 371                 return -EINVAL;
 372
 373         if (reg + sz * 4 >= 0x10000) {
 374                 DRM_ERROR("No such registers in hardware reg=%04x sz=%d\n", reg,
 375                           sz);
 376                 return -EINVAL;
 377         }
 378
 379         if (r300_check_range(reg, sz)) {
 380                 /* go and check everything */
 381                 return r300_emit_carefully_checked_packet0(dev_priv, cmdbuf,
 382                                                            header);
 383         }
 384         /* the rest of the data is safe to emit, whatever the values the user passed */
 385
 386         BEGIN_RING(1 + sz);
 387         OUT_RING(CP_PACKET0(reg, sz - 1));
 388         OUT_RING_TABLE((int *)cmdbuf->buf, sz);
 389         ADVANCE_RING();
 390
 391         cmdbuf->buf += sz * 4;
 392         cmdbuf->bufsz -= sz * 4;
 393
 394         return 0;
 395 }
 396
 397 /**
 398  * Uploads user-supplied vertex program instructions or parameters onto
 399  * the graphics card.
 400  * Called by r300_do_cp_cmdbuf.
 401  */
 402 static __inline__ int r300_emit_vpu(drm_radeon_private_t *dev_priv,
 403                                     drm_radeon_kcmd_buffer_t *cmdbuf,
 404                                     drm_r300_cmd_header_t header)
 405 {
 406         int sz;
 407         int addr;
 408         RING_LOCALS;
 409
 410         sz = header.vpu.count;
 411         addr = (header.vpu.adrhi << 8) | header.vpu.adrlo;
 412
 413         if (!sz)
 414                 return 0;
 415         if (sz * 16 > cmdbuf->bufsz)
 416                 return -EINVAL;
 417
 418         /* VAP is very sensitive so we purge cache before we program it
 419          * and we also flush its state before & after */
 420         BEGIN_RING(6);
 421         OUT_RING(CP_PACKET0(R300_RB3D_DSTCACHE_CTLSTAT, 0));
 422         OUT_RING(R300_RB3D_DC_FLUSH);
 423         OUT_RING(CP_PACKET0(RADEON_WAIT_UNTIL, 0));
 424         OUT_RING(RADEON_WAIT_3D_IDLECLEAN);
 425         OUT_RING(CP_PACKET0(R300_VAP_PVS_STATE_FLUSH_REG, 0));
 426         OUT_RING(0);
 427         ADVANCE_RING();
 428         /* set flush flag */
 429         dev_priv->track_flush |= RADEON_FLUSH_EMITED;
 430
 431         BEGIN_RING(3 + sz * 4);
 432         OUT_RING_REG(R300_VAP_PVS_UPLOAD_ADDRESS, addr);
 433         OUT_RING(CP_PACKET0_TABLE(R300_VAP_PVS_UPLOAD_DATA, sz * 4 - 1));
 434         OUT_RING_TABLE((int *)cmdbuf->buf, sz * 4);
 435         ADVANCE_RING();
 436
 437         BEGIN_RING(2);
 438         OUT_RING(CP_PACKET0(R300_VAP_PVS_STATE_FLUSH_REG, 0));
 439         OUT_RING(0);
 440         ADVANCE_RING();
 441
 442         cmdbuf->buf += sz * 16;
 443         cmdbuf->bufsz -= sz * 16;
 444
 445         return 0;
 446 }
 447
 448 /**
 449  * Emit a clear packet from userspace.
 450  * Called by r300_emit_packet3.
 451  */
 452 static __inline__ int r300_emit_clear(drm_radeon_private_t *dev_priv,
 453                                       drm_radeon_kcmd_buffer_t *cmdbuf)
 454 {
 455         RING_LOCALS;
 456
 457         if (8 * 4 > cmdbuf->bufsz)
 458                 return -EINVAL;
 459
 460         BEGIN_RING(10);
 461         OUT_RING(CP_PACKET3(R200_3D_DRAW_IMMD_2, 8));
 462         OUT_RING(R300_PRIM_TYPE_POINT | R300_PRIM_WALK_RING |
 463                  (1 << R300_PRIM_NUM_VERTICES_SHIFT));
 464         OUT_RING_TABLE((int *)cmdbuf->buf, 8);
 465         ADVANCE_RING();
 466
 467         BEGIN_RING(4);
 468         OUT_RING(CP_PACKET0(R300_RB3D_DSTCACHE_CTLSTAT, 0));
 469         OUT_RING(R300_RB3D_DC_FLUSH);
 470         OUT_RING(CP_PACKET0(RADEON_WAIT_UNTIL, 0));
 471         OUT_RING(RADEON_WAIT_3D_IDLECLEAN);
 472         ADVANCE_RING();
 473         /* set flush flag */
 474         dev_priv->track_flush |= RADEON_FLUSH_EMITED;
 475
 476         cmdbuf->buf += 8 * 4;
 477         cmdbuf->bufsz -= 8 * 4;
 478
 479         return 0;
 480 }
 481
 482 static __inline__ int r300_emit_3d_load_vbpntr(drm_radeon_private_t *dev_priv,
 483                                                drm_radeon_kcmd_buffer_t *cmdbuf,
 484                                                u32 header)
 485 {
 486         int count, i, k;
 487 #define MAX_ARRAY_PACKET  64
 488         u32 payload[MAX_ARRAY_PACKET];
 489         u32 narrays;
 490         RING_LOCALS;
 491
 492         count = (header >> 16) & 0x3fff;
 493
 494         if ((count + 1) > MAX_ARRAY_PACKET) {
 495                 DRM_ERROR("Too large payload in 3D_LOAD_VBPNTR (count=%d)\n",
 496                           count);
 497                 return -EINVAL;
 498         }
 499         memset(payload, 0, MAX_ARRAY_PACKET * 4);
 500         memcpy(payload, cmdbuf->buf + 4, (count + 1) * 4);
 501
 502         /* carefully check packet contents */
 503
 504         narrays = payload[0];
 505         k = 0;
 506         i = 1;
 507         while ((k < narrays) && (i < (count + 1))) {
 508                 i++;            /* skip attribute field */
 509                 if (!radeon_check_offset(dev_priv, payload[i])) {
 510                         DRM_ERROR
 511                             ("Offset failed range check (k=%d i=%d) while processing 3D_LOAD_VBPNTR packet.\n",
 512                              k, i);
 513                         return -EINVAL;
 514                 }
 515                 k++;
 516                 i++;
 517                 if (k == narrays)
 518                         break;
 519                 /* have one more to process, they come in pairs */
 520                 if (!radeon_check_offset(dev_priv, payload[i])) {
 521                         DRM_ERROR
 522                             ("Offset failed range check (k=%d i=%d) while processing 3D_LOAD_VBPNTR packet.\n",
 523                              k, i);
 524                         return -EINVAL;
 525                 }
 526                 k++;
 527                 i++;
 528         }
 529         /* do the counts match what we expect ? */
 530         if ((k != narrays) || (i != (count + 1))) {
 531                 DRM_ERROR
 532                     ("Malformed 3D_LOAD_VBPNTR packet (k=%d i=%d narrays=%d count+1=%d).\n",
 533                      k, i, narrays, count + 1);
 534                 return -EINVAL;
 535         }
 536
 537         /* all clear, output packet */
 538
 539         BEGIN_RING(count + 2);
 540         OUT_RING(header);
 541         OUT_RING_TABLE(payload, count + 1);
 542         ADVANCE_RING();
 543
 544         cmdbuf->buf += (count + 2) * 4;
 545         cmdbuf->bufsz -= (count + 2) * 4;
 546
 547         return 0;
 548 }
 549
 550 static __inline__ int r300_emit_bitblt_multi(drm_radeon_private_t *dev_priv,
 551                                              drm_radeon_kcmd_buffer_t *cmdbuf)
 552 {
 553         u32 *cmd = (u32 *) cmdbuf->buf;
 554         int count, ret;
 555         RING_LOCALS;
 556
 557         count=(cmd[0]>>16) & 0x3fff;
 558
 559         if (cmd[0] & 0x8000) {
 560                 u32 offset;
 561
 562                 if (cmd[1] & (RADEON_GMC_SRC_PITCH_OFFSET_CNTL
 563                               | RADEON_GMC_DST_PITCH_OFFSET_CNTL)) {
 564                         offset = cmd[2] << 10;
 565                         ret = !radeon_check_offset(dev_priv, offset);
 566                         if (ret) {
 567                                 DRM_ERROR("Invalid bitblt first offset is %08X\n", offset);
 568                                 return -EINVAL;
 569                         }
 570                 }
 571
 572                 if ((cmd[1] & RADEON_GMC_SRC_PITCH_OFFSET_CNTL) &&
 573                     (cmd[1] & RADEON_GMC_DST_PITCH_OFFSET_CNTL)) {
 574                         offset = cmd[3] << 10;
 575                         ret = !radeon_check_offset(dev_priv, offset);
 576                         if (ret) {
 577                                 DRM_ERROR("Invalid bitblt second offset is %08X\n", offset);
 578                                 return -EINVAL;
 579                         }
 580
 581                 }
 582         }
 583
 584         BEGIN_RING(count+2);
 585         OUT_RING(cmd[0]);
 586         OUT_RING_TABLE((int *)(cmdbuf->buf + 4), count + 1);
 587         ADVANCE_RING();
 588
 589         cmdbuf->buf += (count+2)*4;
 590         cmdbuf->bufsz -= (count+2)*4;
 591
 592         return 0;
 593 }
 594
 595 static __inline__ int r300_emit_draw_indx_2(drm_radeon_private_t *dev_priv,
 596                                             drm_radeon_kcmd_buffer_t *cmdbuf)
 597 {
 598         u32 *cmd;
 599         int count;
 600         int expected_count;
 601         RING_LOCALS;
 602
 603         cmd = (u32 *) cmdbuf->buf;
 604         count = (cmd[0]>>16) & 0x3fff;
 605         expected_count = cmd[1] >> 16;
 606         if (!(cmd[1] & R300_VAP_VF_CNTL__INDEX_SIZE_32bit))
 607                 expected_count = (expected_count+1)/2;
 608
 609         if (count && count != expected_count) {
 610                 DRM_ERROR("3D_DRAW_INDX_2: packet size %i, expected %i\n",
 611                         count, expected_count);
 612                 return -EINVAL;
 613         }
 614
 615         BEGIN_RING(count+2);
 616         OUT_RING(cmd[0]);
 617         OUT_RING_TABLE((int *)(cmdbuf->buf + 4), count + 1);
 618         ADVANCE_RING();
 619
 620         cmdbuf->buf += (count+2)*4;
 621         cmdbuf->bufsz -= (count+2)*4;
 622
 623         if (!count) {
 624                 drm_r300_cmd_header_t header;
 625
 626                 if (cmdbuf->bufsz < 4*4 + sizeof(header)) {
 627                         DRM_ERROR("3D_DRAW_INDX_2: expect subsequent INDX_BUFFER, but stream is too short.\n");
 628                         return -EINVAL;
 629                 }
 630
 631                 header.u = *(unsigned int *)cmdbuf->buf;
 632
 633                 cmdbuf->buf += sizeof(header);
 634                 cmdbuf->bufsz -= sizeof(header);
 635                 cmd = (u32 *) cmdbuf->buf;
 636
 637                 if (header.header.cmd_type != R300_CMD_PACKET3 ||
 638                     header.packet3.packet != R300_CMD_PACKET3_RAW ||
 639                     cmd[0] != CP_PACKET3(RADEON_CP_INDX_BUFFER, 2)) {
 640                         DRM_ERROR("3D_DRAW_INDX_2: expect subsequent INDX_BUFFER.\n");
 641                         return -EINVAL;
 642                 }
 643
 644                 if ((cmd[1] & 0x8000ffff) != 0x80000810) {
 645                         DRM_ERROR("Invalid indx_buffer reg address %08X\n", cmd[1]);
 646                         return -EINVAL;
 647                 }
 648                 if (!radeon_check_offset(dev_priv, cmd[2])) {
 649                         DRM_ERROR("Invalid indx_buffer offset is %08X\n", cmd[2]);
 650                         return -EINVAL;
 651                 }
 652                 if (cmd[3] != expected_count) {
 653                         DRM_ERROR("INDX_BUFFER: buffer size %i, expected %i\n",
 654                                 cmd[3], expected_count);
 655                         return -EINVAL;
 656                 }
 657
 658                 BEGIN_RING(4);
 659                 OUT_RING(cmd[0]);
 660                 OUT_RING_TABLE((int *)(cmdbuf->buf + 4), 3);
 661                 ADVANCE_RING();
 662
 663                 cmdbuf->buf += 4*4;
 664                 cmdbuf->bufsz -= 4*4;
 665         }
 666
 667         return 0;
 668 }
 669
 670 static __inline__ int r300_emit_raw_packet3(drm_radeon_private_t *dev_priv,
 671                                             drm_radeon_kcmd_buffer_t *cmdbuf)
 672 {
 673         u32 header;
 674         int count;
 675         RING_LOCALS;
 676
 677         if (4 > cmdbuf->bufsz)
 678                 return -EINVAL;
 679
 680         /* Fixme !! This simply emits a packet without much checking.
 681            We need to be smarter. */
 682
 683         /* obtain first word - actual packet3 header */
 684         header = *(u32 *) cmdbuf->buf;
 685
 686         /* Is it packet 3 ? */
 687         if ((header >> 30) != 0x3) {
 688                 DRM_ERROR("Not a packet3 header (0x%08x)\n", header);
 689                 return -EINVAL;
 690         }
 691
 692         count = (header >> 16) & 0x3fff;
 693
 694         /* Check again now that we know how much data to expect */
 695         if ((count + 2) * 4 > cmdbuf->bufsz) {
 696                 DRM_ERROR
 697                     ("Expected packet3 of length %d but have only %d bytes left\n",
 698                      (count + 2) * 4, cmdbuf->bufsz);
 699                 return -EINVAL;
 700         }
 701
 702         /* Is it a packet type we know about ? */
 703         switch (header & 0xff00) {
 704         case RADEON_3D_LOAD_VBPNTR:     /* load vertex array pointers */
 705                 return r300_emit_3d_load_vbpntr(dev_priv, cmdbuf, header);
 706
 707         case RADEON_CNTL_BITBLT_MULTI:
 708                 return r300_emit_bitblt_multi(dev_priv, cmdbuf);
 709
 710         case RADEON_CP_INDX_BUFFER:
 711                 DRM_ERROR("packet3 INDX_BUFFER without preceding 3D_DRAW_INDX_2 is illegal.\n");
 712                 return -EINVAL;
 713         case RADEON_CP_3D_DRAW_IMMD_2:
 714                 /* triggers drawing using in-packet vertex data */
 715         case RADEON_CP_3D_DRAW_VBUF_2:
 716                 /* triggers drawing of vertex buffers setup elsewhere */
 717                 dev_priv->track_flush &= ~(RADEON_FLUSH_EMITED |
 718                                            RADEON_PURGE_EMITED);
 719                 break;
 720         case RADEON_CP_3D_DRAW_INDX_2:
 721                 /* triggers drawing using indices to vertex buffer */
 722                 /* whenever we send vertex we clear flush & purge */
 723                 dev_priv->track_flush &= ~(RADEON_FLUSH_EMITED |
 724                                            RADEON_PURGE_EMITED);
 725                 return r300_emit_draw_indx_2(dev_priv, cmdbuf);
 726         case RADEON_WAIT_FOR_IDLE:
 727         case RADEON_CP_NOP:
 728                 /* these packets are safe */
 729                 break;
 730         default:
 731                 DRM_ERROR("Unknown packet3 header (0x%08x)\n", header);
 732                 return -EINVAL;
 733         }
 734
 735         BEGIN_RING(count + 2);
 736         OUT_RING(header);
 737         OUT_RING_TABLE((int *)(cmdbuf->buf + 4), count + 1);
 738         ADVANCE_RING();
 739
 740         cmdbuf->buf += (count + 2) * 4;
 741         cmdbuf->bufsz -= (count + 2) * 4;
 742
 743         return 0;
 744 }
 745
 746 /**
 747  * Emit a rendering packet3 from userspace.
 748  * Called by r300_do_cp_cmdbuf.
 749  */
 750 static __inline__ int r300_emit_packet3(drm_radeon_private_t *dev_priv,
 751                                         drm_radeon_kcmd_buffer_t *cmdbuf,
 752                                         drm_r300_cmd_header_t header)
 753 {
 754         int n;
 755         int ret;
 756         char *orig_buf = cmdbuf->buf;
 757         int orig_bufsz = cmdbuf->bufsz;
 758
 759         /* This is a do-while-loop so that we run the interior at least once,
 760          * even if cmdbuf->nbox is 0. Compare r300_emit_cliprects for rationale.
 761          */
 762         n = 0;
 763         do {
 764                 if (cmdbuf->nbox > R300_SIMULTANEOUS_CLIPRECTS) {
 765                         ret = r300_emit_cliprects(dev_priv, cmdbuf, n);
 766                         if (ret)
 767                                 return ret;
 768
 769                         cmdbuf->buf = orig_buf;
 770                         cmdbuf->bufsz = orig_bufsz;
 771                 }
 772
 773                 switch (header.packet3.packet) {
 774                 case R300_CMD_PACKET3_CLEAR:
 775                         DRM_DEBUG("R300_CMD_PACKET3_CLEAR\n");
 776                         ret = r300_emit_clear(dev_priv, cmdbuf);
 777                         if (ret) {
 778                                 DRM_ERROR("r300_emit_clear failed\n");
 779                                 return ret;
 780                         }
 781                         break;
 782
 783                 case R300_CMD_PACKET3_RAW:
 784                         DRM_DEBUG("R300_CMD_PACKET3_RAW\n");
 785                         ret = r300_emit_raw_packet3(dev_priv, cmdbuf);
 786                         if (ret) {
 787                                 DRM_ERROR("r300_emit_raw_packet3 failed\n");
 788                                 return ret;
 789                         }
 790                         break;
 791
 792                 default:
 793                         DRM_ERROR("bad packet3 type %i at %p\n",
 794                                   header.packet3.packet,
 795                                   cmdbuf->buf - sizeof(header));
 796                         return -EINVAL;
 797                 }
 798
 799                 n += R300_SIMULTANEOUS_CLIPRECTS;
 800         } while (n < cmdbuf->nbox);
 801
 802         return 0;
 803 }
 804
 805 /* Some of the R300 chips seem to be extremely touchy about the two registers
 806  * that are configured in r300_pacify.
 807  * Among the worst offenders seems to be the R300 ND (0x4E44): When userspace
 808  * sends a command buffer that contains only state setting commands and a
 809  * vertex program/parameter upload sequence, this will eventually lead to a
 810  * lockup, unless the sequence is bracketed by calls to r300_pacify.
 811  * So we should take great care to *always* call r300_pacify before
 812  * *anything* 3D related, and again afterwards. This is what the
 813  * call bracket in r300_do_cp_cmdbuf is for.
 814  */
 815
 816 /**
 817  * Emit the sequence to pacify R300.
 818  */
 819 static __inline__ void r300_pacify(drm_radeon_private_t *dev_priv)
 820 {
 821         uint32_t cache_z, cache_3d, cache_2d;
 822         RING_LOCALS;
 823
 824         cache_z = R300_ZC_FLUSH;
 825         cache_2d = R300_RB2D_DC_FLUSH;
 826         cache_3d = R300_RB3D_DC_FLUSH;
 827         if (!(dev_priv->track_flush & RADEON_PURGE_EMITED)) {
 828                 /* we can purge, primitive where draw since last purge */
 829                 cache_z |= R300_ZC_FREE;
 830                 cache_2d |= R300_RB2D_DC_FREE;
 831                 cache_3d |= R300_RB3D_DC_FREE;
 832         }
 833
 834         /* flush & purge zbuffer */
 835         BEGIN_RING(2);
 836         OUT_RING(CP_PACKET0(R300_ZB_ZCACHE_CTLSTAT, 0));
 837         OUT_RING(cache_z);
 838         ADVANCE_RING();
 839         /* flush & purge 3d */
 840         BEGIN_RING(2);
 841         OUT_RING(CP_PACKET0(R300_RB3D_DSTCACHE_CTLSTAT, 0));
 842         OUT_RING(cache_3d);
 843         ADVANCE_RING();
 844         /* flush & purge texture */
 845         BEGIN_RING(2);
 846         OUT_RING(CP_PACKET0(R300_TX_INVALTAGS, 0));
 847         OUT_RING(0);
 848         ADVANCE_RING();
 849         /* FIXME: is this one really needed ? */
 850         BEGIN_RING(2);
 851         OUT_RING(CP_PACKET0(R300_RB3D_AARESOLVE_CTL, 0));
 852         OUT_RING(0);
 853         ADVANCE_RING();
 854         BEGIN_RING(2);
 855         OUT_RING(CP_PACKET0(RADEON_WAIT_UNTIL, 0));
 856         OUT_RING(RADEON_WAIT_3D_IDLECLEAN);
 857         ADVANCE_RING();
 858         /* flush & purge 2d through E2 as RB2D will trigger lockup */
 859         BEGIN_RING(4);
 860         OUT_RING(CP_PACKET0(R300_DSTCACHE_CTLSTAT, 0));
 861         OUT_RING(cache_2d);
 862         OUT_RING(CP_PACKET0(RADEON_WAIT_UNTIL, 0));
 863         OUT_RING(RADEON_WAIT_2D_IDLECLEAN |
 864                  RADEON_WAIT_HOST_IDLECLEAN);
 865         ADVANCE_RING();
 866         /* set flush & purge flags */
 867         dev_priv->track_flush |= RADEON_FLUSH_EMITED | RADEON_PURGE_EMITED;
 868 }
 869
 870 /**
 871  * Called by r300_do_cp_cmdbuf to update the internal buffer age and state.
 872  * The actual age emit is done by r300_do_cp_cmdbuf, which is why you must
 873  * be careful about how this function is called.
 874  */
 875 static void r300_discard_buffer(struct drm_device * dev, struct drm_buf * buf)
 876 {
 877         drm_radeon_private_t *dev_priv = dev->dev_private;
 878         drm_radeon_buf_priv_t *buf_priv = buf->dev_private;
 879
 880         buf_priv->age = ++dev_priv->sarea_priv->last_dispatch;
 881         buf->pending = 1;
 882         buf->used = 0;
 883 }
 884
 885 static void r300_cmd_wait(drm_radeon_private_t * dev_priv,
 886                           drm_r300_cmd_header_t header)
 887 {
 888         u32 wait_until;
 889         RING_LOCALS;
 890
 891         if (!header.wait.flags)
 892                 return;
 893
 894         wait_until = 0;
 895
 896         switch(header.wait.flags) {
 897         case R300_WAIT_2D:
 898                 wait_until = RADEON_WAIT_2D_IDLE;
 899                 break;
 900         case R300_WAIT_3D:
 901                 wait_until = RADEON_WAIT_3D_IDLE;
 902                 break;
 903         case R300_NEW_WAIT_2D_3D:
 904                 wait_until = RADEON_WAIT_2D_IDLE|RADEON_WAIT_3D_IDLE;
 905                 break;
 906         case R300_NEW_WAIT_2D_2D_CLEAN:
 907                 wait_until = RADEON_WAIT_2D_IDLE|RADEON_WAIT_2D_IDLECLEAN;
 908                 break;
 909         case R300_NEW_WAIT_3D_3D_CLEAN:
 910                 wait_until = RADEON_WAIT_3D_IDLE|RADEON_WAIT_3D_IDLECLEAN;
 911                 break;
 912         case R300_NEW_WAIT_2D_2D_CLEAN_3D_3D_CLEAN:
 913                 wait_until = RADEON_WAIT_2D_IDLE|RADEON_WAIT_2D_IDLECLEAN;
 914                 wait_until |= RADEON_WAIT_3D_IDLE|RADEON_WAIT_3D_IDLECLEAN;
 915                 break;
 916         default:
 917                 return;
 918         }
 919
 920         BEGIN_RING(2);
 921         OUT_RING(CP_PACKET0(RADEON_WAIT_UNTIL, 0));
 922         OUT_RING(wait_until);
 923         ADVANCE_RING();
 924 }
 925
 926 static int r300_scratch(drm_radeon_private_t *dev_priv,
 927                         drm_radeon_kcmd_buffer_t *cmdbuf,
 928                         drm_r300_cmd_header_t header)
 929 {
 930         u32 *ref_age_base;
 931         u32 i, buf_idx, h_pending;
 932         RING_LOCALS;
 933
 934         if (cmdbuf->bufsz < sizeof(uint64_t) + header.scratch.n_bufs * sizeof(buf_idx) ) {
 935                 return -EINVAL;
 936         }
 937
 938         if (header.scratch.reg >= 5) {
 939                 return -EINVAL;
 940         }
 941
 942         dev_priv->scratch_ages[header.scratch.reg] ++;
 943
 944         ref_age_base = (u32 *)(unsigned long)*((uint64_t *)cmdbuf->buf);
 945
 946         cmdbuf->buf += sizeof(uint64_t);
 947         cmdbuf->bufsz -= sizeof(uint64_t);
 948
 949         for (i=0; i < header.scratch.n_bufs; i++) {
 950                 buf_idx = *(u32 *)cmdbuf->buf;
 951                 buf_idx *= 2; /* 8 bytes per buf */
 952
 953                 if (DRM_COPY_TO_USER(ref_age_base + buf_idx, &dev_priv->scratch_ages[header.scratch.reg], sizeof(u32))) {
 954                         return -EINVAL;
 955                 }
 956
 957                 if (DRM_COPY_FROM_USER(&h_pending, ref_age_base + buf_idx + 1, sizeof(u32))) {
 958                         return -EINVAL;
 959                 }
 960
 961                 if (h_pending == 0) {
 962                         return -EINVAL;
 963                 }
 964
 965                 h_pending--;
 966
 967                 if (DRM_COPY_TO_USER(ref_age_base + buf_idx + 1, &h_pending, sizeof(u32))) {
 968                         return -EINVAL;
 969                 }
 970
 971                 cmdbuf->buf += sizeof(buf_idx);
 972                 cmdbuf->bufsz -= sizeof(buf_idx);
 973         }
 974
 975         BEGIN_RING(2);
 976         OUT_RING( CP_PACKET0( RADEON_SCRATCH_REG0 + header.scratch.reg * 4, 0 ) );
 977         OUT_RING( dev_priv->scratch_ages[header.scratch.reg] );
 978         ADVANCE_RING();
 979
 980         return 0;
 981 }
 982
 983 /**
 984  * Uploads user-supplied vertex program instructions or parameters onto
 985  * the graphics card.
 986  * Called by r300_do_cp_cmdbuf.
 987  */
 988 static __inline__ int r300_emit_r500fp(drm_radeon_private_t *dev_priv,
 989                                        drm_radeon_kcmd_buffer_t *cmdbuf,
 990                                        drm_r300_cmd_header_t header)
 991 {
 992         int sz;
 993         int addr;
 994         int type;
 995         int clamp;
 996         int stride;
 997         RING_LOCALS;
 998
 999         sz = header.r500fp.count;
1000         /* address is 9 bits 0 - 8, bit 1 of flags is part of address */
1001         addr = ((header.r500fp.adrhi_flags & 1) << 8) | header.r500fp.adrlo;
1002
1003         type = !!(header.r500fp.adrhi_flags & R500FP_CONSTANT_TYPE);
1004         clamp = !!(header.r500fp.adrhi_flags & R500FP_CONSTANT_CLAMP);
1005
1006         addr |= (type << 16);
1007         addr |= (clamp << 17);
1008
1009         stride = type ? 4 : 6;
1010
1011         DRM_DEBUG("r500fp %d %d type: %d\n", sz, addr, type);
1012         if (!sz)
1013                 return 0;
1014         if (sz * stride * 4 > cmdbuf->bufsz)
1015                 return -EINVAL;
1016
1017         BEGIN_RING(3 + sz * stride);
1018         OUT_RING_REG(R500_GA_US_VECTOR_INDEX, addr);
1019         OUT_RING(CP_PACKET0_TABLE(R500_GA_US_VECTOR_DATA, sz * stride - 1));
1020         OUT_RING_TABLE((int *)cmdbuf->buf, sz * stride);
1021
1022         ADVANCE_RING();
1023
1024         cmdbuf->buf += sz * stride * 4;
1025         cmdbuf->bufsz -= sz * stride * 4;
1026
1027         return 0;
1028 }
1029
1030
1031 /**
1032  * Parses and validates a user-supplied command buffer and emits appropriate
1033  * commands on the DMA ring buffer.
1034  * Called by the ioctl handler function radeon_cp_cmdbuf.
1035  */
1036 int r300_do_cp_cmdbuf(struct drm_device *dev,
1037                       struct drm_file *file_priv,
1038                       drm_radeon_kcmd_buffer_t *cmdbuf)
1039 {
1040         drm_radeon_private_t *dev_priv = dev->dev_private;
1041         struct drm_device_dma *dma = dev->dma;
1042         struct drm_buf *buf = NULL;
1043         int emit_dispatch_age = 0;
1044         int ret = 0;
1045
1046         DRM_DEBUG("\n");
1047
1048         /* pacify */
1049         r300_pacify(dev_priv);
1050
1051         if (cmdbuf->nbox <= R300_SIMULTANEOUS_CLIPRECTS) {
1052                 ret = r300_emit_cliprects(dev_priv, cmdbuf, 0);
1053                 if (ret)
1054                         goto cleanup;
1055         }
1056
1057         while (cmdbuf->bufsz >= sizeof(drm_r300_cmd_header_t)) {
1058                 int idx;
1059                 drm_r300_cmd_header_t header;
1060
1061                 header.u = *(unsigned int *)cmdbuf->buf;
1062
1063                 cmdbuf->buf += sizeof(header);
1064                 cmdbuf->bufsz -= sizeof(header);
1065
1066                 switch (header.header.cmd_type) {
1067                 case R300_CMD_PACKET0:
1068                         ret = r300_emit_packet0(dev_priv, cmdbuf, header);
1069                         if (ret) {
1070                                 DRM_ERROR("r300_emit_packet0 failed\n");
1071                                 goto cleanup;
1072                         }
1073                         break;
1074
1075                 case R300_CMD_VPU:
1076                         DRM_DEBUG("R300_CMD_VPU\n");
1077                         ret = r300_emit_vpu(dev_priv, cmdbuf, header);
1078                         if (ret) {
1079                                 DRM_ERROR("r300_emit_vpu failed\n");
1080                                 goto cleanup;
1081                         }
1082                         break;
1083
1084                 case R300_CMD_PACKET3:
1085                         DRM_DEBUG("R300_CMD_PACKET3\n");
1086                         ret = r300_emit_packet3(dev_priv, cmdbuf, header);
1087                         if (ret) {
1088                                 DRM_ERROR("r300_emit_packet3 failed\n");
1089                                 goto cleanup;
1090                         }
1091                         break;
1092
1093                 case R300_CMD_END3D:
1094                         DRM_DEBUG("R300_CMD_END3D\n");
1095                         /* TODO:
1096                            Ideally userspace driver should not need to issue this call,
1097                            i.e. the drm driver should issue it automatically and prevent
1098                            lockups.
1099
1100                            In practice, we do not understand why this call is needed and what
1101                            it does (except for some vague guesses that it has to do with cache
1102                            coherence) and so the user space driver does it.
1103
1104                            Once we are sure which uses prevent lockups the code could be moved
1105                            into the kernel and the userspace driver will not
1106                            need to use this command.
1107
1108                            Note that issuing this command does not hurt anything
1109                            except, possibly, performance */
1110                         r300_pacify(dev_priv);
1111                         break;
1112
1113                 case R300_CMD_CP_DELAY:
1114                         /* simple enough, we can do it here */
1115                         DRM_DEBUG("R300_CMD_CP_DELAY\n");
1116                         {
1117                                 int i;
1118                                 RING_LOCALS;
1119
1120                                 BEGIN_RING(header.delay.count);
1121                                 for (i = 0; i < header.delay.count; i++)
1122                                         OUT_RING(RADEON_CP_PACKET2);
1123                                 ADVANCE_RING();
1124                         }
1125                         break;
1126
1127                 case R300_CMD_DMA_DISCARD:
1128                         DRM_DEBUG("RADEON_CMD_DMA_DISCARD\n");
1129                         idx = header.dma.buf_idx;
1130                         if (idx < 0 || idx >= dma->buf_count) {
1131                                 DRM_ERROR("buffer index %d (of %d max)\n",
1132                                           idx, dma->buf_count - 1);
1133                                 ret = -EINVAL;
1134                                 goto cleanup;
1135                         }
1136
1137                         buf = dma->buflist[idx];
1138                         if (buf->file_priv != file_priv || buf->pending) {
1139                                 DRM_ERROR("bad buffer %p %p %d\n",
1140                                           buf->file_priv, file_priv,
1141                                           buf->pending);
1142                                 ret = -EINVAL;
1143                                 goto cleanup;
1144                         }
1145
1146                         emit_dispatch_age = 1;
1147                         r300_discard_buffer(dev, buf);
1148                         break;
1149
1150                 case R300_CMD_WAIT:
1151                         DRM_DEBUG("R300_CMD_WAIT\n");
1152                         r300_cmd_wait(dev_priv, header);
1153                         break;
1154
1155                 case R300_CMD_SCRATCH:
1156                         DRM_DEBUG("R300_CMD_SCRATCH\n");
1157                         ret = r300_scratch(dev_priv, cmdbuf, header);
1158                         if (ret) {
1159                                 DRM_ERROR("r300_scratch failed\n");
1160                                 goto cleanup;
1161                         }
1162                         break;
1163
1164                 case R300_CMD_R500FP:
1165                         if ((dev_priv->flags & RADEON_FAMILY_MASK) < CHIP_RV515) {
1166                                 DRM_ERROR("Calling r500 command on r300 card\n");
1167                                 ret = -EINVAL;
1168                                 goto cleanup;
1169                         }
1170                         DRM_DEBUG("R300_CMD_R500FP\n");
1171                         ret = r300_emit_r500fp(dev_priv, cmdbuf, header);
1172                         if (ret) {
1173                                 DRM_ERROR("r300_emit_r500fp failed\n");
1174                                 goto cleanup;
1175                         }
1176                         break;
1177                 default:
1178                         DRM_ERROR("bad cmd_type %i at %p\n",
1179                                   header.header.cmd_type,
1180                                   cmdbuf->buf - sizeof(header));
1181                         ret = -EINVAL;
1182                         goto cleanup;
1183                 }
1184         }
1185
1186         DRM_DEBUG("END\n");
1187
1188       cleanup:
1189         r300_pacify(dev_priv);
1190
1191         /* We emit the vertex buffer age here, outside the pacifier "brackets"
1192          * for two reasons:
1193          *  (1) This may coalesce multiple age emissions into a single one and
1194          *  (2) more importantly, some chips lock up hard when scratch registers
1195          *      are written inside the pacifier bracket.
1196          */
1197         if (emit_dispatch_age) {
1198                 RING_LOCALS;
1199
1200                 /* Emit the vertex buffer age */
1201                 BEGIN_RING(2);
1202                 RADEON_DISPATCH_AGE(dev_priv->sarea_priv->last_dispatch);
1203                 ADVANCE_RING();
1204         }
1205
1206         COMMIT_RING();
1207
1208         return ret;
1209 }