sys/dev/drm/drm_bufs.h

   1 /* drm_bufs.h -- Generic buffer template -*- linux-c -*-
   2  * Created: Thu Nov 23 03:10:50 2000 by gareth@valinux.com
   3  *
   4  * Copyright 1999, 2000 Precision Insight, Inc., Cedar Park, Texas.
   5  * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California.
   6  * All Rights Reserved.
   7  *
   8  * Permission is hereby granted, free of charge, to any person obtaining a
   9  * copy of this software and associated documentation files (the "Software"),
  10  * to deal in the Software without restriction, including without limitation
  11  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  12  * and/or sell copies of the Software, and to permit persons to whom the
  13  * Software is furnished to do so, subject to the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the next
  16  * paragraph) shall be included in all copies or substantial portions of the
  17  * Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  20  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  21  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  22  * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  23  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  24  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  25  * OTHER DEALINGS IN THE SOFTWARE.
  26  *
  27  * Authors:
  28  *    Rickard E. (Rik) Faith <faith@valinux.com>
  29  *    Gareth Hughes <gareth@valinux.com>
  30  *
  31  * $FreeBSD$
  32  */
  33
  34 #define __NO_VERSION__
  35 #ifdef __linux__
  36 #include <linux/vmalloc.h>
  37 #endif /* __linux__ */
  38 #ifdef __FreeBSD__
  39 #include <machine/param.h>
  40 #include <sys/mman.h>
  41 #include <vm/vm.h>
  42 #include <vm/pmap.h>
  43 #include <vm/vm_extern.h>
  44 #include <vm/vm_map.h>
  45 #include <vm/vm_param.h>
  46 #endif /* __FreeBSD__ */
  47 #include "dev/drm/drmP.h"
  48
  49 #ifndef __HAVE_PCI_DMA
  50 #define __HAVE_PCI_DMA          0
  51 #endif
  52
  53 #ifndef __HAVE_SG
  54 #define __HAVE_SG               0
  55 #endif
  56
  57 #ifndef DRIVER_BUF_PRIV_T
  58 #define DRIVER_BUF_PRIV_T               u32
  59 #endif
  60 #ifndef DRIVER_AGP_BUFFERS_MAP
  61 #if __HAVE_AGP && __HAVE_DMA
  62 #error "You must define DRIVER_AGP_BUFFERS_MAP()"
  63 #else
  64 #define DRIVER_AGP_BUFFERS_MAP( dev )   NULL
  65 #endif
  66 #endif
  67
  68 #if __REALLY_HAVE_AGP
  69 int DRM(addbufs_agp)( DRM_OS_IOCTL );
  70 #endif
  71 #if __HAVE_PCI_DMA
  72 int DRM(addbufs_pci)( DRM_OS_IOCTL );
  73 #endif
  74 #if __REALLY_HAVE_SG
  75 int DRM(addbufs_sg)( DRM_OS_IOCTL );
  76 #endif
  77
  78 /*
  79  * Compute order.  Can be made faster.
  80  */
  81 int DRM(order)( unsigned long size )
  82 {
  83         int order;
  84         unsigned long tmp;
  85
  86         for ( order = 0, tmp = size ; tmp >>= 1 ; ++order );
  87
  88         if ( size & ~(1 << order) )
  89                 ++order;
  90
  91         return order;
  92 }
  93
  94 int DRM(addmap)( DRM_OS_IOCTL )
  95 {
  96         DRM_OS_DEVICE;
  97         drm_map_t *map;
  98 #ifdef __linux__
  99         drm_map_list_t *list;
 100 #endif /* __linux__ */
 101 #ifdef __FreeBSD__
 102         drm_map_list_entry_t *list;
 103 #endif /* __FreeBSD__ */
 104
 105 #ifdef __linux__
 106         if ( !(filp->f_mode & 3) )
 107 #endif /* __linux__ */
 108 #ifdef __FreeBSD__
 109         if (!(dev->flags & (FREAD|FWRITE)))
 110 #endif /* __FreeBSD__ */
 111                 return DRM_OS_ERR(EACCES); /* Require read/write */
 112
 113         map = (drm_map_t *) DRM(alloc)( sizeof(*map), DRM_MEM_MAPS );
 114         if ( !map )
 115                 return DRM_OS_ERR(ENOMEM);
 116
 117 #ifdef __linux__
 118         if ( copy_from_user( map, (drm_map_t *)data, sizeof(*map) ) ) {
 119                 DRM(free)( map, sizeof(*map), DRM_MEM_MAPS );
 120                 return DRM_OS_ERR(EFAULT);
 121         }
 122 #endif /* __linux__ */
 123 #ifdef __FreeBSD__
 124         *map = *(drm_map_t *)data;
 125 #endif /* __FreeBSD__ */
 126
 127         /* Only allow shared memory to be removable since we only keep enough
 128          * book keeping information about shared memory to allow for removal
 129          * when processes fork.
 130          */
 131         if ( (map->flags & _DRM_REMOVABLE) && map->type != _DRM_SHM ) {
 132                 DRM(free)( map, sizeof(*map), DRM_MEM_MAPS );
 133                 return DRM_OS_ERR(EINVAL);
 134         }
 135         DRM_DEBUG( "offset = 0x%08lx, size = 0x%08lx, type = %d\n",
 136                    map->offset, map->size, map->type );
 137 #ifdef __linux__
 138         if ( (map->offset & (~PAGE_MASK)) || (map->size & (~PAGE_MASK)) ) {
 139 #endif /* __linux__ */
 140 #ifdef __FreeBSD__
 141         if ( (map->offset & PAGE_MASK) || (map->size & PAGE_MASK) ) {
 142 #endif /* __FreeBSD__ */
 143                 DRM(free)( map, sizeof(*map), DRM_MEM_MAPS );
 144                 return DRM_OS_ERR(EINVAL);
 145         }
 146         map->mtrr   = -1;
 147         map->handle = 0;
 148
 149         switch ( map->type ) {
 150         case _DRM_REGISTERS:
 151         case _DRM_FRAME_BUFFER:
 152 #if !defined(__sparc__) && !defined(__alpha__)
 153                 if ( map->offset + map->size < map->offset
 154 #ifdef __linux__
 155                      || map->offset < virt_to_phys(high_memory)
 156 #endif /* __linux__ */
 157                 ) {
 158                         DRM(free)( map, sizeof(*map), DRM_MEM_MAPS );
 159                         return DRM_OS_ERR(EINVAL);
 160                 }
 161 #endif
 162 #ifdef __alpha__
 163                 map->offset += dev->hose->mem_space->start;
 164 #endif
 165 #if __REALLY_HAVE_MTRR
 166                 if ( map->type == _DRM_FRAME_BUFFER ||
 167                      (map->flags & _DRM_WRITE_COMBINING) ) {
 168                         map->mtrr = mtrr_add( map->offset, map->size,
 169                                               MTRR_TYPE_WRCOMB, 1 );
 170                 }
 171 #endif
 172                 map->handle = DRM(ioremap)( map->offset, map->size );
 173                 break;
 174
 175         case _DRM_SHM:
 176 #ifdef __linux__
 177                 map->handle = vmalloc_32(map->size);
 178 #endif /* __linux__ */
 179 #ifdef __FreeBSD__
 180                 map->handle = (void *)DRM(alloc_pages)
 181                         (DRM(order)(map->size) - PAGE_SHIFT, DRM_MEM_SAREA);
 182 #endif /* __FreeBSD__ */
 183                 DRM_DEBUG( "%ld %d %p\n",
 184                            map->size, DRM(order)( map->size ), map->handle );
 185                 if ( !map->handle ) {
 186                         DRM(free)( map, sizeof(*map), DRM_MEM_MAPS );
 187                         return DRM_OS_ERR(ENOMEM);
 188                 }
 189                 map->offset = (unsigned long)map->handle;
 190                 if ( map->flags & _DRM_CONTAINS_LOCK ) {
 191                         dev->lock.hw_lock = map->handle; /* Pointer to lock */
 192                 }
 193                 break;
 194 #if __REALLY_HAVE_AGP
 195         case _DRM_AGP:
 196 #ifdef __alpha__
 197                 map->offset += dev->hose->mem_space->start;
 198 #endif
 199                 map->offset += dev->agp->base;
 200                 map->mtrr   = dev->agp->agp_mtrr; /* for getmap */
 201                 break;
 202 #endif
 203         case _DRM_SCATTER_GATHER:
 204                 if (!dev->sg) {
 205                         DRM(free)(map, sizeof(*map), DRM_MEM_MAPS);
 206                         return DRM_OS_ERR(EINVAL);
 207                 }
 208                 map->offset = map->offset + dev->sg->handle;
 209                 break;
 210
 211         default:
 212                 DRM(free)( map, sizeof(*map), DRM_MEM_MAPS );
 213                 return DRM_OS_ERR(EINVAL);
 214         }
 215
 216         list = DRM(alloc)(sizeof(*list), DRM_MEM_MAPS);
 217         if(!list) {
 218                 DRM(free)(map, sizeof(*map), DRM_MEM_MAPS);
 219                 return DRM_OS_ERR(EINVAL);
 220         }
 221         memset(list, 0, sizeof(*list));
 222         list->map = map;
 223
 224         DRM_OS_LOCK;
 225 #ifdef __linux__
 226         list_add(&list->head, &dev->maplist->head);
 227 #endif /* __linux__ */
 228 #ifdef __FreeBSD__
 229         TAILQ_INSERT_TAIL(dev->maplist, list, link);
 230 #endif /* __FreeBSD__ */
 231         DRM_OS_UNLOCK;
 232
 233 #ifdef __linux__
 234         if ( copy_to_user( (drm_map_t *)data, map, sizeof(*map) ) )
 235                 return DRM_OS_ERR(EFAULT);
 236 #endif /* __linux__ */
 237 #ifdef __FreeBSD__
 238         *(drm_map_t *)data = *map;
 239 #endif /* __FreeBSD__ */
 240
 241         if ( map->type != _DRM_SHM ) {
 242 #ifdef __linux__
 243                 if ( copy_to_user( &((drm_map_t *)data)->handle,
 244                                    &map->offset,
 245                                    sizeof(map->offset) ) )
 246                         return DRM_OS_ERR(EFAULT);
 247 #endif /* __linux__ */
 248 #ifdef __FreeBSD__
 249                 ((drm_map_t *)data)->handle = (void *)map->offset;
 250 #endif /* __FreeBSD__ */
 251         }
 252         return 0;
 253 }
 254
 255
 256 /* Remove a map private from list and deallocate resources if the mapping
 257  * isn't in use.
 258  */
 259
 260 int DRM(rmmap)( DRM_OS_IOCTL )
 261 {
 262         DRM_OS_DEVICE;
 263 #ifdef __linux__
 264         struct list_head *list;
 265         drm_map_list_t *r_list = NULL;
 266         drm_vma_entry_t *pt, *prev;
 267 #endif /* __linux__ */
 268 #ifdef __FreeBSD__
 269         drm_map_list_entry_t *list;
 270 #endif /* __FreeBSD__ */
 271         drm_map_t *map;
 272         drm_map_t request;
 273         int found_maps = 0;
 274
 275         DRM_OS_KRNFROMUSR( request, (drm_map_t *)data, sizeof(request) );
 276
 277         DRM_OS_LOCK;
 278 #ifdef __linux__
 279         list = &dev->maplist->head;
 280         list_for_each(list, &dev->maplist->head) {
 281                 r_list = (drm_map_list_t *) list;
 282
 283                 if(r_list->map &&
 284                    r_list->map->handle == request.handle &&
 285                    r_list->map->flags & _DRM_REMOVABLE) break;
 286         }
 287
 288         /* List has wrapped around to the head pointer, or its empty we didn't
 289          * find anything.
 290          */
 291         if(list == (&dev->maplist->head)) {
 292                 DRM_OS_UNLOCK;
 293                 return DRM_OS_ERR(EINVAL);
 294         }
 295         map = r_list->map;
 296         list_del(list);
 297 #endif /* __linux__ */
 298 #ifdef __FreeBSD__
 299         TAILQ_FOREACH(list, dev->maplist, link) {
 300                 map = list->map;
 301                 if(map->handle == request.handle &&
 302                    map->flags & _DRM_REMOVABLE) break;
 303         }
 304
 305         /* List has wrapped around to the head pointer, or its empty we didn't
 306          * find anything.
 307          */
 308         if(list == NULL) {
 309                 DRM_OS_UNLOCK;
 310                 return DRM_OS_ERR(EINVAL);
 311         }
 312         TAILQ_REMOVE(dev->maplist, list, link);
 313 #endif /* __FreeBSD__ */
 314         DRM(free)(list, sizeof(*list), DRM_MEM_MAPS);
 315
 316 #ifdef __linux__
 317         for (pt = dev->vmalist, prev = NULL; pt; prev = pt, pt = pt->next) {
 318                 if (pt->vma->vm_private_data == map) found_maps++;
 319         }
 320 #endif /* __linux__ */
 321
 322         if(!found_maps) {
 323                 switch (map->type) {
 324                 case _DRM_REGISTERS:
 325                 case _DRM_FRAME_BUFFER:
 326 #if __REALLY_HAVE_MTRR
 327                         if (map->mtrr >= 0) {
 328                                 int retcode;
 329                                 retcode = mtrr_del(map->mtrr,
 330                                                    map->offset,
 331                                                    map->size);
 332                                 DRM_DEBUG("mtrr_del = %d\n", retcode);
 333                         }
 334 #endif
 335                         DRM(ioremapfree)(map->handle, map->size);
 336                         break;
 337                 case _DRM_SHM:
 338 #ifdef __linux__
 339                         vfree(map->handle);
 340 #endif /* __linux__ */
 341 #ifdef __FreeBSD__
 342                         DRM(free_pages)( (unsigned long)map->handle, DRM(order)(map->size), DRM_MEM_SAREA );
 343 #endif /* __FreeBSD__ */
 344                         break;
 345                 case _DRM_AGP:
 346                 case _DRM_SCATTER_GATHER:
 347                         break;
 348                 }
 349                 DRM(free)(map, sizeof(*map), DRM_MEM_MAPS);
 350         }
 351         DRM_OS_UNLOCK;
 352         return 0;
 353 }
 354
 355 #if __HAVE_DMA
 356
 357
 358 static void DRM(cleanup_buf_error)(drm_buf_entry_t *entry)
 359 {
 360         int i;
 361
 362         if (entry->seg_count) {
 363                 for (i = 0; i < entry->seg_count; i++) {
 364                         DRM(free_pages)(entry->seglist[i],
 365                                         entry->page_order,
 366                                         DRM_MEM_DMA);
 367                 }
 368                 DRM(free)(entry->seglist,
 369                           entry->seg_count *
 370                           sizeof(*entry->seglist),
 371                           DRM_MEM_SEGS);
 372
 373                 entry->seg_count = 0;
 374         }
 375
 376         if(entry->buf_count) {
 377                 for(i = 0; i < entry->buf_count; i++) {
 378                         if(entry->buflist[i].dev_private) {
 379                                 DRM(free)(entry->buflist[i].dev_private,
 380                                           entry->buflist[i].dev_priv_size,
 381                                           DRM_MEM_BUFS);
 382                         }
 383                 }
 384                 DRM(free)(entry->buflist,
 385                           entry->buf_count *
 386                           sizeof(*entry->buflist),
 387                           DRM_MEM_BUFS);
 388
 389 #if __HAVE_DMA_FREELIST
 390                 DRM(freelist_destroy)(&entry->freelist);
 391 #endif
 392
 393                 entry->buf_count = 0;
 394         }
 395 }
 396
 397 #if __REALLY_HAVE_AGP
 398 int DRM(addbufs_agp)( DRM_OS_IOCTL )
 399 {
 400         DRM_OS_DEVICE;
 401         drm_device_dma_t *dma = dev->dma;
 402         drm_buf_desc_t request;
 403         drm_buf_entry_t *entry;
 404         drm_buf_t *buf;
 405         unsigned long offset;
 406         unsigned long agp_offset;
 407         int count;
 408         int order;
 409         int size;
 410         int alignment;
 411         int page_order;
 412         int total;
 413         int byte_count;
 414         int i;
 415         drm_buf_t **temp_buflist;
 416
 417         if ( !dma ) return DRM_OS_ERR(EINVAL);
 418
 419         DRM_OS_KRNFROMUSR( request, (drm_buf_desc_t *)data, sizeof(request) );
 420
 421         count = request.count;
 422         order = DRM(order)( request.size );
 423         size = 1 << order;
 424
 425         alignment  = (request.flags & _DRM_PAGE_ALIGN)
 426 #ifdef __linux__
 427                 ? PAGE_ALIGN(size) : size;
 428 #endif /* __linux__ */
 429 #ifdef __FreeBSD__
 430                 ? round_page(size) : size;
 431 #endif /* __FreeBSD__ */
 432         page_order = order - PAGE_SHIFT > 0 ? order - PAGE_SHIFT : 0;
 433         total = PAGE_SIZE << page_order;
 434
 435         byte_count = 0;
 436         agp_offset = dev->agp->base + request.agp_start;
 437
 438         DRM_DEBUG( "count:      %d\n",  count );
 439         DRM_DEBUG( "order:      %d\n",  order );
 440         DRM_DEBUG( "size:       %d\n",  size );
 441         DRM_DEBUG( "agp_offset: 0x%lx\n", agp_offset );
 442         DRM_DEBUG( "alignment:  %d\n",  alignment );
 443         DRM_DEBUG( "page_order: %d\n",  page_order );
 444         DRM_DEBUG( "total:      %d\n",  total );
 445
 446         if ( order < DRM_MIN_ORDER || order > DRM_MAX_ORDER )
 447                 return DRM_OS_ERR(EINVAL);
 448         if ( dev->queue_count )
 449                 return DRM_OS_ERR(EBUSY); /* Not while in use */
 450
 451         DRM_OS_SPINLOCK( &dev->count_lock );
 452         if ( dev->buf_use ) {
 453                 DRM_OS_SPINUNLOCK( &dev->count_lock );
 454                 return DRM_OS_ERR(EBUSY);
 455         }
 456         atomic_inc( &dev->buf_alloc );
 457         DRM_OS_SPINUNLOCK( &dev->count_lock );
 458
 459         DRM_OS_LOCK;
 460         entry = &dma->bufs[order];
 461         if ( entry->buf_count ) {
 462                 DRM_OS_UNLOCK;
 463                 atomic_dec( &dev->buf_alloc );
 464                 return DRM_OS_ERR(ENOMEM); /* May only call once for each order */
 465         }
 466
 467         if (count < 0 || count > 4096) {
 468                 DRM_OS_UNLOCK;
 469                 atomic_dec( &dev->buf_alloc );
 470                 return DRM_OS_ERR(EINVAL);
 471         }
 472
 473         entry->buflist = DRM(alloc)( count * sizeof(*entry->buflist),
 474                                     DRM_MEM_BUFS );
 475         if ( !entry->buflist ) {
 476                 DRM_OS_UNLOCK;
 477                 atomic_dec( &dev->buf_alloc );
 478                 return DRM_OS_ERR(ENOMEM);
 479         }
 480         memset( entry->buflist, 0, count * sizeof(*entry->buflist) );
 481
 482         entry->buf_size = size;
 483         entry->page_order = page_order;
 484
 485         offset = 0;
 486
 487         while ( entry->buf_count < count ) {
 488                 buf          = &entry->buflist[entry->buf_count];
 489                 buf->idx     = dma->buf_count + entry->buf_count;
 490                 buf->total   = alignment;
 491                 buf->order   = order;
 492                 buf->used    = 0;
 493
 494                 buf->offset  = (dma->byte_count + offset);
 495                 buf->bus_address = agp_offset + offset;
 496                 buf->address = (void *)(agp_offset + offset);
 497                 buf->next    = NULL;
 498                 buf->waiting = 0;
 499                 buf->pending = 0;
 500 #ifdef __linux__
 501                 init_waitqueue_head( &buf->dma_wait );
 502 #endif /* __linux__ */
 503 #ifdef __FreeBSD__
 504                 buf->dma_wait = 0;
 505 #endif /* __FreeBSD__ */
 506                 buf->pid     = 0;
 507
 508                 buf->dev_priv_size = sizeof(DRIVER_BUF_PRIV_T);
 509                 buf->dev_private = DRM(alloc)( sizeof(DRIVER_BUF_PRIV_T),
 510                                                DRM_MEM_BUFS );
 511                 if(!buf->dev_private) {
 512                         /* Set count correctly so we free the proper amount. */
 513                         entry->buf_count = count;
 514                         DRM(cleanup_buf_error)(entry);
 515                 }
 516                 memset( buf->dev_private, 0, buf->dev_priv_size );
 517
 518 #if __HAVE_DMA_HISTOGRAM
 519                 buf->time_queued = 0;
 520                 buf->time_dispatched = 0;
 521                 buf->time_completed = 0;
 522                 buf->time_freed = 0;
 523 #endif
 524
 525                 offset += alignment;
 526                 entry->buf_count++;
 527                 byte_count += PAGE_SIZE << page_order;
 528         }
 529
 530         DRM_DEBUG( "byte_count: %d\n", byte_count );
 531
 532         temp_buflist = DRM(realloc)( dma->buflist,
 533                                      dma->buf_count * sizeof(*dma->buflist),
 534                                      (dma->buf_count + entry->buf_count)
 535                                      * sizeof(*dma->buflist),
 536                                      DRM_MEM_BUFS );
 537         if(!temp_buflist) {
 538                 /* Free the entry because it isn't valid */
 539                 DRM(cleanup_buf_error)(entry);
 540                 DRM_OS_UNLOCK;
 541                 atomic_dec( &dev->buf_alloc );
 542                 return DRM_OS_ERR(ENOMEM);
 543         }
 544         dma->buflist = temp_buflist;
 545
 546         for ( i = 0 ; i < entry->buf_count ; i++ ) {
 547                 dma->buflist[i + dma->buf_count] = &entry->buflist[i];
 548         }
 549
 550         dma->buf_count += entry->buf_count;
 551         dma->byte_count += byte_count;
 552
 553         DRM_DEBUG( "dma->buf_count : %d\n", dma->buf_count );
 554         DRM_DEBUG( "entry->buf_count : %d\n", entry->buf_count );
 555
 556 #if __HAVE_DMA_FREELIST
 557         DRM(freelist_create)( &entry->freelist, entry->buf_count );
 558         for ( i = 0 ; i < entry->buf_count ; i++ ) {
 559                 DRM(freelist_put)( dev, &entry->freelist, &entry->buflist[i] );
 560         }
 561 #endif
 562         DRM_OS_UNLOCK;
 563
 564         request.count = entry->buf_count;
 565         request.size = size;
 566
 567         DRM_OS_KRNTOUSR( (drm_buf_desc_t *)data, request, sizeof(request) );
 568
 569         dma->flags = _DRM_DMA_USE_AGP;
 570
 571         atomic_dec( &dev->buf_alloc );
 572         return 0;
 573 }
 574 #endif /* __REALLY_HAVE_AGP */
 575
 576 #if __HAVE_PCI_DMA
 577 int DRM(addbufs_pci)( DRM_OS_IOCTL )
 578 {
 579         DRM_OS_DEVICE;
 580         drm_device_dma_t *dma = dev->dma;
 581         drm_buf_desc_t request;
 582         int count;
 583         int order;
 584         int size;
 585         int total;
 586         int page_order;
 587         drm_buf_entry_t *entry;
 588         unsigned long page;
 589         drm_buf_t *buf;
 590         int alignment;
 591         unsigned long offset;
 592         int i;
 593         int byte_count;
 594         int page_count;
 595         unsigned long *temp_pagelist;
 596         drm_buf_t **temp_buflist;
 597
 598         if ( !dma ) return DRM_OS_ERR(EINVAL);
 599
 600         DRM_OS_KRNFROMUSR( request, (drm_buf_desc_t *)data, sizeof(request) );
 601
 602         count = request.count;
 603         order = DRM(order)( request.size );
 604         size = 1 << order;
 605
 606         DRM_DEBUG( "count=%d, size=%d (%d), order=%d, queue_count=%d\n",
 607                    request.count, request.size, size,
 608                    order, dev->queue_count );
 609
 610         if ( order < DRM_MIN_ORDER || order > DRM_MAX_ORDER )
 611                 return DRM_OS_ERR(EINVAL);
 612         if ( dev->queue_count )
 613                 return DRM_OS_ERR(EBUSY); /* Not while in use */
 614
 615         alignment = (request.flags & _DRM_PAGE_ALIGN)
 616 #ifdef __linux__
 617                 ? PAGE_ALIGN(size) : size;
 618 #endif /* __linux__ */
 619 #ifdef __FreeBSD__
 620                 ? round_page(size) : size;
 621 #endif /* __FreeBSD__ */
 622         page_order = order - PAGE_SHIFT > 0 ? order - PAGE_SHIFT : 0;
 623         total = PAGE_SIZE << page_order;
 624
 625         DRM_OS_SPINLOCK( &dev->count_lock );
 626         if ( dev->buf_use ) {
 627                 DRM_OS_SPINUNLOCK( &dev->count_lock );
 628                 return DRM_OS_ERR(EBUSY);
 629         }
 630         atomic_inc( &dev->buf_alloc );
 631         DRM_OS_SPINUNLOCK( &dev->count_lock );
 632
 633         DRM_OS_LOCK;
 634         entry = &dma->bufs[order];
 635         if ( entry->buf_count ) {
 636                 DRM_OS_UNLOCK;
 637                 atomic_dec( &dev->buf_alloc );
 638                 return DRM_OS_ERR(ENOMEM);      /* May only call once for each order */
 639         }
 640
 641         if (count < 0 || count > 4096) {
 642                 DRM_OS_UNLOCK;
 643                 atomic_dec( &dev->buf_alloc );
 644                 return DRM_OS_ERR(EINVAL);
 645         }
 646
 647         entry->buflist = DRM(alloc)( count * sizeof(*entry->buflist),
 648                                     DRM_MEM_BUFS );
 649         if ( !entry->buflist ) {
 650                 DRM_OS_UNLOCK;
 651                 atomic_dec( &dev->buf_alloc );
 652                 return DRM_OS_ERR(ENOMEM);
 653         }
 654         memset( entry->buflist, 0, count * sizeof(*entry->buflist) );
 655
 656         entry->seglist = DRM(alloc)( count * sizeof(*entry->seglist),
 657                                     DRM_MEM_SEGS );
 658         if ( !entry->seglist ) {
 659                 DRM(free)( entry->buflist,
 660                           count * sizeof(*entry->buflist),
 661                           DRM_MEM_BUFS );
 662                 DRM_OS_UNLOCK;
 663                 atomic_dec( &dev->buf_alloc );
 664                 return DRM_OS_ERR(ENOMEM);
 665         }
 666         memset( entry->seglist, 0, count * sizeof(*entry->seglist) );
 667
 668         temp_pagelist = DRM(realloc)( dma->pagelist,
 669                                       dma->page_count * sizeof(*dma->pagelist),
 670                                       (dma->page_count + (count << page_order))
 671                                       * sizeof(*dma->pagelist),
 672                                       DRM_MEM_PAGES );
 673         if(!temp_pagelist) {
 674                 DRM(free)( entry->buflist,
 675                            count * sizeof(*entry->buflist),
 676                            DRM_MEM_BUFS );
 677                 DRM(free)( entry->seglist,
 678                            count * sizeof(*entry->seglist),
 679                            DRM_MEM_SEGS );
 680                 DRM_OS_UNLOCK;
 681                 atomic_dec( &dev->buf_alloc );
 682                 return DRM_OS_ERR(ENOMEM);
 683         }
 684
 685         dma->pagelist = temp_pagelist;
 686         DRM_DEBUG( "pagelist: %d entries\n",
 687                    dma->page_count + (count << page_order) );
 688
 689         entry->buf_size = size;
 690         entry->page_order = page_order;
 691         byte_count = 0;
 692         page_count = 0;
 693
 694         while ( entry->buf_count < count ) {
 695                 page = DRM(alloc_pages)( page_order, DRM_MEM_DMA );
 696                 if ( !page ) break;
 697                 entry->seglist[entry->seg_count++] = page;
 698                 for ( i = 0 ; i < (1 << page_order) ; i++ ) {
 699                         DRM_DEBUG( "page %d @ 0x%08lx\n",
 700                                    dma->page_count + page_count,
 701                                    page + PAGE_SIZE * i );
 702                         dma->pagelist[dma->page_count + page_count++]
 703                                 = page + PAGE_SIZE * i;
 704                 }
 705                 for ( offset = 0 ;
 706                       offset + size <= total && entry->buf_count < count ;
 707                       offset += alignment, ++entry->buf_count ) {
 708                         buf          = &entry->buflist[entry->buf_count];
 709                         buf->idx     = dma->buf_count + entry->buf_count;
 710                         buf->total   = alignment;
 711                         buf->order   = order;
 712                         buf->used    = 0;
 713                         buf->offset  = (dma->byte_count + byte_count + offset);
 714                         buf->address = (void *)(page + offset);
 715                         buf->next    = NULL;
 716                         buf->waiting = 0;
 717                         buf->pending = 0;
 718 #ifdef __linux__
 719                         init_waitqueue_head( &buf->dma_wait );
 720 #endif /* __linux__ */
 721 #ifdef __FreeBSD__
 722                         buf->dma_wait = 0;
 723 #endif /* __FreeBSD__ */
 724                         buf->pid     = 0;
 725 #if __HAVE_DMA_HISTOGRAM
 726                         buf->time_queued     = 0;
 727                         buf->time_dispatched = 0;
 728                         buf->time_completed  = 0;
 729                         buf->time_freed      = 0;
 730 #endif
 731                         DRM_DEBUG( "buffer %d @ %p\n",
 732                                    entry->buf_count, buf->address );
 733                 }
 734                 byte_count += PAGE_SIZE << page_order;
 735         }
 736
 737         temp_buflist = DRM(realloc)( dma->buflist,
 738                                      dma->buf_count * sizeof(*dma->buflist),
 739                                      (dma->buf_count + entry->buf_count)
 740                                      * sizeof(*dma->buflist),
 741                                      DRM_MEM_BUFS );
 742         if(!temp_buflist) {
 743                 /* Free the entry because it isn't valid */
 744                 DRM(cleanup_buf_error)(entry);
 745                 DRM_OS_UNLOCK;
 746                 atomic_dec( &dev->buf_alloc );
 747                 return DRM_OS_ERR(ENOMEM);
 748         }
 749         dma->buflist = temp_buflist;
 750
 751         for ( i = 0 ; i < entry->buf_count ; i++ ) {
 752                 dma->buflist[i + dma->buf_count] = &entry->buflist[i];
 753         }
 754
 755         dma->buf_count += entry->buf_count;
 756         dma->seg_count += entry->seg_count;
 757         dma->page_count += entry->seg_count << page_order;
 758         dma->byte_count += PAGE_SIZE * (entry->seg_count << page_order);
 759
 760 #if __HAVE_DMA_FREELIST
 761         DRM(freelist_create)( &entry->freelist, entry->buf_count );
 762         for ( i = 0 ; i < entry->buf_count ; i++ ) {
 763                 DRM(freelist_put)( dev, &entry->freelist, &entry->buflist[i] );
 764         }
 765 #endif
 766         DRM_OS_UNLOCK;
 767
 768         request.count = entry->buf_count;
 769         request.size = size;
 770
 771         DRM_OS_KRNTOUSR( (drm_buf_desc_t *)data, request, sizeof(request) );
 772
 773         atomic_dec( &dev->buf_alloc );
 774         return 0;
 775
 776 }
 777 #endif /* __HAVE_PCI_DMA */
 778
 779 #if __REALLY_HAVE_SG
 780 int DRM(addbufs_sg)( DRM_OS_IOCTL )
 781 {
 782         DRM_OS_DEVICE;
 783         drm_device_dma_t *dma = dev->dma;
 784         drm_buf_desc_t request;
 785         drm_buf_entry_t *entry;
 786         drm_buf_t *buf;
 787         unsigned long offset;
 788         unsigned long agp_offset;
 789         int count;
 790         int order;
 791         int size;
 792         int alignment;
 793         int page_order;
 794         int total;
 795         int byte_count;
 796         int i;
 797         drm_buf_t **temp_buflist;
 798
 799         if ( !dma ) return DRM_OS_ERR(EINVAL);
 800
 801         DRM_OS_KRNFROMUSR( request, (drm_buf_desc_t *)data, sizeof(request) );
 802
 803         count = request.count;
 804         order = DRM(order)( request.size );
 805         size = 1 << order;
 806
 807         alignment  = (request.flags & _DRM_PAGE_ALIGN)
 808 #ifdef __linux__
 809                ? PAGE_ALIGN(size) : size;
 810 #endif /* __linux__ */
 811 #ifdef __FreeBSD__
 812                 ? round_page(size) : size;
 813 #endif /* __FreeBSD__ */
 814         page_order = order - PAGE_SHIFT > 0 ? order - PAGE_SHIFT : 0;
 815         total = PAGE_SIZE << page_order;
 816
 817         byte_count = 0;
 818         agp_offset = request.agp_start;
 819
 820         DRM_DEBUG( "count:      %d\n",  count );
 821         DRM_DEBUG( "order:      %d\n",  order );
 822         DRM_DEBUG( "size:       %d\n",  size );
 823         DRM_DEBUG( "agp_offset: %ld\n", agp_offset );
 824         DRM_DEBUG( "alignment:  %d\n",  alignment );
 825         DRM_DEBUG( "page_order: %d\n",  page_order );
 826         DRM_DEBUG( "total:      %d\n",  total );
 827
 828         if ( order < DRM_MIN_ORDER || order > DRM_MAX_ORDER )
 829                 return DRM_OS_ERR(EINVAL);
 830         if ( dev->queue_count ) return DRM_OS_ERR(EBUSY); /* Not while in use */
 831
 832         DRM_OS_SPINLOCK( &dev->count_lock );
 833         if ( dev->buf_use ) {
 834                 DRM_OS_SPINUNLOCK( &dev->count_lock );
 835                 return DRM_OS_ERR(EBUSY);
 836         }
 837         atomic_inc( &dev->buf_alloc );
 838         DRM_OS_SPINUNLOCK( &dev->count_lock );
 839
 840         DRM_OS_LOCK;
 841         entry = &dma->bufs[order];
 842         if ( entry->buf_count ) {
 843                 DRM_OS_UNLOCK;
 844                 atomic_dec( &dev->buf_alloc );
 845                 return DRM_OS_ERR(ENOMEM); /* May only call once for each order */
 846         }
 847
 848         if (count < 0 || count > 4096) {
 849                 DRM_OS_UNLOCK;
 850                 atomic_dec( &dev->buf_alloc );
 851                 return DRM_OS_ERR(EINVAL);
 852         }
 853
 854         entry->buflist = DRM(alloc)( count * sizeof(*entry->buflist),
 855                                      DRM_MEM_BUFS );
 856         if ( !entry->buflist ) {
 857                 DRM_OS_UNLOCK;
 858                 atomic_dec( &dev->buf_alloc );
 859                 return DRM_OS_ERR(ENOMEM);
 860         }
 861         memset( entry->buflist, 0, count * sizeof(*entry->buflist) );
 862
 863         entry->buf_size = size;
 864         entry->page_order = page_order;
 865
 866         offset = 0;
 867
 868         while ( entry->buf_count < count ) {
 869                 buf          = &entry->buflist[entry->buf_count];
 870                 buf->idx     = dma->buf_count + entry->buf_count;
 871                 buf->total   = alignment;
 872                 buf->order   = order;
 873                 buf->used    = 0;
 874
 875                 buf->offset  = (dma->byte_count + offset);
 876                 buf->bus_address = agp_offset + offset;
 877                 buf->address = (void *)(agp_offset + offset + dev->sg->handle);
 878                 buf->next    = NULL;
 879                 buf->waiting = 0;
 880                 buf->pending = 0;
 881 #ifdef __linux__
 882                 init_waitqueue_head( &buf->dma_wait );
 883 #endif /* __linux__ */
 884 #ifdef __FreeBSD__
 885                 buf->dma_wait = 0;
 886 #endif /* __FreeBSD__ */
 887                 buf->pid     = 0;
 888
 889                 buf->dev_priv_size = sizeof(DRIVER_BUF_PRIV_T);
 890                 buf->dev_private = DRM(alloc)( sizeof(DRIVER_BUF_PRIV_T),
 891                                                DRM_MEM_BUFS );
 892                 if(!buf->dev_private) {
 893                         /* Set count correctly so we free the proper amount. */
 894                         entry->buf_count = count;
 895                         DRM(cleanup_buf_error)(entry);
 896                         DRM_OS_UNLOCK;
 897                         atomic_dec( &dev->buf_alloc );
 898                         return DRM_OS_ERR(ENOMEM);
 899                 }
 900
 901                 memset( buf->dev_private, 0, buf->dev_priv_size );
 902
 903 # if __HAVE_DMA_HISTOGRAM
 904                 buf->time_queued = 0;
 905                 buf->time_dispatched = 0;
 906                 buf->time_completed = 0;
 907                 buf->time_freed = 0;
 908 # endif
 909                 DRM_DEBUG( "buffer %d @ %p\n",
 910                            entry->buf_count, buf->address );
 911
 912                 offset += alignment;
 913                 entry->buf_count++;
 914                 byte_count += PAGE_SIZE << page_order;
 915         }
 916
 917         DRM_DEBUG( "byte_count: %d\n", byte_count );
 918
 919         temp_buflist = DRM(realloc)( dma->buflist,
 920                                      dma->buf_count * sizeof(*dma->buflist),
 921                                      (dma->buf_count + entry->buf_count)
 922                                      * sizeof(*dma->buflist),
 923                                      DRM_MEM_BUFS );
 924         if(!temp_buflist) {
 925                 /* Free the entry because it isn't valid */
 926                 DRM(cleanup_buf_error)(entry);
 927                 DRM_OS_UNLOCK;
 928                 atomic_dec( &dev->buf_alloc );
 929                 return DRM_OS_ERR(ENOMEM);
 930         }
 931         dma->buflist = temp_buflist;
 932
 933         for ( i = 0 ; i < entry->buf_count ; i++ ) {
 934                 dma->buflist[i + dma->buf_count] = &entry->buflist[i];
 935         }
 936
 937         dma->buf_count += entry->buf_count;
 938         dma->byte_count += byte_count;
 939
 940         DRM_DEBUG( "dma->buf_count : %d\n", dma->buf_count );
 941         DRM_DEBUG( "entry->buf_count : %d\n", entry->buf_count );
 942
 943 #if __HAVE_DMA_FREELIST
 944         DRM(freelist_create)( &entry->freelist, entry->buf_count );
 945         for ( i = 0 ; i < entry->buf_count ; i++ ) {
 946                 DRM(freelist_put)( dev, &entry->freelist, &entry->buflist[i] );
 947         }
 948 #endif
 949         DRM_OS_UNLOCK;
 950
 951         request.count = entry->buf_count;
 952         request.size = size;
 953
 954         DRM_OS_KRNTOUSR( (drm_buf_desc_t *)data, request, sizeof(request) );
 955
 956         dma->flags = _DRM_DMA_USE_SG;
 957
 958         atomic_dec( &dev->buf_alloc );
 959         return 0;
 960 }
 961 #endif /* __REALLY_HAVE_SG */
 962
 963 int DRM(addbufs)( DRM_OS_IOCTL )
 964 {
 965         drm_buf_desc_t request;
 966
 967         DRM_OS_KRNFROMUSR( request, (drm_buf_desc_t *)data, sizeof(request) );
 968
 969 #if __REALLY_HAVE_AGP
 970         if ( request.flags & _DRM_AGP_BUFFER )
 971                 return DRM(addbufs_agp)( IOCTL_ARGS_PASS );
 972         else
 973 #endif
 974 #if __REALLY_HAVE_SG
 975         if ( request.flags & _DRM_SG_BUFFER )
 976                 return DRM(addbufs_sg)( IOCTL_ARGS_PASS );
 977         else
 978 #endif
 979 #if __HAVE_PCI_DMA
 980                 return DRM(addbufs_pci)( IOCTL_ARGS_PASS );
 981 #else
 982                 return DRM_OS_ERR(EINVAL);
 983 #endif
 984 }
 985
 986 int DRM(infobufs)( DRM_OS_IOCTL )
 987 {
 988         DRM_OS_DEVICE;
 989         drm_device_dma_t *dma = dev->dma;
 990         drm_buf_info_t request;
 991         int i;
 992         int count;
 993
 994         if ( !dma ) return DRM_OS_ERR(EINVAL);
 995
 996         DRM_OS_SPINLOCK( &dev->count_lock );
 997         if ( atomic_read( &dev->buf_alloc ) ) {
 998                 DRM_OS_SPINUNLOCK( &dev->count_lock );
 999                 return DRM_OS_ERR(EBUSY);
1000         }
1001         ++dev->buf_use;         /* Can't allocate more after this call */
1002         DRM_OS_SPINUNLOCK( &dev->count_lock );
1003
1004         DRM_OS_KRNFROMUSR( request, (drm_buf_info_t *)data, sizeof(request) );
1005
1006         for ( i = 0, count = 0 ; i < DRM_MAX_ORDER + 1 ; i++ ) {
1007                 if ( dma->bufs[i].buf_count ) ++count;
1008         }
1009
1010         DRM_DEBUG( "count = %d\n", count );
1011
1012         if ( request.count >= count ) {
1013                 for ( i = 0, count = 0 ; i < DRM_MAX_ORDER + 1 ; i++ ) {
1014                         if ( dma->bufs[i].buf_count ) {
1015                                 drm_buf_desc_t *to = &request.list[count];
1016                                 drm_buf_entry_t *from = &dma->bufs[i];
1017                                 drm_freelist_t *list = &dma->bufs[i].freelist;
1018                                 if ( DRM_OS_COPYTOUSR( &to->count,
1019                                                    &from->buf_count,
1020                                                    sizeof(from->buf_count) ) ||
1021                                      DRM_OS_COPYTOUSR( &to->size,
1022                                                    &from->buf_size,
1023                                                    sizeof(from->buf_size) ) ||
1024                                      DRM_OS_COPYTOUSR( &to->low_mark,
1025                                                    &list->low_mark,
1026                                                    sizeof(list->low_mark) ) ||
1027                                      DRM_OS_COPYTOUSR( &to->high_mark,
1028                                                    &list->high_mark,
1029                                                    sizeof(list->high_mark) ) )
1030                                         return DRM_OS_ERR(EFAULT);
1031
1032                                 DRM_DEBUG( "%d %d %d %d %d\n",
1033                                            i,
1034                                            dma->bufs[i].buf_count,
1035                                            dma->bufs[i].buf_size,
1036                                            dma->bufs[i].freelist.low_mark,
1037                                            dma->bufs[i].freelist.high_mark );
1038                                 ++count;
1039                         }
1040                 }
1041         }
1042         request.count = count;
1043
1044         DRM_OS_KRNTOUSR( (drm_buf_info_t *)data, request, sizeof(request) );
1045
1046         return 0;
1047 }
1048
1049 int DRM(markbufs)( DRM_OS_IOCTL )
1050 {
1051         DRM_OS_DEVICE;
1052         drm_device_dma_t *dma = dev->dma;
1053         drm_buf_desc_t request;
1054         int order;
1055         drm_buf_entry_t *entry;
1056
1057         if ( !dma ) return DRM_OS_ERR(EINVAL);
1058
1059         DRM_OS_KRNFROMUSR( request, (drm_buf_desc_t *)data, sizeof(request) );
1060
1061         DRM_DEBUG( "%d, %d, %d\n",
1062                    request.size, request.low_mark, request.high_mark );
1063         order = DRM(order)( request.size );
1064         if ( order < DRM_MIN_ORDER || order > DRM_MAX_ORDER )
1065                 return DRM_OS_ERR(EINVAL);
1066         entry = &dma->bufs[order];
1067
1068         if ( request.low_mark < 0 || request.low_mark > entry->buf_count )
1069                 return DRM_OS_ERR(EINVAL);
1070         if ( request.high_mark < 0 || request.high_mark > entry->buf_count )
1071                 return DRM_OS_ERR(EINVAL);
1072
1073         entry->freelist.low_mark  = request.low_mark;
1074         entry->freelist.high_mark = request.high_mark;
1075
1076         return 0;
1077 }
1078
1079 int DRM(freebufs)( DRM_OS_IOCTL )
1080 {
1081         DRM_OS_DEVICE;
1082         drm_device_dma_t *dma = dev->dma;
1083         drm_buf_free_t request;
1084         int i;
1085         int idx;
1086         drm_buf_t *buf;
1087
1088         if ( !dma ) return DRM_OS_ERR(EINVAL);
1089
1090         DRM_OS_KRNFROMUSR( request, (drm_buf_free_t *)data, sizeof(request) );
1091
1092         DRM_DEBUG( "%d\n", request.count );
1093         for ( i = 0 ; i < request.count ; i++ ) {
1094                 if ( DRM_OS_COPYFROMUSR( &idx,
1095                                      &request.list[i],
1096                                      sizeof(idx) ) )
1097                         return DRM_OS_ERR(EFAULT);
1098                 if ( idx < 0 || idx >= dma->buf_count ) {
1099                         DRM_ERROR( "Index %d (of %d max)\n",
1100                                    idx, dma->buf_count - 1 );
1101                         return DRM_OS_ERR(EINVAL);
1102                 }
1103                 buf = dma->buflist[idx];
1104                 if ( buf->pid != DRM_OS_CURRENTPID ) {
1105                         DRM_ERROR( "Process %d freeing buffer owned by %d\n",
1106                                    DRM_OS_CURRENTPID, buf->pid );
1107                         return DRM_OS_ERR(EINVAL);
1108                 }
1109                 DRM(free_buffer)( dev, buf );
1110         }
1111
1112         return 0;
1113 }
1114
1115 int DRM(mapbufs)( DRM_OS_IOCTL )
1116 {
1117         DRM_OS_DEVICE;
1118         drm_device_dma_t *dma = dev->dma;
1119         int retcode = 0;
1120         const int zero = 0;
1121 #ifdef __linux__
1122         unsigned long virtual, address;
1123 #endif /* __linux__ */
1124 #ifdef __FreeBSD__
1125         vm_offset_t virtual, address;
1126 #if __FreeBSD_version >= 500000
1127         struct vmspace *vms = p->td_proc->p_vmspace;
1128 #else
1129         struct vmspace *vms = p->p_vmspace;
1130 #endif
1131 #endif /* __FreeBSD__ */
1132         drm_buf_map_t request;
1133         int i;
1134
1135         if ( !dma ) return DRM_OS_ERR(EINVAL);
1136
1137         DRM_OS_SPINLOCK( &dev->count_lock );
1138         if ( atomic_read( &dev->buf_alloc ) ) {
1139                 DRM_OS_SPINUNLOCK( &dev->count_lock );
1140                 return DRM_OS_ERR(EBUSY);
1141         }
1142         dev->buf_use++;         /* Can't allocate more after this call */
1143         DRM_OS_SPINUNLOCK( &dev->count_lock );
1144
1145         DRM_OS_KRNFROMUSR( request, (drm_buf_map_t *)data, sizeof(request) );
1146
1147         if ( request.count >= dma->buf_count ) {
1148                 if ( (__HAVE_AGP && (dma->flags & _DRM_DMA_USE_AGP)) ||
1149                      (__HAVE_SG && (dma->flags & _DRM_DMA_USE_SG)) ) {
1150                         drm_map_t *map = DRIVER_AGP_BUFFERS_MAP( dev );
1151
1152                         if ( !map ) {
1153                                 retcode = DRM_OS_ERR(EINVAL);
1154                                 goto done;
1155                         }
1156
1157 #ifdef __linux__
1158 #if LINUX_VERSION_CODE <= 0x020402
1159                         down( &current->mm->mmap_sem );
1160 #else
1161                         down_write( &current->mm->mmap_sem );
1162 #endif
1163
1164                         virtual = do_mmap( filp, 0, map->size,
1165                                            PROT_READ | PROT_WRITE,
1166                                            MAP_SHARED,
1167                                            (unsigned long)map->offset );
1168 #if LINUX_VERSION_CODE <= 0x020402
1169                         up( &current->mm->mmap_sem );
1170 #else
1171                         up_write( &current->mm->mmap_sem );
1172 #endif
1173 #endif /* __linux__ */
1174 #ifdef __FreeBSD__
1175                         virtual = round_page((vm_offset_t)vms->vm_daddr + MAXDSIZ);
1176                         retcode = vm_mmap(&vms->vm_map,
1177                                           &virtual,
1178                                           round_page(map->size),
1179                                           PROT_READ|PROT_WRITE, VM_PROT_ALL,
1180                                           MAP_SHARED,
1181                                           SLIST_FIRST(&kdev->si_hlist),
1182                                           (unsigned long)map->offset );
1183 #endif /* __FreeBSD__ */
1184                 } else {
1185 #ifdef __linux__
1186 #if LINUX_VERSION_CODE <= 0x020402
1187                         down( &current->mm->mmap_sem );
1188 #else
1189                         down_write( &current->mm->mmap_sem );
1190 #endif
1191
1192                         virtual = do_mmap( filp, 0, dma->byte_count,
1193                                            PROT_READ | PROT_WRITE,
1194                                            MAP_SHARED, 0 );
1195 #if LINUX_VERSION_CODE <= 0x020402
1196                         up( &current->mm->mmap_sem );
1197 #else
1198                         up_write( &current->mm->mmap_sem );
1199 #endif
1200 #endif /* __linux__ */
1201 #ifdef __FreeBSD__
1202                         virtual = round_page((vm_offset_t)vms->vm_daddr + MAXDSIZ);
1203                         retcode = vm_mmap(&vms->vm_map,
1204                                           &virtual,
1205                                           round_page(dma->byte_count),
1206                                           PROT_READ|PROT_WRITE, VM_PROT_ALL,
1207                                           MAP_SHARED,
1208                                           SLIST_FIRST(&kdev->si_hlist),
1209                                           0);
1210 #endif /* __FreeBSD__ */
1211                 }
1212 #ifdef __linux__
1213                 if ( virtual > -1024UL ) {
1214                         /* Real error */
1215                         retcode = (signed long)virtual;
1216                         goto done;
1217                 }
1218 #endif /* __linux__ */
1219 #ifdef __FreeBSD__
1220                 if (retcode)
1221                         goto done;
1222 #endif /* __FreeBSD__ */
1223                 request.virtual = (void *)virtual;
1224
1225                 for ( i = 0 ; i < dma->buf_count ; i++ ) {
1226                         if ( DRM_OS_COPYTOUSR( &request.list[i].idx,
1227                                            &dma->buflist[i]->idx,
1228                                            sizeof(request.list[0].idx) ) ) {
1229                                 retcode = DRM_OS_ERR(EFAULT);
1230                                 goto done;
1231                         }
1232                         if ( DRM_OS_COPYTOUSR( &request.list[i].total,
1233                                            &dma->buflist[i]->total,
1234                                            sizeof(request.list[0].total) ) ) {
1235                                 retcode = DRM_OS_ERR(EFAULT);
1236                                 goto done;
1237                         }
1238                         if ( DRM_OS_COPYTOUSR( &request.list[i].used,
1239                                            &zero,
1240                                            sizeof(zero) ) ) {
1241                                 retcode = DRM_OS_ERR(EFAULT);
1242                                 goto done;
1243                         }
1244                         address = virtual + dma->buflist[i]->offset; /* *** */
1245                         if ( DRM_OS_COPYTOUSR( &request.list[i].address,
1246                                            &address,
1247                                            sizeof(address) ) ) {
1248                                 retcode = DRM_OS_ERR(EFAULT);
1249                                 goto done;
1250                         }
1251                 }
1252         }
1253  done:
1254         request.count = dma->buf_count;
1255
1256         DRM_DEBUG( "%d buffers, retcode = %d\n", request.count, retcode );
1257
1258         DRM_OS_KRNTOUSR( (drm_buf_map_t *)data, request, sizeof(request) );
1259
1260         return retcode;
1261 }
1262
1263 #endif /* __HAVE_DMA */
1264