sys/x86/x86/busdma_machdep.c

   1 /*-
   2  * Copyright (c) 1997, 1998 Justin T. Gibbs.
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions, and the following disclaimer,
  10  *    without modification, immediately at the beginning of the file.
  11  * 2. The name of the author may not be used to endorse or promote products
  12  *    derived from this software without specific prior written permission.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
  18  * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  */
  26
  27 #include <sys/cdefs.h>
  28 __FBSDID("$FreeBSD$");
  29
  30 #include <sys/param.h>
  31 #include <sys/systm.h>
  32 #include <sys/malloc.h>
  33 #include <sys/bus.h>
  34 #include <sys/interrupt.h>
  35 #include <sys/kernel.h>
  36 #include <sys/ktr.h>
  37 #include <sys/lock.h>
  38 #include <sys/proc.h>
  39 #include <sys/memdesc.h>
  40 #include <sys/mutex.h>
  41 #include <sys/sysctl.h>
  42 #include <sys/uio.h>
  43
  44 #include <vm/vm.h>
  45 #include <vm/vm_extern.h>
  46 #include <vm/vm_kern.h>
  47 #include <vm/vm_page.h>
  48 #include <vm/vm_map.h>
  49
  50 #include <machine/atomic.h>
  51 #include <machine/bus.h>
  52 #include <machine/md_var.h>
  53 #include <machine/specialreg.h>
  54
  55 #ifdef __i386__
  56 #define MAX_BPAGES 512
  57 #else
  58 #define MAX_BPAGES 8192
  59 #endif
  60 #define BUS_DMA_COULD_BOUNCE    BUS_DMA_BUS3
  61 #define BUS_DMA_MIN_ALLOC_COMP  BUS_DMA_BUS4
  62
  63 struct bounce_zone;
  64
  65 struct bus_dma_tag {
  66         bus_dma_tag_t     parent;
  67         bus_size_t        alignment;
  68         bus_addr_t        boundary;
  69         bus_addr_t        lowaddr;
  70         bus_addr_t        highaddr;
  71         bus_dma_filter_t *filter;
  72         void             *filterarg;
  73         bus_size_t        maxsize;
  74         u_int             nsegments;
  75         bus_size_t        maxsegsz;
  76         int               flags;
  77         int               ref_count;
  78         int               map_count;
  79         bus_dma_lock_t   *lockfunc;
  80         void             *lockfuncarg;
  81         bus_dma_segment_t *segments;
  82         struct bounce_zone *bounce_zone;
  83 };
  84
  85 struct bounce_page {
  86         vm_offset_t     vaddr;          /* kva of bounce buffer */
  87         bus_addr_t      busaddr;        /* Physical address */
  88         vm_offset_t     datavaddr;      /* kva of client data */
  89         bus_addr_t      dataaddr;       /* client physical address */
  90         bus_size_t      datacount;      /* client data count */
  91         STAILQ_ENTRY(bounce_page) links;
  92 };
  93
  94 int busdma_swi_pending;
  95
  96 struct bounce_zone {
  97         STAILQ_ENTRY(bounce_zone) links;
  98         STAILQ_HEAD(bp_list, bounce_page) bounce_page_list;
  99         int             total_bpages;
 100         int             free_bpages;
 101         int             reserved_bpages;
 102         int             active_bpages;
 103         int             total_bounced;
 104         int             total_deferred;
 105         int             map_count;
 106         bus_size_t      alignment;
 107         bus_addr_t      lowaddr;
 108         char            zoneid[8];
 109         char            lowaddrid[20];
 110         struct sysctl_ctx_list sysctl_tree;
 111         struct sysctl_oid *sysctl_tree_top;
 112 };
 113
 114 static struct mtx bounce_lock;
 115 static int total_bpages;
 116 static int busdma_zonecount;
 117 static STAILQ_HEAD(, bounce_zone) bounce_zone_list;
 118
 119 static SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD, 0, "Busdma parameters");
 120 SYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bpages, 0,
 121            "Total bounce pages");
 122
 123 struct bus_dmamap {
 124         struct bp_list         bpages;
 125         int                    pagesneeded;
 126         int                    pagesreserved;
 127         bus_dma_tag_t          dmat;
 128         struct memdesc         mem;
 129         bus_dmamap_callback_t *callback;
 130         void                  *callback_arg;
 131         STAILQ_ENTRY(bus_dmamap) links;
 132 };
 133
 134 static STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist;
 135 static STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist;
 136 static struct bus_dmamap nobounce_dmamap, contig_dmamap;
 137
 138 static void init_bounce_pages(void *dummy);
 139 static int alloc_bounce_zone(bus_dma_tag_t dmat);
 140 static int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages);
 141 static int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map,
 142                                 int commit);
 143 static bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map,
 144                                   vm_offset_t vaddr, bus_addr_t addr,
 145                                   bus_size_t size);
 146 static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage);
 147 int run_filter(bus_dma_tag_t dmat, bus_addr_t paddr);
 148 static void _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map,
 149                                     pmap_t pmap, void *buf, bus_size_t buflen,
 150                                     int flags);
 151 static void _bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map,
 152                                    vm_paddr_t buf, bus_size_t buflen,
 153                                    int flags);
 154 static int _bus_dmamap_reserve_pages(bus_dma_tag_t dmat, bus_dmamap_t map,
 155                                      int flags);
 156
 157 #ifdef XEN
 158 #undef pmap_kextract
 159 #define pmap_kextract pmap_kextract_ma
 160 #endif
 161
 162 /*
 163  * Return true if a match is made.
 164  *
 165  * To find a match walk the chain of bus_dma_tag_t's looking for 'paddr'.
 166  *
 167  * If paddr is within the bounds of the dma tag then call the filter callback
 168  * to check for a match, if there is no filter callback then assume a match.
 169  */
 170 int
 171 run_filter(bus_dma_tag_t dmat, bus_addr_t paddr)
 172 {
 173         int retval;
 174
 175         retval = 0;
 176
 177         do {
 178                 if (((paddr > dmat->lowaddr && paddr <= dmat->highaddr)
 179                  || ((paddr & (dmat->alignment - 1)) != 0))
 180                  && (dmat->filter == NULL
 181                   || (*dmat->filter)(dmat->filterarg, paddr) != 0))
 182                         retval = 1;
 183
 184                 dmat = dmat->parent;
 185         } while (retval == 0 && dmat != NULL);
 186         return (retval);
 187 }
 188
 189 /*
 190  * Convenience function for manipulating driver locks from busdma (during
 191  * busdma_swi, for example).  Drivers that don't provide their own locks
 192  * should specify &Giant to dmat->lockfuncarg.  Drivers that use their own
 193  * non-mutex locking scheme don't have to use this at all.
 194  */
 195 void
 196 busdma_lock_mutex(void *arg, bus_dma_lock_op_t op)
 197 {
 198         struct mtx *dmtx;
 199
 200         dmtx = (struct mtx *)arg;
 201         switch (op) {
 202         case BUS_DMA_LOCK:
 203                 mtx_lock(dmtx);
 204                 break;
 205         case BUS_DMA_UNLOCK:
 206                 mtx_unlock(dmtx);
 207                 break;
 208         default:
 209                 panic("Unknown operation 0x%x for busdma_lock_mutex!", op);
 210         }
 211 }
 212
 213 /*
 214  * dflt_lock should never get called.  It gets put into the dma tag when
 215  * lockfunc == NULL, which is only valid if the maps that are associated
 216  * with the tag are meant to never be defered.
 217  * XXX Should have a way to identify which driver is responsible here.
 218  */
 219 static void
 220 dflt_lock(void *arg, bus_dma_lock_op_t op)
 221 {
 222         panic("driver error: busdma dflt_lock called");
 223 }
 224
 225 /*
 226  * Allocate a device specific dma_tag.
 227  */
 228 int
 229 bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment,
 230                    bus_addr_t boundary, bus_addr_t lowaddr,
 231                    bus_addr_t highaddr, bus_dma_filter_t *filter,
 232                    void *filterarg, bus_size_t maxsize, int nsegments,
 233                    bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc,
 234                    void *lockfuncarg, bus_dma_tag_t *dmat)
 235 {
 236         bus_dma_tag_t newtag;
 237         int error = 0;
 238
 239         /* Basic sanity checking */
 240         if (boundary != 0 && boundary < maxsegsz)
 241                 maxsegsz = boundary;
 242
 243         if (maxsegsz == 0) {
 244                 return (EINVAL);
 245         }
 246
 247         /* Return a NULL tag on failure */
 248         *dmat = NULL;
 249
 250         newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_DEVBUF,
 251             M_ZERO | M_NOWAIT);
 252         if (newtag == NULL) {
 253                 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d",
 254                     __func__, newtag, 0, ENOMEM);
 255                 return (ENOMEM);
 256         }
 257
 258         newtag->parent = parent;
 259         newtag->alignment = alignment;
 260         newtag->boundary = boundary;
 261         newtag->lowaddr = trunc_page((vm_paddr_t)lowaddr) + (PAGE_SIZE - 1);
 262         newtag->highaddr = trunc_page((vm_paddr_t)highaddr) + (PAGE_SIZE - 1);
 263         newtag->filter = filter;
 264         newtag->filterarg = filterarg;
 265         newtag->maxsize = maxsize;
 266         newtag->nsegments = nsegments;
 267         newtag->maxsegsz = maxsegsz;
 268         newtag->flags = flags;
 269         newtag->ref_count = 1; /* Count ourself */
 270         newtag->map_count = 0;
 271         if (lockfunc != NULL) {
 272                 newtag->lockfunc = lockfunc;
 273                 newtag->lockfuncarg = lockfuncarg;
 274         } else {
 275                 newtag->lockfunc = dflt_lock;
 276                 newtag->lockfuncarg = NULL;
 277         }
 278         newtag->segments = NULL;
 279
 280         /* Take into account any restrictions imposed by our parent tag */
 281         if (parent != NULL) {
 282                 newtag->lowaddr = MIN(parent->lowaddr, newtag->lowaddr);
 283                 newtag->highaddr = MAX(parent->highaddr, newtag->highaddr);
 284                 if (newtag->boundary == 0)
 285                         newtag->boundary = parent->boundary;
 286                 else if (parent->boundary != 0)
 287                         newtag->boundary = MIN(parent->boundary,
 288                                                newtag->boundary);
 289                 if ((newtag->filter != NULL) ||
 290                     ((parent->flags & BUS_DMA_COULD_BOUNCE) != 0))
 291                         newtag->flags |= BUS_DMA_COULD_BOUNCE;
 292                 if (newtag->filter == NULL) {
 293                         /*
 294                          * Short circuit looking at our parent directly
 295                          * since we have encapsulated all of its information
 296                          */
 297                         newtag->filter = parent->filter;
 298                         newtag->filterarg = parent->filterarg;
 299                         newtag->parent = parent->parent;
 300                 }
 301                 if (newtag->parent != NULL)
 302                         atomic_add_int(&parent->ref_count, 1);
 303         }
 304
 305         if (newtag->lowaddr < ptoa((vm_paddr_t)Maxmem)
 306          || newtag->alignment > 1)
 307                 newtag->flags |= BUS_DMA_COULD_BOUNCE;
 308
 309         if (((newtag->flags & BUS_DMA_COULD_BOUNCE) != 0) &&
 310             (flags & BUS_DMA_ALLOCNOW) != 0) {
 311                 struct bounce_zone *bz;
 312
 313                 /* Must bounce */
 314
 315                 if ((error = alloc_bounce_zone(newtag)) != 0) {
 316                         free(newtag, M_DEVBUF);
 317                         return (error);
 318                 }
 319                 bz = newtag->bounce_zone;
 320
 321                 if (ptoa(bz->total_bpages) < maxsize) {
 322                         int pages;
 323
 324                         pages = atop(maxsize) - bz->total_bpages;
 325
 326                         /* Add pages to our bounce pool */
 327                         if (alloc_bounce_pages(newtag, pages) < pages)
 328                                 error = ENOMEM;
 329                 }
 330                 /* Performed initial allocation */
 331                 newtag->flags |= BUS_DMA_MIN_ALLOC_COMP;
 332         }
 333
 334         if (error != 0) {
 335                 free(newtag, M_DEVBUF);
 336         } else {
 337                 *dmat = newtag;
 338         }
 339         CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d",
 340             __func__, newtag, (newtag != NULL ? newtag->flags : 0), error);
 341         return (error);
 342 }
 343
 344 int
 345 bus_dma_tag_destroy(bus_dma_tag_t dmat)
 346 {
 347         bus_dma_tag_t dmat_copy;
 348         int error;
 349
 350         error = 0;
 351         dmat_copy = dmat;
 352
 353         if (dmat != NULL) {
 354
 355                 if (dmat->map_count != 0) {
 356                         error = EBUSY;
 357                         goto out;
 358                 }
 359
 360                 while (dmat != NULL) {
 361                         bus_dma_tag_t parent;
 362
 363                         parent = dmat->parent;
 364                         atomic_subtract_int(&dmat->ref_count, 1);
 365                         if (dmat->ref_count == 0) {
 366                                 if (dmat->segments != NULL)
 367                                         free(dmat->segments, M_DEVBUF);
 368                                 free(dmat, M_DEVBUF);
 369                                 /*
 370                                  * Last reference count, so
 371                                  * release our reference
 372                                  * count on our parent.
 373                                  */
 374                                 dmat = parent;
 375                         } else
 376                                 dmat = NULL;
 377                 }
 378         }
 379 out:
 380         CTR3(KTR_BUSDMA, "%s tag %p error %d", __func__, dmat_copy, error);
 381         return (error);
 382 }
 383
 384 /*
 385  * Allocate a handle for mapping from kva/uva/physical
 386  * address space into bus device space.
 387  */
 388 int
 389 bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp)
 390 {
 391         int error;
 392
 393         error = 0;
 394
 395         if (dmat->segments == NULL) {
 396                 dmat->segments = (bus_dma_segment_t *)malloc(
 397                     sizeof(bus_dma_segment_t) * dmat->nsegments, M_DEVBUF,
 398                     M_NOWAIT);
 399                 if (dmat->segments == NULL) {
 400                         CTR3(KTR_BUSDMA, "%s: tag %p error %d",
 401                             __func__, dmat, ENOMEM);
 402                         return (ENOMEM);
 403                 }
 404         }
 405
 406         /*
 407          * Bouncing might be required if the driver asks for an active
 408          * exclusion region, a data alignment that is stricter than 1, and/or
 409          * an active address boundary.
 410          */
 411         if (dmat->flags & BUS_DMA_COULD_BOUNCE) {
 412
 413                 /* Must bounce */
 414                 struct bounce_zone *bz;
 415                 int maxpages;
 416
 417                 if (dmat->bounce_zone == NULL) {
 418                         if ((error = alloc_bounce_zone(dmat)) != 0)
 419                                 return (error);
 420                 }
 421                 bz = dmat->bounce_zone;
 422
 423                 *mapp = (bus_dmamap_t)malloc(sizeof(**mapp), M_DEVBUF,
 424                                              M_NOWAIT | M_ZERO);
 425                 if (*mapp == NULL) {
 426                         CTR3(KTR_BUSDMA, "%s: tag %p error %d",
 427                             __func__, dmat, ENOMEM);
 428                         return (ENOMEM);
 429                 }
 430
 431                 /* Initialize the new map */
 432                 STAILQ_INIT(&((*mapp)->bpages));
 433
 434                 /*
 435                  * Attempt to add pages to our pool on a per-instance
 436                  * basis up to a sane limit.
 437                  */
 438                 if (dmat->alignment > 1)
 439                         maxpages = MAX_BPAGES;
 440                 else
 441                         maxpages = MIN(MAX_BPAGES, Maxmem -atop(dmat->lowaddr));
 442                 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0
 443                  || (bz->map_count > 0 && bz->total_bpages < maxpages)) {
 444                         int pages;
 445
 446                         pages = MAX(atop(dmat->maxsize), 1);
 447                         pages = MIN(maxpages - bz->total_bpages, pages);
 448                         pages = MAX(pages, 1);
 449                         if (alloc_bounce_pages(dmat, pages) < pages)
 450                                 error = ENOMEM;
 451
 452                         if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0) {
 453                                 if (error == 0)
 454                                         dmat->flags |= BUS_DMA_MIN_ALLOC_COMP;
 455                         } else {
 456                                 error = 0;
 457                         }
 458                 }
 459                 bz->map_count++;
 460         } else {
 461                 *mapp = NULL;
 462         }
 463         if (error == 0)
 464                 dmat->map_count++;
 465         CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
 466             __func__, dmat, dmat->flags, error);
 467         return (error);
 468 }
 469
 470 /*
 471  * Destroy a handle for mapping from kva/uva/physical
 472  * address space into bus device space.
 473  */
 474 int
 475 bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map)
 476 {
 477         if (map != NULL && map != &nobounce_dmamap && map != &contig_dmamap) {
 478                 if (STAILQ_FIRST(&map->bpages) != NULL) {
 479                         CTR3(KTR_BUSDMA, "%s: tag %p error %d",
 480                             __func__, dmat, EBUSY);
 481                         return (EBUSY);
 482                 }
 483                 if (dmat->bounce_zone)
 484                         dmat->bounce_zone->map_count--;
 485                 free(map, M_DEVBUF);
 486         }
 487         dmat->map_count--;
 488         CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat);
 489         return (0);
 490 }
 491
 492
 493 /*
 494  * Allocate a piece of memory that can be efficiently mapped into
 495  * bus device space based on the constraints lited in the dma tag.
 496  * A dmamap to for use with dmamap_load is also allocated.
 497  */
 498 int
 499 bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags,
 500                  bus_dmamap_t *mapp)
 501 {
 502         vm_memattr_t attr;
 503         int mflags;
 504
 505         if (flags & BUS_DMA_NOWAIT)
 506                 mflags = M_NOWAIT;
 507         else
 508                 mflags = M_WAITOK;
 509
 510         /* If we succeed, no mapping/bouncing will be required */
 511         *mapp = NULL;
 512
 513         if (dmat->segments == NULL) {
 514                 dmat->segments = (bus_dma_segment_t *)malloc(
 515                     sizeof(bus_dma_segment_t) * dmat->nsegments, M_DEVBUF,
 516                     mflags);
 517                 if (dmat->segments == NULL) {
 518                         CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
 519                             __func__, dmat, dmat->flags, ENOMEM);
 520                         return (ENOMEM);
 521                 }
 522         }
 523         if (flags & BUS_DMA_ZERO)
 524                 mflags |= M_ZERO;
 525         if (flags & BUS_DMA_NOCACHE)
 526                 attr = VM_MEMATTR_UNCACHEABLE;
 527         else
 528                 attr = VM_MEMATTR_DEFAULT;
 529
 530         /*
 531          * XXX:
 532          * (dmat->alignment < dmat->maxsize) is just a quick hack; the exact
 533          * alignment guarantees of malloc need to be nailed down, and the
 534          * code below should be rewritten to take that into account.
 535          *
 536          * In the meantime, we'll warn the user if malloc gets it wrong.
 537          */
 538         if ((dmat->maxsize <= PAGE_SIZE) &&
 539            (dmat->alignment < dmat->maxsize) &&
 540             dmat->lowaddr >= ptoa((vm_paddr_t)Maxmem) &&
 541             attr == VM_MEMATTR_DEFAULT) {
 542                 *vaddr = malloc(dmat->maxsize, M_DEVBUF, mflags);
 543         } else if (dmat->nsegments >= btoc(dmat->maxsize) &&
 544             dmat->alignment <= PAGE_SIZE &&
 545             (dmat->boundary == 0 || dmat->boundary >= dmat->lowaddr)) {
 546                 /* Page-based multi-segment allocations allowed */
 547                 *vaddr = (void *)kmem_alloc_attr(kernel_arena, dmat->maxsize,
 548                     mflags, 0ul, dmat->lowaddr, attr);
 549                 *mapp = &contig_dmamap;
 550         } else {
 551                 *vaddr = (void *)kmem_alloc_contig(kernel_arena, dmat->maxsize,
 552                     mflags, 0ul, dmat->lowaddr, dmat->alignment ?
 553                     dmat->alignment : 1ul, dmat->boundary, attr);
 554                 *mapp = &contig_dmamap;
 555         }
 556         if (*vaddr == NULL) {
 557                 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
 558                     __func__, dmat, dmat->flags, ENOMEM);
 559                 return (ENOMEM);
 560         } else if (vtophys(*vaddr) & (dmat->alignment - 1)) {
 561                 printf("bus_dmamem_alloc failed to align memory properly.\n");
 562         }
 563         CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
 564             __func__, dmat, dmat->flags, 0);
 565         return (0);
 566 }
 567
 568 /*
 569  * Free a piece of memory and it's allociated dmamap, that was allocated
 570  * via bus_dmamem_alloc.  Make the same choice for free/contigfree.
 571  */
 572 void
 573 bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map)
 574 {
 575         /*
 576          * dmamem does not need to be bounced, so the map should be
 577          * NULL if malloc() was used and contig_dmamap if
 578          * kmem_alloc_contig() was used.
 579          */
 580         if (!(map == NULL || map == &contig_dmamap))
 581                 panic("bus_dmamem_free: Invalid map freed\n");
 582         if (map == NULL)
 583                 free(vaddr, M_DEVBUF);
 584         else
 585                 kmem_free(kernel_arena, (vm_offset_t)vaddr, dmat->maxsize);
 586         CTR3(KTR_BUSDMA, "%s: tag %p flags 0x%x", __func__, dmat, dmat->flags);
 587 }
 588
 589 static void
 590 _bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf,
 591     bus_size_t buflen, int flags)
 592 {
 593         bus_addr_t curaddr;
 594         bus_size_t sgsize;
 595
 596         if ((map != &nobounce_dmamap && map->pagesneeded == 0)) {
 597                 /*
 598                  * Count the number of bounce pages
 599                  * needed in order to complete this transfer
 600                  */
 601                 curaddr = buf;
 602                 while (buflen != 0) {
 603                         sgsize = MIN(buflen, dmat->maxsegsz);
 604                         if (run_filter(dmat, curaddr)) {
 605                                 sgsize = MIN(sgsize, PAGE_SIZE);
 606                                 map->pagesneeded++;
 607                         }
 608                         curaddr += sgsize;
 609                         buflen -= sgsize;
 610                 }
 611                 CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded);
 612         }
 613 }
 614
 615 static void
 616 _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap,
 617     void *buf, bus_size_t buflen, int flags)
 618 {
 619         vm_offset_t vaddr;
 620         vm_offset_t vendaddr;
 621         bus_addr_t paddr;
 622
 623         if ((map != &nobounce_dmamap && map->pagesneeded == 0)) {
 624                 CTR4(KTR_BUSDMA, "lowaddr= %d Maxmem= %d, boundary= %d, "
 625                     "alignment= %d", dmat->lowaddr, ptoa((vm_paddr_t)Maxmem),
 626                     dmat->boundary, dmat->alignment);
 627                 CTR3(KTR_BUSDMA, "map= %p, nobouncemap= %p, pagesneeded= %d",
 628                     map, &nobounce_dmamap, map->pagesneeded);
 629                 /*
 630                  * Count the number of bounce pages
 631                  * needed in order to complete this transfer
 632                  */
 633                 vaddr = (vm_offset_t)buf;
 634                 vendaddr = (vm_offset_t)buf + buflen;
 635
 636                 while (vaddr < vendaddr) {
 637                         bus_size_t sg_len;
 638
 639                         sg_len = PAGE_SIZE - ((vm_offset_t)vaddr & PAGE_MASK);
 640                         if (pmap == kernel_pmap)
 641                                 paddr = pmap_kextract(vaddr);
 642                         else
 643                                 paddr = pmap_extract(pmap, vaddr);
 644                         if (run_filter(dmat, paddr) != 0) {
 645                                 sg_len = roundup2(sg_len, dmat->alignment);
 646                                 map->pagesneeded++;
 647                         }
 648                         vaddr += sg_len;
 649                 }
 650                 CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded);
 651         }
 652 }
 653
 654 static int
 655 _bus_dmamap_reserve_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int flags)
 656 {
 657
 658         /* Reserve Necessary Bounce Pages */
 659         mtx_lock(&bounce_lock);
 660         if (flags & BUS_DMA_NOWAIT) {
 661                 if (reserve_bounce_pages(dmat, map, 0) != 0) {
 662                         mtx_unlock(&bounce_lock);
 663                         return (ENOMEM);
 664                 }
 665         } else {
 666                 if (reserve_bounce_pages(dmat, map, 1) != 0) {
 667                         /* Queue us for resources */
 668                         STAILQ_INSERT_TAIL(&bounce_map_waitinglist, map, links);
 669                         mtx_unlock(&bounce_lock);
 670                         return (EINPROGRESS);
 671                 }
 672         }
 673         mtx_unlock(&bounce_lock);
 674
 675         return (0);
 676 }
 677
 678 /*
 679  * Add a single contiguous physical range to the segment list.
 680  */
 681 static int
 682 _bus_dmamap_addseg(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t curaddr,
 683                    bus_size_t sgsize, bus_dma_segment_t *segs, int *segp)
 684 {
 685         bus_addr_t baddr, bmask;
 686         int seg;
 687
 688         /*
 689          * Make sure we don't cross any boundaries.
 690          */
 691         bmask = ~(dmat->boundary - 1);
 692         if (dmat->boundary > 0) {
 693                 baddr = (curaddr + dmat->boundary) & bmask;
 694                 if (sgsize > (baddr - curaddr))
 695                         sgsize = (baddr - curaddr);
 696         }
 697
 698         /*
 699          * Insert chunk into a segment, coalescing with
 700          * previous segment if possible.
 701          */
 702         seg = *segp;
 703         if (seg == -1) {
 704                 seg = 0;
 705                 segs[seg].ds_addr = curaddr;
 706                 segs[seg].ds_len = sgsize;
 707         } else {
 708                 if (curaddr == segs[seg].ds_addr + segs[seg].ds_len &&
 709                     (segs[seg].ds_len + sgsize) <= dmat->maxsegsz &&
 710                     (dmat->boundary == 0 ||
 711                      (segs[seg].ds_addr & bmask) == (curaddr & bmask)))
 712                         segs[seg].ds_len += sgsize;
 713                 else {
 714                         if (++seg >= dmat->nsegments)
 715                                 return (0);
 716                         segs[seg].ds_addr = curaddr;
 717                         segs[seg].ds_len = sgsize;
 718                 }
 719         }
 720         *segp = seg;
 721         return (sgsize);
 722 }
 723
 724 /*
 725  * Utility function to load a physical buffer.  segp contains
 726  * the starting segment on entrace, and the ending segment on exit.
 727  */
 728 int
 729 _bus_dmamap_load_phys(bus_dma_tag_t dmat,
 730                       bus_dmamap_t map,
 731                       vm_paddr_t buf, bus_size_t buflen,
 732                       int flags,
 733                       bus_dma_segment_t *segs,
 734                       int *segp)
 735 {
 736         bus_size_t sgsize;
 737         bus_addr_t curaddr;
 738         int error;
 739
 740         if (map == NULL || map == &contig_dmamap)
 741                 map = &nobounce_dmamap;
 742
 743         if (segs == NULL)
 744                 segs = dmat->segments;
 745
 746         if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) {
 747                 _bus_dmamap_count_phys(dmat, map, buf, buflen, flags);
 748                 if (map->pagesneeded != 0) {
 749                         error = _bus_dmamap_reserve_pages(dmat, map, flags);
 750                         if (error)
 751                                 return (error);
 752                 }
 753         }
 754
 755         while (buflen > 0) {
 756                 curaddr = buf;
 757                 sgsize = MIN(buflen, dmat->maxsegsz);
 758                 if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) &&
 759                     map->pagesneeded != 0 && run_filter(dmat, curaddr)) {
 760                         sgsize = MIN(sgsize, PAGE_SIZE);
 761                         curaddr = add_bounce_page(dmat, map, 0, curaddr,
 762                                                   sgsize);
 763                 }
 764                 sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs,
 765                     segp);
 766                 if (sgsize == 0)
 767                         break;
 768                 buf += sgsize;
 769                 buflen -= sgsize;
 770         }
 771
 772         /*
 773          * Did we fit?
 774          */
 775         return (buflen != 0 ? EFBIG : 0); /* XXX better return value here? */
 776 }
 777
 778 /*
 779  * Utility function to load a linear buffer.  segp contains
 780  * the starting segment on entrace, and the ending segment on exit.
 781  */
 782 int
 783 _bus_dmamap_load_buffer(bus_dma_tag_t dmat,
 784                         bus_dmamap_t map,
 785                         void *buf, bus_size_t buflen,
 786                         pmap_t pmap,
 787                         int flags,
 788                         bus_dma_segment_t *segs,
 789                         int *segp)
 790 {
 791         bus_size_t sgsize;
 792         bus_addr_t curaddr;
 793         vm_offset_t vaddr;
 794         int error;
 795
 796         if (map == NULL || map == &contig_dmamap)
 797                 map = &nobounce_dmamap;
 798
 799         if (segs == NULL)
 800                 segs = dmat->segments;
 801
 802         if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) {
 803                 _bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, flags);
 804                 if (map->pagesneeded != 0) {
 805                         error = _bus_dmamap_reserve_pages(dmat, map, flags);
 806                         if (error)
 807                                 return (error);
 808                 }
 809         }
 810
 811         vaddr = (vm_offset_t)buf;
 812
 813         while (buflen > 0) {
 814                 bus_size_t max_sgsize;
 815
 816                 /*
 817                  * Get the physical address for this segment.
 818                  */
 819                 if (pmap == kernel_pmap)
 820                         curaddr = pmap_kextract(vaddr);
 821                 else
 822                         curaddr = pmap_extract(pmap, vaddr);
 823
 824                 /*
 825                  * Compute the segment size, and adjust counts.
 826                  */
 827                 max_sgsize = MIN(buflen, dmat->maxsegsz);
 828                 sgsize = PAGE_SIZE - ((vm_offset_t)curaddr & PAGE_MASK);
 829                 if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) &&
 830                     map->pagesneeded != 0 && run_filter(dmat, curaddr)) {
 831                         sgsize = roundup2(sgsize, dmat->alignment);
 832                         sgsize = MIN(sgsize, max_sgsize);
 833                         curaddr = add_bounce_page(dmat, map, vaddr, curaddr,
 834                                                   sgsize);
 835                 } else {
 836                         sgsize = MIN(sgsize, max_sgsize);
 837                 }
 838                 sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs,
 839                                             segp);
 840                 if (sgsize == 0)
 841                         break;
 842                 vaddr += sgsize;
 843                 buflen -= sgsize;
 844         }
 845
 846         /*
 847          * Did we fit?
 848          */
 849         return (buflen != 0 ? EFBIG : 0); /* XXX better return value here? */
 850 }
 851
 852 void
 853 __bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map,
 854                     struct memdesc *mem, bus_dmamap_callback_t *callback,
 855                     void *callback_arg)
 856 {
 857         if (map != NULL) {
 858                 map->mem = *mem;
 859                 map->dmat = dmat;
 860                 map->callback = callback;
 861                 map->callback_arg = callback_arg;
 862         }
 863 }
 864
 865 bus_dma_segment_t *
 866 _bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map,
 867                       bus_dma_segment_t *segs, int nsegs, int error)
 868 {
 869
 870         if (segs == NULL)
 871                 segs = dmat->segments;
 872         return (segs);
 873 }
 874
 875 /*
 876  * Release the mapping held by map.
 877  */
 878 void
 879 _bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map)
 880 {
 881         struct bounce_page *bpage;
 882
 883         while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) {
 884                 STAILQ_REMOVE_HEAD(&map->bpages, links);
 885                 free_bounce_page(dmat, bpage);
 886         }
 887 }
 888
 889 void
 890 _bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
 891 {
 892         struct bounce_page *bpage;
 893
 894         if ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) {
 895                 /*
 896                  * Handle data bouncing.  We might also
 897                  * want to add support for invalidating
 898                  * the caches on broken hardware
 899                  */
 900                 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x op 0x%x "
 901                     "performing bounce", __func__, dmat, dmat->flags, op);
 902
 903                 if (op & BUS_DMASYNC_PREWRITE) {
 904                         while (bpage != NULL) {
 905                                 if (bpage->datavaddr != 0)
 906                                         bcopy((void *)bpage->datavaddr,
 907                                               (void *)bpage->vaddr,
 908                                               bpage->datacount);
 909                                 else
 910                                         physcopyout(bpage->dataaddr,
 911                                               (void *)bpage->vaddr,
 912                                               bpage->datacount);
 913                                 bpage = STAILQ_NEXT(bpage, links);
 914                         }
 915                         dmat->bounce_zone->total_bounced++;
 916                 }
 917
 918                 if (op & BUS_DMASYNC_POSTREAD) {
 919                         while (bpage != NULL) {
 920                                 if (bpage->datavaddr != 0)
 921                                         bcopy((void *)bpage->vaddr,
 922                                               (void *)bpage->datavaddr,
 923                                               bpage->datacount);
 924                                 else
 925                                         physcopyin((void *)bpage->vaddr,
 926                                               bpage->dataaddr,
 927                                               bpage->datacount);
 928                                 bpage = STAILQ_NEXT(bpage, links);
 929                         }
 930                         dmat->bounce_zone->total_bounced++;
 931                 }
 932         }
 933 }
 934
 935 static void
 936 init_bounce_pages(void *dummy __unused)
 937 {
 938
 939         total_bpages = 0;
 940         STAILQ_INIT(&bounce_zone_list);
 941         STAILQ_INIT(&bounce_map_waitinglist);
 942         STAILQ_INIT(&bounce_map_callbacklist);
 943         mtx_init(&bounce_lock, "bounce pages lock", NULL, MTX_DEF);
 944 }
 945 SYSINIT(bpages, SI_SUB_LOCK, SI_ORDER_ANY, init_bounce_pages, NULL);
 946
 947 static struct sysctl_ctx_list *
 948 busdma_sysctl_tree(struct bounce_zone *bz)
 949 {
 950         return (&bz->sysctl_tree);
 951 }
 952
 953 static struct sysctl_oid *
 954 busdma_sysctl_tree_top(struct bounce_zone *bz)
 955 {
 956         return (bz->sysctl_tree_top);
 957 }
 958
 959 #if defined(__amd64__) || defined(PAE)
 960 #define SYSCTL_ADD_BUS_SIZE_T   SYSCTL_ADD_UQUAD
 961 #else
 962 #define SYSCTL_ADD_BUS_SIZE_T(ctx, parent, nbr, name, flag, ptr, desc)  \
 963         SYSCTL_ADD_UINT(ctx, parent, nbr, name, flag, ptr, 0, desc)
 964 #endif
 965
 966 static int
 967 alloc_bounce_zone(bus_dma_tag_t dmat)
 968 {
 969         struct bounce_zone *bz;
 970
 971         /* Check to see if we already have a suitable zone */
 972         STAILQ_FOREACH(bz, &bounce_zone_list, links) {
 973                 if ((dmat->alignment <= bz->alignment)
 974                  && (dmat->lowaddr >= bz->lowaddr)) {
 975                         dmat->bounce_zone = bz;
 976                         return (0);
 977                 }
 978         }
 979
 980         if ((bz = (struct bounce_zone *)malloc(sizeof(*bz), M_DEVBUF,
 981             M_NOWAIT | M_ZERO)) == NULL)
 982                 return (ENOMEM);
 983
 984         STAILQ_INIT(&bz->bounce_page_list);
 985         bz->free_bpages = 0;
 986         bz->reserved_bpages = 0;
 987         bz->active_bpages = 0;
 988         bz->lowaddr = dmat->lowaddr;
 989         bz->alignment = MAX(dmat->alignment, PAGE_SIZE);
 990         bz->map_count = 0;
 991         snprintf(bz->zoneid, 8, "zone%d", busdma_zonecount);
 992         busdma_zonecount++;
 993         snprintf(bz->lowaddrid, 18, "%#jx", (uintmax_t)bz->lowaddr);
 994         STAILQ_INSERT_TAIL(&bounce_zone_list, bz, links);
 995         dmat->bounce_zone = bz;
 996
 997         sysctl_ctx_init(&bz->sysctl_tree);
 998         bz->sysctl_tree_top = SYSCTL_ADD_NODE(&bz->sysctl_tree,
 999             SYSCTL_STATIC_CHILDREN(_hw_busdma), OID_AUTO, bz->zoneid,
1000             CTLFLAG_RD, 0, "");
1001         if (bz->sysctl_tree_top == NULL) {
1002                 sysctl_ctx_free(&bz->sysctl_tree);
1003                 return (0);     /* XXX error code? */
1004         }
1005
1006         SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1007             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1008             "total_bpages", CTLFLAG_RD, &bz->total_bpages, 0,
1009             "Total bounce pages");
1010         SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1011             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1012             "free_bpages", CTLFLAG_RD, &bz->free_bpages, 0,
1013             "Free bounce pages");
1014         SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1015             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1016             "reserved_bpages", CTLFLAG_RD, &bz->reserved_bpages, 0,
1017             "Reserved bounce pages");
1018         SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1019             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1020             "active_bpages", CTLFLAG_RD, &bz->active_bpages, 0,
1021             "Active bounce pages");
1022         SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1023             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1024             "total_bounced", CTLFLAG_RD, &bz->total_bounced, 0,
1025             "Total bounce requests");
1026         SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1027             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1028             "total_deferred", CTLFLAG_RD, &bz->total_deferred, 0,
1029             "Total bounce requests that were deferred");
1030         SYSCTL_ADD_STRING(busdma_sysctl_tree(bz),
1031             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1032             "lowaddr", CTLFLAG_RD, bz->lowaddrid, 0, "");
1033         SYSCTL_ADD_BUS_SIZE_T(busdma_sysctl_tree(bz),
1034             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1035             "alignment", CTLFLAG_RD, &bz->alignment, "");
1036
1037         return (0);
1038 }
1039
1040 static int
1041 alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages)
1042 {
1043         struct bounce_zone *bz;
1044         int count;
1045
1046         bz = dmat->bounce_zone;
1047         count = 0;
1048         while (numpages > 0) {
1049                 struct bounce_page *bpage;
1050
1051                 bpage = (struct bounce_page *)malloc(sizeof(*bpage), M_DEVBUF,
1052                                                      M_NOWAIT | M_ZERO);
1053
1054                 if (bpage == NULL)
1055                         break;
1056                 bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_DEVBUF,
1057                                                          M_NOWAIT, 0ul,
1058                                                          bz->lowaddr,
1059                                                          PAGE_SIZE,
1060                                                          0);
1061                 if (bpage->vaddr == 0) {
1062                         free(bpage, M_DEVBUF);
1063                         break;
1064                 }
1065                 bpage->busaddr = pmap_kextract(bpage->vaddr);
1066                 mtx_lock(&bounce_lock);
1067                 STAILQ_INSERT_TAIL(&bz->bounce_page_list, bpage, links);
1068                 total_bpages++;
1069                 bz->total_bpages++;
1070                 bz->free_bpages++;
1071                 mtx_unlock(&bounce_lock);
1072                 count++;
1073                 numpages--;
1074         }
1075         return (count);
1076 }
1077
1078 static int
1079 reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int commit)
1080 {
1081         struct bounce_zone *bz;
1082         int pages;
1083
1084         mtx_assert(&bounce_lock, MA_OWNED);
1085         bz = dmat->bounce_zone;
1086         pages = MIN(bz->free_bpages, map->pagesneeded - map->pagesreserved);
1087         if (commit == 0 && map->pagesneeded > (map->pagesreserved + pages))
1088                 return (map->pagesneeded - (map->pagesreserved + pages));
1089         bz->free_bpages -= pages;
1090         bz->reserved_bpages += pages;
1091         map->pagesreserved += pages;
1092         pages = map->pagesneeded - map->pagesreserved;
1093
1094         return (pages);
1095 }
1096
1097 static bus_addr_t
1098 add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr,
1099                 bus_addr_t addr, bus_size_t size)
1100 {
1101         struct bounce_zone *bz;
1102         struct bounce_page *bpage;
1103
1104         KASSERT(dmat->bounce_zone != NULL, ("no bounce zone in dma tag"));
1105         KASSERT(map != NULL && map != &nobounce_dmamap && map != &contig_dmamap,
1106             ("add_bounce_page: bad map %p", map));
1107
1108         bz = dmat->bounce_zone;
1109         if (map->pagesneeded == 0)
1110                 panic("add_bounce_page: map doesn't need any pages");
1111         map->pagesneeded--;
1112
1113         if (map->pagesreserved == 0)
1114                 panic("add_bounce_page: map doesn't need any pages");
1115         map->pagesreserved--;
1116
1117         mtx_lock(&bounce_lock);
1118         bpage = STAILQ_FIRST(&bz->bounce_page_list);
1119         if (bpage == NULL)
1120                 panic("add_bounce_page: free page list is empty");
1121
1122         STAILQ_REMOVE_HEAD(&bz->bounce_page_list, links);
1123         bz->reserved_bpages--;
1124         bz->active_bpages++;
1125         mtx_unlock(&bounce_lock);
1126
1127         if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) {
1128                 /* Page offset needs to be preserved. */
1129                 bpage->vaddr |= vaddr & PAGE_MASK;
1130                 bpage->busaddr |= vaddr & PAGE_MASK;
1131         }
1132         bpage->datavaddr = vaddr;
1133         bpage->dataaddr = addr;
1134         bpage->datacount = size;
1135         STAILQ_INSERT_TAIL(&(map->bpages), bpage, links);
1136         return (bpage->busaddr);
1137 }
1138
1139 static void
1140 free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage)
1141 {
1142         struct bus_dmamap *map;
1143         struct bounce_zone *bz;
1144
1145         bz = dmat->bounce_zone;
1146         bpage->datavaddr = 0;
1147         bpage->datacount = 0;
1148         if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) {
1149                 /*
1150                  * Reset the bounce page to start at offset 0.  Other uses
1151                  * of this bounce page may need to store a full page of
1152                  * data and/or assume it starts on a page boundary.
1153                  */
1154                 bpage->vaddr &= ~PAGE_MASK;
1155                 bpage->busaddr &= ~PAGE_MASK;
1156         }
1157
1158         mtx_lock(&bounce_lock);
1159         STAILQ_INSERT_HEAD(&bz->bounce_page_list, bpage, links);
1160         bz->free_bpages++;
1161         bz->active_bpages--;
1162         if ((map = STAILQ_FIRST(&bounce_map_waitinglist)) != NULL) {
1163                 if (reserve_bounce_pages(map->dmat, map, 1) == 0) {
1164                         STAILQ_REMOVE_HEAD(&bounce_map_waitinglist, links);
1165                         STAILQ_INSERT_TAIL(&bounce_map_callbacklist,
1166                                            map, links);
1167                         busdma_swi_pending = 1;
1168                         bz->total_deferred++;
1169                         swi_sched(vm_ih, 0);
1170                 }
1171         }
1172         mtx_unlock(&bounce_lock);
1173 }
1174
1175 void
1176 busdma_swi(void)
1177 {
1178         bus_dma_tag_t dmat;
1179         struct bus_dmamap *map;
1180
1181         mtx_lock(&bounce_lock);
1182         while ((map = STAILQ_FIRST(&bounce_map_callbacklist)) != NULL) {
1183                 STAILQ_REMOVE_HEAD(&bounce_map_callbacklist, links);
1184                 mtx_unlock(&bounce_lock);
1185                 dmat = map->dmat;
1186                 (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_LOCK);
1187                 bus_dmamap_load_mem(map->dmat, map, &map->mem,
1188                                 map->callback, map->callback_arg,
1189                                 BUS_DMA_WAITOK);
1190                 (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_UNLOCK);
1191                 mtx_lock(&bounce_lock);
1192         }
1193         mtx_unlock(&bounce_lock);
1194 }