sys/arm64/arm64/busdma_bounce.c

   1 /*-
   2  * Copyright (c) 1997, 1998 Justin T. Gibbs.
   3  * Copyright (c) 2015-2016 The FreeBSD Foundation
   4  * All rights reserved.
   5  *
   6  * Portions of this software were developed by Andrew Turner
   7  * under sponsorship of the FreeBSD Foundation.
   8  *
   9  * Portions of this software were developed by Semihalf
  10  * under sponsorship of the FreeBSD Foundation.
  11  *
  12  * Redistribution and use in source and binary forms, with or without
  13  * modification, are permitted provided that the following conditions
  14  * are met:
  15  * 1. Redistributions of source code must retain the above copyright
  16  *    notice, this list of conditions, and the following disclaimer,
  17  *    without modification, immediately at the beginning of the file.
  18  * 2. The name of the author may not be used to endorse or promote products
  19  *    derived from this software without specific prior written permission.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  22  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  23  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  24  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
  25  * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  26  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  27  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  28  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  29  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  30  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  31  * SUCH DAMAGE.
  32  */
  33
  34 #include <sys/cdefs.h>
  35 __FBSDID("$FreeBSD$");
  36
  37 #include <sys/param.h>
  38 #include <sys/systm.h>
  39 #include <sys/malloc.h>
  40 #include <sys/bus.h>
  41 #include <sys/interrupt.h>
  42 #include <sys/kernel.h>
  43 #include <sys/ktr.h>
  44 #include <sys/lock.h>
  45 #include <sys/proc.h>
  46 #include <sys/memdesc.h>
  47 #include <sys/mutex.h>
  48 #include <sys/sysctl.h>
  49 #include <sys/uio.h>
  50
  51 #include <vm/vm.h>
  52 #include <vm/vm_extern.h>
  53 #include <vm/vm_kern.h>
  54 #include <vm/vm_page.h>
  55 #include <vm/vm_map.h>
  56
  57 #include <machine/atomic.h>
  58 #include <machine/bus.h>
  59 #include <machine/md_var.h>
  60 #include <arm64/include/bus_dma_impl.h>
  61
  62 #define MAX_BPAGES 4096
  63
  64 enum {
  65         BF_COULD_BOUNCE         = 0x01,
  66         BF_MIN_ALLOC_COMP       = 0x02,
  67         BF_KMEM_ALLOC           = 0x04,
  68         BF_COHERENT             = 0x10,
  69 };
  70
  71 struct bounce_zone;
  72
  73 struct bus_dma_tag {
  74         struct bus_dma_tag_common common;
  75         size_t                  alloc_size;
  76         size_t                  alloc_alignment;
  77         int                     map_count;
  78         int                     bounce_flags;
  79         bus_dma_segment_t       *segments;
  80         struct bounce_zone      *bounce_zone;
  81 };
  82
  83 struct bounce_page {
  84         vm_offset_t     vaddr;          /* kva of bounce buffer */
  85         bus_addr_t      busaddr;        /* Physical address */
  86         vm_offset_t     datavaddr;      /* kva of client data */
  87         vm_page_t       datapage;       /* physical page of client data */
  88         vm_offset_t     dataoffs;       /* page offset of client data */
  89         bus_size_t      datacount;      /* client data count */
  90         STAILQ_ENTRY(bounce_page) links;
  91 };
  92
  93 int busdma_swi_pending;
  94
  95 struct bounce_zone {
  96         STAILQ_ENTRY(bounce_zone) links;
  97         STAILQ_HEAD(bp_list, bounce_page) bounce_page_list;
  98         int             total_bpages;
  99         int             free_bpages;
 100         int             reserved_bpages;
 101         int             active_bpages;
 102         int             total_bounced;
 103         int             total_deferred;
 104         int             map_count;
 105         bus_size_t      alignment;
 106         bus_addr_t      lowaddr;
 107         char            zoneid[8];
 108         char            lowaddrid[20];
 109         struct sysctl_ctx_list sysctl_tree;
 110         struct sysctl_oid *sysctl_tree_top;
 111 };
 112
 113 static struct mtx bounce_lock;
 114 static int total_bpages;
 115 static int busdma_zonecount;
 116 static STAILQ_HEAD(, bounce_zone) bounce_zone_list;
 117
 118 static SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
 119     "Busdma parameters");
 120 SYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bpages, 0,
 121            "Total bounce pages");
 122
 123 struct sync_list {
 124         vm_offset_t     vaddr;          /* kva of client data */
 125         bus_addr_t      paddr;          /* physical address */
 126         vm_page_t       pages;          /* starting page of client data */
 127         bus_size_t      datacount;      /* client data count */
 128 };
 129
 130 struct bus_dmamap {
 131         struct bp_list         bpages;
 132         int                    pagesneeded;
 133         int                    pagesreserved;
 134         bus_dma_tag_t          dmat;
 135         struct memdesc         mem;
 136         bus_dmamap_callback_t *callback;
 137         void                  *callback_arg;
 138         STAILQ_ENTRY(bus_dmamap) links;
 139         u_int                   flags;
 140 #define DMAMAP_COHERENT         (1 << 0)
 141 #define DMAMAP_FROM_DMAMEM      (1 << 1)
 142         int                     sync_count;
 143         struct sync_list        slist[];
 144 };
 145
 146 static STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist;
 147 static STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist;
 148
 149 static void init_bounce_pages(void *dummy);
 150 static int alloc_bounce_zone(bus_dma_tag_t dmat);
 151 static int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages);
 152 static int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map,
 153     int commit);
 154 static bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map,
 155     vm_offset_t vaddr, bus_addr_t addr, bus_size_t size);
 156 static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage);
 157 int run_filter(bus_dma_tag_t dmat, bus_addr_t paddr);
 158 static bool _bus_dmamap_pagesneeded(bus_dma_tag_t dmat, vm_paddr_t buf,
 159     bus_size_t buflen, int *pagesneeded);
 160 static void _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map,
 161     pmap_t pmap, void *buf, bus_size_t buflen, int flags);
 162 static void _bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map,
 163     vm_paddr_t buf, bus_size_t buflen, int flags);
 164 static int _bus_dmamap_reserve_pages(bus_dma_tag_t dmat, bus_dmamap_t map,
 165     int flags);
 166
 167 static bool
 168 might_bounce(bus_dma_tag_t dmat)
 169 {
 170
 171         if ((dmat->bounce_flags & BF_COULD_BOUNCE) != 0)
 172                 return (true);
 173
 174         return (false);
 175 }
 176
 177 static bool
 178 must_bounce(bus_dma_tag_t dmat, bus_addr_t paddr)
 179 {
 180
 181         if ((dmat->bounce_flags & BF_COULD_BOUNCE) != 0 &&
 182             bus_dma_run_filter(&dmat->common, paddr))
 183                 return (true);
 184
 185         return (false);
 186 }
 187
 188 /*
 189  * Allocate a device specific dma_tag.
 190  */
 191 static int
 192 bounce_bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment,
 193     bus_addr_t boundary, bus_addr_t lowaddr, bus_addr_t highaddr,
 194     bus_dma_filter_t *filter, void *filterarg, bus_size_t maxsize,
 195     int nsegments, bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc,
 196     void *lockfuncarg, bus_dma_tag_t *dmat)
 197 {
 198         bus_dma_tag_t newtag;
 199         int error;
 200
 201         *dmat = NULL;
 202         error = common_bus_dma_tag_create(parent != NULL ? &parent->common :
 203             NULL, alignment, boundary, lowaddr, highaddr, filter, filterarg,
 204             maxsize, nsegments, maxsegsz, flags, lockfunc, lockfuncarg,
 205             sizeof (struct bus_dma_tag), (void **)&newtag);
 206         if (error != 0)
 207                 return (error);
 208
 209         newtag->common.impl = &bus_dma_bounce_impl;
 210         newtag->map_count = 0;
 211         newtag->segments = NULL;
 212
 213         if ((flags & BUS_DMA_COHERENT) != 0) {
 214                 newtag->bounce_flags |= BF_COHERENT;
 215                 newtag->alloc_alignment = newtag->common.alignment;
 216                 newtag->alloc_size = newtag->common.maxsize;
 217         } else {
 218                 /*
 219                  * Ensure the buffer is aligned to a cacheline when allocating
 220                  * a non-coherent buffer. This is so we don't have any data
 221                  * that another CPU may be accessing around DMA buffer
 222                  * causing the cache to become dirty.
 223                  */
 224                 newtag->alloc_alignment = MAX(newtag->common.alignment,
 225                     dcache_line_size);
 226                 newtag->alloc_size = roundup2(newtag->common.maxsize,
 227                     dcache_line_size);
 228         }
 229
 230         if (parent != NULL) {
 231                 if ((newtag->common.filter != NULL ||
 232                     (parent->bounce_flags & BF_COULD_BOUNCE) != 0))
 233                         newtag->bounce_flags |= BF_COULD_BOUNCE;
 234
 235                 /* Copy some flags from the parent */
 236                 newtag->bounce_flags |= parent->bounce_flags & BF_COHERENT;
 237         }
 238
 239         if (newtag->common.lowaddr < ptoa((vm_paddr_t)Maxmem) ||
 240             newtag->common.alignment > 1)
 241                 newtag->bounce_flags |= BF_COULD_BOUNCE;
 242
 243         if (((newtag->bounce_flags & BF_COULD_BOUNCE) != 0) &&
 244             (flags & BUS_DMA_ALLOCNOW) != 0) {
 245                 struct bounce_zone *bz;
 246
 247                 /* Must bounce */
 248                 if ((error = alloc_bounce_zone(newtag)) != 0) {
 249                         free(newtag, M_DEVBUF);
 250                         return (error);
 251                 }
 252                 bz = newtag->bounce_zone;
 253
 254                 if (ptoa(bz->total_bpages) < maxsize) {
 255                         int pages;
 256
 257                         pages = atop(round_page(maxsize)) - bz->total_bpages;
 258
 259                         /* Add pages to our bounce pool */
 260                         if (alloc_bounce_pages(newtag, pages) < pages)
 261                                 error = ENOMEM;
 262                 }
 263                 /* Performed initial allocation */
 264                 newtag->bounce_flags |= BF_MIN_ALLOC_COMP;
 265         } else
 266                 error = 0;
 267
 268         if (error != 0)
 269                 free(newtag, M_DEVBUF);
 270         else
 271                 *dmat = newtag;
 272         CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d",
 273             __func__, newtag, (newtag != NULL ? newtag->common.flags : 0),
 274             error);
 275         return (error);
 276 }
 277
 278 static int
 279 bounce_bus_dma_tag_destroy(bus_dma_tag_t dmat)
 280 {
 281         bus_dma_tag_t dmat_copy, parent;
 282         int error;
 283
 284         error = 0;
 285         dmat_copy = dmat;
 286
 287         if (dmat != NULL) {
 288                 if (dmat->map_count != 0) {
 289                         error = EBUSY;
 290                         goto out;
 291                 }
 292                 while (dmat != NULL) {
 293                         parent = (bus_dma_tag_t)dmat->common.parent;
 294                         atomic_subtract_int(&dmat->common.ref_count, 1);
 295                         if (dmat->common.ref_count == 0) {
 296                                 if (dmat->segments != NULL)
 297                                         free(dmat->segments, M_DEVBUF);
 298                                 free(dmat, M_DEVBUF);
 299                                 /*
 300                                  * Last reference count, so
 301                                  * release our reference
 302                                  * count on our parent.
 303                                  */
 304                                 dmat = parent;
 305                         } else
 306                                 dmat = NULL;
 307                 }
 308         }
 309 out:
 310         CTR3(KTR_BUSDMA, "%s tag %p error %d", __func__, dmat_copy, error);
 311         return (error);
 312 }
 313
 314 static bool
 315 bounce_bus_dma_id_mapped(bus_dma_tag_t dmat, vm_paddr_t buf, bus_size_t buflen)
 316 {
 317
 318         if (!might_bounce(dmat))
 319                 return (true);
 320         return (!_bus_dmamap_pagesneeded(dmat, buf, buflen, NULL));
 321 }
 322
 323 static bus_dmamap_t
 324 alloc_dmamap(bus_dma_tag_t dmat, int flags)
 325 {
 326         u_long mapsize;
 327         bus_dmamap_t map;
 328
 329         mapsize = sizeof(*map);
 330         mapsize += sizeof(struct sync_list) * dmat->common.nsegments;
 331         map = malloc(mapsize, M_DEVBUF, flags | M_ZERO);
 332         if (map == NULL)
 333                 return (NULL);
 334
 335         /* Initialize the new map */
 336         STAILQ_INIT(&map->bpages);
 337
 338         return (map);
 339 }
 340
 341 /*
 342  * Allocate a handle for mapping from kva/uva/physical
 343  * address space into bus device space.
 344  */
 345 static int
 346 bounce_bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp)
 347 {
 348         struct bounce_zone *bz;
 349         int error, maxpages, pages;
 350
 351         error = 0;
 352
 353         if (dmat->segments == NULL) {
 354                 dmat->segments = (bus_dma_segment_t *)malloc(
 355                     sizeof(bus_dma_segment_t) * dmat->common.nsegments,
 356                     M_DEVBUF, M_NOWAIT);
 357                 if (dmat->segments == NULL) {
 358                         CTR3(KTR_BUSDMA, "%s: tag %p error %d",
 359                             __func__, dmat, ENOMEM);
 360                         return (ENOMEM);
 361                 }
 362         }
 363
 364         *mapp = alloc_dmamap(dmat, M_NOWAIT);
 365         if (*mapp == NULL) {
 366                 CTR3(KTR_BUSDMA, "%s: tag %p error %d",
 367                     __func__, dmat, ENOMEM);
 368                 return (ENOMEM);
 369         }
 370
 371         /*
 372          * Bouncing might be required if the driver asks for an active
 373          * exclusion region, a data alignment that is stricter than 1, and/or
 374          * an active address boundary.
 375          */
 376         if (dmat->bounce_flags & BF_COULD_BOUNCE) {
 377                 /* Must bounce */
 378                 if (dmat->bounce_zone == NULL) {
 379                         if ((error = alloc_bounce_zone(dmat)) != 0) {
 380                                 free(*mapp, M_DEVBUF);
 381                                 return (error);
 382                         }
 383                 }
 384                 bz = dmat->bounce_zone;
 385
 386                 /*
 387                  * Attempt to add pages to our pool on a per-instance
 388                  * basis up to a sane limit.
 389                  */
 390                 if (dmat->common.alignment > 1)
 391                         maxpages = MAX_BPAGES;
 392                 else
 393                         maxpages = MIN(MAX_BPAGES, Maxmem -
 394                             atop(dmat->common.lowaddr));
 395                 if ((dmat->bounce_flags & BF_MIN_ALLOC_COMP) == 0 ||
 396                     (bz->map_count > 0 && bz->total_bpages < maxpages)) {
 397                         pages = MAX(atop(dmat->common.maxsize), 1);
 398                         pages = MIN(maxpages - bz->total_bpages, pages);
 399                         pages = MAX(pages, 1);
 400                         if (alloc_bounce_pages(dmat, pages) < pages)
 401                                 error = ENOMEM;
 402                         if ((dmat->bounce_flags & BF_MIN_ALLOC_COMP)
 403                             == 0) {
 404                                 if (error == 0) {
 405                                         dmat->bounce_flags |=
 406                                             BF_MIN_ALLOC_COMP;
 407                                 }
 408                         } else
 409                                 error = 0;
 410                 }
 411                 bz->map_count++;
 412         }
 413         if (error == 0) {
 414                 dmat->map_count++;
 415                 if ((dmat->bounce_flags & BF_COHERENT) != 0)
 416                         (*mapp)->flags |= DMAMAP_COHERENT;
 417         } else {
 418                 free(*mapp, M_DEVBUF);
 419         }
 420         CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
 421             __func__, dmat, dmat->common.flags, error);
 422         return (error);
 423 }
 424
 425 /*
 426  * Destroy a handle for mapping from kva/uva/physical
 427  * address space into bus device space.
 428  */
 429 static int
 430 bounce_bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map)
 431 {
 432
 433         /* Check we are destroying the correct map type */
 434         if ((map->flags & DMAMAP_FROM_DMAMEM) != 0)
 435                 panic("bounce_bus_dmamap_destroy: Invalid map freed\n");
 436
 437         if (STAILQ_FIRST(&map->bpages) != NULL || map->sync_count != 0) {
 438                 CTR3(KTR_BUSDMA, "%s: tag %p error %d", __func__, dmat, EBUSY);
 439                 return (EBUSY);
 440         }
 441         if (dmat->bounce_zone)
 442                 dmat->bounce_zone->map_count--;
 443         free(map, M_DEVBUF);
 444         dmat->map_count--;
 445         CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat);
 446         return (0);
 447 }
 448
 449 /*
 450  * Allocate a piece of memory that can be efficiently mapped into
 451  * bus device space based on the constraints lited in the dma tag.
 452  * A dmamap to for use with dmamap_load is also allocated.
 453  */
 454 static int
 455 bounce_bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags,
 456     bus_dmamap_t *mapp)
 457 {
 458         /*
 459          * XXX ARM64TODO:
 460          * This bus_dma implementation requires IO-Coherent architecutre.
 461          * If IO-Coherency is not guaranteed, the BUS_DMA_COHERENT flag has
 462          * to be implented using non-cacheable memory.
 463          */
 464
 465         vm_memattr_t attr;
 466         int mflags;
 467
 468         if (flags & BUS_DMA_NOWAIT)
 469                 mflags = M_NOWAIT;
 470         else
 471                 mflags = M_WAITOK;
 472
 473         if (dmat->segments == NULL) {
 474                 dmat->segments = (bus_dma_segment_t *)malloc(
 475                     sizeof(bus_dma_segment_t) * dmat->common.nsegments,
 476                     M_DEVBUF, mflags);
 477                 if (dmat->segments == NULL) {
 478                         CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
 479                             __func__, dmat, dmat->common.flags, ENOMEM);
 480                         return (ENOMEM);
 481                 }
 482         }
 483         if (flags & BUS_DMA_ZERO)
 484                 mflags |= M_ZERO;
 485         if (flags & BUS_DMA_NOCACHE)
 486                 attr = VM_MEMATTR_UNCACHEABLE;
 487         else if ((flags & BUS_DMA_COHERENT) != 0 &&
 488             (dmat->bounce_flags & BF_COHERENT) == 0)
 489                 /*
 490                  * If we have a non-coherent tag, and are trying to allocate
 491                  * a coherent block of memory it needs to be uncached.
 492                  */
 493                 attr = VM_MEMATTR_UNCACHEABLE;
 494         else
 495                 attr = VM_MEMATTR_DEFAULT;
 496
 497         /*
 498          * Create the map, but don't set the could bounce flag as
 499          * this allocation should never bounce;
 500          */
 501         *mapp = alloc_dmamap(dmat, mflags);
 502         if (*mapp == NULL) {
 503                 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
 504                     __func__, dmat, dmat->common.flags, ENOMEM);
 505                 return (ENOMEM);
 506         }
 507
 508         /*
 509          * Mark the map as coherent if we used uncacheable memory or the
 510          * tag was already marked as coherent.
 511          */
 512         if (attr == VM_MEMATTR_UNCACHEABLE ||
 513             (dmat->bounce_flags & BF_COHERENT) != 0)
 514                 (*mapp)->flags |= DMAMAP_COHERENT;
 515
 516         (*mapp)->flags |= DMAMAP_FROM_DMAMEM;
 517
 518         /*
 519          * Allocate the buffer from the malloc(9) allocator if...
 520          *  - It's small enough to fit into a single power of two sized bucket.
 521          *  - The alignment is less than or equal to the maximum size
 522          *  - The low address requirement is fulfilled.
 523          * else allocate non-contiguous pages if...
 524          *  - The page count that could get allocated doesn't exceed
 525          *    nsegments also when the maximum segment size is less
 526          *    than PAGE_SIZE.
 527          *  - The alignment constraint isn't larger than a page boundary.
 528          *  - There are no boundary-crossing constraints.
 529          * else allocate a block of contiguous pages because one or more of the
 530          * constraints is something that only the contig allocator can fulfill.
 531          *
 532          * NOTE: The (dmat->common.alignment <= dmat->maxsize) check
 533          * below is just a quick hack. The exact alignment guarantees
 534          * of malloc(9) need to be nailed down, and the code below
 535          * should be rewritten to take that into account.
 536          *
 537          * In the meantime warn the user if malloc gets it wrong.
 538          */
 539         if ((dmat->alloc_size <= PAGE_SIZE) &&
 540            (dmat->alloc_alignment <= dmat->alloc_size) &&
 541             dmat->common.lowaddr >= ptoa((vm_paddr_t)Maxmem) &&
 542             attr == VM_MEMATTR_DEFAULT) {
 543                 *vaddr = malloc(dmat->alloc_size, M_DEVBUF, mflags);
 544         } else if (dmat->common.nsegments >=
 545             howmany(dmat->alloc_size, MIN(dmat->common.maxsegsz, PAGE_SIZE)) &&
 546             dmat->alloc_alignment <= PAGE_SIZE &&
 547             (dmat->common.boundary % PAGE_SIZE) == 0) {
 548                 /* Page-based multi-segment allocations allowed */
 549                 *vaddr = (void *)kmem_alloc_attr(dmat->alloc_size, mflags,
 550                     0ul, dmat->common.lowaddr, attr);
 551                 dmat->bounce_flags |= BF_KMEM_ALLOC;
 552         } else {
 553                 *vaddr = (void *)kmem_alloc_contig(dmat->alloc_size, mflags,
 554                     0ul, dmat->common.lowaddr, dmat->alloc_alignment != 0 ?
 555                     dmat->alloc_alignment : 1ul, dmat->common.boundary, attr);
 556                 dmat->bounce_flags |= BF_KMEM_ALLOC;
 557         }
 558         if (*vaddr == NULL) {
 559                 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
 560                     __func__, dmat, dmat->common.flags, ENOMEM);
 561                 free(*mapp, M_DEVBUF);
 562                 return (ENOMEM);
 563         } else if (vtophys(*vaddr) & (dmat->alloc_alignment - 1)) {
 564                 printf("bus_dmamem_alloc failed to align memory properly.\n");
 565         }
 566         dmat->map_count++;
 567         CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
 568             __func__, dmat, dmat->common.flags, 0);
 569         return (0);
 570 }
 571
 572 /*
 573  * Free a piece of memory and it's allociated dmamap, that was allocated
 574  * via bus_dmamem_alloc.  Make the same choice for free/contigfree.
 575  */
 576 static void
 577 bounce_bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map)
 578 {
 579
 580         /*
 581          * Check the map came from bounce_bus_dmamem_alloc, so the map
 582          * should be NULL and the BF_KMEM_ALLOC flag cleared if malloc()
 583          * was used and set if kmem_alloc_contig() was used.
 584          */
 585         if ((map->flags & DMAMAP_FROM_DMAMEM) == 0)
 586                 panic("bus_dmamem_free: Invalid map freed\n");
 587         if ((dmat->bounce_flags & BF_KMEM_ALLOC) == 0)
 588                 free(vaddr, M_DEVBUF);
 589         else
 590                 kmem_free((vm_offset_t)vaddr, dmat->alloc_size);
 591         free(map, M_DEVBUF);
 592         dmat->map_count--;
 593         CTR3(KTR_BUSDMA, "%s: tag %p flags 0x%x", __func__, dmat,
 594             dmat->bounce_flags);
 595 }
 596
 597 static bool
 598 _bus_dmamap_pagesneeded(bus_dma_tag_t dmat, vm_paddr_t buf, bus_size_t buflen,
 599     int *pagesneeded)
 600 {
 601         bus_addr_t curaddr;
 602         bus_size_t sgsize;
 603         int count;
 604
 605         /*
 606          * Count the number of bounce pages needed in order to
 607          * complete this transfer
 608          */
 609         count = 0;
 610         curaddr = buf;
 611         while (buflen != 0) {
 612                 sgsize = MIN(buflen, dmat->common.maxsegsz);
 613                 if (must_bounce(dmat, curaddr)) {
 614                         sgsize = MIN(sgsize,
 615                             PAGE_SIZE - (curaddr & PAGE_MASK));
 616                         if (pagesneeded == NULL)
 617                                 return (true);
 618                         count++;
 619                 }
 620                 curaddr += sgsize;
 621                 buflen -= sgsize;
 622         }
 623
 624         if (pagesneeded != NULL)
 625                 *pagesneeded = count;
 626         return (count != 0);
 627 }
 628
 629 static void
 630 _bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf,
 631     bus_size_t buflen, int flags)
 632 {
 633
 634         if (map->pagesneeded == 0) {
 635                 _bus_dmamap_pagesneeded(dmat, buf, buflen, &map->pagesneeded);
 636                 CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded);
 637         }
 638 }
 639
 640 static void
 641 _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap,
 642     void *buf, bus_size_t buflen, int flags)
 643 {
 644         vm_offset_t vaddr;
 645         vm_offset_t vendaddr;
 646         bus_addr_t paddr;
 647         bus_size_t sg_len;
 648
 649         if (map->pagesneeded == 0) {
 650                 CTR4(KTR_BUSDMA, "lowaddr= %d Maxmem= %d, boundary= %d, "
 651                     "alignment= %d", dmat->common.lowaddr,
 652                     ptoa((vm_paddr_t)Maxmem),
 653                     dmat->common.boundary, dmat->common.alignment);
 654                 CTR2(KTR_BUSDMA, "map= %p, pagesneeded= %d", map,
 655                     map->pagesneeded);
 656                 /*
 657                  * Count the number of bounce pages
 658                  * needed in order to complete this transfer
 659                  */
 660                 vaddr = (vm_offset_t)buf;
 661                 vendaddr = (vm_offset_t)buf + buflen;
 662
 663                 while (vaddr < vendaddr) {
 664                         sg_len = PAGE_SIZE - ((vm_offset_t)vaddr & PAGE_MASK);
 665                         if (pmap == kernel_pmap)
 666                                 paddr = pmap_kextract(vaddr);
 667                         else
 668                                 paddr = pmap_extract(pmap, vaddr);
 669                         if (must_bounce(dmat, paddr)) {
 670                                 sg_len = roundup2(sg_len,
 671                                     dmat->common.alignment);
 672                                 map->pagesneeded++;
 673                         }
 674                         vaddr += sg_len;
 675                 }
 676                 CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded);
 677         }
 678 }
 679
 680 static int
 681 _bus_dmamap_reserve_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int flags)
 682 {
 683
 684         /* Reserve Necessary Bounce Pages */
 685         mtx_lock(&bounce_lock);
 686         if (flags & BUS_DMA_NOWAIT) {
 687                 if (reserve_bounce_pages(dmat, map, 0) != 0) {
 688                         mtx_unlock(&bounce_lock);
 689                         return (ENOMEM);
 690                 }
 691         } else {
 692                 if (reserve_bounce_pages(dmat, map, 1) != 0) {
 693                         /* Queue us for resources */
 694                         STAILQ_INSERT_TAIL(&bounce_map_waitinglist, map, links);
 695                         mtx_unlock(&bounce_lock);
 696                         return (EINPROGRESS);
 697                 }
 698         }
 699         mtx_unlock(&bounce_lock);
 700
 701         return (0);
 702 }
 703
 704 /*
 705  * Add a single contiguous physical range to the segment list.
 706  */
 707 static bus_size_t
 708 _bus_dmamap_addseg(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t curaddr,
 709     bus_size_t sgsize, bus_dma_segment_t *segs, int *segp)
 710 {
 711         bus_addr_t baddr, bmask;
 712         int seg;
 713
 714         /*
 715          * Make sure we don't cross any boundaries.
 716          */
 717         bmask = ~(dmat->common.boundary - 1);
 718         if (dmat->common.boundary > 0) {
 719                 baddr = (curaddr + dmat->common.boundary) & bmask;
 720                 if (sgsize > (baddr - curaddr))
 721                         sgsize = (baddr - curaddr);
 722         }
 723
 724         /*
 725          * Insert chunk into a segment, coalescing with
 726          * previous segment if possible.
 727          */
 728         seg = *segp;
 729         if (seg == -1) {
 730                 seg = 0;
 731                 segs[seg].ds_addr = curaddr;
 732                 segs[seg].ds_len = sgsize;
 733         } else {
 734                 if (curaddr == segs[seg].ds_addr + segs[seg].ds_len &&
 735                     (segs[seg].ds_len + sgsize) <= dmat->common.maxsegsz &&
 736                     (dmat->common.boundary == 0 ||
 737                      (segs[seg].ds_addr & bmask) == (curaddr & bmask)))
 738                         segs[seg].ds_len += sgsize;
 739                 else {
 740                         if (++seg >= dmat->common.nsegments)
 741                                 return (0);
 742                         segs[seg].ds_addr = curaddr;
 743                         segs[seg].ds_len = sgsize;
 744                 }
 745         }
 746         *segp = seg;
 747         return (sgsize);
 748 }
 749
 750 /*
 751  * Utility function to load a physical buffer.  segp contains
 752  * the starting segment on entrace, and the ending segment on exit.
 753  */
 754 static int
 755 bounce_bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map,
 756     vm_paddr_t buf, bus_size_t buflen, int flags, bus_dma_segment_t *segs,
 757     int *segp)
 758 {
 759         struct sync_list *sl;
 760         bus_size_t sgsize;
 761         bus_addr_t curaddr, sl_end;
 762         int error;
 763
 764         if (segs == NULL)
 765                 segs = dmat->segments;
 766
 767         if (might_bounce(dmat)) {
 768                 _bus_dmamap_count_phys(dmat, map, buf, buflen, flags);
 769                 if (map->pagesneeded != 0) {
 770                         error = _bus_dmamap_reserve_pages(dmat, map, flags);
 771                         if (error)
 772                                 return (error);
 773                 }
 774         }
 775
 776         sl = map->slist + map->sync_count - 1;
 777         sl_end = 0;
 778
 779         while (buflen > 0) {
 780                 curaddr = buf;
 781                 sgsize = MIN(buflen, dmat->common.maxsegsz);
 782                 if (map->pagesneeded != 0 && must_bounce(dmat, curaddr)) {
 783                         sgsize = MIN(sgsize, PAGE_SIZE - (curaddr & PAGE_MASK));
 784                         curaddr = add_bounce_page(dmat, map, 0, curaddr,
 785                             sgsize);
 786                 } else if ((map->flags & DMAMAP_COHERENT) == 0) {
 787                         if (map->sync_count > 0)
 788                                 sl_end = sl->paddr + sl->datacount;
 789
 790                         if (map->sync_count == 0 || curaddr != sl_end) {
 791                                 if (++map->sync_count > dmat->common.nsegments)
 792                                         break;
 793                                 sl++;
 794                                 sl->vaddr = 0;
 795                                 sl->paddr = curaddr;
 796                                 sl->datacount = sgsize;
 797                                 sl->pages = PHYS_TO_VM_PAGE(curaddr);
 798                                 KASSERT(sl->pages != NULL,
 799                                     ("%s: page at PA:0x%08lx is not in "
 800                                     "vm_page_array", __func__, curaddr));
 801                         } else
 802                                 sl->datacount += sgsize;
 803                 }
 804                 sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs,
 805                     segp);
 806                 if (sgsize == 0)
 807                         break;
 808                 buf += sgsize;
 809                 buflen -= sgsize;
 810         }
 811
 812         /*
 813          * Did we fit?
 814          */
 815         return (buflen != 0 ? EFBIG : 0); /* XXX better return value here? */
 816 }
 817
 818 /*
 819  * Utility function to load a linear buffer.  segp contains
 820  * the starting segment on entrace, and the ending segment on exit.
 821  */
 822 static int
 823 bounce_bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf,
 824     bus_size_t buflen, pmap_t pmap, int flags, bus_dma_segment_t *segs,
 825     int *segp)
 826 {
 827         struct sync_list *sl;
 828         bus_size_t sgsize, max_sgsize;
 829         bus_addr_t curaddr, sl_pend;
 830         vm_offset_t kvaddr, vaddr, sl_vend;
 831         int error;
 832
 833         if (segs == NULL)
 834                 segs = dmat->segments;
 835
 836         if (might_bounce(dmat)) {
 837                 _bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, flags);
 838                 if (map->pagesneeded != 0) {
 839                         error = _bus_dmamap_reserve_pages(dmat, map, flags);
 840                         if (error)
 841                                 return (error);
 842                 }
 843         }
 844
 845         sl = map->slist + map->sync_count - 1;
 846         vaddr = (vm_offset_t)buf;
 847         sl_pend = 0;
 848         sl_vend = 0;
 849
 850         while (buflen > 0) {
 851                 /*
 852                  * Get the physical address for this segment.
 853                  */
 854                 if (pmap == kernel_pmap) {
 855                         curaddr = pmap_kextract(vaddr);
 856                         kvaddr = vaddr;
 857                 } else {
 858                         curaddr = pmap_extract(pmap, vaddr);
 859                         kvaddr = 0;
 860                 }
 861
 862                 /*
 863                  * Compute the segment size, and adjust counts.
 864                  */
 865                 max_sgsize = MIN(buflen, dmat->common.maxsegsz);
 866                 sgsize = PAGE_SIZE - (curaddr & PAGE_MASK);
 867                 if (map->pagesneeded != 0 && must_bounce(dmat, curaddr)) {
 868                         sgsize = roundup2(sgsize, dmat->common.alignment);
 869                         sgsize = MIN(sgsize, max_sgsize);
 870                         curaddr = add_bounce_page(dmat, map, kvaddr, curaddr,
 871                             sgsize);
 872                 } else if ((map->flags & DMAMAP_COHERENT) == 0) {
 873                         sgsize = MIN(sgsize, max_sgsize);
 874                         if (map->sync_count > 0) {
 875                                 sl_pend = sl->paddr + sl->datacount;
 876                                 sl_vend = sl->vaddr + sl->datacount;
 877                         }
 878
 879                         if (map->sync_count == 0 ||
 880                             (kvaddr != 0 && kvaddr != sl_vend) ||
 881                             (curaddr != sl_pend)) {
 882                                 if (++map->sync_count > dmat->common.nsegments)
 883                                         goto cleanup;
 884                                 sl++;
 885                                 sl->vaddr = kvaddr;
 886                                 sl->paddr = curaddr;
 887                                 if (kvaddr != 0) {
 888                                         sl->pages = NULL;
 889                                 } else {
 890                                         sl->pages = PHYS_TO_VM_PAGE(curaddr);
 891                                         KASSERT(sl->pages != NULL,
 892                                             ("%s: page at PA:0x%08lx is not "
 893                                             "in vm_page_array", __func__,
 894                                             curaddr));
 895                                 }
 896                                 sl->datacount = sgsize;
 897                         } else
 898                                 sl->datacount += sgsize;
 899                 } else {
 900                         sgsize = MIN(sgsize, max_sgsize);
 901                 }
 902                 sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs,
 903                     segp);
 904                 if (sgsize == 0)
 905                         break;
 906                 vaddr += sgsize;
 907                 buflen -= sgsize;
 908         }
 909
 910 cleanup:
 911         /*
 912          * Did we fit?
 913          */
 914         return (buflen != 0 ? EFBIG : 0); /* XXX better return value here? */
 915 }
 916
 917 static void
 918 bounce_bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map,
 919     struct memdesc *mem, bus_dmamap_callback_t *callback, void *callback_arg)
 920 {
 921
 922         map->mem = *mem;
 923         map->dmat = dmat;
 924         map->callback = callback;
 925         map->callback_arg = callback_arg;
 926 }
 927
 928 static bus_dma_segment_t *
 929 bounce_bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map,
 930     bus_dma_segment_t *segs, int nsegs, int error)
 931 {
 932
 933         if (segs == NULL)
 934                 segs = dmat->segments;
 935         return (segs);
 936 }
 937
 938 /*
 939  * Release the mapping held by map.
 940  */
 941 static void
 942 bounce_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map)
 943 {
 944         struct bounce_page *bpage;
 945
 946         while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) {
 947                 STAILQ_REMOVE_HEAD(&map->bpages, links);
 948                 free_bounce_page(dmat, bpage);
 949         }
 950
 951         map->sync_count = 0;
 952 }
 953
 954 static void
 955 dma_preread_safe(vm_offset_t va, vm_size_t size)
 956 {
 957         /*
 958          * Write back any partial cachelines immediately before and
 959          * after the DMA region.
 960          */
 961         if (va & (dcache_line_size - 1))
 962                 cpu_dcache_wb_range(va, 1);
 963         if ((va + size) & (dcache_line_size - 1))
 964                 cpu_dcache_wb_range(va + size, 1);
 965
 966         cpu_dcache_inv_range(va, size);
 967 }
 968
 969 static void
 970 dma_dcache_sync(struct sync_list *sl, bus_dmasync_op_t op)
 971 {
 972         uint32_t len, offset;
 973         vm_page_t m;
 974         vm_paddr_t pa;
 975         vm_offset_t va, tempva;
 976         bus_size_t size;
 977
 978         offset = sl->paddr & PAGE_MASK;
 979         m = sl->pages;
 980         size = sl->datacount;
 981         pa = sl->paddr;
 982
 983         for ( ; size != 0; size -= len, pa += len, offset = 0, ++m) {
 984                 tempva = 0;
 985                 if (sl->vaddr == 0) {
 986                         len = min(PAGE_SIZE - offset, size);
 987                         tempva = pmap_quick_enter_page(m);
 988                         va = tempva | offset;
 989                         KASSERT(pa == (VM_PAGE_TO_PHYS(m) | offset),
 990                             ("unexpected vm_page_t phys: 0x%16lx != 0x%16lx",
 991                             VM_PAGE_TO_PHYS(m) | offset, pa));
 992                 } else {
 993                         len = sl->datacount;
 994                         va = sl->vaddr;
 995                 }
 996
 997                 switch (op) {
 998                 case BUS_DMASYNC_PREWRITE:
 999                 case BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD:
1000                         cpu_dcache_wb_range(va, len);
1001                         break;
1002                 case BUS_DMASYNC_PREREAD:
1003                         /*
1004                          * An mbuf may start in the middle of a cacheline. There
1005                          * will be no cpu writes to the beginning of that line
1006                          * (which contains the mbuf header) while dma is in
1007                          * progress.  Handle that case by doing a writeback of
1008                          * just the first cacheline before invalidating the
1009                          * overall buffer.  Any mbuf in a chain may have this
1010                          * misalignment.  Buffers which are not mbufs bounce if
1011                          * they are not aligned to a cacheline.
1012                          */
1013                         dma_preread_safe(va, len);
1014                         break;
1015                 case BUS_DMASYNC_POSTREAD:
1016                 case BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE:
1017                         cpu_dcache_inv_range(va, len);
1018                         break;
1019                 default:
1020                         panic("unsupported combination of sync operations: "
1021                               "0x%08x\n", op);
1022                 }
1023
1024                 if (tempva != 0)
1025                         pmap_quick_remove_page(tempva);
1026         }
1027 }
1028
1029 static void
1030 bounce_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map,
1031     bus_dmasync_op_t op)
1032 {
1033         struct bounce_page *bpage;
1034         struct sync_list *sl, *end;
1035         vm_offset_t datavaddr, tempvaddr;
1036
1037         if (op == BUS_DMASYNC_POSTWRITE)
1038                 return;
1039
1040         if ((op & BUS_DMASYNC_POSTREAD) != 0) {
1041                 /*
1042                  * Wait for any DMA operations to complete before the bcopy.
1043                  */
1044                 dsb(sy);
1045         }
1046
1047         if ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) {
1048                 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x op 0x%x "
1049                     "performing bounce", __func__, dmat, dmat->common.flags,
1050                     op);
1051
1052                 if ((op & BUS_DMASYNC_PREWRITE) != 0) {
1053                         while (bpage != NULL) {
1054                                 tempvaddr = 0;
1055                                 datavaddr = bpage->datavaddr;
1056                                 if (datavaddr == 0) {
1057                                         tempvaddr = pmap_quick_enter_page(
1058                                             bpage->datapage);
1059                                         datavaddr = tempvaddr | bpage->dataoffs;
1060                                 }
1061
1062                                 bcopy((void *)datavaddr,
1063                                     (void *)bpage->vaddr, bpage->datacount);
1064                                 if (tempvaddr != 0)
1065                                         pmap_quick_remove_page(tempvaddr);
1066                                 if ((map->flags & DMAMAP_COHERENT) == 0)
1067                                         cpu_dcache_wb_range(bpage->vaddr,
1068                                             bpage->datacount);
1069                                 bpage = STAILQ_NEXT(bpage, links);
1070                         }
1071                         dmat->bounce_zone->total_bounced++;
1072                 } else if ((op & BUS_DMASYNC_PREREAD) != 0) {
1073                         while (bpage != NULL) {
1074                                 if ((map->flags & DMAMAP_COHERENT) == 0)
1075                                         cpu_dcache_wbinv_range(bpage->vaddr,
1076                                             bpage->datacount);
1077                                 bpage = STAILQ_NEXT(bpage, links);
1078                         }
1079                 }
1080
1081                 if ((op & BUS_DMASYNC_POSTREAD) != 0) {
1082                         while (bpage != NULL) {
1083                                 if ((map->flags & DMAMAP_COHERENT) == 0)
1084                                         cpu_dcache_inv_range(bpage->vaddr,
1085                                             bpage->datacount);
1086                                 tempvaddr = 0;
1087                                 datavaddr = bpage->datavaddr;
1088                                 if (datavaddr == 0) {
1089                                         tempvaddr = pmap_quick_enter_page(
1090                                             bpage->datapage);
1091                                         datavaddr = tempvaddr | bpage->dataoffs;
1092                                 }
1093
1094                                 bcopy((void *)bpage->vaddr,
1095                                     (void *)datavaddr, bpage->datacount);
1096
1097                                 if (tempvaddr != 0)
1098                                         pmap_quick_remove_page(tempvaddr);
1099                                 bpage = STAILQ_NEXT(bpage, links);
1100                         }
1101                         dmat->bounce_zone->total_bounced++;
1102                 }
1103         }
1104
1105         /*
1106          * Cache maintenance for normal (non-COHERENT non-bounce) buffers.
1107          */
1108         if (map->sync_count != 0) {
1109                 sl = &map->slist[0];
1110                 end = &map->slist[map->sync_count];
1111                 CTR3(KTR_BUSDMA, "%s: tag %p op 0x%x "
1112                     "performing sync", __func__, dmat, op);
1113
1114                 for ( ; sl != end; ++sl)
1115                         dma_dcache_sync(sl, op);
1116         }
1117
1118         if ((op & (BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE)) != 0) {
1119                 /*
1120                  * Wait for the bcopy to complete before any DMA operations.
1121                  */
1122                 dsb(sy);
1123         }
1124 }
1125
1126 static void
1127 init_bounce_pages(void *dummy __unused)
1128 {
1129
1130         total_bpages = 0;
1131         STAILQ_INIT(&bounce_zone_list);
1132         STAILQ_INIT(&bounce_map_waitinglist);
1133         STAILQ_INIT(&bounce_map_callbacklist);
1134         mtx_init(&bounce_lock, "bounce pages lock", NULL, MTX_DEF);
1135 }
1136 SYSINIT(bpages, SI_SUB_LOCK, SI_ORDER_ANY, init_bounce_pages, NULL);
1137
1138 static struct sysctl_ctx_list *
1139 busdma_sysctl_tree(struct bounce_zone *bz)
1140 {
1141
1142         return (&bz->sysctl_tree);
1143 }
1144
1145 static struct sysctl_oid *
1146 busdma_sysctl_tree_top(struct bounce_zone *bz)
1147 {
1148
1149         return (bz->sysctl_tree_top);
1150 }
1151
1152 static int
1153 alloc_bounce_zone(bus_dma_tag_t dmat)
1154 {
1155         struct bounce_zone *bz;
1156
1157         /* Check to see if we already have a suitable zone */
1158         STAILQ_FOREACH(bz, &bounce_zone_list, links) {
1159                 if ((dmat->common.alignment <= bz->alignment) &&
1160                     (dmat->common.lowaddr >= bz->lowaddr)) {
1161                         dmat->bounce_zone = bz;
1162                         return (0);
1163                 }
1164         }
1165
1166         if ((bz = (struct bounce_zone *)malloc(sizeof(*bz), M_DEVBUF,
1167             M_NOWAIT | M_ZERO)) == NULL)
1168                 return (ENOMEM);
1169
1170         STAILQ_INIT(&bz->bounce_page_list);
1171         bz->free_bpages = 0;
1172         bz->reserved_bpages = 0;
1173         bz->active_bpages = 0;
1174         bz->lowaddr = dmat->common.lowaddr;
1175         bz->alignment = MAX(dmat->common.alignment, PAGE_SIZE);
1176         bz->map_count = 0;
1177         snprintf(bz->zoneid, 8, "zone%d", busdma_zonecount);
1178         busdma_zonecount++;
1179         snprintf(bz->lowaddrid, 18, "%#jx", (uintmax_t)bz->lowaddr);
1180         STAILQ_INSERT_TAIL(&bounce_zone_list, bz, links);
1181         dmat->bounce_zone = bz;
1182
1183         sysctl_ctx_init(&bz->sysctl_tree);
1184         bz->sysctl_tree_top = SYSCTL_ADD_NODE(&bz->sysctl_tree,
1185             SYSCTL_STATIC_CHILDREN(_hw_busdma), OID_AUTO, bz->zoneid,
1186             CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "");
1187         if (bz->sysctl_tree_top == NULL) {
1188                 sysctl_ctx_free(&bz->sysctl_tree);
1189                 return (0);     /* XXX error code? */
1190         }
1191
1192         SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1193             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1194             "total_bpages", CTLFLAG_RD, &bz->total_bpages, 0,
1195             "Total bounce pages");
1196         SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1197             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1198             "free_bpages", CTLFLAG_RD, &bz->free_bpages, 0,
1199             "Free bounce pages");
1200         SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1201             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1202             "reserved_bpages", CTLFLAG_RD, &bz->reserved_bpages, 0,
1203             "Reserved bounce pages");
1204         SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1205             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1206             "active_bpages", CTLFLAG_RD, &bz->active_bpages, 0,
1207             "Active bounce pages");
1208         SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1209             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1210             "total_bounced", CTLFLAG_RD, &bz->total_bounced, 0,
1211             "Total bounce requests");
1212         SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1213             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1214             "total_deferred", CTLFLAG_RD, &bz->total_deferred, 0,
1215             "Total bounce requests that were deferred");
1216         SYSCTL_ADD_STRING(busdma_sysctl_tree(bz),
1217             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1218             "lowaddr", CTLFLAG_RD, bz->lowaddrid, 0, "");
1219         SYSCTL_ADD_UAUTO(busdma_sysctl_tree(bz),
1220             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1221             "alignment", CTLFLAG_RD, &bz->alignment, "");
1222
1223         return (0);
1224 }
1225
1226 static int
1227 alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages)
1228 {
1229         struct bounce_zone *bz;
1230         int count;
1231
1232         bz = dmat->bounce_zone;
1233         count = 0;
1234         while (numpages > 0) {
1235                 struct bounce_page *bpage;
1236
1237                 bpage = (struct bounce_page *)malloc(sizeof(*bpage), M_DEVBUF,
1238                                                      M_NOWAIT | M_ZERO);
1239
1240                 if (bpage == NULL)
1241                         break;
1242                 bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_DEVBUF,
1243                     M_NOWAIT, 0ul, bz->lowaddr, PAGE_SIZE, 0);
1244                 if (bpage->vaddr == 0) {
1245                         free(bpage, M_DEVBUF);
1246                         break;
1247                 }
1248                 bpage->busaddr = pmap_kextract(bpage->vaddr);
1249                 mtx_lock(&bounce_lock);
1250                 STAILQ_INSERT_TAIL(&bz->bounce_page_list, bpage, links);
1251                 total_bpages++;
1252                 bz->total_bpages++;
1253                 bz->free_bpages++;
1254                 mtx_unlock(&bounce_lock);
1255                 count++;
1256                 numpages--;
1257         }
1258         return (count);
1259 }
1260
1261 static int
1262 reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int commit)
1263 {
1264         struct bounce_zone *bz;
1265         int pages;
1266
1267         mtx_assert(&bounce_lock, MA_OWNED);
1268         bz = dmat->bounce_zone;
1269         pages = MIN(bz->free_bpages, map->pagesneeded - map->pagesreserved);
1270         if (commit == 0 && map->pagesneeded > (map->pagesreserved + pages))
1271                 return (map->pagesneeded - (map->pagesreserved + pages));
1272         bz->free_bpages -= pages;
1273         bz->reserved_bpages += pages;
1274         map->pagesreserved += pages;
1275         pages = map->pagesneeded - map->pagesreserved;
1276
1277         return (pages);
1278 }
1279
1280 static bus_addr_t
1281 add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr,
1282                 bus_addr_t addr, bus_size_t size)
1283 {
1284         struct bounce_zone *bz;
1285         struct bounce_page *bpage;
1286
1287         KASSERT(dmat->bounce_zone != NULL, ("no bounce zone in dma tag"));
1288
1289         bz = dmat->bounce_zone;
1290         if (map->pagesneeded == 0)
1291                 panic("add_bounce_page: map doesn't need any pages");
1292         map->pagesneeded--;
1293
1294         if (map->pagesreserved == 0)
1295                 panic("add_bounce_page: map doesn't need any pages");
1296         map->pagesreserved--;
1297
1298         mtx_lock(&bounce_lock);
1299         bpage = STAILQ_FIRST(&bz->bounce_page_list);
1300         if (bpage == NULL)
1301                 panic("add_bounce_page: free page list is empty");
1302
1303         STAILQ_REMOVE_HEAD(&bz->bounce_page_list, links);
1304         bz->reserved_bpages--;
1305         bz->active_bpages++;
1306         mtx_unlock(&bounce_lock);
1307
1308         if (dmat->common.flags & BUS_DMA_KEEP_PG_OFFSET) {
1309                 /* Page offset needs to be preserved. */
1310                 bpage->vaddr |= addr & PAGE_MASK;
1311                 bpage->busaddr |= addr & PAGE_MASK;
1312         }
1313         bpage->datavaddr = vaddr;
1314         bpage->datapage = PHYS_TO_VM_PAGE(addr);
1315         bpage->dataoffs = addr & PAGE_MASK;
1316         bpage->datacount = size;
1317         STAILQ_INSERT_TAIL(&(map->bpages), bpage, links);
1318         return (bpage->busaddr);
1319 }
1320
1321 static void
1322 free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage)
1323 {
1324         struct bus_dmamap *map;
1325         struct bounce_zone *bz;
1326
1327         bz = dmat->bounce_zone;
1328         bpage->datavaddr = 0;
1329         bpage->datacount = 0;
1330         if (dmat->common.flags & BUS_DMA_KEEP_PG_OFFSET) {
1331                 /*
1332                  * Reset the bounce page to start at offset 0.  Other uses
1333                  * of this bounce page may need to store a full page of
1334                  * data and/or assume it starts on a page boundary.
1335                  */
1336                 bpage->vaddr &= ~PAGE_MASK;
1337                 bpage->busaddr &= ~PAGE_MASK;
1338         }
1339
1340         mtx_lock(&bounce_lock);
1341         STAILQ_INSERT_HEAD(&bz->bounce_page_list, bpage, links);
1342         bz->free_bpages++;
1343         bz->active_bpages--;
1344         if ((map = STAILQ_FIRST(&bounce_map_waitinglist)) != NULL) {
1345                 if (reserve_bounce_pages(map->dmat, map, 1) == 0) {
1346                         STAILQ_REMOVE_HEAD(&bounce_map_waitinglist, links);
1347                         STAILQ_INSERT_TAIL(&bounce_map_callbacklist,
1348                             map, links);
1349                         busdma_swi_pending = 1;
1350                         bz->total_deferred++;
1351                         swi_sched(vm_ih, 0);
1352                 }
1353         }
1354         mtx_unlock(&bounce_lock);
1355 }
1356
1357 void
1358 busdma_swi(void)
1359 {
1360         bus_dma_tag_t dmat;
1361         struct bus_dmamap *map;
1362
1363         mtx_lock(&bounce_lock);
1364         while ((map = STAILQ_FIRST(&bounce_map_callbacklist)) != NULL) {
1365                 STAILQ_REMOVE_HEAD(&bounce_map_callbacklist, links);
1366                 mtx_unlock(&bounce_lock);
1367                 dmat = map->dmat;
1368                 (dmat->common.lockfunc)(dmat->common.lockfuncarg, BUS_DMA_LOCK);
1369                 bus_dmamap_load_mem(map->dmat, map, &map->mem,
1370                     map->callback, map->callback_arg, BUS_DMA_WAITOK);
1371                 (dmat->common.lockfunc)(dmat->common.lockfuncarg,
1372                     BUS_DMA_UNLOCK);
1373                 mtx_lock(&bounce_lock);
1374         }
1375         mtx_unlock(&bounce_lock);
1376 }
1377
1378 struct bus_dma_impl bus_dma_bounce_impl = {
1379         .tag_create = bounce_bus_dma_tag_create,
1380         .tag_destroy = bounce_bus_dma_tag_destroy,
1381         .id_mapped = bounce_bus_dma_id_mapped,
1382         .map_create = bounce_bus_dmamap_create,
1383         .map_destroy = bounce_bus_dmamap_destroy,
1384         .mem_alloc = bounce_bus_dmamem_alloc,
1385         .mem_free = bounce_bus_dmamem_free,
1386         .load_phys = bounce_bus_dmamap_load_phys,
1387         .load_buffer = bounce_bus_dmamap_load_buffer,
1388         .load_ma = bus_dmamap_load_ma_triv,
1389         .map_waitok = bounce_bus_dmamap_waitok,
1390         .map_complete = bounce_bus_dmamap_complete,
1391         .map_unload = bounce_bus_dmamap_unload,
1392         .map_sync = bounce_bus_dmamap_sync
1393 };