sys/kern/subr_busdma_bounce.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause
   3  *
   4  * Copyright (c) 1997, 1998 Justin T. Gibbs.
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions, and the following disclaimer,
  12  *    without modification, immediately at the beginning of the file.
  13  * 2. The name of the author may not be used to endorse or promote products
  14  *    derived from this software without specific prior written permission.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
  20  * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  26  * SUCH DAMAGE.
  27  */
  28
  29 /*
  30  * Common code for managing bounce pages for bus_dma backends.  As
  31  * this code currently assumes it can access internal members of
  32  * opaque types like bus_dma_tag_t and bus_dmamap it is #include'd in
  33  * backends rather than being compiled standalone.
  34  *
  35  * Prerequisites:
  36  *
  37  * - M_BUSDMA malloc type
  38  * - struct bus_dmamap
  39  * - hw_busdma SYSCTL_NODE
  40  * - macros to access the following fields of bus_dma_tag_t:
  41  *   - dmat_alignment()
  42  *   - dmat_flags()
  43  *   - dmat_lowaddr()
  44  *   - dmat_lockfunc()
  45  *   - dmat_lockarg()
  46  */
  47
  48 #include <sys/kthread.h>
  49 #include <sys/sched.h>
  50
  51 struct bounce_page {
  52         vm_offset_t     vaddr;          /* kva of bounce buffer */
  53         bus_addr_t      busaddr;        /* Physical address */
  54         vm_offset_t     datavaddr;      /* kva of client data */
  55 #if defined(__amd64__) || defined(__i386__)
  56         vm_page_t       datapage[2];    /* physical page(s) of client data */
  57 #else
  58         vm_page_t       datapage;       /* physical page of client data */
  59 #endif
  60         vm_offset_t     dataoffs;       /* page offset of client data */
  61         bus_size_t      datacount;      /* client data count */
  62         STAILQ_ENTRY(bounce_page) links;
  63 };
  64
  65 struct bounce_zone {
  66         STAILQ_ENTRY(bounce_zone) links;
  67         STAILQ_HEAD(, bounce_page) bounce_page_list;
  68         STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist;
  69         int             total_bpages;
  70         int             free_bpages;
  71         int             reserved_bpages;
  72         int             active_bpages;
  73         int             total_bounced;
  74         int             total_deferred;
  75         int             map_count;
  76 #ifdef dmat_domain
  77         int             domain;
  78 #endif
  79         sbintime_t      total_deferred_time;
  80         bus_size_t      alignment;
  81         bus_addr_t      lowaddr;
  82         char            zoneid[8];
  83         char            lowaddrid[20];
  84         struct sysctl_ctx_list sysctl_tree;
  85         struct sysctl_oid *sysctl_tree_top;
  86 };
  87
  88 static struct mtx bounce_lock;
  89 static int total_bpages;
  90 static int busdma_zonecount;
  91
  92 static STAILQ_HEAD(, bounce_zone) bounce_zone_list;
  93 static STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist;
  94
  95 static MALLOC_DEFINE(M_BOUNCE, "bounce", "busdma bounce pages");
  96
  97 SYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bpages, 0,
  98    "Total bounce pages");
  99
 100 static void busdma_thread(void *);
 101 static int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map,
 102     int commit);
 103
 104 static int
 105 _bus_dmamap_reserve_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int flags)
 106 {
 107         struct bounce_zone *bz;
 108
 109         /* Reserve Necessary Bounce Pages */
 110         mtx_lock(&bounce_lock);
 111         if (flags & BUS_DMA_NOWAIT) {
 112                 if (reserve_bounce_pages(dmat, map, 0) != 0) {
 113                         map->pagesneeded = 0;
 114                         mtx_unlock(&bounce_lock);
 115                         return (ENOMEM);
 116                 }
 117         } else {
 118                 if (reserve_bounce_pages(dmat, map, 1) != 0) {
 119                         /* Queue us for resources */
 120                         bz = dmat->bounce_zone;
 121                         STAILQ_INSERT_TAIL(&bz->bounce_map_waitinglist, map,
 122                             links);
 123                         map->queued_time = sbinuptime();
 124                         mtx_unlock(&bounce_lock);
 125                         return (EINPROGRESS);
 126                 }
 127         }
 128         mtx_unlock(&bounce_lock);
 129
 130         return (0);
 131 }
 132
 133 static void
 134 init_bounce_pages(void *dummy __unused)
 135 {
 136
 137         total_bpages = 0;
 138         STAILQ_INIT(&bounce_zone_list);
 139         STAILQ_INIT(&bounce_map_callbacklist);
 140         mtx_init(&bounce_lock, "bounce pages lock", NULL, MTX_DEF);
 141 }
 142 SYSINIT(bpages, SI_SUB_LOCK, SI_ORDER_ANY, init_bounce_pages, NULL);
 143
 144 static struct sysctl_ctx_list *
 145 busdma_sysctl_tree(struct bounce_zone *bz)
 146 {
 147
 148         return (&bz->sysctl_tree);
 149 }
 150
 151 static struct sysctl_oid *
 152 busdma_sysctl_tree_top(struct bounce_zone *bz)
 153 {
 154
 155         return (bz->sysctl_tree_top);
 156 }
 157
 158 /*
 159  * Returns true if the address falls within the tag's exclusion window, or
 160  * fails to meet its alignment requirements.
 161  */
 162 static bool
 163 addr_needs_bounce(bus_dma_tag_t dmat, bus_addr_t paddr)
 164 {
 165
 166         if (paddr > dmat_lowaddr(dmat) && paddr <= dmat_highaddr(dmat))
 167                 return (true);
 168         if (!vm_addr_align_ok(paddr, dmat_alignment(dmat)))
 169                 return (true);
 170
 171         return (false);
 172 }
 173
 174 static int
 175 alloc_bounce_zone(bus_dma_tag_t dmat)
 176 {
 177         struct bounce_zone *bz;
 178         bool start_thread;
 179
 180         /* Check to see if we already have a suitable zone */
 181         STAILQ_FOREACH(bz, &bounce_zone_list, links) {
 182                 if ((dmat_alignment(dmat) <= bz->alignment) &&
 183 #ifdef dmat_domain
 184                     dmat_domain(dmat) == bz->domain &&
 185 #endif
 186                     (dmat_lowaddr(dmat) >= bz->lowaddr)) {
 187                         dmat->bounce_zone = bz;
 188                         return (0);
 189                 }
 190         }
 191
 192         if ((bz = (struct bounce_zone *)malloc(sizeof(*bz), M_BUSDMA,
 193             M_NOWAIT | M_ZERO)) == NULL)
 194                 return (ENOMEM);
 195
 196         STAILQ_INIT(&bz->bounce_page_list);
 197         STAILQ_INIT(&bz->bounce_map_waitinglist);
 198         bz->free_bpages = 0;
 199         bz->reserved_bpages = 0;
 200         bz->active_bpages = 0;
 201         bz->lowaddr = dmat_lowaddr(dmat);
 202         bz->alignment = MAX(dmat_alignment(dmat), PAGE_SIZE);
 203         bz->map_count = 0;
 204 #ifdef dmat_domain
 205         bz->domain = dmat_domain(dmat);
 206 #endif
 207         snprintf(bz->zoneid, sizeof(bz->zoneid), "zone%d", busdma_zonecount);
 208         busdma_zonecount++;
 209         snprintf(bz->lowaddrid, sizeof(bz->lowaddrid), "%#jx",
 210             (uintmax_t)bz->lowaddr);
 211         start_thread = STAILQ_EMPTY(&bounce_zone_list);
 212         STAILQ_INSERT_TAIL(&bounce_zone_list, bz, links);
 213         dmat->bounce_zone = bz;
 214
 215         sysctl_ctx_init(&bz->sysctl_tree);
 216         bz->sysctl_tree_top = SYSCTL_ADD_NODE(&bz->sysctl_tree,
 217             SYSCTL_STATIC_CHILDREN(_hw_busdma), OID_AUTO, bz->zoneid,
 218             CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "");
 219         if (bz->sysctl_tree_top == NULL) {
 220                 sysctl_ctx_free(&bz->sysctl_tree);
 221                 return (0);     /* XXX error code? */
 222         }
 223
 224         SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
 225             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
 226             "total_bpages", CTLFLAG_RD, &bz->total_bpages, 0,
 227             "Total bounce pages");
 228         SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
 229             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
 230             "free_bpages", CTLFLAG_RD, &bz->free_bpages, 0,
 231             "Free bounce pages");
 232         SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
 233             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
 234             "reserved_bpages", CTLFLAG_RD, &bz->reserved_bpages, 0,
 235             "Reserved bounce pages");
 236         SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
 237             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
 238             "active_bpages", CTLFLAG_RD, &bz->active_bpages, 0,
 239             "Active bounce pages");
 240         SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
 241             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
 242             "total_bounced", CTLFLAG_RD, &bz->total_bounced, 0,
 243             "Total bounce requests (pages bounced)");
 244         SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
 245             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
 246             "total_deferred", CTLFLAG_RD, &bz->total_deferred, 0,
 247             "Total bounce requests that were deferred");
 248         SYSCTL_ADD_STRING(busdma_sysctl_tree(bz),
 249             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
 250             "lowaddr", CTLFLAG_RD, bz->lowaddrid, 0, "");
 251         SYSCTL_ADD_UAUTO(busdma_sysctl_tree(bz),
 252             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
 253             "alignment", CTLFLAG_RD, &bz->alignment, "");
 254 #ifdef dmat_domain
 255         SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
 256             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
 257             "domain", CTLFLAG_RD, &bz->domain, 0,
 258             "memory domain");
 259 #endif
 260         SYSCTL_ADD_SBINTIME_USEC(busdma_sysctl_tree(bz),
 261             SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
 262             "total_deferred_time", CTLFLAG_RD, &bz->total_deferred_time,
 263             "Cumulative time busdma requests are deferred (us)");
 264         if (start_thread) {
 265                 if (kproc_create(busdma_thread, NULL, NULL, 0, 0, "busdma") !=
 266                     0)
 267                         printf("failed to create busdma thread");
 268         }
 269         return (0);
 270 }
 271
 272 static int
 273 alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages)
 274 {
 275         struct bounce_zone *bz;
 276         int count;
 277
 278         bz = dmat->bounce_zone;
 279         count = 0;
 280         while (numpages > 0) {
 281                 struct bounce_page *bpage;
 282
 283 #ifdef dmat_domain
 284                 bpage = malloc_domainset(sizeof(*bpage), M_BUSDMA,
 285                     DOMAINSET_PREF(bz->domain), M_NOWAIT | M_ZERO);
 286 #else
 287                 bpage = malloc(sizeof(*bpage), M_BUSDMA, M_NOWAIT | M_ZERO);
 288 #endif
 289
 290                 if (bpage == NULL)
 291                         break;
 292 #ifdef dmat_domain
 293                 bpage->vaddr = (vm_offset_t)contigmalloc_domainset(PAGE_SIZE,
 294                     M_BOUNCE, DOMAINSET_PREF(bz->domain), M_NOWAIT,
 295                     0ul, bz->lowaddr, PAGE_SIZE, 0);
 296 #else
 297                 bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_BOUNCE,
 298                     M_NOWAIT, 0ul, bz->lowaddr, PAGE_SIZE, 0);
 299 #endif
 300                 if (bpage->vaddr == 0) {
 301                         free(bpage, M_BUSDMA);
 302                         break;
 303                 }
 304                 bpage->busaddr = pmap_kextract(bpage->vaddr);
 305                 mtx_lock(&bounce_lock);
 306                 STAILQ_INSERT_TAIL(&bz->bounce_page_list, bpage, links);
 307                 total_bpages++;
 308                 bz->total_bpages++;
 309                 bz->free_bpages++;
 310                 mtx_unlock(&bounce_lock);
 311                 count++;
 312                 numpages--;
 313         }
 314         return (count);
 315 }
 316
 317 static int
 318 reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int commit)
 319 {
 320         struct bounce_zone *bz;
 321         int pages;
 322
 323         mtx_assert(&bounce_lock, MA_OWNED);
 324         bz = dmat->bounce_zone;
 325         pages = MIN(bz->free_bpages, map->pagesneeded - map->pagesreserved);
 326         if (commit == 0 && map->pagesneeded > (map->pagesreserved + pages))
 327                 return (map->pagesneeded - (map->pagesreserved + pages));
 328         bz->free_bpages -= pages;
 329         bz->reserved_bpages += pages;
 330         map->pagesreserved += pages;
 331         pages = map->pagesneeded - map->pagesreserved;
 332
 333         return (pages);
 334 }
 335
 336 #if defined(__amd64__) || defined(__i386__)
 337 static bus_addr_t
 338 add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr,
 339     vm_paddr_t addr1, vm_paddr_t addr2, bus_size_t size)
 340 #else
 341 static bus_addr_t
 342 add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr,
 343     bus_addr_t addr, bus_size_t size)
 344 #endif
 345 {
 346         struct bounce_zone *bz;
 347         struct bounce_page *bpage;
 348
 349         KASSERT(dmat->bounce_zone != NULL, ("no bounce zone in dma tag"));
 350         KASSERT(map != NULL, ("add_bounce_page: bad map %p", map));
 351 #if defined(__amd64__) || defined(__i386__)
 352         KASSERT(map != &nobounce_dmamap, ("add_bounce_page: bad map %p", map));
 353 #endif
 354 #ifdef __riscv
 355         KASSERT((map->flags & DMAMAP_COULD_BOUNCE) != 0,
 356             ("add_bounce_page: bad map %p", map));
 357 #endif
 358
 359         bz = dmat->bounce_zone;
 360         if (map->pagesneeded == 0)
 361                 panic("add_bounce_page: map doesn't need any pages");
 362         map->pagesneeded--;
 363
 364         if (map->pagesreserved == 0)
 365                 panic("add_bounce_page: map doesn't need any pages");
 366         map->pagesreserved--;
 367
 368         mtx_lock(&bounce_lock);
 369         bpage = STAILQ_FIRST(&bz->bounce_page_list);
 370         if (bpage == NULL)
 371                 panic("add_bounce_page: free page list is empty");
 372
 373         STAILQ_REMOVE_HEAD(&bz->bounce_page_list, links);
 374         bz->reserved_bpages--;
 375         bz->active_bpages++;
 376         mtx_unlock(&bounce_lock);
 377
 378         if (dmat_flags(dmat) & BUS_DMA_KEEP_PG_OFFSET) {
 379                 /* Page offset needs to be preserved. */
 380 #if defined(__amd64__) || defined(__i386__)
 381                 bpage->vaddr |= addr1 & PAGE_MASK;
 382                 bpage->busaddr |= addr1 & PAGE_MASK;
 383                 KASSERT(addr2 == 0,
 384             ("Trying to bounce multiple pages with BUS_DMA_KEEP_PG_OFFSET"));
 385 #else
 386                 bpage->vaddr |= addr & PAGE_MASK;
 387                 bpage->busaddr |= addr & PAGE_MASK;
 388 #endif
 389         }
 390         bpage->datavaddr = vaddr;
 391 #if defined(__amd64__) || defined(__i386__)
 392         bpage->datapage[0] = PHYS_TO_VM_PAGE(addr1);
 393         KASSERT((addr2 & PAGE_MASK) == 0, ("Second page is not aligned"));
 394         bpage->datapage[1] = PHYS_TO_VM_PAGE(addr2);
 395         bpage->dataoffs = addr1 & PAGE_MASK;
 396 #else
 397         bpage->datapage = PHYS_TO_VM_PAGE(addr);
 398         bpage->dataoffs = addr & PAGE_MASK;
 399 #endif
 400         bpage->datacount = size;
 401         STAILQ_INSERT_TAIL(&(map->bpages), bpage, links);
 402         return (bpage->busaddr);
 403 }
 404
 405 static void
 406 free_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map)
 407 {
 408         struct bounce_page *bpage;
 409         struct bounce_zone *bz;
 410         bool schedule_thread;
 411         u_int count;
 412
 413         if (STAILQ_EMPTY(&map->bpages))
 414                 return;
 415
 416         bz = dmat->bounce_zone;
 417         count = 0;
 418         schedule_thread = false;
 419         STAILQ_FOREACH(bpage, &map->bpages, links) {
 420                 bpage->datavaddr = 0;
 421                 bpage->datacount = 0;
 422
 423                 if (dmat_flags(dmat) & BUS_DMA_KEEP_PG_OFFSET) {
 424                         /*
 425                          * Reset the bounce page to start at offset 0.
 426                          * Other uses of this bounce page may need to
 427                          * store a full page of data and/or assume it
 428                          * starts on a page boundary.
 429                          */
 430                         bpage->vaddr &= ~PAGE_MASK;
 431                         bpage->busaddr &= ~PAGE_MASK;
 432                 }
 433                 count++;
 434         }
 435
 436         mtx_lock(&bounce_lock);
 437         STAILQ_CONCAT(&bz->bounce_page_list, &map->bpages);
 438         bz->free_bpages += count;
 439         bz->active_bpages -= count;
 440         while ((map = STAILQ_FIRST(&bz->bounce_map_waitinglist)) != NULL) {
 441                 if (reserve_bounce_pages(map->dmat, map, 1) != 0)
 442                         break;
 443
 444                 STAILQ_REMOVE_HEAD(&bz->bounce_map_waitinglist, links);
 445                 STAILQ_INSERT_TAIL(&bounce_map_callbacklist, map, links);
 446                 bz->total_deferred++;
 447                 schedule_thread = true;
 448         }
 449         mtx_unlock(&bounce_lock);
 450         if (schedule_thread)
 451                 wakeup(&bounce_map_callbacklist);
 452 }
 453
 454 static void
 455 busdma_thread(void *dummy __unused)
 456 {
 457         STAILQ_HEAD(, bus_dmamap) callbacklist;
 458         bus_dma_tag_t dmat;
 459         struct bus_dmamap *map, *nmap;
 460         struct bounce_zone *bz;
 461
 462         thread_lock(curthread);
 463         sched_class(curthread, PRI_ITHD);
 464         sched_ithread_prio(curthread, PI_SWI(SWI_BUSDMA));
 465         thread_unlock(curthread);
 466         for (;;) {
 467                 mtx_lock(&bounce_lock);
 468                 while (STAILQ_EMPTY(&bounce_map_callbacklist))
 469                         mtx_sleep(&bounce_map_callbacklist, &bounce_lock, 0,
 470                             "-", 0);
 471                 STAILQ_INIT(&callbacklist);
 472                 STAILQ_CONCAT(&callbacklist, &bounce_map_callbacklist);
 473                 mtx_unlock(&bounce_lock);
 474
 475                 STAILQ_FOREACH_SAFE(map, &callbacklist, links, nmap) {
 476                         dmat = map->dmat;
 477                         bz = dmat->bounce_zone;
 478                         dmat_lockfunc(dmat)(dmat_lockfuncarg(dmat),
 479                             BUS_DMA_LOCK);
 480                         bz->total_deferred_time += (sbinuptime() - map->queued_time);
 481                         bus_dmamap_load_mem(map->dmat, map, &map->mem,
 482                             map->callback, map->callback_arg, BUS_DMA_WAITOK);
 483                         dmat_lockfunc(dmat)(dmat_lockfuncarg(dmat),
 484                             BUS_DMA_UNLOCK);
 485                 }
 486         }
 487 }