sys/dev/usb/usb_busdma.c

   1 /* $FreeBSD$ */
   2 /*-
   3  * Copyright (c) 2008 Hans Petter Selasky. All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  */
  26
  27 #include <sys/stdint.h>
  28 #include <sys/stddef.h>
  29 #include <sys/param.h>
  30 #include <sys/queue.h>
  31 #include <sys/types.h>
  32 #include <sys/systm.h>
  33 #include <sys/kernel.h>
  34 #include <sys/bus.h>
  35 #include <sys/linker_set.h>
  36 #include <sys/module.h>
  37 #include <sys/lock.h>
  38 #include <sys/mutex.h>
  39 #include <sys/condvar.h>
  40 #include <sys/sysctl.h>
  41 #include <sys/sx.h>
  42 #include <sys/unistd.h>
  43 #include <sys/callout.h>
  44 #include <sys/malloc.h>
  45 #include <sys/priv.h>
  46
  47 #include <dev/usb/usb.h>
  48 #include <dev/usb/usbdi.h>
  49 #include <dev/usb/usbdi_util.h>
  50
  51 #define USB_DEBUG_VAR usb_debug
  52
  53 #include <dev/usb/usb_core.h>
  54 #include <dev/usb/usb_busdma.h>
  55 #include <dev/usb/usb_process.h>
  56 #include <dev/usb/usb_transfer.h>
  57 #include <dev/usb/usb_device.h>
  58 #include <dev/usb/usb_util.h>
  59 #include <dev/usb/usb_debug.h>
  60
  61 #include <dev/usb/usb_controller.h>
  62 #include <dev/usb/usb_bus.h>
  63
  64 #if USB_HAVE_BUSDMA
  65 static void     usb_dma_tag_create(struct usb_dma_tag *, usb_size_t, usb_size_t);
  66 static void     usb_dma_tag_destroy(struct usb_dma_tag *);
  67 static void     usb_dma_lock_cb(void *, bus_dma_lock_op_t);
  68 static void     usb_pc_alloc_mem_cb(void *, bus_dma_segment_t *, int, int);
  69 static void     usb_pc_load_mem_cb(void *, bus_dma_segment_t *, int, int);
  70 static void     usb_pc_common_mem_cb(void *, bus_dma_segment_t *, int, int,
  71                     uint8_t);
  72 #endif
  73
  74 /*------------------------------------------------------------------------*
  75  *  usbd_get_page - lookup DMA-able memory for the given offset
  76  *
  77  * NOTE: Only call this function when the "page_cache" structure has
  78  * been properly initialized !
  79  *------------------------------------------------------------------------*/
  80 void
  81 usbd_get_page(struct usb_page_cache *pc, usb_frlength_t offset,
  82     struct usb_page_search *res)
  83 {
  84         struct usb_page *page;
  85
  86 #if USB_HAVE_BUSDMA
  87         if (pc->page_start) {
  88
  89                 /* Case 1 - something has been loaded into DMA */
  90
  91                 if (pc->buffer) {
  92
  93                         /* Case 1a - Kernel Virtual Address */
  94
  95                         res->buffer = USB_ADD_BYTES(pc->buffer, offset);
  96                 }
  97                 offset += pc->page_offset_buf;
  98
  99                 /* compute destination page */
 100
 101                 page = pc->page_start;
 102
 103                 if (pc->ismultiseg) {
 104
 105                         page += (offset / USB_PAGE_SIZE);
 106
 107                         offset %= USB_PAGE_SIZE;
 108
 109                         res->length = USB_PAGE_SIZE - offset;
 110                         res->physaddr = page->physaddr + offset;
 111                 } else {
 112                         res->length = 0 - 1;
 113                         res->physaddr = page->physaddr + offset;
 114                 }
 115                 if (!pc->buffer) {
 116
 117                         /* Case 1b - Non Kernel Virtual Address */
 118
 119                         res->buffer = USB_ADD_BYTES(page->buffer, offset);
 120                 }
 121                 return;
 122         }
 123 #endif
 124         /* Case 2 - Plain PIO */
 125
 126         res->buffer = USB_ADD_BYTES(pc->buffer, offset);
 127         res->length = 0 - 1;
 128 #if USB_HAVE_BUSDMA
 129         res->physaddr = 0;
 130 #endif
 131 }
 132
 133 /*------------------------------------------------------------------------*
 134  *  usbd_copy_in - copy directly to DMA-able memory
 135  *------------------------------------------------------------------------*/
 136 void
 137 usbd_copy_in(struct usb_page_cache *cache, usb_frlength_t offset,
 138     const void *ptr, usb_frlength_t len)
 139 {
 140         struct usb_page_search buf_res;
 141
 142         while (len != 0) {
 143
 144                 usbd_get_page(cache, offset, &buf_res);
 145
 146                 if (buf_res.length > len) {
 147                         buf_res.length = len;
 148                 }
 149                 bcopy(ptr, buf_res.buffer, buf_res.length);
 150
 151                 offset += buf_res.length;
 152                 len -= buf_res.length;
 153                 ptr = USB_ADD_BYTES(ptr, buf_res.length);
 154         }
 155 }
 156
 157 /*------------------------------------------------------------------------*
 158  *  usbd_copy_in_user - copy directly to DMA-able memory from userland
 159  *
 160  * Return values:
 161  *    0: Success
 162  * Else: Failure
 163  *------------------------------------------------------------------------*/
 164 #if USB_HAVE_USER_IO
 165 int
 166 usbd_copy_in_user(struct usb_page_cache *cache, usb_frlength_t offset,
 167     const void *ptr, usb_frlength_t len)
 168 {
 169         struct usb_page_search buf_res;
 170         int error;
 171
 172         while (len != 0) {
 173
 174                 usbd_get_page(cache, offset, &buf_res);
 175
 176                 if (buf_res.length > len) {
 177                         buf_res.length = len;
 178                 }
 179                 error = copyin(ptr, buf_res.buffer, buf_res.length);
 180                 if (error)
 181                         return (error);
 182
 183                 offset += buf_res.length;
 184                 len -= buf_res.length;
 185                 ptr = USB_ADD_BYTES(ptr, buf_res.length);
 186         }
 187         return (0);                     /* success */
 188 }
 189 #endif
 190
 191 /*------------------------------------------------------------------------*
 192  *  usbd_m_copy_in - copy a mbuf chain directly into DMA-able memory
 193  *------------------------------------------------------------------------*/
 194 #if USB_HAVE_MBUF
 195 struct usb_m_copy_in_arg {
 196         struct usb_page_cache *cache;
 197         usb_frlength_t dst_offset;
 198 };
 199
 200 static int
 201 usbd_m_copy_in_cb(void *arg, void *src, uint32_t count)
 202 {
 203         register struct usb_m_copy_in_arg *ua = arg;
 204
 205         usbd_copy_in(ua->cache, ua->dst_offset, src, count);
 206         ua->dst_offset += count;
 207         return (0);
 208 }
 209
 210 void
 211 usbd_m_copy_in(struct usb_page_cache *cache, usb_frlength_t dst_offset,
 212     struct mbuf *m, usb_size_t src_offset, usb_frlength_t src_len)
 213 {
 214         struct usb_m_copy_in_arg arg = {cache, dst_offset};
 215         int error;
 216
 217         error = m_apply(m, src_offset, src_len, &usbd_m_copy_in_cb, &arg);
 218 }
 219 #endif
 220
 221 /*------------------------------------------------------------------------*
 222  *  usb_uiomove - factored out code
 223  *------------------------------------------------------------------------*/
 224 #if USB_HAVE_USER_IO
 225 int
 226 usb_uiomove(struct usb_page_cache *pc, struct uio *uio,
 227     usb_frlength_t pc_offset, usb_frlength_t len)
 228 {
 229         struct usb_page_search res;
 230         int error = 0;
 231
 232         while (len != 0) {
 233
 234                 usbd_get_page(pc, pc_offset, &res);
 235
 236                 if (res.length > len) {
 237                         res.length = len;
 238                 }
 239                 /*
 240                  * "uiomove()" can sleep so one needs to make a wrapper,
 241                  * exiting the mutex and checking things
 242                  */
 243                 error = uiomove(res.buffer, res.length, uio);
 244
 245                 if (error) {
 246                         break;
 247                 }
 248                 pc_offset += res.length;
 249                 len -= res.length;
 250         }
 251         return (error);
 252 }
 253 #endif
 254
 255 /*------------------------------------------------------------------------*
 256  *  usbd_copy_out - copy directly from DMA-able memory
 257  *------------------------------------------------------------------------*/
 258 void
 259 usbd_copy_out(struct usb_page_cache *cache, usb_frlength_t offset,
 260     void *ptr, usb_frlength_t len)
 261 {
 262         struct usb_page_search res;
 263
 264         while (len != 0) {
 265
 266                 usbd_get_page(cache, offset, &res);
 267
 268                 if (res.length > len) {
 269                         res.length = len;
 270                 }
 271                 bcopy(res.buffer, ptr, res.length);
 272
 273                 offset += res.length;
 274                 len -= res.length;
 275                 ptr = USB_ADD_BYTES(ptr, res.length);
 276         }
 277 }
 278
 279 /*------------------------------------------------------------------------*
 280  *  usbd_copy_out_user - copy directly from DMA-able memory to userland
 281  *
 282  * Return values:
 283  *    0: Success
 284  * Else: Failure
 285  *------------------------------------------------------------------------*/
 286 #if USB_HAVE_USER_IO
 287 int
 288 usbd_copy_out_user(struct usb_page_cache *cache, usb_frlength_t offset,
 289     void *ptr, usb_frlength_t len)
 290 {
 291         struct usb_page_search res;
 292         int error;
 293
 294         while (len != 0) {
 295
 296                 usbd_get_page(cache, offset, &res);
 297
 298                 if (res.length > len) {
 299                         res.length = len;
 300                 }
 301                 error = copyout(res.buffer, ptr, res.length);
 302                 if (error)
 303                         return (error);
 304
 305                 offset += res.length;
 306                 len -= res.length;
 307                 ptr = USB_ADD_BYTES(ptr, res.length);
 308         }
 309         return (0);                     /* success */
 310 }
 311 #endif
 312
 313 /*------------------------------------------------------------------------*
 314  *  usbd_frame_zero - zero DMA-able memory
 315  *------------------------------------------------------------------------*/
 316 void
 317 usbd_frame_zero(struct usb_page_cache *cache, usb_frlength_t offset,
 318     usb_frlength_t len)
 319 {
 320         struct usb_page_search res;
 321
 322         while (len != 0) {
 323
 324                 usbd_get_page(cache, offset, &res);
 325
 326                 if (res.length > len) {
 327                         res.length = len;
 328                 }
 329                 bzero(res.buffer, res.length);
 330
 331                 offset += res.length;
 332                 len -= res.length;
 333         }
 334 }
 335
 336 #if USB_HAVE_BUSDMA
 337
 338 /*------------------------------------------------------------------------*
 339  *      usb_dma_lock_cb - dummy callback
 340  *------------------------------------------------------------------------*/
 341 static void
 342 usb_dma_lock_cb(void *arg, bus_dma_lock_op_t op)
 343 {
 344         /* we use "mtx_owned()" instead of this function */
 345 }
 346
 347 /*------------------------------------------------------------------------*
 348  *      usb_dma_tag_create - allocate a DMA tag
 349  *
 350  * NOTE: If the "align" parameter has a value of 1 the DMA-tag will
 351  * allow multi-segment mappings. Else all mappings are single-segment.
 352  *------------------------------------------------------------------------*/
 353 static void
 354 usb_dma_tag_create(struct usb_dma_tag *udt,
 355     usb_size_t size, usb_size_t align)
 356 {
 357         bus_dma_tag_t tag;
 358
 359         if (bus_dma_tag_create
 360             ( /* parent    */ udt->tag_parent->tag,
 361              /* alignment */ align,
 362              /* boundary  */ (align == 1) ?
 363             USB_PAGE_SIZE : 0,
 364              /* lowaddr   */ (2ULL << (udt->tag_parent->dma_bits - 1)) - 1,
 365              /* highaddr  */ BUS_SPACE_MAXADDR,
 366              /* filter    */ NULL,
 367              /* filterarg */ NULL,
 368              /* maxsize   */ size,
 369              /* nsegments */ (align == 1) ?
 370             (2 + (size / USB_PAGE_SIZE)) : 1,
 371              /* maxsegsz  */ (align == 1) ?
 372             USB_PAGE_SIZE : size,
 373              /* flags     */ BUS_DMA_KEEP_PG_OFFSET,
 374              /* lockfn    */ &usb_dma_lock_cb,
 375              /* lockarg   */ NULL,
 376             &tag)) {
 377                 tag = NULL;
 378         }
 379         udt->tag = tag;
 380 }
 381
 382 /*------------------------------------------------------------------------*
 383  *      usb_dma_tag_free - free a DMA tag
 384  *------------------------------------------------------------------------*/
 385 static void
 386 usb_dma_tag_destroy(struct usb_dma_tag *udt)
 387 {
 388         bus_dma_tag_destroy(udt->tag);
 389 }
 390
 391 /*------------------------------------------------------------------------*
 392  *      usb_pc_alloc_mem_cb - BUS-DMA callback function
 393  *------------------------------------------------------------------------*/
 394 static void
 395 usb_pc_alloc_mem_cb(void *arg, bus_dma_segment_t *segs,
 396     int nseg, int error)
 397 {
 398         usb_pc_common_mem_cb(arg, segs, nseg, error, 0);
 399 }
 400
 401 /*------------------------------------------------------------------------*
 402  *      usb_pc_load_mem_cb - BUS-DMA callback function
 403  *------------------------------------------------------------------------*/
 404 static void
 405 usb_pc_load_mem_cb(void *arg, bus_dma_segment_t *segs,
 406     int nseg, int error)
 407 {
 408         usb_pc_common_mem_cb(arg, segs, nseg, error, 1);
 409 }
 410
 411 /*------------------------------------------------------------------------*
 412  *      usb_pc_common_mem_cb - BUS-DMA callback function
 413  *------------------------------------------------------------------------*/
 414 static void
 415 usb_pc_common_mem_cb(void *arg, bus_dma_segment_t *segs,
 416     int nseg, int error, uint8_t isload)
 417 {
 418         struct usb_dma_parent_tag *uptag;
 419         struct usb_page_cache *pc;
 420         struct usb_page *pg;
 421         usb_size_t rem;
 422         uint8_t owned;
 423
 424         pc = arg;
 425         uptag = pc->tag_parent;
 426
 427         /*
 428          * XXX There is sometimes recursive locking here.
 429          * XXX We should try to find a better solution.
 430          * XXX Until further the "owned" variable does
 431          * XXX the trick.
 432          */
 433
 434         if (error) {
 435                 goto done;
 436         }
 437         pg = pc->page_start;
 438         pg->physaddr = segs->ds_addr & ~(USB_PAGE_SIZE - 1);
 439         rem = segs->ds_addr & (USB_PAGE_SIZE - 1);
 440         pc->page_offset_buf = rem;
 441         pc->page_offset_end += rem;
 442         nseg--;
 443 #ifdef USB_DEBUG
 444         if (rem != (USB_P2U(pc->buffer) & (USB_PAGE_SIZE - 1))) {
 445                 /*
 446                  * This check verifies that the physical address is correct:
 447                  */
 448                 DPRINTFN(0, "Page offset was not preserved!\n");
 449                 error = 1;
 450                 goto done;
 451         }
 452 #endif
 453         while (nseg > 0) {
 454                 nseg--;
 455                 segs++;
 456                 pg++;
 457                 pg->physaddr = segs->ds_addr & ~(USB_PAGE_SIZE - 1);
 458         }
 459
 460 done:
 461         owned = mtx_owned(uptag->mtx);
 462         if (!owned)
 463                 mtx_lock(uptag->mtx);
 464
 465         uptag->dma_error = (error ? 1 : 0);
 466         if (isload) {
 467                 (uptag->func) (uptag);
 468         } else {
 469                 cv_broadcast(uptag->cv);
 470         }
 471         if (!owned)
 472                 mtx_unlock(uptag->mtx);
 473 }
 474
 475 /*------------------------------------------------------------------------*
 476  *      usb_pc_alloc_mem - allocate DMA'able memory
 477  *
 478  * Returns:
 479  *    0: Success
 480  * Else: Failure
 481  *------------------------------------------------------------------------*/
 482 uint8_t
 483 usb_pc_alloc_mem(struct usb_page_cache *pc, struct usb_page *pg,
 484     usb_size_t size, usb_size_t align)
 485 {
 486         struct usb_dma_parent_tag *uptag;
 487         struct usb_dma_tag *utag;
 488         bus_dmamap_t map;
 489         void *ptr;
 490         int err;
 491
 492         uptag = pc->tag_parent;
 493
 494         if (align != 1) {
 495                 /*
 496                  * The alignment must be greater or equal to the
 497                  * "size" else the object can be split between two
 498                  * memory pages and we get a problem!
 499                  */
 500                 while (align < size) {
 501                         align *= 2;
 502                         if (align == 0) {
 503                                 goto error;
 504                         }
 505                 }
 506 #if 1
 507                 /*
 508                  * XXX BUS-DMA workaround - FIXME later:
 509                  *
 510                  * We assume that that the aligment at this point of
 511                  * the code is greater than or equal to the size and
 512                  * less than two times the size, so that if we double
 513                  * the size, the size will be greater than the
 514                  * alignment.
 515                  *
 516                  * The bus-dma system has a check for "alignment"
 517                  * being less than "size". If that check fails we end
 518                  * up using contigmalloc which is page based even for
 519                  * small allocations. Try to avoid that to save
 520                  * memory, hence we sometimes to a large number of
 521                  * small allocations!
 522                  */
 523                 if (size <= (USB_PAGE_SIZE / 2)) {
 524                         size *= 2;
 525                 }
 526 #endif
 527         }
 528         /* get the correct DMA tag */
 529         utag = usb_dma_tag_find(uptag, size, align);
 530         if (utag == NULL) {
 531                 goto error;
 532         }
 533         /* allocate memory */
 534         if (bus_dmamem_alloc(
 535             utag->tag, &ptr, (BUS_DMA_WAITOK | BUS_DMA_COHERENT), &map)) {
 536                 goto error;
 537         }
 538         /* setup page cache */
 539         pc->buffer = ptr;
 540         pc->page_start = pg;
 541         pc->page_offset_buf = 0;
 542         pc->page_offset_end = size;
 543         pc->map = map;
 544         pc->tag = utag->tag;
 545         pc->ismultiseg = (align == 1);
 546
 547         mtx_lock(uptag->mtx);
 548
 549         /* load memory into DMA */
 550         err = bus_dmamap_load(
 551             utag->tag, map, ptr, size, &usb_pc_alloc_mem_cb,
 552             pc, (BUS_DMA_WAITOK | BUS_DMA_COHERENT));
 553
 554         if (err == EINPROGRESS) {
 555                 cv_wait(uptag->cv, uptag->mtx);
 556                 err = 0;
 557         }
 558         mtx_unlock(uptag->mtx);
 559
 560         if (err || uptag->dma_error) {
 561                 bus_dmamem_free(utag->tag, ptr, map);
 562                 goto error;
 563         }
 564         bzero(ptr, size);
 565
 566         usb_pc_cpu_flush(pc);
 567
 568         return (0);
 569
 570 error:
 571         /* reset most of the page cache */
 572         pc->buffer = NULL;
 573         pc->page_start = NULL;
 574         pc->page_offset_buf = 0;
 575         pc->page_offset_end = 0;
 576         pc->map = NULL;
 577         pc->tag = NULL;
 578         return (1);
 579 }
 580
 581 /*------------------------------------------------------------------------*
 582  *      usb_pc_free_mem - free DMA memory
 583  *
 584  * This function is NULL safe.
 585  *------------------------------------------------------------------------*/
 586 void
 587 usb_pc_free_mem(struct usb_page_cache *pc)
 588 {
 589         if (pc && pc->buffer) {
 590
 591                 bus_dmamap_unload(pc->tag, pc->map);
 592
 593                 bus_dmamem_free(pc->tag, pc->buffer, pc->map);
 594
 595                 pc->buffer = NULL;
 596         }
 597 }
 598
 599 /*------------------------------------------------------------------------*
 600  *      usb_pc_load_mem - load virtual memory into DMA
 601  *
 602  * Return values:
 603  * 0: Success
 604  * Else: Error
 605  *------------------------------------------------------------------------*/
 606 uint8_t
 607 usb_pc_load_mem(struct usb_page_cache *pc, usb_size_t size, uint8_t sync)
 608 {
 609         /* setup page cache */
 610         pc->page_offset_buf = 0;
 611         pc->page_offset_end = size;
 612         pc->ismultiseg = 1;
 613
 614         mtx_assert(pc->tag_parent->mtx, MA_OWNED);
 615
 616         if (size > 0) {
 617                 if (sync) {
 618                         struct usb_dma_parent_tag *uptag;
 619                         int err;
 620
 621                         uptag = pc->tag_parent;
 622
 623                         /*
 624                          * We have to unload the previous loaded DMA
 625                          * pages before trying to load a new one!
 626                          */
 627                         bus_dmamap_unload(pc->tag, pc->map);
 628
 629                         /*
 630                          * Try to load memory into DMA.
 631                          */
 632                         err = bus_dmamap_load(
 633                             pc->tag, pc->map, pc->buffer, size,
 634                             &usb_pc_alloc_mem_cb, pc, BUS_DMA_WAITOK);
 635                         if (err == EINPROGRESS) {
 636                                 cv_wait(uptag->cv, uptag->mtx);
 637                                 err = 0;
 638                         }
 639                         if (err || uptag->dma_error) {
 640                                 return (1);
 641                         }
 642                 } else {
 643
 644                         /*
 645                          * We have to unload the previous loaded DMA
 646                          * pages before trying to load a new one!
 647                          */
 648                         bus_dmamap_unload(pc->tag, pc->map);
 649
 650                         /*
 651                          * Try to load memory into DMA. The callback
 652                          * will be called in all cases:
 653                          */
 654                         if (bus_dmamap_load(
 655                             pc->tag, pc->map, pc->buffer, size,
 656                             &usb_pc_load_mem_cb, pc, BUS_DMA_WAITOK)) {
 657                         }
 658                 }
 659         } else {
 660                 if (!sync) {
 661                         /*
 662                          * Call callback so that refcount is decremented
 663                          * properly:
 664                          */
 665                         pc->tag_parent->dma_error = 0;
 666                         (pc->tag_parent->func) (pc->tag_parent);
 667                 }
 668         }
 669         return (0);
 670 }
 671
 672 /*------------------------------------------------------------------------*
 673  *      usb_pc_cpu_invalidate - invalidate CPU cache
 674  *------------------------------------------------------------------------*/
 675 void
 676 usb_pc_cpu_invalidate(struct usb_page_cache *pc)
 677 {
 678         if (pc->page_offset_end == pc->page_offset_buf) {
 679                 /* nothing has been loaded into this page cache! */
 680                 return;
 681         }
 682         bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_POSTREAD);
 683         bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_PREREAD);
 684 }
 685
 686 /*------------------------------------------------------------------------*
 687  *      usb_pc_cpu_flush - flush CPU cache
 688  *------------------------------------------------------------------------*/
 689 void
 690 usb_pc_cpu_flush(struct usb_page_cache *pc)
 691 {
 692         if (pc->page_offset_end == pc->page_offset_buf) {
 693                 /* nothing has been loaded into this page cache! */
 694                 return;
 695         }
 696         bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_PREWRITE);
 697 }
 698
 699 /*------------------------------------------------------------------------*
 700  *      usb_pc_dmamap_create - create a DMA map
 701  *
 702  * Returns:
 703  *    0: Success
 704  * Else: Failure
 705  *------------------------------------------------------------------------*/
 706 uint8_t
 707 usb_pc_dmamap_create(struct usb_page_cache *pc, usb_size_t size)
 708 {
 709         struct usb_xfer_root *info;
 710         struct usb_dma_tag *utag;
 711
 712         /* get info */
 713         info = USB_DMATAG_TO_XROOT(pc->tag_parent);
 714
 715         /* sanity check */
 716         if (info == NULL) {
 717                 goto error;
 718         }
 719         utag = usb_dma_tag_find(pc->tag_parent, size, 1);
 720         if (utag == NULL) {
 721                 goto error;
 722         }
 723         /* create DMA map */
 724         if (bus_dmamap_create(utag->tag, 0, &pc->map)) {
 725                 goto error;
 726         }
 727         pc->tag = utag->tag;
 728         return 0;                       /* success */
 729
 730 error:
 731         pc->map = NULL;
 732         pc->tag = NULL;
 733         return 1;                       /* failure */
 734 }
 735
 736 /*------------------------------------------------------------------------*
 737  *      usb_pc_dmamap_destroy
 738  *
 739  * This function is NULL safe.
 740  *------------------------------------------------------------------------*/
 741 void
 742 usb_pc_dmamap_destroy(struct usb_page_cache *pc)
 743 {
 744         if (pc && pc->tag) {
 745                 bus_dmamap_destroy(pc->tag, pc->map);
 746                 pc->tag = NULL;
 747                 pc->map = NULL;
 748         }
 749 }
 750
 751 /*------------------------------------------------------------------------*
 752  *      usb_dma_tag_find - factored out code
 753  *------------------------------------------------------------------------*/
 754 struct usb_dma_tag *
 755 usb_dma_tag_find(struct usb_dma_parent_tag *udpt,
 756     usb_size_t size, usb_size_t align)
 757 {
 758         struct usb_dma_tag *udt;
 759         uint8_t nudt;
 760
 761         USB_ASSERT(align > 0, ("Invalid parameter align = 0!\n"));
 762         USB_ASSERT(size > 0, ("Invalid parameter size = 0!\n"));
 763
 764         udt = udpt->utag_first;
 765         nudt = udpt->utag_max;
 766
 767         while (nudt--) {
 768
 769                 if (udt->align == 0) {
 770                         usb_dma_tag_create(udt, size, align);
 771                         if (udt->tag == NULL) {
 772                                 return (NULL);
 773                         }
 774                         udt->align = align;
 775                         udt->size = size;
 776                         return (udt);
 777                 }
 778                 if ((udt->align == align) && (udt->size == size)) {
 779                         return (udt);
 780                 }
 781                 udt++;
 782         }
 783         return (NULL);
 784 }
 785
 786 /*------------------------------------------------------------------------*
 787  *      usb_dma_tag_setup - initialise USB DMA tags
 788  *------------------------------------------------------------------------*/
 789 void
 790 usb_dma_tag_setup(struct usb_dma_parent_tag *udpt,
 791     struct usb_dma_tag *udt, bus_dma_tag_t dmat,
 792     struct mtx *mtx, usb_dma_callback_t *func,
 793     uint8_t ndmabits, uint8_t nudt)
 794 {
 795         bzero(udpt, sizeof(*udpt));
 796
 797         /* sanity checking */
 798         if ((nudt == 0) ||
 799             (ndmabits == 0) ||
 800             (mtx == NULL)) {
 801                 /* something is corrupt */
 802                 return;
 803         }
 804         /* initialise condition variable */
 805         cv_init(udpt->cv, "USB DMA CV");
 806
 807         /* store some information */
 808         udpt->mtx = mtx;
 809         udpt->func = func;
 810         udpt->tag = dmat;
 811         udpt->utag_first = udt;
 812         udpt->utag_max = nudt;
 813         udpt->dma_bits = ndmabits;
 814
 815         while (nudt--) {
 816                 bzero(udt, sizeof(*udt));
 817                 udt->tag_parent = udpt;
 818                 udt++;
 819         }
 820 }
 821
 822 /*------------------------------------------------------------------------*
 823  *      usb_bus_tag_unsetup - factored out code
 824  *------------------------------------------------------------------------*/
 825 void
 826 usb_dma_tag_unsetup(struct usb_dma_parent_tag *udpt)
 827 {
 828         struct usb_dma_tag *udt;
 829         uint8_t nudt;
 830
 831         udt = udpt->utag_first;
 832         nudt = udpt->utag_max;
 833
 834         while (nudt--) {
 835
 836                 if (udt->align) {
 837                         /* destroy the USB DMA tag */
 838                         usb_dma_tag_destroy(udt);
 839                         udt->align = 0;
 840                 }
 841                 udt++;
 842         }
 843
 844         if (udpt->utag_max) {
 845                 /* destroy the condition variable */
 846                 cv_destroy(udpt->cv);
 847         }
 848 }
 849
 850 /*------------------------------------------------------------------------*
 851  *      usb_bdma_work_loop
 852  *
 853  * This function handles loading of virtual buffers into DMA and is
 854  * only called when "dma_refcount" is zero.
 855  *------------------------------------------------------------------------*/
 856 void
 857 usb_bdma_work_loop(struct usb_xfer_queue *pq)
 858 {
 859         struct usb_xfer_root *info;
 860         struct usb_xfer *xfer;
 861         usb_frcount_t nframes;
 862
 863         xfer = pq->curr;
 864         info = xfer->xroot;
 865
 866         mtx_assert(info->xfer_mtx, MA_OWNED);
 867
 868         if (xfer->error) {
 869                 /* some error happened */
 870                 USB_BUS_LOCK(info->bus);
 871                 usbd_transfer_done(xfer, 0);
 872                 USB_BUS_UNLOCK(info->bus);
 873                 return;
 874         }
 875         if (!xfer->flags_int.bdma_setup) {
 876                 struct usb_page *pg;
 877                 usb_frlength_t frlength_0;
 878                 uint8_t isread;
 879
 880                 xfer->flags_int.bdma_setup = 1;
 881
 882                 /* reset BUS-DMA load state */
 883
 884                 info->dma_error = 0;
 885
 886                 if (xfer->flags_int.isochronous_xfr) {
 887                         /* only one frame buffer */
 888                         nframes = 1;
 889                         frlength_0 = xfer->sumlen;
 890                 } else {
 891                         /* can be multiple frame buffers */
 892                         nframes = xfer->nframes;
 893                         frlength_0 = xfer->frlengths[0];
 894                 }
 895
 896                 /*
 897                  * Set DMA direction first. This is needed to
 898                  * select the correct cache invalidate and cache
 899                  * flush operations.
 900                  */
 901                 isread = USB_GET_DATA_ISREAD(xfer);
 902                 pg = xfer->dma_page_ptr;
 903
 904                 if (xfer->flags_int.control_xfr &&
 905                     xfer->flags_int.control_hdr) {
 906                         /* special case */
 907                         if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) {
 908                                 /* The device controller writes to memory */
 909                                 xfer->frbuffers[0].isread = 1;
 910                         } else {
 911                                 /* The host controller reads from memory */
 912                                 xfer->frbuffers[0].isread = 0;
 913                         }
 914                 } else {
 915                         /* default case */
 916                         xfer->frbuffers[0].isread = isread;
 917                 }
 918
 919                 /*
 920                  * Setup the "page_start" pointer which points to an array of
 921                  * USB pages where information about the physical address of a
 922                  * page will be stored. Also initialise the "isread" field of
 923                  * the USB page caches.
 924                  */
 925                 xfer->frbuffers[0].page_start = pg;
 926
 927                 info->dma_nframes = nframes;
 928                 info->dma_currframe = 0;
 929                 info->dma_frlength_0 = frlength_0;
 930
 931                 pg += (frlength_0 / USB_PAGE_SIZE);
 932                 pg += 2;
 933
 934                 while (--nframes > 0) {
 935                         xfer->frbuffers[nframes].isread = isread;
 936                         xfer->frbuffers[nframes].page_start = pg;
 937
 938                         pg += (xfer->frlengths[nframes] / USB_PAGE_SIZE);
 939                         pg += 2;
 940                 }
 941
 942         }
 943         if (info->dma_error) {
 944                 USB_BUS_LOCK(info->bus);
 945                 usbd_transfer_done(xfer, USB_ERR_DMA_LOAD_FAILED);
 946                 USB_BUS_UNLOCK(info->bus);
 947                 return;
 948         }
 949         if (info->dma_currframe != info->dma_nframes) {
 950
 951                 if (info->dma_currframe == 0) {
 952                         /* special case */
 953                         usb_pc_load_mem(xfer->frbuffers,
 954                             info->dma_frlength_0, 0);
 955                 } else {
 956                         /* default case */
 957                         nframes = info->dma_currframe;
 958                         usb_pc_load_mem(xfer->frbuffers + nframes,
 959                             xfer->frlengths[nframes], 0);
 960                 }
 961
 962                 /* advance frame index */
 963                 info->dma_currframe++;
 964
 965                 return;
 966         }
 967         /* go ahead */
 968         usb_bdma_pre_sync(xfer);
 969
 970         /* start loading next USB transfer, if any */
 971         usb_command_wrapper(pq, NULL);
 972
 973         /* finally start the hardware */
 974         usbd_pipe_enter(xfer);
 975 }
 976
 977 /*------------------------------------------------------------------------*
 978  *      usb_bdma_done_event
 979  *
 980  * This function is called when the BUS-DMA has loaded virtual memory
 981  * into DMA, if any.
 982  *------------------------------------------------------------------------*/
 983 void
 984 usb_bdma_done_event(struct usb_dma_parent_tag *udpt)
 985 {
 986         struct usb_xfer_root *info;
 987
 988         info = USB_DMATAG_TO_XROOT(udpt);
 989
 990         mtx_assert(info->xfer_mtx, MA_OWNED);
 991
 992         /* copy error */
 993         info->dma_error = udpt->dma_error;
 994
 995         /* enter workloop again */
 996         usb_command_wrapper(&info->dma_q,
 997             info->dma_q.curr);
 998 }
 999
1000 /*------------------------------------------------------------------------*
1001  *      usb_bdma_pre_sync
1002  *
1003  * This function handles DMA synchronisation that must be done before
1004  * an USB transfer is started.
1005  *------------------------------------------------------------------------*/
1006 void
1007 usb_bdma_pre_sync(struct usb_xfer *xfer)
1008 {
1009         struct usb_page_cache *pc;
1010         usb_frcount_t nframes;
1011
1012         if (xfer->flags_int.isochronous_xfr) {
1013                 /* only one frame buffer */
1014                 nframes = 1;
1015         } else {
1016                 /* can be multiple frame buffers */
1017                 nframes = xfer->nframes;
1018         }
1019
1020         pc = xfer->frbuffers;
1021
1022         while (nframes--) {
1023
1024                 if (pc->isread) {
1025                         usb_pc_cpu_invalidate(pc);
1026                 } else {
1027                         usb_pc_cpu_flush(pc);
1028                 }
1029                 pc++;
1030         }
1031 }
1032
1033 /*------------------------------------------------------------------------*
1034  *      usb_bdma_post_sync
1035  *
1036  * This function handles DMA synchronisation that must be done after
1037  * an USB transfer is complete.
1038  *------------------------------------------------------------------------*/
1039 void
1040 usb_bdma_post_sync(struct usb_xfer *xfer)
1041 {
1042         struct usb_page_cache *pc;
1043         usb_frcount_t nframes;
1044
1045         if (xfer->flags_int.isochronous_xfr) {
1046                 /* only one frame buffer */
1047                 nframes = 1;
1048         } else {
1049                 /* can be multiple frame buffers */
1050                 nframes = xfer->nframes;
1051         }
1052
1053         pc = xfer->frbuffers;
1054
1055         while (nframes--) {
1056                 if (pc->isread) {
1057                         usb_pc_cpu_invalidate(pc);
1058                 }
1059                 pc++;
1060         }
1061 }
1062
1063 #endif