sys/dev/usb/usb_busdma.c

   1 /* $FreeBSD$ */
   2 /*-
   3  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   4  *
   5  * Copyright (c) 2008 Hans Petter Selasky. All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  26  * SUCH DAMAGE.
  27  */
  28
  29 #ifdef USB_GLOBAL_INCLUDE_FILE
  30 #include USB_GLOBAL_INCLUDE_FILE
  31 #else
  32 #include <sys/stdint.h>
  33 #include <sys/stddef.h>
  34 #include <sys/param.h>
  35 #include <sys/queue.h>
  36 #include <sys/types.h>
  37 #include <sys/systm.h>
  38 #include <sys/kernel.h>
  39 #include <sys/bus.h>
  40 #include <sys/module.h>
  41 #include <sys/lock.h>
  42 #include <sys/mutex.h>
  43 #include <sys/condvar.h>
  44 #include <sys/sysctl.h>
  45 #include <sys/sx.h>
  46 #include <sys/unistd.h>
  47 #include <sys/callout.h>
  48 #include <sys/malloc.h>
  49 #include <sys/priv.h>
  50
  51 #include <dev/usb/usb.h>
  52 #include <dev/usb/usbdi.h>
  53 #include <dev/usb/usbdi_util.h>
  54
  55 #define USB_DEBUG_VAR usb_debug
  56
  57 #include <dev/usb/usb_core.h>
  58 #include <dev/usb/usb_busdma.h>
  59 #include <dev/usb/usb_process.h>
  60 #include <dev/usb/usb_transfer.h>
  61 #include <dev/usb/usb_device.h>
  62 #include <dev/usb/usb_util.h>
  63 #include <dev/usb/usb_debug.h>
  64
  65 #include <dev/usb/usb_controller.h>
  66 #include <dev/usb/usb_bus.h>
  67 #endif                  /* USB_GLOBAL_INCLUDE_FILE */
  68
  69 #if USB_HAVE_BUSDMA
  70 static void     usb_dma_tag_create(struct usb_dma_tag *, usb_size_t, usb_size_t);
  71 static void     usb_dma_tag_destroy(struct usb_dma_tag *);
  72 static void     usb_dma_lock_cb(void *, bus_dma_lock_op_t);
  73 static void     usb_pc_alloc_mem_cb(void *, bus_dma_segment_t *, int, int);
  74 static void     usb_pc_load_mem_cb(void *, bus_dma_segment_t *, int, int);
  75 static void     usb_pc_common_mem_cb(void *, bus_dma_segment_t *, int, int,
  76                     uint8_t);
  77 #endif
  78
  79 /*------------------------------------------------------------------------*
  80  *  usbd_get_page - lookup DMA-able memory for the given offset
  81  *
  82  * NOTE: Only call this function when the "page_cache" structure has
  83  * been properly initialized !
  84  *------------------------------------------------------------------------*/
  85 void
  86 usbd_get_page(struct usb_page_cache *pc, usb_frlength_t offset,
  87     struct usb_page_search *res)
  88 {
  89 #if USB_HAVE_BUSDMA
  90         struct usb_page *page;
  91
  92         if (pc->page_start) {
  93
  94                 /* Case 1 - something has been loaded into DMA */
  95
  96                 if (pc->buffer) {
  97
  98                         /* Case 1a - Kernel Virtual Address */
  99
 100                         res->buffer = USB_ADD_BYTES(pc->buffer, offset);
 101                 }
 102                 offset += pc->page_offset_buf;
 103
 104                 /* compute destination page */
 105
 106                 page = pc->page_start;
 107
 108                 if (pc->ismultiseg) {
 109
 110                         page += (offset / USB_PAGE_SIZE);
 111
 112                         offset %= USB_PAGE_SIZE;
 113
 114                         res->length = USB_PAGE_SIZE - offset;
 115                         res->physaddr = page->physaddr + offset;
 116                 } else {
 117                         res->length = (usb_size_t)-1;
 118                         res->physaddr = page->physaddr + offset;
 119                 }
 120                 if (!pc->buffer) {
 121
 122                         /* Case 1b - Non Kernel Virtual Address */
 123
 124                         res->buffer = USB_ADD_BYTES(page->buffer, offset);
 125                 }
 126                 return;
 127         }
 128 #endif
 129         /* Case 2 - Plain PIO */
 130
 131         res->buffer = USB_ADD_BYTES(pc->buffer, offset);
 132         res->length = (usb_size_t)-1;
 133 #if USB_HAVE_BUSDMA
 134         res->physaddr = 0;
 135 #endif
 136 }
 137
 138 /*------------------------------------------------------------------------*
 139  *  usb_pc_buffer_is_aligned - verify alignment
 140  *
 141  * This function is used to check if a page cache buffer is properly
 142  * aligned to reduce the use of bounce buffers in PIO mode.
 143  *------------------------------------------------------------------------*/
 144 uint8_t
 145 usb_pc_buffer_is_aligned(struct usb_page_cache *pc, usb_frlength_t offset,
 146     usb_frlength_t len, usb_frlength_t mask)
 147 {
 148         struct usb_page_search buf_res;
 149
 150         while (len != 0) {
 151
 152                 usbd_get_page(pc, offset, &buf_res);
 153
 154                 if (buf_res.length > len)
 155                         buf_res.length = len;
 156                 if (USB_P2U(buf_res.buffer) & mask)
 157                         return (0);
 158                 if (buf_res.length & mask)
 159                         return (0);
 160
 161                 offset += buf_res.length;
 162                 len -= buf_res.length;
 163         }
 164         return (1);
 165 }
 166
 167 /*------------------------------------------------------------------------*
 168  *  usbd_copy_in - copy directly to DMA-able memory
 169  *------------------------------------------------------------------------*/
 170 void
 171 usbd_copy_in(struct usb_page_cache *cache, usb_frlength_t offset,
 172     const void *ptr, usb_frlength_t len)
 173 {
 174         struct usb_page_search buf_res;
 175
 176         while (len != 0) {
 177
 178                 usbd_get_page(cache, offset, &buf_res);
 179
 180                 if (buf_res.length > len) {
 181                         buf_res.length = len;
 182                 }
 183                 memcpy(buf_res.buffer, ptr, buf_res.length);
 184
 185                 offset += buf_res.length;
 186                 len -= buf_res.length;
 187                 ptr = USB_ADD_BYTES(ptr, buf_res.length);
 188         }
 189 }
 190
 191 /*------------------------------------------------------------------------*
 192  *  usbd_copy_in_user - copy directly to DMA-able memory from userland
 193  *
 194  * Return values:
 195  *    0: Success
 196  * Else: Failure
 197  *------------------------------------------------------------------------*/
 198 #if USB_HAVE_USER_IO
 199 int
 200 usbd_copy_in_user(struct usb_page_cache *cache, usb_frlength_t offset,
 201     const void *ptr, usb_frlength_t len)
 202 {
 203         struct usb_page_search buf_res;
 204         int error;
 205
 206         while (len != 0) {
 207
 208                 usbd_get_page(cache, offset, &buf_res);
 209
 210                 if (buf_res.length > len) {
 211                         buf_res.length = len;
 212                 }
 213                 error = copyin(ptr, buf_res.buffer, buf_res.length);
 214                 if (error)
 215                         return (error);
 216
 217                 offset += buf_res.length;
 218                 len -= buf_res.length;
 219                 ptr = USB_ADD_BYTES(ptr, buf_res.length);
 220         }
 221         return (0);                     /* success */
 222 }
 223 #endif
 224
 225 /*------------------------------------------------------------------------*
 226  *  usbd_m_copy_in - copy a mbuf chain directly into DMA-able memory
 227  *------------------------------------------------------------------------*/
 228 #if USB_HAVE_MBUF
 229 struct usb_m_copy_in_arg {
 230         struct usb_page_cache *cache;
 231         usb_frlength_t dst_offset;
 232 };
 233
 234 static int
 235 usbd_m_copy_in_cb(void *arg, void *src, uint32_t count)
 236 {
 237         struct usb_m_copy_in_arg *ua = arg;
 238
 239         usbd_copy_in(ua->cache, ua->dst_offset, src, count);
 240         ua->dst_offset += count;
 241         return (0);
 242 }
 243
 244 void
 245 usbd_m_copy_in(struct usb_page_cache *cache, usb_frlength_t dst_offset,
 246     struct mbuf *m, usb_size_t src_offset, usb_frlength_t src_len)
 247 {
 248         struct usb_m_copy_in_arg arg = {cache, dst_offset};
 249         (void) m_apply(m, src_offset, src_len, &usbd_m_copy_in_cb, &arg);
 250 }
 251 #endif
 252
 253 /*------------------------------------------------------------------------*
 254  *  usb_uiomove - factored out code
 255  *------------------------------------------------------------------------*/
 256 #if USB_HAVE_USER_IO
 257 int
 258 usb_uiomove(struct usb_page_cache *pc, struct uio *uio,
 259     usb_frlength_t pc_offset, usb_frlength_t len)
 260 {
 261         struct usb_page_search res;
 262         int error = 0;
 263
 264         while (len != 0) {
 265
 266                 usbd_get_page(pc, pc_offset, &res);
 267
 268                 if (res.length > len) {
 269                         res.length = len;
 270                 }
 271                 /*
 272                  * "uiomove()" can sleep so one needs to make a wrapper,
 273                  * exiting the mutex and checking things
 274                  */
 275                 error = uiomove(res.buffer, res.length, uio);
 276
 277                 if (error) {
 278                         break;
 279                 }
 280                 pc_offset += res.length;
 281                 len -= res.length;
 282         }
 283         return (error);
 284 }
 285 #endif
 286
 287 /*------------------------------------------------------------------------*
 288  *  usbd_copy_out - copy directly from DMA-able memory
 289  *------------------------------------------------------------------------*/
 290 void
 291 usbd_copy_out(struct usb_page_cache *cache, usb_frlength_t offset,
 292     void *ptr, usb_frlength_t len)
 293 {
 294         struct usb_page_search res;
 295
 296         while (len != 0) {
 297
 298                 usbd_get_page(cache, offset, &res);
 299
 300                 if (res.length > len) {
 301                         res.length = len;
 302                 }
 303                 memcpy(ptr, res.buffer, res.length);
 304
 305                 offset += res.length;
 306                 len -= res.length;
 307                 ptr = USB_ADD_BYTES(ptr, res.length);
 308         }
 309 }
 310
 311 /*------------------------------------------------------------------------*
 312  *  usbd_copy_out_user - copy directly from DMA-able memory to userland
 313  *
 314  * Return values:
 315  *    0: Success
 316  * Else: Failure
 317  *------------------------------------------------------------------------*/
 318 #if USB_HAVE_USER_IO
 319 int
 320 usbd_copy_out_user(struct usb_page_cache *cache, usb_frlength_t offset,
 321     void *ptr, usb_frlength_t len)
 322 {
 323         struct usb_page_search res;
 324         int error;
 325
 326         while (len != 0) {
 327
 328                 usbd_get_page(cache, offset, &res);
 329
 330                 if (res.length > len) {
 331                         res.length = len;
 332                 }
 333                 error = copyout(res.buffer, ptr, res.length);
 334                 if (error)
 335                         return (error);
 336
 337                 offset += res.length;
 338                 len -= res.length;
 339                 ptr = USB_ADD_BYTES(ptr, res.length);
 340         }
 341         return (0);                     /* success */
 342 }
 343 #endif
 344
 345 /*------------------------------------------------------------------------*
 346  *  usbd_frame_zero - zero DMA-able memory
 347  *------------------------------------------------------------------------*/
 348 void
 349 usbd_frame_zero(struct usb_page_cache *cache, usb_frlength_t offset,
 350     usb_frlength_t len)
 351 {
 352         struct usb_page_search res;
 353
 354         while (len != 0) {
 355
 356                 usbd_get_page(cache, offset, &res);
 357
 358                 if (res.length > len) {
 359                         res.length = len;
 360                 }
 361                 memset(res.buffer, 0, res.length);
 362
 363                 offset += res.length;
 364                 len -= res.length;
 365         }
 366 }
 367
 368 #if USB_HAVE_BUSDMA
 369
 370 /*------------------------------------------------------------------------*
 371  *      usb_dma_lock_cb - dummy callback
 372  *------------------------------------------------------------------------*/
 373 static void
 374 usb_dma_lock_cb(void *arg, bus_dma_lock_op_t op)
 375 {
 376         /* we use "mtx_owned()" instead of this function */
 377 }
 378
 379 /*------------------------------------------------------------------------*
 380  *      usb_dma_tag_create - allocate a DMA tag
 381  *
 382  * NOTE: If the "align" parameter has a value of 1 the DMA-tag will
 383  * allow multi-segment mappings. Else all mappings are single-segment.
 384  *------------------------------------------------------------------------*/
 385 static void
 386 usb_dma_tag_create(struct usb_dma_tag *udt,
 387     usb_size_t size, usb_size_t align)
 388 {
 389         bus_dma_tag_t tag;
 390
 391         if (bus_dma_tag_create
 392             ( /* parent    */ udt->tag_parent->tag,
 393              /* alignment */ align,
 394              /* boundary  */ 0,
 395              /* lowaddr   */ (2ULL << (udt->tag_parent->dma_bits - 1)) - 1,
 396              /* highaddr  */ BUS_SPACE_MAXADDR,
 397              /* filter    */ NULL,
 398              /* filterarg */ NULL,
 399              /* maxsize   */ size,
 400              /* nsegments */ (align == 1 && size > 1) ?
 401             (2 + (size / USB_PAGE_SIZE)) : 1,
 402              /* maxsegsz  */ (align == 1 && size > USB_PAGE_SIZE) ?
 403             USB_PAGE_SIZE : size,
 404              /* flags     */ BUS_DMA_KEEP_PG_OFFSET,
 405              /* lockfn    */ &usb_dma_lock_cb,
 406              /* lockarg   */ NULL,
 407             &tag)) {
 408                 tag = NULL;
 409         }
 410         udt->tag = tag;
 411 }
 412
 413 /*------------------------------------------------------------------------*
 414  *      usb_dma_tag_free - free a DMA tag
 415  *------------------------------------------------------------------------*/
 416 static void
 417 usb_dma_tag_destroy(struct usb_dma_tag *udt)
 418 {
 419         bus_dma_tag_destroy(udt->tag);
 420 }
 421
 422 /*------------------------------------------------------------------------*
 423  *      usb_pc_alloc_mem_cb - BUS-DMA callback function
 424  *------------------------------------------------------------------------*/
 425 static void
 426 usb_pc_alloc_mem_cb(void *arg, bus_dma_segment_t *segs,
 427     int nseg, int error)
 428 {
 429         usb_pc_common_mem_cb(arg, segs, nseg, error, 0);
 430 }
 431
 432 /*------------------------------------------------------------------------*
 433  *      usb_pc_load_mem_cb - BUS-DMA callback function
 434  *------------------------------------------------------------------------*/
 435 static void
 436 usb_pc_load_mem_cb(void *arg, bus_dma_segment_t *segs,
 437     int nseg, int error)
 438 {
 439         usb_pc_common_mem_cb(arg, segs, nseg, error, 1);
 440 }
 441
 442 /*------------------------------------------------------------------------*
 443  *      usb_pc_common_mem_cb - BUS-DMA callback function
 444  *------------------------------------------------------------------------*/
 445 static void
 446 usb_pc_common_mem_cb(void *arg, bus_dma_segment_t *segs,
 447     int nseg, int error, uint8_t isload)
 448 {
 449         struct usb_dma_parent_tag *uptag;
 450         struct usb_page_cache *pc;
 451         struct usb_page *pg;
 452         usb_size_t rem;
 453         bus_size_t off;
 454         uint8_t owned;
 455
 456         pc = arg;
 457         uptag = pc->tag_parent;
 458
 459         /*
 460          * XXX There is sometimes recursive locking here.
 461          * XXX We should try to find a better solution.
 462          * XXX Until further the "owned" variable does
 463          * XXX the trick.
 464          */
 465
 466         if (error) {
 467                 goto done;
 468         }
 469
 470         off = 0;
 471         pg = pc->page_start;
 472         pg->physaddr = rounddown2(segs->ds_addr, USB_PAGE_SIZE);
 473         rem = segs->ds_addr & (USB_PAGE_SIZE - 1);
 474         pc->page_offset_buf = rem;
 475         pc->page_offset_end += rem;
 476 #ifdef USB_DEBUG
 477         if (nseg > 1) {
 478                 int x;
 479
 480                 for (x = 0; x != nseg - 1; x++) {
 481                         if (((segs[x].ds_addr + segs[x].ds_len) & (USB_PAGE_SIZE - 1)) ==
 482                             ((segs[x + 1].ds_addr & (USB_PAGE_SIZE - 1))))
 483                                 continue;
 484                         /*
 485                          * This check verifies there is no page offset
 486                          * hole between any of the segments. See the
 487                          * BUS_DMA_KEEP_PG_OFFSET flag.
 488                          */
 489                         DPRINTFN(0, "Page offset was not preserved\n");
 490                         error = 1;
 491                         goto done;
 492                 }
 493         }
 494 #endif
 495         while (pc->ismultiseg) {
 496                 off += USB_PAGE_SIZE;
 497                 if (off >= (segs->ds_len + rem)) {
 498                         /* page crossing */
 499                         nseg--;
 500                         segs++;
 501                         off = 0;
 502                         rem = 0;
 503                         if (nseg == 0)
 504                                 break;
 505                 }
 506                 pg++;
 507                 pg->physaddr = rounddown2(segs->ds_addr + off, USB_PAGE_SIZE);
 508         }
 509
 510 done:
 511         owned = mtx_owned(uptag->mtx);
 512         if (!owned)
 513                 USB_MTX_LOCK(uptag->mtx);
 514
 515         uptag->dma_error = (error ? 1 : 0);
 516         if (isload) {
 517                 (uptag->func) (uptag);
 518         } else {
 519                 cv_broadcast(uptag->cv);
 520         }
 521         if (!owned)
 522                 USB_MTX_UNLOCK(uptag->mtx);
 523 }
 524
 525 /*------------------------------------------------------------------------*
 526  *      usb_pc_alloc_mem - allocate DMA'able memory
 527  *
 528  * Returns:
 529  *    0: Success
 530  * Else: Failure
 531  *------------------------------------------------------------------------*/
 532 uint8_t
 533 usb_pc_alloc_mem(struct usb_page_cache *pc, struct usb_page *pg,
 534     usb_size_t size, usb_size_t align)
 535 {
 536         struct usb_dma_parent_tag *uptag;
 537         struct usb_dma_tag *utag;
 538         bus_dmamap_t map;
 539         void *ptr;
 540         int err;
 541
 542         uptag = pc->tag_parent;
 543
 544         if (align != 1) {
 545                 /*
 546                  * The alignment must be greater or equal to the
 547                  * "size" else the object can be split between two
 548                  * memory pages and we get a problem!
 549                  */
 550                 while (align < size) {
 551                         align *= 2;
 552                         if (align == 0) {
 553                                 goto error;
 554                         }
 555                 }
 556 #if 1
 557                 /*
 558                  * XXX BUS-DMA workaround - FIXME later:
 559                  *
 560                  * We assume that that the aligment at this point of
 561                  * the code is greater than or equal to the size and
 562                  * less than two times the size, so that if we double
 563                  * the size, the size will be greater than the
 564                  * alignment.
 565                  *
 566                  * The bus-dma system has a check for "alignment"
 567                  * being less than "size". If that check fails we end
 568                  * up using contigmalloc which is page based even for
 569                  * small allocations. Try to avoid that to save
 570                  * memory, hence we sometimes to a large number of
 571                  * small allocations!
 572                  */
 573                 if (size <= (USB_PAGE_SIZE / 2)) {
 574                         size *= 2;
 575                 }
 576 #endif
 577         }
 578         /* get the correct DMA tag */
 579         utag = usb_dma_tag_find(uptag, size, align);
 580         if (utag == NULL) {
 581                 goto error;
 582         }
 583         /* allocate memory */
 584         if (bus_dmamem_alloc(
 585             utag->tag, &ptr, (BUS_DMA_WAITOK | BUS_DMA_COHERENT), &map)) {
 586                 goto error;
 587         }
 588         /* setup page cache */
 589         pc->buffer = ptr;
 590         pc->page_start = pg;
 591         pc->page_offset_buf = 0;
 592         pc->page_offset_end = size;
 593         pc->map = map;
 594         pc->tag = utag->tag;
 595         pc->ismultiseg = (align == 1);
 596
 597         USB_MTX_LOCK(uptag->mtx);
 598
 599         /* load memory into DMA */
 600         err = bus_dmamap_load(
 601             utag->tag, map, ptr, size, &usb_pc_alloc_mem_cb,
 602             pc, (BUS_DMA_WAITOK | BUS_DMA_COHERENT));
 603
 604         if (err == EINPROGRESS) {
 605                 cv_wait(uptag->cv, uptag->mtx);
 606                 err = 0;
 607         }
 608         USB_MTX_UNLOCK(uptag->mtx);
 609
 610         if (err || uptag->dma_error) {
 611                 bus_dmamem_free(utag->tag, ptr, map);
 612                 goto error;
 613         }
 614         memset(ptr, 0, size);
 615
 616         usb_pc_cpu_flush(pc);
 617
 618         return (0);
 619
 620 error:
 621         /* reset most of the page cache */
 622         pc->buffer = NULL;
 623         pc->page_start = NULL;
 624         pc->page_offset_buf = 0;
 625         pc->page_offset_end = 0;
 626         pc->map = NULL;
 627         pc->tag = NULL;
 628         return (1);
 629 }
 630
 631 /*------------------------------------------------------------------------*
 632  *      usb_pc_free_mem - free DMA memory
 633  *
 634  * This function is NULL safe.
 635  *------------------------------------------------------------------------*/
 636 void
 637 usb_pc_free_mem(struct usb_page_cache *pc)
 638 {
 639         if (pc && pc->buffer) {
 640
 641                 bus_dmamap_unload(pc->tag, pc->map);
 642
 643                 bus_dmamem_free(pc->tag, pc->buffer, pc->map);
 644
 645                 pc->buffer = NULL;
 646         }
 647 }
 648
 649 /*------------------------------------------------------------------------*
 650  *      usb_pc_load_mem - load virtual memory into DMA
 651  *
 652  * Return values:
 653  * 0: Success
 654  * Else: Error
 655  *------------------------------------------------------------------------*/
 656 uint8_t
 657 usb_pc_load_mem(struct usb_page_cache *pc, usb_size_t size, uint8_t sync)
 658 {
 659         /* setup page cache */
 660         pc->page_offset_buf = 0;
 661         pc->page_offset_end = size;
 662         pc->ismultiseg = 1;
 663
 664         USB_MTX_ASSERT(pc->tag_parent->mtx, MA_OWNED);
 665
 666         if (size > 0) {
 667                 if (sync) {
 668                         struct usb_dma_parent_tag *uptag;
 669                         int err;
 670
 671                         uptag = pc->tag_parent;
 672
 673                         /*
 674                          * We have to unload the previous loaded DMA
 675                          * pages before trying to load a new one!
 676                          */
 677                         bus_dmamap_unload(pc->tag, pc->map);
 678
 679                         /*
 680                          * Try to load memory into DMA.
 681                          */
 682                         err = bus_dmamap_load(
 683                             pc->tag, pc->map, pc->buffer, size,
 684                             &usb_pc_alloc_mem_cb, pc, BUS_DMA_WAITOK);
 685                         if (err == EINPROGRESS) {
 686                                 cv_wait(uptag->cv, uptag->mtx);
 687                                 err = 0;
 688                         }
 689                         if (err || uptag->dma_error) {
 690                                 return (1);
 691                         }
 692                 } else {
 693
 694                         /*
 695                          * We have to unload the previous loaded DMA
 696                          * pages before trying to load a new one!
 697                          */
 698                         bus_dmamap_unload(pc->tag, pc->map);
 699
 700                         /*
 701                          * Try to load memory into DMA. The callback
 702                          * will be called in all cases:
 703                          */
 704                         if (bus_dmamap_load(
 705                             pc->tag, pc->map, pc->buffer, size,
 706                             &usb_pc_load_mem_cb, pc, BUS_DMA_WAITOK)) {
 707                         }
 708                 }
 709         } else {
 710                 if (!sync) {
 711                         /*
 712                          * Call callback so that refcount is decremented
 713                          * properly:
 714                          */
 715                         pc->tag_parent->dma_error = 0;
 716                         (pc->tag_parent->func) (pc->tag_parent);
 717                 }
 718         }
 719         return (0);
 720 }
 721
 722 /*------------------------------------------------------------------------*
 723  *      usb_pc_cpu_invalidate - invalidate CPU cache
 724  *------------------------------------------------------------------------*/
 725 void
 726 usb_pc_cpu_invalidate(struct usb_page_cache *pc)
 727 {
 728         if (pc->page_offset_end == pc->page_offset_buf) {
 729                 /* nothing has been loaded into this page cache! */
 730                 return;
 731         }
 732
 733         /*
 734          * TODO: We currently do XXX_POSTREAD and XXX_PREREAD at the
 735          * same time, but in the future we should try to isolate the
 736          * different cases to optimise the code. --HPS
 737          */
 738         bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_POSTREAD);
 739         bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_PREREAD);
 740 }
 741
 742 /*------------------------------------------------------------------------*
 743  *      usb_pc_cpu_flush - flush CPU cache
 744  *------------------------------------------------------------------------*/
 745 void
 746 usb_pc_cpu_flush(struct usb_page_cache *pc)
 747 {
 748         if (pc->page_offset_end == pc->page_offset_buf) {
 749                 /* nothing has been loaded into this page cache! */
 750                 return;
 751         }
 752         bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_PREWRITE);
 753 }
 754
 755 /*------------------------------------------------------------------------*
 756  *      usb_pc_dmamap_create - create a DMA map
 757  *
 758  * Returns:
 759  *    0: Success
 760  * Else: Failure
 761  *------------------------------------------------------------------------*/
 762 uint8_t
 763 usb_pc_dmamap_create(struct usb_page_cache *pc, usb_size_t size)
 764 {
 765         struct usb_xfer_root *info;
 766         struct usb_dma_tag *utag;
 767
 768         /* get info */
 769         info = USB_DMATAG_TO_XROOT(pc->tag_parent);
 770
 771         /* sanity check */
 772         if (info == NULL) {
 773                 goto error;
 774         }
 775         utag = usb_dma_tag_find(pc->tag_parent, size, 1);
 776         if (utag == NULL) {
 777                 goto error;
 778         }
 779         /* create DMA map */
 780         if (bus_dmamap_create(utag->tag, 0, &pc->map)) {
 781                 goto error;
 782         }
 783         pc->tag = utag->tag;
 784         return 0;                       /* success */
 785
 786 error:
 787         pc->map = NULL;
 788         pc->tag = NULL;
 789         return 1;                       /* failure */
 790 }
 791
 792 /*------------------------------------------------------------------------*
 793  *      usb_pc_dmamap_destroy
 794  *
 795  * This function is NULL safe.
 796  *------------------------------------------------------------------------*/
 797 void
 798 usb_pc_dmamap_destroy(struct usb_page_cache *pc)
 799 {
 800         if (pc && pc->tag) {
 801                 bus_dmamap_destroy(pc->tag, pc->map);
 802                 pc->tag = NULL;
 803                 pc->map = NULL;
 804         }
 805 }
 806
 807 /*------------------------------------------------------------------------*
 808  *      usb_dma_tag_find - factored out code
 809  *------------------------------------------------------------------------*/
 810 struct usb_dma_tag *
 811 usb_dma_tag_find(struct usb_dma_parent_tag *udpt,
 812     usb_size_t size, usb_size_t align)
 813 {
 814         struct usb_dma_tag *udt;
 815         uint8_t nudt;
 816
 817         USB_ASSERT(align > 0, ("Invalid parameter align = 0\n"));
 818         USB_ASSERT(size > 0, ("Invalid parameter size = 0\n"));
 819
 820         udt = udpt->utag_first;
 821         nudt = udpt->utag_max;
 822
 823         while (nudt--) {
 824
 825                 if (udt->align == 0) {
 826                         usb_dma_tag_create(udt, size, align);
 827                         if (udt->tag == NULL) {
 828                                 return (NULL);
 829                         }
 830                         udt->align = align;
 831                         udt->size = size;
 832                         return (udt);
 833                 }
 834                 if ((udt->align == align) && (udt->size == size)) {
 835                         return (udt);
 836                 }
 837                 udt++;
 838         }
 839         return (NULL);
 840 }
 841
 842 /*------------------------------------------------------------------------*
 843  *      usb_dma_tag_setup - initialise USB DMA tags
 844  *------------------------------------------------------------------------*/
 845 void
 846 usb_dma_tag_setup(struct usb_dma_parent_tag *udpt,
 847     struct usb_dma_tag *udt, bus_dma_tag_t dmat,
 848     struct mtx *mtx, usb_dma_callback_t *func,
 849     uint8_t ndmabits, uint8_t nudt)
 850 {
 851         memset(udpt, 0, sizeof(*udpt));
 852
 853         /* sanity checking */
 854         if ((nudt == 0) ||
 855             (ndmabits == 0) ||
 856             (mtx == NULL)) {
 857                 /* something is corrupt */
 858                 return;
 859         }
 860         /* initialise condition variable */
 861         cv_init(udpt->cv, "USB DMA CV");
 862
 863         /* store some information */
 864         udpt->mtx = mtx;
 865         udpt->func = func;
 866         udpt->tag = dmat;
 867         udpt->utag_first = udt;
 868         udpt->utag_max = nudt;
 869         udpt->dma_bits = ndmabits;
 870
 871         while (nudt--) {
 872                 memset(udt, 0, sizeof(*udt));
 873                 udt->tag_parent = udpt;
 874                 udt++;
 875         }
 876 }
 877
 878 /*------------------------------------------------------------------------*
 879  *      usb_bus_tag_unsetup - factored out code
 880  *------------------------------------------------------------------------*/
 881 void
 882 usb_dma_tag_unsetup(struct usb_dma_parent_tag *udpt)
 883 {
 884         struct usb_dma_tag *udt;
 885         uint8_t nudt;
 886
 887         udt = udpt->utag_first;
 888         nudt = udpt->utag_max;
 889
 890         while (nudt--) {
 891
 892                 if (udt->align) {
 893                         /* destroy the USB DMA tag */
 894                         usb_dma_tag_destroy(udt);
 895                         udt->align = 0;
 896                 }
 897                 udt++;
 898         }
 899
 900         if (udpt->utag_max) {
 901                 /* destroy the condition variable */
 902                 cv_destroy(udpt->cv);
 903         }
 904 }
 905
 906 /*------------------------------------------------------------------------*
 907  *      usb_bdma_work_loop
 908  *
 909  * This function handles loading of virtual buffers into DMA and is
 910  * only called when "dma_refcount" is zero.
 911  *------------------------------------------------------------------------*/
 912 void
 913 usb_bdma_work_loop(struct usb_xfer_queue *pq)
 914 {
 915         struct usb_xfer_root *info;
 916         struct usb_xfer *xfer;
 917         usb_frcount_t nframes;
 918
 919         xfer = pq->curr;
 920         info = xfer->xroot;
 921
 922         USB_MTX_ASSERT(info->xfer_mtx, MA_OWNED);
 923
 924         if (xfer->error) {
 925                 /* some error happened */
 926                 USB_BUS_LOCK(info->bus);
 927                 usbd_transfer_done(xfer, 0);
 928                 USB_BUS_UNLOCK(info->bus);
 929                 return;
 930         }
 931         if (!xfer->flags_int.bdma_setup) {
 932                 struct usb_page *pg;
 933                 usb_frlength_t frlength_0;
 934                 uint8_t isread;
 935
 936                 xfer->flags_int.bdma_setup = 1;
 937
 938                 /* reset BUS-DMA load state */
 939
 940                 info->dma_error = 0;
 941
 942                 if (xfer->flags_int.isochronous_xfr) {
 943                         /* only one frame buffer */
 944                         nframes = 1;
 945                         frlength_0 = xfer->sumlen;
 946                 } else {
 947                         /* can be multiple frame buffers */
 948                         nframes = xfer->nframes;
 949                         frlength_0 = xfer->frlengths[0];
 950                 }
 951
 952                 /*
 953                  * Set DMA direction first. This is needed to
 954                  * select the correct cache invalidate and cache
 955                  * flush operations.
 956                  */
 957                 isread = USB_GET_DATA_ISREAD(xfer);
 958                 pg = xfer->dma_page_ptr;
 959
 960                 if (xfer->flags_int.control_xfr &&
 961                     xfer->flags_int.control_hdr) {
 962                         /* special case */
 963                         if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) {
 964                                 /* The device controller writes to memory */
 965                                 xfer->frbuffers[0].isread = 1;
 966                         } else {
 967                                 /* The host controller reads from memory */
 968                                 xfer->frbuffers[0].isread = 0;
 969                         }
 970                 } else {
 971                         /* default case */
 972                         xfer->frbuffers[0].isread = isread;
 973                 }
 974
 975                 /*
 976                  * Setup the "page_start" pointer which points to an array of
 977                  * USB pages where information about the physical address of a
 978                  * page will be stored. Also initialise the "isread" field of
 979                  * the USB page caches.
 980                  */
 981                 xfer->frbuffers[0].page_start = pg;
 982
 983                 info->dma_nframes = nframes;
 984                 info->dma_currframe = 0;
 985                 info->dma_frlength_0 = frlength_0;
 986
 987                 pg += (frlength_0 / USB_PAGE_SIZE);
 988                 pg += 2;
 989
 990                 while (--nframes > 0) {
 991                         xfer->frbuffers[nframes].isread = isread;
 992                         xfer->frbuffers[nframes].page_start = pg;
 993
 994                         pg += (xfer->frlengths[nframes] / USB_PAGE_SIZE);
 995                         pg += 2;
 996                 }
 997
 998         }
 999         if (info->dma_error) {
1000                 USB_BUS_LOCK(info->bus);
1001                 usbd_transfer_done(xfer, USB_ERR_DMA_LOAD_FAILED);
1002                 USB_BUS_UNLOCK(info->bus);
1003                 return;
1004         }
1005         if (info->dma_currframe != info->dma_nframes) {
1006
1007                 if (info->dma_currframe == 0) {
1008                         /* special case */
1009                         usb_pc_load_mem(xfer->frbuffers,
1010                             info->dma_frlength_0, 0);
1011                 } else {
1012                         /* default case */
1013                         nframes = info->dma_currframe;
1014                         usb_pc_load_mem(xfer->frbuffers + nframes,
1015                             xfer->frlengths[nframes], 0);
1016                 }
1017
1018                 /* advance frame index */
1019                 info->dma_currframe++;
1020
1021                 return;
1022         }
1023         /* go ahead */
1024         usb_bdma_pre_sync(xfer);
1025
1026         /* start loading next USB transfer, if any */
1027         usb_command_wrapper(pq, NULL);
1028
1029         /* finally start the hardware */
1030         usbd_pipe_enter(xfer);
1031 }
1032
1033 /*------------------------------------------------------------------------*
1034  *      usb_bdma_done_event
1035  *
1036  * This function is called when the BUS-DMA has loaded virtual memory
1037  * into DMA, if any.
1038  *------------------------------------------------------------------------*/
1039 void
1040 usb_bdma_done_event(struct usb_dma_parent_tag *udpt)
1041 {
1042         struct usb_xfer_root *info;
1043
1044         info = USB_DMATAG_TO_XROOT(udpt);
1045
1046         USB_MTX_ASSERT(info->xfer_mtx, MA_OWNED);
1047
1048         /* copy error */
1049         info->dma_error = udpt->dma_error;
1050
1051         /* enter workloop again */
1052         usb_command_wrapper(&info->dma_q,
1053             info->dma_q.curr);
1054 }
1055
1056 /*------------------------------------------------------------------------*
1057  *      usb_bdma_pre_sync
1058  *
1059  * This function handles DMA synchronisation that must be done before
1060  * an USB transfer is started.
1061  *------------------------------------------------------------------------*/
1062 void
1063 usb_bdma_pre_sync(struct usb_xfer *xfer)
1064 {
1065         struct usb_page_cache *pc;
1066         usb_frcount_t nframes;
1067
1068         if (xfer->flags_int.isochronous_xfr) {
1069                 /* only one frame buffer */
1070                 nframes = 1;
1071         } else {
1072                 /* can be multiple frame buffers */
1073                 nframes = xfer->nframes;
1074         }
1075
1076         pc = xfer->frbuffers;
1077
1078         while (nframes--) {
1079
1080                 if (pc->isread) {
1081                         usb_pc_cpu_invalidate(pc);
1082                 } else {
1083                         usb_pc_cpu_flush(pc);
1084                 }
1085                 pc++;
1086         }
1087 }
1088
1089 /*------------------------------------------------------------------------*
1090  *      usb_bdma_post_sync
1091  *
1092  * This function handles DMA synchronisation that must be done after
1093  * an USB transfer is complete.
1094  *------------------------------------------------------------------------*/
1095 void
1096 usb_bdma_post_sync(struct usb_xfer *xfer)
1097 {
1098         struct usb_page_cache *pc;
1099         usb_frcount_t nframes;
1100
1101         if (xfer->flags_int.isochronous_xfr) {
1102                 /* only one frame buffer */
1103                 nframes = 1;
1104         } else {
1105                 /* can be multiple frame buffers */
1106                 nframes = xfer->nframes;
1107         }
1108
1109         pc = xfer->frbuffers;
1110
1111         while (nframes--) {
1112                 if (pc->isread) {
1113                         usb_pc_cpu_invalidate(pc);
1114                 }
1115                 pc++;
1116         }
1117 }
1118
1119 #endif