sys/dev/usb/usb_busdma.c

   1 /* $FreeBSD$ */
   2 /*-
   3  * Copyright (c) 2008 Hans Petter Selasky. All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  */
  26
  27 #ifdef USB_GLOBAL_INCLUDE_FILE
  28 #include USB_GLOBAL_INCLUDE_FILE
  29 #else
  30 #include <sys/stdint.h>
  31 #include <sys/stddef.h>
  32 #include <sys/param.h>
  33 #include <sys/queue.h>
  34 #include <sys/types.h>
  35 #include <sys/systm.h>
  36 #include <sys/kernel.h>
  37 #include <sys/bus.h>
  38 #include <sys/module.h>
  39 #include <sys/lock.h>
  40 #include <sys/mutex.h>
  41 #include <sys/condvar.h>
  42 #include <sys/sysctl.h>
  43 #include <sys/sx.h>
  44 #include <sys/unistd.h>
  45 #include <sys/callout.h>
  46 #include <sys/malloc.h>
  47 #include <sys/priv.h>
  48
  49 #include <dev/usb/usb.h>
  50 #include <dev/usb/usbdi.h>
  51 #include <dev/usb/usbdi_util.h>
  52
  53 #define USB_DEBUG_VAR usb_debug
  54
  55 #include <dev/usb/usb_core.h>
  56 #include <dev/usb/usb_busdma.h>
  57 #include <dev/usb/usb_process.h>
  58 #include <dev/usb/usb_transfer.h>
  59 #include <dev/usb/usb_device.h>
  60 #include <dev/usb/usb_util.h>
  61 #include <dev/usb/usb_debug.h>
  62
  63 #include <dev/usb/usb_controller.h>
  64 #include <dev/usb/usb_bus.h>
  65 #endif                  /* USB_GLOBAL_INCLUDE_FILE */
  66
  67 #if USB_HAVE_BUSDMA
  68 static void     usb_dma_tag_create(struct usb_dma_tag *, usb_size_t, usb_size_t);
  69 static void     usb_dma_tag_destroy(struct usb_dma_tag *);
  70 static void     usb_dma_lock_cb(void *, bus_dma_lock_op_t);
  71 static void     usb_pc_alloc_mem_cb(void *, bus_dma_segment_t *, int, int);
  72 static void     usb_pc_load_mem_cb(void *, bus_dma_segment_t *, int, int);
  73 static void     usb_pc_common_mem_cb(void *, bus_dma_segment_t *, int, int,
  74                     uint8_t);
  75 #endif
  76
  77 /*------------------------------------------------------------------------*
  78  *  usbd_get_page - lookup DMA-able memory for the given offset
  79  *
  80  * NOTE: Only call this function when the "page_cache" structure has
  81  * been properly initialized !
  82  *------------------------------------------------------------------------*/
  83 void
  84 usbd_get_page(struct usb_page_cache *pc, usb_frlength_t offset,
  85     struct usb_page_search *res)
  86 {
  87 #if USB_HAVE_BUSDMA
  88         struct usb_page *page;
  89
  90         if (pc->page_start) {
  91
  92                 /* Case 1 - something has been loaded into DMA */
  93
  94                 if (pc->buffer) {
  95
  96                         /* Case 1a - Kernel Virtual Address */
  97
  98                         res->buffer = USB_ADD_BYTES(pc->buffer, offset);
  99                 }
 100                 offset += pc->page_offset_buf;
 101
 102                 /* compute destination page */
 103
 104                 page = pc->page_start;
 105
 106                 if (pc->ismultiseg) {
 107
 108                         page += (offset / USB_PAGE_SIZE);
 109
 110                         offset %= USB_PAGE_SIZE;
 111
 112                         res->length = USB_PAGE_SIZE - offset;
 113                         res->physaddr = page->physaddr + offset;
 114                 } else {
 115                         res->length = (usb_size_t)-1;
 116                         res->physaddr = page->physaddr + offset;
 117                 }
 118                 if (!pc->buffer) {
 119
 120                         /* Case 1b - Non Kernel Virtual Address */
 121
 122                         res->buffer = USB_ADD_BYTES(page->buffer, offset);
 123                 }
 124                 return;
 125         }
 126 #endif
 127         /* Case 2 - Plain PIO */
 128
 129         res->buffer = USB_ADD_BYTES(pc->buffer, offset);
 130         res->length = (usb_size_t)-1;
 131 #if USB_HAVE_BUSDMA
 132         res->physaddr = 0;
 133 #endif
 134 }
 135
 136 /*------------------------------------------------------------------------*
 137  *  usb_pc_buffer_is_aligned - verify alignment
 138  *
 139  * This function is used to check if a page cache buffer is properly
 140  * aligned to reduce the use of bounce buffers in PIO mode.
 141  *------------------------------------------------------------------------*/
 142 uint8_t
 143 usb_pc_buffer_is_aligned(struct usb_page_cache *pc, usb_frlength_t offset,
 144     usb_frlength_t len, usb_frlength_t mask)
 145 {
 146         struct usb_page_search buf_res;
 147
 148         while (len != 0) {
 149
 150                 usbd_get_page(pc, offset, &buf_res);
 151
 152                 if (buf_res.length > len)
 153                         buf_res.length = len;
 154                 if (USB_P2U(buf_res.buffer) & mask)
 155                         return (0);
 156                 if (buf_res.length & mask)
 157                         return (0);
 158
 159                 offset += buf_res.length;
 160                 len -= buf_res.length;
 161         }
 162         return (1);
 163 }
 164
 165 /*------------------------------------------------------------------------*
 166  *  usbd_copy_in - copy directly to DMA-able memory
 167  *------------------------------------------------------------------------*/
 168 void
 169 usbd_copy_in(struct usb_page_cache *cache, usb_frlength_t offset,
 170     const void *ptr, usb_frlength_t len)
 171 {
 172         struct usb_page_search buf_res;
 173
 174         while (len != 0) {
 175
 176                 usbd_get_page(cache, offset, &buf_res);
 177
 178                 if (buf_res.length > len) {
 179                         buf_res.length = len;
 180                 }
 181                 memcpy(buf_res.buffer, ptr, buf_res.length);
 182
 183                 offset += buf_res.length;
 184                 len -= buf_res.length;
 185                 ptr = USB_ADD_BYTES(ptr, buf_res.length);
 186         }
 187 }
 188
 189 /*------------------------------------------------------------------------*
 190  *  usbd_copy_in_user - copy directly to DMA-able memory from userland
 191  *
 192  * Return values:
 193  *    0: Success
 194  * Else: Failure
 195  *------------------------------------------------------------------------*/
 196 #if USB_HAVE_USER_IO
 197 int
 198 usbd_copy_in_user(struct usb_page_cache *cache, usb_frlength_t offset,
 199     const void *ptr, usb_frlength_t len)
 200 {
 201         struct usb_page_search buf_res;
 202         int error;
 203
 204         while (len != 0) {
 205
 206                 usbd_get_page(cache, offset, &buf_res);
 207
 208                 if (buf_res.length > len) {
 209                         buf_res.length = len;
 210                 }
 211                 error = copyin(ptr, buf_res.buffer, buf_res.length);
 212                 if (error)
 213                         return (error);
 214
 215                 offset += buf_res.length;
 216                 len -= buf_res.length;
 217                 ptr = USB_ADD_BYTES(ptr, buf_res.length);
 218         }
 219         return (0);                     /* success */
 220 }
 221 #endif
 222
 223 /*------------------------------------------------------------------------*
 224  *  usbd_m_copy_in - copy a mbuf chain directly into DMA-able memory
 225  *------------------------------------------------------------------------*/
 226 #if USB_HAVE_MBUF
 227 struct usb_m_copy_in_arg {
 228         struct usb_page_cache *cache;
 229         usb_frlength_t dst_offset;
 230 };
 231
 232 static int
 233 usbd_m_copy_in_cb(void *arg, void *src, uint32_t count)
 234 {
 235         struct usb_m_copy_in_arg *ua = arg;
 236
 237         usbd_copy_in(ua->cache, ua->dst_offset, src, count);
 238         ua->dst_offset += count;
 239         return (0);
 240 }
 241
 242 void
 243 usbd_m_copy_in(struct usb_page_cache *cache, usb_frlength_t dst_offset,
 244     struct mbuf *m, usb_size_t src_offset, usb_frlength_t src_len)
 245 {
 246         struct usb_m_copy_in_arg arg = {cache, dst_offset};
 247         (void) m_apply(m, src_offset, src_len, &usbd_m_copy_in_cb, &arg);
 248 }
 249 #endif
 250
 251 /*------------------------------------------------------------------------*
 252  *  usb_uiomove - factored out code
 253  *------------------------------------------------------------------------*/
 254 #if USB_HAVE_USER_IO
 255 int
 256 usb_uiomove(struct usb_page_cache *pc, struct uio *uio,
 257     usb_frlength_t pc_offset, usb_frlength_t len)
 258 {
 259         struct usb_page_search res;
 260         int error = 0;
 261
 262         while (len != 0) {
 263
 264                 usbd_get_page(pc, pc_offset, &res);
 265
 266                 if (res.length > len) {
 267                         res.length = len;
 268                 }
 269                 /*
 270                  * "uiomove()" can sleep so one needs to make a wrapper,
 271                  * exiting the mutex and checking things
 272                  */
 273                 error = uiomove(res.buffer, res.length, uio);
 274
 275                 if (error) {
 276                         break;
 277                 }
 278                 pc_offset += res.length;
 279                 len -= res.length;
 280         }
 281         return (error);
 282 }
 283 #endif
 284
 285 /*------------------------------------------------------------------------*
 286  *  usbd_copy_out - copy directly from DMA-able memory
 287  *------------------------------------------------------------------------*/
 288 void
 289 usbd_copy_out(struct usb_page_cache *cache, usb_frlength_t offset,
 290     void *ptr, usb_frlength_t len)
 291 {
 292         struct usb_page_search res;
 293
 294         while (len != 0) {
 295
 296                 usbd_get_page(cache, offset, &res);
 297
 298                 if (res.length > len) {
 299                         res.length = len;
 300                 }
 301                 memcpy(ptr, res.buffer, res.length);
 302
 303                 offset += res.length;
 304                 len -= res.length;
 305                 ptr = USB_ADD_BYTES(ptr, res.length);
 306         }
 307 }
 308
 309 /*------------------------------------------------------------------------*
 310  *  usbd_copy_out_user - copy directly from DMA-able memory to userland
 311  *
 312  * Return values:
 313  *    0: Success
 314  * Else: Failure
 315  *------------------------------------------------------------------------*/
 316 #if USB_HAVE_USER_IO
 317 int
 318 usbd_copy_out_user(struct usb_page_cache *cache, usb_frlength_t offset,
 319     void *ptr, usb_frlength_t len)
 320 {
 321         struct usb_page_search res;
 322         int error;
 323
 324         while (len != 0) {
 325
 326                 usbd_get_page(cache, offset, &res);
 327
 328                 if (res.length > len) {
 329                         res.length = len;
 330                 }
 331                 error = copyout(res.buffer, ptr, res.length);
 332                 if (error)
 333                         return (error);
 334
 335                 offset += res.length;
 336                 len -= res.length;
 337                 ptr = USB_ADD_BYTES(ptr, res.length);
 338         }
 339         return (0);                     /* success */
 340 }
 341 #endif
 342
 343 /*------------------------------------------------------------------------*
 344  *  usbd_frame_zero - zero DMA-able memory
 345  *------------------------------------------------------------------------*/
 346 void
 347 usbd_frame_zero(struct usb_page_cache *cache, usb_frlength_t offset,
 348     usb_frlength_t len)
 349 {
 350         struct usb_page_search res;
 351
 352         while (len != 0) {
 353
 354                 usbd_get_page(cache, offset, &res);
 355
 356                 if (res.length > len) {
 357                         res.length = len;
 358                 }
 359                 memset(res.buffer, 0, res.length);
 360
 361                 offset += res.length;
 362                 len -= res.length;
 363         }
 364 }
 365
 366 #if USB_HAVE_BUSDMA
 367
 368 /*------------------------------------------------------------------------*
 369  *      usb_dma_lock_cb - dummy callback
 370  *------------------------------------------------------------------------*/
 371 static void
 372 usb_dma_lock_cb(void *arg, bus_dma_lock_op_t op)
 373 {
 374         /* we use "mtx_owned()" instead of this function */
 375 }
 376
 377 /*------------------------------------------------------------------------*
 378  *      usb_dma_tag_create - allocate a DMA tag
 379  *
 380  * NOTE: If the "align" parameter has a value of 1 the DMA-tag will
 381  * allow multi-segment mappings. Else all mappings are single-segment.
 382  *------------------------------------------------------------------------*/
 383 static void
 384 usb_dma_tag_create(struct usb_dma_tag *udt,
 385     usb_size_t size, usb_size_t align)
 386 {
 387         bus_dma_tag_t tag;
 388
 389         if (bus_dma_tag_create
 390             ( /* parent    */ udt->tag_parent->tag,
 391              /* alignment */ align,
 392              /* boundary  */ 0,
 393              /* lowaddr   */ (2ULL << (udt->tag_parent->dma_bits - 1)) - 1,
 394              /* highaddr  */ BUS_SPACE_MAXADDR,
 395              /* filter    */ NULL,
 396              /* filterarg */ NULL,
 397              /* maxsize   */ size,
 398              /* nsegments */ (align == 1 && size > 1) ?
 399             (2 + (size / USB_PAGE_SIZE)) : 1,
 400              /* maxsegsz  */ (align == 1 && size > USB_PAGE_SIZE) ?
 401             USB_PAGE_SIZE : size,
 402              /* flags     */ BUS_DMA_KEEP_PG_OFFSET,
 403              /* lockfn    */ &usb_dma_lock_cb,
 404              /* lockarg   */ NULL,
 405             &tag)) {
 406                 tag = NULL;
 407         }
 408         udt->tag = tag;
 409 }
 410
 411 /*------------------------------------------------------------------------*
 412  *      usb_dma_tag_free - free a DMA tag
 413  *------------------------------------------------------------------------*/
 414 static void
 415 usb_dma_tag_destroy(struct usb_dma_tag *udt)
 416 {
 417         bus_dma_tag_destroy(udt->tag);
 418 }
 419
 420 /*------------------------------------------------------------------------*
 421  *      usb_pc_alloc_mem_cb - BUS-DMA callback function
 422  *------------------------------------------------------------------------*/
 423 static void
 424 usb_pc_alloc_mem_cb(void *arg, bus_dma_segment_t *segs,
 425     int nseg, int error)
 426 {
 427         usb_pc_common_mem_cb(arg, segs, nseg, error, 0);
 428 }
 429
 430 /*------------------------------------------------------------------------*
 431  *      usb_pc_load_mem_cb - BUS-DMA callback function
 432  *------------------------------------------------------------------------*/
 433 static void
 434 usb_pc_load_mem_cb(void *arg, bus_dma_segment_t *segs,
 435     int nseg, int error)
 436 {
 437         usb_pc_common_mem_cb(arg, segs, nseg, error, 1);
 438 }
 439
 440 /*------------------------------------------------------------------------*
 441  *      usb_pc_common_mem_cb - BUS-DMA callback function
 442  *------------------------------------------------------------------------*/
 443 static void
 444 usb_pc_common_mem_cb(void *arg, bus_dma_segment_t *segs,
 445     int nseg, int error, uint8_t isload)
 446 {
 447         struct usb_dma_parent_tag *uptag;
 448         struct usb_page_cache *pc;
 449         struct usb_page *pg;
 450         usb_size_t rem;
 451         bus_size_t off;
 452         uint8_t owned;
 453
 454         pc = arg;
 455         uptag = pc->tag_parent;
 456
 457         /*
 458          * XXX There is sometimes recursive locking here.
 459          * XXX We should try to find a better solution.
 460          * XXX Until further the "owned" variable does
 461          * XXX the trick.
 462          */
 463
 464         if (error) {
 465                 goto done;
 466         }
 467
 468         off = 0;
 469         pg = pc->page_start;
 470         pg->physaddr = rounddown2(segs->ds_addr, USB_PAGE_SIZE);
 471         rem = segs->ds_addr & (USB_PAGE_SIZE - 1);
 472         pc->page_offset_buf = rem;
 473         pc->page_offset_end += rem;
 474 #ifdef USB_DEBUG
 475         if (nseg > 1) {
 476                 int x;
 477
 478                 for (x = 0; x != nseg - 1; x++) {
 479                         if (((segs[x].ds_addr + segs[x].ds_len) & (USB_PAGE_SIZE - 1)) ==
 480                             ((segs[x + 1].ds_addr & (USB_PAGE_SIZE - 1))))
 481                                 continue;
 482                         /*
 483                          * This check verifies there is no page offset
 484                          * hole between any of the segments. See the
 485                          * BUS_DMA_KEEP_PG_OFFSET flag.
 486                          */
 487                         DPRINTFN(0, "Page offset was not preserved\n");
 488                         error = 1;
 489                         goto done;
 490                 }
 491         }
 492 #endif
 493         while (pc->ismultiseg) {
 494                 off += USB_PAGE_SIZE;
 495                 if (off >= (segs->ds_len + rem)) {
 496                         /* page crossing */
 497                         nseg--;
 498                         segs++;
 499                         off = 0;
 500                         rem = 0;
 501                         if (nseg == 0)
 502                                 break;
 503                 }
 504                 pg++;
 505                 pg->physaddr = rounddown2(segs->ds_addr + off, USB_PAGE_SIZE);
 506         }
 507
 508 done:
 509         owned = mtx_owned(uptag->mtx);
 510         if (!owned)
 511                 mtx_lock(uptag->mtx);
 512
 513         uptag->dma_error = (error ? 1 : 0);
 514         if (isload) {
 515                 (uptag->func) (uptag);
 516         } else {
 517                 cv_broadcast(uptag->cv);
 518         }
 519         if (!owned)
 520                 mtx_unlock(uptag->mtx);
 521 }
 522
 523 /*------------------------------------------------------------------------*
 524  *      usb_pc_alloc_mem - allocate DMA'able memory
 525  *
 526  * Returns:
 527  *    0: Success
 528  * Else: Failure
 529  *------------------------------------------------------------------------*/
 530 uint8_t
 531 usb_pc_alloc_mem(struct usb_page_cache *pc, struct usb_page *pg,
 532     usb_size_t size, usb_size_t align)
 533 {
 534         struct usb_dma_parent_tag *uptag;
 535         struct usb_dma_tag *utag;
 536         bus_dmamap_t map;
 537         void *ptr;
 538         int err;
 539
 540         uptag = pc->tag_parent;
 541
 542         if (align != 1) {
 543                 /*
 544                  * The alignment must be greater or equal to the
 545                  * "size" else the object can be split between two
 546                  * memory pages and we get a problem!
 547                  */
 548                 while (align < size) {
 549                         align *= 2;
 550                         if (align == 0) {
 551                                 goto error;
 552                         }
 553                 }
 554 #if 1
 555                 /*
 556                  * XXX BUS-DMA workaround - FIXME later:
 557                  *
 558                  * We assume that that the aligment at this point of
 559                  * the code is greater than or equal to the size and
 560                  * less than two times the size, so that if we double
 561                  * the size, the size will be greater than the
 562                  * alignment.
 563                  *
 564                  * The bus-dma system has a check for "alignment"
 565                  * being less than "size". If that check fails we end
 566                  * up using contigmalloc which is page based even for
 567                  * small allocations. Try to avoid that to save
 568                  * memory, hence we sometimes to a large number of
 569                  * small allocations!
 570                  */
 571                 if (size <= (USB_PAGE_SIZE / 2)) {
 572                         size *= 2;
 573                 }
 574 #endif
 575         }
 576         /* get the correct DMA tag */
 577         utag = usb_dma_tag_find(uptag, size, align);
 578         if (utag == NULL) {
 579                 goto error;
 580         }
 581         /* allocate memory */
 582         if (bus_dmamem_alloc(
 583             utag->tag, &ptr, (BUS_DMA_WAITOK | BUS_DMA_COHERENT), &map)) {
 584                 goto error;
 585         }
 586         /* setup page cache */
 587         pc->buffer = ptr;
 588         pc->page_start = pg;
 589         pc->page_offset_buf = 0;
 590         pc->page_offset_end = size;
 591         pc->map = map;
 592         pc->tag = utag->tag;
 593         pc->ismultiseg = (align == 1);
 594
 595         mtx_lock(uptag->mtx);
 596
 597         /* load memory into DMA */
 598         err = bus_dmamap_load(
 599             utag->tag, map, ptr, size, &usb_pc_alloc_mem_cb,
 600             pc, (BUS_DMA_WAITOK | BUS_DMA_COHERENT));
 601
 602         if (err == EINPROGRESS) {
 603                 cv_wait(uptag->cv, uptag->mtx);
 604                 err = 0;
 605         }
 606         mtx_unlock(uptag->mtx);
 607
 608         if (err || uptag->dma_error) {
 609                 bus_dmamem_free(utag->tag, ptr, map);
 610                 goto error;
 611         }
 612         memset(ptr, 0, size);
 613
 614         usb_pc_cpu_flush(pc);
 615
 616         return (0);
 617
 618 error:
 619         /* reset most of the page cache */
 620         pc->buffer = NULL;
 621         pc->page_start = NULL;
 622         pc->page_offset_buf = 0;
 623         pc->page_offset_end = 0;
 624         pc->map = NULL;
 625         pc->tag = NULL;
 626         return (1);
 627 }
 628
 629 /*------------------------------------------------------------------------*
 630  *      usb_pc_free_mem - free DMA memory
 631  *
 632  * This function is NULL safe.
 633  *------------------------------------------------------------------------*/
 634 void
 635 usb_pc_free_mem(struct usb_page_cache *pc)
 636 {
 637         if (pc && pc->buffer) {
 638
 639                 bus_dmamap_unload(pc->tag, pc->map);
 640
 641                 bus_dmamem_free(pc->tag, pc->buffer, pc->map);
 642
 643                 pc->buffer = NULL;
 644         }
 645 }
 646
 647 /*------------------------------------------------------------------------*
 648  *      usb_pc_load_mem - load virtual memory into DMA
 649  *
 650  * Return values:
 651  * 0: Success
 652  * Else: Error
 653  *------------------------------------------------------------------------*/
 654 uint8_t
 655 usb_pc_load_mem(struct usb_page_cache *pc, usb_size_t size, uint8_t sync)
 656 {
 657         /* setup page cache */
 658         pc->page_offset_buf = 0;
 659         pc->page_offset_end = size;
 660         pc->ismultiseg = 1;
 661
 662         mtx_assert(pc->tag_parent->mtx, MA_OWNED);
 663
 664         if (size > 0) {
 665                 if (sync) {
 666                         struct usb_dma_parent_tag *uptag;
 667                         int err;
 668
 669                         uptag = pc->tag_parent;
 670
 671                         /*
 672                          * We have to unload the previous loaded DMA
 673                          * pages before trying to load a new one!
 674                          */
 675                         bus_dmamap_unload(pc->tag, pc->map);
 676
 677                         /*
 678                          * Try to load memory into DMA.
 679                          */
 680                         err = bus_dmamap_load(
 681                             pc->tag, pc->map, pc->buffer, size,
 682                             &usb_pc_alloc_mem_cb, pc, BUS_DMA_WAITOK);
 683                         if (err == EINPROGRESS) {
 684                                 cv_wait(uptag->cv, uptag->mtx);
 685                                 err = 0;
 686                         }
 687                         if (err || uptag->dma_error) {
 688                                 return (1);
 689                         }
 690                 } else {
 691
 692                         /*
 693                          * We have to unload the previous loaded DMA
 694                          * pages before trying to load a new one!
 695                          */
 696                         bus_dmamap_unload(pc->tag, pc->map);
 697
 698                         /*
 699                          * Try to load memory into DMA. The callback
 700                          * will be called in all cases:
 701                          */
 702                         if (bus_dmamap_load(
 703                             pc->tag, pc->map, pc->buffer, size,
 704                             &usb_pc_load_mem_cb, pc, BUS_DMA_WAITOK)) {
 705                         }
 706                 }
 707         } else {
 708                 if (!sync) {
 709                         /*
 710                          * Call callback so that refcount is decremented
 711                          * properly:
 712                          */
 713                         pc->tag_parent->dma_error = 0;
 714                         (pc->tag_parent->func) (pc->tag_parent);
 715                 }
 716         }
 717         return (0);
 718 }
 719
 720 /*------------------------------------------------------------------------*
 721  *      usb_pc_cpu_invalidate - invalidate CPU cache
 722  *------------------------------------------------------------------------*/
 723 void
 724 usb_pc_cpu_invalidate(struct usb_page_cache *pc)
 725 {
 726         if (pc->page_offset_end == pc->page_offset_buf) {
 727                 /* nothing has been loaded into this page cache! */
 728                 return;
 729         }
 730
 731         /*
 732          * TODO: We currently do XXX_POSTREAD and XXX_PREREAD at the
 733          * same time, but in the future we should try to isolate the
 734          * different cases to optimise the code. --HPS
 735          */
 736         bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_POSTREAD);
 737         bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_PREREAD);
 738 }
 739
 740 /*------------------------------------------------------------------------*
 741  *      usb_pc_cpu_flush - flush CPU cache
 742  *------------------------------------------------------------------------*/
 743 void
 744 usb_pc_cpu_flush(struct usb_page_cache *pc)
 745 {
 746         if (pc->page_offset_end == pc->page_offset_buf) {
 747                 /* nothing has been loaded into this page cache! */
 748                 return;
 749         }
 750         bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_PREWRITE);
 751 }
 752
 753 /*------------------------------------------------------------------------*
 754  *      usb_pc_dmamap_create - create a DMA map
 755  *
 756  * Returns:
 757  *    0: Success
 758  * Else: Failure
 759  *------------------------------------------------------------------------*/
 760 uint8_t
 761 usb_pc_dmamap_create(struct usb_page_cache *pc, usb_size_t size)
 762 {
 763         struct usb_xfer_root *info;
 764         struct usb_dma_tag *utag;
 765
 766         /* get info */
 767         info = USB_DMATAG_TO_XROOT(pc->tag_parent);
 768
 769         /* sanity check */
 770         if (info == NULL) {
 771                 goto error;
 772         }
 773         utag = usb_dma_tag_find(pc->tag_parent, size, 1);
 774         if (utag == NULL) {
 775                 goto error;
 776         }
 777         /* create DMA map */
 778         if (bus_dmamap_create(utag->tag, 0, &pc->map)) {
 779                 goto error;
 780         }
 781         pc->tag = utag->tag;
 782         return 0;                       /* success */
 783
 784 error:
 785         pc->map = NULL;
 786         pc->tag = NULL;
 787         return 1;                       /* failure */
 788 }
 789
 790 /*------------------------------------------------------------------------*
 791  *      usb_pc_dmamap_destroy
 792  *
 793  * This function is NULL safe.
 794  *------------------------------------------------------------------------*/
 795 void
 796 usb_pc_dmamap_destroy(struct usb_page_cache *pc)
 797 {
 798         if (pc && pc->tag) {
 799                 bus_dmamap_destroy(pc->tag, pc->map);
 800                 pc->tag = NULL;
 801                 pc->map = NULL;
 802         }
 803 }
 804
 805 /*------------------------------------------------------------------------*
 806  *      usb_dma_tag_find - factored out code
 807  *------------------------------------------------------------------------*/
 808 struct usb_dma_tag *
 809 usb_dma_tag_find(struct usb_dma_parent_tag *udpt,
 810     usb_size_t size, usb_size_t align)
 811 {
 812         struct usb_dma_tag *udt;
 813         uint8_t nudt;
 814
 815         USB_ASSERT(align > 0, ("Invalid parameter align = 0\n"));
 816         USB_ASSERT(size > 0, ("Invalid parameter size = 0\n"));
 817
 818         udt = udpt->utag_first;
 819         nudt = udpt->utag_max;
 820
 821         while (nudt--) {
 822
 823                 if (udt->align == 0) {
 824                         usb_dma_tag_create(udt, size, align);
 825                         if (udt->tag == NULL) {
 826                                 return (NULL);
 827                         }
 828                         udt->align = align;
 829                         udt->size = size;
 830                         return (udt);
 831                 }
 832                 if ((udt->align == align) && (udt->size == size)) {
 833                         return (udt);
 834                 }
 835                 udt++;
 836         }
 837         return (NULL);
 838 }
 839
 840 /*------------------------------------------------------------------------*
 841  *      usb_dma_tag_setup - initialise USB DMA tags
 842  *------------------------------------------------------------------------*/
 843 void
 844 usb_dma_tag_setup(struct usb_dma_parent_tag *udpt,
 845     struct usb_dma_tag *udt, bus_dma_tag_t dmat,
 846     struct mtx *mtx, usb_dma_callback_t *func,
 847     uint8_t ndmabits, uint8_t nudt)
 848 {
 849         memset(udpt, 0, sizeof(*udpt));
 850
 851         /* sanity checking */
 852         if ((nudt == 0) ||
 853             (ndmabits == 0) ||
 854             (mtx == NULL)) {
 855                 /* something is corrupt */
 856                 return;
 857         }
 858         /* initialise condition variable */
 859         cv_init(udpt->cv, "USB DMA CV");
 860
 861         /* store some information */
 862         udpt->mtx = mtx;
 863         udpt->func = func;
 864         udpt->tag = dmat;
 865         udpt->utag_first = udt;
 866         udpt->utag_max = nudt;
 867         udpt->dma_bits = ndmabits;
 868
 869         while (nudt--) {
 870                 memset(udt, 0, sizeof(*udt));
 871                 udt->tag_parent = udpt;
 872                 udt++;
 873         }
 874 }
 875
 876 /*------------------------------------------------------------------------*
 877  *      usb_bus_tag_unsetup - factored out code
 878  *------------------------------------------------------------------------*/
 879 void
 880 usb_dma_tag_unsetup(struct usb_dma_parent_tag *udpt)
 881 {
 882         struct usb_dma_tag *udt;
 883         uint8_t nudt;
 884
 885         udt = udpt->utag_first;
 886         nudt = udpt->utag_max;
 887
 888         while (nudt--) {
 889
 890                 if (udt->align) {
 891                         /* destroy the USB DMA tag */
 892                         usb_dma_tag_destroy(udt);
 893                         udt->align = 0;
 894                 }
 895                 udt++;
 896         }
 897
 898         if (udpt->utag_max) {
 899                 /* destroy the condition variable */
 900                 cv_destroy(udpt->cv);
 901         }
 902 }
 903
 904 /*------------------------------------------------------------------------*
 905  *      usb_bdma_work_loop
 906  *
 907  * This function handles loading of virtual buffers into DMA and is
 908  * only called when "dma_refcount" is zero.
 909  *------------------------------------------------------------------------*/
 910 void
 911 usb_bdma_work_loop(struct usb_xfer_queue *pq)
 912 {
 913         struct usb_xfer_root *info;
 914         struct usb_xfer *xfer;
 915         usb_frcount_t nframes;
 916
 917         xfer = pq->curr;
 918         info = xfer->xroot;
 919
 920         mtx_assert(info->xfer_mtx, MA_OWNED);
 921
 922         if (xfer->error) {
 923                 /* some error happened */
 924                 USB_BUS_LOCK(info->bus);
 925                 usbd_transfer_done(xfer, 0);
 926                 USB_BUS_UNLOCK(info->bus);
 927                 return;
 928         }
 929         if (!xfer->flags_int.bdma_setup) {
 930                 struct usb_page *pg;
 931                 usb_frlength_t frlength_0;
 932                 uint8_t isread;
 933
 934                 xfer->flags_int.bdma_setup = 1;
 935
 936                 /* reset BUS-DMA load state */
 937
 938                 info->dma_error = 0;
 939
 940                 if (xfer->flags_int.isochronous_xfr) {
 941                         /* only one frame buffer */
 942                         nframes = 1;
 943                         frlength_0 = xfer->sumlen;
 944                 } else {
 945                         /* can be multiple frame buffers */
 946                         nframes = xfer->nframes;
 947                         frlength_0 = xfer->frlengths[0];
 948                 }
 949
 950                 /*
 951                  * Set DMA direction first. This is needed to
 952                  * select the correct cache invalidate and cache
 953                  * flush operations.
 954                  */
 955                 isread = USB_GET_DATA_ISREAD(xfer);
 956                 pg = xfer->dma_page_ptr;
 957
 958                 if (xfer->flags_int.control_xfr &&
 959                     xfer->flags_int.control_hdr) {
 960                         /* special case */
 961                         if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) {
 962                                 /* The device controller writes to memory */
 963                                 xfer->frbuffers[0].isread = 1;
 964                         } else {
 965                                 /* The host controller reads from memory */
 966                                 xfer->frbuffers[0].isread = 0;
 967                         }
 968                 } else {
 969                         /* default case */
 970                         xfer->frbuffers[0].isread = isread;
 971                 }
 972
 973                 /*
 974                  * Setup the "page_start" pointer which points to an array of
 975                  * USB pages where information about the physical address of a
 976                  * page will be stored. Also initialise the "isread" field of
 977                  * the USB page caches.
 978                  */
 979                 xfer->frbuffers[0].page_start = pg;
 980
 981                 info->dma_nframes = nframes;
 982                 info->dma_currframe = 0;
 983                 info->dma_frlength_0 = frlength_0;
 984
 985                 pg += (frlength_0 / USB_PAGE_SIZE);
 986                 pg += 2;
 987
 988                 while (--nframes > 0) {
 989                         xfer->frbuffers[nframes].isread = isread;
 990                         xfer->frbuffers[nframes].page_start = pg;
 991
 992                         pg += (xfer->frlengths[nframes] / USB_PAGE_SIZE);
 993                         pg += 2;
 994                 }
 995
 996         }
 997         if (info->dma_error) {
 998                 USB_BUS_LOCK(info->bus);
 999                 usbd_transfer_done(xfer, USB_ERR_DMA_LOAD_FAILED);
1000                 USB_BUS_UNLOCK(info->bus);
1001                 return;
1002         }
1003         if (info->dma_currframe != info->dma_nframes) {
1004
1005                 if (info->dma_currframe == 0) {
1006                         /* special case */
1007                         usb_pc_load_mem(xfer->frbuffers,
1008                             info->dma_frlength_0, 0);
1009                 } else {
1010                         /* default case */
1011                         nframes = info->dma_currframe;
1012                         usb_pc_load_mem(xfer->frbuffers + nframes,
1013                             xfer->frlengths[nframes], 0);
1014                 }
1015
1016                 /* advance frame index */
1017                 info->dma_currframe++;
1018
1019                 return;
1020         }
1021         /* go ahead */
1022         usb_bdma_pre_sync(xfer);
1023
1024         /* start loading next USB transfer, if any */
1025         usb_command_wrapper(pq, NULL);
1026
1027         /* finally start the hardware */
1028         usbd_pipe_enter(xfer);
1029 }
1030
1031 /*------------------------------------------------------------------------*
1032  *      usb_bdma_done_event
1033  *
1034  * This function is called when the BUS-DMA has loaded virtual memory
1035  * into DMA, if any.
1036  *------------------------------------------------------------------------*/
1037 void
1038 usb_bdma_done_event(struct usb_dma_parent_tag *udpt)
1039 {
1040         struct usb_xfer_root *info;
1041
1042         info = USB_DMATAG_TO_XROOT(udpt);
1043
1044         mtx_assert(info->xfer_mtx, MA_OWNED);
1045
1046         /* copy error */
1047         info->dma_error = udpt->dma_error;
1048
1049         /* enter workloop again */
1050         usb_command_wrapper(&info->dma_q,
1051             info->dma_q.curr);
1052 }
1053
1054 /*------------------------------------------------------------------------*
1055  *      usb_bdma_pre_sync
1056  *
1057  * This function handles DMA synchronisation that must be done before
1058  * an USB transfer is started.
1059  *------------------------------------------------------------------------*/
1060 void
1061 usb_bdma_pre_sync(struct usb_xfer *xfer)
1062 {
1063         struct usb_page_cache *pc;
1064         usb_frcount_t nframes;
1065
1066         if (xfer->flags_int.isochronous_xfr) {
1067                 /* only one frame buffer */
1068                 nframes = 1;
1069         } else {
1070                 /* can be multiple frame buffers */
1071                 nframes = xfer->nframes;
1072         }
1073
1074         pc = xfer->frbuffers;
1075
1076         while (nframes--) {
1077
1078                 if (pc->isread) {
1079                         usb_pc_cpu_invalidate(pc);
1080                 } else {
1081                         usb_pc_cpu_flush(pc);
1082                 }
1083                 pc++;
1084         }
1085 }
1086
1087 /*------------------------------------------------------------------------*
1088  *      usb_bdma_post_sync
1089  *
1090  * This function handles DMA synchronisation that must be done after
1091  * an USB transfer is complete.
1092  *------------------------------------------------------------------------*/
1093 void
1094 usb_bdma_post_sync(struct usb_xfer *xfer)
1095 {
1096         struct usb_page_cache *pc;
1097         usb_frcount_t nframes;
1098
1099         if (xfer->flags_int.isochronous_xfr) {
1100                 /* only one frame buffer */
1101                 nframes = 1;
1102         } else {
1103                 /* can be multiple frame buffers */
1104                 nframes = xfer->nframes;
1105         }
1106
1107         pc = xfer->frbuffers;
1108
1109         while (nframes--) {
1110                 if (pc->isread) {
1111                         usb_pc_cpu_invalidate(pc);
1112                 }
1113                 pc++;
1114         }
1115 }
1116
1117 #endif