sys/dev/usb/usb_transfer.c

   1 /* $FreeBSD$ */
   2 /*-
   3  * Copyright (c) 2008 Hans Petter Selasky. All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  */
  26
  27 #include <sys/stdint.h>
  28 #include <sys/stddef.h>
  29 #include <sys/param.h>
  30 #include <sys/queue.h>
  31 #include <sys/types.h>
  32 #include <sys/systm.h>
  33 #include <sys/kernel.h>
  34 #include <sys/bus.h>
  35 #include <sys/module.h>
  36 #include <sys/lock.h>
  37 #include <sys/mutex.h>
  38 #include <sys/condvar.h>
  39 #include <sys/sysctl.h>
  40 #include <sys/sx.h>
  41 #include <sys/unistd.h>
  42 #include <sys/callout.h>
  43 #include <sys/malloc.h>
  44 #include <sys/priv.h>
  45 #include <sys/proc.h>
  46
  47 #include <dev/usb/usb.h>
  48 #include <dev/usb/usbdi.h>
  49 #include <dev/usb/usbdi_util.h>
  50
  51 #define USB_DEBUG_VAR usb_debug
  52
  53 #include <dev/usb/usb_core.h>
  54 #include <dev/usb/usb_busdma.h>
  55 #include <dev/usb/usb_process.h>
  56 #include <dev/usb/usb_transfer.h>
  57 #include <dev/usb/usb_device.h>
  58 #include <dev/usb/usb_debug.h>
  59 #include <dev/usb/usb_util.h>
  60
  61 #include <dev/usb/usb_controller.h>
  62 #include <dev/usb/usb_bus.h>
  63 #include <dev/usb/usb_pf.h>
  64
  65 struct usb_std_packet_size {
  66         struct {
  67                 uint16_t min;           /* inclusive */
  68                 uint16_t max;           /* inclusive */
  69         }       range;
  70
  71         uint16_t fixed[4];
  72 };
  73
  74 static usb_callback_t usb_request_callback;
  75
  76 static const struct usb_config usb_control_ep_cfg[USB_CTRL_XFER_MAX] = {
  77
  78         /* This transfer is used for generic control endpoint transfers */
  79
  80         [0] = {
  81                 .type = UE_CONTROL,
  82                 .endpoint = 0x00,       /* Control endpoint */
  83                 .direction = UE_DIR_ANY,
  84                 .bufsize = USB_EP0_BUFSIZE,     /* bytes */
  85                 .flags = {.proxy_buffer = 1,},
  86                 .callback = &usb_request_callback,
  87                 .usb_mode = USB_MODE_DUAL,      /* both modes */
  88         },
  89
  90         /* This transfer is used for generic clear stall only */
  91
  92         [1] = {
  93                 .type = UE_CONTROL,
  94                 .endpoint = 0x00,       /* Control pipe */
  95                 .direction = UE_DIR_ANY,
  96                 .bufsize = sizeof(struct usb_device_request),
  97                 .callback = &usb_do_clear_stall_callback,
  98                 .timeout = 1000,        /* 1 second */
  99                 .interval = 50, /* 50ms */
 100                 .usb_mode = USB_MODE_HOST,
 101         },
 102 };
 103
 104 /* function prototypes */
 105
 106 static void     usbd_update_max_frame_size(struct usb_xfer *);
 107 static void     usbd_transfer_unsetup_sub(struct usb_xfer_root *, uint8_t);
 108 static void     usbd_control_transfer_init(struct usb_xfer *);
 109 static int      usbd_setup_ctrl_transfer(struct usb_xfer *);
 110 static void     usb_callback_proc(struct usb_proc_msg *);
 111 static void     usbd_callback_ss_done_defer(struct usb_xfer *);
 112 static void     usbd_callback_wrapper(struct usb_xfer_queue *);
 113 static void     usbd_transfer_start_cb(void *);
 114 static uint8_t  usbd_callback_wrapper_sub(struct usb_xfer *);
 115 static void     usbd_get_std_packet_size(struct usb_std_packet_size *ptr,
 116                     uint8_t type, enum usb_dev_speed speed);
 117
 118 /*------------------------------------------------------------------------*
 119  *      usb_request_callback
 120  *------------------------------------------------------------------------*/
 121 static void
 122 usb_request_callback(struct usb_xfer *xfer, usb_error_t error)
 123 {
 124         if (xfer->flags_int.usb_mode == USB_MODE_DEVICE)
 125                 usb_handle_request_callback(xfer, error);
 126         else
 127                 usbd_do_request_callback(xfer, error);
 128 }
 129
 130 /*------------------------------------------------------------------------*
 131  *      usbd_update_max_frame_size
 132  *
 133  * This function updates the maximum frame size, hence high speed USB
 134  * can transfer multiple consecutive packets.
 135  *------------------------------------------------------------------------*/
 136 static void
 137 usbd_update_max_frame_size(struct usb_xfer *xfer)
 138 {
 139         /* compute maximum frame size */
 140         /* this computation should not overflow 16-bit */
 141         /* max = 15 * 1024 */
 142
 143         xfer->max_frame_size = xfer->max_packet_size * xfer->max_packet_count;
 144 }
 145
 146 /*------------------------------------------------------------------------*
 147  *      usbd_get_dma_delay
 148  *
 149  * The following function is called when we need to
 150  * synchronize with DMA hardware.
 151  *
 152  * Returns:
 153  *    0: no DMA delay required
 154  * Else: milliseconds of DMA delay
 155  *------------------------------------------------------------------------*/
 156 usb_timeout_t
 157 usbd_get_dma_delay(struct usb_device *udev)
 158 {
 159         struct usb_bus_methods *mtod;
 160         uint32_t temp;
 161
 162         mtod = udev->bus->methods;
 163         temp = 0;
 164
 165         if (mtod->get_dma_delay) {
 166                 (mtod->get_dma_delay) (udev, &temp);
 167                 /*
 168                  * Round up and convert to milliseconds. Note that we use
 169                  * 1024 milliseconds per second. to save a division.
 170                  */
 171                 temp += 0x3FF;
 172                 temp /= 0x400;
 173         }
 174         return (temp);
 175 }
 176
 177 /*------------------------------------------------------------------------*
 178  *      usbd_transfer_setup_sub_malloc
 179  *
 180  * This function will allocate one or more DMA'able memory chunks
 181  * according to "size", "align" and "count" arguments. "ppc" is
 182  * pointed to a linear array of USB page caches afterwards.
 183  *
 184  * Returns:
 185  *    0: Success
 186  * Else: Failure
 187  *------------------------------------------------------------------------*/
 188 #if USB_HAVE_BUSDMA
 189 uint8_t
 190 usbd_transfer_setup_sub_malloc(struct usb_setup_params *parm,
 191     struct usb_page_cache **ppc, usb_size_t size, usb_size_t align,
 192     usb_size_t count)
 193 {
 194         struct usb_page_cache *pc;
 195         struct usb_page *pg;
 196         void *buf;
 197         usb_size_t n_dma_pc;
 198         usb_size_t n_obj;
 199         usb_size_t x;
 200         usb_size_t y;
 201         usb_size_t r;
 202         usb_size_t z;
 203
 204         USB_ASSERT(align > 1, ("Invalid alignment, 0x%08x\n",
 205             align));
 206         USB_ASSERT(size > 0, ("Invalid size = 0\n"));
 207
 208         if (count == 0) {
 209                 return (0);             /* nothing to allocate */
 210         }
 211         /*
 212          * Make sure that the size is aligned properly.
 213          */
 214         size = -((-size) & (-align));
 215
 216         /*
 217          * Try multi-allocation chunks to reduce the number of DMA
 218          * allocations, hence DMA allocations are slow.
 219          */
 220         if (size >= USB_PAGE_SIZE) {
 221                 n_dma_pc = count;
 222                 n_obj = 1;
 223         } else {
 224                 /* compute number of objects per page */
 225                 n_obj = (USB_PAGE_SIZE / size);
 226                 /*
 227                  * Compute number of DMA chunks, rounded up
 228                  * to nearest one:
 229                  */
 230                 n_dma_pc = ((count + n_obj - 1) / n_obj);
 231         }
 232
 233         if (parm->buf == NULL) {
 234                 /* for the future */
 235                 parm->dma_page_ptr += n_dma_pc;
 236                 parm->dma_page_cache_ptr += n_dma_pc;
 237                 parm->dma_page_ptr += count;
 238                 parm->xfer_page_cache_ptr += count;
 239                 return (0);
 240         }
 241         for (x = 0; x != n_dma_pc; x++) {
 242                 /* need to initialize the page cache */
 243                 parm->dma_page_cache_ptr[x].tag_parent =
 244                     &parm->curr_xfer->xroot->dma_parent_tag;
 245         }
 246         for (x = 0; x != count; x++) {
 247                 /* need to initialize the page cache */
 248                 parm->xfer_page_cache_ptr[x].tag_parent =
 249                     &parm->curr_xfer->xroot->dma_parent_tag;
 250         }
 251
 252         if (ppc) {
 253                 *ppc = parm->xfer_page_cache_ptr;
 254         }
 255         r = count;                      /* set remainder count */
 256         z = n_obj * size;               /* set allocation size */
 257         pc = parm->xfer_page_cache_ptr;
 258         pg = parm->dma_page_ptr;
 259
 260         for (x = 0; x != n_dma_pc; x++) {
 261
 262                 if (r < n_obj) {
 263                         /* compute last remainder */
 264                         z = r * size;
 265                         n_obj = r;
 266                 }
 267                 if (usb_pc_alloc_mem(parm->dma_page_cache_ptr,
 268                     pg, z, align)) {
 269                         return (1);     /* failure */
 270                 }
 271                 /* Set beginning of current buffer */
 272                 buf = parm->dma_page_cache_ptr->buffer;
 273                 /* Make room for one DMA page cache and one page */
 274                 parm->dma_page_cache_ptr++;
 275                 pg++;
 276
 277                 for (y = 0; (y != n_obj); y++, r--, pc++, pg++) {
 278
 279                         /* Load sub-chunk into DMA */
 280                         if (usb_pc_dmamap_create(pc, size)) {
 281                                 return (1);     /* failure */
 282                         }
 283                         pc->buffer = USB_ADD_BYTES(buf, y * size);
 284                         pc->page_start = pg;
 285
 286                         mtx_lock(pc->tag_parent->mtx);
 287                         if (usb_pc_load_mem(pc, size, 1 /* synchronous */ )) {
 288                                 mtx_unlock(pc->tag_parent->mtx);
 289                                 return (1);     /* failure */
 290                         }
 291                         mtx_unlock(pc->tag_parent->mtx);
 292                 }
 293         }
 294
 295         parm->xfer_page_cache_ptr = pc;
 296         parm->dma_page_ptr = pg;
 297         return (0);
 298 }
 299 #endif
 300
 301 /*------------------------------------------------------------------------*
 302  *      usbd_transfer_setup_sub - transfer setup subroutine
 303  *
 304  * This function must be called from the "xfer_setup" callback of the
 305  * USB Host or Device controller driver when setting up an USB
 306  * transfer. This function will setup correct packet sizes, buffer
 307  * sizes, flags and more, that are stored in the "usb_xfer"
 308  * structure.
 309  *------------------------------------------------------------------------*/
 310 void
 311 usbd_transfer_setup_sub(struct usb_setup_params *parm)
 312 {
 313         enum {
 314                 REQ_SIZE = 8,
 315                 MIN_PKT = 8,
 316         };
 317         struct usb_xfer *xfer = parm->curr_xfer;
 318         const struct usb_config *setup = parm->curr_setup;
 319         struct usb_endpoint_ss_comp_descriptor *ecomp;
 320         struct usb_endpoint_descriptor *edesc;
 321         struct usb_std_packet_size std_size;
 322         usb_frcount_t n_frlengths;
 323         usb_frcount_t n_frbuffers;
 324         usb_frcount_t x;
 325         uint8_t type;
 326         uint8_t zmps;
 327
 328         /*
 329          * Sanity check. The following parameters must be initialized before
 330          * calling this function.
 331          */
 332         if ((parm->hc_max_packet_size == 0) ||
 333             (parm->hc_max_packet_count == 0) ||
 334             (parm->hc_max_frame_size == 0)) {
 335                 parm->err = USB_ERR_INVAL;
 336                 goto done;
 337         }
 338         edesc = xfer->endpoint->edesc;
 339         ecomp = xfer->endpoint->ecomp;
 340
 341         type = (edesc->bmAttributes & UE_XFERTYPE);
 342
 343         xfer->flags = setup->flags;
 344         xfer->nframes = setup->frames;
 345         xfer->timeout = setup->timeout;
 346         xfer->callback = setup->callback;
 347         xfer->interval = setup->interval;
 348         xfer->endpointno = edesc->bEndpointAddress;
 349         xfer->max_packet_size = UGETW(edesc->wMaxPacketSize);
 350         xfer->max_packet_count = 1;
 351         /* make a shadow copy: */
 352         xfer->flags_int.usb_mode = parm->udev->flags.usb_mode;
 353
 354         parm->bufsize = setup->bufsize;
 355
 356         switch (parm->speed) {
 357         case USB_SPEED_HIGH:
 358                 switch (type) {
 359                 case UE_ISOCHRONOUS:
 360                 case UE_INTERRUPT:
 361                         xfer->max_packet_count += (xfer->max_packet_size >> 11) & 3;
 362
 363                         /* check for invalid max packet count */
 364                         if (xfer->max_packet_count > 3)
 365                                 xfer->max_packet_count = 3;
 366                         break;
 367                 default:
 368                         break;
 369                 }
 370                 xfer->max_packet_size &= 0x7FF;
 371                 break;
 372         case USB_SPEED_SUPER:
 373                 xfer->max_packet_count += (xfer->max_packet_size >> 11) & 3;
 374
 375                 if (ecomp != NULL)
 376                         xfer->max_packet_count += ecomp->bMaxBurst;
 377
 378                 if ((xfer->max_packet_count == 0) ||
 379                     (xfer->max_packet_count > 16))
 380                         xfer->max_packet_count = 16;
 381
 382                 switch (type) {
 383                 case UE_CONTROL:
 384                         xfer->max_packet_count = 1;
 385                         break;
 386                 case UE_ISOCHRONOUS:
 387                         if (ecomp != NULL) {
 388                                 uint8_t mult;
 389
 390                                 mult = (ecomp->bmAttributes & 3) + 1;
 391                                 if (mult > 3)
 392                                         mult = 3;
 393
 394                                 xfer->max_packet_count *= mult;
 395                         }
 396                         break;
 397                 default:
 398                         break;
 399                 }
 400                 xfer->max_packet_size &= 0x7FF;
 401                 break;
 402         default:
 403                 break;
 404         }
 405         /* range check "max_packet_count" */
 406
 407         if (xfer->max_packet_count > parm->hc_max_packet_count) {
 408                 xfer->max_packet_count = parm->hc_max_packet_count;
 409         }
 410         /* filter "wMaxPacketSize" according to HC capabilities */
 411
 412         if ((xfer->max_packet_size > parm->hc_max_packet_size) ||
 413             (xfer->max_packet_size == 0)) {
 414                 xfer->max_packet_size = parm->hc_max_packet_size;
 415         }
 416         /* filter "wMaxPacketSize" according to standard sizes */
 417
 418         usbd_get_std_packet_size(&std_size, type, parm->speed);
 419
 420         if (std_size.range.min || std_size.range.max) {
 421
 422                 if (xfer->max_packet_size < std_size.range.min) {
 423                         xfer->max_packet_size = std_size.range.min;
 424                 }
 425                 if (xfer->max_packet_size > std_size.range.max) {
 426                         xfer->max_packet_size = std_size.range.max;
 427                 }
 428         } else {
 429
 430                 if (xfer->max_packet_size >= std_size.fixed[3]) {
 431                         xfer->max_packet_size = std_size.fixed[3];
 432                 } else if (xfer->max_packet_size >= std_size.fixed[2]) {
 433                         xfer->max_packet_size = std_size.fixed[2];
 434                 } else if (xfer->max_packet_size >= std_size.fixed[1]) {
 435                         xfer->max_packet_size = std_size.fixed[1];
 436                 } else {
 437                         /* only one possibility left */
 438                         xfer->max_packet_size = std_size.fixed[0];
 439                 }
 440         }
 441
 442         /* compute "max_frame_size" */
 443
 444         usbd_update_max_frame_size(xfer);
 445
 446         /* check interrupt interval and transfer pre-delay */
 447
 448         if (type == UE_ISOCHRONOUS) {
 449
 450                 uint16_t frame_limit;
 451
 452                 xfer->interval = 0;     /* not used, must be zero */
 453                 xfer->flags_int.isochronous_xfr = 1;    /* set flag */
 454
 455                 if (xfer->timeout == 0) {
 456                         /*
 457                          * set a default timeout in
 458                          * case something goes wrong!
 459                          */
 460                         xfer->timeout = 1000 / 4;
 461                 }
 462                 switch (parm->speed) {
 463                 case USB_SPEED_LOW:
 464                 case USB_SPEED_FULL:
 465                         frame_limit = USB_MAX_FS_ISOC_FRAMES_PER_XFER;
 466                         xfer->fps_shift = 0;
 467                         break;
 468                 default:
 469                         frame_limit = USB_MAX_HS_ISOC_FRAMES_PER_XFER;
 470                         xfer->fps_shift = edesc->bInterval;
 471                         if (xfer->fps_shift > 0)
 472                                 xfer->fps_shift--;
 473                         if (xfer->fps_shift > 3)
 474                                 xfer->fps_shift = 3;
 475                         if (xfer->flags.pre_scale_frames != 0)
 476                                 xfer->nframes <<= (3 - xfer->fps_shift);
 477                         break;
 478                 }
 479
 480                 if (xfer->nframes > frame_limit) {
 481                         /*
 482                          * this is not going to work
 483                          * cross hardware
 484                          */
 485                         parm->err = USB_ERR_INVAL;
 486                         goto done;
 487                 }
 488                 if (xfer->nframes == 0) {
 489                         /*
 490                          * this is not a valid value
 491                          */
 492                         parm->err = USB_ERR_ZERO_NFRAMES;
 493                         goto done;
 494                 }
 495         } else {
 496
 497                 /*
 498                  * If a value is specified use that else check the
 499                  * endpoint descriptor!
 500                  */
 501                 if (type == UE_INTERRUPT) {
 502
 503                         uint32_t temp;
 504
 505                         if (xfer->interval == 0) {
 506
 507                                 xfer->interval = edesc->bInterval;
 508
 509                                 switch (parm->speed) {
 510                                 case USB_SPEED_LOW:
 511                                 case USB_SPEED_FULL:
 512                                         break;
 513                                 default:
 514                                         /* 125us -> 1ms */
 515                                         if (xfer->interval < 4)
 516                                                 xfer->interval = 1;
 517                                         else if (xfer->interval > 16)
 518                                                 xfer->interval = (1 << (16 - 4));
 519                                         else
 520                                                 xfer->interval =
 521                                                     (1 << (xfer->interval - 4));
 522                                         break;
 523                                 }
 524                         }
 525
 526                         if (xfer->interval == 0) {
 527                                 /*
 528                                  * One millisecond is the smallest
 529                                  * interval we support:
 530                                  */
 531                                 xfer->interval = 1;
 532                         }
 533
 534                         xfer->fps_shift = 0;
 535                         temp = 1;
 536
 537                         while ((temp != 0) && (temp < xfer->interval)) {
 538                                 xfer->fps_shift++;
 539                                 temp *= 2;
 540                         }
 541
 542                         switch (parm->speed) {
 543                         case USB_SPEED_LOW:
 544                         case USB_SPEED_FULL:
 545                                 break;
 546                         default:
 547                                 xfer->fps_shift += 3;
 548                                 break;
 549                         }
 550                 }
 551         }
 552
 553         /*
 554          * NOTE: we do not allow "max_packet_size" or "max_frame_size"
 555          * to be equal to zero when setting up USB transfers, hence
 556          * this leads to alot of extra code in the USB kernel.
 557          */
 558
 559         if ((xfer->max_frame_size == 0) ||
 560             (xfer->max_packet_size == 0)) {
 561
 562                 zmps = 1;
 563
 564                 if ((parm->bufsize <= MIN_PKT) &&
 565                     (type != UE_CONTROL) &&
 566                     (type != UE_BULK)) {
 567
 568                         /* workaround */
 569                         xfer->max_packet_size = MIN_PKT;
 570                         xfer->max_packet_count = 1;
 571                         parm->bufsize = 0;      /* automatic setup length */
 572                         usbd_update_max_frame_size(xfer);
 573
 574                 } else {
 575                         parm->err = USB_ERR_ZERO_MAXP;
 576                         goto done;
 577                 }
 578
 579         } else {
 580                 zmps = 0;
 581         }
 582
 583         /*
 584          * check if we should setup a default
 585          * length:
 586          */
 587
 588         if (parm->bufsize == 0) {
 589
 590                 parm->bufsize = xfer->max_frame_size;
 591
 592                 if (type == UE_ISOCHRONOUS) {
 593                         parm->bufsize *= xfer->nframes;
 594                 }
 595         }
 596         /*
 597          * check if we are about to setup a proxy
 598          * type of buffer:
 599          */
 600
 601         if (xfer->flags.proxy_buffer) {
 602
 603                 /* round bufsize up */
 604
 605                 parm->bufsize += (xfer->max_frame_size - 1);
 606
 607                 if (parm->bufsize < xfer->max_frame_size) {
 608                         /* length wrapped around */
 609                         parm->err = USB_ERR_INVAL;
 610                         goto done;
 611                 }
 612                 /* subtract remainder */
 613
 614                 parm->bufsize -= (parm->bufsize % xfer->max_frame_size);
 615
 616                 /* add length of USB device request structure, if any */
 617
 618                 if (type == UE_CONTROL) {
 619                         parm->bufsize += REQ_SIZE;      /* SETUP message */
 620                 }
 621         }
 622         xfer->max_data_length = parm->bufsize;
 623
 624         /* Setup "n_frlengths" and "n_frbuffers" */
 625
 626         if (type == UE_ISOCHRONOUS) {
 627                 n_frlengths = xfer->nframes;
 628                 n_frbuffers = 1;
 629         } else {
 630
 631                 if (type == UE_CONTROL) {
 632                         xfer->flags_int.control_xfr = 1;
 633                         if (xfer->nframes == 0) {
 634                                 if (parm->bufsize <= REQ_SIZE) {
 635                                         /*
 636                                          * there will never be any data
 637                                          * stage
 638                                          */
 639                                         xfer->nframes = 1;
 640                                 } else {
 641                                         xfer->nframes = 2;
 642                                 }
 643                         }
 644                 } else {
 645                         if (xfer->nframes == 0) {
 646                                 xfer->nframes = 1;
 647                         }
 648                 }
 649
 650                 n_frlengths = xfer->nframes;
 651                 n_frbuffers = xfer->nframes;
 652         }
 653
 654         /*
 655          * check if we have room for the
 656          * USB device request structure:
 657          */
 658
 659         if (type == UE_CONTROL) {
 660
 661                 if (xfer->max_data_length < REQ_SIZE) {
 662                         /* length wrapped around or too small bufsize */
 663                         parm->err = USB_ERR_INVAL;
 664                         goto done;
 665                 }
 666                 xfer->max_data_length -= REQ_SIZE;
 667         }
 668         /*
 669          * Setup "frlengths" and shadow "frlengths" for keeping the
 670          * initial frame lengths when a USB transfer is complete. This
 671          * information is useful when computing isochronous offsets.
 672          */
 673         xfer->frlengths = parm->xfer_length_ptr;
 674         parm->xfer_length_ptr += 2 * n_frlengths;
 675
 676         /* setup "frbuffers" */
 677         xfer->frbuffers = parm->xfer_page_cache_ptr;
 678         parm->xfer_page_cache_ptr += n_frbuffers;
 679
 680         /* initialize max frame count */
 681         xfer->max_frame_count = xfer->nframes;
 682
 683         /*
 684          * check if we need to setup
 685          * a local buffer:
 686          */
 687
 688         if (!xfer->flags.ext_buffer) {
 689
 690                 /* align data */
 691                 parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
 692
 693                 if (parm->buf) {
 694
 695                         xfer->local_buffer =
 696                             USB_ADD_BYTES(parm->buf, parm->size[0]);
 697
 698                         usbd_xfer_set_frame_offset(xfer, 0, 0);
 699
 700                         if ((type == UE_CONTROL) && (n_frbuffers > 1)) {
 701                                 usbd_xfer_set_frame_offset(xfer, REQ_SIZE, 1);
 702                         }
 703                 }
 704                 parm->size[0] += parm->bufsize;
 705
 706                 /* align data again */
 707                 parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
 708         }
 709         /*
 710          * Compute maximum buffer size
 711          */
 712
 713         if (parm->bufsize_max < parm->bufsize) {
 714                 parm->bufsize_max = parm->bufsize;
 715         }
 716 #if USB_HAVE_BUSDMA
 717         if (xfer->flags_int.bdma_enable) {
 718                 /*
 719                  * Setup "dma_page_ptr".
 720                  *
 721                  * Proof for formula below:
 722                  *
 723                  * Assume there are three USB frames having length "a", "b" and
 724                  * "c". These USB frames will at maximum need "z"
 725                  * "usb_page" structures. "z" is given by:
 726                  *
 727                  * z = ((a / USB_PAGE_SIZE) + 2) + ((b / USB_PAGE_SIZE) + 2) +
 728                  * ((c / USB_PAGE_SIZE) + 2);
 729                  *
 730                  * Constraining "a", "b" and "c" like this:
 731                  *
 732                  * (a + b + c) <= parm->bufsize
 733                  *
 734                  * We know that:
 735                  *
 736                  * z <= ((parm->bufsize / USB_PAGE_SIZE) + (3*2));
 737                  *
 738                  * Here is the general formula:
 739                  */
 740                 xfer->dma_page_ptr = parm->dma_page_ptr;
 741                 parm->dma_page_ptr += (2 * n_frbuffers);
 742                 parm->dma_page_ptr += (parm->bufsize / USB_PAGE_SIZE);
 743         }
 744 #endif
 745         if (zmps) {
 746                 /* correct maximum data length */
 747                 xfer->max_data_length = 0;
 748         }
 749         /* subtract USB frame remainder from "hc_max_frame_size" */
 750
 751         xfer->max_hc_frame_size =
 752             (parm->hc_max_frame_size -
 753             (parm->hc_max_frame_size % xfer->max_frame_size));
 754
 755         if (xfer->max_hc_frame_size == 0) {
 756                 parm->err = USB_ERR_INVAL;
 757                 goto done;
 758         }
 759
 760         /* initialize frame buffers */
 761
 762         if (parm->buf) {
 763                 for (x = 0; x != n_frbuffers; x++) {
 764                         xfer->frbuffers[x].tag_parent =
 765                             &xfer->xroot->dma_parent_tag;
 766 #if USB_HAVE_BUSDMA
 767                         if (xfer->flags_int.bdma_enable &&
 768                             (parm->bufsize_max > 0)) {
 769
 770                                 if (usb_pc_dmamap_create(
 771                                     xfer->frbuffers + x,
 772                                     parm->bufsize_max)) {
 773                                         parm->err = USB_ERR_NOMEM;
 774                                         goto done;
 775                                 }
 776                         }
 777 #endif
 778                 }
 779         }
 780 done:
 781         if (parm->err) {
 782                 /*
 783                  * Set some dummy values so that we avoid division by zero:
 784                  */
 785                 xfer->max_hc_frame_size = 1;
 786                 xfer->max_frame_size = 1;
 787                 xfer->max_packet_size = 1;
 788                 xfer->max_data_length = 0;
 789                 xfer->nframes = 0;
 790                 xfer->max_frame_count = 0;
 791         }
 792 }
 793
 794 /*------------------------------------------------------------------------*
 795  *      usbd_transfer_setup - setup an array of USB transfers
 796  *
 797  * NOTE: You must always call "usbd_transfer_unsetup" after calling
 798  * "usbd_transfer_setup" if success was returned.
 799  *
 800  * The idea is that the USB device driver should pre-allocate all its
 801  * transfers by one call to this function.
 802  *
 803  * Return values:
 804  *    0: Success
 805  * Else: Failure
 806  *------------------------------------------------------------------------*/
 807 usb_error_t
 808 usbd_transfer_setup(struct usb_device *udev,
 809     const uint8_t *ifaces, struct usb_xfer **ppxfer,
 810     const struct usb_config *setup_start, uint16_t n_setup,
 811     void *priv_sc, struct mtx *xfer_mtx)
 812 {
 813         const struct usb_config *setup_end = setup_start + n_setup;
 814         const struct usb_config *setup;
 815         struct usb_setup_params *parm;
 816         struct usb_endpoint *ep;
 817         struct usb_xfer_root *info;
 818         struct usb_xfer *xfer;
 819         void *buf = NULL;
 820         usb_error_t error = 0;
 821         uint16_t n;
 822         uint16_t refcount;
 823         uint8_t do_unlock;
 824
 825         WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
 826             "usbd_transfer_setup can sleep!");
 827
 828         /* do some checking first */
 829
 830         if (n_setup == 0) {
 831                 DPRINTFN(6, "setup array has zero length!\n");
 832                 return (USB_ERR_INVAL);
 833         }
 834         if (ifaces == 0) {
 835                 DPRINTFN(6, "ifaces array is NULL!\n");
 836                 return (USB_ERR_INVAL);
 837         }
 838         if (xfer_mtx == NULL) {
 839                 DPRINTFN(6, "using global lock\n");
 840                 xfer_mtx = &Giant;
 841         }
 842
 843         /* more sanity checks */
 844
 845         for (setup = setup_start, n = 0;
 846             setup != setup_end; setup++, n++) {
 847                 if (setup->bufsize == (usb_frlength_t)-1) {
 848                         error = USB_ERR_BAD_BUFSIZE;
 849                         DPRINTF("invalid bufsize\n");
 850                 }
 851                 if (setup->callback == NULL) {
 852                         error = USB_ERR_NO_CALLBACK;
 853                         DPRINTF("no callback\n");
 854                 }
 855                 ppxfer[n] = NULL;
 856         }
 857
 858         if (error)
 859                 return (error);
 860
 861         /* Protect scratch area */
 862         do_unlock = usbd_enum_lock(udev);
 863
 864         refcount = 0;
 865         info = NULL;
 866
 867         parm = &udev->scratch.xfer_setup[0].parm;
 868         memset(parm, 0, sizeof(*parm));
 869
 870         parm->udev = udev;
 871         parm->speed = usbd_get_speed(udev);
 872         parm->hc_max_packet_count = 1;
 873
 874         if (parm->speed >= USB_SPEED_MAX) {
 875                 parm->err = USB_ERR_INVAL;
 876                 goto done;
 877         }
 878         /* setup all transfers */
 879
 880         while (1) {
 881
 882                 if (buf) {
 883                         /*
 884                          * Initialize the "usb_xfer_root" structure,
 885                          * which is common for all our USB transfers.
 886                          */
 887                         info = USB_ADD_BYTES(buf, 0);
 888
 889                         info->memory_base = buf;
 890                         info->memory_size = parm->size[0];
 891
 892 #if USB_HAVE_BUSDMA
 893                         info->dma_page_cache_start = USB_ADD_BYTES(buf, parm->size[4]);
 894                         info->dma_page_cache_end = USB_ADD_BYTES(buf, parm->size[5]);
 895 #endif
 896                         info->xfer_page_cache_start = USB_ADD_BYTES(buf, parm->size[5]);
 897                         info->xfer_page_cache_end = USB_ADD_BYTES(buf, parm->size[2]);
 898
 899                         cv_init(&info->cv_drain, "WDRAIN");
 900
 901                         info->xfer_mtx = xfer_mtx;
 902 #if USB_HAVE_BUSDMA
 903                         usb_dma_tag_setup(&info->dma_parent_tag,
 904                             parm->dma_tag_p, udev->bus->dma_parent_tag[0].tag,
 905                             xfer_mtx, &usb_bdma_done_event, 32, parm->dma_tag_max);
 906 #endif
 907
 908                         info->bus = udev->bus;
 909                         info->udev = udev;
 910
 911                         TAILQ_INIT(&info->done_q.head);
 912                         info->done_q.command = &usbd_callback_wrapper;
 913 #if USB_HAVE_BUSDMA
 914                         TAILQ_INIT(&info->dma_q.head);
 915                         info->dma_q.command = &usb_bdma_work_loop;
 916 #endif
 917                         info->done_m[0].hdr.pm_callback = &usb_callback_proc;
 918                         info->done_m[0].xroot = info;
 919                         info->done_m[1].hdr.pm_callback = &usb_callback_proc;
 920                         info->done_m[1].xroot = info;
 921
 922                         /*
 923                          * In device side mode control endpoint
 924                          * requests need to run from a separate
 925                          * context, else there is a chance of
 926                          * deadlock!
 927                          */
 928                         if (setup_start == usb_control_ep_cfg)
 929                                 info->done_p =
 930                                     &udev->bus->control_xfer_proc;
 931                         else if (xfer_mtx == &Giant)
 932                                 info->done_p =
 933                                     &udev->bus->giant_callback_proc;
 934                         else
 935                                 info->done_p =
 936                                     &udev->bus->non_giant_callback_proc;
 937                 }
 938                 /* reset sizes */
 939
 940                 parm->size[0] = 0;
 941                 parm->buf = buf;
 942                 parm->size[0] += sizeof(info[0]);
 943
 944                 for (setup = setup_start, n = 0;
 945                     setup != setup_end; setup++, n++) {
 946
 947                         /* skip USB transfers without callbacks: */
 948                         if (setup->callback == NULL) {
 949                                 continue;
 950                         }
 951                         /* see if there is a matching endpoint */
 952                         ep = usbd_get_endpoint(udev,
 953                             ifaces[setup->if_index], setup);
 954
 955                         if ((ep == NULL) || (ep->methods == NULL)) {
 956                                 if (setup->flags.no_pipe_ok)
 957                                         continue;
 958                                 if ((setup->usb_mode != USB_MODE_DUAL) &&
 959                                     (setup->usb_mode != udev->flags.usb_mode))
 960                                         continue;
 961                                 parm->err = USB_ERR_NO_PIPE;
 962                                 goto done;
 963                         }
 964
 965                         /* align data properly */
 966                         parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
 967
 968                         /* store current setup pointer */
 969                         parm->curr_setup = setup;
 970
 971                         if (buf) {
 972                                 /*
 973                                  * Common initialization of the
 974                                  * "usb_xfer" structure.
 975                                  */
 976                                 xfer = USB_ADD_BYTES(buf, parm->size[0]);
 977                                 xfer->address = udev->address;
 978                                 xfer->priv_sc = priv_sc;
 979                                 xfer->xroot = info;
 980
 981                                 usb_callout_init_mtx(&xfer->timeout_handle,
 982                                     &udev->bus->bus_mtx, 0);
 983                         } else {
 984                                 /*
 985                                  * Setup a dummy xfer, hence we are
 986                                  * writing to the "usb_xfer"
 987                                  * structure pointed to by "xfer"
 988                                  * before we have allocated any
 989                                  * memory:
 990                                  */
 991                                 xfer = &udev->scratch.xfer_setup[0].dummy;
 992                                 memset(xfer, 0, sizeof(*xfer));
 993                                 refcount++;
 994                         }
 995
 996                         /* set transfer endpoint pointer */
 997                         xfer->endpoint = ep;
 998
 999                         parm->size[0] += sizeof(xfer[0]);
1000                         parm->methods = xfer->endpoint->methods;
1001                         parm->curr_xfer = xfer;
1002
1003                         /*
1004                          * Call the Host or Device controller transfer
1005                          * setup routine:
1006                          */
1007                         (udev->bus->methods->xfer_setup) (parm);
1008
1009                         /* check for error */
1010                         if (parm->err)
1011                                 goto done;
1012
1013                         if (buf) {
1014                                 /*
1015                                  * Increment the endpoint refcount. This
1016                                  * basically prevents setting a new
1017                                  * configuration and alternate setting
1018                                  * when USB transfers are in use on
1019                                  * the given interface. Search the USB
1020                                  * code for "endpoint->refcount_alloc" if you
1021                                  * want more information.
1022                                  */
1023                                 USB_BUS_LOCK(info->bus);
1024                                 if (xfer->endpoint->refcount_alloc >= USB_EP_REF_MAX)
1025                                         parm->err = USB_ERR_INVAL;
1026
1027                                 xfer->endpoint->refcount_alloc++;
1028
1029                                 if (xfer->endpoint->refcount_alloc == 0)
1030                                         panic("usbd_transfer_setup(): Refcount wrapped to zero\n");
1031                                 USB_BUS_UNLOCK(info->bus);
1032
1033                                 /*
1034                                  * Whenever we set ppxfer[] then we
1035                                  * also need to increment the
1036                                  * "setup_refcount":
1037                                  */
1038                                 info->setup_refcount++;
1039
1040                                 /*
1041                                  * Transfer is successfully setup and
1042                                  * can be used:
1043                                  */
1044                                 ppxfer[n] = xfer;
1045                         }
1046
1047                         /* check for error */
1048                         if (parm->err)
1049                                 goto done;
1050                 }
1051
1052                 if (buf != NULL || parm->err != 0)
1053                         goto done;
1054
1055                 /* if no transfers, nothing to do */
1056                 if (refcount == 0)
1057                         goto done;
1058
1059                 /* align data properly */
1060                 parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
1061
1062                 /* store offset temporarily */
1063                 parm->size[1] = parm->size[0];
1064
1065                 /*
1066                  * The number of DMA tags required depends on
1067                  * the number of endpoints. The current estimate
1068                  * for maximum number of DMA tags per endpoint
1069                  * is two.
1070                  */
1071                 parm->dma_tag_max += 2 * MIN(n_setup, USB_EP_MAX);
1072
1073                 /*
1074                  * DMA tags for QH, TD, Data and more.
1075                  */
1076                 parm->dma_tag_max += 8;
1077
1078                 parm->dma_tag_p += parm->dma_tag_max;
1079
1080                 parm->size[0] += ((uint8_t *)parm->dma_tag_p) -
1081                     ((uint8_t *)0);
1082
1083                 /* align data properly */
1084                 parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
1085
1086                 /* store offset temporarily */
1087                 parm->size[3] = parm->size[0];
1088
1089                 parm->size[0] += ((uint8_t *)parm->dma_page_ptr) -
1090                     ((uint8_t *)0);
1091
1092                 /* align data properly */
1093                 parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
1094
1095                 /* store offset temporarily */
1096                 parm->size[4] = parm->size[0];
1097
1098                 parm->size[0] += ((uint8_t *)parm->dma_page_cache_ptr) -
1099                     ((uint8_t *)0);
1100
1101                 /* store end offset temporarily */
1102                 parm->size[5] = parm->size[0];
1103
1104                 parm->size[0] += ((uint8_t *)parm->xfer_page_cache_ptr) -
1105                     ((uint8_t *)0);
1106
1107                 /* store end offset temporarily */
1108
1109                 parm->size[2] = parm->size[0];
1110
1111                 /* align data properly */
1112                 parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
1113
1114                 parm->size[6] = parm->size[0];
1115
1116                 parm->size[0] += ((uint8_t *)parm->xfer_length_ptr) -
1117                     ((uint8_t *)0);
1118
1119                 /* align data properly */
1120                 parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
1121
1122                 /* allocate zeroed memory */
1123                 buf = malloc(parm->size[0], M_USB, M_WAITOK | M_ZERO);
1124
1125                 if (buf == NULL) {
1126                         parm->err = USB_ERR_NOMEM;
1127                         DPRINTFN(0, "cannot allocate memory block for "
1128                             "configuration (%d bytes)\n",
1129                             parm->size[0]);
1130                         goto done;
1131                 }
1132                 parm->dma_tag_p = USB_ADD_BYTES(buf, parm->size[1]);
1133                 parm->dma_page_ptr = USB_ADD_BYTES(buf, parm->size[3]);
1134                 parm->dma_page_cache_ptr = USB_ADD_BYTES(buf, parm->size[4]);
1135                 parm->xfer_page_cache_ptr = USB_ADD_BYTES(buf, parm->size[5]);
1136                 parm->xfer_length_ptr = USB_ADD_BYTES(buf, parm->size[6]);
1137         }
1138
1139 done:
1140         if (buf) {
1141                 if (info->setup_refcount == 0) {
1142                         /*
1143                          * "usbd_transfer_unsetup_sub" will unlock
1144                          * the bus mutex before returning !
1145                          */
1146                         USB_BUS_LOCK(info->bus);
1147
1148                         /* something went wrong */
1149                         usbd_transfer_unsetup_sub(info, 0);
1150                 }
1151         }
1152
1153         /* check if any errors happened */
1154         if (parm->err)
1155                 usbd_transfer_unsetup(ppxfer, n_setup);
1156
1157         error = parm->err;
1158
1159         if (do_unlock)
1160                 usbd_enum_unlock(udev);
1161
1162         return (error);
1163 }
1164
1165 /*------------------------------------------------------------------------*
1166  *      usbd_transfer_unsetup_sub - factored out code
1167  *------------------------------------------------------------------------*/
1168 static void
1169 usbd_transfer_unsetup_sub(struct usb_xfer_root *info, uint8_t needs_delay)
1170 {
1171 #if USB_HAVE_BUSDMA
1172         struct usb_page_cache *pc;
1173 #endif
1174
1175         USB_BUS_LOCK_ASSERT(info->bus, MA_OWNED);
1176
1177         /* wait for any outstanding DMA operations */
1178
1179         if (needs_delay) {
1180                 usb_timeout_t temp;
1181                 temp = usbd_get_dma_delay(info->udev);
1182                 if (temp != 0) {
1183                         usb_pause_mtx(&info->bus->bus_mtx,
1184                             USB_MS_TO_TICKS(temp));
1185                 }
1186         }
1187
1188         /* make sure that our done messages are not queued anywhere */
1189         usb_proc_mwait(info->done_p, &info->done_m[0], &info->done_m[1]);
1190
1191         USB_BUS_UNLOCK(info->bus);
1192
1193 #if USB_HAVE_BUSDMA
1194         /* free DMA'able memory, if any */
1195         pc = info->dma_page_cache_start;
1196         while (pc != info->dma_page_cache_end) {
1197                 usb_pc_free_mem(pc);
1198                 pc++;
1199         }
1200
1201         /* free DMA maps in all "xfer->frbuffers" */
1202         pc = info->xfer_page_cache_start;
1203         while (pc != info->xfer_page_cache_end) {
1204                 usb_pc_dmamap_destroy(pc);
1205                 pc++;
1206         }
1207
1208         /* free all DMA tags */
1209         usb_dma_tag_unsetup(&info->dma_parent_tag);
1210 #endif
1211
1212         cv_destroy(&info->cv_drain);
1213
1214         /*
1215          * free the "memory_base" last, hence the "info" structure is
1216          * contained within the "memory_base"!
1217          */
1218         free(info->memory_base, M_USB);
1219 }
1220
1221 /*------------------------------------------------------------------------*
1222  *      usbd_transfer_unsetup - unsetup/free an array of USB transfers
1223  *
1224  * NOTE: All USB transfers in progress will get called back passing
1225  * the error code "USB_ERR_CANCELLED" before this function
1226  * returns.
1227  *------------------------------------------------------------------------*/
1228 void
1229 usbd_transfer_unsetup(struct usb_xfer **pxfer, uint16_t n_setup)
1230 {
1231         struct usb_xfer *xfer;
1232         struct usb_xfer_root *info;
1233         uint8_t needs_delay = 0;
1234
1235         WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
1236             "usbd_transfer_unsetup can sleep!");
1237
1238         while (n_setup--) {
1239                 xfer = pxfer[n_setup];
1240
1241                 if (xfer == NULL)
1242                         continue;
1243
1244                 info = xfer->xroot;
1245
1246                 USB_XFER_LOCK(xfer);
1247                 USB_BUS_LOCK(info->bus);
1248
1249                 /*
1250                  * HINT: when you start/stop a transfer, it might be a
1251                  * good idea to directly use the "pxfer[]" structure:
1252                  *
1253                  * usbd_transfer_start(sc->pxfer[0]);
1254                  * usbd_transfer_stop(sc->pxfer[0]);
1255                  *
1256                  * That way, if your code has many parts that will not
1257                  * stop running under the same lock, in other words
1258                  * "xfer_mtx", the usbd_transfer_start and
1259                  * usbd_transfer_stop functions will simply return
1260                  * when they detect a NULL pointer argument.
1261                  *
1262                  * To avoid any races we clear the "pxfer[]" pointer
1263                  * while holding the private mutex of the driver:
1264                  */
1265                 pxfer[n_setup] = NULL;
1266
1267                 USB_BUS_UNLOCK(info->bus);
1268                 USB_XFER_UNLOCK(xfer);
1269
1270                 usbd_transfer_drain(xfer);
1271
1272 #if USB_HAVE_BUSDMA
1273                 if (xfer->flags_int.bdma_enable)
1274                         needs_delay = 1;
1275 #endif
1276                 /*
1277                  * NOTE: default endpoint does not have an
1278                  * interface, even if endpoint->iface_index == 0
1279                  */
1280                 USB_BUS_LOCK(info->bus);
1281                 xfer->endpoint->refcount_alloc--;
1282                 USB_BUS_UNLOCK(info->bus);
1283
1284                 usb_callout_drain(&xfer->timeout_handle);
1285
1286                 USB_BUS_LOCK(info->bus);
1287
1288                 USB_ASSERT(info->setup_refcount != 0, ("Invalid setup "
1289                     "reference count\n"));
1290
1291                 info->setup_refcount--;
1292
1293                 if (info->setup_refcount == 0) {
1294                         usbd_transfer_unsetup_sub(info,
1295                             needs_delay);
1296                 } else {
1297                         USB_BUS_UNLOCK(info->bus);
1298                 }
1299         }
1300 }
1301
1302 /*------------------------------------------------------------------------*
1303  *      usbd_control_transfer_init - factored out code
1304  *
1305  * In USB Device Mode we have to wait for the SETUP packet which
1306  * containst the "struct usb_device_request" structure, before we can
1307  * transfer any data. In USB Host Mode we already have the SETUP
1308  * packet at the moment the USB transfer is started. This leads us to
1309  * having to setup the USB transfer at two different places in
1310  * time. This function just contains factored out control transfer
1311  * initialisation code, so that we don't duplicate the code.
1312  *------------------------------------------------------------------------*/
1313 static void
1314 usbd_control_transfer_init(struct usb_xfer *xfer)
1315 {
1316         struct usb_device_request req;
1317
1318         /* copy out the USB request header */
1319
1320         usbd_copy_out(xfer->frbuffers, 0, &req, sizeof(req));
1321
1322         /* setup remainder */
1323
1324         xfer->flags_int.control_rem = UGETW(req.wLength);
1325
1326         /* copy direction to endpoint variable */
1327
1328         xfer->endpointno &= ~(UE_DIR_IN | UE_DIR_OUT);
1329         xfer->endpointno |=
1330             (req.bmRequestType & UT_READ) ? UE_DIR_IN : UE_DIR_OUT;
1331 }
1332
1333 /*------------------------------------------------------------------------*
1334  *      usbd_setup_ctrl_transfer
1335  *
1336  * This function handles initialisation of control transfers. Control
1337  * transfers are special in that regard that they can both transmit
1338  * and receive data.
1339  *
1340  * Return values:
1341  *    0: Success
1342  * Else: Failure
1343  *------------------------------------------------------------------------*/
1344 static int
1345 usbd_setup_ctrl_transfer(struct usb_xfer *xfer)
1346 {
1347         usb_frlength_t len;
1348
1349         /* Check for control endpoint stall */
1350         if (xfer->flags.stall_pipe && xfer->flags_int.control_act) {
1351                 /* the control transfer is no longer active */
1352                 xfer->flags_int.control_stall = 1;
1353                 xfer->flags_int.control_act = 0;
1354         } else {
1355                 /* don't stall control transfer by default */
1356                 xfer->flags_int.control_stall = 0;
1357         }
1358
1359         /* Check for invalid number of frames */
1360         if (xfer->nframes > 2) {
1361                 /*
1362                  * If you need to split a control transfer, you
1363                  * have to do one part at a time. Only with
1364                  * non-control transfers you can do multiple
1365                  * parts a time.
1366                  */
1367                 DPRINTFN(0, "Too many frames: %u\n",
1368                     (unsigned int)xfer->nframes);
1369                 goto error;
1370         }
1371
1372         /*
1373          * Check if there is a control
1374          * transfer in progress:
1375          */
1376         if (xfer->flags_int.control_act) {
1377
1378                 if (xfer->flags_int.control_hdr) {
1379
1380                         /* clear send header flag */
1381
1382                         xfer->flags_int.control_hdr = 0;
1383
1384                         /* setup control transfer */
1385                         if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) {
1386                                 usbd_control_transfer_init(xfer);
1387                         }
1388                 }
1389                 /* get data length */
1390
1391                 len = xfer->sumlen;
1392
1393         } else {
1394
1395                 /* the size of the SETUP structure is hardcoded ! */
1396
1397                 if (xfer->frlengths[0] != sizeof(struct usb_device_request)) {
1398                         DPRINTFN(0, "Wrong framelength %u != %zu\n",
1399                             xfer->frlengths[0], sizeof(struct
1400                             usb_device_request));
1401                         goto error;
1402                 }
1403                 /* check USB mode */
1404                 if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) {
1405
1406                         /* check number of frames */
1407                         if (xfer->nframes != 1) {
1408                                 /*
1409                                  * We need to receive the setup
1410                                  * message first so that we know the
1411                                  * data direction!
1412                                  */
1413                                 DPRINTF("Misconfigured transfer\n");
1414                                 goto error;
1415                         }
1416                         /*
1417                          * Set a dummy "control_rem" value.  This
1418                          * variable will be overwritten later by a
1419                          * call to "usbd_control_transfer_init()" !
1420                          */
1421                         xfer->flags_int.control_rem = 0xFFFF;
1422                 } else {
1423
1424                         /* setup "endpoint" and "control_rem" */
1425
1426                         usbd_control_transfer_init(xfer);
1427                 }
1428
1429                 /* set transfer-header flag */
1430
1431                 xfer->flags_int.control_hdr = 1;
1432
1433                 /* get data length */
1434
1435                 len = (xfer->sumlen - sizeof(struct usb_device_request));
1436         }
1437
1438         /* check if there is a length mismatch */
1439
1440         if (len > xfer->flags_int.control_rem) {
1441                 DPRINTFN(0, "Length (%d) greater than "
1442                     "remaining length (%d)\n", len,
1443                     xfer->flags_int.control_rem);
1444                 goto error;
1445         }
1446         /* check if we are doing a short transfer */
1447
1448         if (xfer->flags.force_short_xfer) {
1449                 xfer->flags_int.control_rem = 0;
1450         } else {
1451                 if ((len != xfer->max_data_length) &&
1452                     (len != xfer->flags_int.control_rem) &&
1453                     (xfer->nframes != 1)) {
1454                         DPRINTFN(0, "Short control transfer without "
1455                             "force_short_xfer set\n");
1456                         goto error;
1457                 }
1458                 xfer->flags_int.control_rem -= len;
1459         }
1460
1461         /* the status part is executed when "control_act" is 0 */
1462
1463         if ((xfer->flags_int.control_rem > 0) ||
1464             (xfer->flags.manual_status)) {
1465                 /* don't execute the STATUS stage yet */
1466                 xfer->flags_int.control_act = 1;
1467
1468                 /* sanity check */
1469                 if ((!xfer->flags_int.control_hdr) &&
1470                     (xfer->nframes == 1)) {
1471                         /*
1472                          * This is not a valid operation!
1473                          */
1474                         DPRINTFN(0, "Invalid parameter "
1475                             "combination\n");
1476                         goto error;
1477                 }
1478         } else {
1479                 /* time to execute the STATUS stage */
1480                 xfer->flags_int.control_act = 0;
1481         }
1482         return (0);                     /* success */
1483
1484 error:
1485         return (1);                     /* failure */
1486 }
1487
1488 /*------------------------------------------------------------------------*
1489  *      usbd_transfer_submit - start USB hardware for the given transfer
1490  *
1491  * This function should only be called from the USB callback.
1492  *------------------------------------------------------------------------*/
1493 void
1494 usbd_transfer_submit(struct usb_xfer *xfer)
1495 {
1496         struct usb_xfer_root *info;
1497         struct usb_bus *bus;
1498         usb_frcount_t x;
1499
1500         info = xfer->xroot;
1501         bus = info->bus;
1502
1503         DPRINTF("xfer=%p, endpoint=%p, nframes=%d, dir=%s\n",
1504             xfer, xfer->endpoint, xfer->nframes, USB_GET_DATA_ISREAD(xfer) ?
1505             "read" : "write");
1506
1507 #ifdef USB_DEBUG
1508         if (USB_DEBUG_VAR > 0) {
1509                 USB_BUS_LOCK(bus);
1510
1511                 usb_dump_endpoint(xfer->endpoint);
1512
1513                 USB_BUS_UNLOCK(bus);
1514         }
1515 #endif
1516
1517         USB_XFER_LOCK_ASSERT(xfer, MA_OWNED);
1518         USB_BUS_LOCK_ASSERT(bus, MA_NOTOWNED);
1519
1520         /* Only open the USB transfer once! */
1521         if (!xfer->flags_int.open) {
1522                 xfer->flags_int.open = 1;
1523
1524                 DPRINTF("open\n");
1525
1526                 USB_BUS_LOCK(bus);
1527                 (xfer->endpoint->methods->open) (xfer);
1528                 USB_BUS_UNLOCK(bus);
1529         }
1530         /* set "transferring" flag */
1531         xfer->flags_int.transferring = 1;
1532
1533 #if USB_HAVE_POWERD
1534         /* increment power reference */
1535         usbd_transfer_power_ref(xfer, 1);
1536 #endif
1537         /*
1538          * Check if the transfer is waiting on a queue, most
1539          * frequently the "done_q":
1540          */
1541         if (xfer->wait_queue) {
1542                 USB_BUS_LOCK(bus);
1543                 usbd_transfer_dequeue(xfer);
1544                 USB_BUS_UNLOCK(bus);
1545         }
1546         /* clear "did_dma_delay" flag */
1547         xfer->flags_int.did_dma_delay = 0;
1548
1549         /* clear "did_close" flag */
1550         xfer->flags_int.did_close = 0;
1551
1552 #if USB_HAVE_BUSDMA
1553         /* clear "bdma_setup" flag */
1554         xfer->flags_int.bdma_setup = 0;
1555 #endif
1556         /* by default we cannot cancel any USB transfer immediately */
1557         xfer->flags_int.can_cancel_immed = 0;
1558
1559         /* clear lengths and frame counts by default */
1560         xfer->sumlen = 0;
1561         xfer->actlen = 0;
1562         xfer->aframes = 0;
1563
1564         /* clear any previous errors */
1565         xfer->error = 0;
1566
1567         /* Check if the device is still alive */
1568         if (info->udev->state < USB_STATE_POWERED) {
1569                 USB_BUS_LOCK(bus);
1570                 /*
1571                  * Must return cancelled error code else
1572                  * device drivers can hang.
1573                  */
1574                 usbd_transfer_done(xfer, USB_ERR_CANCELLED);
1575                 USB_BUS_UNLOCK(bus);
1576                 return;
1577         }
1578
1579         /* sanity check */
1580         if (xfer->nframes == 0) {
1581                 if (xfer->flags.stall_pipe) {
1582                         /*
1583                          * Special case - want to stall without transferring
1584                          * any data:
1585                          */
1586                         DPRINTF("xfer=%p nframes=0: stall "
1587                             "or clear stall!\n", xfer);
1588                         USB_BUS_LOCK(bus);
1589                         xfer->flags_int.can_cancel_immed = 1;
1590                         /* start the transfer */
1591                         usb_command_wrapper(&xfer->endpoint->endpoint_q, xfer);
1592                         USB_BUS_UNLOCK(bus);
1593                         return;
1594                 }
1595                 USB_BUS_LOCK(bus);
1596                 usbd_transfer_done(xfer, USB_ERR_INVAL);
1597                 USB_BUS_UNLOCK(bus);
1598                 return;
1599         }
1600         /* compute some variables */
1601
1602         for (x = 0; x != xfer->nframes; x++) {
1603                 /* make a copy of the frlenghts[] */
1604                 xfer->frlengths[x + xfer->max_frame_count] = xfer->frlengths[x];
1605                 /* compute total transfer length */
1606                 xfer->sumlen += xfer->frlengths[x];
1607                 if (xfer->sumlen < xfer->frlengths[x]) {
1608                         /* length wrapped around */
1609                         USB_BUS_LOCK(bus);
1610                         usbd_transfer_done(xfer, USB_ERR_INVAL);
1611                         USB_BUS_UNLOCK(bus);
1612                         return;
1613                 }
1614         }
1615
1616         /* clear some internal flags */
1617
1618         xfer->flags_int.short_xfer_ok = 0;
1619         xfer->flags_int.short_frames_ok = 0;
1620
1621         /* check if this is a control transfer */
1622
1623         if (xfer->flags_int.control_xfr) {
1624
1625                 if (usbd_setup_ctrl_transfer(xfer)) {
1626                         USB_BUS_LOCK(bus);
1627                         usbd_transfer_done(xfer, USB_ERR_STALLED);
1628                         USB_BUS_UNLOCK(bus);
1629                         return;
1630                 }
1631         }
1632         /*
1633          * Setup filtered version of some transfer flags,
1634          * in case of data read direction
1635          */
1636         if (USB_GET_DATA_ISREAD(xfer)) {
1637
1638                 if (xfer->flags.short_frames_ok) {
1639                         xfer->flags_int.short_xfer_ok = 1;
1640                         xfer->flags_int.short_frames_ok = 1;
1641                 } else if (xfer->flags.short_xfer_ok) {
1642                         xfer->flags_int.short_xfer_ok = 1;
1643
1644                         /* check for control transfer */
1645                         if (xfer->flags_int.control_xfr) {
1646                                 /*
1647                                  * 1) Control transfers do not support
1648                                  * reception of multiple short USB
1649                                  * frames in host mode and device side
1650                                  * mode, with exception of:
1651                                  *
1652                                  * 2) Due to sometimes buggy device
1653                                  * side firmware we need to do a
1654                                  * STATUS stage in case of short
1655                                  * control transfers in USB host mode.
1656                                  * The STATUS stage then becomes the
1657                                  * "alt_next" to the DATA stage.
1658                                  */
1659                                 xfer->flags_int.short_frames_ok = 1;
1660                         }
1661                 }
1662         }
1663         /*
1664          * Check if BUS-DMA support is enabled and try to load virtual
1665          * buffers into DMA, if any:
1666          */
1667 #if USB_HAVE_BUSDMA
1668         if (xfer->flags_int.bdma_enable) {
1669                 /* insert the USB transfer last in the BUS-DMA queue */
1670                 usb_command_wrapper(&xfer->xroot->dma_q, xfer);
1671                 return;
1672         }
1673 #endif
1674         /*
1675          * Enter the USB transfer into the Host Controller or
1676          * Device Controller schedule:
1677          */
1678         usbd_pipe_enter(xfer);
1679 }
1680
1681 /*------------------------------------------------------------------------*
1682  *      usbd_pipe_enter - factored out code
1683  *------------------------------------------------------------------------*/
1684 void
1685 usbd_pipe_enter(struct usb_xfer *xfer)
1686 {
1687         struct usb_endpoint *ep;
1688
1689         USB_XFER_LOCK_ASSERT(xfer, MA_OWNED);
1690
1691         USB_BUS_LOCK(xfer->xroot->bus);
1692
1693         ep = xfer->endpoint;
1694
1695         DPRINTF("enter\n");
1696
1697         /* the transfer can now be cancelled */
1698         xfer->flags_int.can_cancel_immed = 1;
1699
1700         /* enter the transfer */
1701         (ep->methods->enter) (xfer);
1702
1703         /* check for transfer error */
1704         if (xfer->error) {
1705                 /* some error has happened */
1706                 usbd_transfer_done(xfer, 0);
1707                 USB_BUS_UNLOCK(xfer->xroot->bus);
1708                 return;
1709         }
1710
1711         /* start the transfer */
1712         usb_command_wrapper(&ep->endpoint_q, xfer);
1713         USB_BUS_UNLOCK(xfer->xroot->bus);
1714 }
1715
1716 /*------------------------------------------------------------------------*
1717  *      usbd_transfer_start - start an USB transfer
1718  *
1719  * NOTE: Calling this function more than one time will only
1720  *       result in a single transfer start, until the USB transfer
1721  *       completes.
1722  *------------------------------------------------------------------------*/
1723 void
1724 usbd_transfer_start(struct usb_xfer *xfer)
1725 {
1726         if (xfer == NULL) {
1727                 /* transfer is gone */
1728                 return;
1729         }
1730         USB_XFER_LOCK_ASSERT(xfer, MA_OWNED);
1731
1732         /* mark the USB transfer started */
1733
1734         if (!xfer->flags_int.started) {
1735                 /* lock the BUS lock to avoid races updating flags_int */
1736                 USB_BUS_LOCK(xfer->xroot->bus);
1737                 xfer->flags_int.started = 1;
1738                 USB_BUS_UNLOCK(xfer->xroot->bus);
1739         }
1740         /* check if the USB transfer callback is already transferring */
1741
1742         if (xfer->flags_int.transferring) {
1743                 return;
1744         }
1745         USB_BUS_LOCK(xfer->xroot->bus);
1746         /* call the USB transfer callback */
1747         usbd_callback_ss_done_defer(xfer);
1748         USB_BUS_UNLOCK(xfer->xroot->bus);
1749 }
1750
1751 /*------------------------------------------------------------------------*
1752  *      usbd_transfer_stop - stop an USB transfer
1753  *
1754  * NOTE: Calling this function more than one time will only
1755  *       result in a single transfer stop.
1756  * NOTE: When this function returns it is not safe to free nor
1757  *       reuse any DMA buffers. See "usbd_transfer_drain()".
1758  *------------------------------------------------------------------------*/
1759 void
1760 usbd_transfer_stop(struct usb_xfer *xfer)
1761 {
1762         struct usb_endpoint *ep;
1763
1764         if (xfer == NULL) {
1765                 /* transfer is gone */
1766                 return;
1767         }
1768         USB_XFER_LOCK_ASSERT(xfer, MA_OWNED);
1769
1770         /* check if the USB transfer was ever opened */
1771
1772         if (!xfer->flags_int.open) {
1773                 if (xfer->flags_int.started) {
1774                         /* nothing to do except clearing the "started" flag */
1775                         /* lock the BUS lock to avoid races updating flags_int */
1776                         USB_BUS_LOCK(xfer->xroot->bus);
1777                         xfer->flags_int.started = 0;
1778                         USB_BUS_UNLOCK(xfer->xroot->bus);
1779                 }
1780                 return;
1781         }
1782         /* try to stop the current USB transfer */
1783
1784         USB_BUS_LOCK(xfer->xroot->bus);
1785         /* override any previous error */
1786         xfer->error = USB_ERR_CANCELLED;
1787
1788         /*
1789          * Clear "open" and "started" when both private and USB lock
1790          * is locked so that we don't get a race updating "flags_int"
1791          */
1792         xfer->flags_int.open = 0;
1793         xfer->flags_int.started = 0;
1794
1795         /*
1796          * Check if we can cancel the USB transfer immediately.
1797          */
1798         if (xfer->flags_int.transferring) {
1799                 if (xfer->flags_int.can_cancel_immed &&
1800                     (!xfer->flags_int.did_close)) {
1801                         DPRINTF("close\n");
1802                         /*
1803                          * The following will lead to an USB_ERR_CANCELLED
1804                          * error code being passed to the USB callback.
1805                          */
1806                         (xfer->endpoint->methods->close) (xfer);
1807                         /* only close once */
1808                         xfer->flags_int.did_close = 1;
1809                 } else {
1810                         /* need to wait for the next done callback */
1811                 }
1812         } else {
1813                 DPRINTF("close\n");
1814
1815                 /* close here and now */
1816                 (xfer->endpoint->methods->close) (xfer);
1817
1818                 /*
1819                  * Any additional DMA delay is done by
1820                  * "usbd_transfer_unsetup()".
1821                  */
1822
1823                 /*
1824                  * Special case. Check if we need to restart a blocked
1825                  * endpoint.
1826                  */
1827                 ep = xfer->endpoint;
1828
1829                 /*
1830                  * If the current USB transfer is completing we need
1831                  * to start the next one:
1832                  */
1833                 if (ep->endpoint_q.curr == xfer) {
1834                         usb_command_wrapper(&ep->endpoint_q, NULL);
1835                 }
1836         }
1837
1838         USB_BUS_UNLOCK(xfer->xroot->bus);
1839 }
1840
1841 /*------------------------------------------------------------------------*
1842  *      usbd_transfer_pending
1843  *
1844  * This function will check if an USB transfer is pending which is a
1845  * little bit complicated!
1846  * Return values:
1847  * 0: Not pending
1848  * 1: Pending: The USB transfer will receive a callback in the future.
1849  *------------------------------------------------------------------------*/
1850 uint8_t
1851 usbd_transfer_pending(struct usb_xfer *xfer)
1852 {
1853         struct usb_xfer_root *info;
1854         struct usb_xfer_queue *pq;
1855
1856         if (xfer == NULL) {
1857                 /* transfer is gone */
1858                 return (0);
1859         }
1860         USB_XFER_LOCK_ASSERT(xfer, MA_OWNED);
1861
1862         if (xfer->flags_int.transferring) {
1863                 /* trivial case */
1864                 return (1);
1865         }
1866         USB_BUS_LOCK(xfer->xroot->bus);
1867         if (xfer->wait_queue) {
1868                 /* we are waiting on a queue somewhere */
1869                 USB_BUS_UNLOCK(xfer->xroot->bus);
1870                 return (1);
1871         }
1872         info = xfer->xroot;
1873         pq = &info->done_q;
1874
1875         if (pq->curr == xfer) {
1876                 /* we are currently scheduled for callback */
1877                 USB_BUS_UNLOCK(xfer->xroot->bus);
1878                 return (1);
1879         }
1880         /* we are not pending */
1881         USB_BUS_UNLOCK(xfer->xroot->bus);
1882         return (0);
1883 }
1884
1885 /*------------------------------------------------------------------------*
1886  *      usbd_transfer_drain
1887  *
1888  * This function will stop the USB transfer and wait for any
1889  * additional BUS-DMA and HW-DMA operations to complete. Buffers that
1890  * are loaded into DMA can safely be freed or reused after that this
1891  * function has returned.
1892  *------------------------------------------------------------------------*/
1893 void
1894 usbd_transfer_drain(struct usb_xfer *xfer)
1895 {
1896         WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
1897             "usbd_transfer_drain can sleep!");
1898
1899         if (xfer == NULL) {
1900                 /* transfer is gone */
1901                 return;
1902         }
1903         if (xfer->xroot->xfer_mtx != &Giant) {
1904                 USB_XFER_LOCK_ASSERT(xfer, MA_NOTOWNED);
1905         }
1906         USB_XFER_LOCK(xfer);
1907
1908         usbd_transfer_stop(xfer);
1909
1910         while (usbd_transfer_pending(xfer) ||
1911             xfer->flags_int.doing_callback) {
1912
1913                 /*
1914                  * It is allowed that the callback can drop its
1915                  * transfer mutex. In that case checking only
1916                  * "usbd_transfer_pending()" is not enough to tell if
1917                  * the USB transfer is fully drained. We also need to
1918                  * check the internal "doing_callback" flag.
1919                  */
1920                 xfer->flags_int.draining = 1;
1921
1922                 /*
1923                  * Wait until the current outstanding USB
1924                  * transfer is complete !
1925                  */
1926                 cv_wait(&xfer->xroot->cv_drain, xfer->xroot->xfer_mtx);
1927         }
1928         USB_XFER_UNLOCK(xfer);
1929 }
1930
1931 struct usb_page_cache *
1932 usbd_xfer_get_frame(struct usb_xfer *xfer, usb_frcount_t frindex)
1933 {
1934         KASSERT(frindex < xfer->max_frame_count, ("frame index overflow"));
1935
1936         return (&xfer->frbuffers[frindex]);
1937 }
1938
1939 void *
1940 usbd_xfer_get_frame_buffer(struct usb_xfer *xfer, usb_frcount_t frindex)
1941 {
1942         struct usb_page_search page_info;
1943
1944         KASSERT(frindex < xfer->max_frame_count, ("frame index overflow"));
1945
1946         usbd_get_page(&xfer->frbuffers[frindex], 0, &page_info);
1947         return (page_info.buffer);
1948 }
1949
1950 /*------------------------------------------------------------------------*
1951  *      usbd_xfer_get_fps_shift
1952  *
1953  * The following function is only useful for isochronous transfers. It
1954  * returns how many times the frame execution rate has been shifted
1955  * down.
1956  *
1957  * Return value:
1958  * Success: 0..3
1959  * Failure: 0
1960  *------------------------------------------------------------------------*/
1961 uint8_t
1962 usbd_xfer_get_fps_shift(struct usb_xfer *xfer)
1963 {
1964         return (xfer->fps_shift);
1965 }
1966
1967 usb_frlength_t
1968 usbd_xfer_frame_len(struct usb_xfer *xfer, usb_frcount_t frindex)
1969 {
1970         KASSERT(frindex < xfer->max_frame_count, ("frame index overflow"));
1971
1972         return (xfer->frlengths[frindex]);
1973 }
1974
1975 /*------------------------------------------------------------------------*
1976  *      usbd_xfer_set_frame_data
1977  *
1978  * This function sets the pointer of the buffer that should
1979  * loaded directly into DMA for the given USB frame. Passing "ptr"
1980  * equal to NULL while the corresponding "frlength" is greater
1981  * than zero gives undefined results!
1982  *------------------------------------------------------------------------*/
1983 void
1984 usbd_xfer_set_frame_data(struct usb_xfer *xfer, usb_frcount_t frindex,
1985     void *ptr, usb_frlength_t len)
1986 {
1987         KASSERT(frindex < xfer->max_frame_count, ("frame index overflow"));
1988
1989         /* set virtual address to load and length */
1990         xfer->frbuffers[frindex].buffer = ptr;
1991         usbd_xfer_set_frame_len(xfer, frindex, len);
1992 }
1993
1994 void
1995 usbd_xfer_frame_data(struct usb_xfer *xfer, usb_frcount_t frindex,
1996     void **ptr, int *len)
1997 {
1998         KASSERT(frindex < xfer->max_frame_count, ("frame index overflow"));
1999
2000         if (ptr != NULL)
2001                 *ptr = xfer->frbuffers[frindex].buffer;
2002         if (len != NULL)
2003                 *len = xfer->frlengths[frindex];
2004 }
2005
2006 /*------------------------------------------------------------------------*
2007  *      usbd_xfer_old_frame_length
2008  *
2009  * This function returns the framelength of the given frame at the
2010  * time the transfer was submitted. This function can be used to
2011  * compute the starting data pointer of the next isochronous frame
2012  * when an isochronous transfer has completed.
2013  *------------------------------------------------------------------------*/
2014 usb_frlength_t
2015 usbd_xfer_old_frame_length(struct usb_xfer *xfer, usb_frcount_t frindex)
2016 {
2017         KASSERT(frindex < xfer->max_frame_count, ("frame index overflow"));
2018
2019         return (xfer->frlengths[frindex + xfer->max_frame_count]);
2020 }
2021
2022 void
2023 usbd_xfer_status(struct usb_xfer *xfer, int *actlen, int *sumlen, int *aframes,
2024     int *nframes)
2025 {
2026         if (actlen != NULL)
2027                 *actlen = xfer->actlen;
2028         if (sumlen != NULL)
2029                 *sumlen = xfer->sumlen;
2030         if (aframes != NULL)
2031                 *aframes = xfer->aframes;
2032         if (nframes != NULL)
2033                 *nframes = xfer->nframes;
2034 }
2035
2036 /*------------------------------------------------------------------------*
2037  *      usbd_xfer_set_frame_offset
2038  *
2039  * This function sets the frame data buffer offset relative to the beginning
2040  * of the USB DMA buffer allocated for this USB transfer.
2041  *------------------------------------------------------------------------*/
2042 void
2043 usbd_xfer_set_frame_offset(struct usb_xfer *xfer, usb_frlength_t offset,
2044     usb_frcount_t frindex)
2045 {
2046         KASSERT(!xfer->flags.ext_buffer, ("Cannot offset data frame "
2047             "when the USB buffer is external\n"));
2048         KASSERT(frindex < xfer->max_frame_count, ("frame index overflow"));
2049
2050         /* set virtual address to load */
2051         xfer->frbuffers[frindex].buffer =
2052             USB_ADD_BYTES(xfer->local_buffer, offset);
2053 }
2054
2055 void
2056 usbd_xfer_set_interval(struct usb_xfer *xfer, int i)
2057 {
2058         xfer->interval = i;
2059 }
2060
2061 void
2062 usbd_xfer_set_timeout(struct usb_xfer *xfer, int t)
2063 {
2064         xfer->timeout = t;
2065 }
2066
2067 void
2068 usbd_xfer_set_frames(struct usb_xfer *xfer, usb_frcount_t n)
2069 {
2070         xfer->nframes = n;
2071 }
2072
2073 usb_frcount_t
2074 usbd_xfer_max_frames(struct usb_xfer *xfer)
2075 {
2076         return (xfer->max_frame_count);
2077 }
2078
2079 usb_frlength_t
2080 usbd_xfer_max_len(struct usb_xfer *xfer)
2081 {
2082         return (xfer->max_data_length);
2083 }
2084
2085 usb_frlength_t
2086 usbd_xfer_max_framelen(struct usb_xfer *xfer)
2087 {
2088         return (xfer->max_frame_size);
2089 }
2090
2091 void
2092 usbd_xfer_set_frame_len(struct usb_xfer *xfer, usb_frcount_t frindex,
2093     usb_frlength_t len)
2094 {
2095         KASSERT(frindex < xfer->max_frame_count, ("frame index overflow"));
2096
2097         xfer->frlengths[frindex] = len;
2098 }
2099
2100 /*------------------------------------------------------------------------*
2101  *      usb_callback_proc - factored out code
2102  *
2103  * This function performs USB callbacks.
2104  *------------------------------------------------------------------------*/
2105 static void
2106 usb_callback_proc(struct usb_proc_msg *_pm)
2107 {
2108         struct usb_done_msg *pm = (void *)_pm;
2109         struct usb_xfer_root *info = pm->xroot;
2110
2111         /* Change locking order */
2112         USB_BUS_UNLOCK(info->bus);
2113
2114         /*
2115          * We exploit the fact that the mutex is the same for all
2116          * callbacks that will be called from this thread:
2117          */
2118         mtx_lock(info->xfer_mtx);
2119         USB_BUS_LOCK(info->bus);
2120
2121         /* Continue where we lost track */
2122         usb_command_wrapper(&info->done_q,
2123             info->done_q.curr);
2124
2125         mtx_unlock(info->xfer_mtx);
2126 }
2127
2128 /*------------------------------------------------------------------------*
2129  *      usbd_callback_ss_done_defer
2130  *
2131  * This function will defer the start, stop and done callback to the
2132  * correct thread.
2133  *------------------------------------------------------------------------*/
2134 static void
2135 usbd_callback_ss_done_defer(struct usb_xfer *xfer)
2136 {
2137         struct usb_xfer_root *info = xfer->xroot;
2138         struct usb_xfer_queue *pq = &info->done_q;
2139
2140         USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
2141
2142         if (pq->curr != xfer) {
2143                 usbd_transfer_enqueue(pq, xfer);
2144         }
2145         if (!pq->recurse_1) {
2146
2147                 /*
2148                  * We have to postpone the callback due to the fact we
2149                  * will have a Lock Order Reversal, LOR, if we try to
2150                  * proceed !
2151                  */
2152                 if (usb_proc_msignal(info->done_p,
2153                     &info->done_m[0], &info->done_m[1])) {
2154                         /* ignore */
2155                 }
2156         } else {
2157                 /* clear second recurse flag */
2158                 pq->recurse_2 = 0;
2159         }
2160         return;
2161
2162 }
2163
2164 /*------------------------------------------------------------------------*
2165  *      usbd_callback_wrapper
2166  *
2167  * This is a wrapper for USB callbacks. This wrapper does some
2168  * auto-magic things like figuring out if we can call the callback
2169  * directly from the current context or if we need to wakeup the
2170  * interrupt process.
2171  *------------------------------------------------------------------------*/
2172 static void
2173 usbd_callback_wrapper(struct usb_xfer_queue *pq)
2174 {
2175         struct usb_xfer *xfer = pq->curr;
2176         struct usb_xfer_root *info = xfer->xroot;
2177
2178         USB_BUS_LOCK_ASSERT(info->bus, MA_OWNED);
2179         if (!mtx_owned(info->xfer_mtx) && !SCHEDULER_STOPPED()) {
2180                 /*
2181                  * Cases that end up here:
2182                  *
2183                  * 5) HW interrupt done callback or other source.
2184                  */
2185                 DPRINTFN(3, "case 5\n");
2186
2187                 /*
2188                  * We have to postpone the callback due to the fact we
2189                  * will have a Lock Order Reversal, LOR, if we try to
2190                  * proceed !
2191                  */
2192                 if (usb_proc_msignal(info->done_p,
2193                     &info->done_m[0], &info->done_m[1])) {
2194                         /* ignore */
2195                 }
2196                 return;
2197         }
2198         /*
2199          * Cases that end up here:
2200          *
2201          * 1) We are starting a transfer
2202          * 2) We are prematurely calling back a transfer
2203          * 3) We are stopping a transfer
2204          * 4) We are doing an ordinary callback
2205          */
2206         DPRINTFN(3, "case 1-4\n");
2207         /* get next USB transfer in the queue */
2208         info->done_q.curr = NULL;
2209
2210         /* set flag in case of drain */
2211         xfer->flags_int.doing_callback = 1;
2212
2213         USB_BUS_UNLOCK(info->bus);
2214         USB_BUS_LOCK_ASSERT(info->bus, MA_NOTOWNED);
2215
2216         /* set correct USB state for callback */
2217         if (!xfer->flags_int.transferring) {
2218                 xfer->usb_state = USB_ST_SETUP;
2219                 if (!xfer->flags_int.started) {
2220                         /* we got stopped before we even got started */
2221                         USB_BUS_LOCK(info->bus);
2222                         goto done;
2223                 }
2224         } else {
2225
2226                 if (usbd_callback_wrapper_sub(xfer)) {
2227                         /* the callback has been deferred */
2228                         USB_BUS_LOCK(info->bus);
2229                         goto done;
2230                 }
2231 #if USB_HAVE_POWERD
2232                 /* decrement power reference */
2233                 usbd_transfer_power_ref(xfer, -1);
2234 #endif
2235                 xfer->flags_int.transferring = 0;
2236
2237                 if (xfer->error) {
2238                         xfer->usb_state = USB_ST_ERROR;
2239                 } else {
2240                         /* set transferred state */
2241                         xfer->usb_state = USB_ST_TRANSFERRED;
2242 #if USB_HAVE_BUSDMA
2243                         /* sync DMA memory, if any */
2244                         if (xfer->flags_int.bdma_enable &&
2245                             (!xfer->flags_int.bdma_no_post_sync)) {
2246                                 usb_bdma_post_sync(xfer);
2247                         }
2248 #endif
2249                 }
2250         }
2251
2252 #if USB_HAVE_PF
2253         if (xfer->usb_state != USB_ST_SETUP)
2254                 usbpf_xfertap(xfer, USBPF_XFERTAP_DONE);
2255 #endif
2256         /* call processing routine */
2257         (xfer->callback) (xfer, xfer->error);
2258
2259         /* pickup the USB mutex again */
2260         USB_BUS_LOCK(info->bus);
2261
2262         /*
2263          * Check if we got started after that we got cancelled, but
2264          * before we managed to do the callback.
2265          */
2266         if ((!xfer->flags_int.open) &&
2267             (xfer->flags_int.started) &&
2268             (xfer->usb_state == USB_ST_ERROR)) {
2269                 /* clear flag in case of drain */
2270                 xfer->flags_int.doing_callback = 0;
2271                 /* try to loop, but not recursivly */
2272                 usb_command_wrapper(&info->done_q, xfer);
2273                 return;
2274         }
2275
2276 done:
2277         /* clear flag in case of drain */
2278         xfer->flags_int.doing_callback = 0;
2279
2280         /*
2281          * Check if we are draining.
2282          */
2283         if (xfer->flags_int.draining &&
2284             (!xfer->flags_int.transferring)) {
2285                 /* "usbd_transfer_drain()" is waiting for end of transfer */
2286                 xfer->flags_int.draining = 0;
2287                 cv_broadcast(&info->cv_drain);
2288         }
2289
2290         /* do the next callback, if any */
2291         usb_command_wrapper(&info->done_q,
2292             info->done_q.curr);
2293 }
2294
2295 /*------------------------------------------------------------------------*
2296  *      usb_dma_delay_done_cb
2297  *
2298  * This function is called when the DMA delay has been exectuded, and
2299  * will make sure that the callback is called to complete the USB
2300  * transfer. This code path is ususally only used when there is an USB
2301  * error like USB_ERR_CANCELLED.
2302  *------------------------------------------------------------------------*/
2303 void
2304 usb_dma_delay_done_cb(struct usb_xfer *xfer)
2305 {
2306         USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
2307
2308         DPRINTFN(3, "Completed %p\n", xfer);
2309
2310         /* queue callback for execution, again */
2311         usbd_transfer_done(xfer, 0);
2312 }
2313
2314 /*------------------------------------------------------------------------*
2315  *      usbd_transfer_dequeue
2316  *
2317  *  - This function is used to remove an USB transfer from a USB
2318  *  transfer queue.
2319  *
2320  *  - This function can be called multiple times in a row.
2321  *------------------------------------------------------------------------*/
2322 void
2323 usbd_transfer_dequeue(struct usb_xfer *xfer)
2324 {
2325         struct usb_xfer_queue *pq;
2326
2327         pq = xfer->wait_queue;
2328         if (pq) {
2329                 TAILQ_REMOVE(&pq->head, xfer, wait_entry);
2330                 xfer->wait_queue = NULL;
2331         }
2332 }
2333
2334 /*------------------------------------------------------------------------*
2335  *      usbd_transfer_enqueue
2336  *
2337  *  - This function is used to insert an USB transfer into a USB *
2338  *  transfer queue.
2339  *
2340  *  - This function can be called multiple times in a row.
2341  *------------------------------------------------------------------------*/
2342 void
2343 usbd_transfer_enqueue(struct usb_xfer_queue *pq, struct usb_xfer *xfer)
2344 {
2345         /*
2346          * Insert the USB transfer into the queue, if it is not
2347          * already on a USB transfer queue:
2348          */
2349         if (xfer->wait_queue == NULL) {
2350                 xfer->wait_queue = pq;
2351                 TAILQ_INSERT_TAIL(&pq->head, xfer, wait_entry);
2352         }
2353 }
2354
2355 /*------------------------------------------------------------------------*
2356  *      usbd_transfer_done
2357  *
2358  *  - This function is used to remove an USB transfer from the busdma,
2359  *  pipe or interrupt queue.
2360  *
2361  *  - This function is used to queue the USB transfer on the done
2362  *  queue.
2363  *
2364  *  - This function is used to stop any USB transfer timeouts.
2365  *------------------------------------------------------------------------*/
2366 void
2367 usbd_transfer_done(struct usb_xfer *xfer, usb_error_t error)
2368 {
2369         struct usb_xfer_root *info = xfer->xroot;
2370
2371         USB_BUS_LOCK_ASSERT(info->bus, MA_OWNED);
2372
2373         DPRINTF("err=%s\n", usbd_errstr(error));
2374
2375         /*
2376          * If we are not transferring then just return.
2377          * This can happen during transfer cancel.
2378          */
2379         if (!xfer->flags_int.transferring) {
2380                 DPRINTF("not transferring\n");
2381                 /* end of control transfer, if any */
2382                 xfer->flags_int.control_act = 0;
2383                 return;
2384         }
2385         /* only set transfer error, if not already set */
2386         if (xfer->error == USB_ERR_NORMAL_COMPLETION)
2387                 xfer->error = error;
2388
2389         /* stop any callouts */
2390         usb_callout_stop(&xfer->timeout_handle);
2391
2392         /*
2393          * If we are waiting on a queue, just remove the USB transfer
2394          * from the queue, if any. We should have the required locks
2395          * locked to do the remove when this function is called.
2396          */
2397         usbd_transfer_dequeue(xfer);
2398
2399 #if USB_HAVE_BUSDMA
2400         if (mtx_owned(info->xfer_mtx)) {
2401                 struct usb_xfer_queue *pq;
2402
2403                 /*
2404                  * If the private USB lock is not locked, then we assume
2405                  * that the BUS-DMA load stage has been passed:
2406                  */
2407                 pq = &info->dma_q;
2408
2409                 if (pq->curr == xfer) {
2410                         /* start the next BUS-DMA load, if any */
2411                         usb_command_wrapper(pq, NULL);
2412                 }
2413         }
2414 #endif
2415         /* keep some statistics */
2416         if (xfer->error) {
2417                 info->bus->stats_err.uds_requests
2418                     [xfer->endpoint->edesc->bmAttributes & UE_XFERTYPE]++;
2419         } else {
2420                 info->bus->stats_ok.uds_requests
2421                     [xfer->endpoint->edesc->bmAttributes & UE_XFERTYPE]++;
2422         }
2423
2424         /* call the USB transfer callback */
2425         usbd_callback_ss_done_defer(xfer);
2426 }
2427
2428 /*------------------------------------------------------------------------*
2429  *      usbd_transfer_start_cb
2430  *
2431  * This function is called to start the USB transfer when
2432  * "xfer->interval" is greater than zero, and and the endpoint type is
2433  * BULK or CONTROL.
2434  *------------------------------------------------------------------------*/
2435 static void
2436 usbd_transfer_start_cb(void *arg)
2437 {
2438         struct usb_xfer *xfer = arg;
2439         struct usb_endpoint *ep = xfer->endpoint;
2440
2441         USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
2442
2443         DPRINTF("start\n");
2444
2445 #if USB_HAVE_PF
2446         usbpf_xfertap(xfer, USBPF_XFERTAP_SUBMIT);
2447 #endif
2448
2449         /* the transfer can now be cancelled */
2450         xfer->flags_int.can_cancel_immed = 1;
2451
2452         /* start USB transfer, if no error */
2453         if (xfer->error == 0)
2454                 (ep->methods->start) (xfer);
2455
2456         /* check for transfer error */
2457         if (xfer->error) {
2458                 /* some error has happened */
2459                 usbd_transfer_done(xfer, 0);
2460         }
2461 }
2462
2463 /*------------------------------------------------------------------------*
2464  *      usbd_xfer_set_stall
2465  *
2466  * This function is used to set the stall flag outside the
2467  * callback. This function is NULL safe.
2468  *------------------------------------------------------------------------*/
2469 void
2470 usbd_xfer_set_stall(struct usb_xfer *xfer)
2471 {
2472         if (xfer == NULL) {
2473                 /* tearing down */
2474                 return;
2475         }
2476         USB_XFER_LOCK_ASSERT(xfer, MA_OWNED);
2477
2478         /* avoid any races by locking the USB mutex */
2479         USB_BUS_LOCK(xfer->xroot->bus);
2480         xfer->flags.stall_pipe = 1;
2481         USB_BUS_UNLOCK(xfer->xroot->bus);
2482 }
2483
2484 int
2485 usbd_xfer_is_stalled(struct usb_xfer *xfer)
2486 {
2487         return (xfer->endpoint->is_stalled);
2488 }
2489
2490 /*------------------------------------------------------------------------*
2491  *      usbd_transfer_clear_stall
2492  *
2493  * This function is used to clear the stall flag outside the
2494  * callback. This function is NULL safe.
2495  *------------------------------------------------------------------------*/
2496 void
2497 usbd_transfer_clear_stall(struct usb_xfer *xfer)
2498 {
2499         if (xfer == NULL) {
2500                 /* tearing down */
2501                 return;
2502         }
2503         USB_XFER_LOCK_ASSERT(xfer, MA_OWNED);
2504
2505         /* avoid any races by locking the USB mutex */
2506         USB_BUS_LOCK(xfer->xroot->bus);
2507
2508         xfer->flags.stall_pipe = 0;
2509
2510         USB_BUS_UNLOCK(xfer->xroot->bus);
2511 }
2512
2513 /*------------------------------------------------------------------------*
2514  *      usbd_pipe_start
2515  *
2516  * This function is used to add an USB transfer to the pipe transfer list.
2517  *------------------------------------------------------------------------*/
2518 void
2519 usbd_pipe_start(struct usb_xfer_queue *pq)
2520 {
2521         struct usb_endpoint *ep;
2522         struct usb_xfer *xfer;
2523         uint8_t type;
2524
2525         xfer = pq->curr;
2526         ep = xfer->endpoint;
2527
2528         USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
2529
2530         /*
2531          * If the endpoint is already stalled we do nothing !
2532          */
2533         if (ep->is_stalled) {
2534                 return;
2535         }
2536         /*
2537          * Check if we are supposed to stall the endpoint:
2538          */
2539         if (xfer->flags.stall_pipe) {
2540                 struct usb_device *udev;
2541                 struct usb_xfer_root *info;
2542
2543                 /* clear stall command */
2544                 xfer->flags.stall_pipe = 0;
2545
2546                 /* get pointer to USB device */
2547                 info = xfer->xroot;
2548                 udev = info->udev;
2549
2550                 /*
2551                  * Only stall BULK and INTERRUPT endpoints.
2552                  */
2553                 type = (ep->edesc->bmAttributes & UE_XFERTYPE);
2554                 if ((type == UE_BULK) ||
2555                     (type == UE_INTERRUPT)) {
2556                         uint8_t did_stall;
2557
2558                         did_stall = 1;
2559
2560                         if (udev->flags.usb_mode == USB_MODE_DEVICE) {
2561                                 (udev->bus->methods->set_stall) (
2562                                     udev, NULL, ep, &did_stall);
2563                         } else if (udev->ctrl_xfer[1]) {
2564                                 info = udev->ctrl_xfer[1]->xroot;
2565                                 usb_proc_msignal(
2566                                     &info->bus->non_giant_callback_proc,
2567                                     &udev->cs_msg[0], &udev->cs_msg[1]);
2568                         } else {
2569                                 /* should not happen */
2570                                 DPRINTFN(0, "No stall handler\n");
2571                         }
2572                         /*
2573                          * Check if we should stall. Some USB hardware
2574                          * handles set- and clear-stall in hardware.
2575                          */
2576                         if (did_stall) {
2577                                 /*
2578                                  * The transfer will be continued when
2579                                  * the clear-stall control endpoint
2580                                  * message is received.
2581                                  */
2582                                 ep->is_stalled = 1;
2583                                 return;
2584                         }
2585                 } else if (type == UE_ISOCHRONOUS) {
2586
2587                         /*
2588                          * Make sure any FIFO overflow or other FIFO
2589                          * error conditions go away by resetting the
2590                          * endpoint FIFO through the clear stall
2591                          * method.
2592                          */
2593                         if (udev->flags.usb_mode == USB_MODE_DEVICE) {
2594                                 (udev->bus->methods->clear_stall) (udev, ep);
2595                         }
2596                 }
2597         }
2598         /* Set or clear stall complete - special case */
2599         if (xfer->nframes == 0) {
2600                 /* we are complete */
2601                 xfer->aframes = 0;
2602                 usbd_transfer_done(xfer, 0);
2603                 return;
2604         }
2605         /*
2606          * Handled cases:
2607          *
2608          * 1) Start the first transfer queued.
2609          *
2610          * 2) Re-start the current USB transfer.
2611          */
2612         /*
2613          * Check if there should be any
2614          * pre transfer start delay:
2615          */
2616         if (xfer->interval > 0) {
2617                 type = (ep->edesc->bmAttributes & UE_XFERTYPE);
2618                 if ((type == UE_BULK) ||
2619                     (type == UE_CONTROL)) {
2620                         usbd_transfer_timeout_ms(xfer,
2621                             &usbd_transfer_start_cb,
2622                             xfer->interval);
2623                         return;
2624                 }
2625         }
2626         DPRINTF("start\n");
2627
2628 #if USB_HAVE_PF
2629         usbpf_xfertap(xfer, USBPF_XFERTAP_SUBMIT);
2630 #endif
2631         /* the transfer can now be cancelled */
2632         xfer->flags_int.can_cancel_immed = 1;
2633
2634         /* start USB transfer, if no error */
2635         if (xfer->error == 0)
2636                 (ep->methods->start) (xfer);
2637
2638         /* check for transfer error */
2639         if (xfer->error) {
2640                 /* some error has happened */
2641                 usbd_transfer_done(xfer, 0);
2642         }
2643 }
2644
2645 /*------------------------------------------------------------------------*
2646  *      usbd_transfer_timeout_ms
2647  *
2648  * This function is used to setup a timeout on the given USB
2649  * transfer. If the timeout has been deferred the callback given by
2650  * "cb" will get called after "ms" milliseconds.
2651  *------------------------------------------------------------------------*/
2652 void
2653 usbd_transfer_timeout_ms(struct usb_xfer *xfer,
2654     void (*cb) (void *arg), usb_timeout_t ms)
2655 {
2656         USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
2657
2658         /* defer delay */
2659         usb_callout_reset(&xfer->timeout_handle,
2660             USB_MS_TO_TICKS(ms) + USB_CALLOUT_ZERO_TICKS, cb, xfer);
2661 }
2662
2663 /*------------------------------------------------------------------------*
2664  *      usbd_callback_wrapper_sub
2665  *
2666  *  - This function will update variables in an USB transfer after
2667  *  that the USB transfer is complete.
2668  *
2669  *  - This function is used to start the next USB transfer on the
2670  *  ep transfer queue, if any.
2671  *
2672  * NOTE: In some special cases the USB transfer will not be removed from
2673  * the pipe queue, but remain first. To enforce USB transfer removal call
2674  * this function passing the error code "USB_ERR_CANCELLED".
2675  *
2676  * Return values:
2677  * 0: Success.
2678  * Else: The callback has been deferred.
2679  *------------------------------------------------------------------------*/
2680 static uint8_t
2681 usbd_callback_wrapper_sub(struct usb_xfer *xfer)
2682 {
2683         struct usb_endpoint *ep;
2684         struct usb_bus *bus;
2685         usb_frcount_t x;
2686
2687         bus = xfer->xroot->bus;
2688
2689         if ((!xfer->flags_int.open) &&
2690             (!xfer->flags_int.did_close)) {
2691                 DPRINTF("close\n");
2692                 USB_BUS_LOCK(bus);
2693                 (xfer->endpoint->methods->close) (xfer);
2694                 USB_BUS_UNLOCK(bus);
2695                 /* only close once */
2696                 xfer->flags_int.did_close = 1;
2697                 return (1);             /* wait for new callback */
2698         }
2699         /*
2700          * If we have a non-hardware induced error we
2701          * need to do the DMA delay!
2702          */
2703         if (xfer->error != 0 && !xfer->flags_int.did_dma_delay &&
2704             (xfer->error == USB_ERR_CANCELLED ||
2705             xfer->error == USB_ERR_TIMEOUT ||
2706             bus->methods->start_dma_delay != NULL)) {
2707
2708                 usb_timeout_t temp;
2709
2710                 /* only delay once */
2711                 xfer->flags_int.did_dma_delay = 1;
2712
2713                 /* we can not cancel this delay */
2714                 xfer->flags_int.can_cancel_immed = 0;
2715
2716                 temp = usbd_get_dma_delay(xfer->xroot->udev);
2717
2718                 DPRINTFN(3, "DMA delay, %u ms, "
2719                     "on %p\n", temp, xfer);
2720
2721                 if (temp != 0) {
2722                         USB_BUS_LOCK(bus);
2723                         /*
2724                          * Some hardware solutions have dedicated
2725                          * events when it is safe to free DMA'ed
2726                          * memory. For the other hardware platforms we
2727                          * use a static delay.
2728                          */
2729                         if (bus->methods->start_dma_delay != NULL) {
2730                                 (bus->methods->start_dma_delay) (xfer);
2731                         } else {
2732                                 usbd_transfer_timeout_ms(xfer,
2733                                     (void (*)(void *))&usb_dma_delay_done_cb,
2734                                     temp);
2735                         }
2736                         USB_BUS_UNLOCK(bus);
2737                         return (1);     /* wait for new callback */
2738                 }
2739         }
2740         /* check actual number of frames */
2741         if (xfer->aframes > xfer->nframes) {
2742                 if (xfer->error == 0) {
2743                         panic("%s: actual number of frames, %d, is "
2744                             "greater than initial number of frames, %d\n",
2745                             __FUNCTION__, xfer->aframes, xfer->nframes);
2746                 } else {
2747                         /* just set some valid value */
2748                         xfer->aframes = xfer->nframes;
2749                 }
2750         }
2751         /* compute actual length */
2752         xfer->actlen = 0;
2753
2754         for (x = 0; x != xfer->aframes; x++) {
2755                 xfer->actlen += xfer->frlengths[x];
2756         }
2757
2758         /*
2759          * Frames that were not transferred get zero actual length in
2760          * case the USB device driver does not check the actual number
2761          * of frames transferred, "xfer->aframes":
2762          */
2763         for (; x < xfer->nframes; x++) {
2764                 usbd_xfer_set_frame_len(xfer, x, 0);
2765         }
2766
2767         /* check actual length */
2768         if (xfer->actlen > xfer->sumlen) {
2769                 if (xfer->error == 0) {
2770                         panic("%s: actual length, %d, is greater than "
2771                             "initial length, %d\n",
2772                             __FUNCTION__, xfer->actlen, xfer->sumlen);
2773                 } else {
2774                         /* just set some valid value */
2775                         xfer->actlen = xfer->sumlen;
2776                 }
2777         }
2778         DPRINTFN(1, "xfer=%p endpoint=%p sts=%d alen=%d, slen=%d, afrm=%d, nfrm=%d\n",
2779             xfer, xfer->endpoint, xfer->error, xfer->actlen, xfer->sumlen,
2780             xfer->aframes, xfer->nframes);
2781
2782         if (xfer->error) {
2783                 /* end of control transfer, if any */
2784                 xfer->flags_int.control_act = 0;
2785
2786 #if USB_HAVE_TT_SUPPORT
2787                 switch (xfer->error) {
2788                 case USB_ERR_NORMAL_COMPLETION:
2789                 case USB_ERR_SHORT_XFER:
2790                 case USB_ERR_STALLED:
2791                 case USB_ERR_CANCELLED:
2792                         /* nothing to do */
2793                         break;
2794                 default:
2795                         /* try to reset the TT, if any */
2796                         USB_BUS_LOCK(bus);
2797                         uhub_tt_buffer_reset_async_locked(xfer->xroot->udev, xfer->endpoint);
2798                         USB_BUS_UNLOCK(bus);
2799                         break;
2800                 }
2801 #endif
2802                 /* check if we should block the execution queue */
2803                 if ((xfer->error != USB_ERR_CANCELLED) &&
2804                     (xfer->flags.pipe_bof)) {
2805                         DPRINTFN(2, "xfer=%p: Block On Failure "
2806                             "on endpoint=%p\n", xfer, xfer->endpoint);
2807                         goto done;
2808                 }
2809         } else {
2810                 /* check for short transfers */
2811                 if (xfer->actlen < xfer->sumlen) {
2812
2813                         /* end of control transfer, if any */
2814                         xfer->flags_int.control_act = 0;
2815
2816                         if (!xfer->flags_int.short_xfer_ok) {
2817                                 xfer->error = USB_ERR_SHORT_XFER;
2818                                 if (xfer->flags.pipe_bof) {
2819                                         DPRINTFN(2, "xfer=%p: Block On Failure on "
2820                                             "Short Transfer on endpoint %p.\n",
2821                                             xfer, xfer->endpoint);
2822                                         goto done;
2823                                 }
2824                         }
2825                 } else {
2826                         /*
2827                          * Check if we are in the middle of a
2828                          * control transfer:
2829                          */
2830                         if (xfer->flags_int.control_act) {
2831                                 DPRINTFN(5, "xfer=%p: Control transfer "
2832                                     "active on endpoint=%p\n", xfer, xfer->endpoint);
2833                                 goto done;
2834                         }
2835                 }
2836         }
2837
2838         ep = xfer->endpoint;
2839
2840         /*
2841          * If the current USB transfer is completing we need to start the
2842          * next one:
2843          */
2844         USB_BUS_LOCK(bus);
2845         if (ep->endpoint_q.curr == xfer) {
2846                 usb_command_wrapper(&ep->endpoint_q, NULL);
2847
2848                 if (ep->endpoint_q.curr || TAILQ_FIRST(&ep->endpoint_q.head)) {
2849                         /* there is another USB transfer waiting */
2850                 } else {
2851                         /* this is the last USB transfer */
2852                         /* clear isochronous sync flag */
2853                         xfer->endpoint->is_synced = 0;
2854                 }
2855         }
2856         USB_BUS_UNLOCK(bus);
2857 done:
2858         return (0);
2859 }
2860
2861 /*------------------------------------------------------------------------*
2862  *      usb_command_wrapper
2863  *
2864  * This function is used to execute commands non-recursivly on an USB
2865  * transfer.
2866  *------------------------------------------------------------------------*/
2867 void
2868 usb_command_wrapper(struct usb_xfer_queue *pq, struct usb_xfer *xfer)
2869 {
2870         if (xfer) {
2871                 /*
2872                  * If the transfer is not already processing,
2873                  * queue it!
2874                  */
2875                 if (pq->curr != xfer) {
2876                         usbd_transfer_enqueue(pq, xfer);
2877                         if (pq->curr != NULL) {
2878                                 /* something is already processing */
2879                                 DPRINTFN(6, "busy %p\n", pq->curr);
2880                                 return;
2881                         }
2882                 }
2883         } else {
2884                 /* Get next element in queue */
2885                 pq->curr = NULL;
2886         }
2887
2888         if (!pq->recurse_1) {
2889
2890                 do {
2891
2892                         /* set both recurse flags */
2893                         pq->recurse_1 = 1;
2894                         pq->recurse_2 = 1;
2895
2896                         if (pq->curr == NULL) {
2897                                 xfer = TAILQ_FIRST(&pq->head);
2898                                 if (xfer) {
2899                                         TAILQ_REMOVE(&pq->head, xfer,
2900                                             wait_entry);
2901                                         xfer->wait_queue = NULL;
2902                                         pq->curr = xfer;
2903                                 } else {
2904                                         break;
2905                                 }
2906                         }
2907                         DPRINTFN(6, "cb %p (enter)\n", pq->curr);
2908                         (pq->command) (pq);
2909                         DPRINTFN(6, "cb %p (leave)\n", pq->curr);
2910
2911                 } while (!pq->recurse_2);
2912
2913                 /* clear first recurse flag */
2914                 pq->recurse_1 = 0;
2915
2916         } else {
2917                 /* clear second recurse flag */
2918                 pq->recurse_2 = 0;
2919         }
2920 }
2921
2922 /*------------------------------------------------------------------------*
2923  *      usbd_ctrl_transfer_setup
2924  *
2925  * This function is used to setup the default USB control endpoint
2926  * transfer.
2927  *------------------------------------------------------------------------*/
2928 void
2929 usbd_ctrl_transfer_setup(struct usb_device *udev)
2930 {
2931         struct usb_xfer *xfer;
2932         uint8_t no_resetup;
2933         uint8_t iface_index;
2934
2935         /* check for root HUB */
2936         if (udev->parent_hub == NULL)
2937                 return;
2938 repeat:
2939
2940         xfer = udev->ctrl_xfer[0];
2941         if (xfer) {
2942                 USB_XFER_LOCK(xfer);
2943                 no_resetup =
2944                     ((xfer->address == udev->address) &&
2945                     (udev->ctrl_ep_desc.wMaxPacketSize[0] ==
2946                     udev->ddesc.bMaxPacketSize));
2947                 if (udev->flags.usb_mode == USB_MODE_DEVICE) {
2948                         if (no_resetup) {
2949                                 /*
2950                                  * NOTE: checking "xfer->address" and
2951                                  * starting the USB transfer must be
2952                                  * atomic!
2953                                  */
2954                                 usbd_transfer_start(xfer);
2955                         }
2956                 }
2957                 USB_XFER_UNLOCK(xfer);
2958         } else {
2959                 no_resetup = 0;
2960         }
2961
2962         if (no_resetup) {
2963                 /*
2964                  * All parameters are exactly the same like before.
2965                  * Just return.
2966                  */
2967                 return;
2968         }
2969         /*
2970          * Update wMaxPacketSize for the default control endpoint:
2971          */
2972         udev->ctrl_ep_desc.wMaxPacketSize[0] =
2973             udev->ddesc.bMaxPacketSize;
2974
2975         /*
2976          * Unsetup any existing USB transfer:
2977          */
2978         usbd_transfer_unsetup(udev->ctrl_xfer, USB_CTRL_XFER_MAX);
2979
2980         /*
2981          * Reset clear stall error counter.
2982          */
2983         udev->clear_stall_errors = 0;
2984
2985         /*
2986          * Try to setup a new USB transfer for the
2987          * default control endpoint:
2988          */
2989         iface_index = 0;
2990         if (usbd_transfer_setup(udev, &iface_index,
2991             udev->ctrl_xfer, usb_control_ep_cfg, USB_CTRL_XFER_MAX, NULL,
2992             &udev->device_mtx)) {
2993                 DPRINTFN(0, "could not setup default "
2994                     "USB transfer\n");
2995         } else {
2996                 goto repeat;
2997         }
2998 }
2999
3000 /*------------------------------------------------------------------------*
3001  *      usbd_clear_data_toggle - factored out code
3002  *
3003  * NOTE: the intention of this function is not to reset the hardware
3004  * data toggle.
3005  *------------------------------------------------------------------------*/
3006 void
3007 usbd_clear_stall_locked(struct usb_device *udev, struct usb_endpoint *ep)
3008 {
3009         USB_BUS_LOCK_ASSERT(udev->bus, MA_OWNED);
3010
3011         /* check that we have a valid case */
3012         if (udev->flags.usb_mode == USB_MODE_HOST &&
3013             udev->parent_hub != NULL &&
3014             udev->bus->methods->clear_stall != NULL &&
3015             ep->methods != NULL) {
3016                 (udev->bus->methods->clear_stall) (udev, ep);
3017         }
3018 }
3019
3020 /*------------------------------------------------------------------------*
3021  *      usbd_clear_data_toggle - factored out code
3022  *
3023  * NOTE: the intention of this function is not to reset the hardware
3024  * data toggle on the USB device side.
3025  *------------------------------------------------------------------------*/
3026 void
3027 usbd_clear_data_toggle(struct usb_device *udev, struct usb_endpoint *ep)
3028 {
3029         DPRINTFN(5, "udev=%p endpoint=%p\n", udev, ep);
3030
3031         USB_BUS_LOCK(udev->bus);
3032         ep->toggle_next = 0;
3033         /* some hardware needs a callback to clear the data toggle */
3034         usbd_clear_stall_locked(udev, ep);
3035         USB_BUS_UNLOCK(udev->bus);
3036 }
3037
3038 /*------------------------------------------------------------------------*
3039  *      usbd_clear_stall_callback - factored out clear stall callback
3040  *
3041  * Input parameters:
3042  *  xfer1: Clear Stall Control Transfer
3043  *  xfer2: Stalled USB Transfer
3044  *
3045  * This function is NULL safe.
3046  *
3047  * Return values:
3048  *   0: In progress
3049  *   Else: Finished
3050  *
3051  * Clear stall config example:
3052  *
3053  * static const struct usb_config my_clearstall =  {
3054  *      .type = UE_CONTROL,
3055  *      .endpoint = 0,
3056  *      .direction = UE_DIR_ANY,
3057  *      .interval = 50, //50 milliseconds
3058  *      .bufsize = sizeof(struct usb_device_request),
3059  *      .timeout = 1000, //1.000 seconds
3060  *      .callback = &my_clear_stall_callback, // **
3061  *      .usb_mode = USB_MODE_HOST,
3062  * };
3063  *
3064  * ** "my_clear_stall_callback" calls "usbd_clear_stall_callback"
3065  * passing the correct parameters.
3066  *------------------------------------------------------------------------*/
3067 uint8_t
3068 usbd_clear_stall_callback(struct usb_xfer *xfer1,
3069     struct usb_xfer *xfer2)
3070 {
3071         struct usb_device_request req;
3072
3073         if (xfer2 == NULL) {
3074                 /* looks like we are tearing down */
3075                 DPRINTF("NULL input parameter\n");
3076                 return (0);
3077         }
3078         USB_XFER_LOCK_ASSERT(xfer1, MA_OWNED);
3079         USB_XFER_LOCK_ASSERT(xfer2, MA_OWNED);
3080
3081         switch (USB_GET_STATE(xfer1)) {
3082         case USB_ST_SETUP:
3083
3084                 /*
3085                  * pre-clear the data toggle to DATA0 ("umass.c" and
3086                  * "ata-usb.c" depends on this)
3087                  */
3088
3089                 usbd_clear_data_toggle(xfer2->xroot->udev, xfer2->endpoint);
3090
3091                 /* setup a clear-stall packet */
3092
3093                 req.bmRequestType = UT_WRITE_ENDPOINT;
3094                 req.bRequest = UR_CLEAR_FEATURE;
3095                 USETW(req.wValue, UF_ENDPOINT_HALT);
3096                 req.wIndex[0] = xfer2->endpoint->edesc->bEndpointAddress;
3097                 req.wIndex[1] = 0;
3098                 USETW(req.wLength, 0);
3099
3100                 /*
3101                  * "usbd_transfer_setup_sub()" will ensure that
3102                  * we have sufficient room in the buffer for
3103                  * the request structure!
3104                  */
3105
3106                 /* copy in the transfer */
3107
3108                 usbd_copy_in(xfer1->frbuffers, 0, &req, sizeof(req));
3109
3110                 /* set length */
3111                 xfer1->frlengths[0] = sizeof(req);
3112                 xfer1->nframes = 1;
3113
3114                 usbd_transfer_submit(xfer1);
3115                 return (0);
3116
3117         case USB_ST_TRANSFERRED:
3118                 break;
3119
3120         default:                        /* Error */
3121                 if (xfer1->error == USB_ERR_CANCELLED) {
3122                         return (0);
3123                 }
3124                 break;
3125         }
3126         return (1);                     /* Clear Stall Finished */
3127 }
3128
3129 /*------------------------------------------------------------------------*
3130  *      usbd_transfer_poll
3131  *
3132  * The following function gets called from the USB keyboard driver and
3133  * UMASS when the system has paniced.
3134  *
3135  * NOTE: It is currently not possible to resume normal operation on
3136  * the USB controller which has been polled, due to clearing of the
3137  * "up_dsleep" and "up_msleep" flags.
3138  *------------------------------------------------------------------------*/
3139 void
3140 usbd_transfer_poll(struct usb_xfer **ppxfer, uint16_t max)
3141 {
3142         struct usb_xfer *xfer;
3143         struct usb_xfer_root *xroot;
3144         struct usb_device *udev;
3145         struct usb_proc_msg *pm;
3146         uint16_t n;
3147         uint16_t drop_bus;
3148         uint16_t drop_xfer;
3149
3150         for (n = 0; n != max; n++) {
3151                 /* Extra checks to avoid panic */
3152                 xfer = ppxfer[n];
3153                 if (xfer == NULL)
3154                         continue;       /* no USB transfer */
3155                 xroot = xfer->xroot;
3156                 if (xroot == NULL)
3157                         continue;       /* no USB root */
3158                 udev = xroot->udev;
3159                 if (udev == NULL)
3160                         continue;       /* no USB device */
3161                 if (udev->bus == NULL)
3162                         continue;       /* no BUS structure */
3163                 if (udev->bus->methods == NULL)
3164                         continue;       /* no BUS methods */
3165                 if (udev->bus->methods->xfer_poll == NULL)
3166                         continue;       /* no poll method */
3167
3168                 /* make sure that the BUS mutex is not locked */
3169                 drop_bus = 0;
3170                 while (mtx_owned(&xroot->udev->bus->bus_mtx) && !SCHEDULER_STOPPED()) {
3171                         mtx_unlock(&xroot->udev->bus->bus_mtx);
3172                         drop_bus++;
3173                 }
3174
3175                 /* make sure that the transfer mutex is not locked */
3176                 drop_xfer = 0;
3177                 while (mtx_owned(xroot->xfer_mtx) && !SCHEDULER_STOPPED()) {
3178                         mtx_unlock(xroot->xfer_mtx);
3179                         drop_xfer++;
3180                 }
3181
3182                 /* Make sure cv_signal() and cv_broadcast() is not called */
3183                 udev->bus->control_xfer_proc.up_msleep = 0;
3184                 udev->bus->explore_proc.up_msleep = 0;
3185                 udev->bus->giant_callback_proc.up_msleep = 0;
3186                 udev->bus->non_giant_callback_proc.up_msleep = 0;
3187
3188                 /* poll USB hardware */
3189                 (udev->bus->methods->xfer_poll) (udev->bus);
3190
3191                 USB_BUS_LOCK(xroot->bus);
3192
3193                 /* check for clear stall */
3194                 if (udev->ctrl_xfer[1] != NULL) {
3195
3196                         /* poll clear stall start */
3197                         pm = &udev->cs_msg[0].hdr;
3198                         (pm->pm_callback) (pm);
3199                         /* poll clear stall done thread */
3200                         pm = &udev->ctrl_xfer[1]->
3201                             xroot->done_m[0].hdr;
3202                         (pm->pm_callback) (pm);
3203                 }
3204
3205                 /* poll done thread */
3206                 pm = &xroot->done_m[0].hdr;
3207                 (pm->pm_callback) (pm);
3208
3209                 USB_BUS_UNLOCK(xroot->bus);
3210
3211                 /* restore transfer mutex */
3212                 while (drop_xfer--)
3213                         mtx_lock(xroot->xfer_mtx);
3214
3215                 /* restore BUS mutex */
3216                 while (drop_bus--)
3217                         mtx_lock(&xroot->udev->bus->bus_mtx);
3218         }
3219 }
3220
3221 static void
3222 usbd_get_std_packet_size(struct usb_std_packet_size *ptr,
3223     uint8_t type, enum usb_dev_speed speed)
3224 {
3225         static const uint16_t intr_range_max[USB_SPEED_MAX] = {
3226                 [USB_SPEED_LOW] = 8,
3227                 [USB_SPEED_FULL] = 64,
3228                 [USB_SPEED_HIGH] = 1024,
3229                 [USB_SPEED_VARIABLE] = 1024,
3230                 [USB_SPEED_SUPER] = 1024,
3231         };
3232
3233         static const uint16_t isoc_range_max[USB_SPEED_MAX] = {
3234                 [USB_SPEED_LOW] = 0,    /* invalid */
3235                 [USB_SPEED_FULL] = 1023,
3236                 [USB_SPEED_HIGH] = 1024,
3237                 [USB_SPEED_VARIABLE] = 3584,
3238                 [USB_SPEED_SUPER] = 1024,
3239         };
3240
3241         static const uint16_t control_min[USB_SPEED_MAX] = {
3242                 [USB_SPEED_LOW] = 8,
3243                 [USB_SPEED_FULL] = 8,
3244                 [USB_SPEED_HIGH] = 64,
3245                 [USB_SPEED_VARIABLE] = 512,
3246                 [USB_SPEED_SUPER] = 512,
3247         };
3248
3249         static const uint16_t bulk_min[USB_SPEED_MAX] = {
3250                 [USB_SPEED_LOW] = 8,
3251                 [USB_SPEED_FULL] = 8,
3252                 [USB_SPEED_HIGH] = 512,
3253                 [USB_SPEED_VARIABLE] = 512,
3254                 [USB_SPEED_SUPER] = 1024,
3255         };
3256
3257         uint16_t temp;
3258
3259         memset(ptr, 0, sizeof(*ptr));
3260
3261         switch (type) {
3262         case UE_INTERRUPT:
3263                 ptr->range.max = intr_range_max[speed];
3264                 break;
3265         case UE_ISOCHRONOUS:
3266                 ptr->range.max = isoc_range_max[speed];
3267                 break;
3268         default:
3269                 if (type == UE_BULK)
3270                         temp = bulk_min[speed];
3271                 else /* UE_CONTROL */
3272                         temp = control_min[speed];
3273
3274                 /* default is fixed */
3275                 ptr->fixed[0] = temp;
3276                 ptr->fixed[1] = temp;
3277                 ptr->fixed[2] = temp;
3278                 ptr->fixed[3] = temp;
3279
3280                 if (speed == USB_SPEED_FULL) {
3281                         /* multiple sizes */
3282                         ptr->fixed[1] = 16;
3283                         ptr->fixed[2] = 32;
3284                         ptr->fixed[3] = 64;
3285                 }
3286                 if ((speed == USB_SPEED_VARIABLE) &&
3287                     (type == UE_BULK)) {
3288                         /* multiple sizes */
3289                         ptr->fixed[2] = 1024;
3290                         ptr->fixed[3] = 1536;
3291                 }
3292                 break;
3293         }
3294 }
3295
3296 void    *
3297 usbd_xfer_softc(struct usb_xfer *xfer)
3298 {
3299         return (xfer->priv_sc);
3300 }
3301
3302 void *
3303 usbd_xfer_get_priv(struct usb_xfer *xfer)
3304 {
3305         return (xfer->priv_fifo);
3306 }
3307
3308 void
3309 usbd_xfer_set_priv(struct usb_xfer *xfer, void *ptr)
3310 {
3311         xfer->priv_fifo = ptr;
3312 }
3313
3314 uint8_t
3315 usbd_xfer_state(struct usb_xfer *xfer)
3316 {
3317         return (xfer->usb_state);
3318 }
3319
3320 void
3321 usbd_xfer_set_flag(struct usb_xfer *xfer, int flag)
3322 {
3323         switch (flag) {
3324                 case USB_FORCE_SHORT_XFER:
3325                         xfer->flags.force_short_xfer = 1;
3326                         break;
3327                 case USB_SHORT_XFER_OK:
3328                         xfer->flags.short_xfer_ok = 1;
3329                         break;
3330                 case USB_MULTI_SHORT_OK:
3331                         xfer->flags.short_frames_ok = 1;
3332                         break;
3333                 case USB_MANUAL_STATUS:
3334                         xfer->flags.manual_status = 1;
3335                         break;
3336         }
3337 }
3338
3339 void
3340 usbd_xfer_clr_flag(struct usb_xfer *xfer, int flag)
3341 {
3342         switch (flag) {
3343                 case USB_FORCE_SHORT_XFER:
3344                         xfer->flags.force_short_xfer = 0;
3345                         break;
3346                 case USB_SHORT_XFER_OK:
3347                         xfer->flags.short_xfer_ok = 0;
3348                         break;
3349                 case USB_MULTI_SHORT_OK:
3350                         xfer->flags.short_frames_ok = 0;
3351                         break;
3352                 case USB_MANUAL_STATUS:
3353                         xfer->flags.manual_status = 0;
3354                         break;
3355         }
3356 }
3357
3358 /*
3359  * The following function returns in milliseconds when the isochronous
3360  * transfer was completed by the hardware. The returned value wraps
3361  * around 65536 milliseconds.
3362  */
3363 uint16_t
3364 usbd_xfer_get_timestamp(struct usb_xfer *xfer)
3365 {
3366         return (xfer->isoc_time_complete);
3367 }