sys/dev/usb/usb_transfer.c

   1 /* $FreeBSD$ */
   2 /*-
   3  * Copyright (c) 2008 Hans Petter Selasky. All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  */
  26
  27 #include <sys/stdint.h>
  28 #include <sys/stddef.h>
  29 #include <sys/param.h>
  30 #include <sys/queue.h>
  31 #include <sys/types.h>
  32 #include <sys/systm.h>
  33 #include <sys/kernel.h>
  34 #include <sys/bus.h>
  35 #include <sys/module.h>
  36 #include <sys/lock.h>
  37 #include <sys/mutex.h>
  38 #include <sys/condvar.h>
  39 #include <sys/sysctl.h>
  40 #include <sys/sx.h>
  41 #include <sys/unistd.h>
  42 #include <sys/callout.h>
  43 #include <sys/malloc.h>
  44 #include <sys/priv.h>
  45 #include <sys/proc.h>
  46
  47 #include <dev/usb/usb.h>
  48 #include <dev/usb/usbdi.h>
  49 #include <dev/usb/usbdi_util.h>
  50
  51 #define USB_DEBUG_VAR usb_debug
  52
  53 #include <dev/usb/usb_core.h>
  54 #include <dev/usb/usb_busdma.h>
  55 #include <dev/usb/usb_process.h>
  56 #include <dev/usb/usb_transfer.h>
  57 #include <dev/usb/usb_device.h>
  58 #include <dev/usb/usb_debug.h>
  59 #include <dev/usb/usb_util.h>
  60
  61 #include <dev/usb/usb_controller.h>
  62 #include <dev/usb/usb_bus.h>
  63 #include <dev/usb/usb_pf.h>
  64
  65 struct usb_std_packet_size {
  66         struct {
  67                 uint16_t min;           /* inclusive */
  68                 uint16_t max;           /* inclusive */
  69         }       range;
  70
  71         uint16_t fixed[4];
  72 };
  73
  74 static usb_callback_t usb_request_callback;
  75
  76 static const struct usb_config usb_control_ep_cfg[USB_CTRL_XFER_MAX] = {
  77
  78         /* This transfer is used for generic control endpoint transfers */
  79
  80         [0] = {
  81                 .type = UE_CONTROL,
  82                 .endpoint = 0x00,       /* Control endpoint */
  83                 .direction = UE_DIR_ANY,
  84                 .bufsize = USB_EP0_BUFSIZE,     /* bytes */
  85                 .flags = {.proxy_buffer = 1,},
  86                 .callback = &usb_request_callback,
  87                 .usb_mode = USB_MODE_DUAL,      /* both modes */
  88         },
  89
  90         /* This transfer is used for generic clear stall only */
  91
  92         [1] = {
  93                 .type = UE_CONTROL,
  94                 .endpoint = 0x00,       /* Control pipe */
  95                 .direction = UE_DIR_ANY,
  96                 .bufsize = sizeof(struct usb_device_request),
  97                 .callback = &usb_do_clear_stall_callback,
  98                 .timeout = 1000,        /* 1 second */
  99                 .interval = 50, /* 50ms */
 100                 .usb_mode = USB_MODE_HOST,
 101         },
 102 };
 103
 104 /* function prototypes */
 105
 106 static void     usbd_update_max_frame_size(struct usb_xfer *);
 107 static void     usbd_transfer_unsetup_sub(struct usb_xfer_root *, uint8_t);
 108 static void     usbd_control_transfer_init(struct usb_xfer *);
 109 static int      usbd_setup_ctrl_transfer(struct usb_xfer *);
 110 static void     usb_callback_proc(struct usb_proc_msg *);
 111 static void     usbd_callback_ss_done_defer(struct usb_xfer *);
 112 static void     usbd_callback_wrapper(struct usb_xfer_queue *);
 113 static void     usbd_transfer_start_cb(void *);
 114 static uint8_t  usbd_callback_wrapper_sub(struct usb_xfer *);
 115 static void     usbd_get_std_packet_size(struct usb_std_packet_size *ptr,
 116                     uint8_t type, enum usb_dev_speed speed);
 117
 118 /*------------------------------------------------------------------------*
 119  *      usb_request_callback
 120  *------------------------------------------------------------------------*/
 121 static void
 122 usb_request_callback(struct usb_xfer *xfer, usb_error_t error)
 123 {
 124         if (xfer->flags_int.usb_mode == USB_MODE_DEVICE)
 125                 usb_handle_request_callback(xfer, error);
 126         else
 127                 usbd_do_request_callback(xfer, error);
 128 }
 129
 130 /*------------------------------------------------------------------------*
 131  *      usbd_update_max_frame_size
 132  *
 133  * This function updates the maximum frame size, hence high speed USB
 134  * can transfer multiple consecutive packets.
 135  *------------------------------------------------------------------------*/
 136 static void
 137 usbd_update_max_frame_size(struct usb_xfer *xfer)
 138 {
 139         /* compute maximum frame size */
 140         /* this computation should not overflow 16-bit */
 141         /* max = 15 * 1024 */
 142
 143         xfer->max_frame_size = xfer->max_packet_size * xfer->max_packet_count;
 144 }
 145
 146 /*------------------------------------------------------------------------*
 147  *      usbd_get_dma_delay
 148  *
 149  * The following function is called when we need to
 150  * synchronize with DMA hardware.
 151  *
 152  * Returns:
 153  *    0: no DMA delay required
 154  * Else: milliseconds of DMA delay
 155  *------------------------------------------------------------------------*/
 156 usb_timeout_t
 157 usbd_get_dma_delay(struct usb_device *udev)
 158 {
 159         struct usb_bus_methods *mtod;
 160         uint32_t temp;
 161
 162         mtod = udev->bus->methods;
 163         temp = 0;
 164
 165         if (mtod->get_dma_delay) {
 166                 (mtod->get_dma_delay) (udev, &temp);
 167                 /*
 168                  * Round up and convert to milliseconds. Note that we use
 169                  * 1024 milliseconds per second. to save a division.
 170                  */
 171                 temp += 0x3FF;
 172                 temp /= 0x400;
 173         }
 174         return (temp);
 175 }
 176
 177 /*------------------------------------------------------------------------*
 178  *      usbd_transfer_setup_sub_malloc
 179  *
 180  * This function will allocate one or more DMA'able memory chunks
 181  * according to "size", "align" and "count" arguments. "ppc" is
 182  * pointed to a linear array of USB page caches afterwards.
 183  *
 184  * Returns:
 185  *    0: Success
 186  * Else: Failure
 187  *------------------------------------------------------------------------*/
 188 #if USB_HAVE_BUSDMA
 189 uint8_t
 190 usbd_transfer_setup_sub_malloc(struct usb_setup_params *parm,
 191     struct usb_page_cache **ppc, usb_size_t size, usb_size_t align,
 192     usb_size_t count)
 193 {
 194         struct usb_page_cache *pc;
 195         struct usb_page *pg;
 196         void *buf;
 197         usb_size_t n_dma_pc;
 198         usb_size_t n_obj;
 199         usb_size_t x;
 200         usb_size_t y;
 201         usb_size_t r;
 202         usb_size_t z;
 203
 204         USB_ASSERT(align > 1, ("Invalid alignment, 0x%08x\n",
 205             align));
 206         USB_ASSERT(size > 0, ("Invalid size = 0\n"));
 207
 208         if (count == 0) {
 209                 return (0);             /* nothing to allocate */
 210         }
 211         /*
 212          * Make sure that the size is aligned properly.
 213          */
 214         size = -((-size) & (-align));
 215
 216         /*
 217          * Try multi-allocation chunks to reduce the number of DMA
 218          * allocations, hence DMA allocations are slow.
 219          */
 220         if (size >= USB_PAGE_SIZE) {
 221                 n_dma_pc = count;
 222                 n_obj = 1;
 223         } else {
 224                 /* compute number of objects per page */
 225                 n_obj = (USB_PAGE_SIZE / size);
 226                 /*
 227                  * Compute number of DMA chunks, rounded up
 228                  * to nearest one:
 229                  */
 230                 n_dma_pc = ((count + n_obj - 1) / n_obj);
 231         }
 232
 233         if (parm->buf == NULL) {
 234                 /* for the future */
 235                 parm->dma_page_ptr += n_dma_pc;
 236                 parm->dma_page_cache_ptr += n_dma_pc;
 237                 parm->dma_page_ptr += count;
 238                 parm->xfer_page_cache_ptr += count;
 239                 return (0);
 240         }
 241         for (x = 0; x != n_dma_pc; x++) {
 242                 /* need to initialize the page cache */
 243                 parm->dma_page_cache_ptr[x].tag_parent =
 244                     &parm->curr_xfer->xroot->dma_parent_tag;
 245         }
 246         for (x = 0; x != count; x++) {
 247                 /* need to initialize the page cache */
 248                 parm->xfer_page_cache_ptr[x].tag_parent =
 249                     &parm->curr_xfer->xroot->dma_parent_tag;
 250         }
 251
 252         if (ppc) {
 253                 *ppc = parm->xfer_page_cache_ptr;
 254         }
 255         r = count;                      /* set remainder count */
 256         z = n_obj * size;               /* set allocation size */
 257         pc = parm->xfer_page_cache_ptr;
 258         pg = parm->dma_page_ptr;
 259
 260         for (x = 0; x != n_dma_pc; x++) {
 261
 262                 if (r < n_obj) {
 263                         /* compute last remainder */
 264                         z = r * size;
 265                         n_obj = r;
 266                 }
 267                 if (usb_pc_alloc_mem(parm->dma_page_cache_ptr,
 268                     pg, z, align)) {
 269                         return (1);     /* failure */
 270                 }
 271                 /* Set beginning of current buffer */
 272                 buf = parm->dma_page_cache_ptr->buffer;
 273                 /* Make room for one DMA page cache and one page */
 274                 parm->dma_page_cache_ptr++;
 275                 pg++;
 276
 277                 for (y = 0; (y != n_obj); y++, r--, pc++, pg++) {
 278
 279                         /* Load sub-chunk into DMA */
 280                         if (usb_pc_dmamap_create(pc, size)) {
 281                                 return (1);     /* failure */
 282                         }
 283                         pc->buffer = USB_ADD_BYTES(buf, y * size);
 284                         pc->page_start = pg;
 285
 286                         mtx_lock(pc->tag_parent->mtx);
 287                         if (usb_pc_load_mem(pc, size, 1 /* synchronous */ )) {
 288                                 mtx_unlock(pc->tag_parent->mtx);
 289                                 return (1);     /* failure */
 290                         }
 291                         mtx_unlock(pc->tag_parent->mtx);
 292                 }
 293         }
 294
 295         parm->xfer_page_cache_ptr = pc;
 296         parm->dma_page_ptr = pg;
 297         return (0);
 298 }
 299 #endif
 300
 301 /*------------------------------------------------------------------------*
 302  *      usbd_get_max_frame_length
 303  *
 304  * This function returns the maximum single frame length as computed by
 305  * usbd_transfer_setup(). It is useful when computing buffer sizes for
 306  * devices having multiple alternate settings. The SuperSpeed endpoint
 307  * companion pointer is allowed to be NULL.
 308  *------------------------------------------------------------------------*/
 309 uint32_t
 310 usbd_get_max_frame_length(const struct usb_endpoint_descriptor *edesc,
 311     const struct usb_endpoint_ss_comp_descriptor *ecomp,
 312     enum usb_dev_speed speed)
 313 {
 314         uint32_t max_packet_size;
 315         uint32_t max_packet_count;
 316         uint8_t type;
 317
 318         max_packet_size = UGETW(edesc->wMaxPacketSize);
 319         max_packet_count = 1;
 320         type = (edesc->bmAttributes & UE_XFERTYPE);
 321
 322         switch (speed) {
 323         case USB_SPEED_HIGH:
 324                 switch (type) {
 325                 case UE_ISOCHRONOUS:
 326                 case UE_INTERRUPT:
 327                         max_packet_count +=
 328                             (max_packet_size >> 11) & 3;
 329
 330                         /* check for invalid max packet count */
 331                         if (max_packet_count > 3)
 332                                 max_packet_count = 3;
 333                         break;
 334                 default:
 335                         break;
 336                 }
 337                 max_packet_size &= 0x7FF;
 338                 break;
 339         case USB_SPEED_SUPER:
 340                 max_packet_count += (max_packet_size >> 11) & 3;
 341
 342                 if (ecomp != NULL)
 343                         max_packet_count += ecomp->bMaxBurst;
 344
 345                 if ((max_packet_count == 0) ||
 346                     (max_packet_count > 16))
 347                         max_packet_count = 16;
 348
 349                 switch (type) {
 350                 case UE_CONTROL:
 351                         max_packet_count = 1;
 352                         break;
 353                 case UE_ISOCHRONOUS:
 354                         if (ecomp != NULL) {
 355                                 uint8_t mult;
 356
 357                                 mult = UE_GET_SS_ISO_MULT(
 358                                     ecomp->bmAttributes) + 1;
 359                                 if (mult > 3)
 360                                         mult = 3;
 361
 362                                 max_packet_count *= mult;
 363                         }
 364                         break;
 365                 default:
 366                         break;
 367                 }
 368                 max_packet_size &= 0x7FF;
 369                 break;
 370         default:
 371                 break;
 372         }
 373         return (max_packet_size * max_packet_count);
 374 }
 375
 376 /*------------------------------------------------------------------------*
 377  *      usbd_transfer_setup_sub - transfer setup subroutine
 378  *
 379  * This function must be called from the "xfer_setup" callback of the
 380  * USB Host or Device controller driver when setting up an USB
 381  * transfer. This function will setup correct packet sizes, buffer
 382  * sizes, flags and more, that are stored in the "usb_xfer"
 383  * structure.
 384  *------------------------------------------------------------------------*/
 385 void
 386 usbd_transfer_setup_sub(struct usb_setup_params *parm)
 387 {
 388         enum {
 389                 REQ_SIZE = 8,
 390                 MIN_PKT = 8,
 391         };
 392         struct usb_xfer *xfer = parm->curr_xfer;
 393         const struct usb_config *setup = parm->curr_setup;
 394         struct usb_endpoint_ss_comp_descriptor *ecomp;
 395         struct usb_endpoint_descriptor *edesc;
 396         struct usb_std_packet_size std_size;
 397         usb_frcount_t n_frlengths;
 398         usb_frcount_t n_frbuffers;
 399         usb_frcount_t x;
 400         uint16_t maxp_old;
 401         uint8_t type;
 402         uint8_t zmps;
 403
 404         /*
 405          * Sanity check. The following parameters must be initialized before
 406          * calling this function.
 407          */
 408         if ((parm->hc_max_packet_size == 0) ||
 409             (parm->hc_max_packet_count == 0) ||
 410             (parm->hc_max_frame_size == 0)) {
 411                 parm->err = USB_ERR_INVAL;
 412                 goto done;
 413         }
 414         edesc = xfer->endpoint->edesc;
 415         ecomp = xfer->endpoint->ecomp;
 416
 417         type = (edesc->bmAttributes & UE_XFERTYPE);
 418
 419         xfer->flags = setup->flags;
 420         xfer->nframes = setup->frames;
 421         xfer->timeout = setup->timeout;
 422         xfer->callback = setup->callback;
 423         xfer->interval = setup->interval;
 424         xfer->endpointno = edesc->bEndpointAddress;
 425         xfer->max_packet_size = UGETW(edesc->wMaxPacketSize);
 426         xfer->max_packet_count = 1;
 427         /* make a shadow copy: */
 428         xfer->flags_int.usb_mode = parm->udev->flags.usb_mode;
 429
 430         parm->bufsize = setup->bufsize;
 431
 432         switch (parm->speed) {
 433         case USB_SPEED_HIGH:
 434                 switch (type) {
 435                 case UE_ISOCHRONOUS:
 436                 case UE_INTERRUPT:
 437                         xfer->max_packet_count += (xfer->max_packet_size >> 11) & 3;
 438
 439                         /* check for invalid max packet count */
 440                         if (xfer->max_packet_count > 3)
 441                                 xfer->max_packet_count = 3;
 442                         break;
 443                 default:
 444                         break;
 445                 }
 446                 xfer->max_packet_size &= 0x7FF;
 447                 break;
 448         case USB_SPEED_SUPER:
 449                 xfer->max_packet_count += (xfer->max_packet_size >> 11) & 3;
 450
 451                 if (ecomp != NULL)
 452                         xfer->max_packet_count += ecomp->bMaxBurst;
 453
 454                 if ((xfer->max_packet_count == 0) ||
 455                     (xfer->max_packet_count > 16))
 456                         xfer->max_packet_count = 16;
 457
 458                 switch (type) {
 459                 case UE_CONTROL:
 460                         xfer->max_packet_count = 1;
 461                         break;
 462                 case UE_ISOCHRONOUS:
 463                         if (ecomp != NULL) {
 464                                 uint8_t mult;
 465
 466                                 mult = (ecomp->bmAttributes & 3) + 1;
 467                                 if (mult > 3)
 468                                         mult = 3;
 469
 470                                 xfer->max_packet_count *= mult;
 471                         }
 472                         break;
 473                 default:
 474                         break;
 475                 }
 476                 xfer->max_packet_size &= 0x7FF;
 477                 break;
 478         default:
 479                 break;
 480         }
 481         /* range check "max_packet_count" */
 482
 483         if (xfer->max_packet_count > parm->hc_max_packet_count) {
 484                 xfer->max_packet_count = parm->hc_max_packet_count;
 485         }
 486
 487         /* store max packet size value before filtering */
 488
 489         maxp_old = xfer->max_packet_size;
 490
 491         /* filter "wMaxPacketSize" according to HC capabilities */
 492
 493         if ((xfer->max_packet_size > parm->hc_max_packet_size) ||
 494             (xfer->max_packet_size == 0)) {
 495                 xfer->max_packet_size = parm->hc_max_packet_size;
 496         }
 497         /* filter "wMaxPacketSize" according to standard sizes */
 498
 499         usbd_get_std_packet_size(&std_size, type, parm->speed);
 500
 501         if (std_size.range.min || std_size.range.max) {
 502
 503                 if (xfer->max_packet_size < std_size.range.min) {
 504                         xfer->max_packet_size = std_size.range.min;
 505                 }
 506                 if (xfer->max_packet_size > std_size.range.max) {
 507                         xfer->max_packet_size = std_size.range.max;
 508                 }
 509         } else {
 510
 511                 if (xfer->max_packet_size >= std_size.fixed[3]) {
 512                         xfer->max_packet_size = std_size.fixed[3];
 513                 } else if (xfer->max_packet_size >= std_size.fixed[2]) {
 514                         xfer->max_packet_size = std_size.fixed[2];
 515                 } else if (xfer->max_packet_size >= std_size.fixed[1]) {
 516                         xfer->max_packet_size = std_size.fixed[1];
 517                 } else {
 518                         /* only one possibility left */
 519                         xfer->max_packet_size = std_size.fixed[0];
 520                 }
 521         }
 522
 523         /*
 524          * Check if the max packet size was outside its allowed range
 525          * and clamped to a valid value:
 526          */
 527         if (maxp_old != xfer->max_packet_size)
 528                 xfer->flags_int.maxp_was_clamped = 1;
 529
 530         /* compute "max_frame_size" */
 531
 532         usbd_update_max_frame_size(xfer);
 533
 534         /* check interrupt interval and transfer pre-delay */
 535
 536         if (type == UE_ISOCHRONOUS) {
 537
 538                 uint16_t frame_limit;
 539
 540                 xfer->interval = 0;     /* not used, must be zero */
 541                 xfer->flags_int.isochronous_xfr = 1;    /* set flag */
 542
 543                 if (xfer->timeout == 0) {
 544                         /*
 545                          * set a default timeout in
 546                          * case something goes wrong!
 547                          */
 548                         xfer->timeout = 1000 / 4;
 549                 }
 550                 switch (parm->speed) {
 551                 case USB_SPEED_LOW:
 552                 case USB_SPEED_FULL:
 553                         frame_limit = USB_MAX_FS_ISOC_FRAMES_PER_XFER;
 554                         xfer->fps_shift = 0;
 555                         break;
 556                 default:
 557                         frame_limit = USB_MAX_HS_ISOC_FRAMES_PER_XFER;
 558                         xfer->fps_shift = edesc->bInterval;
 559                         if (xfer->fps_shift > 0)
 560                                 xfer->fps_shift--;
 561                         if (xfer->fps_shift > 3)
 562                                 xfer->fps_shift = 3;
 563                         if (xfer->flags.pre_scale_frames != 0)
 564                                 xfer->nframes <<= (3 - xfer->fps_shift);
 565                         break;
 566                 }
 567
 568                 if (xfer->nframes > frame_limit) {
 569                         /*
 570                          * this is not going to work
 571                          * cross hardware
 572                          */
 573                         parm->err = USB_ERR_INVAL;
 574                         goto done;
 575                 }
 576                 if (xfer->nframes == 0) {
 577                         /*
 578                          * this is not a valid value
 579                          */
 580                         parm->err = USB_ERR_ZERO_NFRAMES;
 581                         goto done;
 582                 }
 583         } else {
 584
 585                 /*
 586                  * If a value is specified use that else check the
 587                  * endpoint descriptor!
 588                  */
 589                 if (type == UE_INTERRUPT) {
 590
 591                         uint32_t temp;
 592
 593                         if (xfer->interval == 0) {
 594
 595                                 xfer->interval = edesc->bInterval;
 596
 597                                 switch (parm->speed) {
 598                                 case USB_SPEED_LOW:
 599                                 case USB_SPEED_FULL:
 600                                         break;
 601                                 default:
 602                                         /* 125us -> 1ms */
 603                                         if (xfer->interval < 4)
 604                                                 xfer->interval = 1;
 605                                         else if (xfer->interval > 16)
 606                                                 xfer->interval = (1 << (16 - 4));
 607                                         else
 608                                                 xfer->interval =
 609                                                     (1 << (xfer->interval - 4));
 610                                         break;
 611                                 }
 612                         }
 613
 614                         if (xfer->interval == 0) {
 615                                 /*
 616                                  * One millisecond is the smallest
 617                                  * interval we support:
 618                                  */
 619                                 xfer->interval = 1;
 620                         }
 621
 622                         xfer->fps_shift = 0;
 623                         temp = 1;
 624
 625                         while ((temp != 0) && (temp < xfer->interval)) {
 626                                 xfer->fps_shift++;
 627                                 temp *= 2;
 628                         }
 629
 630                         switch (parm->speed) {
 631                         case USB_SPEED_LOW:
 632                         case USB_SPEED_FULL:
 633                                 break;
 634                         default:
 635                                 xfer->fps_shift += 3;
 636                                 break;
 637                         }
 638                 }
 639         }
 640
 641         /*
 642          * NOTE: we do not allow "max_packet_size" or "max_frame_size"
 643          * to be equal to zero when setting up USB transfers, hence
 644          * this leads to alot of extra code in the USB kernel.
 645          */
 646
 647         if ((xfer->max_frame_size == 0) ||
 648             (xfer->max_packet_size == 0)) {
 649
 650                 zmps = 1;
 651
 652                 if ((parm->bufsize <= MIN_PKT) &&
 653                     (type != UE_CONTROL) &&
 654                     (type != UE_BULK)) {
 655
 656                         /* workaround */
 657                         xfer->max_packet_size = MIN_PKT;
 658                         xfer->max_packet_count = 1;
 659                         parm->bufsize = 0;      /* automatic setup length */
 660                         usbd_update_max_frame_size(xfer);
 661
 662                 } else {
 663                         parm->err = USB_ERR_ZERO_MAXP;
 664                         goto done;
 665                 }
 666
 667         } else {
 668                 zmps = 0;
 669         }
 670
 671         /*
 672          * check if we should setup a default
 673          * length:
 674          */
 675
 676         if (parm->bufsize == 0) {
 677
 678                 parm->bufsize = xfer->max_frame_size;
 679
 680                 if (type == UE_ISOCHRONOUS) {
 681                         parm->bufsize *= xfer->nframes;
 682                 }
 683         }
 684         /*
 685          * check if we are about to setup a proxy
 686          * type of buffer:
 687          */
 688
 689         if (xfer->flags.proxy_buffer) {
 690
 691                 /* round bufsize up */
 692
 693                 parm->bufsize += (xfer->max_frame_size - 1);
 694
 695                 if (parm->bufsize < xfer->max_frame_size) {
 696                         /* length wrapped around */
 697                         parm->err = USB_ERR_INVAL;
 698                         goto done;
 699                 }
 700                 /* subtract remainder */
 701
 702                 parm->bufsize -= (parm->bufsize % xfer->max_frame_size);
 703
 704                 /* add length of USB device request structure, if any */
 705
 706                 if (type == UE_CONTROL) {
 707                         parm->bufsize += REQ_SIZE;      /* SETUP message */
 708                 }
 709         }
 710         xfer->max_data_length = parm->bufsize;
 711
 712         /* Setup "n_frlengths" and "n_frbuffers" */
 713
 714         if (type == UE_ISOCHRONOUS) {
 715                 n_frlengths = xfer->nframes;
 716                 n_frbuffers = 1;
 717         } else {
 718
 719                 if (type == UE_CONTROL) {
 720                         xfer->flags_int.control_xfr = 1;
 721                         if (xfer->nframes == 0) {
 722                                 if (parm->bufsize <= REQ_SIZE) {
 723                                         /*
 724                                          * there will never be any data
 725                                          * stage
 726                                          */
 727                                         xfer->nframes = 1;
 728                                 } else {
 729                                         xfer->nframes = 2;
 730                                 }
 731                         }
 732                 } else {
 733                         if (xfer->nframes == 0) {
 734                                 xfer->nframes = 1;
 735                         }
 736                 }
 737
 738                 n_frlengths = xfer->nframes;
 739                 n_frbuffers = xfer->nframes;
 740         }
 741
 742         /*
 743          * check if we have room for the
 744          * USB device request structure:
 745          */
 746
 747         if (type == UE_CONTROL) {
 748
 749                 if (xfer->max_data_length < REQ_SIZE) {
 750                         /* length wrapped around or too small bufsize */
 751                         parm->err = USB_ERR_INVAL;
 752                         goto done;
 753                 }
 754                 xfer->max_data_length -= REQ_SIZE;
 755         }
 756         /*
 757          * Setup "frlengths" and shadow "frlengths" for keeping the
 758          * initial frame lengths when a USB transfer is complete. This
 759          * information is useful when computing isochronous offsets.
 760          */
 761         xfer->frlengths = parm->xfer_length_ptr;
 762         parm->xfer_length_ptr += 2 * n_frlengths;
 763
 764         /* setup "frbuffers" */
 765         xfer->frbuffers = parm->xfer_page_cache_ptr;
 766         parm->xfer_page_cache_ptr += n_frbuffers;
 767
 768         /* initialize max frame count */
 769         xfer->max_frame_count = xfer->nframes;
 770
 771         /*
 772          * check if we need to setup
 773          * a local buffer:
 774          */
 775
 776         if (!xfer->flags.ext_buffer) {
 777
 778                 /* align data */
 779                 parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
 780
 781                 if (parm->buf) {
 782
 783                         xfer->local_buffer =
 784                             USB_ADD_BYTES(parm->buf, parm->size[0]);
 785
 786                         usbd_xfer_set_frame_offset(xfer, 0, 0);
 787
 788                         if ((type == UE_CONTROL) && (n_frbuffers > 1)) {
 789                                 usbd_xfer_set_frame_offset(xfer, REQ_SIZE, 1);
 790                         }
 791                 }
 792                 parm->size[0] += parm->bufsize;
 793
 794                 /* align data again */
 795                 parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
 796         }
 797         /*
 798          * Compute maximum buffer size
 799          */
 800
 801         if (parm->bufsize_max < parm->bufsize) {
 802                 parm->bufsize_max = parm->bufsize;
 803         }
 804 #if USB_HAVE_BUSDMA
 805         if (xfer->flags_int.bdma_enable) {
 806                 /*
 807                  * Setup "dma_page_ptr".
 808                  *
 809                  * Proof for formula below:
 810                  *
 811                  * Assume there are three USB frames having length "a", "b" and
 812                  * "c". These USB frames will at maximum need "z"
 813                  * "usb_page" structures. "z" is given by:
 814                  *
 815                  * z = ((a / USB_PAGE_SIZE) + 2) + ((b / USB_PAGE_SIZE) + 2) +
 816                  * ((c / USB_PAGE_SIZE) + 2);
 817                  *
 818                  * Constraining "a", "b" and "c" like this:
 819                  *
 820                  * (a + b + c) <= parm->bufsize
 821                  *
 822                  * We know that:
 823                  *
 824                  * z <= ((parm->bufsize / USB_PAGE_SIZE) + (3*2));
 825                  *
 826                  * Here is the general formula:
 827                  */
 828                 xfer->dma_page_ptr = parm->dma_page_ptr;
 829                 parm->dma_page_ptr += (2 * n_frbuffers);
 830                 parm->dma_page_ptr += (parm->bufsize / USB_PAGE_SIZE);
 831         }
 832 #endif
 833         if (zmps) {
 834                 /* correct maximum data length */
 835                 xfer->max_data_length = 0;
 836         }
 837         /* subtract USB frame remainder from "hc_max_frame_size" */
 838
 839         xfer->max_hc_frame_size =
 840             (parm->hc_max_frame_size -
 841             (parm->hc_max_frame_size % xfer->max_frame_size));
 842
 843         if (xfer->max_hc_frame_size == 0) {
 844                 parm->err = USB_ERR_INVAL;
 845                 goto done;
 846         }
 847
 848         /* initialize frame buffers */
 849
 850         if (parm->buf) {
 851                 for (x = 0; x != n_frbuffers; x++) {
 852                         xfer->frbuffers[x].tag_parent =
 853                             &xfer->xroot->dma_parent_tag;
 854 #if USB_HAVE_BUSDMA
 855                         if (xfer->flags_int.bdma_enable &&
 856                             (parm->bufsize_max > 0)) {
 857
 858                                 if (usb_pc_dmamap_create(
 859                                     xfer->frbuffers + x,
 860                                     parm->bufsize_max)) {
 861                                         parm->err = USB_ERR_NOMEM;
 862                                         goto done;
 863                                 }
 864                         }
 865 #endif
 866                 }
 867         }
 868 done:
 869         if (parm->err) {
 870                 /*
 871                  * Set some dummy values so that we avoid division by zero:
 872                  */
 873                 xfer->max_hc_frame_size = 1;
 874                 xfer->max_frame_size = 1;
 875                 xfer->max_packet_size = 1;
 876                 xfer->max_data_length = 0;
 877                 xfer->nframes = 0;
 878                 xfer->max_frame_count = 0;
 879         }
 880 }
 881
 882 /*------------------------------------------------------------------------*
 883  *      usbd_transfer_setup - setup an array of USB transfers
 884  *
 885  * NOTE: You must always call "usbd_transfer_unsetup" after calling
 886  * "usbd_transfer_setup" if success was returned.
 887  *
 888  * The idea is that the USB device driver should pre-allocate all its
 889  * transfers by one call to this function.
 890  *
 891  * Return values:
 892  *    0: Success
 893  * Else: Failure
 894  *------------------------------------------------------------------------*/
 895 usb_error_t
 896 usbd_transfer_setup(struct usb_device *udev,
 897     const uint8_t *ifaces, struct usb_xfer **ppxfer,
 898     const struct usb_config *setup_start, uint16_t n_setup,
 899     void *priv_sc, struct mtx *xfer_mtx)
 900 {
 901         const struct usb_config *setup_end = setup_start + n_setup;
 902         const struct usb_config *setup;
 903         struct usb_setup_params *parm;
 904         struct usb_endpoint *ep;
 905         struct usb_xfer_root *info;
 906         struct usb_xfer *xfer;
 907         void *buf = NULL;
 908         usb_error_t error = 0;
 909         uint16_t n;
 910         uint16_t refcount;
 911         uint8_t do_unlock;
 912
 913         WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
 914             "usbd_transfer_setup can sleep!");
 915
 916         /* do some checking first */
 917
 918         if (n_setup == 0) {
 919                 DPRINTFN(6, "setup array has zero length!\n");
 920                 return (USB_ERR_INVAL);
 921         }
 922         if (ifaces == 0) {
 923                 DPRINTFN(6, "ifaces array is NULL!\n");
 924                 return (USB_ERR_INVAL);
 925         }
 926         if (xfer_mtx == NULL) {
 927                 DPRINTFN(6, "using global lock\n");
 928                 xfer_mtx = &Giant;
 929         }
 930
 931         /* more sanity checks */
 932
 933         for (setup = setup_start, n = 0;
 934             setup != setup_end; setup++, n++) {
 935                 if (setup->bufsize == (usb_frlength_t)-1) {
 936                         error = USB_ERR_BAD_BUFSIZE;
 937                         DPRINTF("invalid bufsize\n");
 938                 }
 939                 if (setup->callback == NULL) {
 940                         error = USB_ERR_NO_CALLBACK;
 941                         DPRINTF("no callback\n");
 942                 }
 943                 ppxfer[n] = NULL;
 944         }
 945
 946         if (error)
 947                 return (error);
 948
 949         /* Protect scratch area */
 950         do_unlock = usbd_ctrl_lock(udev);
 951
 952         refcount = 0;
 953         info = NULL;
 954
 955         parm = &udev->scratch.xfer_setup[0].parm;
 956         memset(parm, 0, sizeof(*parm));
 957
 958         parm->udev = udev;
 959         parm->speed = usbd_get_speed(udev);
 960         parm->hc_max_packet_count = 1;
 961
 962         if (parm->speed >= USB_SPEED_MAX) {
 963                 parm->err = USB_ERR_INVAL;
 964                 goto done;
 965         }
 966         /* setup all transfers */
 967
 968         while (1) {
 969
 970                 if (buf) {
 971                         /*
 972                          * Initialize the "usb_xfer_root" structure,
 973                          * which is common for all our USB transfers.
 974                          */
 975                         info = USB_ADD_BYTES(buf, 0);
 976
 977                         info->memory_base = buf;
 978                         info->memory_size = parm->size[0];
 979
 980 #if USB_HAVE_BUSDMA
 981                         info->dma_page_cache_start = USB_ADD_BYTES(buf, parm->size[4]);
 982                         info->dma_page_cache_end = USB_ADD_BYTES(buf, parm->size[5]);
 983 #endif
 984                         info->xfer_page_cache_start = USB_ADD_BYTES(buf, parm->size[5]);
 985                         info->xfer_page_cache_end = USB_ADD_BYTES(buf, parm->size[2]);
 986
 987                         cv_init(&info->cv_drain, "WDRAIN");
 988
 989                         info->xfer_mtx = xfer_mtx;
 990 #if USB_HAVE_BUSDMA
 991                         usb_dma_tag_setup(&info->dma_parent_tag,
 992                             parm->dma_tag_p, udev->bus->dma_parent_tag[0].tag,
 993                             xfer_mtx, &usb_bdma_done_event, udev->bus->dma_bits,
 994                             parm->dma_tag_max);
 995 #endif
 996
 997                         info->bus = udev->bus;
 998                         info->udev = udev;
 999
1000                         TAILQ_INIT(&info->done_q.head);
1001                         info->done_q.command = &usbd_callback_wrapper;
1002 #if USB_HAVE_BUSDMA
1003                         TAILQ_INIT(&info->dma_q.head);
1004                         info->dma_q.command = &usb_bdma_work_loop;
1005 #endif
1006                         info->done_m[0].hdr.pm_callback = &usb_callback_proc;
1007                         info->done_m[0].xroot = info;
1008                         info->done_m[1].hdr.pm_callback = &usb_callback_proc;
1009                         info->done_m[1].xroot = info;
1010
1011                         /*
1012                          * In device side mode control endpoint
1013                          * requests need to run from a separate
1014                          * context, else there is a chance of
1015                          * deadlock!
1016                          */
1017                         if (setup_start == usb_control_ep_cfg)
1018                                 info->done_p =
1019                                     &udev->bus->control_xfer_proc;
1020                         else if (xfer_mtx == &Giant)
1021                                 info->done_p =
1022                                     &udev->bus->giant_callback_proc;
1023                         else
1024                                 info->done_p =
1025                                     &udev->bus->non_giant_callback_proc;
1026                 }
1027                 /* reset sizes */
1028
1029                 parm->size[0] = 0;
1030                 parm->buf = buf;
1031                 parm->size[0] += sizeof(info[0]);
1032
1033                 for (setup = setup_start, n = 0;
1034                     setup != setup_end; setup++, n++) {
1035
1036                         /* skip USB transfers without callbacks: */
1037                         if (setup->callback == NULL) {
1038                                 continue;
1039                         }
1040                         /* see if there is a matching endpoint */
1041                         ep = usbd_get_endpoint(udev,
1042                             ifaces[setup->if_index], setup);
1043
1044                         if ((ep == NULL) || (ep->methods == NULL)) {
1045                                 if (setup->flags.no_pipe_ok)
1046                                         continue;
1047                                 if ((setup->usb_mode != USB_MODE_DUAL) &&
1048                                     (setup->usb_mode != udev->flags.usb_mode))
1049                                         continue;
1050                                 parm->err = USB_ERR_NO_PIPE;
1051                                 goto done;
1052                         }
1053
1054                         /* align data properly */
1055                         parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
1056
1057                         /* store current setup pointer */
1058                         parm->curr_setup = setup;
1059
1060                         if (buf) {
1061                                 /*
1062                                  * Common initialization of the
1063                                  * "usb_xfer" structure.
1064                                  */
1065                                 xfer = USB_ADD_BYTES(buf, parm->size[0]);
1066                                 xfer->address = udev->address;
1067                                 xfer->priv_sc = priv_sc;
1068                                 xfer->xroot = info;
1069
1070                                 usb_callout_init_mtx(&xfer->timeout_handle,
1071                                     &udev->bus->bus_mtx, 0);
1072                         } else {
1073                                 /*
1074                                  * Setup a dummy xfer, hence we are
1075                                  * writing to the "usb_xfer"
1076                                  * structure pointed to by "xfer"
1077                                  * before we have allocated any
1078                                  * memory:
1079                                  */
1080                                 xfer = &udev->scratch.xfer_setup[0].dummy;
1081                                 memset(xfer, 0, sizeof(*xfer));
1082                                 refcount++;
1083                         }
1084
1085                         /* set transfer endpoint pointer */
1086                         xfer->endpoint = ep;
1087
1088                         parm->size[0] += sizeof(xfer[0]);
1089                         parm->methods = xfer->endpoint->methods;
1090                         parm->curr_xfer = xfer;
1091
1092                         /*
1093                          * Call the Host or Device controller transfer
1094                          * setup routine:
1095                          */
1096                         (udev->bus->methods->xfer_setup) (parm);
1097
1098                         /* check for error */
1099                         if (parm->err)
1100                                 goto done;
1101
1102                         if (buf) {
1103                                 /*
1104                                  * Increment the endpoint refcount. This
1105                                  * basically prevents setting a new
1106                                  * configuration and alternate setting
1107                                  * when USB transfers are in use on
1108                                  * the given interface. Search the USB
1109                                  * code for "endpoint->refcount_alloc" if you
1110                                  * want more information.
1111                                  */
1112                                 USB_BUS_LOCK(info->bus);
1113                                 if (xfer->endpoint->refcount_alloc >= USB_EP_REF_MAX)
1114                                         parm->err = USB_ERR_INVAL;
1115
1116                                 xfer->endpoint->refcount_alloc++;
1117
1118                                 if (xfer->endpoint->refcount_alloc == 0)
1119                                         panic("usbd_transfer_setup(): Refcount wrapped to zero\n");
1120                                 USB_BUS_UNLOCK(info->bus);
1121
1122                                 /*
1123                                  * Whenever we set ppxfer[] then we
1124                                  * also need to increment the
1125                                  * "setup_refcount":
1126                                  */
1127                                 info->setup_refcount++;
1128
1129                                 /*
1130                                  * Transfer is successfully setup and
1131                                  * can be used:
1132                                  */
1133                                 ppxfer[n] = xfer;
1134                         }
1135
1136                         /* check for error */
1137                         if (parm->err)
1138                                 goto done;
1139                 }
1140
1141                 if (buf != NULL || parm->err != 0)
1142                         goto done;
1143
1144                 /* if no transfers, nothing to do */
1145                 if (refcount == 0)
1146                         goto done;
1147
1148                 /* align data properly */
1149                 parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
1150
1151                 /* store offset temporarily */
1152                 parm->size[1] = parm->size[0];
1153
1154                 /*
1155                  * The number of DMA tags required depends on
1156                  * the number of endpoints. The current estimate
1157                  * for maximum number of DMA tags per endpoint
1158                  * is two.
1159                  */
1160                 parm->dma_tag_max += 2 * MIN(n_setup, USB_EP_MAX);
1161
1162                 /*
1163                  * DMA tags for QH, TD, Data and more.
1164                  */
1165                 parm->dma_tag_max += 8;
1166
1167                 parm->dma_tag_p += parm->dma_tag_max;
1168
1169                 parm->size[0] += ((uint8_t *)parm->dma_tag_p) -
1170                     ((uint8_t *)0);
1171
1172                 /* align data properly */
1173                 parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
1174
1175                 /* store offset temporarily */
1176                 parm->size[3] = parm->size[0];
1177
1178                 parm->size[0] += ((uint8_t *)parm->dma_page_ptr) -
1179                     ((uint8_t *)0);
1180
1181                 /* align data properly */
1182                 parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
1183
1184                 /* store offset temporarily */
1185                 parm->size[4] = parm->size[0];
1186
1187                 parm->size[0] += ((uint8_t *)parm->dma_page_cache_ptr) -
1188                     ((uint8_t *)0);
1189
1190                 /* store end offset temporarily */
1191                 parm->size[5] = parm->size[0];
1192
1193                 parm->size[0] += ((uint8_t *)parm->xfer_page_cache_ptr) -
1194                     ((uint8_t *)0);
1195
1196                 /* store end offset temporarily */
1197
1198                 parm->size[2] = parm->size[0];
1199
1200                 /* align data properly */
1201                 parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
1202
1203                 parm->size[6] = parm->size[0];
1204
1205                 parm->size[0] += ((uint8_t *)parm->xfer_length_ptr) -
1206                     ((uint8_t *)0);
1207
1208                 /* align data properly */
1209                 parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
1210
1211                 /* allocate zeroed memory */
1212                 buf = malloc(parm->size[0], M_USB, M_WAITOK | M_ZERO);
1213
1214                 if (buf == NULL) {
1215                         parm->err = USB_ERR_NOMEM;
1216                         DPRINTFN(0, "cannot allocate memory block for "
1217                             "configuration (%d bytes)\n",
1218                             parm->size[0]);
1219                         goto done;
1220                 }
1221                 parm->dma_tag_p = USB_ADD_BYTES(buf, parm->size[1]);
1222                 parm->dma_page_ptr = USB_ADD_BYTES(buf, parm->size[3]);
1223                 parm->dma_page_cache_ptr = USB_ADD_BYTES(buf, parm->size[4]);
1224                 parm->xfer_page_cache_ptr = USB_ADD_BYTES(buf, parm->size[5]);
1225                 parm->xfer_length_ptr = USB_ADD_BYTES(buf, parm->size[6]);
1226         }
1227
1228 done:
1229         if (buf) {
1230                 if (info->setup_refcount == 0) {
1231                         /*
1232                          * "usbd_transfer_unsetup_sub" will unlock
1233                          * the bus mutex before returning !
1234                          */
1235                         USB_BUS_LOCK(info->bus);
1236
1237                         /* something went wrong */
1238                         usbd_transfer_unsetup_sub(info, 0);
1239                 }
1240         }
1241
1242         /* check if any errors happened */
1243         if (parm->err)
1244                 usbd_transfer_unsetup(ppxfer, n_setup);
1245
1246         error = parm->err;
1247
1248         if (do_unlock)
1249                 usbd_ctrl_unlock(udev);
1250
1251         return (error);
1252 }
1253
1254 /*------------------------------------------------------------------------*
1255  *      usbd_transfer_unsetup_sub - factored out code
1256  *------------------------------------------------------------------------*/
1257 static void
1258 usbd_transfer_unsetup_sub(struct usb_xfer_root *info, uint8_t needs_delay)
1259 {
1260 #if USB_HAVE_BUSDMA
1261         struct usb_page_cache *pc;
1262 #endif
1263
1264         USB_BUS_LOCK_ASSERT(info->bus, MA_OWNED);
1265
1266         /* wait for any outstanding DMA operations */
1267
1268         if (needs_delay) {
1269                 usb_timeout_t temp;
1270                 temp = usbd_get_dma_delay(info->udev);
1271                 if (temp != 0) {
1272                         usb_pause_mtx(&info->bus->bus_mtx,
1273                             USB_MS_TO_TICKS(temp));
1274                 }
1275         }
1276
1277         /* make sure that our done messages are not queued anywhere */
1278         usb_proc_mwait(info->done_p, &info->done_m[0], &info->done_m[1]);
1279
1280         USB_BUS_UNLOCK(info->bus);
1281
1282 #if USB_HAVE_BUSDMA
1283         /* free DMA'able memory, if any */
1284         pc = info->dma_page_cache_start;
1285         while (pc != info->dma_page_cache_end) {
1286                 usb_pc_free_mem(pc);
1287                 pc++;
1288         }
1289
1290         /* free DMA maps in all "xfer->frbuffers" */
1291         pc = info->xfer_page_cache_start;
1292         while (pc != info->xfer_page_cache_end) {
1293                 usb_pc_dmamap_destroy(pc);
1294                 pc++;
1295         }
1296
1297         /* free all DMA tags */
1298         usb_dma_tag_unsetup(&info->dma_parent_tag);
1299 #endif
1300
1301         cv_destroy(&info->cv_drain);
1302
1303         /*
1304          * free the "memory_base" last, hence the "info" structure is
1305          * contained within the "memory_base"!
1306          */
1307         free(info->memory_base, M_USB);
1308 }
1309
1310 /*------------------------------------------------------------------------*
1311  *      usbd_transfer_unsetup - unsetup/free an array of USB transfers
1312  *
1313  * NOTE: All USB transfers in progress will get called back passing
1314  * the error code "USB_ERR_CANCELLED" before this function
1315  * returns.
1316  *------------------------------------------------------------------------*/
1317 void
1318 usbd_transfer_unsetup(struct usb_xfer **pxfer, uint16_t n_setup)
1319 {
1320         struct usb_xfer *xfer;
1321         struct usb_xfer_root *info;
1322         uint8_t needs_delay = 0;
1323
1324         WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
1325             "usbd_transfer_unsetup can sleep!");
1326
1327         while (n_setup--) {
1328                 xfer = pxfer[n_setup];
1329
1330                 if (xfer == NULL)
1331                         continue;
1332
1333                 info = xfer->xroot;
1334
1335                 USB_XFER_LOCK(xfer);
1336                 USB_BUS_LOCK(info->bus);
1337
1338                 /*
1339                  * HINT: when you start/stop a transfer, it might be a
1340                  * good idea to directly use the "pxfer[]" structure:
1341                  *
1342                  * usbd_transfer_start(sc->pxfer[0]);
1343                  * usbd_transfer_stop(sc->pxfer[0]);
1344                  *
1345                  * That way, if your code has many parts that will not
1346                  * stop running under the same lock, in other words
1347                  * "xfer_mtx", the usbd_transfer_start and
1348                  * usbd_transfer_stop functions will simply return
1349                  * when they detect a NULL pointer argument.
1350                  *
1351                  * To avoid any races we clear the "pxfer[]" pointer
1352                  * while holding the private mutex of the driver:
1353                  */
1354                 pxfer[n_setup] = NULL;
1355
1356                 USB_BUS_UNLOCK(info->bus);
1357                 USB_XFER_UNLOCK(xfer);
1358
1359                 usbd_transfer_drain(xfer);
1360
1361 #if USB_HAVE_BUSDMA
1362                 if (xfer->flags_int.bdma_enable)
1363                         needs_delay = 1;
1364 #endif
1365                 /*
1366                  * NOTE: default endpoint does not have an
1367                  * interface, even if endpoint->iface_index == 0
1368                  */
1369                 USB_BUS_LOCK(info->bus);
1370                 xfer->endpoint->refcount_alloc--;
1371                 USB_BUS_UNLOCK(info->bus);
1372
1373                 usb_callout_drain(&xfer->timeout_handle);
1374
1375                 USB_BUS_LOCK(info->bus);
1376
1377                 USB_ASSERT(info->setup_refcount != 0, ("Invalid setup "
1378                     "reference count\n"));
1379
1380                 info->setup_refcount--;
1381
1382                 if (info->setup_refcount == 0) {
1383                         usbd_transfer_unsetup_sub(info,
1384                             needs_delay);
1385                 } else {
1386                         USB_BUS_UNLOCK(info->bus);
1387                 }
1388         }
1389 }
1390
1391 /*------------------------------------------------------------------------*
1392  *      usbd_control_transfer_init - factored out code
1393  *
1394  * In USB Device Mode we have to wait for the SETUP packet which
1395  * containst the "struct usb_device_request" structure, before we can
1396  * transfer any data. In USB Host Mode we already have the SETUP
1397  * packet at the moment the USB transfer is started. This leads us to
1398  * having to setup the USB transfer at two different places in
1399  * time. This function just contains factored out control transfer
1400  * initialisation code, so that we don't duplicate the code.
1401  *------------------------------------------------------------------------*/
1402 static void
1403 usbd_control_transfer_init(struct usb_xfer *xfer)
1404 {
1405         struct usb_device_request req;
1406
1407         /* copy out the USB request header */
1408
1409         usbd_copy_out(xfer->frbuffers, 0, &req, sizeof(req));
1410
1411         /* setup remainder */
1412
1413         xfer->flags_int.control_rem = UGETW(req.wLength);
1414
1415         /* copy direction to endpoint variable */
1416
1417         xfer->endpointno &= ~(UE_DIR_IN | UE_DIR_OUT);
1418         xfer->endpointno |=
1419             (req.bmRequestType & UT_READ) ? UE_DIR_IN : UE_DIR_OUT;
1420 }
1421
1422 /*------------------------------------------------------------------------*
1423  *      usbd_control_transfer_did_data
1424  *
1425  * This function returns non-zero if a control endpoint has
1426  * transferred the first DATA packet after the SETUP packet.
1427  * Else it returns zero.
1428  *------------------------------------------------------------------------*/
1429 static uint8_t
1430 usbd_control_transfer_did_data(struct usb_xfer *xfer)
1431 {
1432         struct usb_device_request req;
1433
1434         /* SETUP packet is not yet sent */
1435         if (xfer->flags_int.control_hdr != 0)
1436                 return (0);
1437
1438         /* copy out the USB request header */
1439         usbd_copy_out(xfer->frbuffers, 0, &req, sizeof(req));
1440
1441         /* compare remainder to the initial value */
1442         return (xfer->flags_int.control_rem != UGETW(req.wLength));
1443 }
1444
1445 /*------------------------------------------------------------------------*
1446  *      usbd_setup_ctrl_transfer
1447  *
1448  * This function handles initialisation of control transfers. Control
1449  * transfers are special in that regard that they can both transmit
1450  * and receive data.
1451  *
1452  * Return values:
1453  *    0: Success
1454  * Else: Failure
1455  *------------------------------------------------------------------------*/
1456 static int
1457 usbd_setup_ctrl_transfer(struct usb_xfer *xfer)
1458 {
1459         usb_frlength_t len;
1460
1461         /* Check for control endpoint stall */
1462         if (xfer->flags.stall_pipe && xfer->flags_int.control_act) {
1463                 /* the control transfer is no longer active */
1464                 xfer->flags_int.control_stall = 1;
1465                 xfer->flags_int.control_act = 0;
1466         } else {
1467                 /* don't stall control transfer by default */
1468                 xfer->flags_int.control_stall = 0;
1469         }
1470
1471         /* Check for invalid number of frames */
1472         if (xfer->nframes > 2) {
1473                 /*
1474                  * If you need to split a control transfer, you
1475                  * have to do one part at a time. Only with
1476                  * non-control transfers you can do multiple
1477                  * parts a time.
1478                  */
1479                 DPRINTFN(0, "Too many frames: %u\n",
1480                     (unsigned int)xfer->nframes);
1481                 goto error;
1482         }
1483
1484         /*
1485          * Check if there is a control
1486          * transfer in progress:
1487          */
1488         if (xfer->flags_int.control_act) {
1489
1490                 if (xfer->flags_int.control_hdr) {
1491
1492                         /* clear send header flag */
1493
1494                         xfer->flags_int.control_hdr = 0;
1495
1496                         /* setup control transfer */
1497                         if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) {
1498                                 usbd_control_transfer_init(xfer);
1499                         }
1500                 }
1501                 /* get data length */
1502
1503                 len = xfer->sumlen;
1504
1505         } else {
1506
1507                 /* the size of the SETUP structure is hardcoded ! */
1508
1509                 if (xfer->frlengths[0] != sizeof(struct usb_device_request)) {
1510                         DPRINTFN(0, "Wrong framelength %u != %zu\n",
1511                             xfer->frlengths[0], sizeof(struct
1512                             usb_device_request));
1513                         goto error;
1514                 }
1515                 /* check USB mode */
1516                 if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) {
1517
1518                         /* check number of frames */
1519                         if (xfer->nframes != 1) {
1520                                 /*
1521                                  * We need to receive the setup
1522                                  * message first so that we know the
1523                                  * data direction!
1524                                  */
1525                                 DPRINTF("Misconfigured transfer\n");
1526                                 goto error;
1527                         }
1528                         /*
1529                          * Set a dummy "control_rem" value.  This
1530                          * variable will be overwritten later by a
1531                          * call to "usbd_control_transfer_init()" !
1532                          */
1533                         xfer->flags_int.control_rem = 0xFFFF;
1534                 } else {
1535
1536                         /* setup "endpoint" and "control_rem" */
1537
1538                         usbd_control_transfer_init(xfer);
1539                 }
1540
1541                 /* set transfer-header flag */
1542
1543                 xfer->flags_int.control_hdr = 1;
1544
1545                 /* get data length */
1546
1547                 len = (xfer->sumlen - sizeof(struct usb_device_request));
1548         }
1549
1550         /* update did data flag */
1551
1552         xfer->flags_int.control_did_data =
1553             usbd_control_transfer_did_data(xfer);
1554
1555         /* check if there is a length mismatch */
1556
1557         if (len > xfer->flags_int.control_rem) {
1558                 DPRINTFN(0, "Length (%d) greater than "
1559                     "remaining length (%d)\n", len,
1560                     xfer->flags_int.control_rem);
1561                 goto error;
1562         }
1563         /* check if we are doing a short transfer */
1564
1565         if (xfer->flags.force_short_xfer) {
1566                 xfer->flags_int.control_rem = 0;
1567         } else {
1568                 if ((len != xfer->max_data_length) &&
1569                     (len != xfer->flags_int.control_rem) &&
1570                     (xfer->nframes != 1)) {
1571                         DPRINTFN(0, "Short control transfer without "
1572                             "force_short_xfer set\n");
1573                         goto error;
1574                 }
1575                 xfer->flags_int.control_rem -= len;
1576         }
1577
1578         /* the status part is executed when "control_act" is 0 */
1579
1580         if ((xfer->flags_int.control_rem > 0) ||
1581             (xfer->flags.manual_status)) {
1582                 /* don't execute the STATUS stage yet */
1583                 xfer->flags_int.control_act = 1;
1584
1585                 /* sanity check */
1586                 if ((!xfer->flags_int.control_hdr) &&
1587                     (xfer->nframes == 1)) {
1588                         /*
1589                          * This is not a valid operation!
1590                          */
1591                         DPRINTFN(0, "Invalid parameter "
1592                             "combination\n");
1593                         goto error;
1594                 }
1595         } else {
1596                 /* time to execute the STATUS stage */
1597                 xfer->flags_int.control_act = 0;
1598         }
1599         return (0);                     /* success */
1600
1601 error:
1602         return (1);                     /* failure */
1603 }
1604
1605 /*------------------------------------------------------------------------*
1606  *      usbd_transfer_submit - start USB hardware for the given transfer
1607  *
1608  * This function should only be called from the USB callback.
1609  *------------------------------------------------------------------------*/
1610 void
1611 usbd_transfer_submit(struct usb_xfer *xfer)
1612 {
1613         struct usb_xfer_root *info;
1614         struct usb_bus *bus;
1615         usb_frcount_t x;
1616
1617         info = xfer->xroot;
1618         bus = info->bus;
1619
1620         DPRINTF("xfer=%p, endpoint=%p, nframes=%d, dir=%s\n",
1621             xfer, xfer->endpoint, xfer->nframes, USB_GET_DATA_ISREAD(xfer) ?
1622             "read" : "write");
1623
1624 #ifdef USB_DEBUG
1625         if (USB_DEBUG_VAR > 0) {
1626                 USB_BUS_LOCK(bus);
1627
1628                 usb_dump_endpoint(xfer->endpoint);
1629
1630                 USB_BUS_UNLOCK(bus);
1631         }
1632 #endif
1633
1634         USB_XFER_LOCK_ASSERT(xfer, MA_OWNED);
1635         USB_BUS_LOCK_ASSERT(bus, MA_NOTOWNED);
1636
1637         /* Only open the USB transfer once! */
1638         if (!xfer->flags_int.open) {
1639                 xfer->flags_int.open = 1;
1640
1641                 DPRINTF("open\n");
1642
1643                 USB_BUS_LOCK(bus);
1644                 (xfer->endpoint->methods->open) (xfer);
1645                 USB_BUS_UNLOCK(bus);
1646         }
1647         /* set "transferring" flag */
1648         xfer->flags_int.transferring = 1;
1649
1650 #if USB_HAVE_POWERD
1651         /* increment power reference */
1652         usbd_transfer_power_ref(xfer, 1);
1653 #endif
1654         /*
1655          * Check if the transfer is waiting on a queue, most
1656          * frequently the "done_q":
1657          */
1658         if (xfer->wait_queue) {
1659                 USB_BUS_LOCK(bus);
1660                 usbd_transfer_dequeue(xfer);
1661                 USB_BUS_UNLOCK(bus);
1662         }
1663         /* clear "did_dma_delay" flag */
1664         xfer->flags_int.did_dma_delay = 0;
1665
1666         /* clear "did_close" flag */
1667         xfer->flags_int.did_close = 0;
1668
1669 #if USB_HAVE_BUSDMA
1670         /* clear "bdma_setup" flag */
1671         xfer->flags_int.bdma_setup = 0;
1672 #endif
1673         /* by default we cannot cancel any USB transfer immediately */
1674         xfer->flags_int.can_cancel_immed = 0;
1675
1676         /* clear lengths and frame counts by default */
1677         xfer->sumlen = 0;
1678         xfer->actlen = 0;
1679         xfer->aframes = 0;
1680
1681         /* clear any previous errors */
1682         xfer->error = 0;
1683
1684         /* Check if the device is still alive */
1685         if (info->udev->state < USB_STATE_POWERED) {
1686                 USB_BUS_LOCK(bus);
1687                 /*
1688                  * Must return cancelled error code else
1689                  * device drivers can hang.
1690                  */
1691                 usbd_transfer_done(xfer, USB_ERR_CANCELLED);
1692                 USB_BUS_UNLOCK(bus);
1693                 return;
1694         }
1695
1696         /* sanity check */
1697         if (xfer->nframes == 0) {
1698                 if (xfer->flags.stall_pipe) {
1699                         /*
1700                          * Special case - want to stall without transferring
1701                          * any data:
1702                          */
1703                         DPRINTF("xfer=%p nframes=0: stall "
1704                             "or clear stall!\n", xfer);
1705                         USB_BUS_LOCK(bus);
1706                         xfer->flags_int.can_cancel_immed = 1;
1707                         /* start the transfer */
1708                         usb_command_wrapper(&xfer->endpoint->endpoint_q, xfer);
1709                         USB_BUS_UNLOCK(bus);
1710                         return;
1711                 }
1712                 USB_BUS_LOCK(bus);
1713                 usbd_transfer_done(xfer, USB_ERR_INVAL);
1714                 USB_BUS_UNLOCK(bus);
1715                 return;
1716         }
1717         /* compute some variables */
1718
1719         for (x = 0; x != xfer->nframes; x++) {
1720                 /* make a copy of the frlenghts[] */
1721                 xfer->frlengths[x + xfer->max_frame_count] = xfer->frlengths[x];
1722                 /* compute total transfer length */
1723                 xfer->sumlen += xfer->frlengths[x];
1724                 if (xfer->sumlen < xfer->frlengths[x]) {
1725                         /* length wrapped around */
1726                         USB_BUS_LOCK(bus);
1727                         usbd_transfer_done(xfer, USB_ERR_INVAL);
1728                         USB_BUS_UNLOCK(bus);
1729                         return;
1730                 }
1731         }
1732
1733         /* clear some internal flags */
1734
1735         xfer->flags_int.short_xfer_ok = 0;
1736         xfer->flags_int.short_frames_ok = 0;
1737
1738         /* check if this is a control transfer */
1739
1740         if (xfer->flags_int.control_xfr) {
1741
1742                 if (usbd_setup_ctrl_transfer(xfer)) {
1743                         USB_BUS_LOCK(bus);
1744                         usbd_transfer_done(xfer, USB_ERR_STALLED);
1745                         USB_BUS_UNLOCK(bus);
1746                         return;
1747                 }
1748         }
1749         /*
1750          * Setup filtered version of some transfer flags,
1751          * in case of data read direction
1752          */
1753         if (USB_GET_DATA_ISREAD(xfer)) {
1754
1755                 if (xfer->flags.short_frames_ok) {
1756                         xfer->flags_int.short_xfer_ok = 1;
1757                         xfer->flags_int.short_frames_ok = 1;
1758                 } else if (xfer->flags.short_xfer_ok) {
1759                         xfer->flags_int.short_xfer_ok = 1;
1760
1761                         /* check for control transfer */
1762                         if (xfer->flags_int.control_xfr) {
1763                                 /*
1764                                  * 1) Control transfers do not support
1765                                  * reception of multiple short USB
1766                                  * frames in host mode and device side
1767                                  * mode, with exception of:
1768                                  *
1769                                  * 2) Due to sometimes buggy device
1770                                  * side firmware we need to do a
1771                                  * STATUS stage in case of short
1772                                  * control transfers in USB host mode.
1773                                  * The STATUS stage then becomes the
1774                                  * "alt_next" to the DATA stage.
1775                                  */
1776                                 xfer->flags_int.short_frames_ok = 1;
1777                         }
1778                 }
1779         }
1780         /*
1781          * Check if BUS-DMA support is enabled and try to load virtual
1782          * buffers into DMA, if any:
1783          */
1784 #if USB_HAVE_BUSDMA
1785         if (xfer->flags_int.bdma_enable) {
1786                 /* insert the USB transfer last in the BUS-DMA queue */
1787                 usb_command_wrapper(&xfer->xroot->dma_q, xfer);
1788                 return;
1789         }
1790 #endif
1791         /*
1792          * Enter the USB transfer into the Host Controller or
1793          * Device Controller schedule:
1794          */
1795         usbd_pipe_enter(xfer);
1796 }
1797
1798 /*------------------------------------------------------------------------*
1799  *      usbd_pipe_enter - factored out code
1800  *------------------------------------------------------------------------*/
1801 void
1802 usbd_pipe_enter(struct usb_xfer *xfer)
1803 {
1804         struct usb_endpoint *ep;
1805
1806         USB_XFER_LOCK_ASSERT(xfer, MA_OWNED);
1807
1808         USB_BUS_LOCK(xfer->xroot->bus);
1809
1810         ep = xfer->endpoint;
1811
1812         DPRINTF("enter\n");
1813
1814         /* the transfer can now be cancelled */
1815         xfer->flags_int.can_cancel_immed = 1;
1816
1817         /* enter the transfer */
1818         (ep->methods->enter) (xfer);
1819
1820         /* check for transfer error */
1821         if (xfer->error) {
1822                 /* some error has happened */
1823                 usbd_transfer_done(xfer, 0);
1824                 USB_BUS_UNLOCK(xfer->xroot->bus);
1825                 return;
1826         }
1827
1828         /* start the transfer */
1829         usb_command_wrapper(&ep->endpoint_q, xfer);
1830         USB_BUS_UNLOCK(xfer->xroot->bus);
1831 }
1832
1833 /*------------------------------------------------------------------------*
1834  *      usbd_transfer_start - start an USB transfer
1835  *
1836  * NOTE: Calling this function more than one time will only
1837  *       result in a single transfer start, until the USB transfer
1838  *       completes.
1839  *------------------------------------------------------------------------*/
1840 void
1841 usbd_transfer_start(struct usb_xfer *xfer)
1842 {
1843         if (xfer == NULL) {
1844                 /* transfer is gone */
1845                 return;
1846         }
1847         USB_XFER_LOCK_ASSERT(xfer, MA_OWNED);
1848
1849         /* mark the USB transfer started */
1850
1851         if (!xfer->flags_int.started) {
1852                 /* lock the BUS lock to avoid races updating flags_int */
1853                 USB_BUS_LOCK(xfer->xroot->bus);
1854                 xfer->flags_int.started = 1;
1855                 USB_BUS_UNLOCK(xfer->xroot->bus);
1856         }
1857         /* check if the USB transfer callback is already transferring */
1858
1859         if (xfer->flags_int.transferring) {
1860                 return;
1861         }
1862         USB_BUS_LOCK(xfer->xroot->bus);
1863         /* call the USB transfer callback */
1864         usbd_callback_ss_done_defer(xfer);
1865         USB_BUS_UNLOCK(xfer->xroot->bus);
1866 }
1867
1868 /*------------------------------------------------------------------------*
1869  *      usbd_transfer_stop - stop an USB transfer
1870  *
1871  * NOTE: Calling this function more than one time will only
1872  *       result in a single transfer stop.
1873  * NOTE: When this function returns it is not safe to free nor
1874  *       reuse any DMA buffers. See "usbd_transfer_drain()".
1875  *------------------------------------------------------------------------*/
1876 void
1877 usbd_transfer_stop(struct usb_xfer *xfer)
1878 {
1879         struct usb_endpoint *ep;
1880
1881         if (xfer == NULL) {
1882                 /* transfer is gone */
1883                 return;
1884         }
1885         USB_XFER_LOCK_ASSERT(xfer, MA_OWNED);
1886
1887         /* check if the USB transfer was ever opened */
1888
1889         if (!xfer->flags_int.open) {
1890                 if (xfer->flags_int.started) {
1891                         /* nothing to do except clearing the "started" flag */
1892                         /* lock the BUS lock to avoid races updating flags_int */
1893                         USB_BUS_LOCK(xfer->xroot->bus);
1894                         xfer->flags_int.started = 0;
1895                         USB_BUS_UNLOCK(xfer->xroot->bus);
1896                 }
1897                 return;
1898         }
1899         /* try to stop the current USB transfer */
1900
1901         USB_BUS_LOCK(xfer->xroot->bus);
1902         /* override any previous error */
1903         xfer->error = USB_ERR_CANCELLED;
1904
1905         /*
1906          * Clear "open" and "started" when both private and USB lock
1907          * is locked so that we don't get a race updating "flags_int"
1908          */
1909         xfer->flags_int.open = 0;
1910         xfer->flags_int.started = 0;
1911
1912         /*
1913          * Check if we can cancel the USB transfer immediately.
1914          */
1915         if (xfer->flags_int.transferring) {
1916                 if (xfer->flags_int.can_cancel_immed &&
1917                     (!xfer->flags_int.did_close)) {
1918                         DPRINTF("close\n");
1919                         /*
1920                          * The following will lead to an USB_ERR_CANCELLED
1921                          * error code being passed to the USB callback.
1922                          */
1923                         (xfer->endpoint->methods->close) (xfer);
1924                         /* only close once */
1925                         xfer->flags_int.did_close = 1;
1926                 } else {
1927                         /* need to wait for the next done callback */
1928                 }
1929         } else {
1930                 DPRINTF("close\n");
1931
1932                 /* close here and now */
1933                 (xfer->endpoint->methods->close) (xfer);
1934
1935                 /*
1936                  * Any additional DMA delay is done by
1937                  * "usbd_transfer_unsetup()".
1938                  */
1939
1940                 /*
1941                  * Special case. Check if we need to restart a blocked
1942                  * endpoint.
1943                  */
1944                 ep = xfer->endpoint;
1945
1946                 /*
1947                  * If the current USB transfer is completing we need
1948                  * to start the next one:
1949                  */
1950                 if (ep->endpoint_q.curr == xfer) {
1951                         usb_command_wrapper(&ep->endpoint_q, NULL);
1952                 }
1953         }
1954
1955         USB_BUS_UNLOCK(xfer->xroot->bus);
1956 }
1957
1958 /*------------------------------------------------------------------------*
1959  *      usbd_transfer_pending
1960  *
1961  * This function will check if an USB transfer is pending which is a
1962  * little bit complicated!
1963  * Return values:
1964  * 0: Not pending
1965  * 1: Pending: The USB transfer will receive a callback in the future.
1966  *------------------------------------------------------------------------*/
1967 uint8_t
1968 usbd_transfer_pending(struct usb_xfer *xfer)
1969 {
1970         struct usb_xfer_root *info;
1971         struct usb_xfer_queue *pq;
1972
1973         if (xfer == NULL) {
1974                 /* transfer is gone */
1975                 return (0);
1976         }
1977         USB_XFER_LOCK_ASSERT(xfer, MA_OWNED);
1978
1979         if (xfer->flags_int.transferring) {
1980                 /* trivial case */
1981                 return (1);
1982         }
1983         USB_BUS_LOCK(xfer->xroot->bus);
1984         if (xfer->wait_queue) {
1985                 /* we are waiting on a queue somewhere */
1986                 USB_BUS_UNLOCK(xfer->xroot->bus);
1987                 return (1);
1988         }
1989         info = xfer->xroot;
1990         pq = &info->done_q;
1991
1992         if (pq->curr == xfer) {
1993                 /* we are currently scheduled for callback */
1994                 USB_BUS_UNLOCK(xfer->xroot->bus);
1995                 return (1);
1996         }
1997         /* we are not pending */
1998         USB_BUS_UNLOCK(xfer->xroot->bus);
1999         return (0);
2000 }
2001
2002 /*------------------------------------------------------------------------*
2003  *      usbd_transfer_drain
2004  *
2005  * This function will stop the USB transfer and wait for any
2006  * additional BUS-DMA and HW-DMA operations to complete. Buffers that
2007  * are loaded into DMA can safely be freed or reused after that this
2008  * function has returned.
2009  *------------------------------------------------------------------------*/
2010 void
2011 usbd_transfer_drain(struct usb_xfer *xfer)
2012 {
2013         WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
2014             "usbd_transfer_drain can sleep!");
2015
2016         if (xfer == NULL) {
2017                 /* transfer is gone */
2018                 return;
2019         }
2020         if (xfer->xroot->xfer_mtx != &Giant) {
2021                 USB_XFER_LOCK_ASSERT(xfer, MA_NOTOWNED);
2022         }
2023         USB_XFER_LOCK(xfer);
2024
2025         usbd_transfer_stop(xfer);
2026
2027         while (usbd_transfer_pending(xfer) ||
2028             xfer->flags_int.doing_callback) {
2029
2030                 /*
2031                  * It is allowed that the callback can drop its
2032                  * transfer mutex. In that case checking only
2033                  * "usbd_transfer_pending()" is not enough to tell if
2034                  * the USB transfer is fully drained. We also need to
2035                  * check the internal "doing_callback" flag.
2036                  */
2037                 xfer->flags_int.draining = 1;
2038
2039                 /*
2040                  * Wait until the current outstanding USB
2041                  * transfer is complete !
2042                  */
2043                 cv_wait(&xfer->xroot->cv_drain, xfer->xroot->xfer_mtx);
2044         }
2045         USB_XFER_UNLOCK(xfer);
2046 }
2047
2048 struct usb_page_cache *
2049 usbd_xfer_get_frame(struct usb_xfer *xfer, usb_frcount_t frindex)
2050 {
2051         KASSERT(frindex < xfer->max_frame_count, ("frame index overflow"));
2052
2053         return (&xfer->frbuffers[frindex]);
2054 }
2055
2056 void *
2057 usbd_xfer_get_frame_buffer(struct usb_xfer *xfer, usb_frcount_t frindex)
2058 {
2059         struct usb_page_search page_info;
2060
2061         KASSERT(frindex < xfer->max_frame_count, ("frame index overflow"));
2062
2063         usbd_get_page(&xfer->frbuffers[frindex], 0, &page_info);
2064         return (page_info.buffer);
2065 }
2066
2067 /*------------------------------------------------------------------------*
2068  *      usbd_xfer_get_fps_shift
2069  *
2070  * The following function is only useful for isochronous transfers. It
2071  * returns how many times the frame execution rate has been shifted
2072  * down.
2073  *
2074  * Return value:
2075  * Success: 0..3
2076  * Failure: 0
2077  *------------------------------------------------------------------------*/
2078 uint8_t
2079 usbd_xfer_get_fps_shift(struct usb_xfer *xfer)
2080 {
2081         return (xfer->fps_shift);
2082 }
2083
2084 usb_frlength_t
2085 usbd_xfer_frame_len(struct usb_xfer *xfer, usb_frcount_t frindex)
2086 {
2087         KASSERT(frindex < xfer->max_frame_count, ("frame index overflow"));
2088
2089         return (xfer->frlengths[frindex]);
2090 }
2091
2092 /*------------------------------------------------------------------------*
2093  *      usbd_xfer_set_frame_data
2094  *
2095  * This function sets the pointer of the buffer that should
2096  * loaded directly into DMA for the given USB frame. Passing "ptr"
2097  * equal to NULL while the corresponding "frlength" is greater
2098  * than zero gives undefined results!
2099  *------------------------------------------------------------------------*/
2100 void
2101 usbd_xfer_set_frame_data(struct usb_xfer *xfer, usb_frcount_t frindex,
2102     void *ptr, usb_frlength_t len)
2103 {
2104         KASSERT(frindex < xfer->max_frame_count, ("frame index overflow"));
2105
2106         /* set virtual address to load and length */
2107         xfer->frbuffers[frindex].buffer = ptr;
2108         usbd_xfer_set_frame_len(xfer, frindex, len);
2109 }
2110
2111 void
2112 usbd_xfer_frame_data(struct usb_xfer *xfer, usb_frcount_t frindex,
2113     void **ptr, int *len)
2114 {
2115         KASSERT(frindex < xfer->max_frame_count, ("frame index overflow"));
2116
2117         if (ptr != NULL)
2118                 *ptr = xfer->frbuffers[frindex].buffer;
2119         if (len != NULL)
2120                 *len = xfer->frlengths[frindex];
2121 }
2122
2123 /*------------------------------------------------------------------------*
2124  *      usbd_xfer_old_frame_length
2125  *
2126  * This function returns the framelength of the given frame at the
2127  * time the transfer was submitted. This function can be used to
2128  * compute the starting data pointer of the next isochronous frame
2129  * when an isochronous transfer has completed.
2130  *------------------------------------------------------------------------*/
2131 usb_frlength_t
2132 usbd_xfer_old_frame_length(struct usb_xfer *xfer, usb_frcount_t frindex)
2133 {
2134         KASSERT(frindex < xfer->max_frame_count, ("frame index overflow"));
2135
2136         return (xfer->frlengths[frindex + xfer->max_frame_count]);
2137 }
2138
2139 void
2140 usbd_xfer_status(struct usb_xfer *xfer, int *actlen, int *sumlen, int *aframes,
2141     int *nframes)
2142 {
2143         if (actlen != NULL)
2144                 *actlen = xfer->actlen;
2145         if (sumlen != NULL)
2146                 *sumlen = xfer->sumlen;
2147         if (aframes != NULL)
2148                 *aframes = xfer->aframes;
2149         if (nframes != NULL)
2150                 *nframes = xfer->nframes;
2151 }
2152
2153 /*------------------------------------------------------------------------*
2154  *      usbd_xfer_set_frame_offset
2155  *
2156  * This function sets the frame data buffer offset relative to the beginning
2157  * of the USB DMA buffer allocated for this USB transfer.
2158  *------------------------------------------------------------------------*/
2159 void
2160 usbd_xfer_set_frame_offset(struct usb_xfer *xfer, usb_frlength_t offset,
2161     usb_frcount_t frindex)
2162 {
2163         KASSERT(!xfer->flags.ext_buffer, ("Cannot offset data frame "
2164             "when the USB buffer is external\n"));
2165         KASSERT(frindex < xfer->max_frame_count, ("frame index overflow"));
2166
2167         /* set virtual address to load */
2168         xfer->frbuffers[frindex].buffer =
2169             USB_ADD_BYTES(xfer->local_buffer, offset);
2170 }
2171
2172 void
2173 usbd_xfer_set_interval(struct usb_xfer *xfer, int i)
2174 {
2175         xfer->interval = i;
2176 }
2177
2178 void
2179 usbd_xfer_set_timeout(struct usb_xfer *xfer, int t)
2180 {
2181         xfer->timeout = t;
2182 }
2183
2184 void
2185 usbd_xfer_set_frames(struct usb_xfer *xfer, usb_frcount_t n)
2186 {
2187         xfer->nframes = n;
2188 }
2189
2190 usb_frcount_t
2191 usbd_xfer_max_frames(struct usb_xfer *xfer)
2192 {
2193         return (xfer->max_frame_count);
2194 }
2195
2196 usb_frlength_t
2197 usbd_xfer_max_len(struct usb_xfer *xfer)
2198 {
2199         return (xfer->max_data_length);
2200 }
2201
2202 usb_frlength_t
2203 usbd_xfer_max_framelen(struct usb_xfer *xfer)
2204 {
2205         return (xfer->max_frame_size);
2206 }
2207
2208 void
2209 usbd_xfer_set_frame_len(struct usb_xfer *xfer, usb_frcount_t frindex,
2210     usb_frlength_t len)
2211 {
2212         KASSERT(frindex < xfer->max_frame_count, ("frame index overflow"));
2213
2214         xfer->frlengths[frindex] = len;
2215 }
2216
2217 /*------------------------------------------------------------------------*
2218  *      usb_callback_proc - factored out code
2219  *
2220  * This function performs USB callbacks.
2221  *------------------------------------------------------------------------*/
2222 static void
2223 usb_callback_proc(struct usb_proc_msg *_pm)
2224 {
2225         struct usb_done_msg *pm = (void *)_pm;
2226         struct usb_xfer_root *info = pm->xroot;
2227
2228         /* Change locking order */
2229         USB_BUS_UNLOCK(info->bus);
2230
2231         /*
2232          * We exploit the fact that the mutex is the same for all
2233          * callbacks that will be called from this thread:
2234          */
2235         mtx_lock(info->xfer_mtx);
2236         USB_BUS_LOCK(info->bus);
2237
2238         /* Continue where we lost track */
2239         usb_command_wrapper(&info->done_q,
2240             info->done_q.curr);
2241
2242         mtx_unlock(info->xfer_mtx);
2243 }
2244
2245 /*------------------------------------------------------------------------*
2246  *      usbd_callback_ss_done_defer
2247  *
2248  * This function will defer the start, stop and done callback to the
2249  * correct thread.
2250  *------------------------------------------------------------------------*/
2251 static void
2252 usbd_callback_ss_done_defer(struct usb_xfer *xfer)
2253 {
2254         struct usb_xfer_root *info = xfer->xroot;
2255         struct usb_xfer_queue *pq = &info->done_q;
2256
2257         USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
2258
2259         if (pq->curr != xfer) {
2260                 usbd_transfer_enqueue(pq, xfer);
2261         }
2262         if (!pq->recurse_1) {
2263
2264                 /*
2265                  * We have to postpone the callback due to the fact we
2266                  * will have a Lock Order Reversal, LOR, if we try to
2267                  * proceed !
2268                  */
2269                 if (usb_proc_msignal(info->done_p,
2270                     &info->done_m[0], &info->done_m[1])) {
2271                         /* ignore */
2272                 }
2273         } else {
2274                 /* clear second recurse flag */
2275                 pq->recurse_2 = 0;
2276         }
2277         return;
2278
2279 }
2280
2281 /*------------------------------------------------------------------------*
2282  *      usbd_callback_wrapper
2283  *
2284  * This is a wrapper for USB callbacks. This wrapper does some
2285  * auto-magic things like figuring out if we can call the callback
2286  * directly from the current context or if we need to wakeup the
2287  * interrupt process.
2288  *------------------------------------------------------------------------*/
2289 static void
2290 usbd_callback_wrapper(struct usb_xfer_queue *pq)
2291 {
2292         struct usb_xfer *xfer = pq->curr;
2293         struct usb_xfer_root *info = xfer->xroot;
2294
2295         USB_BUS_LOCK_ASSERT(info->bus, MA_OWNED);
2296         if (!mtx_owned(info->xfer_mtx) && !SCHEDULER_STOPPED()) {
2297                 /*
2298                  * Cases that end up here:
2299                  *
2300                  * 5) HW interrupt done callback or other source.
2301                  */
2302                 DPRINTFN(3, "case 5\n");
2303
2304                 /*
2305                  * We have to postpone the callback due to the fact we
2306                  * will have a Lock Order Reversal, LOR, if we try to
2307                  * proceed !
2308                  */
2309                 if (usb_proc_msignal(info->done_p,
2310                     &info->done_m[0], &info->done_m[1])) {
2311                         /* ignore */
2312                 }
2313                 return;
2314         }
2315         /*
2316          * Cases that end up here:
2317          *
2318          * 1) We are starting a transfer
2319          * 2) We are prematurely calling back a transfer
2320          * 3) We are stopping a transfer
2321          * 4) We are doing an ordinary callback
2322          */
2323         DPRINTFN(3, "case 1-4\n");
2324         /* get next USB transfer in the queue */
2325         info->done_q.curr = NULL;
2326
2327         /* set flag in case of drain */
2328         xfer->flags_int.doing_callback = 1;
2329
2330         USB_BUS_UNLOCK(info->bus);
2331         USB_BUS_LOCK_ASSERT(info->bus, MA_NOTOWNED);
2332
2333         /* set correct USB state for callback */
2334         if (!xfer->flags_int.transferring) {
2335                 xfer->usb_state = USB_ST_SETUP;
2336                 if (!xfer->flags_int.started) {
2337                         /* we got stopped before we even got started */
2338                         USB_BUS_LOCK(info->bus);
2339                         goto done;
2340                 }
2341         } else {
2342
2343                 if (usbd_callback_wrapper_sub(xfer)) {
2344                         /* the callback has been deferred */
2345                         USB_BUS_LOCK(info->bus);
2346                         goto done;
2347                 }
2348 #if USB_HAVE_POWERD
2349                 /* decrement power reference */
2350                 usbd_transfer_power_ref(xfer, -1);
2351 #endif
2352                 xfer->flags_int.transferring = 0;
2353
2354                 if (xfer->error) {
2355                         xfer->usb_state = USB_ST_ERROR;
2356                 } else {
2357                         /* set transferred state */
2358                         xfer->usb_state = USB_ST_TRANSFERRED;
2359 #if USB_HAVE_BUSDMA
2360                         /* sync DMA memory, if any */
2361                         if (xfer->flags_int.bdma_enable &&
2362                             (!xfer->flags_int.bdma_no_post_sync)) {
2363                                 usb_bdma_post_sync(xfer);
2364                         }
2365 #endif
2366                 }
2367         }
2368
2369 #if USB_HAVE_PF
2370         if (xfer->usb_state != USB_ST_SETUP) {
2371                 USB_BUS_LOCK(info->bus);
2372                 usbpf_xfertap(xfer, USBPF_XFERTAP_DONE);
2373                 USB_BUS_UNLOCK(info->bus);
2374         }
2375 #endif
2376         /* call processing routine */
2377         (xfer->callback) (xfer, xfer->error);
2378
2379         /* pickup the USB mutex again */
2380         USB_BUS_LOCK(info->bus);
2381
2382         /*
2383          * Check if we got started after that we got cancelled, but
2384          * before we managed to do the callback.
2385          */
2386         if ((!xfer->flags_int.open) &&
2387             (xfer->flags_int.started) &&
2388             (xfer->usb_state == USB_ST_ERROR)) {
2389                 /* clear flag in case of drain */
2390                 xfer->flags_int.doing_callback = 0;
2391                 /* try to loop, but not recursivly */
2392                 usb_command_wrapper(&info->done_q, xfer);
2393                 return;
2394         }
2395
2396 done:
2397         /* clear flag in case of drain */
2398         xfer->flags_int.doing_callback = 0;
2399
2400         /*
2401          * Check if we are draining.
2402          */
2403         if (xfer->flags_int.draining &&
2404             (!xfer->flags_int.transferring)) {
2405                 /* "usbd_transfer_drain()" is waiting for end of transfer */
2406                 xfer->flags_int.draining = 0;
2407                 cv_broadcast(&info->cv_drain);
2408         }
2409
2410         /* do the next callback, if any */
2411         usb_command_wrapper(&info->done_q,
2412             info->done_q.curr);
2413 }
2414
2415 /*------------------------------------------------------------------------*
2416  *      usb_dma_delay_done_cb
2417  *
2418  * This function is called when the DMA delay has been exectuded, and
2419  * will make sure that the callback is called to complete the USB
2420  * transfer. This code path is ususally only used when there is an USB
2421  * error like USB_ERR_CANCELLED.
2422  *------------------------------------------------------------------------*/
2423 void
2424 usb_dma_delay_done_cb(struct usb_xfer *xfer)
2425 {
2426         USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
2427
2428         DPRINTFN(3, "Completed %p\n", xfer);
2429
2430         /* queue callback for execution, again */
2431         usbd_transfer_done(xfer, 0);
2432 }
2433
2434 /*------------------------------------------------------------------------*
2435  *      usbd_transfer_dequeue
2436  *
2437  *  - This function is used to remove an USB transfer from a USB
2438  *  transfer queue.
2439  *
2440  *  - This function can be called multiple times in a row.
2441  *------------------------------------------------------------------------*/
2442 void
2443 usbd_transfer_dequeue(struct usb_xfer *xfer)
2444 {
2445         struct usb_xfer_queue *pq;
2446
2447         pq = xfer->wait_queue;
2448         if (pq) {
2449                 TAILQ_REMOVE(&pq->head, xfer, wait_entry);
2450                 xfer->wait_queue = NULL;
2451         }
2452 }
2453
2454 /*------------------------------------------------------------------------*
2455  *      usbd_transfer_enqueue
2456  *
2457  *  - This function is used to insert an USB transfer into a USB *
2458  *  transfer queue.
2459  *
2460  *  - This function can be called multiple times in a row.
2461  *------------------------------------------------------------------------*/
2462 void
2463 usbd_transfer_enqueue(struct usb_xfer_queue *pq, struct usb_xfer *xfer)
2464 {
2465         /*
2466          * Insert the USB transfer into the queue, if it is not
2467          * already on a USB transfer queue:
2468          */
2469         if (xfer->wait_queue == NULL) {
2470                 xfer->wait_queue = pq;
2471                 TAILQ_INSERT_TAIL(&pq->head, xfer, wait_entry);
2472         }
2473 }
2474
2475 /*------------------------------------------------------------------------*
2476  *      usbd_transfer_done
2477  *
2478  *  - This function is used to remove an USB transfer from the busdma,
2479  *  pipe or interrupt queue.
2480  *
2481  *  - This function is used to queue the USB transfer on the done
2482  *  queue.
2483  *
2484  *  - This function is used to stop any USB transfer timeouts.
2485  *------------------------------------------------------------------------*/
2486 void
2487 usbd_transfer_done(struct usb_xfer *xfer, usb_error_t error)
2488 {
2489         struct usb_xfer_root *info = xfer->xroot;
2490
2491         USB_BUS_LOCK_ASSERT(info->bus, MA_OWNED);
2492
2493         DPRINTF("err=%s\n", usbd_errstr(error));
2494
2495         /*
2496          * If we are not transferring then just return.
2497          * This can happen during transfer cancel.
2498          */
2499         if (!xfer->flags_int.transferring) {
2500                 DPRINTF("not transferring\n");
2501                 /* end of control transfer, if any */
2502                 xfer->flags_int.control_act = 0;
2503                 return;
2504         }
2505         /* only set transfer error, if not already set */
2506         if (xfer->error == USB_ERR_NORMAL_COMPLETION)
2507                 xfer->error = error;
2508
2509         /* stop any callouts */
2510         usb_callout_stop(&xfer->timeout_handle);
2511
2512         /*
2513          * If we are waiting on a queue, just remove the USB transfer
2514          * from the queue, if any. We should have the required locks
2515          * locked to do the remove when this function is called.
2516          */
2517         usbd_transfer_dequeue(xfer);
2518
2519 #if USB_HAVE_BUSDMA
2520         if (mtx_owned(info->xfer_mtx)) {
2521                 struct usb_xfer_queue *pq;
2522
2523                 /*
2524                  * If the private USB lock is not locked, then we assume
2525                  * that the BUS-DMA load stage has been passed:
2526                  */
2527                 pq = &info->dma_q;
2528
2529                 if (pq->curr == xfer) {
2530                         /* start the next BUS-DMA load, if any */
2531                         usb_command_wrapper(pq, NULL);
2532                 }
2533         }
2534 #endif
2535         /* keep some statistics */
2536         if (xfer->error) {
2537                 info->bus->stats_err.uds_requests
2538                     [xfer->endpoint->edesc->bmAttributes & UE_XFERTYPE]++;
2539         } else {
2540                 info->bus->stats_ok.uds_requests
2541                     [xfer->endpoint->edesc->bmAttributes & UE_XFERTYPE]++;
2542         }
2543
2544         /* call the USB transfer callback */
2545         usbd_callback_ss_done_defer(xfer);
2546 }
2547
2548 /*------------------------------------------------------------------------*
2549  *      usbd_transfer_start_cb
2550  *
2551  * This function is called to start the USB transfer when
2552  * "xfer->interval" is greater than zero, and and the endpoint type is
2553  * BULK or CONTROL.
2554  *------------------------------------------------------------------------*/
2555 static void
2556 usbd_transfer_start_cb(void *arg)
2557 {
2558         struct usb_xfer *xfer = arg;
2559         struct usb_endpoint *ep = xfer->endpoint;
2560
2561         USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
2562
2563         DPRINTF("start\n");
2564
2565 #if USB_HAVE_PF
2566         usbpf_xfertap(xfer, USBPF_XFERTAP_SUBMIT);
2567 #endif
2568
2569         /* the transfer can now be cancelled */
2570         xfer->flags_int.can_cancel_immed = 1;
2571
2572         /* start USB transfer, if no error */
2573         if (xfer->error == 0)
2574                 (ep->methods->start) (xfer);
2575
2576         /* check for transfer error */
2577         if (xfer->error) {
2578                 /* some error has happened */
2579                 usbd_transfer_done(xfer, 0);
2580         }
2581 }
2582
2583 /*------------------------------------------------------------------------*
2584  *      usbd_xfer_set_stall
2585  *
2586  * This function is used to set the stall flag outside the
2587  * callback. This function is NULL safe.
2588  *------------------------------------------------------------------------*/
2589 void
2590 usbd_xfer_set_stall(struct usb_xfer *xfer)
2591 {
2592         if (xfer == NULL) {
2593                 /* tearing down */
2594                 return;
2595         }
2596         USB_XFER_LOCK_ASSERT(xfer, MA_OWNED);
2597
2598         /* avoid any races by locking the USB mutex */
2599         USB_BUS_LOCK(xfer->xroot->bus);
2600         xfer->flags.stall_pipe = 1;
2601         USB_BUS_UNLOCK(xfer->xroot->bus);
2602 }
2603
2604 int
2605 usbd_xfer_is_stalled(struct usb_xfer *xfer)
2606 {
2607         return (xfer->endpoint->is_stalled);
2608 }
2609
2610 /*------------------------------------------------------------------------*
2611  *      usbd_transfer_clear_stall
2612  *
2613  * This function is used to clear the stall flag outside the
2614  * callback. This function is NULL safe.
2615  *------------------------------------------------------------------------*/
2616 void
2617 usbd_transfer_clear_stall(struct usb_xfer *xfer)
2618 {
2619         if (xfer == NULL) {
2620                 /* tearing down */
2621                 return;
2622         }
2623         USB_XFER_LOCK_ASSERT(xfer, MA_OWNED);
2624
2625         /* avoid any races by locking the USB mutex */
2626         USB_BUS_LOCK(xfer->xroot->bus);
2627
2628         xfer->flags.stall_pipe = 0;
2629
2630         USB_BUS_UNLOCK(xfer->xroot->bus);
2631 }
2632
2633 /*------------------------------------------------------------------------*
2634  *      usbd_pipe_start
2635  *
2636  * This function is used to add an USB transfer to the pipe transfer list.
2637  *------------------------------------------------------------------------*/
2638 void
2639 usbd_pipe_start(struct usb_xfer_queue *pq)
2640 {
2641         struct usb_endpoint *ep;
2642         struct usb_xfer *xfer;
2643         uint8_t type;
2644
2645         xfer = pq->curr;
2646         ep = xfer->endpoint;
2647
2648         USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
2649
2650         /*
2651          * If the endpoint is already stalled we do nothing !
2652          */
2653         if (ep->is_stalled) {
2654                 return;
2655         }
2656         /*
2657          * Check if we are supposed to stall the endpoint:
2658          */
2659         if (xfer->flags.stall_pipe) {
2660                 struct usb_device *udev;
2661                 struct usb_xfer_root *info;
2662
2663                 /* clear stall command */
2664                 xfer->flags.stall_pipe = 0;
2665
2666                 /* get pointer to USB device */
2667                 info = xfer->xroot;
2668                 udev = info->udev;
2669
2670                 /*
2671                  * Only stall BULK and INTERRUPT endpoints.
2672                  */
2673                 type = (ep->edesc->bmAttributes & UE_XFERTYPE);
2674                 if ((type == UE_BULK) ||
2675                     (type == UE_INTERRUPT)) {
2676                         uint8_t did_stall;
2677
2678                         did_stall = 1;
2679
2680                         if (udev->flags.usb_mode == USB_MODE_DEVICE) {
2681                                 (udev->bus->methods->set_stall) (
2682                                     udev, NULL, ep, &did_stall);
2683                         } else if (udev->ctrl_xfer[1]) {
2684                                 info = udev->ctrl_xfer[1]->xroot;
2685                                 usb_proc_msignal(
2686                                     &info->bus->non_giant_callback_proc,
2687                                     &udev->cs_msg[0], &udev->cs_msg[1]);
2688                         } else {
2689                                 /* should not happen */
2690                                 DPRINTFN(0, "No stall handler\n");
2691                         }
2692                         /*
2693                          * Check if we should stall. Some USB hardware
2694                          * handles set- and clear-stall in hardware.
2695                          */
2696                         if (did_stall) {
2697                                 /*
2698                                  * The transfer will be continued when
2699                                  * the clear-stall control endpoint
2700                                  * message is received.
2701                                  */
2702                                 ep->is_stalled = 1;
2703                                 return;
2704                         }
2705                 } else if (type == UE_ISOCHRONOUS) {
2706
2707                         /*
2708                          * Make sure any FIFO overflow or other FIFO
2709                          * error conditions go away by resetting the
2710                          * endpoint FIFO through the clear stall
2711                          * method.
2712                          */
2713                         if (udev->flags.usb_mode == USB_MODE_DEVICE) {
2714                                 (udev->bus->methods->clear_stall) (udev, ep);
2715                         }
2716                 }
2717         }
2718         /* Set or clear stall complete - special case */
2719         if (xfer->nframes == 0) {
2720                 /* we are complete */
2721                 xfer->aframes = 0;
2722                 usbd_transfer_done(xfer, 0);
2723                 return;
2724         }
2725         /*
2726          * Handled cases:
2727          *
2728          * 1) Start the first transfer queued.
2729          *
2730          * 2) Re-start the current USB transfer.
2731          */
2732         /*
2733          * Check if there should be any
2734          * pre transfer start delay:
2735          */
2736         if (xfer->interval > 0) {
2737                 type = (ep->edesc->bmAttributes & UE_XFERTYPE);
2738                 if ((type == UE_BULK) ||
2739                     (type == UE_CONTROL)) {
2740                         usbd_transfer_timeout_ms(xfer,
2741                             &usbd_transfer_start_cb,
2742                             xfer->interval);
2743                         return;
2744                 }
2745         }
2746         DPRINTF("start\n");
2747
2748 #if USB_HAVE_PF
2749         usbpf_xfertap(xfer, USBPF_XFERTAP_SUBMIT);
2750 #endif
2751         /* the transfer can now be cancelled */
2752         xfer->flags_int.can_cancel_immed = 1;
2753
2754         /* start USB transfer, if no error */
2755         if (xfer->error == 0)
2756                 (ep->methods->start) (xfer);
2757
2758         /* check for transfer error */
2759         if (xfer->error) {
2760                 /* some error has happened */
2761                 usbd_transfer_done(xfer, 0);
2762         }
2763 }
2764
2765 /*------------------------------------------------------------------------*
2766  *      usbd_transfer_timeout_ms
2767  *
2768  * This function is used to setup a timeout on the given USB
2769  * transfer. If the timeout has been deferred the callback given by
2770  * "cb" will get called after "ms" milliseconds.
2771  *------------------------------------------------------------------------*/
2772 void
2773 usbd_transfer_timeout_ms(struct usb_xfer *xfer,
2774     void (*cb) (void *arg), usb_timeout_t ms)
2775 {
2776         USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
2777
2778         /* defer delay */
2779         usb_callout_reset(&xfer->timeout_handle,
2780             USB_MS_TO_TICKS(ms) + USB_CALLOUT_ZERO_TICKS, cb, xfer);
2781 }
2782
2783 /*------------------------------------------------------------------------*
2784  *      usbd_callback_wrapper_sub
2785  *
2786  *  - This function will update variables in an USB transfer after
2787  *  that the USB transfer is complete.
2788  *
2789  *  - This function is used to start the next USB transfer on the
2790  *  ep transfer queue, if any.
2791  *
2792  * NOTE: In some special cases the USB transfer will not be removed from
2793  * the pipe queue, but remain first. To enforce USB transfer removal call
2794  * this function passing the error code "USB_ERR_CANCELLED".
2795  *
2796  * Return values:
2797  * 0: Success.
2798  * Else: The callback has been deferred.
2799  *------------------------------------------------------------------------*/
2800 static uint8_t
2801 usbd_callback_wrapper_sub(struct usb_xfer *xfer)
2802 {
2803         struct usb_endpoint *ep;
2804         struct usb_bus *bus;
2805         usb_frcount_t x;
2806
2807         bus = xfer->xroot->bus;
2808
2809         if ((!xfer->flags_int.open) &&
2810             (!xfer->flags_int.did_close)) {
2811                 DPRINTF("close\n");
2812                 USB_BUS_LOCK(bus);
2813                 (xfer->endpoint->methods->close) (xfer);
2814                 USB_BUS_UNLOCK(bus);
2815                 /* only close once */
2816                 xfer->flags_int.did_close = 1;
2817                 return (1);             /* wait for new callback */
2818         }
2819         /*
2820          * If we have a non-hardware induced error we
2821          * need to do the DMA delay!
2822          */
2823         if (xfer->error != 0 && !xfer->flags_int.did_dma_delay &&
2824             (xfer->error == USB_ERR_CANCELLED ||
2825             xfer->error == USB_ERR_TIMEOUT ||
2826             bus->methods->start_dma_delay != NULL)) {
2827
2828                 usb_timeout_t temp;
2829
2830                 /* only delay once */
2831                 xfer->flags_int.did_dma_delay = 1;
2832
2833                 /* we can not cancel this delay */
2834                 xfer->flags_int.can_cancel_immed = 0;
2835
2836                 temp = usbd_get_dma_delay(xfer->xroot->udev);
2837
2838                 DPRINTFN(3, "DMA delay, %u ms, "
2839                     "on %p\n", temp, xfer);
2840
2841                 if (temp != 0) {
2842                         USB_BUS_LOCK(bus);
2843                         /*
2844                          * Some hardware solutions have dedicated
2845                          * events when it is safe to free DMA'ed
2846                          * memory. For the other hardware platforms we
2847                          * use a static delay.
2848                          */
2849                         if (bus->methods->start_dma_delay != NULL) {
2850                                 (bus->methods->start_dma_delay) (xfer);
2851                         } else {
2852                                 usbd_transfer_timeout_ms(xfer,
2853                                     (void (*)(void *))&usb_dma_delay_done_cb,
2854                                     temp);
2855                         }
2856                         USB_BUS_UNLOCK(bus);
2857                         return (1);     /* wait for new callback */
2858                 }
2859         }
2860         /* check actual number of frames */
2861         if (xfer->aframes > xfer->nframes) {
2862                 if (xfer->error == 0) {
2863                         panic("%s: actual number of frames, %d, is "
2864                             "greater than initial number of frames, %d\n",
2865                             __FUNCTION__, xfer->aframes, xfer->nframes);
2866                 } else {
2867                         /* just set some valid value */
2868                         xfer->aframes = xfer->nframes;
2869                 }
2870         }
2871         /* compute actual length */
2872         xfer->actlen = 0;
2873
2874         for (x = 0; x != xfer->aframes; x++) {
2875                 xfer->actlen += xfer->frlengths[x];
2876         }
2877
2878         /*
2879          * Frames that were not transferred get zero actual length in
2880          * case the USB device driver does not check the actual number
2881          * of frames transferred, "xfer->aframes":
2882          */
2883         for (; x < xfer->nframes; x++) {
2884                 usbd_xfer_set_frame_len(xfer, x, 0);
2885         }
2886
2887         /* check actual length */
2888         if (xfer->actlen > xfer->sumlen) {
2889                 if (xfer->error == 0) {
2890                         panic("%s: actual length, %d, is greater than "
2891                             "initial length, %d\n",
2892                             __FUNCTION__, xfer->actlen, xfer->sumlen);
2893                 } else {
2894                         /* just set some valid value */
2895                         xfer->actlen = xfer->sumlen;
2896                 }
2897         }
2898         DPRINTFN(1, "xfer=%p endpoint=%p sts=%d alen=%d, slen=%d, afrm=%d, nfrm=%d\n",
2899             xfer, xfer->endpoint, xfer->error, xfer->actlen, xfer->sumlen,
2900             xfer->aframes, xfer->nframes);
2901
2902         if (xfer->error) {
2903                 /* end of control transfer, if any */
2904                 xfer->flags_int.control_act = 0;
2905
2906 #if USB_HAVE_TT_SUPPORT
2907                 switch (xfer->error) {
2908                 case USB_ERR_NORMAL_COMPLETION:
2909                 case USB_ERR_SHORT_XFER:
2910                 case USB_ERR_STALLED:
2911                 case USB_ERR_CANCELLED:
2912                         /* nothing to do */
2913                         break;
2914                 default:
2915                         /* try to reset the TT, if any */
2916                         USB_BUS_LOCK(bus);
2917                         uhub_tt_buffer_reset_async_locked(xfer->xroot->udev, xfer->endpoint);
2918                         USB_BUS_UNLOCK(bus);
2919                         break;
2920                 }
2921 #endif
2922                 /* check if we should block the execution queue */
2923                 if ((xfer->error != USB_ERR_CANCELLED) &&
2924                     (xfer->flags.pipe_bof)) {
2925                         DPRINTFN(2, "xfer=%p: Block On Failure "
2926                             "on endpoint=%p\n", xfer, xfer->endpoint);
2927                         goto done;
2928                 }
2929         } else {
2930                 /* check for short transfers */
2931                 if (xfer->actlen < xfer->sumlen) {
2932
2933                         /* end of control transfer, if any */
2934                         xfer->flags_int.control_act = 0;
2935
2936                         if (!xfer->flags_int.short_xfer_ok) {
2937                                 xfer->error = USB_ERR_SHORT_XFER;
2938                                 if (xfer->flags.pipe_bof) {
2939                                         DPRINTFN(2, "xfer=%p: Block On Failure on "
2940                                             "Short Transfer on endpoint %p.\n",
2941                                             xfer, xfer->endpoint);
2942                                         goto done;
2943                                 }
2944                         }
2945                 } else {
2946                         /*
2947                          * Check if we are in the middle of a
2948                          * control transfer:
2949                          */
2950                         if (xfer->flags_int.control_act) {
2951                                 DPRINTFN(5, "xfer=%p: Control transfer "
2952                                     "active on endpoint=%p\n", xfer, xfer->endpoint);
2953                                 goto done;
2954                         }
2955                 }
2956         }
2957
2958         ep = xfer->endpoint;
2959
2960         /*
2961          * If the current USB transfer is completing we need to start the
2962          * next one:
2963          */
2964         USB_BUS_LOCK(bus);
2965         if (ep->endpoint_q.curr == xfer) {
2966                 usb_command_wrapper(&ep->endpoint_q, NULL);
2967
2968                 if (ep->endpoint_q.curr || TAILQ_FIRST(&ep->endpoint_q.head)) {
2969                         /* there is another USB transfer waiting */
2970                 } else {
2971                         /* this is the last USB transfer */
2972                         /* clear isochronous sync flag */
2973                         xfer->endpoint->is_synced = 0;
2974                 }
2975         }
2976         USB_BUS_UNLOCK(bus);
2977 done:
2978         return (0);
2979 }
2980
2981 /*------------------------------------------------------------------------*
2982  *      usb_command_wrapper
2983  *
2984  * This function is used to execute commands non-recursivly on an USB
2985  * transfer.
2986  *------------------------------------------------------------------------*/
2987 void
2988 usb_command_wrapper(struct usb_xfer_queue *pq, struct usb_xfer *xfer)
2989 {
2990         if (xfer) {
2991                 /*
2992                  * If the transfer is not already processing,
2993                  * queue it!
2994                  */
2995                 if (pq->curr != xfer) {
2996                         usbd_transfer_enqueue(pq, xfer);
2997                         if (pq->curr != NULL) {
2998                                 /* something is already processing */
2999                                 DPRINTFN(6, "busy %p\n", pq->curr);
3000                                 return;
3001                         }
3002                 }
3003         } else {
3004                 /* Get next element in queue */
3005                 pq->curr = NULL;
3006         }
3007
3008         if (!pq->recurse_1) {
3009
3010                 do {
3011
3012                         /* set both recurse flags */
3013                         pq->recurse_1 = 1;
3014                         pq->recurse_2 = 1;
3015
3016                         if (pq->curr == NULL) {
3017                                 xfer = TAILQ_FIRST(&pq->head);
3018                                 if (xfer) {
3019                                         TAILQ_REMOVE(&pq->head, xfer,
3020                                             wait_entry);
3021                                         xfer->wait_queue = NULL;
3022                                         pq->curr = xfer;
3023                                 } else {
3024                                         break;
3025                                 }
3026                         }
3027                         DPRINTFN(6, "cb %p (enter)\n", pq->curr);
3028                         (pq->command) (pq);
3029                         DPRINTFN(6, "cb %p (leave)\n", pq->curr);
3030
3031                 } while (!pq->recurse_2);
3032
3033                 /* clear first recurse flag */
3034                 pq->recurse_1 = 0;
3035
3036         } else {
3037                 /* clear second recurse flag */
3038                 pq->recurse_2 = 0;
3039         }
3040 }
3041
3042 /*------------------------------------------------------------------------*
3043  *      usbd_ctrl_transfer_setup
3044  *
3045  * This function is used to setup the default USB control endpoint
3046  * transfer.
3047  *------------------------------------------------------------------------*/
3048 void
3049 usbd_ctrl_transfer_setup(struct usb_device *udev)
3050 {
3051         struct usb_xfer *xfer;
3052         uint8_t no_resetup;
3053         uint8_t iface_index;
3054
3055         /* check for root HUB */
3056         if (udev->parent_hub == NULL)
3057                 return;
3058 repeat:
3059
3060         xfer = udev->ctrl_xfer[0];
3061         if (xfer) {
3062                 USB_XFER_LOCK(xfer);
3063                 no_resetup =
3064                     ((xfer->address == udev->address) &&
3065                     (udev->ctrl_ep_desc.wMaxPacketSize[0] ==
3066                     udev->ddesc.bMaxPacketSize));
3067                 if (udev->flags.usb_mode == USB_MODE_DEVICE) {
3068                         if (no_resetup) {
3069                                 /*
3070                                  * NOTE: checking "xfer->address" and
3071                                  * starting the USB transfer must be
3072                                  * atomic!
3073                                  */
3074                                 usbd_transfer_start(xfer);
3075                         }
3076                 }
3077                 USB_XFER_UNLOCK(xfer);
3078         } else {
3079                 no_resetup = 0;
3080         }
3081
3082         if (no_resetup) {
3083                 /*
3084                  * All parameters are exactly the same like before.
3085                  * Just return.
3086                  */
3087                 return;
3088         }
3089         /*
3090          * Update wMaxPacketSize for the default control endpoint:
3091          */
3092         udev->ctrl_ep_desc.wMaxPacketSize[0] =
3093             udev->ddesc.bMaxPacketSize;
3094
3095         /*
3096          * Unsetup any existing USB transfer:
3097          */
3098         usbd_transfer_unsetup(udev->ctrl_xfer, USB_CTRL_XFER_MAX);
3099
3100         /*
3101          * Reset clear stall error counter.
3102          */
3103         udev->clear_stall_errors = 0;
3104
3105         /*
3106          * Try to setup a new USB transfer for the
3107          * default control endpoint:
3108          */
3109         iface_index = 0;
3110         if (usbd_transfer_setup(udev, &iface_index,
3111             udev->ctrl_xfer, usb_control_ep_cfg, USB_CTRL_XFER_MAX, NULL,
3112             &udev->device_mtx)) {
3113                 DPRINTFN(0, "could not setup default "
3114                     "USB transfer\n");
3115         } else {
3116                 goto repeat;
3117         }
3118 }
3119
3120 /*------------------------------------------------------------------------*
3121  *      usbd_clear_data_toggle - factored out code
3122  *
3123  * NOTE: the intention of this function is not to reset the hardware
3124  * data toggle.
3125  *------------------------------------------------------------------------*/
3126 void
3127 usbd_clear_stall_locked(struct usb_device *udev, struct usb_endpoint *ep)
3128 {
3129         USB_BUS_LOCK_ASSERT(udev->bus, MA_OWNED);
3130
3131         /* check that we have a valid case */
3132         if (udev->flags.usb_mode == USB_MODE_HOST &&
3133             udev->parent_hub != NULL &&
3134             udev->bus->methods->clear_stall != NULL &&
3135             ep->methods != NULL) {
3136                 (udev->bus->methods->clear_stall) (udev, ep);
3137         }
3138 }
3139
3140 /*------------------------------------------------------------------------*
3141  *      usbd_clear_data_toggle - factored out code
3142  *
3143  * NOTE: the intention of this function is not to reset the hardware
3144  * data toggle on the USB device side.
3145  *------------------------------------------------------------------------*/
3146 void
3147 usbd_clear_data_toggle(struct usb_device *udev, struct usb_endpoint *ep)
3148 {
3149         DPRINTFN(5, "udev=%p endpoint=%p\n", udev, ep);
3150
3151         USB_BUS_LOCK(udev->bus);
3152         ep->toggle_next = 0;
3153         /* some hardware needs a callback to clear the data toggle */
3154         usbd_clear_stall_locked(udev, ep);
3155         USB_BUS_UNLOCK(udev->bus);
3156 }
3157
3158 /*------------------------------------------------------------------------*
3159  *      usbd_clear_stall_callback - factored out clear stall callback
3160  *
3161  * Input parameters:
3162  *  xfer1: Clear Stall Control Transfer
3163  *  xfer2: Stalled USB Transfer
3164  *
3165  * This function is NULL safe.
3166  *
3167  * Return values:
3168  *   0: In progress
3169  *   Else: Finished
3170  *
3171  * Clear stall config example:
3172  *
3173  * static const struct usb_config my_clearstall =  {
3174  *      .type = UE_CONTROL,
3175  *      .endpoint = 0,
3176  *      .direction = UE_DIR_ANY,
3177  *      .interval = 50, //50 milliseconds
3178  *      .bufsize = sizeof(struct usb_device_request),
3179  *      .timeout = 1000, //1.000 seconds
3180  *      .callback = &my_clear_stall_callback, // **
3181  *      .usb_mode = USB_MODE_HOST,
3182  * };
3183  *
3184  * ** "my_clear_stall_callback" calls "usbd_clear_stall_callback"
3185  * passing the correct parameters.
3186  *------------------------------------------------------------------------*/
3187 uint8_t
3188 usbd_clear_stall_callback(struct usb_xfer *xfer1,
3189     struct usb_xfer *xfer2)
3190 {
3191         struct usb_device_request req;
3192
3193         if (xfer2 == NULL) {
3194                 /* looks like we are tearing down */
3195                 DPRINTF("NULL input parameter\n");
3196                 return (0);
3197         }
3198         USB_XFER_LOCK_ASSERT(xfer1, MA_OWNED);
3199         USB_XFER_LOCK_ASSERT(xfer2, MA_OWNED);
3200
3201         switch (USB_GET_STATE(xfer1)) {
3202         case USB_ST_SETUP:
3203
3204                 /*
3205                  * pre-clear the data toggle to DATA0 ("umass.c" and
3206                  * "ata-usb.c" depends on this)
3207                  */
3208
3209                 usbd_clear_data_toggle(xfer2->xroot->udev, xfer2->endpoint);
3210
3211                 /* setup a clear-stall packet */
3212
3213                 req.bmRequestType = UT_WRITE_ENDPOINT;
3214                 req.bRequest = UR_CLEAR_FEATURE;
3215                 USETW(req.wValue, UF_ENDPOINT_HALT);
3216                 req.wIndex[0] = xfer2->endpoint->edesc->bEndpointAddress;
3217                 req.wIndex[1] = 0;
3218                 USETW(req.wLength, 0);
3219
3220                 /*
3221                  * "usbd_transfer_setup_sub()" will ensure that
3222                  * we have sufficient room in the buffer for
3223                  * the request structure!
3224                  */
3225
3226                 /* copy in the transfer */
3227
3228                 usbd_copy_in(xfer1->frbuffers, 0, &req, sizeof(req));
3229
3230                 /* set length */
3231                 xfer1->frlengths[0] = sizeof(req);
3232                 xfer1->nframes = 1;
3233
3234                 usbd_transfer_submit(xfer1);
3235                 return (0);
3236
3237         case USB_ST_TRANSFERRED:
3238                 break;
3239
3240         default:                        /* Error */
3241                 if (xfer1->error == USB_ERR_CANCELLED) {
3242                         return (0);
3243                 }
3244                 break;
3245         }
3246         return (1);                     /* Clear Stall Finished */
3247 }
3248
3249 /*------------------------------------------------------------------------*
3250  *      usbd_transfer_poll
3251  *
3252  * The following function gets called from the USB keyboard driver and
3253  * UMASS when the system has paniced.
3254  *
3255  * NOTE: It is currently not possible to resume normal operation on
3256  * the USB controller which has been polled, due to clearing of the
3257  * "up_dsleep" and "up_msleep" flags.
3258  *------------------------------------------------------------------------*/
3259 void
3260 usbd_transfer_poll(struct usb_xfer **ppxfer, uint16_t max)
3261 {
3262         struct usb_xfer *xfer;
3263         struct usb_xfer_root *xroot;
3264         struct usb_device *udev;
3265         struct usb_proc_msg *pm;
3266         uint16_t n;
3267         uint16_t drop_bus;
3268         uint16_t drop_xfer;
3269
3270         for (n = 0; n != max; n++) {
3271                 /* Extra checks to avoid panic */
3272                 xfer = ppxfer[n];
3273                 if (xfer == NULL)
3274                         continue;       /* no USB transfer */
3275                 xroot = xfer->xroot;
3276                 if (xroot == NULL)
3277                         continue;       /* no USB root */
3278                 udev = xroot->udev;
3279                 if (udev == NULL)
3280                         continue;       /* no USB device */
3281                 if (udev->bus == NULL)
3282                         continue;       /* no BUS structure */
3283                 if (udev->bus->methods == NULL)
3284                         continue;       /* no BUS methods */
3285                 if (udev->bus->methods->xfer_poll == NULL)
3286                         continue;       /* no poll method */
3287
3288                 /* make sure that the BUS mutex is not locked */
3289                 drop_bus = 0;
3290                 while (mtx_owned(&xroot->udev->bus->bus_mtx) && !SCHEDULER_STOPPED()) {
3291                         mtx_unlock(&xroot->udev->bus->bus_mtx);
3292                         drop_bus++;
3293                 }
3294
3295                 /* make sure that the transfer mutex is not locked */
3296                 drop_xfer = 0;
3297                 while (mtx_owned(xroot->xfer_mtx) && !SCHEDULER_STOPPED()) {
3298                         mtx_unlock(xroot->xfer_mtx);
3299                         drop_xfer++;
3300                 }
3301
3302                 /* Make sure cv_signal() and cv_broadcast() is not called */
3303                 udev->bus->control_xfer_proc.up_msleep = 0;
3304                 udev->bus->explore_proc.up_msleep = 0;
3305                 udev->bus->giant_callback_proc.up_msleep = 0;
3306                 udev->bus->non_giant_callback_proc.up_msleep = 0;
3307
3308                 /* poll USB hardware */
3309                 (udev->bus->methods->xfer_poll) (udev->bus);
3310
3311                 USB_BUS_LOCK(xroot->bus);
3312
3313                 /* check for clear stall */
3314                 if (udev->ctrl_xfer[1] != NULL) {
3315
3316                         /* poll clear stall start */
3317                         pm = &udev->cs_msg[0].hdr;
3318                         (pm->pm_callback) (pm);
3319                         /* poll clear stall done thread */
3320                         pm = &udev->ctrl_xfer[1]->
3321                             xroot->done_m[0].hdr;
3322                         (pm->pm_callback) (pm);
3323                 }
3324
3325                 /* poll done thread */
3326                 pm = &xroot->done_m[0].hdr;
3327                 (pm->pm_callback) (pm);
3328
3329                 USB_BUS_UNLOCK(xroot->bus);
3330
3331                 /* restore transfer mutex */
3332                 while (drop_xfer--)
3333                         mtx_lock(xroot->xfer_mtx);
3334
3335                 /* restore BUS mutex */
3336                 while (drop_bus--)
3337                         mtx_lock(&xroot->udev->bus->bus_mtx);
3338         }
3339 }
3340
3341 static void
3342 usbd_get_std_packet_size(struct usb_std_packet_size *ptr,
3343     uint8_t type, enum usb_dev_speed speed)
3344 {
3345         static const uint16_t intr_range_max[USB_SPEED_MAX] = {
3346                 [USB_SPEED_LOW] = 8,
3347                 [USB_SPEED_FULL] = 64,
3348                 [USB_SPEED_HIGH] = 1024,
3349                 [USB_SPEED_VARIABLE] = 1024,
3350                 [USB_SPEED_SUPER] = 1024,
3351         };
3352
3353         static const uint16_t isoc_range_max[USB_SPEED_MAX] = {
3354                 [USB_SPEED_LOW] = 0,    /* invalid */
3355                 [USB_SPEED_FULL] = 1023,
3356                 [USB_SPEED_HIGH] = 1024,
3357                 [USB_SPEED_VARIABLE] = 3584,
3358                 [USB_SPEED_SUPER] = 1024,
3359         };
3360
3361         static const uint16_t control_min[USB_SPEED_MAX] = {
3362                 [USB_SPEED_LOW] = 8,
3363                 [USB_SPEED_FULL] = 8,
3364                 [USB_SPEED_HIGH] = 64,
3365                 [USB_SPEED_VARIABLE] = 512,
3366                 [USB_SPEED_SUPER] = 512,
3367         };
3368
3369         static const uint16_t bulk_min[USB_SPEED_MAX] = {
3370                 [USB_SPEED_LOW] = 8,
3371                 [USB_SPEED_FULL] = 8,
3372                 [USB_SPEED_HIGH] = 512,
3373                 [USB_SPEED_VARIABLE] = 512,
3374                 [USB_SPEED_SUPER] = 1024,
3375         };
3376
3377         uint16_t temp;
3378
3379         memset(ptr, 0, sizeof(*ptr));
3380
3381         switch (type) {
3382         case UE_INTERRUPT:
3383                 ptr->range.max = intr_range_max[speed];
3384                 break;
3385         case UE_ISOCHRONOUS:
3386                 ptr->range.max = isoc_range_max[speed];
3387                 break;
3388         default:
3389                 if (type == UE_BULK)
3390                         temp = bulk_min[speed];
3391                 else /* UE_CONTROL */
3392                         temp = control_min[speed];
3393
3394                 /* default is fixed */
3395                 ptr->fixed[0] = temp;
3396                 ptr->fixed[1] = temp;
3397                 ptr->fixed[2] = temp;
3398                 ptr->fixed[3] = temp;
3399
3400                 if (speed == USB_SPEED_FULL) {
3401                         /* multiple sizes */
3402                         ptr->fixed[1] = 16;
3403                         ptr->fixed[2] = 32;
3404                         ptr->fixed[3] = 64;
3405                 }
3406                 if ((speed == USB_SPEED_VARIABLE) &&
3407                     (type == UE_BULK)) {
3408                         /* multiple sizes */
3409                         ptr->fixed[2] = 1024;
3410                         ptr->fixed[3] = 1536;
3411                 }
3412                 break;
3413         }
3414 }
3415
3416 void    *
3417 usbd_xfer_softc(struct usb_xfer *xfer)
3418 {
3419         return (xfer->priv_sc);
3420 }
3421
3422 void *
3423 usbd_xfer_get_priv(struct usb_xfer *xfer)
3424 {
3425         return (xfer->priv_fifo);
3426 }
3427
3428 void
3429 usbd_xfer_set_priv(struct usb_xfer *xfer, void *ptr)
3430 {
3431         xfer->priv_fifo = ptr;
3432 }
3433
3434 uint8_t
3435 usbd_xfer_state(struct usb_xfer *xfer)
3436 {
3437         return (xfer->usb_state);
3438 }
3439
3440 void
3441 usbd_xfer_set_flag(struct usb_xfer *xfer, int flag)
3442 {
3443         switch (flag) {
3444                 case USB_FORCE_SHORT_XFER:
3445                         xfer->flags.force_short_xfer = 1;
3446                         break;
3447                 case USB_SHORT_XFER_OK:
3448                         xfer->flags.short_xfer_ok = 1;
3449                         break;
3450                 case USB_MULTI_SHORT_OK:
3451                         xfer->flags.short_frames_ok = 1;
3452                         break;
3453                 case USB_MANUAL_STATUS:
3454                         xfer->flags.manual_status = 1;
3455                         break;
3456         }
3457 }
3458
3459 void
3460 usbd_xfer_clr_flag(struct usb_xfer *xfer, int flag)
3461 {
3462         switch (flag) {
3463                 case USB_FORCE_SHORT_XFER:
3464                         xfer->flags.force_short_xfer = 0;
3465                         break;
3466                 case USB_SHORT_XFER_OK:
3467                         xfer->flags.short_xfer_ok = 0;
3468                         break;
3469                 case USB_MULTI_SHORT_OK:
3470                         xfer->flags.short_frames_ok = 0;
3471                         break;
3472                 case USB_MANUAL_STATUS:
3473                         xfer->flags.manual_status = 0;
3474                         break;
3475         }
3476 }
3477
3478 /*
3479  * The following function returns in milliseconds when the isochronous
3480  * transfer was completed by the hardware. The returned value wraps
3481  * around 65536 milliseconds.
3482  */
3483 uint16_t
3484 usbd_xfer_get_timestamp(struct usb_xfer *xfer)
3485 {
3486         return (xfer->isoc_time_complete);
3487 }
3488
3489 /*
3490  * The following function returns non-zero if the max packet size
3491  * field was clamped to a valid value. Else it returns zero.
3492  */
3493 uint8_t
3494 usbd_xfer_maxp_was_clamped(struct usb_xfer *xfer)
3495 {
3496         return (xfer->flags_int.maxp_was_clamped);
3497 }