sys/dev/usb/controller/dwc_otg.c

   1 /* $FreeBSD$ */
   2 /*-
   3  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   4  *
   5  * Copyright (c) 2015 Daisuke Aoyama. All rights reserved.
   6  * Copyright (c) 2012-2015 Hans Petter Selasky. All rights reserved.
   7  * Copyright (c) 2010-2011 Aleksandr Rybalko. All rights reserved.
   8  *
   9  * Redistribution and use in source and binary forms, with or without
  10  * modification, are permitted provided that the following conditions
  11  * are met:
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions and the following disclaimer.
  14  * 2. Redistributions in binary form must reproduce the above copyright
  15  *    notice, this list of conditions and the following disclaimer in the
  16  *    documentation and/or other materials provided with the distribution.
  17  *
  18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  21  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  28  * SUCH DAMAGE.
  29  */
  30
  31 /*
  32  * This file contains the driver for the DesignWare series USB 2.0 OTG
  33  * Controller.
  34  */
  35
  36 /*
  37  * LIMITATION: Drivers must be bound to all OUT endpoints in the
  38  * active configuration for this driver to work properly. Blocking any
  39  * OUT endpoint will block all OUT endpoints including the control
  40  * endpoint. Usually this is not a problem.
  41  */
  42
  43 /*
  44  * NOTE: Writing to non-existing registers appears to cause an
  45  * internal reset.
  46  */
  47
  48 #ifdef USB_GLOBAL_INCLUDE_FILE
  49 #include USB_GLOBAL_INCLUDE_FILE
  50 #else
  51 #include <sys/stdint.h>
  52 #include <sys/stddef.h>
  53 #include <sys/param.h>
  54 #include <sys/queue.h>
  55 #include <sys/types.h>
  56 #include <sys/systm.h>
  57 #include <sys/kernel.h>
  58 #include <sys/bus.h>
  59 #include <sys/module.h>
  60 #include <sys/lock.h>
  61 #include <sys/mutex.h>
  62 #include <sys/condvar.h>
  63 #include <sys/sysctl.h>
  64 #include <sys/sx.h>
  65 #include <sys/unistd.h>
  66 #include <sys/callout.h>
  67 #include <sys/malloc.h>
  68 #include <sys/priv.h>
  69
  70 #include <dev/usb/usb.h>
  71 #include <dev/usb/usbdi.h>
  72
  73 #define USB_DEBUG_VAR dwc_otg_debug
  74
  75 #include <dev/usb/usb_core.h>
  76 #include <dev/usb/usb_debug.h>
  77 #include <dev/usb/usb_busdma.h>
  78 #include <dev/usb/usb_process.h>
  79 #include <dev/usb/usb_transfer.h>
  80 #include <dev/usb/usb_device.h>
  81 #include <dev/usb/usb_hub.h>
  82 #include <dev/usb/usb_util.h>
  83
  84 #include <dev/usb/usb_controller.h>
  85 #include <dev/usb/usb_bus.h>
  86 #endif                  /* USB_GLOBAL_INCLUDE_FILE */
  87
  88 #include <dev/usb/controller/dwc_otg.h>
  89 #include <dev/usb/controller/dwc_otgreg.h>
  90
  91 #define DWC_OTG_BUS2SC(bus) \
  92    ((struct dwc_otg_softc *)(((uint8_t *)(bus)) - \
  93     ((uint8_t *)&(((struct dwc_otg_softc *)0)->sc_bus))))
  94
  95 #define DWC_OTG_PC2UDEV(pc) \
  96    (USB_DMATAG_TO_XROOT((pc)->tag_parent)->udev)
  97
  98 #define DWC_OTG_MSK_GINT_THREAD_IRQ                             \
  99    (GINTSTS_USBRST | GINTSTS_ENUMDONE | GINTSTS_PRTINT |        \
 100    GINTSTS_WKUPINT | GINTSTS_USBSUSP | GINTMSK_OTGINTMSK |      \
 101    GINTSTS_SESSREQINT)
 102
 103 #ifndef DWC_OTG_PHY_DEFAULT
 104 #define DWC_OTG_PHY_DEFAULT DWC_OTG_PHY_ULPI
 105 #endif
 106
 107 static int dwc_otg_phy_type = DWC_OTG_PHY_DEFAULT;
 108
 109 static SYSCTL_NODE(_hw_usb, OID_AUTO, dwc_otg, CTLFLAG_RW, 0, "USB DWC OTG");
 110 SYSCTL_INT(_hw_usb_dwc_otg, OID_AUTO, phy_type, CTLFLAG_RDTUN,
 111     &dwc_otg_phy_type, 0, "DWC OTG PHY TYPE - 0/1/2/3 - ULPI/HSIC/INTERNAL/UTMI+");
 112
 113 #ifdef USB_DEBUG
 114 static int dwc_otg_debug = 0;
 115
 116 SYSCTL_INT(_hw_usb_dwc_otg, OID_AUTO, debug, CTLFLAG_RWTUN,
 117     &dwc_otg_debug, 0, "DWC OTG debug level");
 118 #endif
 119
 120 #define DWC_OTG_INTR_ENDPT 1
 121
 122 /* prototypes */
 123
 124 static const struct usb_bus_methods dwc_otg_bus_methods;
 125 static const struct usb_pipe_methods dwc_otg_device_non_isoc_methods;
 126 static const struct usb_pipe_methods dwc_otg_device_isoc_methods;
 127
 128 static dwc_otg_cmd_t dwc_otg_setup_rx;
 129 static dwc_otg_cmd_t dwc_otg_data_rx;
 130 static dwc_otg_cmd_t dwc_otg_data_tx;
 131 static dwc_otg_cmd_t dwc_otg_data_tx_sync;
 132
 133 static dwc_otg_cmd_t dwc_otg_host_setup_tx;
 134 static dwc_otg_cmd_t dwc_otg_host_data_tx;
 135 static dwc_otg_cmd_t dwc_otg_host_data_rx;
 136
 137 static void dwc_otg_device_done(struct usb_xfer *, usb_error_t);
 138 static void dwc_otg_do_poll(struct usb_bus *);
 139 static void dwc_otg_standard_done(struct usb_xfer *);
 140 static void dwc_otg_root_intr(struct dwc_otg_softc *);
 141 static void dwc_otg_interrupt_poll_locked(struct dwc_otg_softc *);
 142
 143 /*
 144  * Here is a configuration that the chip supports.
 145  */
 146 static const struct usb_hw_ep_profile dwc_otg_ep_profile[1] = {
 147
 148         [0] = {
 149                 .max_in_frame_size = 64,/* fixed */
 150                 .max_out_frame_size = 64,       /* fixed */
 151                 .is_simplex = 1,
 152                 .support_control = 1,
 153         }
 154 };
 155
 156 static void
 157 dwc_otg_get_hw_ep_profile(struct usb_device *udev,
 158     const struct usb_hw_ep_profile **ppf, uint8_t ep_addr)
 159 {
 160         struct dwc_otg_softc *sc;
 161
 162         sc = DWC_OTG_BUS2SC(udev->bus);
 163
 164         if (ep_addr < sc->sc_dev_ep_max)
 165                 *ppf = &sc->sc_hw_ep_profile[ep_addr].usb;
 166         else
 167                 *ppf = NULL;
 168 }
 169
 170 static void
 171 dwc_otg_write_fifo(struct dwc_otg_softc *sc, struct usb_page_cache *pc,
 172     uint32_t offset, uint32_t fifo, uint32_t count)
 173 {
 174         uint32_t temp;
 175
 176         /* round down length to nearest 4-bytes */
 177         temp = count & ~3;
 178
 179         /* check if we can write the data directly */
 180         if (temp != 0 && usb_pc_buffer_is_aligned(pc, offset, temp, 3)) {
 181                 struct usb_page_search buf_res;
 182
 183                 /* pre-subtract length */
 184                 count -= temp;
 185
 186                 /* iterate buffer list */
 187                 do {
 188                         /* get current buffer pointer */
 189                         usbd_get_page(pc, offset, &buf_res);
 190
 191                         if (buf_res.length > temp)
 192                                 buf_res.length = temp;
 193
 194                         /* transfer data into FIFO */
 195                         bus_space_write_region_4(sc->sc_io_tag, sc->sc_io_hdl,
 196                             fifo, buf_res.buffer, buf_res.length / 4);
 197
 198                         offset += buf_res.length;
 199                         fifo += buf_res.length;
 200                         temp -= buf_res.length;
 201                 } while (temp != 0);
 202         }
 203
 204         /* check for remainder */
 205         if (count != 0) {
 206                 /* clear topmost word before copy */
 207                 sc->sc_bounce_buffer[(count - 1) / 4] = 0;
 208
 209                 /* copy out data */
 210                 usbd_copy_out(pc, offset,
 211                     sc->sc_bounce_buffer, count);
 212
 213                 /* transfer data into FIFO */
 214                 bus_space_write_region_4(sc->sc_io_tag,
 215                     sc->sc_io_hdl, fifo, sc->sc_bounce_buffer,
 216                     (count + 3) / 4);
 217         }
 218 }
 219
 220 static void
 221 dwc_otg_read_fifo(struct dwc_otg_softc *sc, struct usb_page_cache *pc,
 222     uint32_t offset, uint32_t count)
 223 {
 224         uint32_t temp;
 225
 226         /* round down length to nearest 4-bytes */
 227         temp = count & ~3;
 228
 229         /* check if we can read the data directly */
 230         if (temp != 0 && usb_pc_buffer_is_aligned(pc, offset, temp, 3)) {
 231                 struct usb_page_search buf_res;
 232
 233                 /* pre-subtract length */
 234                 count -= temp;
 235
 236                 /* iterate buffer list */
 237                 do {
 238                         /* get current buffer pointer */
 239                         usbd_get_page(pc, offset, &buf_res);
 240
 241                         if (buf_res.length > temp)
 242                                 buf_res.length = temp;
 243
 244                         /* transfer data from FIFO */
 245                         bus_space_read_region_4(sc->sc_io_tag, sc->sc_io_hdl,
 246                             sc->sc_current_rx_fifo, buf_res.buffer, buf_res.length / 4);
 247
 248                         offset += buf_res.length;
 249                         sc->sc_current_rx_fifo += buf_res.length;
 250                         sc->sc_current_rx_bytes -= buf_res.length;
 251                         temp -= buf_res.length;
 252                 } while (temp != 0);
 253         }
 254
 255         /* check for remainder */
 256         if (count != 0) {
 257                 /* read data into bounce buffer */
 258                 bus_space_read_region_4(sc->sc_io_tag, sc->sc_io_hdl,
 259                         sc->sc_current_rx_fifo,
 260                         sc->sc_bounce_buffer, (count + 3) / 4);
 261
 262                 /* store data into proper buffer */
 263                 usbd_copy_in(pc, offset, sc->sc_bounce_buffer, count);
 264
 265                 /* round length up to nearest 4 bytes */
 266                 count = (count + 3) & ~3;
 267
 268                 /* update counters */
 269                 sc->sc_current_rx_bytes -= count;
 270                 sc->sc_current_rx_fifo += count;
 271         }
 272 }
 273
 274 static void
 275 dwc_otg_tx_fifo_reset(struct dwc_otg_softc *sc, uint32_t value)
 276 {
 277         uint32_t temp;
 278
 279         /* reset FIFO */
 280         DWC_OTG_WRITE_4(sc, DOTG_GRSTCTL, value);
 281
 282         /* wait for reset to complete */
 283         for (temp = 0; temp != 16; temp++) {
 284                 value = DWC_OTG_READ_4(sc, DOTG_GRSTCTL);
 285                 if (!(value & (GRSTCTL_TXFFLSH | GRSTCTL_RXFFLSH)))
 286                         break;
 287         }
 288 }
 289
 290 static int
 291 dwc_otg_init_fifo(struct dwc_otg_softc *sc, uint8_t mode)
 292 {
 293         struct dwc_otg_profile *pf;
 294         uint32_t fifo_size;
 295         uint32_t fifo_regs;
 296         uint32_t tx_start;
 297         uint8_t x;
 298
 299         fifo_size = sc->sc_fifo_size;
 300
 301         /*
 302          * NOTE: Reserved fixed size area at end of RAM, which must
 303          * not be allocated to the FIFOs:
 304          */
 305         fifo_regs = 4 * 16;
 306
 307         if (fifo_size < fifo_regs) {
 308                 DPRINTF("Too little FIFO\n");
 309                 return (EINVAL);
 310         }
 311
 312         /* subtract FIFO regs from total once */
 313         fifo_size -= fifo_regs;
 314
 315         /* split equally for IN and OUT */
 316         fifo_size /= 2;
 317
 318         /* Align to 4 bytes boundary (refer to PGM) */
 319         fifo_size &= ~3;
 320
 321         /* set global receive FIFO size */
 322         DWC_OTG_WRITE_4(sc, DOTG_GRXFSIZ, fifo_size / 4);
 323
 324         tx_start = fifo_size;
 325
 326         if (fifo_size < 64) {
 327                 DPRINTFN(-1, "Not enough data space for EP0 FIFO.\n");
 328                 return (EINVAL);
 329         }
 330
 331         if (mode == DWC_MODE_HOST) {
 332
 333                 /* reset active endpoints */
 334                 sc->sc_active_rx_ep = 0;
 335
 336                 /* split equally for periodic and non-periodic */
 337                 fifo_size /= 2;
 338
 339                 DPRINTF("PTX/NPTX FIFO=%u\n", fifo_size);
 340
 341                 /* align to 4 bytes boundary */
 342                 fifo_size &= ~3;
 343
 344                 DWC_OTG_WRITE_4(sc, DOTG_GNPTXFSIZ,
 345                     ((fifo_size / 4) << 16) |
 346                     (tx_start / 4));
 347
 348                 tx_start += fifo_size;
 349
 350                 for (x = 0; x != sc->sc_host_ch_max; x++) {
 351                         /* enable all host interrupts */
 352                         DWC_OTG_WRITE_4(sc, DOTG_HCINTMSK(x),
 353                             HCINT_DEFAULT_MASK);
 354                 }
 355
 356                 DWC_OTG_WRITE_4(sc, DOTG_HPTXFSIZ,
 357                     ((fifo_size / 4) << 16) |
 358                     (tx_start / 4));
 359
 360                 /* reset host channel state */
 361                 memset(sc->sc_chan_state, 0, sizeof(sc->sc_chan_state));
 362
 363                 /* enable all host channel interrupts */
 364                 DWC_OTG_WRITE_4(sc, DOTG_HAINTMSK,
 365                     (1U << sc->sc_host_ch_max) - 1U);
 366
 367                 /* enable proper host channel interrupts */
 368                 sc->sc_irq_mask |= GINTMSK_HCHINTMSK;
 369                 sc->sc_irq_mask &= ~GINTMSK_IEPINTMSK;
 370                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 371         }
 372
 373         if (mode == DWC_MODE_DEVICE) {
 374
 375             DWC_OTG_WRITE_4(sc, DOTG_GNPTXFSIZ,
 376                 (0x10 << 16) | (tx_start / 4));
 377             fifo_size -= 0x40;
 378             tx_start += 0x40;
 379
 380             /* setup control endpoint profile */
 381             sc->sc_hw_ep_profile[0].usb = dwc_otg_ep_profile[0];
 382
 383             /* reset active endpoints */
 384             sc->sc_active_rx_ep = 1;
 385
 386             for (x = 1; x != sc->sc_dev_ep_max; x++) {
 387
 388                 pf = sc->sc_hw_ep_profile + x;
 389
 390                 pf->usb.max_out_frame_size = 1024 * 3;
 391                 pf->usb.is_simplex = 0; /* assume duplex */
 392                 pf->usb.support_bulk = 1;
 393                 pf->usb.support_interrupt = 1;
 394                 pf->usb.support_isochronous = 1;
 395                 pf->usb.support_out = 1;
 396
 397                 if (x < sc->sc_dev_in_ep_max) {
 398                         uint32_t limit;
 399
 400                         limit = (x == 1) ? MIN(DWC_OTG_TX_MAX_FIFO_SIZE,
 401                             DWC_OTG_MAX_TXN) : MIN(DWC_OTG_MAX_TXN / 2,
 402                             DWC_OTG_TX_MAX_FIFO_SIZE);
 403
 404                         /* see if there is enough FIFO space */
 405                         if (limit <= fifo_size) {
 406                                 pf->max_buffer = limit;
 407                                 pf->usb.support_in = 1;
 408                         } else {
 409                                 limit = MIN(DWC_OTG_TX_MAX_FIFO_SIZE, 0x40);
 410                                 if (limit <= fifo_size) {
 411                                         pf->usb.support_in = 1;
 412                                 } else {
 413                                         pf->usb.is_simplex = 1;
 414                                         limit = 0;
 415                                 }
 416                         }
 417                         /* set FIFO size */
 418                         DWC_OTG_WRITE_4(sc, DOTG_DIEPTXF(x),
 419                             ((limit / 4) << 16) | (tx_start / 4));
 420                         tx_start += limit;
 421                         fifo_size -= limit;
 422                         pf->usb.max_in_frame_size = limit;
 423                 } else {
 424                         pf->usb.is_simplex = 1;
 425                 }
 426
 427                 DPRINTF("FIFO%d = IN:%d / OUT:%d\n", x,
 428                     pf->usb.max_in_frame_size,
 429                     pf->usb.max_out_frame_size);
 430             }
 431
 432             /* enable proper device channel interrupts */
 433             sc->sc_irq_mask &= ~GINTMSK_HCHINTMSK;
 434             sc->sc_irq_mask |= GINTMSK_IEPINTMSK;
 435             DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 436         }
 437
 438         /* reset RX FIFO */
 439         dwc_otg_tx_fifo_reset(sc, GRSTCTL_RXFFLSH);
 440
 441         if (mode != DWC_MODE_OTG) {
 442                 /* reset all TX FIFOs */
 443                 dwc_otg_tx_fifo_reset(sc,
 444                     GRSTCTL_TXFIFO(0x10) |
 445                     GRSTCTL_TXFFLSH);
 446         } else {
 447                 /* reset active endpoints */
 448                 sc->sc_active_rx_ep = 0;
 449
 450                 /* reset host channel state */
 451                 memset(sc->sc_chan_state, 0, sizeof(sc->sc_chan_state));
 452         }
 453         return (0);
 454 }
 455
 456 static uint8_t
 457 dwc_otg_uses_split(struct usb_device *udev)
 458 {
 459         /*
 460          * When a LOW or FULL speed device is connected directly to
 461          * the USB port we don't use split transactions:
 462          */
 463         return (udev->speed != USB_SPEED_HIGH &&
 464             udev->parent_hs_hub != NULL &&
 465             udev->parent_hs_hub->parent_hub != NULL);
 466 }
 467
 468 static void
 469 dwc_otg_update_host_frame_interval(struct dwc_otg_softc *sc)
 470 {
 471
 472   /*
 473    * Disabled until further. Assuming that the register is already
 474    * programmed correctly by the boot loader.
 475    */
 476 #if 0
 477         uint32_t temp;
 478
 479         /* setup HOST frame interval register, based on existing value */
 480         temp = DWC_OTG_READ_4(sc, DOTG_HFIR) & HFIR_FRINT_MASK;
 481         if (temp >= 10000)
 482                 temp /= 1000;
 483         else
 484                 temp /= 125;
 485
 486         /* figure out nearest X-tal value */
 487         if (temp >= 54)
 488                 temp = 60;      /* MHz */
 489         else if (temp >= 39)
 490                 temp = 48;      /* MHz */
 491         else
 492                 temp = 30;      /* MHz */
 493
 494         if (sc->sc_flags.status_high_speed)
 495                 temp *= 125;
 496         else
 497                 temp *= 1000;
 498
 499         DPRINTF("HFIR=0x%08x\n", temp);
 500
 501         DWC_OTG_WRITE_4(sc, DOTG_HFIR, temp);
 502 #endif
 503 }
 504
 505 static void
 506 dwc_otg_clocks_on(struct dwc_otg_softc *sc)
 507 {
 508         if (sc->sc_flags.clocks_off &&
 509             sc->sc_flags.port_powered) {
 510
 511                 DPRINTFN(5, "\n");
 512
 513                 /* TODO - platform specific */
 514
 515                 sc->sc_flags.clocks_off = 0;
 516         }
 517 }
 518
 519 static void
 520 dwc_otg_clocks_off(struct dwc_otg_softc *sc)
 521 {
 522         if (!sc->sc_flags.clocks_off) {
 523
 524                 DPRINTFN(5, "\n");
 525
 526                 /* TODO - platform specific */
 527
 528                 sc->sc_flags.clocks_off = 1;
 529         }
 530 }
 531
 532 static void
 533 dwc_otg_pull_up(struct dwc_otg_softc *sc)
 534 {
 535         uint32_t temp;
 536
 537         /* pullup D+, if possible */
 538
 539         if (!sc->sc_flags.d_pulled_up &&
 540             sc->sc_flags.port_powered) {
 541                 sc->sc_flags.d_pulled_up = 1;
 542
 543                 temp = DWC_OTG_READ_4(sc, DOTG_DCTL);
 544                 temp &= ~DCTL_SFTDISCON;
 545                 DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
 546         }
 547 }
 548
 549 static void
 550 dwc_otg_pull_down(struct dwc_otg_softc *sc)
 551 {
 552         uint32_t temp;
 553
 554         /* pulldown D+, if possible */
 555
 556         if (sc->sc_flags.d_pulled_up) {
 557                 sc->sc_flags.d_pulled_up = 0;
 558
 559                 temp = DWC_OTG_READ_4(sc, DOTG_DCTL);
 560                 temp |= DCTL_SFTDISCON;
 561                 DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
 562         }
 563 }
 564
 565 static void
 566 dwc_otg_enable_sof_irq(struct dwc_otg_softc *sc)
 567 {
 568         /* In device mode we don't use the SOF interrupt */
 569         if (sc->sc_flags.status_device_mode != 0)
 570                 return;
 571         /* Ensure the SOF interrupt is not disabled */
 572         sc->sc_needsof = 1;
 573         /* Check if the SOF interrupt is already enabled */
 574         if ((sc->sc_irq_mask & GINTMSK_SOFMSK) != 0)
 575                 return;
 576         sc->sc_irq_mask |= GINTMSK_SOFMSK;
 577         DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 578 }
 579
 580 static void
 581 dwc_otg_resume_irq(struct dwc_otg_softc *sc)
 582 {
 583         if (sc->sc_flags.status_suspend) {
 584                 /* update status bits */
 585                 sc->sc_flags.status_suspend = 0;
 586                 sc->sc_flags.change_suspend = 1;
 587
 588                 if (sc->sc_flags.status_device_mode) {
 589                         /*
 590                          * Disable resume interrupt and enable suspend
 591                          * interrupt:
 592                          */
 593                         sc->sc_irq_mask &= ~GINTMSK_WKUPINTMSK;
 594                         sc->sc_irq_mask |= GINTMSK_USBSUSPMSK;
 595                         DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 596                 }
 597
 598                 /* complete root HUB interrupt endpoint */
 599                 dwc_otg_root_intr(sc);
 600         }
 601 }
 602
 603 static void
 604 dwc_otg_suspend_irq(struct dwc_otg_softc *sc)
 605 {
 606         if (!sc->sc_flags.status_suspend) {
 607                 /* update status bits */
 608                 sc->sc_flags.status_suspend = 1;
 609                 sc->sc_flags.change_suspend = 1;
 610
 611                 if (sc->sc_flags.status_device_mode) {
 612                         /*
 613                          * Disable suspend interrupt and enable resume
 614                          * interrupt:
 615                          */
 616                         sc->sc_irq_mask &= ~GINTMSK_USBSUSPMSK;
 617                         sc->sc_irq_mask |= GINTMSK_WKUPINTMSK;
 618                         DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 619                 }
 620
 621                 /* complete root HUB interrupt endpoint */
 622                 dwc_otg_root_intr(sc);
 623         }
 624 }
 625
 626 static void
 627 dwc_otg_wakeup_peer(struct dwc_otg_softc *sc)
 628 {
 629         if (!sc->sc_flags.status_suspend)
 630                 return;
 631
 632         DPRINTFN(5, "Remote wakeup\n");
 633
 634         if (sc->sc_flags.status_device_mode) {
 635                 uint32_t temp;
 636
 637                 /* enable remote wakeup signalling */
 638                 temp = DWC_OTG_READ_4(sc, DOTG_DCTL);
 639                 temp |= DCTL_RMTWKUPSIG;
 640                 DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
 641
 642                 /* Wait 8ms for remote wakeup to complete. */
 643                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 125);
 644
 645                 temp &= ~DCTL_RMTWKUPSIG;
 646                 DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
 647         } else {
 648                 /* enable USB port */
 649                 DWC_OTG_WRITE_4(sc, DOTG_PCGCCTL, 0);
 650
 651                 /* wait 10ms */
 652                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 100);
 653
 654                 /* resume port */
 655                 sc->sc_hprt_val |= HPRT_PRTRES;
 656                 DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
 657
 658                 /* Wait 100ms for resume signalling to complete. */
 659                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 10);
 660
 661                 /* clear suspend and resume */
 662                 sc->sc_hprt_val &= ~(HPRT_PRTSUSP | HPRT_PRTRES);
 663                 DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
 664
 665                 /* Wait 4ms */
 666                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 250);
 667         }
 668
 669         /* need to fake resume IRQ */
 670         dwc_otg_resume_irq(sc);
 671 }
 672
 673 static void
 674 dwc_otg_set_address(struct dwc_otg_softc *sc, uint8_t addr)
 675 {
 676         uint32_t temp;
 677
 678         DPRINTFN(5, "addr=%d\n", addr);
 679
 680         temp = DWC_OTG_READ_4(sc, DOTG_DCFG);
 681         temp &= ~DCFG_DEVADDR_SET(0x7F);
 682         temp |= DCFG_DEVADDR_SET(addr);
 683         DWC_OTG_WRITE_4(sc, DOTG_DCFG, temp);
 684 }
 685
 686 static void
 687 dwc_otg_common_rx_ack(struct dwc_otg_softc *sc)
 688 {
 689         DPRINTFN(5, "RX status clear\n");
 690
 691         /* enable RX FIFO level interrupt */
 692         sc->sc_irq_mask |= GINTMSK_RXFLVLMSK;
 693         DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 694
 695         if (sc->sc_current_rx_bytes != 0) {
 696                 /* need to dump remaining data */
 697                 bus_space_read_region_4(sc->sc_io_tag, sc->sc_io_hdl,
 698                     sc->sc_current_rx_fifo, sc->sc_bounce_buffer,
 699                     sc->sc_current_rx_bytes / 4);
 700                 /* clear number of active bytes to receive */
 701                 sc->sc_current_rx_bytes = 0;
 702         }
 703         /* clear cached status */
 704         sc->sc_last_rx_status = 0;
 705 }
 706
 707 static void
 708 dwc_otg_clear_hcint(struct dwc_otg_softc *sc, uint8_t x)
 709 {
 710         uint32_t hcint;
 711
 712         /* clear all pending interrupts */
 713         hcint = DWC_OTG_READ_4(sc, DOTG_HCINT(x));
 714         DWC_OTG_WRITE_4(sc, DOTG_HCINT(x), hcint);
 715
 716         /* clear buffered interrupts */
 717         sc->sc_chan_state[x].hcint = 0;
 718 }
 719
 720 static uint8_t
 721 dwc_otg_host_check_tx_fifo_empty(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
 722 {
 723         uint32_t temp;
 724
 725         temp = DWC_OTG_READ_4(sc, DOTG_GINTSTS);
 726
 727         if (td->ep_type == UE_ISOCHRONOUS) {
 728                 /*
 729                  * NOTE: USB INTERRUPT transactions are executed like
 730                  * USB CONTROL transactions! See the setup standard
 731                  * chain function for more information.
 732                  */
 733                 if (!(temp & GINTSTS_PTXFEMP)) {
 734                         DPRINTF("Periodic TX FIFO is not empty\n");
 735                         if (!(sc->sc_irq_mask & GINTMSK_PTXFEMPMSK)) {
 736                                 sc->sc_irq_mask |= GINTMSK_PTXFEMPMSK;
 737                                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 738                         }
 739                         return (1);     /* busy */
 740                 }
 741         } else {
 742                 if (!(temp & GINTSTS_NPTXFEMP)) {
 743                         DPRINTF("Non-periodic TX FIFO is not empty\n");
 744                         if (!(sc->sc_irq_mask & GINTMSK_NPTXFEMPMSK)) {
 745                                 sc->sc_irq_mask |= GINTMSK_NPTXFEMPMSK;
 746                                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 747                         }
 748                         return (1);     /* busy */
 749                 }
 750         }
 751         return (0);     /* ready for transmit */
 752 }
 753
 754 static uint8_t
 755 dwc_otg_host_channel_alloc(struct dwc_otg_softc *sc,
 756     struct dwc_otg_td *td, uint8_t is_out)
 757 {
 758         uint8_t x;
 759         uint8_t y;
 760         uint8_t z;
 761
 762         if (td->channel[0] < DWC_OTG_MAX_CHANNELS)
 763                 return (0);             /* already allocated */
 764
 765         /* check if device is suspended */
 766         if (DWC_OTG_PC2UDEV(td->pc)->flags.self_suspended != 0)
 767                 return (1);             /* busy - cannot transfer data */
 768
 769         /* compute needed TX FIFO size */
 770         if (is_out != 0) {
 771                 if (dwc_otg_host_check_tx_fifo_empty(sc, td) != 0)
 772                         return (1);     /* busy - cannot transfer data */
 773         }
 774         z = td->max_packet_count;
 775         for (x = y = 0; x != sc->sc_host_ch_max; x++) {
 776                 /* check if channel is allocated */
 777                 if (sc->sc_chan_state[x].allocated != 0)
 778                         continue;
 779                 /* check if channel is still enabled */
 780                 if (sc->sc_chan_state[x].wait_halted != 0)
 781                         continue;
 782                 /* store channel number */
 783                 td->channel[y++] = x;
 784                 /* check if we got all channels */
 785                 if (y == z)
 786                         break;
 787         }
 788         if (y != z) {
 789                 /* reset channel variable */
 790                 td->channel[0] = DWC_OTG_MAX_CHANNELS;
 791                 td->channel[1] = DWC_OTG_MAX_CHANNELS;
 792                 td->channel[2] = DWC_OTG_MAX_CHANNELS;
 793                 /* wait a bit */
 794                 dwc_otg_enable_sof_irq(sc);
 795                 return (1);     /* busy - not enough channels */
 796         }
 797
 798         for (y = 0; y != z; y++) {
 799                 x = td->channel[y];
 800
 801                 /* set allocated */
 802                 sc->sc_chan_state[x].allocated = 1;
 803
 804                 /* set wait halted */
 805                 sc->sc_chan_state[x].wait_halted = 1;
 806
 807                 /* clear interrupts */
 808                 dwc_otg_clear_hcint(sc, x);
 809
 810                 DPRINTF("CH=%d HCCHAR=0x%08x "
 811                     "HCSPLT=0x%08x\n", x, td->hcchar, td->hcsplt);
 812
 813                 /* set active channel */
 814                 sc->sc_active_rx_ep |= (1 << x);
 815         }
 816         return (0);     /* allocated */
 817 }
 818
 819 static void
 820 dwc_otg_host_channel_free_sub(struct dwc_otg_softc *sc, struct dwc_otg_td *td, uint8_t index)
 821 {
 822         uint32_t hcchar;
 823         uint8_t x;
 824
 825         if (td->channel[index] >= DWC_OTG_MAX_CHANNELS)
 826                 return;         /* already freed */
 827
 828         /* free channel */
 829         x = td->channel[index];
 830         td->channel[index] = DWC_OTG_MAX_CHANNELS;
 831
 832         DPRINTF("CH=%d\n", x);
 833
 834         /*
 835          * We need to let programmed host channels run till complete
 836          * else the host channel will stop functioning.
 837          */
 838         sc->sc_chan_state[x].allocated = 0;
 839
 840         /* ack any pending messages */
 841         if (sc->sc_last_rx_status != 0 &&
 842             GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) == x) {
 843                 dwc_otg_common_rx_ack(sc);
 844         }
 845
 846         /* clear active channel */
 847         sc->sc_active_rx_ep &= ~(1 << x);
 848
 849         /* check if already halted */
 850         if (sc->sc_chan_state[x].wait_halted == 0)
 851                 return;
 852
 853         /* disable host channel */
 854         hcchar = DWC_OTG_READ_4(sc, DOTG_HCCHAR(x));
 855         if (hcchar & HCCHAR_CHENA) {
 856                 DPRINTF("Halting channel %d\n", x);
 857                 DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(x),
 858                     hcchar | HCCHAR_CHDIS);
 859                 /* don't write HCCHAR until the channel is halted */
 860         } else {
 861                 sc->sc_chan_state[x].wait_halted = 0;
 862         }
 863 }
 864
 865 static void
 866 dwc_otg_host_channel_free(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
 867 {
 868         uint8_t x;
 869         for (x = 0; x != td->max_packet_count; x++)
 870                 dwc_otg_host_channel_free_sub(sc, td, x);
 871 }
 872
 873 static void
 874 dwc_otg_host_dump_rx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
 875 {
 876         uint8_t x;
 877         /* dump any pending messages */
 878         if (sc->sc_last_rx_status == 0)
 879                 return;
 880         for (x = 0; x != td->max_packet_count; x++) {
 881                 if (td->channel[x] >= DWC_OTG_MAX_CHANNELS ||
 882                     td->channel[x] != GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status))
 883                         continue;
 884                 dwc_otg_common_rx_ack(sc);
 885                 break;
 886         }
 887 }
 888
 889 static uint8_t
 890 dwc_otg_host_setup_tx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
 891 {
 892         struct usb_device_request req __aligned(4);
 893         uint32_t hcint;
 894         uint32_t hcchar;
 895         uint8_t delta;
 896
 897         dwc_otg_host_dump_rx(sc, td);
 898
 899         if (td->channel[0] < DWC_OTG_MAX_CHANNELS) {
 900                 hcint = sc->sc_chan_state[td->channel[0]].hcint;
 901
 902                 DPRINTF("CH=%d ST=%d HCINT=0x%08x HCCHAR=0x%08x HCTSIZ=0x%08x\n",
 903                     td->channel[0], td->state, hcint,
 904                     DWC_OTG_READ_4(sc, DOTG_HCCHAR(td->channel[0])),
 905                     DWC_OTG_READ_4(sc, DOTG_HCTSIZ(td->channel[0])));
 906         } else {
 907                 hcint = 0;
 908                 goto check_state;
 909         }
 910
 911         if (hcint & (HCINT_RETRY |
 912             HCINT_ACK | HCINT_NYET)) {
 913                 /* give success bits priority over failure bits */
 914         } else if (hcint & HCINT_STALL) {
 915                 DPRINTF("CH=%d STALL\n", td->channel[0]);
 916                 td->error_stall = 1;
 917                 td->error_any = 1;
 918                 goto complete;
 919         } else if (hcint & HCINT_ERRORS) {
 920                 DPRINTF("CH=%d ERROR\n", td->channel[0]);
 921                 td->errcnt++;
 922                 if (td->hcsplt != 0 || td->errcnt >= 3) {
 923                         td->error_any = 1;
 924                         goto complete;
 925                 }
 926         }
 927
 928         if (hcint & (HCINT_ERRORS | HCINT_RETRY |
 929             HCINT_ACK | HCINT_NYET)) {
 930                 if (!(hcint & HCINT_ERRORS))
 931                         td->errcnt = 0;
 932         }
 933
 934 check_state:
 935         switch (td->state) {
 936         case DWC_CHAN_ST_START:
 937                 goto send_pkt;
 938
 939         case DWC_CHAN_ST_WAIT_ANE:
 940                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
 941                         td->did_nak = 1;
 942                         td->tt_scheduled = 0;
 943                         goto send_pkt;
 944                 } else if (hcint & (HCINT_ACK | HCINT_NYET)) {
 945                         td->offset += td->tx_bytes;
 946                         td->remainder -= td->tx_bytes;
 947                         td->toggle = 1;
 948                         td->tt_scheduled = 0;
 949                         goto complete;
 950                 }
 951                 break;
 952
 953         case DWC_CHAN_ST_WAIT_S_ANE:
 954                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
 955                         td->did_nak = 1;
 956                         td->tt_scheduled = 0;
 957                         goto send_pkt;
 958                 } else if (hcint & (HCINT_ACK | HCINT_NYET)) {
 959                         goto send_cpkt;
 960                 }
 961                 break;
 962
 963         case DWC_CHAN_ST_WAIT_C_ANE:
 964                 if (hcint & HCINT_NYET) {
 965                         goto send_cpkt;
 966                 } else if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
 967                         td->did_nak = 1;
 968                         td->tt_scheduled = 0;
 969                         goto send_pkt;
 970                 } else if (hcint & HCINT_ACK) {
 971                         td->offset += td->tx_bytes;
 972                         td->remainder -= td->tx_bytes;
 973                         td->toggle = 1;
 974                         goto complete;
 975                 }
 976                 break;
 977
 978         case DWC_CHAN_ST_WAIT_C_PKT:
 979                 goto send_cpkt;
 980
 981         default:
 982                 break;
 983         }
 984         goto busy;
 985
 986 send_pkt:
 987         /* free existing channel, if any */
 988         dwc_otg_host_channel_free(sc, td);
 989
 990         if (sizeof(req) != td->remainder) {
 991                 td->error_any = 1;
 992                 goto complete;
 993         }
 994
 995         if (td->hcsplt != 0) {
 996                 delta = td->tt_start_slot - sc->sc_last_frame_num - 1;
 997                 if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
 998                         td->state = DWC_CHAN_ST_START;
 999                         goto busy;
1000                 }
1001                 delta = sc->sc_last_frame_num - td->tt_start_slot;
1002                 if (delta > 5) {
1003                         /* missed it */
1004                         td->tt_scheduled = 0;
1005                         td->state = DWC_CHAN_ST_START;
1006                         goto busy;
1007                 }
1008         }
1009
1010         /* allocate a new channel */
1011         if (dwc_otg_host_channel_alloc(sc, td, 1)) {
1012                 td->state = DWC_CHAN_ST_START;
1013                 goto busy;
1014         }
1015
1016         if (td->hcsplt != 0) {
1017                 td->hcsplt &= ~HCSPLT_COMPSPLT;
1018                 td->state = DWC_CHAN_ST_WAIT_S_ANE;
1019         } else {
1020                 td->state = DWC_CHAN_ST_WAIT_ANE;
1021         }
1022
1023         /* copy out control request */
1024         usbd_copy_out(td->pc, 0, &req, sizeof(req));
1025
1026         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(td->channel[0]),
1027             (sizeof(req) << HCTSIZ_XFERSIZE_SHIFT) |
1028             (1 << HCTSIZ_PKTCNT_SHIFT) |
1029             (HCTSIZ_PID_SETUP << HCTSIZ_PID_SHIFT));
1030
1031         DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(td->channel[0]), td->hcsplt);
1032
1033         hcchar = td->hcchar;
1034         hcchar &= ~(HCCHAR_EPDIR_IN | HCCHAR_EPTYPE_MASK);
1035         hcchar |= UE_CONTROL << HCCHAR_EPTYPE_SHIFT;
1036
1037         /* must enable channel before writing data to FIFO */
1038         DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(td->channel[0]), hcchar);
1039
1040         /* transfer data into FIFO */
1041         bus_space_write_region_4(sc->sc_io_tag, sc->sc_io_hdl,
1042             DOTG_DFIFO(td->channel[0]), (uint32_t *)&req, sizeof(req) / 4);
1043
1044         /* wait until next slot before trying complete split */
1045         td->tt_complete_slot = sc->sc_last_frame_num + 1;
1046
1047         /* store number of bytes transmitted */
1048         td->tx_bytes = sizeof(req);
1049         goto busy;
1050
1051 send_cpkt:
1052         /* free existing channel, if any */
1053         dwc_otg_host_channel_free(sc, td);
1054
1055         delta = td->tt_complete_slot - sc->sc_last_frame_num - 1;
1056         if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
1057                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
1058                 goto busy;
1059         }
1060         delta = sc->sc_last_frame_num - td->tt_start_slot;
1061         if (delta > DWC_OTG_TT_SLOT_MAX) {
1062                 /* we missed the service interval */
1063                 if (td->ep_type != UE_ISOCHRONOUS)
1064                         td->error_any = 1;
1065                 goto complete;
1066         }
1067         /* allocate a new channel */
1068         if (dwc_otg_host_channel_alloc(sc, td, 0)) {
1069                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
1070                 goto busy;
1071         }
1072
1073         /* wait until next slot before trying complete split */
1074         td->tt_complete_slot = sc->sc_last_frame_num + 1;
1075
1076         td->hcsplt |= HCSPLT_COMPSPLT;
1077         td->state = DWC_CHAN_ST_WAIT_C_ANE;
1078
1079         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(td->channel[0]),
1080             (HCTSIZ_PID_SETUP << HCTSIZ_PID_SHIFT));
1081
1082         DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(td->channel[0]), td->hcsplt);
1083
1084         hcchar = td->hcchar;
1085         hcchar &= ~(HCCHAR_EPDIR_IN | HCCHAR_EPTYPE_MASK);
1086         hcchar |= UE_CONTROL << HCCHAR_EPTYPE_SHIFT;
1087
1088         /* must enable channel before writing data to FIFO */
1089         DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(td->channel[0]), hcchar);
1090
1091 busy:
1092         return (1);     /* busy */
1093
1094 complete:
1095         dwc_otg_host_channel_free(sc, td);
1096         return (0);     /* complete */
1097 }
1098
1099 static uint8_t
1100 dwc_otg_setup_rx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1101 {
1102         struct usb_device_request req __aligned(4);
1103         uint32_t temp;
1104         uint16_t count;
1105
1106         /* check endpoint status */
1107
1108         if (sc->sc_last_rx_status == 0)
1109                 goto not_complete;
1110
1111         if (GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) != 0)
1112                 goto not_complete;
1113
1114         if ((sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) !=
1115             GRXSTSRD_STP_DATA) {
1116                 if ((sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) !=
1117                     GRXSTSRD_STP_COMPLETE || td->remainder != 0) {
1118                         /* release FIFO */
1119                         dwc_otg_common_rx_ack(sc);
1120                         goto not_complete;
1121                 }
1122                 /* release FIFO */
1123                 dwc_otg_common_rx_ack(sc);
1124                 return (0);     /* complete */
1125         }
1126
1127         if ((sc->sc_last_rx_status & GRXSTSRD_DPID_MASK) !=
1128             GRXSTSRD_DPID_DATA0) {
1129                 /* release FIFO */
1130                 dwc_otg_common_rx_ack(sc);
1131                 goto not_complete;
1132         }
1133
1134         DPRINTFN(5, "GRXSTSR=0x%08x\n", sc->sc_last_rx_status);
1135
1136         /* clear did stall */
1137         td->did_stall = 0;
1138
1139         /* get the packet byte count */
1140         count = GRXSTSRD_BCNT_GET(sc->sc_last_rx_status);
1141
1142         if (count != sizeof(req)) {
1143                 DPRINTFN(0, "Unsupported SETUP packet "
1144                     "length, %d bytes\n", count);
1145                 /* release FIFO */
1146                 dwc_otg_common_rx_ack(sc);
1147                 goto not_complete;
1148         }
1149
1150         /* read FIFO */
1151         dwc_otg_read_fifo(sc, td->pc, 0, sizeof(req));
1152
1153         /* copy out control request */
1154         usbd_copy_out(td->pc, 0, &req, sizeof(req));
1155
1156         td->offset = sizeof(req);
1157         td->remainder = 0;
1158
1159         /* sneak peek the set address */
1160         if ((req.bmRequestType == UT_WRITE_DEVICE) &&
1161             (req.bRequest == UR_SET_ADDRESS)) {
1162                 /* must write address before ZLP */
1163                 dwc_otg_set_address(sc, req.wValue[0] & 0x7F);
1164         }
1165
1166         /* don't send any data by default */
1167         DWC_OTG_WRITE_4(sc, DOTG_DIEPTSIZ(0), DIEPCTL_EPDIS);
1168         DWC_OTG_WRITE_4(sc, DOTG_DOEPTSIZ(0), DOEPCTL_EPDIS);
1169
1170         /* reset IN endpoint buffer */
1171         dwc_otg_tx_fifo_reset(sc,
1172             GRSTCTL_TXFIFO(0) |
1173             GRSTCTL_TXFFLSH);
1174
1175         /* acknowledge RX status */
1176         dwc_otg_common_rx_ack(sc);
1177         td->did_stall = 1;
1178
1179 not_complete:
1180         /* abort any ongoing transfer, before enabling again */
1181         if (!td->did_stall) {
1182                 td->did_stall = 1;
1183
1184                 DPRINTFN(5, "stalling IN and OUT direction\n");
1185
1186                 temp = sc->sc_out_ctl[0];
1187
1188                 /* set stall after enabling endpoint */
1189                 DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(0),
1190                     temp | DOEPCTL_STALL);
1191
1192                 temp = sc->sc_in_ctl[0];
1193
1194                 /* set stall assuming endpoint is enabled */
1195                 DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(0),
1196                     temp | DIEPCTL_STALL);
1197         }
1198         return (1);                     /* not complete */
1199 }
1200
1201 static uint8_t
1202 dwc_otg_host_rate_check_interrupt(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1203 {
1204         uint8_t delta;
1205
1206         delta = sc->sc_tmr_val - td->tmr_val;
1207         if (delta >= 128)
1208                 return (1);     /* busy */
1209
1210         td->tmr_val = sc->sc_tmr_val + td->tmr_res;
1211
1212         /* set toggle, if any */
1213         if (td->set_toggle) {
1214                 td->set_toggle = 0;
1215                 td->toggle = 1;
1216         }
1217         return (0);
1218 }
1219
1220 static uint8_t
1221 dwc_otg_host_rate_check(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1222 {
1223         uint8_t frame_num = (uint8_t)sc->sc_last_frame_num;
1224
1225         if (td->ep_type == UE_ISOCHRONOUS) {
1226                 /* non TT isochronous traffic */
1227                 if (frame_num & (td->tmr_res - 1))
1228                         goto busy;
1229                 if ((frame_num ^ td->tmr_val) & td->tmr_res)
1230                         goto busy;
1231                 td->tmr_val = td->tmr_res + sc->sc_last_frame_num;
1232                 td->toggle = 0;
1233                 return (0);
1234         } else if (td->ep_type == UE_INTERRUPT) {
1235                 if (!td->tt_scheduled)
1236                         goto busy;
1237                 td->tt_scheduled = 0;
1238                 return (0);
1239         } else if (td->did_nak != 0) {
1240                 /* check if we should pause sending queries for 125us */
1241                 if (td->tmr_res == frame_num) {
1242                         /* wait a bit */
1243                         dwc_otg_enable_sof_irq(sc);
1244                         goto busy;
1245                 }
1246         } else if (td->set_toggle) {
1247                 td->set_toggle = 0;
1248                 td->toggle = 1;
1249         }
1250         /* query for data one more time */
1251         td->tmr_res = frame_num;
1252         td->did_nak = 0;
1253         return (0);
1254 busy:
1255         return (1);
1256 }
1257
1258 static uint8_t
1259 dwc_otg_host_data_rx_sub(struct dwc_otg_softc *sc, struct dwc_otg_td *td,
1260     uint8_t channel)
1261 {
1262         uint32_t count;
1263
1264         /* check endpoint status */
1265         if (sc->sc_last_rx_status == 0)
1266                 goto busy;
1267
1268         if (channel >= DWC_OTG_MAX_CHANNELS)
1269                 goto busy;
1270
1271         if (GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) != channel)
1272                 goto busy;
1273
1274         switch (sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) {
1275         case GRXSTSRH_IN_DATA:
1276
1277                 DPRINTF("DATA ST=%d STATUS=0x%08x\n",
1278                     (int)td->state, (int)sc->sc_last_rx_status);
1279
1280                 if (sc->sc_chan_state[channel].hcint & HCINT_SOFTWARE_ONLY) {
1281                         /*
1282                          * When using SPLIT transactions on interrupt
1283                          * endpoints, sometimes data occurs twice.
1284                          */
1285                         DPRINTF("Data already received\n");
1286                         break;
1287                 }
1288
1289                 /* get the packet byte count */
1290                 count = GRXSTSRD_BCNT_GET(sc->sc_last_rx_status);
1291
1292                 /* check for ISOCHRONOUS endpoint */
1293                 if (td->ep_type == UE_ISOCHRONOUS) {
1294                         if ((sc->sc_last_rx_status & GRXSTSRD_DPID_MASK) !=
1295                             GRXSTSRD_DPID_DATA0) {
1296                                 /* more data to be received */
1297                                 td->tt_xactpos = HCSPLT_XACTPOS_MIDDLE;
1298                         } else {
1299                                 /* all data received */
1300                                 td->tt_xactpos = HCSPLT_XACTPOS_BEGIN;
1301                                 /* verify the packet byte count */
1302                                 if (count != td->remainder) {
1303                                         /* we have a short packet */
1304                                         td->short_pkt = 1;
1305                                         td->got_short = 1;
1306                                 }
1307                         }
1308                 } else {
1309                         /* verify the packet byte count */
1310                         if (count != td->max_packet_size) {
1311                                 if (count < td->max_packet_size) {
1312                                         /* we have a short packet */
1313                                         td->short_pkt = 1;
1314                                         td->got_short = 1;
1315                                 } else {
1316                                         /* invalid USB packet */
1317                                         td->error_any = 1;
1318
1319                                         /* release FIFO */
1320                                         dwc_otg_common_rx_ack(sc);
1321                                         goto complete;
1322                                 }
1323                         }
1324                         td->toggle ^= 1;
1325                         td->tt_scheduled = 0;
1326                 }
1327
1328                 /* verify the packet byte count */
1329                 if (count > td->remainder) {
1330                         /* invalid USB packet */
1331                         td->error_any = 1;
1332
1333                         /* release FIFO */
1334                         dwc_otg_common_rx_ack(sc);
1335                         goto complete;
1336                 }
1337
1338                 /* read data from FIFO */
1339                 dwc_otg_read_fifo(sc, td->pc, td->offset, count);
1340
1341                 td->remainder -= count;
1342                 td->offset += count;
1343                 sc->sc_chan_state[channel].hcint |= HCINT_SOFTWARE_ONLY;
1344                 break;
1345         default:
1346                 break;
1347         }
1348         /* release FIFO */
1349         dwc_otg_common_rx_ack(sc);
1350 busy:
1351         return (0);
1352 complete:
1353         return (1);
1354 }
1355
1356 static uint8_t
1357 dwc_otg_host_data_rx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1358 {
1359         uint32_t hcint = 0;
1360         uint32_t hcchar;
1361         uint8_t delta;
1362         uint8_t channel;
1363         uint8_t x;
1364
1365         for (x = 0; x != td->max_packet_count; x++) {
1366                 channel = td->channel[x];
1367                 if (channel >= DWC_OTG_MAX_CHANNELS)
1368                         continue;
1369                 hcint |= sc->sc_chan_state[channel].hcint;
1370
1371                 DPRINTF("CH=%d ST=%d HCINT=0x%08x HCCHAR=0x%08x HCTSIZ=0x%08x\n",
1372                     channel, td->state, hcint,
1373                     DWC_OTG_READ_4(sc, DOTG_HCCHAR(channel)),
1374                     DWC_OTG_READ_4(sc, DOTG_HCTSIZ(channel)));
1375
1376                 /* check interrupt bits */
1377                 if (hcint & (HCINT_RETRY |
1378                     HCINT_ACK | HCINT_NYET)) {
1379                         /* give success bits priority over failure bits */
1380                 } else if (hcint & HCINT_STALL) {
1381                         DPRINTF("CH=%d STALL\n", channel);
1382                         td->error_stall = 1;
1383                         td->error_any = 1;
1384                         goto complete;
1385                 } else if (hcint & HCINT_ERRORS) {
1386                         DPRINTF("CH=%d ERROR\n", channel);
1387                         td->errcnt++;
1388                         if (td->hcsplt != 0 || td->errcnt >= 3) {
1389                                 if (td->ep_type != UE_ISOCHRONOUS) {
1390                                         td->error_any = 1;
1391                                         goto complete;
1392                                 }
1393                         }
1394                 }
1395
1396                 /* check channels for data, if any */
1397                 if (dwc_otg_host_data_rx_sub(sc, td, channel))
1398                         goto complete;
1399
1400                 /* refresh interrupt status */
1401                 hcint |= sc->sc_chan_state[channel].hcint;
1402
1403                 if (hcint & (HCINT_ERRORS | HCINT_RETRY |
1404                     HCINT_ACK | HCINT_NYET)) {
1405                         if (!(hcint & HCINT_ERRORS))
1406                                 td->errcnt = 0;
1407                 }
1408         }
1409
1410         switch (td->state) {
1411         case DWC_CHAN_ST_START:
1412                 if (td->hcsplt != 0)
1413                         goto receive_spkt;
1414                 else
1415                         goto receive_pkt;
1416
1417         case DWC_CHAN_ST_WAIT_ANE:
1418                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1419                         if (td->ep_type == UE_INTERRUPT) {
1420                                 /*
1421                                  * The USB specification does not
1422                                  * mandate a particular data toggle
1423                                  * value for USB INTERRUPT
1424                                  * transfers. Switch the data toggle
1425                                  * value to receive the packet
1426                                  * correctly:
1427                                  */
1428                                 if (hcint & HCINT_DATATGLERR) {
1429                                         DPRINTF("Retrying packet due to "
1430                                             "data toggle error\n");
1431                                         td->toggle ^= 1;
1432                                         goto receive_pkt;
1433                                 }
1434                         } else if (td->ep_type == UE_ISOCHRONOUS) {
1435                                 goto complete;
1436                         }
1437                         td->did_nak = 1;
1438                         td->tt_scheduled = 0;
1439                         if (td->hcsplt != 0)
1440                                 goto receive_spkt;
1441                         else
1442                                 goto receive_pkt;
1443                 } else if (hcint & HCINT_NYET) {
1444                         if (td->hcsplt != 0) {
1445                                 /* try again */
1446                                 goto receive_pkt;
1447                         } else {
1448                                 /* not a valid token for IN endpoints */
1449                                 td->error_any = 1;
1450                                 goto complete;
1451                         }
1452                 } else if (hcint & HCINT_ACK) {
1453                         /* wait for data - ACK arrived first */
1454                         if (!(hcint & HCINT_SOFTWARE_ONLY))
1455                                 goto busy;
1456
1457                         if (td->ep_type == UE_ISOCHRONOUS) {
1458                                 /* check if we are complete */
1459                                 if (td->tt_xactpos == HCSPLT_XACTPOS_BEGIN) {
1460                                         goto complete;
1461                                 } else {
1462                                         /* get more packets */
1463                                         goto busy;
1464                                 }
1465                         } else {
1466                                 /* check if we are complete */
1467                                 if ((td->remainder == 0) || (td->got_short != 0)) {
1468                                         if (td->short_pkt)
1469                                                 goto complete;
1470
1471                                         /*
1472                                          * Else need to receive a zero length
1473                                          * packet.
1474                                          */
1475                                 }
1476                                 td->tt_scheduled = 0;
1477                                 td->did_nak = 0;
1478                                 if (td->hcsplt != 0)
1479                                         goto receive_spkt;
1480                                 else
1481                                         goto receive_pkt;
1482                         }
1483                 }
1484                 break;
1485
1486         case DWC_CHAN_ST_WAIT_S_ANE:
1487                 /*
1488                  * NOTE: The DWC OTG hardware provides a fake ACK in
1489                  * case of interrupt and isochronous transfers:
1490                  */
1491                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1492                         td->did_nak = 1;
1493                         td->tt_scheduled = 0;
1494                         goto receive_spkt;
1495                 } else if (hcint & HCINT_NYET) {
1496                         td->tt_scheduled = 0;
1497                         goto receive_spkt;
1498                 } else if (hcint & HCINT_ACK) {
1499                         td->did_nak = 0;
1500                         goto receive_pkt;
1501                 }
1502                 break;
1503
1504         case DWC_CHAN_ST_WAIT_C_PKT:
1505                 goto receive_pkt;
1506
1507         default:
1508                 break;
1509         }
1510         goto busy;
1511
1512 receive_pkt:
1513         /* free existing channel, if any */
1514         dwc_otg_host_channel_free(sc, td);
1515
1516         if (td->hcsplt != 0) {
1517                 delta = td->tt_complete_slot - sc->sc_last_frame_num - 1;
1518                 if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
1519                         td->state = DWC_CHAN_ST_WAIT_C_PKT;
1520                         goto busy;
1521                 }
1522                 delta = sc->sc_last_frame_num - td->tt_start_slot;
1523                 if (delta > DWC_OTG_TT_SLOT_MAX) {
1524                         if (td->ep_type != UE_ISOCHRONOUS) {
1525                                 /* we missed the service interval */
1526                                 td->error_any = 1;
1527                         }
1528                         goto complete;
1529                 }
1530                 /* complete split */
1531                 td->hcsplt |= HCSPLT_COMPSPLT;
1532         } else if (dwc_otg_host_rate_check(sc, td)) {
1533                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
1534                 goto busy;
1535         }
1536
1537         /* allocate a new channel */
1538         if (dwc_otg_host_channel_alloc(sc, td, 0)) {
1539                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
1540                 goto busy;
1541         }
1542
1543         /* set toggle, if any */
1544         if (td->set_toggle) {
1545                 td->set_toggle = 0;
1546                 td->toggle = 1;
1547         }
1548
1549         td->state = DWC_CHAN_ST_WAIT_ANE;
1550
1551         for (x = 0; x != td->max_packet_count; x++) {
1552                 channel = td->channel[x];
1553
1554                 /* receive one packet */
1555                 DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1556                     (td->max_packet_size << HCTSIZ_XFERSIZE_SHIFT) |
1557                     (1 << HCTSIZ_PKTCNT_SHIFT) |
1558                     (td->toggle ? (HCTSIZ_PID_DATA1 << HCTSIZ_PID_SHIFT) :
1559                     (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT)));
1560
1561                 DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(channel), td->hcsplt);
1562
1563                 hcchar = td->hcchar;
1564                 hcchar |= HCCHAR_EPDIR_IN;
1565
1566                 /* receive complete split ASAP */
1567                 if ((sc->sc_last_frame_num & 1) != 0 &&
1568                     td->ep_type == UE_ISOCHRONOUS)
1569                         hcchar |= HCCHAR_ODDFRM;
1570                 else
1571                         hcchar &= ~HCCHAR_ODDFRM;
1572
1573                 /* must enable channel before data can be received */
1574                 DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(channel), hcchar);
1575         }
1576         /* wait until next slot before trying complete split */
1577         td->tt_complete_slot = sc->sc_last_frame_num + 1;
1578
1579         goto busy;
1580
1581 receive_spkt:
1582         /* free existing channel(s), if any */
1583         dwc_otg_host_channel_free(sc, td);
1584
1585         delta = td->tt_start_slot - sc->sc_last_frame_num - 1;
1586         if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
1587                 td->state = DWC_CHAN_ST_START;
1588                 goto busy;
1589         }
1590         delta = sc->sc_last_frame_num - td->tt_start_slot;
1591         if (delta > 5) {
1592                 /* missed it */
1593                 td->tt_scheduled = 0;
1594                 td->state = DWC_CHAN_ST_START;
1595                 goto busy;
1596         }
1597
1598         /* allocate a new channel */
1599         if (dwc_otg_host_channel_alloc(sc, td, 0)) {
1600                 td->state = DWC_CHAN_ST_START;
1601                 goto busy;
1602         }
1603
1604         channel = td->channel[0];
1605
1606         td->hcsplt &= ~HCSPLT_COMPSPLT;
1607         td->state = DWC_CHAN_ST_WAIT_S_ANE;
1608
1609         /* receive one packet */
1610         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1611             (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT));
1612
1613         DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(channel), td->hcsplt);
1614
1615         /* send after next SOF event */
1616         if ((sc->sc_last_frame_num & 1) == 0 &&
1617             td->ep_type == UE_ISOCHRONOUS)
1618                 td->hcchar |= HCCHAR_ODDFRM;
1619         else
1620                 td->hcchar &= ~HCCHAR_ODDFRM;
1621
1622         hcchar = td->hcchar;
1623         hcchar |= HCCHAR_EPDIR_IN;
1624
1625         /* wait until next slot before trying complete split */
1626         td->tt_complete_slot = sc->sc_last_frame_num + 1;
1627
1628         /* must enable channel before data can be received */
1629         DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(channel), hcchar);
1630 busy:
1631         return (1);     /* busy */
1632
1633 complete:
1634         dwc_otg_host_channel_free(sc, td);
1635         return (0);     /* complete */
1636 }
1637
1638 static uint8_t
1639 dwc_otg_data_rx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1640 {
1641         uint32_t temp;
1642         uint16_t count;
1643         uint8_t got_short;
1644
1645         got_short = 0;
1646
1647         /* check endpoint status */
1648         if (sc->sc_last_rx_status == 0)
1649                 goto not_complete;
1650
1651         if (GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) != td->ep_no)
1652                 goto not_complete;
1653
1654         /* check for SETUP packet */
1655         if ((sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) ==
1656             GRXSTSRD_STP_DATA ||
1657             (sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) ==
1658             GRXSTSRD_STP_COMPLETE) {
1659                 if (td->remainder == 0) {
1660                         /*
1661                          * We are actually complete and have
1662                          * received the next SETUP
1663                          */
1664                         DPRINTFN(5, "faking complete\n");
1665                         return (0);     /* complete */
1666                 }
1667                 /*
1668                  * USB Host Aborted the transfer.
1669                  */
1670                 td->error_any = 1;
1671                 return (0);             /* complete */
1672         }
1673
1674         if ((sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) !=
1675             GRXSTSRD_OUT_DATA) {
1676                 /* release FIFO */
1677                 dwc_otg_common_rx_ack(sc);
1678                 goto not_complete;
1679         }
1680
1681         /* get the packet byte count */
1682         count = GRXSTSRD_BCNT_GET(sc->sc_last_rx_status);
1683
1684         /* verify the packet byte count */
1685         if (count != td->max_packet_size) {
1686                 if (count < td->max_packet_size) {
1687                         /* we have a short packet */
1688                         td->short_pkt = 1;
1689                         got_short = 1;
1690                 } else {
1691                         /* invalid USB packet */
1692                         td->error_any = 1;
1693
1694                         /* release FIFO */
1695                         dwc_otg_common_rx_ack(sc);
1696                         return (0);     /* we are complete */
1697                 }
1698         }
1699         /* verify the packet byte count */
1700         if (count > td->remainder) {
1701                 /* invalid USB packet */
1702                 td->error_any = 1;
1703
1704                 /* release FIFO */
1705                 dwc_otg_common_rx_ack(sc);
1706                 return (0);             /* we are complete */
1707         }
1708
1709         /* read data from FIFO */
1710         dwc_otg_read_fifo(sc, td->pc, td->offset, count);
1711
1712         td->remainder -= count;
1713         td->offset += count;
1714
1715         /* release FIFO */
1716         dwc_otg_common_rx_ack(sc);
1717
1718         temp = sc->sc_out_ctl[td->ep_no];
1719
1720         /* check for isochronous mode */
1721         if ((temp & DIEPCTL_EPTYPE_MASK) ==
1722             (DIEPCTL_EPTYPE_ISOC << DIEPCTL_EPTYPE_SHIFT)) {
1723                 /* toggle odd or even frame bit */
1724                 if (temp & DIEPCTL_SETD1PID) {
1725                         temp &= ~DIEPCTL_SETD1PID;
1726                         temp |= DIEPCTL_SETD0PID;
1727                 } else {
1728                         temp &= ~DIEPCTL_SETD0PID;
1729                         temp |= DIEPCTL_SETD1PID;
1730                 }
1731                 sc->sc_out_ctl[td->ep_no] = temp;
1732         }
1733
1734         /* check if we are complete */
1735         if ((td->remainder == 0) || got_short) {
1736                 if (td->short_pkt) {
1737                         /* we are complete */
1738                         return (0);
1739                 }
1740                 /* else need to receive a zero length packet */
1741         }
1742
1743 not_complete:
1744
1745         /* enable SETUP and transfer complete interrupt */
1746         if (td->ep_no == 0) {
1747                 DWC_OTG_WRITE_4(sc, DOTG_DOEPTSIZ(0),
1748                     DXEPTSIZ_SET_MULTI(3) |
1749                     DXEPTSIZ_SET_NPKT(1) |
1750                     DXEPTSIZ_SET_NBYTES(td->max_packet_size));
1751         } else {
1752                 /* allow reception of multiple packets */
1753                 DWC_OTG_WRITE_4(sc, DOTG_DOEPTSIZ(td->ep_no),
1754                     DXEPTSIZ_SET_MULTI(1) |
1755                     DXEPTSIZ_SET_NPKT(4) |
1756                     DXEPTSIZ_SET_NBYTES(4 *
1757                         ((td->max_packet_size + 3) & ~3)));
1758         }
1759         temp = sc->sc_out_ctl[td->ep_no];
1760         DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(td->ep_no), temp |
1761             DOEPCTL_EPENA | DOEPCTL_CNAK);
1762
1763         return (1);                     /* not complete */
1764 }
1765
1766 static uint8_t
1767 dwc_otg_host_data_tx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1768 {
1769         uint32_t count;
1770         uint32_t hcint;
1771         uint32_t hcchar;
1772         uint8_t delta;
1773         uint8_t channel;
1774         uint8_t x;
1775
1776         dwc_otg_host_dump_rx(sc, td);
1777
1778         /* check that last channel is complete */
1779         channel = td->channel[td->npkt];
1780
1781         if (channel < DWC_OTG_MAX_CHANNELS) {
1782                 hcint = sc->sc_chan_state[channel].hcint;
1783
1784                 DPRINTF("CH=%d ST=%d HCINT=0x%08x HCCHAR=0x%08x HCTSIZ=0x%08x\n",
1785                     channel, td->state, hcint,
1786                     DWC_OTG_READ_4(sc, DOTG_HCCHAR(channel)),
1787                     DWC_OTG_READ_4(sc, DOTG_HCTSIZ(channel)));
1788
1789                 if (hcint & (HCINT_RETRY |
1790                     HCINT_ACK | HCINT_NYET)) {
1791                         /* give success bits priority over failure bits */
1792                 } else if (hcint & HCINT_STALL) {
1793                         DPRINTF("CH=%d STALL\n", channel);
1794                         td->error_stall = 1;
1795                         td->error_any = 1;
1796                         goto complete;
1797                 } else if (hcint & HCINT_ERRORS) {
1798                         DPRINTF("CH=%d ERROR\n", channel);
1799                         td->errcnt++;
1800                         if (td->hcsplt != 0 || td->errcnt >= 3) {
1801                                 td->error_any = 1;
1802                                 goto complete;
1803                         }
1804                 }
1805
1806                 if (hcint & (HCINT_ERRORS | HCINT_RETRY |
1807                     HCINT_ACK | HCINT_NYET)) {
1808
1809                         if (!(hcint & HCINT_ERRORS))
1810                                 td->errcnt = 0;
1811                 }
1812         } else {
1813                 hcint = 0;
1814         }
1815
1816         switch (td->state) {
1817         case DWC_CHAN_ST_START:
1818                 goto send_pkt;
1819
1820         case DWC_CHAN_ST_WAIT_ANE:
1821                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1822                         td->did_nak = 1;
1823                         td->tt_scheduled = 0;
1824                         goto send_pkt;
1825                 } else if (hcint & (HCINT_ACK | HCINT_NYET)) {
1826                         td->offset += td->tx_bytes;
1827                         td->remainder -= td->tx_bytes;
1828                         td->toggle ^= 1;
1829                         /* check if next response will be a NAK */
1830                         if (hcint & HCINT_NYET)
1831                                 td->did_nak = 1;
1832                         else
1833                                 td->did_nak = 0;
1834                         td->tt_scheduled = 0;
1835
1836                         /* check remainder */
1837                         if (td->remainder == 0) {
1838                                 if (td->short_pkt)
1839                                         goto complete;
1840
1841                                 /*
1842                                  * Else we need to transmit a short
1843                                  * packet:
1844                                  */
1845                         }
1846                         goto send_pkt;
1847                 }
1848                 break;
1849
1850         case DWC_CHAN_ST_WAIT_S_ANE:
1851                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1852                         td->did_nak = 1;
1853                         td->tt_scheduled = 0;
1854                         goto send_pkt;
1855                 } else if (hcint & (HCINT_ACK | HCINT_NYET)) {
1856                         td->did_nak = 0;
1857                         goto send_cpkt;
1858                 }
1859                 break;
1860
1861         case DWC_CHAN_ST_WAIT_C_ANE:
1862                 if (hcint & HCINT_NYET) {
1863                         goto send_cpkt;
1864                 } else if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1865                         td->did_nak = 1;
1866                         td->tt_scheduled = 0;
1867                         goto send_pkt;
1868                 } else if (hcint & HCINT_ACK) {
1869                         td->offset += td->tx_bytes;
1870                         td->remainder -= td->tx_bytes;
1871                         td->toggle ^= 1;
1872                         td->did_nak = 0;
1873                         td->tt_scheduled = 0;
1874
1875                         /* check remainder */
1876                         if (td->remainder == 0) {
1877                                 if (td->short_pkt)
1878                                         goto complete;
1879
1880                                 /* else we need to transmit a short packet */
1881                         }
1882                         goto send_pkt;
1883                 }
1884                 break;
1885
1886         case DWC_CHAN_ST_WAIT_C_PKT:
1887                 goto send_cpkt;
1888
1889         case DWC_CHAN_ST_TX_WAIT_ISOC:
1890                 /* Check if ISOCHRONOUS OUT traffic is complete */
1891                 if ((hcint & HCINT_HCH_DONE_MASK) == 0)
1892                         break;
1893
1894                 td->offset += td->tx_bytes;
1895                 td->remainder -= td->tx_bytes;
1896                 goto complete;
1897         default:
1898                 break;
1899         }
1900         goto busy;
1901
1902 send_pkt:
1903         /* free existing channel(s), if any */
1904         dwc_otg_host_channel_free(sc, td);
1905
1906         if (td->hcsplt != 0) {
1907                 delta = td->tt_start_slot - sc->sc_last_frame_num - 1;
1908                 if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
1909                         td->state = DWC_CHAN_ST_START;
1910                         goto busy;
1911                 }
1912                 delta = sc->sc_last_frame_num - td->tt_start_slot;
1913                 if (delta > 5) {
1914                         /* missed it */
1915                         td->tt_scheduled = 0;
1916                         td->state = DWC_CHAN_ST_START;
1917                         goto busy;
1918                 }
1919         } else if (dwc_otg_host_rate_check(sc, td)) {
1920                 td->state = DWC_CHAN_ST_START;
1921                 goto busy;
1922         }
1923
1924         /* allocate a new channel */
1925         if (dwc_otg_host_channel_alloc(sc, td, 1)) {
1926                 td->state = DWC_CHAN_ST_START;
1927                 goto busy;
1928         }
1929
1930         /* set toggle, if any */
1931         if (td->set_toggle) {
1932                 td->set_toggle = 0;
1933                 td->toggle = 1;
1934         }
1935
1936         if (td->ep_type == UE_ISOCHRONOUS) {
1937                 /* ISOCHRONOUS OUT transfers don't have any ACKs */
1938                 td->state = DWC_CHAN_ST_TX_WAIT_ISOC;
1939                 td->hcsplt &= ~HCSPLT_COMPSPLT;
1940                 if (td->hcsplt != 0) {
1941                         /* get maximum transfer length */
1942                         count = td->remainder;
1943                         if (count > HCSPLT_XACTLEN_BURST) {
1944                                 DPRINTF("TT overflow\n");
1945                                 td->error_any = 1;
1946                                 goto complete;
1947                         }
1948                         /* Update transaction position */
1949                         td->hcsplt &= ~HCSPLT_XACTPOS_MASK;
1950                         td->hcsplt |= (HCSPLT_XACTPOS_ALL << HCSPLT_XACTPOS_SHIFT);
1951                 }
1952         } else if (td->hcsplt != 0) {
1953                 td->hcsplt &= ~HCSPLT_COMPSPLT;
1954                 /* Wait for ACK/NAK/ERR from TT */
1955                 td->state = DWC_CHAN_ST_WAIT_S_ANE;
1956         } else {
1957                 /* Wait for ACK/NAK/STALL from device */
1958                 td->state = DWC_CHAN_ST_WAIT_ANE;
1959         }
1960
1961         td->tx_bytes = 0;
1962
1963         for (x = 0; x != td->max_packet_count; x++) {
1964                 uint32_t rem_bytes;
1965
1966                 channel = td->channel[x];
1967
1968                 /* send one packet at a time */
1969                 count = td->max_packet_size;
1970                 rem_bytes = td->remainder - td->tx_bytes;
1971                 if (rem_bytes < count) {
1972                         /* we have a short packet */
1973                         td->short_pkt = 1;
1974                         count = rem_bytes;
1975                 }
1976                 if (count == rem_bytes) {
1977                         /* last packet */
1978                         switch (x) {
1979                         case 0:
1980                                 DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1981                                     (count << HCTSIZ_XFERSIZE_SHIFT) |
1982                                     (1 << HCTSIZ_PKTCNT_SHIFT) |
1983                                     (td->toggle ? (HCTSIZ_PID_DATA1 << HCTSIZ_PID_SHIFT) :
1984                                     (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT)));
1985                                 break;
1986                         case 1:
1987                                 DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1988                                     (count << HCTSIZ_XFERSIZE_SHIFT) |
1989                                     (1 << HCTSIZ_PKTCNT_SHIFT) |
1990                                     (HCTSIZ_PID_DATA1 << HCTSIZ_PID_SHIFT));
1991                                 break;
1992                         default:
1993                                 DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1994                                     (count << HCTSIZ_XFERSIZE_SHIFT) |
1995                                     (1 << HCTSIZ_PKTCNT_SHIFT) |
1996                                     (HCTSIZ_PID_DATA2 << HCTSIZ_PID_SHIFT));
1997                                 break;
1998                         }
1999                 } else if (td->ep_type == UE_ISOCHRONOUS &&
2000                            td->max_packet_count > 1) {
2001                         /* ISOCHRONOUS multi packet */
2002                         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
2003                             (count << HCTSIZ_XFERSIZE_SHIFT) |
2004                             (1 << HCTSIZ_PKTCNT_SHIFT) |
2005                             (HCTSIZ_PID_MDATA << HCTSIZ_PID_SHIFT));
2006                 } else {
2007                         /* TODO: HCTSIZ_DOPNG */
2008                         /* standard BULK/INTERRUPT/CONTROL packet */
2009                         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
2010                             (count << HCTSIZ_XFERSIZE_SHIFT) |
2011                             (1 << HCTSIZ_PKTCNT_SHIFT) |
2012                             (td->toggle ? (HCTSIZ_PID_DATA1 << HCTSIZ_PID_SHIFT) :
2013                             (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT)));
2014                 }
2015
2016                 DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(channel), td->hcsplt);
2017
2018                 hcchar = td->hcchar;
2019                 hcchar &= ~HCCHAR_EPDIR_IN;
2020
2021                 /* send after next SOF event */
2022                 if ((sc->sc_last_frame_num & 1) == 0 &&
2023                     td->ep_type == UE_ISOCHRONOUS)
2024                         hcchar |= HCCHAR_ODDFRM;
2025                 else
2026                         hcchar &= ~HCCHAR_ODDFRM;
2027
2028                 /* must enable before writing data to FIFO */
2029                 DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(channel), hcchar);
2030
2031                 if (count != 0) {
2032                         /* write data into FIFO */
2033                         dwc_otg_write_fifo(sc, td->pc, td->offset +
2034                             td->tx_bytes, DOTG_DFIFO(channel), count);
2035                 }
2036
2037                 /* store number of bytes transmitted */
2038                 td->tx_bytes += count;
2039
2040                 /* store last packet index */
2041                 td->npkt = x;
2042
2043                 /* check for last packet */
2044                 if (count == rem_bytes)
2045                         break;
2046         }
2047         goto busy;
2048
2049 send_cpkt:
2050         /* free existing channel, if any */
2051         dwc_otg_host_channel_free(sc, td);
2052
2053         delta = td->tt_complete_slot - sc->sc_last_frame_num - 1;
2054         if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
2055                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
2056                 goto busy;
2057         }
2058         delta = sc->sc_last_frame_num - td->tt_start_slot;
2059         if (delta > DWC_OTG_TT_SLOT_MAX) {
2060                 /* we missed the service interval */
2061                 if (td->ep_type != UE_ISOCHRONOUS)
2062                         td->error_any = 1;
2063                 goto complete;
2064         }
2065
2066         /* allocate a new channel */
2067         if (dwc_otg_host_channel_alloc(sc, td, 0)) {
2068                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
2069                 goto busy;
2070         }
2071
2072         channel = td->channel[0];
2073
2074         td->hcsplt |= HCSPLT_COMPSPLT;
2075         td->state = DWC_CHAN_ST_WAIT_C_ANE;
2076
2077         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
2078             (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT));
2079
2080         DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(channel), td->hcsplt);
2081
2082         hcchar = td->hcchar;
2083         hcchar &= ~HCCHAR_EPDIR_IN;
2084
2085         /* receive complete split ASAP */
2086         if ((sc->sc_last_frame_num & 1) != 0 &&
2087             td->ep_type == UE_ISOCHRONOUS)
2088                 hcchar |= HCCHAR_ODDFRM;
2089         else
2090                 hcchar &= ~HCCHAR_ODDFRM;
2091
2092         /* must enable channel before data can be received */
2093         DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(channel), hcchar);
2094
2095         /* wait until next slot before trying complete split */
2096         td->tt_complete_slot = sc->sc_last_frame_num + 1;
2097 busy:
2098         return (1);     /* busy */
2099
2100 complete:
2101         dwc_otg_host_channel_free(sc, td);
2102         return (0);     /* complete */
2103 }
2104
2105 static uint8_t
2106 dwc_otg_data_tx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
2107 {
2108         uint32_t max_buffer;
2109         uint32_t count;
2110         uint32_t fifo_left;
2111         uint32_t mpkt;
2112         uint32_t temp;
2113         uint8_t to;
2114
2115         to = 3;                         /* don't loop forever! */
2116
2117         max_buffer = sc->sc_hw_ep_profile[td->ep_no].max_buffer;
2118
2119 repeat:
2120         /* check for for endpoint 0 data */
2121
2122         temp = sc->sc_last_rx_status;
2123
2124         if ((td->ep_no == 0) && (temp != 0) &&
2125             (GRXSTSRD_CHNUM_GET(temp) == 0)) {
2126
2127                 if ((temp & GRXSTSRD_PKTSTS_MASK) !=
2128                     GRXSTSRD_STP_DATA &&
2129                     (temp & GRXSTSRD_PKTSTS_MASK) !=
2130                     GRXSTSRD_STP_COMPLETE) {
2131
2132                         /* dump data - wrong direction */
2133                         dwc_otg_common_rx_ack(sc);
2134                 } else {
2135                         /*
2136                          * The current transfer was cancelled
2137                          * by the USB Host:
2138                          */
2139                         td->error_any = 1;
2140                         return (0);             /* complete */
2141                 }
2142         }
2143
2144         /* fill in more TX data, if possible */
2145         if (td->tx_bytes != 0) {
2146
2147                 uint16_t cpkt;
2148
2149                 /* check if packets have been transferred */
2150                 temp = DWC_OTG_READ_4(sc, DOTG_DIEPTSIZ(td->ep_no));
2151
2152                 /* get current packet number */
2153                 cpkt = DXEPTSIZ_GET_NPKT(temp);
2154
2155                 if (cpkt >= td->npkt) {
2156                         fifo_left = 0;
2157                 } else {
2158                         if (max_buffer != 0) {
2159                                 fifo_left = (td->npkt - cpkt) *
2160                                     td->max_packet_size;
2161
2162                                 if (fifo_left > max_buffer)
2163                                         fifo_left = max_buffer;
2164                         } else {
2165                                 fifo_left = td->max_packet_size;
2166                         }
2167                 }
2168
2169                 count = td->tx_bytes;
2170                 if (count > fifo_left)
2171                         count = fifo_left;
2172
2173                 if (count != 0) {
2174                         /* write data into FIFO */
2175                         dwc_otg_write_fifo(sc, td->pc, td->offset,
2176                             DOTG_DFIFO(td->ep_no), count);
2177
2178                         td->tx_bytes -= count;
2179                         td->remainder -= count;
2180                         td->offset += count;
2181                         td->npkt = cpkt;
2182                 }
2183                 if (td->tx_bytes != 0)
2184                         goto not_complete;
2185
2186                 /* check remainder */
2187                 if (td->remainder == 0) {
2188                         if (td->short_pkt)
2189                                 return (0);     /* complete */
2190
2191                         /* else we need to transmit a short packet */
2192                 }
2193         }
2194
2195         if (!to--)
2196                 goto not_complete;
2197
2198         /* check if not all packets have been transferred */
2199         temp = DWC_OTG_READ_4(sc, DOTG_DIEPTSIZ(td->ep_no));
2200
2201         if (DXEPTSIZ_GET_NPKT(temp) != 0) {
2202
2203                 DPRINTFN(5, "busy ep=%d npkt=%d DIEPTSIZ=0x%08x "
2204                     "DIEPCTL=0x%08x\n", td->ep_no,
2205                     DXEPTSIZ_GET_NPKT(temp),
2206                     temp, DWC_OTG_READ_4(sc, DOTG_DIEPCTL(td->ep_no)));
2207
2208                 goto not_complete;
2209         }
2210
2211         DPRINTFN(5, "rem=%u ep=%d\n", td->remainder, td->ep_no);
2212
2213         /* try to optimise by sending more data */
2214         if ((max_buffer != 0) && ((td->max_packet_size & 3) == 0)) {
2215
2216                 /* send multiple packets at the same time */
2217                 mpkt = max_buffer / td->max_packet_size;
2218
2219                 if (mpkt > 0x3FE)
2220                         mpkt = 0x3FE;
2221
2222                 count = td->remainder;
2223                 if (count > 0x7FFFFF)
2224                         count = 0x7FFFFF - (0x7FFFFF % td->max_packet_size);
2225
2226                 td->npkt = count / td->max_packet_size;
2227
2228                 /*
2229                  * NOTE: We could use 0x3FE instead of "mpkt" in the
2230                  * check below to get more throughput, but then we
2231                  * have a dependency towards non-generic chip features
2232                  * to disable the TX-FIFO-EMPTY interrupts on a per
2233                  * endpoint basis. Increase the maximum buffer size of
2234                  * the IN endpoint to increase the performance.
2235                  */
2236                 if (td->npkt > mpkt) {
2237                         td->npkt = mpkt;
2238                         count = td->max_packet_size * mpkt;
2239                 } else if ((count == 0) || (count % td->max_packet_size)) {
2240                         /* we are transmitting a short packet */
2241                         td->npkt++;
2242                         td->short_pkt = 1;
2243                 }
2244         } else {
2245                 /* send one packet at a time */
2246                 mpkt = 1;
2247                 count = td->max_packet_size;
2248                 if (td->remainder < count) {
2249                         /* we have a short packet */
2250                         td->short_pkt = 1;
2251                         count = td->remainder;
2252                 }
2253                 td->npkt = 1;
2254         }
2255         DWC_OTG_WRITE_4(sc, DOTG_DIEPTSIZ(td->ep_no),
2256             DXEPTSIZ_SET_MULTI(1) |
2257             DXEPTSIZ_SET_NPKT(td->npkt) |
2258             DXEPTSIZ_SET_NBYTES(count));
2259
2260         /* make room for buffering */
2261         td->npkt += mpkt;
2262
2263         temp = sc->sc_in_ctl[td->ep_no];
2264
2265         /* check for isochronous mode */
2266         if ((temp & DIEPCTL_EPTYPE_MASK) ==
2267             (DIEPCTL_EPTYPE_ISOC << DIEPCTL_EPTYPE_SHIFT)) {
2268                 /* toggle odd or even frame bit */
2269                 if (temp & DIEPCTL_SETD1PID) {
2270                         temp &= ~DIEPCTL_SETD1PID;
2271                         temp |= DIEPCTL_SETD0PID;
2272                 } else {
2273                         temp &= ~DIEPCTL_SETD0PID;
2274                         temp |= DIEPCTL_SETD1PID;
2275                 }
2276                 sc->sc_in_ctl[td->ep_no] = temp;
2277         }
2278
2279         /* must enable before writing data to FIFO */
2280         DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(td->ep_no), temp |
2281             DIEPCTL_EPENA | DIEPCTL_CNAK);
2282
2283         td->tx_bytes = count;
2284
2285         /* check remainder */
2286         if (td->tx_bytes == 0 &&
2287             td->remainder == 0) {
2288                 if (td->short_pkt)
2289                         return (0);     /* complete */
2290
2291                 /* else we need to transmit a short packet */
2292         }
2293         goto repeat;
2294
2295 not_complete:
2296         return (1);                     /* not complete */
2297 }
2298
2299 static uint8_t
2300 dwc_otg_data_tx_sync(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
2301 {
2302         uint32_t temp;
2303
2304         /*
2305          * If all packets are transferred we are complete:
2306          */
2307         temp = DWC_OTG_READ_4(sc, DOTG_DIEPTSIZ(td->ep_no));
2308
2309         /* check that all packets have been transferred */
2310         if (DXEPTSIZ_GET_NPKT(temp) != 0) {
2311                 DPRINTFN(5, "busy ep=%d\n", td->ep_no);
2312                 goto not_complete;
2313         }
2314         return (0);
2315
2316 not_complete:
2317
2318         /* we only want to know if there is a SETUP packet or free IN packet */
2319
2320         temp = sc->sc_last_rx_status;
2321
2322         if ((td->ep_no == 0) && (temp != 0) &&
2323             (GRXSTSRD_CHNUM_GET(temp) == 0)) {
2324
2325                 if ((temp & GRXSTSRD_PKTSTS_MASK) ==
2326                     GRXSTSRD_STP_DATA ||
2327                     (temp & GRXSTSRD_PKTSTS_MASK) ==
2328                     GRXSTSRD_STP_COMPLETE) {
2329                         DPRINTFN(5, "faking complete\n");
2330                         /*
2331                          * Race condition: We are complete!
2332                          */
2333                         return (0);
2334                 } else {
2335                         /* dump data - wrong direction */
2336                         dwc_otg_common_rx_ack(sc);
2337                 }
2338         }
2339         return (1);                     /* not complete */
2340 }
2341
2342 static void
2343 dwc_otg_xfer_do_fifo(struct dwc_otg_softc *sc, struct usb_xfer *xfer)
2344 {
2345         struct dwc_otg_td *td;
2346         uint8_t toggle;
2347         uint8_t tmr_val;
2348         uint8_t tmr_res;
2349
2350         DPRINTFN(9, "\n");
2351
2352         td = xfer->td_transfer_cache;
2353         if (td == NULL)
2354                 return;
2355
2356         while (1) {
2357                 if ((td->func) (sc, td)) {
2358                         /* operation in progress */
2359                         break;
2360                 }
2361                 if (((void *)td) == xfer->td_transfer_last) {
2362                         goto done;
2363                 }
2364                 if (td->error_any) {
2365                         goto done;
2366                 } else if (td->remainder > 0) {
2367                         /*
2368                          * We had a short transfer. If there is no alternate
2369                          * next, stop processing !
2370                          */
2371                         if (!td->alt_next)
2372                                 goto done;
2373                 }
2374
2375                 /*
2376                  * Fetch the next transfer descriptor and transfer
2377                  * some flags to the next transfer descriptor
2378                  */
2379                 tmr_res = td->tmr_res;
2380                 tmr_val = td->tmr_val;
2381                 toggle = td->toggle;
2382                 td = td->obj_next;
2383                 xfer->td_transfer_cache = td;
2384                 td->toggle = toggle;    /* transfer toggle */
2385                 td->tmr_res = tmr_res;
2386                 td->tmr_val = tmr_val;
2387         }
2388         return;
2389
2390 done:
2391         xfer->td_transfer_cache = NULL;
2392         sc->sc_xfer_complete = 1;
2393 }
2394
2395 static uint8_t
2396 dwc_otg_xfer_do_complete_locked(struct dwc_otg_softc *sc, struct usb_xfer *xfer)
2397 {
2398         struct dwc_otg_td *td;
2399
2400         DPRINTFN(9, "\n");
2401
2402         td = xfer->td_transfer_cache;
2403         if (td == NULL) {
2404                 /* compute all actual lengths */
2405                 dwc_otg_standard_done(xfer);
2406                 return (1);
2407         }
2408         return (0);
2409 }
2410
2411 static void
2412 dwc_otg_timer(void *_sc)
2413 {
2414         struct dwc_otg_softc *sc = _sc;
2415
2416         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
2417
2418         DPRINTF("\n");
2419
2420         USB_BUS_SPIN_LOCK(&sc->sc_bus);
2421
2422         /* increment timer value */
2423         sc->sc_tmr_val++;
2424
2425         /* enable SOF interrupt, which will poll jobs */
2426         dwc_otg_enable_sof_irq(sc);
2427
2428         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
2429
2430         if (sc->sc_timer_active) {
2431                 /* restart timer */
2432                 usb_callout_reset(&sc->sc_timer,
2433                     hz / (1000 / DWC_OTG_HOST_TIMER_RATE),
2434                     &dwc_otg_timer, sc);
2435         }
2436 }
2437
2438 static void
2439 dwc_otg_timer_start(struct dwc_otg_softc *sc)
2440 {
2441         if (sc->sc_timer_active != 0)
2442                 return;
2443
2444         sc->sc_timer_active = 1;
2445
2446         /* restart timer */
2447         usb_callout_reset(&sc->sc_timer,
2448             hz / (1000 / DWC_OTG_HOST_TIMER_RATE),
2449             &dwc_otg_timer, sc);
2450 }
2451
2452 static void
2453 dwc_otg_timer_stop(struct dwc_otg_softc *sc)
2454 {
2455         if (sc->sc_timer_active == 0)
2456                 return;
2457
2458         sc->sc_timer_active = 0;
2459
2460         /* stop timer */
2461         usb_callout_stop(&sc->sc_timer);
2462 }
2463
2464 static uint16_t
2465 dwc_otg_compute_isoc_rx_tt_slot(struct dwc_otg_tt_info *pinfo)
2466 {
2467         if (pinfo->slot_index < DWC_OTG_TT_SLOT_MAX)
2468                 pinfo->slot_index++;
2469         return (pinfo->slot_index);
2470 }
2471
2472 static uint8_t
2473 dwc_otg_update_host_transfer_schedule_locked(struct dwc_otg_softc *sc)
2474 {
2475         TAILQ_HEAD(, usb_xfer) head;
2476         struct usb_xfer *xfer;
2477         struct usb_xfer *xfer_next;
2478         struct dwc_otg_td *td;
2479         uint16_t temp;
2480         uint16_t slot;
2481
2482         temp = DWC_OTG_READ_4(sc, DOTG_HFNUM) & DWC_OTG_FRAME_MASK;
2483
2484         if (sc->sc_last_frame_num == temp)
2485                 return (0);
2486
2487         sc->sc_last_frame_num = temp;
2488
2489         TAILQ_INIT(&head);
2490
2491         if ((temp & 7) == 0) {
2492
2493                 /* reset the schedule */
2494                 memset(sc->sc_tt_info, 0, sizeof(sc->sc_tt_info));
2495
2496                 TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2497                         td = xfer->td_transfer_cache;
2498                         if (td == NULL || td->ep_type != UE_ISOCHRONOUS)
2499                                 continue;
2500
2501                         /* check for IN direction */
2502                         if ((td->hcchar & HCCHAR_EPDIR_IN) != 0)
2503                                 continue;
2504
2505                         sc->sc_needsof = 1;
2506
2507                         if (td->hcsplt == 0 || td->tt_scheduled != 0)
2508                                 continue;
2509
2510                         /* compute slot */
2511                         slot = dwc_otg_compute_isoc_rx_tt_slot(
2512                             sc->sc_tt_info + td->tt_index);
2513                         if (slot > 3) {
2514                                 /*
2515                                  * Not enough time to get complete
2516                                  * split executed.
2517                                  */
2518                                 continue;
2519                         }
2520                         /* Delayed start */
2521                         td->tt_start_slot = temp + slot;
2522                         td->tt_scheduled = 1;
2523                         TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2524                         TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2525                 }
2526
2527                 TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2528                         td = xfer->td_transfer_cache;
2529                         if (td == NULL || td->ep_type != UE_ISOCHRONOUS)
2530                                 continue;
2531
2532                         /* check for OUT direction */
2533                         if ((td->hcchar & HCCHAR_EPDIR_IN) == 0)
2534                                 continue;
2535
2536                         sc->sc_needsof = 1;
2537
2538                         if (td->hcsplt == 0 || td->tt_scheduled != 0)
2539                                 continue;
2540
2541                         /* Start ASAP */
2542                         td->tt_start_slot = temp;
2543                         td->tt_scheduled = 1;
2544                         TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2545                         TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2546                 }
2547
2548                 TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2549                         td = xfer->td_transfer_cache;
2550                         if (td == NULL || td->ep_type != UE_INTERRUPT)
2551                                 continue;
2552
2553                         if (td->tt_scheduled != 0) {
2554                                 sc->sc_needsof = 1;
2555                                 continue;
2556                         }
2557
2558                         if (dwc_otg_host_rate_check_interrupt(sc, td))
2559                                 continue;
2560
2561                         if (td->hcsplt == 0) {
2562                                 sc->sc_needsof = 1;
2563                                 td->tt_scheduled = 1;
2564                                 continue;
2565                         }
2566
2567                         /* start ASAP */
2568                         td->tt_start_slot = temp;
2569                         sc->sc_needsof = 1;
2570                         td->tt_scheduled = 1;
2571                         TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2572                         TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2573                 }
2574
2575                 TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2576                         td = xfer->td_transfer_cache;
2577                         if (td == NULL ||
2578                             td->ep_type != UE_CONTROL) {
2579                                 continue;
2580                         }
2581
2582                         sc->sc_needsof = 1;
2583
2584                         if (td->hcsplt == 0 || td->tt_scheduled != 0)
2585                                 continue;
2586
2587                         /* start ASAP */
2588                         td->tt_start_slot = temp;
2589                         td->tt_scheduled = 1;
2590                         TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2591                         TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2592                 }
2593         }
2594         if ((temp & 7) < 6) {
2595                 TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2596                         td = xfer->td_transfer_cache;
2597                         if (td == NULL ||
2598                             td->ep_type != UE_BULK) {
2599                                 continue;
2600                         }
2601
2602                         sc->sc_needsof = 1;
2603
2604                         if (td->hcsplt == 0 || td->tt_scheduled != 0)
2605                                 continue;
2606
2607                         /* start ASAP */
2608                         td->tt_start_slot = temp;
2609                         td->tt_scheduled = 1;
2610                         TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2611                         TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2612                 }
2613         }
2614
2615         /* Put TT transfers in execution order at the end */
2616         TAILQ_CONCAT(&sc->sc_bus.intr_q.head, &head, wait_entry);
2617
2618         /* move all TT transfers in front, keeping the current order */
2619         TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2620                 td = xfer->td_transfer_cache;
2621                 if (td == NULL || td->hcsplt == 0)
2622                         continue;
2623                 TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2624                 TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2625         }
2626         TAILQ_CONCAT(&head, &sc->sc_bus.intr_q.head, wait_entry);
2627         TAILQ_CONCAT(&sc->sc_bus.intr_q.head, &head, wait_entry);
2628
2629         /* put non-TT non-ISOCHRONOUS transfers last */
2630         TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2631                 td = xfer->td_transfer_cache;
2632                 if (td == NULL || td->hcsplt != 0 || td->ep_type == UE_ISOCHRONOUS)
2633                         continue;
2634                 TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2635                 TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2636         }
2637         TAILQ_CONCAT(&sc->sc_bus.intr_q.head, &head, wait_entry);
2638
2639         if ((temp & 7) == 0) {
2640
2641                 DPRINTFN(12, "SOF interrupt #%d, needsof=%d\n",
2642                     (int)temp, (int)sc->sc_needsof);
2643
2644                 /* update SOF IRQ mask */
2645                 if (sc->sc_irq_mask & GINTMSK_SOFMSK) {
2646                         if (sc->sc_needsof == 0) {
2647                                 sc->sc_irq_mask &= ~GINTMSK_SOFMSK;
2648                                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2649                         }
2650                 } else {
2651                         if (sc->sc_needsof != 0) {
2652                                 sc->sc_irq_mask |= GINTMSK_SOFMSK;
2653                                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2654                         }
2655                 }
2656
2657                 /* clear need SOF flag */
2658                 sc->sc_needsof = 0;
2659         }
2660         return (1);
2661 }
2662
2663 static void
2664 dwc_otg_interrupt_poll_locked(struct dwc_otg_softc *sc)
2665 {
2666         struct usb_xfer *xfer;
2667         uint32_t count;
2668         uint32_t temp;
2669         uint32_t haint;
2670         uint8_t got_rx_status;
2671         uint8_t x;
2672
2673         if (sc->sc_flags.status_device_mode == 0) {
2674                 /*
2675                  * Update host transfer schedule, so that new
2676                  * transfers can be issued:
2677                  */
2678                 dwc_otg_update_host_transfer_schedule_locked(sc);
2679         }
2680         count = 0;
2681 repeat:
2682         if (++count == 16) {
2683                 /* give other interrupts a chance */
2684                 DPRINTF("Yield\n");
2685                 return;
2686         }
2687
2688         /* get all host channel interrupts */
2689         haint = DWC_OTG_READ_4(sc, DOTG_HAINT);
2690         while (1) {
2691                 x = ffs(haint) - 1;
2692                 if (x >= sc->sc_host_ch_max)
2693                         break;
2694                 temp = DWC_OTG_READ_4(sc, DOTG_HCINT(x));
2695                 DWC_OTG_WRITE_4(sc, DOTG_HCINT(x), temp);
2696                 temp &= ~HCINT_SOFTWARE_ONLY;
2697                 sc->sc_chan_state[x].hcint |= temp;
2698                 haint &= ~(1U << x);
2699         }
2700
2701         if (sc->sc_last_rx_status == 0) {
2702
2703                 temp = DWC_OTG_READ_4(sc, DOTG_GINTSTS);
2704                 if (temp & GINTSTS_RXFLVL) {
2705                         /* pop current status */
2706                         sc->sc_last_rx_status =
2707                             DWC_OTG_READ_4(sc, DOTG_GRXSTSPD);
2708                 }
2709
2710                 if (sc->sc_last_rx_status != 0) {
2711
2712                         uint8_t ep_no;
2713
2714                         temp = sc->sc_last_rx_status &
2715                             GRXSTSRD_PKTSTS_MASK;
2716
2717                         /* non-data messages we simply skip */
2718                         if (temp != GRXSTSRD_STP_DATA &&
2719                             temp != GRXSTSRD_STP_COMPLETE &&
2720                             temp != GRXSTSRD_OUT_DATA) {
2721                                 /* check for halted channel */
2722                                 if (temp == GRXSTSRH_HALTED) {
2723                                         ep_no = GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status);
2724                                         sc->sc_chan_state[ep_no].wait_halted = 0;
2725                                         DPRINTFN(5, "channel halt complete ch=%u\n", ep_no);
2726                                 }
2727                                 /* store bytes and FIFO offset */
2728                                 sc->sc_current_rx_bytes = 0;
2729                                 sc->sc_current_rx_fifo = 0;
2730
2731                                 /* acknowledge status */
2732                                 dwc_otg_common_rx_ack(sc);
2733                                 goto repeat;
2734                         }
2735
2736                         temp = GRXSTSRD_BCNT_GET(
2737                             sc->sc_last_rx_status);
2738                         ep_no = GRXSTSRD_CHNUM_GET(
2739                             sc->sc_last_rx_status);
2740
2741                         /* store bytes and FIFO offset */
2742                         sc->sc_current_rx_bytes = (temp + 3) & ~3;
2743                         sc->sc_current_rx_fifo = DOTG_DFIFO(ep_no);
2744
2745                         DPRINTF("Reading %d bytes from ep %d\n", temp, ep_no);
2746
2747                         /* check if we should dump the data */
2748                         if (!(sc->sc_active_rx_ep & (1U << ep_no))) {
2749                                 dwc_otg_common_rx_ack(sc);
2750                                 goto repeat;
2751                         }
2752
2753                         got_rx_status = 1;
2754
2755                         DPRINTFN(5, "RX status = 0x%08x: ch=%d pid=%d bytes=%d sts=%d\n",
2756                             sc->sc_last_rx_status, ep_no,
2757                             (sc->sc_last_rx_status >> 15) & 3,
2758                             GRXSTSRD_BCNT_GET(sc->sc_last_rx_status),
2759                             (sc->sc_last_rx_status >> 17) & 15);
2760                 } else {
2761                         got_rx_status = 0;
2762                 }
2763         } else {
2764                 uint8_t ep_no;
2765
2766                 ep_no = GRXSTSRD_CHNUM_GET(
2767                     sc->sc_last_rx_status);
2768
2769                 /* check if we should dump the data */
2770                 if (!(sc->sc_active_rx_ep & (1U << ep_no))) {
2771                         dwc_otg_common_rx_ack(sc);
2772                         goto repeat;
2773                 }
2774
2775                 got_rx_status = 1;
2776         }
2777
2778         /* execute FIFOs */
2779         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry)
2780                 dwc_otg_xfer_do_fifo(sc, xfer);
2781
2782         if (got_rx_status) {
2783                 /* check if data was consumed */
2784                 if (sc->sc_last_rx_status == 0)
2785                         goto repeat;
2786
2787                 /* disable RX FIFO level interrupt */
2788                 sc->sc_irq_mask &= ~GINTMSK_RXFLVLMSK;
2789                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2790         }
2791 }
2792
2793 static void
2794 dwc_otg_interrupt_complete_locked(struct dwc_otg_softc *sc)
2795 {
2796         struct usb_xfer *xfer;
2797 repeat:
2798         /* scan for completion events */
2799         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
2800                 if (dwc_otg_xfer_do_complete_locked(sc, xfer))
2801                         goto repeat;
2802         }
2803 }
2804
2805 static void
2806 dwc_otg_vbus_interrupt(struct dwc_otg_softc *sc, uint8_t is_on)
2807 {
2808         DPRINTFN(5, "vbus = %u\n", is_on);
2809
2810         /*
2811          * If the USB host mode is forced, then assume VBUS is always
2812          * present else rely on the input to this function:
2813          */
2814         if ((is_on != 0) || (sc->sc_mode == DWC_MODE_HOST)) {
2815
2816                 if (!sc->sc_flags.status_vbus) {
2817                         sc->sc_flags.status_vbus = 1;
2818
2819                         /* complete root HUB interrupt endpoint */
2820
2821                         dwc_otg_root_intr(sc);
2822                 }
2823         } else {
2824                 if (sc->sc_flags.status_vbus) {
2825                         sc->sc_flags.status_vbus = 0;
2826                         sc->sc_flags.status_bus_reset = 0;
2827                         sc->sc_flags.status_suspend = 0;
2828                         sc->sc_flags.change_suspend = 0;
2829                         sc->sc_flags.change_connect = 1;
2830
2831                         /* complete root HUB interrupt endpoint */
2832
2833                         dwc_otg_root_intr(sc);
2834                 }
2835         }
2836 }
2837
2838 int
2839 dwc_otg_filter_interrupt(void *arg)
2840 {
2841         struct dwc_otg_softc *sc = arg;
2842         int retval = FILTER_HANDLED;
2843         uint32_t status;
2844
2845         USB_BUS_SPIN_LOCK(&sc->sc_bus);
2846
2847         /* read and clear interrupt status */
2848         status = DWC_OTG_READ_4(sc, DOTG_GINTSTS);
2849
2850         /* clear interrupts we are handling here */
2851         DWC_OTG_WRITE_4(sc, DOTG_GINTSTS, status & ~DWC_OTG_MSK_GINT_THREAD_IRQ);
2852
2853         /* check for USB state change interrupts */
2854         if ((status & DWC_OTG_MSK_GINT_THREAD_IRQ) != 0)
2855                 retval = FILTER_SCHEDULE_THREAD;
2856
2857         /* clear FIFO empty interrupts */
2858         if (status & sc->sc_irq_mask &
2859             (GINTSTS_PTXFEMP | GINTSTS_NPTXFEMP)) {
2860                 sc->sc_irq_mask &= ~(GINTSTS_PTXFEMP | GINTSTS_NPTXFEMP);
2861                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2862         }
2863         /* clear all IN endpoint interrupts */
2864         if (status & GINTSTS_IEPINT) {
2865                 uint32_t temp;
2866                 uint8_t x;
2867
2868                 for (x = 0; x != sc->sc_dev_in_ep_max; x++) {
2869                         temp = DWC_OTG_READ_4(sc, DOTG_DIEPINT(x));
2870                         /*
2871                          * NOTE: Need to clear all interrupt bits,
2872                          * because some appears to be unmaskable and
2873                          * can cause an interrupt loop:
2874                          */
2875                         if (temp != 0)
2876                                 DWC_OTG_WRITE_4(sc, DOTG_DIEPINT(x), temp);
2877                 }
2878         }
2879
2880         /* poll FIFOs, if any */
2881         dwc_otg_interrupt_poll_locked(sc);
2882
2883         if (sc->sc_xfer_complete != 0)
2884                 retval = FILTER_SCHEDULE_THREAD;
2885
2886         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
2887
2888         return (retval);
2889 }
2890
2891 void
2892 dwc_otg_interrupt(void *arg)
2893 {
2894         struct dwc_otg_softc *sc = arg;
2895         uint32_t status;
2896
2897         USB_BUS_LOCK(&sc->sc_bus);
2898         USB_BUS_SPIN_LOCK(&sc->sc_bus);
2899
2900         /* read and clear interrupt status */
2901         status = DWC_OTG_READ_4(sc, DOTG_GINTSTS);
2902
2903         /* clear interrupts we are handling here */
2904         DWC_OTG_WRITE_4(sc, DOTG_GINTSTS, status & DWC_OTG_MSK_GINT_THREAD_IRQ);
2905
2906         DPRINTFN(14, "GINTSTS=0x%08x HAINT=0x%08x HFNUM=0x%08x\n",
2907             status, DWC_OTG_READ_4(sc, DOTG_HAINT),
2908             DWC_OTG_READ_4(sc, DOTG_HFNUM));
2909
2910         if (status & GINTSTS_USBRST) {
2911
2912                 /* set correct state */
2913                 sc->sc_flags.status_device_mode = 1;
2914                 sc->sc_flags.status_bus_reset = 0;
2915                 sc->sc_flags.status_suspend = 0;
2916                 sc->sc_flags.change_suspend = 0;
2917                 sc->sc_flags.change_connect = 1;
2918
2919                 /* Disable SOF interrupt */
2920                 sc->sc_irq_mask &= ~GINTMSK_SOFMSK;
2921                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2922
2923                 /* complete root HUB interrupt endpoint */
2924                 dwc_otg_root_intr(sc);
2925         }
2926
2927         /* check for any bus state change interrupts */
2928         if (status & GINTSTS_ENUMDONE) {
2929
2930                 uint32_t temp;
2931
2932                 DPRINTFN(5, "end of reset\n");
2933
2934                 /* set correct state */
2935                 sc->sc_flags.status_device_mode = 1;
2936                 sc->sc_flags.status_bus_reset = 1;
2937                 sc->sc_flags.status_suspend = 0;
2938                 sc->sc_flags.change_suspend = 0;
2939                 sc->sc_flags.change_connect = 1;
2940                 sc->sc_flags.status_low_speed = 0;
2941                 sc->sc_flags.port_enabled = 1;
2942
2943                 /* reset FIFOs */
2944                 (void) dwc_otg_init_fifo(sc, DWC_MODE_DEVICE);
2945
2946                 /* reset function address */
2947                 dwc_otg_set_address(sc, 0);
2948
2949                 /* figure out enumeration speed */
2950                 temp = DWC_OTG_READ_4(sc, DOTG_DSTS);
2951                 if (DSTS_ENUMSPD_GET(temp) == DSTS_ENUMSPD_HI)
2952                         sc->sc_flags.status_high_speed = 1;
2953                 else
2954                         sc->sc_flags.status_high_speed = 0;
2955
2956                 /*
2957                  * Disable resume and SOF interrupt, and enable
2958                  * suspend and RX frame interrupt:
2959                  */
2960                 sc->sc_irq_mask &= ~(GINTMSK_WKUPINTMSK | GINTMSK_SOFMSK);
2961                 sc->sc_irq_mask |= GINTMSK_USBSUSPMSK;
2962                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2963
2964                 /* complete root HUB interrupt endpoint */
2965                 dwc_otg_root_intr(sc);
2966         }
2967
2968         if (status & GINTSTS_PRTINT) {
2969                 uint32_t hprt;
2970
2971                 hprt = DWC_OTG_READ_4(sc, DOTG_HPRT);
2972
2973                 /* clear change bits */
2974                 DWC_OTG_WRITE_4(sc, DOTG_HPRT, (hprt & (
2975                     HPRT_PRTPWR | HPRT_PRTENCHNG |
2976                     HPRT_PRTCONNDET | HPRT_PRTOVRCURRCHNG)) |
2977                     sc->sc_hprt_val);
2978
2979                 DPRINTFN(12, "GINTSTS=0x%08x, HPRT=0x%08x\n", status, hprt);
2980
2981                 sc->sc_flags.status_device_mode = 0;
2982
2983                 if (hprt & HPRT_PRTCONNSTS)
2984                         sc->sc_flags.status_bus_reset = 1;
2985                 else
2986                         sc->sc_flags.status_bus_reset = 0;
2987
2988                 if ((hprt & HPRT_PRTENCHNG) &&
2989                     (hprt & HPRT_PRTENA) == 0)
2990                         sc->sc_flags.change_enabled = 1;
2991
2992                 if (hprt & HPRT_PRTENA)
2993                         sc->sc_flags.port_enabled = 1;
2994                 else
2995                         sc->sc_flags.port_enabled = 0;
2996
2997                 if (hprt & HPRT_PRTOVRCURRCHNG)
2998                         sc->sc_flags.change_over_current = 1;
2999
3000                 if (hprt & HPRT_PRTOVRCURRACT)
3001                         sc->sc_flags.port_over_current = 1;
3002                 else
3003                         sc->sc_flags.port_over_current = 0;
3004
3005                 if (hprt & HPRT_PRTPWR)
3006                         sc->sc_flags.port_powered = 1;
3007                 else
3008                         sc->sc_flags.port_powered = 0;
3009
3010                 if (((hprt & HPRT_PRTSPD_MASK)
3011                     >> HPRT_PRTSPD_SHIFT) == HPRT_PRTSPD_LOW)
3012                         sc->sc_flags.status_low_speed = 1;
3013                 else
3014                         sc->sc_flags.status_low_speed = 0;
3015
3016                 if (((hprt & HPRT_PRTSPD_MASK)
3017                     >> HPRT_PRTSPD_SHIFT) == HPRT_PRTSPD_HIGH)
3018                         sc->sc_flags.status_high_speed = 1;
3019                 else
3020                         sc->sc_flags.status_high_speed = 0;
3021
3022                 if (hprt & HPRT_PRTCONNDET)
3023                         sc->sc_flags.change_connect = 1;
3024
3025                 if (hprt & HPRT_PRTSUSP)
3026                         dwc_otg_suspend_irq(sc);
3027                 else
3028                         dwc_otg_resume_irq(sc);
3029
3030                 /* complete root HUB interrupt endpoint */
3031                 dwc_otg_root_intr(sc);
3032
3033                 /* update host frame interval */
3034                 dwc_otg_update_host_frame_interval(sc);
3035         }
3036
3037         /*
3038          * If resume and suspend is set at the same time we interpret
3039          * that like RESUME. Resume is set when there is at least 3
3040          * milliseconds of inactivity on the USB BUS.
3041          */
3042         if (status & GINTSTS_WKUPINT) {
3043
3044                 DPRINTFN(5, "resume interrupt\n");
3045
3046                 dwc_otg_resume_irq(sc);
3047
3048         } else if (status & GINTSTS_USBSUSP) {
3049
3050                 DPRINTFN(5, "suspend interrupt\n");
3051
3052                 dwc_otg_suspend_irq(sc);
3053         }
3054         /* check VBUS */
3055         if (status & (GINTSTS_USBSUSP |
3056             GINTSTS_USBRST |
3057             GINTMSK_OTGINTMSK |
3058             GINTSTS_SESSREQINT)) {
3059                 uint32_t temp;
3060
3061                 temp = DWC_OTG_READ_4(sc, DOTG_GOTGCTL);
3062
3063                 DPRINTFN(5, "GOTGCTL=0x%08x\n", temp);
3064
3065                 dwc_otg_vbus_interrupt(sc,
3066                     (temp & (GOTGCTL_ASESVLD | GOTGCTL_BSESVLD)) ? 1 : 0);
3067         }
3068
3069         if (sc->sc_xfer_complete != 0) {
3070                 sc->sc_xfer_complete = 0;
3071
3072                 /* complete FIFOs, if any */
3073                 dwc_otg_interrupt_complete_locked(sc);
3074         }
3075         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3076         USB_BUS_UNLOCK(&sc->sc_bus);
3077 }
3078
3079 static void
3080 dwc_otg_setup_standard_chain_sub(struct dwc_otg_std_temp *temp)
3081 {
3082         struct dwc_otg_td *td;
3083
3084         /* get current Transfer Descriptor */
3085         td = temp->td_next;
3086         temp->td = td;
3087
3088         /* prepare for next TD */
3089         temp->td_next = td->obj_next;
3090
3091         /* fill out the Transfer Descriptor */
3092         td->func = temp->func;
3093         td->pc = temp->pc;
3094         td->offset = temp->offset;
3095         td->remainder = temp->len;
3096         td->tx_bytes = 0;
3097         td->error_any = 0;
3098         td->error_stall = 0;
3099         td->npkt = 0;
3100         td->did_stall = temp->did_stall;
3101         td->short_pkt = temp->short_pkt;
3102         td->alt_next = temp->setup_alt_next;
3103         td->set_toggle = 0;
3104         td->got_short = 0;
3105         td->did_nak = 0;
3106         td->channel[0] = DWC_OTG_MAX_CHANNELS;
3107         td->channel[1] = DWC_OTG_MAX_CHANNELS;
3108         td->channel[2] = DWC_OTG_MAX_CHANNELS;
3109         td->state = 0;
3110         td->errcnt = 0;
3111         td->tt_scheduled = 0;
3112         td->tt_xactpos = HCSPLT_XACTPOS_BEGIN;
3113 }
3114
3115 static void
3116 dwc_otg_setup_standard_chain(struct usb_xfer *xfer)
3117 {
3118         struct dwc_otg_std_temp temp;
3119         struct dwc_otg_td *td;
3120         uint32_t x;
3121         uint8_t need_sync;
3122         uint8_t is_host;
3123
3124         DPRINTFN(9, "addr=%d endpt=%d sumlen=%d speed=%d\n",
3125             xfer->address, UE_GET_ADDR(xfer->endpointno),
3126             xfer->sumlen, usbd_get_speed(xfer->xroot->udev));
3127
3128         temp.max_frame_size = xfer->max_frame_size;
3129
3130         td = xfer->td_start[0];
3131         xfer->td_transfer_first = td;
3132         xfer->td_transfer_cache = td;
3133
3134         /* setup temp */
3135
3136         temp.pc = NULL;
3137         temp.td = NULL;
3138         temp.td_next = xfer->td_start[0];
3139         temp.offset = 0;
3140         temp.setup_alt_next = xfer->flags_int.short_frames_ok ||
3141             xfer->flags_int.isochronous_xfr;
3142         temp.did_stall = !xfer->flags_int.control_stall;
3143
3144         is_host = (xfer->xroot->udev->flags.usb_mode == USB_MODE_HOST);
3145
3146         /* check if we should prepend a setup message */
3147
3148         if (xfer->flags_int.control_xfr) {
3149                 if (xfer->flags_int.control_hdr) {
3150
3151                         if (is_host)
3152                                 temp.func = &dwc_otg_host_setup_tx;
3153                         else
3154                                 temp.func = &dwc_otg_setup_rx;
3155
3156                         temp.len = xfer->frlengths[0];
3157                         temp.pc = xfer->frbuffers + 0;
3158                         temp.short_pkt = temp.len ? 1 : 0;
3159
3160                         /* check for last frame */
3161                         if (xfer->nframes == 1) {
3162                                 /* no STATUS stage yet, SETUP is last */
3163                                 if (xfer->flags_int.control_act)
3164                                         temp.setup_alt_next = 0;
3165                         }
3166
3167                         dwc_otg_setup_standard_chain_sub(&temp);
3168                 }
3169                 x = 1;
3170         } else {
3171                 x = 0;
3172         }
3173
3174         if (x != xfer->nframes) {
3175                 if (xfer->endpointno & UE_DIR_IN) {
3176                         if (is_host) {
3177                                 temp.func = &dwc_otg_host_data_rx;
3178                                 need_sync = 0;
3179                         } else {
3180                                 temp.func = &dwc_otg_data_tx;
3181                                 need_sync = 1;
3182                         }
3183                 } else {
3184                         if (is_host) {
3185                                 temp.func = &dwc_otg_host_data_tx;
3186                                 need_sync = 0;
3187                         } else {
3188                                 temp.func = &dwc_otg_data_rx;
3189                                 need_sync = 0;
3190                         }
3191                 }
3192
3193                 /* setup "pc" pointer */
3194                 temp.pc = xfer->frbuffers + x;
3195         } else {
3196                 need_sync = 0;
3197         }
3198         while (x != xfer->nframes) {
3199
3200                 /* DATA0 / DATA1 message */
3201
3202                 temp.len = xfer->frlengths[x];
3203
3204                 x++;
3205
3206                 if (x == xfer->nframes) {
3207                         if (xfer->flags_int.control_xfr) {
3208                                 if (xfer->flags_int.control_act) {
3209                                         temp.setup_alt_next = 0;
3210                                 }
3211                         } else {
3212                                 temp.setup_alt_next = 0;
3213                         }
3214                 }
3215                 if (temp.len == 0) {
3216
3217                         /* make sure that we send an USB packet */
3218
3219                         temp.short_pkt = 0;
3220
3221                 } else {
3222
3223                         /* regular data transfer */
3224
3225                         temp.short_pkt = (xfer->flags.force_short_xfer ? 0 : 1);
3226                 }
3227
3228                 dwc_otg_setup_standard_chain_sub(&temp);
3229
3230                 if (xfer->flags_int.isochronous_xfr) {
3231                         temp.offset += temp.len;
3232                 } else {
3233                         /* get next Page Cache pointer */
3234                         temp.pc = xfer->frbuffers + x;
3235                 }
3236         }
3237
3238         if (xfer->flags_int.control_xfr) {
3239
3240                 /* always setup a valid "pc" pointer for status and sync */
3241                 temp.pc = xfer->frbuffers + 0;
3242                 temp.len = 0;
3243                 temp.short_pkt = 0;
3244                 temp.setup_alt_next = 0;
3245
3246                 /* check if we need to sync */
3247                 if (need_sync) {
3248                         /* we need a SYNC point after TX */
3249                         temp.func = &dwc_otg_data_tx_sync;
3250                         dwc_otg_setup_standard_chain_sub(&temp);
3251                 }
3252
3253                 /* check if we should append a status stage */
3254                 if (!xfer->flags_int.control_act) {
3255
3256                         /*
3257                          * Send a DATA1 message and invert the current
3258                          * endpoint direction.
3259                          */
3260                         if (xfer->endpointno & UE_DIR_IN) {
3261                                 if (is_host) {
3262                                         temp.func = &dwc_otg_host_data_tx;
3263                                         need_sync = 0;
3264                                 } else {
3265                                         temp.func = &dwc_otg_data_rx;
3266                                         need_sync = 0;
3267                                 }
3268                         } else {
3269                                 if (is_host) {
3270                                         temp.func = &dwc_otg_host_data_rx;
3271                                         need_sync = 0;
3272                                 } else {
3273                                         temp.func = &dwc_otg_data_tx;
3274                                         need_sync = 1;
3275                                 }
3276                         }
3277
3278                         dwc_otg_setup_standard_chain_sub(&temp);
3279
3280                         /* data toggle should be DATA1 */
3281                         td = temp.td;
3282                         td->set_toggle = 1;
3283
3284                         if (need_sync) {
3285                                 /* we need a SYNC point after TX */
3286                                 temp.func = &dwc_otg_data_tx_sync;
3287                                 dwc_otg_setup_standard_chain_sub(&temp);
3288                         }
3289                 }
3290         } else {
3291                 /* check if we need to sync */
3292                 if (need_sync) {
3293
3294                         temp.pc = xfer->frbuffers + 0;
3295                         temp.len = 0;
3296                         temp.short_pkt = 0;
3297                         temp.setup_alt_next = 0;
3298
3299                         /* we need a SYNC point after TX */
3300                         temp.func = &dwc_otg_data_tx_sync;
3301                         dwc_otg_setup_standard_chain_sub(&temp);
3302                 }
3303         }
3304
3305         /* must have at least one frame! */
3306         td = temp.td;
3307         xfer->td_transfer_last = td;
3308
3309         if (is_host) {
3310
3311                 struct dwc_otg_softc *sc;
3312                 uint32_t hcchar;
3313                 uint32_t hcsplt;
3314
3315                 sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
3316
3317                 /* get first again */
3318                 td = xfer->td_transfer_first;
3319                 td->toggle = (xfer->endpoint->toggle_next ? 1 : 0);
3320
3321                 hcchar =
3322                         (xfer->address << HCCHAR_DEVADDR_SHIFT) |
3323                         ((xfer->endpointno & UE_ADDR) << HCCHAR_EPNUM_SHIFT) |
3324                         (xfer->max_packet_size << HCCHAR_MPS_SHIFT) |
3325                         HCCHAR_CHENA;
3326
3327                 /*
3328                  * We are not always able to meet the timing
3329                  * requirements of the USB interrupt endpoint's
3330                  * complete split token, when doing transfers going
3331                  * via a transaction translator. Use the CONTROL
3332                  * transfer type instead of the INTERRUPT transfer
3333                  * type in general, as a means to workaround
3334                  * that. This trick should work for both FULL and LOW
3335                  * speed USB traffic going through a TT. For non-TT
3336                  * traffic it works as well. The reason for using
3337                  * CONTROL type instead of BULK is that some TTs might
3338                  * reject LOW speed BULK traffic.
3339                  */
3340                 if (td->ep_type == UE_INTERRUPT)
3341                         hcchar |= (UE_CONTROL << HCCHAR_EPTYPE_SHIFT);
3342                 else
3343                         hcchar |= (td->ep_type << HCCHAR_EPTYPE_SHIFT);
3344
3345                 if (UE_GET_DIR(xfer->endpointno) == UE_DIR_IN)
3346                         hcchar |= HCCHAR_EPDIR_IN;
3347
3348                 switch (xfer->xroot->udev->speed) {
3349                 case USB_SPEED_LOW:
3350                         hcchar |= HCCHAR_LSPDDEV;
3351                         /* FALLTHROUGH */
3352                 case USB_SPEED_FULL:
3353                         /* check if root HUB port is running High Speed */
3354                         if (dwc_otg_uses_split(xfer->xroot->udev)) {
3355                                 hcsplt = HCSPLT_SPLTENA |
3356                                     (xfer->xroot->udev->hs_port_no <<
3357                                     HCSPLT_PRTADDR_SHIFT) |
3358                                     (xfer->xroot->udev->hs_hub_addr <<
3359                                     HCSPLT_HUBADDR_SHIFT);
3360                         } else {
3361                                 hcsplt = 0;
3362                         }
3363                         if (td->ep_type == UE_INTERRUPT) {
3364                                 uint32_t ival;
3365                                 ival = xfer->interval / DWC_OTG_HOST_TIMER_RATE;
3366                                 if (ival == 0)
3367                                         ival = 1;
3368                                 else if (ival > 127)
3369                                         ival = 127;
3370                                 td->tmr_val = sc->sc_tmr_val + ival;
3371                                 td->tmr_res = ival;
3372                         } else if (td->ep_type == UE_ISOCHRONOUS) {
3373                                 td->tmr_res = 1;
3374                                 td->tmr_val = sc->sc_last_frame_num;
3375                                 if (td->hcchar & HCCHAR_EPDIR_IN)
3376                                         td->tmr_val++;
3377                         } else {
3378                                 td->tmr_val = 0;
3379                                 td->tmr_res = (uint8_t)sc->sc_last_frame_num;
3380                         }
3381                         break;
3382                 case USB_SPEED_HIGH:
3383                         hcsplt = 0;
3384                         if (td->ep_type == UE_INTERRUPT) {
3385                                 uint32_t ival;
3386                                 hcchar |= ((xfer->max_packet_count & 3)
3387                                     << HCCHAR_MC_SHIFT);
3388                                 ival = xfer->interval / DWC_OTG_HOST_TIMER_RATE;
3389                                 if (ival == 0)
3390                                         ival = 1;
3391                                 else if (ival > 127)
3392                                         ival = 127;
3393                                 td->tmr_val = sc->sc_tmr_val + ival;
3394                                 td->tmr_res = ival;
3395                         } else if (td->ep_type == UE_ISOCHRONOUS) {
3396                                 hcchar |= ((xfer->max_packet_count & 3)
3397                                     << HCCHAR_MC_SHIFT);
3398                                 td->tmr_res = 1 << usbd_xfer_get_fps_shift(xfer);
3399                                 td->tmr_val = sc->sc_last_frame_num;
3400                                 if (td->hcchar & HCCHAR_EPDIR_IN)
3401                                         td->tmr_val += td->tmr_res;
3402
3403                         } else {
3404                                 td->tmr_val = 0;
3405                                 td->tmr_res = (uint8_t)sc->sc_last_frame_num;
3406                         }
3407                         break;
3408                 default:
3409                         hcsplt = 0;
3410                         td->tmr_val = 0;
3411                         td->tmr_res = 0;
3412                         break;
3413                 }
3414
3415                 /* store configuration in all TD's */
3416                 while (1) {
3417                         td->hcchar = hcchar;
3418                         td->hcsplt = hcsplt;
3419
3420                         if (((void *)td) == xfer->td_transfer_last)
3421                                 break;
3422
3423                         td = td->obj_next;
3424                 }
3425         }
3426 }
3427
3428 static void
3429 dwc_otg_timeout(void *arg)
3430 {
3431         struct usb_xfer *xfer = arg;
3432
3433         DPRINTF("xfer=%p\n", xfer);
3434
3435         USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
3436
3437         /* transfer is transferred */
3438         dwc_otg_device_done(xfer, USB_ERR_TIMEOUT);
3439 }
3440
3441 static void
3442 dwc_otg_start_standard_chain(struct usb_xfer *xfer)
3443 {
3444         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
3445
3446         DPRINTFN(9, "\n");
3447
3448         /*
3449          * Poll one time in device mode, which will turn on the
3450          * endpoint interrupts. Else wait for SOF interrupt in host
3451          * mode.
3452          */
3453         USB_BUS_SPIN_LOCK(&sc->sc_bus);
3454
3455         if (sc->sc_flags.status_device_mode != 0) {
3456                 dwc_otg_xfer_do_fifo(sc, xfer);
3457                 if (dwc_otg_xfer_do_complete_locked(sc, xfer))
3458                         goto done;
3459         } else {
3460                 struct dwc_otg_td *td = xfer->td_transfer_cache;
3461                 if (td->ep_type == UE_ISOCHRONOUS &&
3462                     (td->hcchar & HCCHAR_EPDIR_IN) == 0) {
3463                         /*
3464                          * Need to start ISOCHRONOUS OUT transfer ASAP
3465                          * because execution is delayed by one 125us
3466                          * microframe:
3467                          */
3468                         dwc_otg_xfer_do_fifo(sc, xfer);
3469                         if (dwc_otg_xfer_do_complete_locked(sc, xfer))
3470                                 goto done;
3471                 }
3472         }
3473
3474         /* put transfer on interrupt queue */
3475         usbd_transfer_enqueue(&xfer->xroot->bus->intr_q, xfer);
3476
3477         /* start timeout, if any */
3478         if (xfer->timeout != 0) {
3479                 usbd_transfer_timeout_ms(xfer,
3480                     &dwc_otg_timeout, xfer->timeout);
3481         }
3482
3483         if (sc->sc_flags.status_device_mode != 0)
3484                 goto done;
3485
3486         /* enable SOF interrupt, if any */
3487         dwc_otg_enable_sof_irq(sc);
3488 done:
3489         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3490 }
3491
3492 static void
3493 dwc_otg_root_intr(struct dwc_otg_softc *sc)
3494 {
3495         DPRINTFN(9, "\n");
3496
3497         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
3498
3499         /* set port bit */
3500         sc->sc_hub_idata[0] = 0x02;     /* we only have one port */
3501
3502         uhub_root_intr(&sc->sc_bus, sc->sc_hub_idata,
3503             sizeof(sc->sc_hub_idata));
3504 }
3505
3506 static usb_error_t
3507 dwc_otg_standard_done_sub(struct usb_xfer *xfer)
3508 {
3509         struct dwc_otg_td *td;
3510         uint32_t len;
3511         usb_error_t error;
3512
3513         DPRINTFN(9, "\n");
3514
3515         td = xfer->td_transfer_cache;
3516
3517         do {
3518                 len = td->remainder;
3519
3520                 /* store last data toggle */
3521                 xfer->endpoint->toggle_next = td->toggle;
3522
3523                 if (xfer->aframes != xfer->nframes) {
3524                         /*
3525                          * Verify the length and subtract
3526                          * the remainder from "frlengths[]":
3527                          */
3528                         if (len > xfer->frlengths[xfer->aframes]) {
3529                                 td->error_any = 1;
3530                         } else {
3531                                 xfer->frlengths[xfer->aframes] -= len;
3532                         }
3533                 }
3534                 /* Check for transfer error */
3535                 if (td->error_any) {
3536                         /* the transfer is finished */
3537                         error = (td->error_stall ?
3538                             USB_ERR_STALLED : USB_ERR_IOERROR);
3539                         td = NULL;
3540                         break;
3541                 }
3542                 /* Check for short transfer */
3543                 if (len > 0) {
3544                         if (xfer->flags_int.short_frames_ok ||
3545                             xfer->flags_int.isochronous_xfr) {
3546                                 /* follow alt next */
3547                                 if (td->alt_next) {
3548                                         td = td->obj_next;
3549                                 } else {
3550                                         td = NULL;
3551                                 }
3552                         } else {
3553                                 /* the transfer is finished */
3554                                 td = NULL;
3555                         }
3556                         error = 0;
3557                         break;
3558                 }
3559                 td = td->obj_next;
3560
3561                 /* this USB frame is complete */
3562                 error = 0;
3563                 break;
3564
3565         } while (0);
3566
3567         /* update transfer cache */
3568
3569         xfer->td_transfer_cache = td;
3570
3571         return (error);
3572 }
3573
3574 static void
3575 dwc_otg_standard_done(struct usb_xfer *xfer)
3576 {
3577         usb_error_t err = 0;
3578
3579         DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
3580             xfer, xfer->endpoint);
3581
3582         /* reset scanner */
3583
3584         xfer->td_transfer_cache = xfer->td_transfer_first;
3585
3586         if (xfer->flags_int.control_xfr) {
3587
3588                 if (xfer->flags_int.control_hdr) {
3589
3590                         err = dwc_otg_standard_done_sub(xfer);
3591                 }
3592                 xfer->aframes = 1;
3593
3594                 if (xfer->td_transfer_cache == NULL) {
3595                         goto done;
3596                 }
3597         }
3598         while (xfer->aframes != xfer->nframes) {
3599
3600                 err = dwc_otg_standard_done_sub(xfer);
3601                 xfer->aframes++;
3602
3603                 if (xfer->td_transfer_cache == NULL) {
3604                         goto done;
3605                 }
3606         }
3607
3608         if (xfer->flags_int.control_xfr &&
3609             !xfer->flags_int.control_act) {
3610
3611                 err = dwc_otg_standard_done_sub(xfer);
3612         }
3613 done:
3614         dwc_otg_device_done(xfer, err);
3615 }
3616
3617 /*------------------------------------------------------------------------*
3618  *      dwc_otg_device_done
3619  *
3620  * NOTE: this function can be called more than one time on the
3621  * same USB transfer!
3622  *------------------------------------------------------------------------*/
3623 static void
3624 dwc_otg_device_done(struct usb_xfer *xfer, usb_error_t error)
3625 {
3626         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
3627
3628         DPRINTFN(9, "xfer=%p, endpoint=%p, error=%d\n",
3629             xfer, xfer->endpoint, error);
3630
3631         USB_BUS_SPIN_LOCK(&sc->sc_bus);
3632
3633         if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) {
3634                 /* Interrupts are cleared by the interrupt handler */
3635         } else {
3636                 struct dwc_otg_td *td;
3637
3638                 td = xfer->td_transfer_cache;
3639                 if (td != NULL)
3640                         dwc_otg_host_channel_free(sc, td);
3641         }
3642         /* dequeue transfer and start next transfer */
3643         usbd_transfer_done(xfer, error);
3644
3645         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3646 }
3647
3648 static void
3649 dwc_otg_xfer_stall(struct usb_xfer *xfer)
3650 {
3651         dwc_otg_device_done(xfer, USB_ERR_STALLED);
3652 }
3653
3654 static void
3655 dwc_otg_set_stall(struct usb_device *udev,
3656     struct usb_endpoint *ep, uint8_t *did_stall)
3657 {
3658         struct dwc_otg_softc *sc;
3659         uint32_t temp;
3660         uint32_t reg;
3661         uint8_t ep_no;
3662
3663         USB_BUS_LOCK_ASSERT(udev->bus, MA_OWNED);
3664
3665         /* check mode */
3666         if (udev->flags.usb_mode != USB_MODE_DEVICE) {
3667                 /* not supported */
3668                 return;
3669         }
3670
3671         sc = DWC_OTG_BUS2SC(udev->bus);
3672
3673         USB_BUS_SPIN_LOCK(&sc->sc_bus);
3674
3675         /* get endpoint address */
3676         ep_no = ep->edesc->bEndpointAddress;
3677
3678         DPRINTFN(5, "endpoint=0x%x\n", ep_no);
3679
3680         if (ep_no & UE_DIR_IN) {
3681                 reg = DOTG_DIEPCTL(ep_no & UE_ADDR);
3682                 temp = sc->sc_in_ctl[ep_no & UE_ADDR];
3683         } else {
3684                 reg = DOTG_DOEPCTL(ep_no & UE_ADDR);
3685                 temp = sc->sc_out_ctl[ep_no & UE_ADDR];
3686         }
3687
3688         /* disable and stall endpoint */
3689         DWC_OTG_WRITE_4(sc, reg, temp | DOEPCTL_EPDIS);
3690         DWC_OTG_WRITE_4(sc, reg, temp | DOEPCTL_STALL);
3691
3692         /* clear active OUT ep */
3693         if (!(ep_no & UE_DIR_IN)) {
3694
3695                 sc->sc_active_rx_ep &= ~(1U << (ep_no & UE_ADDR));
3696
3697                 if (sc->sc_last_rx_status != 0 &&
3698                     (ep_no & UE_ADDR) == GRXSTSRD_CHNUM_GET(
3699                     sc->sc_last_rx_status)) {
3700                         /* dump data */
3701                         dwc_otg_common_rx_ack(sc);
3702                         /* poll interrupt */
3703                         dwc_otg_interrupt_poll_locked(sc);
3704                         dwc_otg_interrupt_complete_locked(sc);
3705                 }
3706         }
3707         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3708 }
3709
3710 static void
3711 dwc_otg_clear_stall_sub_locked(struct dwc_otg_softc *sc, uint32_t mps,
3712     uint8_t ep_no, uint8_t ep_type, uint8_t ep_dir)
3713 {
3714         uint32_t reg;
3715         uint32_t temp;
3716
3717         if (ep_type == UE_CONTROL) {
3718                 /* clearing stall is not needed */
3719                 return;
3720         }
3721
3722         if (ep_dir) {
3723                 reg = DOTG_DIEPCTL(ep_no);
3724         } else {
3725                 reg = DOTG_DOEPCTL(ep_no);
3726                 sc->sc_active_rx_ep |= (1U << ep_no);
3727         }
3728
3729         /* round up and mask away the multiplier count */
3730         mps = (mps + 3) & 0x7FC;
3731
3732         if (ep_type == UE_BULK) {
3733                 temp = DIEPCTL_EPTYPE_SET(
3734                     DIEPCTL_EPTYPE_BULK) |
3735                     DIEPCTL_USBACTEP;
3736         } else if (ep_type == UE_INTERRUPT) {
3737                 temp = DIEPCTL_EPTYPE_SET(
3738                     DIEPCTL_EPTYPE_INTERRUPT) |
3739                     DIEPCTL_USBACTEP;
3740         } else {
3741                 temp = DIEPCTL_EPTYPE_SET(
3742                     DIEPCTL_EPTYPE_ISOC) |
3743                     DIEPCTL_USBACTEP;
3744         }
3745
3746         temp |= DIEPCTL_MPS_SET(mps);
3747         temp |= DIEPCTL_TXFNUM_SET(ep_no);
3748
3749         if (ep_dir)
3750                 sc->sc_in_ctl[ep_no] = temp;
3751         else
3752                 sc->sc_out_ctl[ep_no] = temp;
3753
3754         DWC_OTG_WRITE_4(sc, reg, temp | DOEPCTL_EPDIS);
3755         DWC_OTG_WRITE_4(sc, reg, temp | DOEPCTL_SETD0PID);
3756         DWC_OTG_WRITE_4(sc, reg, temp | DIEPCTL_SNAK);
3757
3758         /* we only reset the transmit FIFO */
3759         if (ep_dir) {
3760                 dwc_otg_tx_fifo_reset(sc,
3761                     GRSTCTL_TXFIFO(ep_no) |
3762                     GRSTCTL_TXFFLSH);
3763
3764                 DWC_OTG_WRITE_4(sc,
3765                     DOTG_DIEPTSIZ(ep_no), 0);
3766         }
3767
3768         /* poll interrupt */
3769         dwc_otg_interrupt_poll_locked(sc);
3770         dwc_otg_interrupt_complete_locked(sc);
3771 }
3772
3773 static void
3774 dwc_otg_clear_stall(struct usb_device *udev, struct usb_endpoint *ep)
3775 {
3776         struct dwc_otg_softc *sc;
3777         struct usb_endpoint_descriptor *ed;
3778
3779         DPRINTFN(5, "endpoint=%p\n", ep);
3780
3781         USB_BUS_LOCK_ASSERT(udev->bus, MA_OWNED);
3782
3783         /* check mode */
3784         if (udev->flags.usb_mode != USB_MODE_DEVICE) {
3785                 /* not supported */
3786                 return;
3787         }
3788         /* get softc */
3789         sc = DWC_OTG_BUS2SC(udev->bus);
3790
3791         USB_BUS_SPIN_LOCK(&sc->sc_bus);
3792
3793         /* get endpoint descriptor */
3794         ed = ep->edesc;
3795
3796         /* reset endpoint */
3797         dwc_otg_clear_stall_sub_locked(sc,
3798             UGETW(ed->wMaxPacketSize),
3799             (ed->bEndpointAddress & UE_ADDR),
3800             (ed->bmAttributes & UE_XFERTYPE),
3801             (ed->bEndpointAddress & (UE_DIR_IN | UE_DIR_OUT)));
3802
3803         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3804 }
3805
3806 static void
3807 dwc_otg_device_state_change(struct usb_device *udev)
3808 {
3809         struct dwc_otg_softc *sc;
3810         uint8_t x;
3811
3812         /* check mode */
3813         if (udev->flags.usb_mode != USB_MODE_DEVICE) {
3814                 /* not supported */
3815                 return;
3816         }
3817
3818         /* get softc */
3819         sc = DWC_OTG_BUS2SC(udev->bus);
3820
3821         /* deactivate all other endpoint but the control endpoint */
3822         if (udev->state == USB_STATE_CONFIGURED ||
3823             udev->state == USB_STATE_ADDRESSED) {
3824
3825                 USB_BUS_LOCK(&sc->sc_bus);
3826
3827                 for (x = 1; x != sc->sc_dev_ep_max; x++) {
3828
3829                         if (x < sc->sc_dev_in_ep_max) {
3830                                 DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(x),
3831                                     DIEPCTL_EPDIS);
3832                                 DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(x), 0);
3833                         }
3834
3835                         DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(x),
3836                             DOEPCTL_EPDIS);
3837                         DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(x), 0);
3838                 }
3839                 USB_BUS_UNLOCK(&sc->sc_bus);
3840         }
3841 }
3842
3843 int
3844 dwc_otg_init(struct dwc_otg_softc *sc)
3845 {
3846         uint32_t temp;
3847
3848         DPRINTF("start\n");
3849
3850         /* set up the bus structure */
3851         sc->sc_bus.usbrev = USB_REV_2_0;
3852         sc->sc_bus.methods = &dwc_otg_bus_methods;
3853
3854         usb_callout_init_mtx(&sc->sc_timer,
3855             &sc->sc_bus.bus_mtx, 0);
3856
3857         USB_BUS_LOCK(&sc->sc_bus);
3858
3859         /* turn on clocks */
3860         dwc_otg_clocks_on(sc);
3861
3862         temp = DWC_OTG_READ_4(sc, DOTG_GSNPSID);
3863         DPRINTF("Version = 0x%08x\n", temp);
3864
3865         /* disconnect */
3866         DWC_OTG_WRITE_4(sc, DOTG_DCTL,
3867             DCTL_SFTDISCON);
3868
3869         /* wait for host to detect disconnect */
3870         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 32);
3871
3872         DWC_OTG_WRITE_4(sc, DOTG_GRSTCTL,
3873             GRSTCTL_CSFTRST);
3874
3875         /* wait a little bit for block to reset */
3876         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 128);
3877
3878         switch (sc->sc_mode) {
3879         case DWC_MODE_DEVICE:
3880                 temp = GUSBCFG_FORCEDEVMODE;
3881                 break;
3882         case DWC_MODE_HOST:
3883                 temp = GUSBCFG_FORCEHOSTMODE;
3884                 break;
3885         default:
3886                 temp = 0;
3887                 break;
3888         }
3889
3890         if (sc->sc_phy_type == 0)
3891                 sc->sc_phy_type = dwc_otg_phy_type + 1;
3892         if (sc->sc_phy_bits == 0)
3893                 sc->sc_phy_bits = 16;
3894
3895         /* select HSIC, ULPI, UTMI+ or internal PHY mode */
3896         switch (sc->sc_phy_type) {
3897         case DWC_OTG_PHY_HSIC:
3898                 DWC_OTG_WRITE_4(sc, DOTG_GUSBCFG,
3899                     GUSBCFG_PHYIF |
3900                     GUSBCFG_TRD_TIM_SET(5) | temp);
3901                 DWC_OTG_WRITE_4(sc, DOTG_GOTGCTL,
3902                     0x000000EC);
3903
3904                 temp = DWC_OTG_READ_4(sc, DOTG_GLPMCFG);
3905                 DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3906                     temp & ~GLPMCFG_HSIC_CONN);
3907                 DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3908                     temp | GLPMCFG_HSIC_CONN);
3909                 break;
3910         case DWC_OTG_PHY_ULPI:
3911                 DWC_OTG_WRITE_4(sc, DOTG_GUSBCFG,
3912                     GUSBCFG_ULPI_UTMI_SEL |
3913                     GUSBCFG_TRD_TIM_SET(5) | temp);
3914                 DWC_OTG_WRITE_4(sc, DOTG_GOTGCTL, 0);
3915
3916                 temp = DWC_OTG_READ_4(sc, DOTG_GLPMCFG);
3917                 DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3918                     temp & ~GLPMCFG_HSIC_CONN);
3919                 break;
3920         case DWC_OTG_PHY_UTMI:
3921                 DWC_OTG_WRITE_4(sc, DOTG_GUSBCFG,
3922                     (sc->sc_phy_bits == 16 ? GUSBCFG_PHYIF : 0) |
3923                     GUSBCFG_TRD_TIM_SET(5) | temp);
3924                 DWC_OTG_WRITE_4(sc, DOTG_GOTGCTL, 0);
3925
3926                 temp = DWC_OTG_READ_4(sc, DOTG_GLPMCFG);
3927                 DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3928                     temp & ~GLPMCFG_HSIC_CONN);
3929                 break;
3930         case DWC_OTG_PHY_INTERNAL:
3931                 DWC_OTG_WRITE_4(sc, DOTG_GUSBCFG,
3932                     GUSBCFG_PHYSEL |
3933                     GUSBCFG_TRD_TIM_SET(5) | temp);
3934                 DWC_OTG_WRITE_4(sc, DOTG_GOTGCTL, 0);
3935
3936                 temp = DWC_OTG_READ_4(sc, DOTG_GLPMCFG);
3937                 DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3938                     temp & ~GLPMCFG_HSIC_CONN);
3939
3940                 temp = DWC_OTG_READ_4(sc, DOTG_GGPIO);
3941                 temp &= ~(DOTG_GGPIO_NOVBUSSENS | DOTG_GGPIO_I2CPADEN);
3942                 temp |= (DOTG_GGPIO_VBUSASEN | DOTG_GGPIO_VBUSBSEN |
3943                     DOTG_GGPIO_PWRDWN);
3944                 DWC_OTG_WRITE_4(sc, DOTG_GGPIO, temp);
3945                 break;
3946         default:
3947                 break;
3948         }
3949
3950         /* clear global nak */
3951         DWC_OTG_WRITE_4(sc, DOTG_DCTL,
3952             DCTL_CGOUTNAK |
3953             DCTL_CGNPINNAK);
3954
3955         /* disable USB port */
3956         DWC_OTG_WRITE_4(sc, DOTG_PCGCCTL, 0xFFFFFFFF);
3957
3958         /* wait 10ms */
3959         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 100);
3960
3961         /* enable USB port */
3962         DWC_OTG_WRITE_4(sc, DOTG_PCGCCTL, 0);
3963
3964         /* wait 10ms */
3965         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 100);
3966
3967         temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG3);
3968
3969         sc->sc_fifo_size = 4 * GHWCFG3_DFIFODEPTH_GET(temp);
3970
3971         temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG2);
3972
3973         sc->sc_dev_ep_max = GHWCFG2_NUMDEVEPS_GET(temp);
3974
3975         if (sc->sc_dev_ep_max > DWC_OTG_MAX_ENDPOINTS)
3976                 sc->sc_dev_ep_max = DWC_OTG_MAX_ENDPOINTS;
3977
3978         sc->sc_host_ch_max = GHWCFG2_NUMHSTCHNL_GET(temp);
3979
3980         if (sc->sc_host_ch_max > DWC_OTG_MAX_CHANNELS)
3981                 sc->sc_host_ch_max = DWC_OTG_MAX_CHANNELS;
3982
3983         temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG4);
3984
3985         sc->sc_dev_in_ep_max = GHWCFG4_NUM_IN_EP_GET(temp);
3986
3987         DPRINTF("Total FIFO size = %d bytes, Device EPs = %d/%d Host CHs = %d\n",
3988             sc->sc_fifo_size, sc->sc_dev_ep_max, sc->sc_dev_in_ep_max,
3989             sc->sc_host_ch_max);
3990
3991         /* setup FIFO */
3992         if (dwc_otg_init_fifo(sc, sc->sc_mode)) {
3993                 USB_BUS_UNLOCK(&sc->sc_bus);
3994                 return (EINVAL);
3995         }
3996
3997         /* enable interrupts */
3998         sc->sc_irq_mask |= DWC_OTG_MSK_GINT_THREAD_IRQ;
3999         DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
4000
4001         if (sc->sc_mode == DWC_MODE_OTG || sc->sc_mode == DWC_MODE_DEVICE) {
4002
4003                 /* enable all endpoint interrupts */
4004                 temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG2);
4005                 if (temp & GHWCFG2_MPI) {
4006                         uint8_t x;
4007
4008                         DPRINTF("Disable Multi Process Interrupts\n");
4009
4010                         for (x = 0; x != sc->sc_dev_in_ep_max; x++) {
4011                                 DWC_OTG_WRITE_4(sc, DOTG_DIEPEACHINTMSK(x), 0);
4012                                 DWC_OTG_WRITE_4(sc, DOTG_DOEPEACHINTMSK(x), 0);
4013                         }
4014                         DWC_OTG_WRITE_4(sc, DOTG_DEACHINTMSK, 0);
4015                 }
4016                 DWC_OTG_WRITE_4(sc, DOTG_DIEPMSK,
4017                     DIEPMSK_XFERCOMPLMSK);
4018                 DWC_OTG_WRITE_4(sc, DOTG_DOEPMSK, 0);
4019                 DWC_OTG_WRITE_4(sc, DOTG_DAINTMSK, 0xFFFF);
4020         }
4021
4022         if (sc->sc_mode == DWC_MODE_OTG || sc->sc_mode == DWC_MODE_HOST) {
4023                 /* setup clocks */
4024                 temp = DWC_OTG_READ_4(sc, DOTG_HCFG);
4025                 temp &= ~(HCFG_FSLSSUPP | HCFG_FSLSPCLKSEL_MASK);
4026                 temp |= (1 << HCFG_FSLSPCLKSEL_SHIFT);
4027                 DWC_OTG_WRITE_4(sc, DOTG_HCFG, temp);
4028         }
4029
4030         /* only enable global IRQ */
4031         DWC_OTG_WRITE_4(sc, DOTG_GAHBCFG,
4032             GAHBCFG_GLBLINTRMSK);
4033
4034         /* turn off clocks */
4035         dwc_otg_clocks_off(sc);
4036
4037         /* read initial VBUS state */
4038
4039         temp = DWC_OTG_READ_4(sc, DOTG_GOTGCTL);
4040
4041         DPRINTFN(5, "GOTCTL=0x%08x\n", temp);
4042
4043         dwc_otg_vbus_interrupt(sc,
4044             (temp & (GOTGCTL_ASESVLD | GOTGCTL_BSESVLD)) ? 1 : 0);
4045
4046         USB_BUS_UNLOCK(&sc->sc_bus);
4047
4048         /* catch any lost interrupts */
4049
4050         dwc_otg_do_poll(&sc->sc_bus);
4051
4052         return (0);                     /* success */
4053 }
4054
4055 void
4056 dwc_otg_uninit(struct dwc_otg_softc *sc)
4057 {
4058         USB_BUS_LOCK(&sc->sc_bus);
4059
4060         /* stop host timer */
4061         dwc_otg_timer_stop(sc);
4062
4063         /* set disconnect */
4064         DWC_OTG_WRITE_4(sc, DOTG_DCTL,
4065             DCTL_SFTDISCON);
4066
4067         /* turn off global IRQ */
4068         DWC_OTG_WRITE_4(sc, DOTG_GAHBCFG, 0);
4069
4070         sc->sc_flags.port_enabled = 0;
4071         sc->sc_flags.port_powered = 0;
4072         sc->sc_flags.status_vbus = 0;
4073         sc->sc_flags.status_bus_reset = 0;
4074         sc->sc_flags.status_suspend = 0;
4075         sc->sc_flags.change_suspend = 0;
4076         sc->sc_flags.change_connect = 1;
4077
4078         dwc_otg_pull_down(sc);
4079         dwc_otg_clocks_off(sc);
4080
4081         USB_BUS_UNLOCK(&sc->sc_bus);
4082
4083         usb_callout_drain(&sc->sc_timer);
4084 }
4085
4086 static void
4087 dwc_otg_suspend(struct dwc_otg_softc *sc)
4088 {
4089         return;
4090 }
4091
4092 static void
4093 dwc_otg_resume(struct dwc_otg_softc *sc)
4094 {
4095         return;
4096 }
4097
4098 static void
4099 dwc_otg_do_poll(struct usb_bus *bus)
4100 {
4101         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(bus);
4102
4103         USB_BUS_LOCK(&sc->sc_bus);
4104         USB_BUS_SPIN_LOCK(&sc->sc_bus);
4105         dwc_otg_interrupt_poll_locked(sc);
4106         dwc_otg_interrupt_complete_locked(sc);
4107         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
4108         USB_BUS_UNLOCK(&sc->sc_bus);
4109 }
4110
4111 /*------------------------------------------------------------------------*
4112  * DWC OTG bulk support
4113  * DWC OTG control support
4114  * DWC OTG interrupt support
4115  *------------------------------------------------------------------------*/
4116 static void
4117 dwc_otg_device_non_isoc_open(struct usb_xfer *xfer)
4118 {
4119 }
4120
4121 static void
4122 dwc_otg_device_non_isoc_close(struct usb_xfer *xfer)
4123 {
4124         dwc_otg_device_done(xfer, USB_ERR_CANCELLED);
4125 }
4126
4127 static void
4128 dwc_otg_device_non_isoc_enter(struct usb_xfer *xfer)
4129 {
4130 }
4131
4132 static void
4133 dwc_otg_device_non_isoc_start(struct usb_xfer *xfer)
4134 {
4135         /* setup TDs */
4136         dwc_otg_setup_standard_chain(xfer);
4137         dwc_otg_start_standard_chain(xfer);
4138 }
4139
4140 static const struct usb_pipe_methods dwc_otg_device_non_isoc_methods =
4141 {
4142         .open = dwc_otg_device_non_isoc_open,
4143         .close = dwc_otg_device_non_isoc_close,
4144         .enter = dwc_otg_device_non_isoc_enter,
4145         .start = dwc_otg_device_non_isoc_start,
4146 };
4147
4148 /*------------------------------------------------------------------------*
4149  * DWC OTG full speed isochronous support
4150  *------------------------------------------------------------------------*/
4151 static void
4152 dwc_otg_device_isoc_open(struct usb_xfer *xfer)
4153 {
4154 }
4155
4156 static void
4157 dwc_otg_device_isoc_close(struct usb_xfer *xfer)
4158 {
4159         dwc_otg_device_done(xfer, USB_ERR_CANCELLED);
4160 }
4161
4162 static void
4163 dwc_otg_device_isoc_enter(struct usb_xfer *xfer)
4164 {
4165 }
4166
4167 static void
4168 dwc_otg_device_isoc_start(struct usb_xfer *xfer)
4169 {
4170         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
4171         uint32_t temp;
4172         uint32_t msframes;
4173         uint32_t framenum;
4174         uint8_t shift = usbd_xfer_get_fps_shift(xfer);
4175
4176         DPRINTFN(6, "xfer=%p next=%d nframes=%d\n",
4177             xfer, xfer->endpoint->isoc_next, xfer->nframes);
4178
4179         if (xfer->xroot->udev->flags.usb_mode == USB_MODE_HOST) {
4180                 temp = DWC_OTG_READ_4(sc, DOTG_HFNUM);
4181
4182                 /* get the current frame index */
4183                 framenum = (temp & HFNUM_FRNUM_MASK);
4184         } else {
4185                 temp = DWC_OTG_READ_4(sc, DOTG_DSTS);
4186
4187                 /* get the current frame index */
4188                 framenum = DSTS_SOFFN_GET(temp);
4189         }
4190
4191         /*
4192          * Check if port is doing 8000 or 1000 frames per second:
4193          */
4194         if (sc->sc_flags.status_high_speed)
4195                 framenum /= 8;
4196
4197         framenum &= DWC_OTG_FRAME_MASK;
4198
4199         /*
4200          * Compute number of milliseconds worth of data traffic for
4201          * this USB transfer:
4202          */
4203         if (xfer->xroot->udev->speed == USB_SPEED_HIGH)
4204                 msframes = ((xfer->nframes << shift) + 7) / 8;
4205         else
4206                 msframes = xfer->nframes;
4207
4208         /*
4209          * check if the frame index is within the window where the frames
4210          * will be inserted
4211          */
4212         temp = (framenum - xfer->endpoint->isoc_next) & DWC_OTG_FRAME_MASK;
4213
4214         if ((xfer->endpoint->is_synced == 0) || (temp < msframes)) {
4215                 /*
4216                  * If there is data underflow or the pipe queue is
4217                  * empty we schedule the transfer a few frames ahead
4218                  * of the current frame position. Else two isochronous
4219                  * transfers might overlap.
4220                  */
4221                 xfer->endpoint->isoc_next = (framenum + 3) & DWC_OTG_FRAME_MASK;
4222                 xfer->endpoint->is_synced = 1;
4223                 DPRINTFN(3, "start next=%d\n", xfer->endpoint->isoc_next);
4224         }
4225         /*
4226          * compute how many milliseconds the insertion is ahead of the
4227          * current frame position:
4228          */
4229         temp = (xfer->endpoint->isoc_next - framenum) & DWC_OTG_FRAME_MASK;
4230
4231         /*
4232          * pre-compute when the isochronous transfer will be finished:
4233          */
4234         xfer->isoc_time_complete =
4235                 usb_isoc_time_expand(&sc->sc_bus, framenum) + temp + msframes;
4236
4237         /* setup TDs */
4238         dwc_otg_setup_standard_chain(xfer);
4239
4240         /* compute frame number for next insertion */
4241         xfer->endpoint->isoc_next += msframes;
4242
4243         /* start TD chain */
4244         dwc_otg_start_standard_chain(xfer);
4245 }
4246
4247 static const struct usb_pipe_methods dwc_otg_device_isoc_methods =
4248 {
4249         .open = dwc_otg_device_isoc_open,
4250         .close = dwc_otg_device_isoc_close,
4251         .enter = dwc_otg_device_isoc_enter,
4252         .start = dwc_otg_device_isoc_start,
4253 };
4254
4255 /*------------------------------------------------------------------------*
4256  * DWC OTG root control support
4257  *------------------------------------------------------------------------*
4258  * Simulate a hardware HUB by handling all the necessary requests.
4259  *------------------------------------------------------------------------*/
4260
4261 static const struct usb_device_descriptor dwc_otg_devd = {
4262         .bLength = sizeof(struct usb_device_descriptor),
4263         .bDescriptorType = UDESC_DEVICE,
4264         .bcdUSB = {0x00, 0x02},
4265         .bDeviceClass = UDCLASS_HUB,
4266         .bDeviceSubClass = UDSUBCLASS_HUB,
4267         .bDeviceProtocol = UDPROTO_HSHUBSTT,
4268         .bMaxPacketSize = 64,
4269         .bcdDevice = {0x00, 0x01},
4270         .iManufacturer = 1,
4271         .iProduct = 2,
4272         .bNumConfigurations = 1,
4273 };
4274
4275 static const struct dwc_otg_config_desc dwc_otg_confd = {
4276         .confd = {
4277                 .bLength = sizeof(struct usb_config_descriptor),
4278                 .bDescriptorType = UDESC_CONFIG,
4279                 .wTotalLength[0] = sizeof(dwc_otg_confd),
4280                 .bNumInterface = 1,
4281                 .bConfigurationValue = 1,
4282                 .iConfiguration = 0,
4283                 .bmAttributes = UC_SELF_POWERED,
4284                 .bMaxPower = 0,
4285         },
4286         .ifcd = {
4287                 .bLength = sizeof(struct usb_interface_descriptor),
4288                 .bDescriptorType = UDESC_INTERFACE,
4289                 .bNumEndpoints = 1,
4290                 .bInterfaceClass = UICLASS_HUB,
4291                 .bInterfaceSubClass = UISUBCLASS_HUB,
4292                 .bInterfaceProtocol = 0,
4293         },
4294         .endpd = {
4295                 .bLength = sizeof(struct usb_endpoint_descriptor),
4296                 .bDescriptorType = UDESC_ENDPOINT,
4297                 .bEndpointAddress = (UE_DIR_IN | DWC_OTG_INTR_ENDPT),
4298                 .bmAttributes = UE_INTERRUPT,
4299                 .wMaxPacketSize[0] = 8,
4300                 .bInterval = 255,
4301         },
4302 };
4303
4304 #define HSETW(ptr, val) ptr = { (uint8_t)(val), (uint8_t)((val) >> 8) }
4305
4306 static const struct usb_hub_descriptor_min dwc_otg_hubd = {
4307         .bDescLength = sizeof(dwc_otg_hubd),
4308         .bDescriptorType = UDESC_HUB,
4309         .bNbrPorts = 1,
4310         HSETW(.wHubCharacteristics, (UHD_PWR_NO_SWITCH | UHD_OC_INDIVIDUAL)),
4311         .bPwrOn2PwrGood = 50,
4312         .bHubContrCurrent = 0,
4313         .DeviceRemovable = {0},         /* port is removable */
4314 };
4315
4316 #define STRING_VENDOR \
4317   "D\0W\0C\0O\0T\0G"
4318
4319 #define STRING_PRODUCT \
4320   "O\0T\0G\0 \0R\0o\0o\0t\0 \0H\0U\0B"
4321
4322 USB_MAKE_STRING_DESC(STRING_VENDOR, dwc_otg_vendor);
4323 USB_MAKE_STRING_DESC(STRING_PRODUCT, dwc_otg_product);
4324
4325 static usb_error_t
4326 dwc_otg_roothub_exec(struct usb_device *udev,
4327     struct usb_device_request *req, const void **pptr, uint16_t *plength)
4328 {
4329         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(udev->bus);
4330         const void *ptr;
4331         uint16_t len;
4332         uint16_t value;
4333         uint16_t index;
4334         usb_error_t err;
4335
4336         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
4337
4338         /* buffer reset */
4339         ptr = (const void *)&sc->sc_hub_temp;
4340         len = 0;
4341         err = 0;
4342
4343         value = UGETW(req->wValue);
4344         index = UGETW(req->wIndex);
4345
4346         /* demultiplex the control request */
4347
4348         switch (req->bmRequestType) {
4349         case UT_READ_DEVICE:
4350                 switch (req->bRequest) {
4351                 case UR_GET_DESCRIPTOR:
4352                         goto tr_handle_get_descriptor;
4353                 case UR_GET_CONFIG:
4354                         goto tr_handle_get_config;
4355                 case UR_GET_STATUS:
4356                         goto tr_handle_get_status;
4357                 default:
4358                         goto tr_stalled;
4359                 }
4360                 break;
4361
4362         case UT_WRITE_DEVICE:
4363                 switch (req->bRequest) {
4364                 case UR_SET_ADDRESS:
4365                         goto tr_handle_set_address;
4366                 case UR_SET_CONFIG:
4367                         goto tr_handle_set_config;
4368                 case UR_CLEAR_FEATURE:
4369                         goto tr_valid;  /* nop */
4370                 case UR_SET_DESCRIPTOR:
4371                         goto tr_valid;  /* nop */
4372                 case UR_SET_FEATURE:
4373                 default:
4374                         goto tr_stalled;
4375                 }
4376                 break;
4377
4378         case UT_WRITE_ENDPOINT:
4379                 switch (req->bRequest) {
4380                 case UR_CLEAR_FEATURE:
4381                         switch (UGETW(req->wValue)) {
4382                         case UF_ENDPOINT_HALT:
4383                                 goto tr_handle_clear_halt;
4384                         case UF_DEVICE_REMOTE_WAKEUP:
4385                                 goto tr_handle_clear_wakeup;
4386                         default:
4387                                 goto tr_stalled;
4388                         }
4389                         break;
4390                 case UR_SET_FEATURE:
4391                         switch (UGETW(req->wValue)) {
4392                         case UF_ENDPOINT_HALT:
4393                                 goto tr_handle_set_halt;
4394                         case UF_DEVICE_REMOTE_WAKEUP:
4395                                 goto tr_handle_set_wakeup;
4396                         default:
4397                                 goto tr_stalled;
4398                         }
4399                         break;
4400                 case UR_SYNCH_FRAME:
4401                         goto tr_valid;  /* nop */
4402                 default:
4403                         goto tr_stalled;
4404                 }
4405                 break;
4406
4407         case UT_READ_ENDPOINT:
4408                 switch (req->bRequest) {
4409                 case UR_GET_STATUS:
4410                         goto tr_handle_get_ep_status;
4411                 default:
4412                         goto tr_stalled;
4413                 }
4414                 break;
4415
4416         case UT_WRITE_INTERFACE:
4417                 switch (req->bRequest) {
4418                 case UR_SET_INTERFACE:
4419                         goto tr_handle_set_interface;
4420                 case UR_CLEAR_FEATURE:
4421                         goto tr_valid;  /* nop */
4422                 case UR_SET_FEATURE:
4423                 default:
4424                         goto tr_stalled;
4425                 }
4426                 break;
4427
4428         case UT_READ_INTERFACE:
4429                 switch (req->bRequest) {
4430                 case UR_GET_INTERFACE:
4431                         goto tr_handle_get_interface;
4432                 case UR_GET_STATUS:
4433                         goto tr_handle_get_iface_status;
4434                 default:
4435                         goto tr_stalled;
4436                 }
4437                 break;
4438
4439         case UT_WRITE_CLASS_INTERFACE:
4440         case UT_WRITE_VENDOR_INTERFACE:
4441                 /* XXX forward */
4442                 break;
4443
4444         case UT_READ_CLASS_INTERFACE:
4445         case UT_READ_VENDOR_INTERFACE:
4446                 /* XXX forward */
4447                 break;
4448
4449         case UT_WRITE_CLASS_DEVICE:
4450                 switch (req->bRequest) {
4451                 case UR_CLEAR_FEATURE:
4452                         goto tr_valid;
4453                 case UR_SET_DESCRIPTOR:
4454                 case UR_SET_FEATURE:
4455                         break;
4456                 default:
4457                         goto tr_stalled;
4458                 }
4459                 break;
4460
4461         case UT_WRITE_CLASS_OTHER:
4462                 switch (req->bRequest) {
4463                 case UR_CLEAR_FEATURE:
4464                         goto tr_handle_clear_port_feature;
4465                 case UR_SET_FEATURE:
4466                         goto tr_handle_set_port_feature;
4467                 case UR_CLEAR_TT_BUFFER:
4468                 case UR_RESET_TT:
4469                 case UR_STOP_TT:
4470                         goto tr_valid;
4471
4472                 default:
4473                         goto tr_stalled;
4474                 }
4475                 break;
4476
4477         case UT_READ_CLASS_OTHER:
4478                 switch (req->bRequest) {
4479                 case UR_GET_TT_STATE:
4480                         goto tr_handle_get_tt_state;
4481                 case UR_GET_STATUS:
4482                         goto tr_handle_get_port_status;
4483                 default:
4484                         goto tr_stalled;
4485                 }
4486                 break;
4487
4488         case UT_READ_CLASS_DEVICE:
4489                 switch (req->bRequest) {
4490                 case UR_GET_DESCRIPTOR:
4491                         goto tr_handle_get_class_descriptor;
4492                 case UR_GET_STATUS:
4493                         goto tr_handle_get_class_status;
4494
4495                 default:
4496                         goto tr_stalled;
4497                 }
4498                 break;
4499         default:
4500                 goto tr_stalled;
4501         }
4502         goto tr_valid;
4503
4504 tr_handle_get_descriptor:
4505         switch (value >> 8) {
4506         case UDESC_DEVICE:
4507                 if (value & 0xff) {
4508                         goto tr_stalled;
4509                 }
4510                 len = sizeof(dwc_otg_devd);
4511                 ptr = (const void *)&dwc_otg_devd;
4512                 goto tr_valid;
4513         case UDESC_CONFIG:
4514                 if (value & 0xff) {
4515                         goto tr_stalled;
4516                 }
4517                 len = sizeof(dwc_otg_confd);
4518                 ptr = (const void *)&dwc_otg_confd;
4519                 goto tr_valid;
4520         case UDESC_STRING:
4521                 switch (value & 0xff) {
4522                 case 0:         /* Language table */
4523                         len = sizeof(usb_string_lang_en);
4524                         ptr = (const void *)&usb_string_lang_en;
4525                         goto tr_valid;
4526
4527                 case 1:         /* Vendor */
4528                         len = sizeof(dwc_otg_vendor);
4529                         ptr = (const void *)&dwc_otg_vendor;
4530                         goto tr_valid;
4531
4532                 case 2:         /* Product */
4533                         len = sizeof(dwc_otg_product);
4534                         ptr = (const void *)&dwc_otg_product;
4535                         goto tr_valid;
4536                 default:
4537                         break;
4538                 }
4539                 break;
4540         default:
4541                 goto tr_stalled;
4542         }
4543         goto tr_stalled;
4544
4545 tr_handle_get_config:
4546         len = 1;
4547         sc->sc_hub_temp.wValue[0] = sc->sc_conf;
4548         goto tr_valid;
4549
4550 tr_handle_get_status:
4551         len = 2;
4552         USETW(sc->sc_hub_temp.wValue, UDS_SELF_POWERED);
4553         goto tr_valid;
4554
4555 tr_handle_set_address:
4556         if (value & 0xFF00) {
4557                 goto tr_stalled;
4558         }
4559         sc->sc_rt_addr = value;
4560         goto tr_valid;
4561
4562 tr_handle_set_config:
4563         if (value >= 2) {
4564                 goto tr_stalled;
4565         }
4566         sc->sc_conf = value;
4567         goto tr_valid;
4568
4569 tr_handle_get_interface:
4570         len = 1;
4571         sc->sc_hub_temp.wValue[0] = 0;
4572         goto tr_valid;
4573
4574 tr_handle_get_tt_state:
4575 tr_handle_get_class_status:
4576 tr_handle_get_iface_status:
4577 tr_handle_get_ep_status:
4578         len = 2;
4579         USETW(sc->sc_hub_temp.wValue, 0);
4580         goto tr_valid;
4581
4582 tr_handle_set_halt:
4583 tr_handle_set_interface:
4584 tr_handle_set_wakeup:
4585 tr_handle_clear_wakeup:
4586 tr_handle_clear_halt:
4587         goto tr_valid;
4588
4589 tr_handle_clear_port_feature:
4590         if (index != 1)
4591                 goto tr_stalled;
4592
4593         DPRINTFN(9, "UR_CLEAR_PORT_FEATURE on port %d\n", index);
4594
4595         switch (value) {
4596         case UHF_PORT_SUSPEND:
4597                 dwc_otg_wakeup_peer(sc);
4598                 break;
4599
4600         case UHF_PORT_ENABLE:
4601                 if (sc->sc_flags.status_device_mode == 0) {
4602                         DWC_OTG_WRITE_4(sc, DOTG_HPRT,
4603                             sc->sc_hprt_val | HPRT_PRTENA);
4604                 }
4605                 sc->sc_flags.port_enabled = 0;
4606                 break;
4607
4608         case UHF_C_PORT_RESET:
4609                 sc->sc_flags.change_reset = 0;
4610                 break;
4611
4612         case UHF_C_PORT_ENABLE:
4613                 sc->sc_flags.change_enabled = 0;
4614                 break;
4615
4616         case UHF_C_PORT_OVER_CURRENT:
4617                 sc->sc_flags.change_over_current = 0;
4618                 break;
4619
4620         case UHF_PORT_TEST:
4621         case UHF_PORT_INDICATOR:
4622                 /* nops */
4623                 break;
4624
4625         case UHF_PORT_POWER:
4626                 sc->sc_flags.port_powered = 0;
4627                 if (sc->sc_mode == DWC_MODE_HOST || sc->sc_mode == DWC_MODE_OTG) {
4628                         sc->sc_hprt_val = 0;
4629                         DWC_OTG_WRITE_4(sc, DOTG_HPRT, HPRT_PRTENA);
4630                 }
4631                 dwc_otg_pull_down(sc);
4632                 dwc_otg_clocks_off(sc);
4633                 break;
4634
4635         case UHF_C_PORT_CONNECTION:
4636                 /* clear connect change flag */
4637                 sc->sc_flags.change_connect = 0;
4638                 break;
4639
4640         case UHF_C_PORT_SUSPEND:
4641                 sc->sc_flags.change_suspend = 0;
4642                 break;
4643
4644         default:
4645                 err = USB_ERR_IOERROR;
4646                 goto done;
4647         }
4648         goto tr_valid;
4649
4650 tr_handle_set_port_feature:
4651         if (index != 1) {
4652                 goto tr_stalled;
4653         }
4654         DPRINTFN(9, "UR_SET_PORT_FEATURE\n");
4655
4656         switch (value) {
4657         case UHF_PORT_ENABLE:
4658                 break;
4659
4660         case UHF_PORT_SUSPEND:
4661                 if (sc->sc_flags.status_device_mode == 0) {
4662                         /* set suspend BIT */
4663                         sc->sc_hprt_val |= HPRT_PRTSUSP;
4664                         DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
4665
4666                         /* generate HUB suspend event */
4667                         dwc_otg_suspend_irq(sc);
4668                 }
4669                 break;
4670
4671         case UHF_PORT_RESET:
4672                 if (sc->sc_flags.status_device_mode == 0) {
4673
4674                         DPRINTF("PORT RESET\n");
4675
4676                         /* enable PORT reset */
4677                         DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val | HPRT_PRTRST);
4678
4679                         /* Wait 62.5ms for reset to complete */
4680                         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 16);
4681
4682                         DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
4683
4684                         /* Wait 62.5ms for reset to complete */
4685                         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 16);
4686
4687                         /* reset FIFOs */
4688                         (void) dwc_otg_init_fifo(sc, DWC_MODE_HOST);
4689
4690                         sc->sc_flags.change_reset = 1;
4691                 } else {
4692                         err = USB_ERR_IOERROR;
4693                 }
4694                 break;
4695
4696         case UHF_PORT_TEST:
4697         case UHF_PORT_INDICATOR:
4698                 /* nops */
4699                 break;
4700         case UHF_PORT_POWER:
4701                 sc->sc_flags.port_powered = 1;
4702                 if (sc->sc_mode == DWC_MODE_HOST || sc->sc_mode == DWC_MODE_OTG) {
4703                         sc->sc_hprt_val |= HPRT_PRTPWR;
4704                         DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
4705                 }
4706                 if (sc->sc_mode == DWC_MODE_DEVICE || sc->sc_mode == DWC_MODE_OTG) {
4707                         /* pull up D+, if any */
4708                         dwc_otg_pull_up(sc);
4709                 }
4710                 break;
4711         default:
4712                 err = USB_ERR_IOERROR;
4713                 goto done;
4714         }
4715         goto tr_valid;
4716
4717 tr_handle_get_port_status:
4718
4719         DPRINTFN(9, "UR_GET_PORT_STATUS\n");
4720
4721         if (index != 1)
4722                 goto tr_stalled;
4723
4724         if (sc->sc_flags.status_vbus)
4725                 dwc_otg_clocks_on(sc);
4726         else
4727                 dwc_otg_clocks_off(sc);
4728
4729         /* Select Device Side Mode */
4730
4731         if (sc->sc_flags.status_device_mode) {
4732                 value = UPS_PORT_MODE_DEVICE;
4733                 dwc_otg_timer_stop(sc);
4734         } else {
4735                 value = 0;
4736                 dwc_otg_timer_start(sc);
4737         }
4738
4739         if (sc->sc_flags.status_high_speed)
4740                 value |= UPS_HIGH_SPEED;
4741         else if (sc->sc_flags.status_low_speed)
4742                 value |= UPS_LOW_SPEED;
4743
4744         if (sc->sc_flags.port_powered)
4745                 value |= UPS_PORT_POWER;
4746
4747         if (sc->sc_flags.port_enabled)
4748                 value |= UPS_PORT_ENABLED;
4749
4750         if (sc->sc_flags.port_over_current)
4751                 value |= UPS_OVERCURRENT_INDICATOR;
4752
4753         if (sc->sc_flags.status_vbus &&
4754             sc->sc_flags.status_bus_reset)
4755                 value |= UPS_CURRENT_CONNECT_STATUS;
4756
4757         if (sc->sc_flags.status_suspend)
4758                 value |= UPS_SUSPEND;
4759
4760         USETW(sc->sc_hub_temp.ps.wPortStatus, value);
4761
4762         value = 0;
4763
4764         if (sc->sc_flags.change_enabled)
4765                 value |= UPS_C_PORT_ENABLED;
4766         if (sc->sc_flags.change_connect)
4767                 value |= UPS_C_CONNECT_STATUS;
4768         if (sc->sc_flags.change_suspend)
4769                 value |= UPS_C_SUSPEND;
4770         if (sc->sc_flags.change_reset)
4771                 value |= UPS_C_PORT_RESET;
4772         if (sc->sc_flags.change_over_current)
4773                 value |= UPS_C_OVERCURRENT_INDICATOR;
4774
4775         USETW(sc->sc_hub_temp.ps.wPortChange, value);
4776         len = sizeof(sc->sc_hub_temp.ps);
4777         goto tr_valid;
4778
4779 tr_handle_get_class_descriptor:
4780         if (value & 0xFF) {
4781                 goto tr_stalled;
4782         }
4783         ptr = (const void *)&dwc_otg_hubd;
4784         len = sizeof(dwc_otg_hubd);
4785         goto tr_valid;
4786
4787 tr_stalled:
4788         err = USB_ERR_STALLED;
4789 tr_valid:
4790 done:
4791         *plength = len;
4792         *pptr = ptr;
4793         return (err);
4794 }
4795
4796 static void
4797 dwc_otg_xfer_setup(struct usb_setup_params *parm)
4798 {
4799         struct usb_xfer *xfer;
4800         void *last_obj;
4801         uint32_t ntd;
4802         uint32_t n;
4803         uint8_t ep_no;
4804         uint8_t ep_type;
4805
4806         xfer = parm->curr_xfer;
4807
4808         /*
4809          * NOTE: This driver does not use any of the parameters that
4810          * are computed from the following values. Just set some
4811          * reasonable dummies:
4812          */
4813         parm->hc_max_packet_size = 0x500;
4814         parm->hc_max_packet_count = 3;
4815         parm->hc_max_frame_size = 3 * 0x500;
4816
4817         usbd_transfer_setup_sub(parm);
4818
4819         /*
4820          * compute maximum number of TDs
4821          */
4822         ep_type = (xfer->endpoint->edesc->bmAttributes & UE_XFERTYPE);
4823
4824         if (ep_type == UE_CONTROL) {
4825
4826                 ntd = xfer->nframes + 1 /* STATUS */ + 1 /* SYNC 1 */
4827                     + 1 /* SYNC 2 */ + 1 /* SYNC 3 */;
4828         } else {
4829
4830                 ntd = xfer->nframes + 1 /* SYNC */ ;
4831         }
4832
4833         /*
4834          * check if "usbd_transfer_setup_sub" set an error
4835          */
4836         if (parm->err)
4837                 return;
4838
4839         /*
4840          * allocate transfer descriptors
4841          */
4842         last_obj = NULL;
4843
4844         ep_no = xfer->endpointno & UE_ADDR;
4845
4846         /*
4847          * Check for a valid endpoint profile in USB device mode:
4848          */
4849         if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) {
4850                 const struct usb_hw_ep_profile *pf;
4851
4852                 dwc_otg_get_hw_ep_profile(parm->udev, &pf, ep_no);
4853
4854                 if (pf == NULL) {
4855                         /* should not happen */
4856                         parm->err = USB_ERR_INVAL;
4857                         return;
4858                 }
4859         }
4860
4861         /* align data */
4862         parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
4863
4864         for (n = 0; n != ntd; n++) {
4865
4866                 struct dwc_otg_td *td;
4867
4868                 if (parm->buf) {
4869
4870                         td = USB_ADD_BYTES(parm->buf, parm->size[0]);
4871
4872                         /* compute shared bandwidth resource index for TT */
4873                         if (dwc_otg_uses_split(parm->udev)) {
4874                                 if (parm->udev->parent_hs_hub->ddesc.bDeviceProtocol == UDPROTO_HSHUBMTT)
4875                                         td->tt_index = parm->udev->device_index;
4876                                 else
4877                                         td->tt_index = parm->udev->parent_hs_hub->device_index;
4878                         } else {
4879                                 td->tt_index = parm->udev->device_index;
4880                         }
4881
4882                         /* init TD */
4883                         td->max_packet_size = xfer->max_packet_size;
4884                         td->max_packet_count = xfer->max_packet_count;
4885                         /* range check */
4886                         if (td->max_packet_count == 0 || td->max_packet_count > 3)
4887                                 td->max_packet_count = 1;
4888                         td->ep_no = ep_no;
4889                         td->ep_type = ep_type;
4890                         td->obj_next = last_obj;
4891
4892                         last_obj = td;
4893                 }
4894                 parm->size[0] += sizeof(*td);
4895         }
4896
4897         xfer->td_start[0] = last_obj;
4898 }
4899
4900 static void
4901 dwc_otg_xfer_unsetup(struct usb_xfer *xfer)
4902 {
4903         return;
4904 }
4905
4906 static void
4907 dwc_otg_ep_init(struct usb_device *udev, struct usb_endpoint_descriptor *edesc,
4908     struct usb_endpoint *ep)
4909 {
4910         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(udev->bus);
4911
4912         DPRINTFN(2, "endpoint=%p, addr=%d, endpt=%d, mode=%d (%d,%d)\n",
4913             ep, udev->address,
4914             edesc->bEndpointAddress, udev->flags.usb_mode,
4915             sc->sc_rt_addr, udev->device_index);
4916
4917         if (udev->device_index != sc->sc_rt_addr) {
4918
4919                 if (udev->flags.usb_mode == USB_MODE_DEVICE) {
4920                         if (udev->speed != USB_SPEED_FULL &&
4921                             udev->speed != USB_SPEED_HIGH) {
4922                                 /* not supported */
4923                                 return;
4924                         }
4925                 } else {
4926                         if (udev->speed == USB_SPEED_HIGH &&
4927                             (edesc->wMaxPacketSize[1] & 0x18) != 0 &&
4928                             (edesc->bmAttributes & UE_XFERTYPE) != UE_ISOCHRONOUS) {
4929                                 /* not supported */
4930                                 DPRINTFN(-1, "Non-isochronous high bandwidth "
4931                                     "endpoint not supported\n");
4932                                 return;
4933                         }
4934                 }
4935                 if ((edesc->bmAttributes & UE_XFERTYPE) == UE_ISOCHRONOUS)
4936                         ep->methods = &dwc_otg_device_isoc_methods;
4937                 else
4938                         ep->methods = &dwc_otg_device_non_isoc_methods;
4939         }
4940 }
4941
4942 static void
4943 dwc_otg_set_hw_power_sleep(struct usb_bus *bus, uint32_t state)
4944 {
4945         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(bus);
4946
4947         switch (state) {
4948         case USB_HW_POWER_SUSPEND:
4949                 dwc_otg_suspend(sc);
4950                 break;
4951         case USB_HW_POWER_SHUTDOWN:
4952                 dwc_otg_uninit(sc);
4953                 break;
4954         case USB_HW_POWER_RESUME:
4955                 dwc_otg_resume(sc);
4956                 break;
4957         default:
4958                 break;
4959         }
4960 }
4961
4962 static void
4963 dwc_otg_get_dma_delay(struct usb_device *udev, uint32_t *pus)
4964 {
4965         /* DMA delay - wait until any use of memory is finished */
4966         *pus = (2125);                  /* microseconds */
4967 }
4968
4969 static void
4970 dwc_otg_device_resume(struct usb_device *udev)
4971 {
4972         DPRINTF("\n");
4973
4974         /* poll all transfers again to restart resumed ones */
4975         dwc_otg_do_poll(udev->bus);
4976 }
4977
4978 static void
4979 dwc_otg_device_suspend(struct usb_device *udev)
4980 {
4981         DPRINTF("\n");
4982 }
4983
4984 static const struct usb_bus_methods dwc_otg_bus_methods =
4985 {
4986         .endpoint_init = &dwc_otg_ep_init,
4987         .xfer_setup = &dwc_otg_xfer_setup,
4988         .xfer_unsetup = &dwc_otg_xfer_unsetup,
4989         .get_hw_ep_profile = &dwc_otg_get_hw_ep_profile,
4990         .xfer_stall = &dwc_otg_xfer_stall,
4991         .set_stall = &dwc_otg_set_stall,
4992         .clear_stall = &dwc_otg_clear_stall,
4993         .roothub_exec = &dwc_otg_roothub_exec,
4994         .xfer_poll = &dwc_otg_do_poll,
4995         .device_state_change = &dwc_otg_device_state_change,
4996         .set_hw_power_sleep = &dwc_otg_set_hw_power_sleep,
4997         .get_dma_delay = &dwc_otg_get_dma_delay,
4998         .device_resume = &dwc_otg_device_resume,
4999         .device_suspend = &dwc_otg_device_suspend,
5000 };