sys/dev/usb/controller/dwc_otg.c

   1 /* $FreeBSD$ */
   2 /*-
   3  * Copyright (c) 2015 Daisuke Aoyama. All rights reserved.
   4  * Copyright (c) 2012-2015 Hans Petter Selasky. All rights reserved.
   5  * Copyright (c) 2010-2011 Aleksandr Rybalko. All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  26  * SUCH DAMAGE.
  27  */
  28
  29 /*
  30  * This file contains the driver for the DesignWare series USB 2.0 OTG
  31  * Controller.
  32  */
  33
  34 /*
  35  * LIMITATION: Drivers must be bound to all OUT endpoints in the
  36  * active configuration for this driver to work properly. Blocking any
  37  * OUT endpoint will block all OUT endpoints including the control
  38  * endpoint. Usually this is not a problem.
  39  */
  40
  41 /*
  42  * NOTE: Writing to non-existing registers appears to cause an
  43  * internal reset.
  44  */
  45
  46 #ifdef USB_GLOBAL_INCLUDE_FILE
  47 #include USB_GLOBAL_INCLUDE_FILE
  48 #else
  49 #include <sys/stdint.h>
  50 #include <sys/stddef.h>
  51 #include <sys/param.h>
  52 #include <sys/queue.h>
  53 #include <sys/types.h>
  54 #include <sys/systm.h>
  55 #include <sys/kernel.h>
  56 #include <sys/bus.h>
  57 #include <sys/module.h>
  58 #include <sys/lock.h>
  59 #include <sys/mutex.h>
  60 #include <sys/condvar.h>
  61 #include <sys/sysctl.h>
  62 #include <sys/sx.h>
  63 #include <sys/unistd.h>
  64 #include <sys/callout.h>
  65 #include <sys/malloc.h>
  66 #include <sys/priv.h>
  67
  68 #include <dev/usb/usb.h>
  69 #include <dev/usb/usbdi.h>
  70
  71 #define USB_DEBUG_VAR dwc_otg_debug
  72
  73 #include <dev/usb/usb_core.h>
  74 #include <dev/usb/usb_debug.h>
  75 #include <dev/usb/usb_busdma.h>
  76 #include <dev/usb/usb_process.h>
  77 #include <dev/usb/usb_transfer.h>
  78 #include <dev/usb/usb_device.h>
  79 #include <dev/usb/usb_hub.h>
  80 #include <dev/usb/usb_util.h>
  81
  82 #include <dev/usb/usb_controller.h>
  83 #include <dev/usb/usb_bus.h>
  84 #endif                  /* USB_GLOBAL_INCLUDE_FILE */
  85
  86 #include <dev/usb/controller/dwc_otg.h>
  87 #include <dev/usb/controller/dwc_otgreg.h>
  88
  89 #define DWC_OTG_BUS2SC(bus) \
  90    ((struct dwc_otg_softc *)(((uint8_t *)(bus)) - \
  91     ((uint8_t *)&(((struct dwc_otg_softc *)0)->sc_bus))))
  92
  93 #define DWC_OTG_PC2UDEV(pc) \
  94    (USB_DMATAG_TO_XROOT((pc)->tag_parent)->udev)
  95
  96 #define DWC_OTG_MSK_GINT_ENABLED        \
  97    (GINTMSK_ENUMDONEMSK |               \
  98    GINTMSK_USBRSTMSK |                  \
  99    GINTMSK_USBSUSPMSK |                 \
 100    GINTMSK_IEPINTMSK |                  \
 101    GINTMSK_SESSREQINTMSK |              \
 102    GINTMSK_RXFLVLMSK |                  \
 103    GINTMSK_HCHINTMSK |                  \
 104    GINTMSK_OTGINTMSK |                  \
 105    GINTMSK_PRTINTMSK)
 106
 107 #define DWC_OTG_MSK_GINT_THREAD_IRQ                             \
 108    (GINTSTS_USBRST | GINTSTS_ENUMDONE | GINTSTS_PRTINT |        \
 109    GINTSTS_WKUPINT | GINTSTS_USBSUSP | GINTMSK_OTGINTMSK |      \
 110    GINTSTS_SESSREQINT)
 111
 112 #define DWC_OTG_PHY_ULPI 0
 113 #define DWC_OTG_PHY_HSIC 1
 114 #define DWC_OTG_PHY_INTERNAL 2
 115
 116 #ifndef DWC_OTG_PHY_DEFAULT
 117 #define DWC_OTG_PHY_DEFAULT DWC_OTG_PHY_ULPI
 118 #endif
 119
 120 static int dwc_otg_phy_type = DWC_OTG_PHY_DEFAULT;
 121
 122 static SYSCTL_NODE(_hw_usb, OID_AUTO, dwc_otg, CTLFLAG_RW, 0, "USB DWC OTG");
 123 SYSCTL_INT(_hw_usb_dwc_otg, OID_AUTO, phy_type, CTLFLAG_RDTUN,
 124     &dwc_otg_phy_type, 0, "DWC OTG PHY TYPE - 0/1/2 - ULPI/HSIC/INTERNAL");
 125 TUNABLE_INT("hw.usb.dwc_otg.phy_type", &dwc_otg_phy_type);
 126
 127 #ifdef USB_DEBUG
 128 static int dwc_otg_debug;
 129
 130 SYSCTL_INT(_hw_usb_dwc_otg, OID_AUTO, debug, CTLFLAG_RW,
 131     &dwc_otg_debug, 0, "DWC OTG debug level");
 132 #endif
 133
 134 #define DWC_OTG_INTR_ENDPT 1
 135
 136 /* prototypes */
 137
 138 struct usb_bus_methods dwc_otg_bus_methods;
 139 struct usb_pipe_methods dwc_otg_device_non_isoc_methods;
 140 struct usb_pipe_methods dwc_otg_device_isoc_methods;
 141
 142 static dwc_otg_cmd_t dwc_otg_setup_rx;
 143 static dwc_otg_cmd_t dwc_otg_data_rx;
 144 static dwc_otg_cmd_t dwc_otg_data_tx;
 145 static dwc_otg_cmd_t dwc_otg_data_tx_sync;
 146
 147 static dwc_otg_cmd_t dwc_otg_host_setup_tx;
 148 static dwc_otg_cmd_t dwc_otg_host_data_tx;
 149 static dwc_otg_cmd_t dwc_otg_host_data_rx;
 150
 151 static void dwc_otg_device_done(struct usb_xfer *, usb_error_t);
 152 static void dwc_otg_do_poll(struct usb_bus *);
 153 static void dwc_otg_standard_done(struct usb_xfer *);
 154 static void dwc_otg_root_intr(struct dwc_otg_softc *);
 155 static void dwc_otg_interrupt_poll_locked(struct dwc_otg_softc *);
 156
 157 /*
 158  * Here is a configuration that the chip supports.
 159  */
 160 static const struct usb_hw_ep_profile dwc_otg_ep_profile[1] = {
 161
 162         [0] = {
 163                 .max_in_frame_size = 64,/* fixed */
 164                 .max_out_frame_size = 64,       /* fixed */
 165                 .is_simplex = 1,
 166                 .support_control = 1,
 167         }
 168 };
 169
 170 static void
 171 dwc_otg_get_hw_ep_profile(struct usb_device *udev,
 172     const struct usb_hw_ep_profile **ppf, uint8_t ep_addr)
 173 {
 174         struct dwc_otg_softc *sc;
 175
 176         sc = DWC_OTG_BUS2SC(udev->bus);
 177
 178         if (ep_addr < sc->sc_dev_ep_max)
 179                 *ppf = &sc->sc_hw_ep_profile[ep_addr].usb;
 180         else
 181                 *ppf = NULL;
 182 }
 183
 184 static void
 185 dwc_otg_write_fifo(struct dwc_otg_softc *sc, struct usb_page_cache *pc,
 186     uint32_t offset, uint32_t fifo, uint32_t count)
 187 {
 188         uint32_t temp;
 189
 190         /* round down length to nearest 4-bytes */
 191         temp = count & ~3;
 192
 193         /* check if we can write the data directly */
 194         if (temp != 0 && usb_pc_buffer_is_aligned(pc, offset, temp, 3)) {
 195                 struct usb_page_search buf_res;
 196
 197                 /* pre-subtract length */
 198                 count -= temp;
 199
 200                 /* iterate buffer list */
 201                 do {
 202                         /* get current buffer pointer */
 203                         usbd_get_page(pc, offset, &buf_res);
 204
 205                         if (buf_res.length > temp)
 206                                 buf_res.length = temp;
 207
 208                         /* transfer data into FIFO */
 209                         bus_space_write_region_4(sc->sc_io_tag, sc->sc_io_hdl,
 210                             fifo, buf_res.buffer, buf_res.length / 4);
 211
 212                         offset += buf_res.length;
 213                         fifo += buf_res.length;
 214                         temp -= buf_res.length;
 215                 } while (temp != 0);
 216         }
 217
 218         /* check for remainder */
 219         if (count != 0) {
 220                 /* clear topmost word before copy */
 221                 sc->sc_bounce_buffer[(count - 1) / 4] = 0;
 222
 223                 /* copy out data */
 224                 usbd_copy_out(pc, offset,
 225                     sc->sc_bounce_buffer, count);
 226
 227                 /* transfer data into FIFO */
 228                 bus_space_write_region_4(sc->sc_io_tag,
 229                     sc->sc_io_hdl, fifo, sc->sc_bounce_buffer,
 230                     (count + 3) / 4);
 231         }
 232 }
 233
 234 static void
 235 dwc_otg_read_fifo(struct dwc_otg_softc *sc, struct usb_page_cache *pc,
 236     uint32_t offset, uint32_t count)
 237 {
 238         uint32_t temp;
 239
 240         /* round down length to nearest 4-bytes */
 241         temp = count & ~3;
 242
 243         /* check if we can read the data directly */
 244         if (temp != 0 && usb_pc_buffer_is_aligned(pc, offset, temp, 3)) {
 245                 struct usb_page_search buf_res;
 246
 247                 /* pre-subtract length */
 248                 count -= temp;
 249
 250                 /* iterate buffer list */
 251                 do {
 252                         /* get current buffer pointer */
 253                         usbd_get_page(pc, offset, &buf_res);
 254
 255                         if (buf_res.length > temp)
 256                                 buf_res.length = temp;
 257
 258                         /* transfer data from FIFO */
 259                         bus_space_read_region_4(sc->sc_io_tag, sc->sc_io_hdl,
 260                             sc->sc_current_rx_fifo, buf_res.buffer, buf_res.length / 4);
 261
 262                         offset += buf_res.length;
 263                         sc->sc_current_rx_fifo += buf_res.length;
 264                         sc->sc_current_rx_bytes -= buf_res.length;
 265                         temp -= buf_res.length;
 266                 } while (temp != 0);
 267         }
 268
 269         /* check for remainder */
 270         if (count != 0) {
 271                 /* read data into bounce buffer */
 272                 bus_space_read_region_4(sc->sc_io_tag, sc->sc_io_hdl,
 273                         sc->sc_current_rx_fifo,
 274                         sc->sc_bounce_buffer, (count + 3) / 4);
 275
 276                 /* store data into proper buffer */
 277                 usbd_copy_in(pc, offset, sc->sc_bounce_buffer, count);
 278
 279                 /* round length up to nearest 4 bytes */
 280                 count = (count + 3) & ~3;
 281
 282                 /* update counters */
 283                 sc->sc_current_rx_bytes -= count;
 284                 sc->sc_current_rx_fifo += count;
 285         }
 286 }
 287
 288 static void
 289 dwc_otg_tx_fifo_reset(struct dwc_otg_softc *sc, uint32_t value)
 290 {
 291         uint32_t temp;
 292
 293         /* reset FIFO */
 294         DWC_OTG_WRITE_4(sc, DOTG_GRSTCTL, value);
 295
 296         /* wait for reset to complete */
 297         for (temp = 0; temp != 16; temp++) {
 298                 value = DWC_OTG_READ_4(sc, DOTG_GRSTCTL);
 299                 if (!(value & (GRSTCTL_TXFFLSH | GRSTCTL_RXFFLSH)))
 300                         break;
 301         }
 302 }
 303
 304 static int
 305 dwc_otg_init_fifo(struct dwc_otg_softc *sc, uint8_t mode)
 306 {
 307         struct dwc_otg_profile *pf;
 308         uint32_t fifo_size;
 309         uint32_t fifo_regs;
 310         uint32_t tx_start;
 311         uint8_t x;
 312
 313         fifo_size = sc->sc_fifo_size;
 314
 315         /*
 316          * NOTE: Reserved fixed size area at end of RAM, which must
 317          * not be allocated to the FIFOs:
 318          */
 319         fifo_regs = 4 * 16;
 320
 321         if (fifo_size < fifo_regs) {
 322                 DPRINTF("Too little FIFO\n");
 323                 return (EINVAL);
 324         }
 325
 326         /* subtract FIFO regs from total once */
 327         fifo_size -= fifo_regs;
 328
 329         /* split equally for IN and OUT */
 330         fifo_size /= 2;
 331
 332         /* Align to 4 bytes boundary (refer to PGM) */
 333         fifo_size &= ~3;
 334
 335         /* set global receive FIFO size */
 336         DWC_OTG_WRITE_4(sc, DOTG_GRXFSIZ, fifo_size / 4);
 337
 338         tx_start = fifo_size;
 339
 340         if (fifo_size < 64) {
 341                 DPRINTFN(-1, "Not enough data space for EP0 FIFO.\n");
 342                 return (EINVAL);
 343         }
 344
 345         if (mode == DWC_MODE_HOST) {
 346
 347                 /* reset active endpoints */
 348                 sc->sc_active_rx_ep = 0;
 349
 350                 /* split equally for periodic and non-periodic */
 351                 fifo_size /= 2;
 352
 353                 DPRINTF("PTX/NPTX FIFO=%u\n", fifo_size);
 354
 355                 /* align to 4 bytes boundary */
 356                 fifo_size &= ~3;
 357
 358                 DWC_OTG_WRITE_4(sc, DOTG_GNPTXFSIZ,
 359                     ((fifo_size / 4) << 16) |
 360                     (tx_start / 4));
 361
 362                 tx_start += fifo_size;
 363
 364                 for (x = 0; x != sc->sc_host_ch_max; x++) {
 365                         /* enable all host interrupts */
 366                         DWC_OTG_WRITE_4(sc, DOTG_HCINTMSK(x),
 367                             HCINT_DEFAULT_MASK);
 368                 }
 369
 370                 DWC_OTG_WRITE_4(sc, DOTG_HPTXFSIZ,
 371                     ((fifo_size / 4) << 16) |
 372                     (tx_start / 4));
 373
 374                 /* reset host channel state */
 375                 memset(sc->sc_chan_state, 0, sizeof(sc->sc_chan_state));
 376
 377                 /* enable all host channel interrupts */
 378                 DWC_OTG_WRITE_4(sc, DOTG_HAINTMSK,
 379                     (1U << sc->sc_host_ch_max) - 1U);
 380         }
 381
 382         if (mode == DWC_MODE_DEVICE) {
 383
 384             DWC_OTG_WRITE_4(sc, DOTG_GNPTXFSIZ,
 385                 (0x10 << 16) | (tx_start / 4));
 386             fifo_size -= 0x40;
 387             tx_start += 0x40;
 388
 389             /* setup control endpoint profile */
 390             sc->sc_hw_ep_profile[0].usb = dwc_otg_ep_profile[0];
 391
 392             /* reset active endpoints */
 393             sc->sc_active_rx_ep = 1;
 394
 395             for (x = 1; x != sc->sc_dev_ep_max; x++) {
 396
 397                 pf = sc->sc_hw_ep_profile + x;
 398
 399                 pf->usb.max_out_frame_size = 1024 * 3;
 400                 pf->usb.is_simplex = 0; /* assume duplex */
 401                 pf->usb.support_bulk = 1;
 402                 pf->usb.support_interrupt = 1;
 403                 pf->usb.support_isochronous = 1;
 404                 pf->usb.support_out = 1;
 405
 406                 if (x < sc->sc_dev_in_ep_max) {
 407                         uint32_t limit;
 408
 409                         limit = (x == 1) ? MIN(DWC_OTG_TX_MAX_FIFO_SIZE,
 410                             DWC_OTG_MAX_TXN) : MIN(DWC_OTG_MAX_TXN / 2,
 411                             DWC_OTG_TX_MAX_FIFO_SIZE);
 412
 413                         /* see if there is enough FIFO space */
 414                         if (limit <= fifo_size) {
 415                                 pf->max_buffer = limit;
 416                                 pf->usb.support_in = 1;
 417                         } else {
 418                                 limit = MIN(DWC_OTG_TX_MAX_FIFO_SIZE, 0x40);
 419                                 if (limit <= fifo_size) {
 420                                         pf->usb.support_in = 1;
 421                                 } else {
 422                                         pf->usb.is_simplex = 1;
 423                                         limit = 0;
 424                                 }
 425                         }
 426                         /* set FIFO size */
 427                         DWC_OTG_WRITE_4(sc, DOTG_DIEPTXF(x),
 428                             ((limit / 4) << 16) | (tx_start / 4));
 429                         tx_start += limit;
 430                         fifo_size -= limit;
 431                         pf->usb.max_in_frame_size = limit;
 432                 } else {
 433                         pf->usb.is_simplex = 1;
 434                 }
 435
 436                 DPRINTF("FIFO%d = IN:%d / OUT:%d\n", x,
 437                     pf->usb.max_in_frame_size,
 438                     pf->usb.max_out_frame_size);
 439             }
 440         }
 441
 442         /* reset RX FIFO */
 443         dwc_otg_tx_fifo_reset(sc, GRSTCTL_RXFFLSH);
 444
 445         if (mode != DWC_MODE_OTG) {
 446                 /* reset all TX FIFOs */
 447                 dwc_otg_tx_fifo_reset(sc,
 448                     GRSTCTL_TXFIFO(0x10) |
 449                     GRSTCTL_TXFFLSH);
 450         } else {
 451                 /* reset active endpoints */
 452                 sc->sc_active_rx_ep = 0;
 453
 454                 /* reset host channel state */
 455                 memset(sc->sc_chan_state, 0, sizeof(sc->sc_chan_state));
 456         }
 457         return (0);
 458 }
 459
 460 static uint8_t
 461 dwc_otg_uses_split(struct usb_device *udev)
 462 {
 463         /*
 464          * When a LOW or FULL speed device is connected directly to
 465          * the USB port we don't use split transactions:
 466          */
 467         return (udev->speed != USB_SPEED_HIGH &&
 468             udev->parent_hs_hub != NULL &&
 469             udev->parent_hs_hub->parent_hub != NULL);
 470 }
 471
 472 static void
 473 dwc_otg_update_host_frame_interval(struct dwc_otg_softc *sc)
 474 {
 475
 476   /*
 477    * Disabled until further. Assuming that the register is already
 478    * programmed correctly by the boot loader.
 479    */
 480 #if 0
 481         uint32_t temp;
 482
 483         /* setup HOST frame interval register, based on existing value */
 484         temp = DWC_OTG_READ_4(sc, DOTG_HFIR) & HFIR_FRINT_MASK;
 485         if (temp >= 10000)
 486                 temp /= 1000;
 487         else
 488                 temp /= 125;
 489
 490         /* figure out nearest X-tal value */
 491         if (temp >= 54)
 492                 temp = 60;      /* MHz */
 493         else if (temp >= 39)
 494                 temp = 48;      /* MHz */
 495         else
 496                 temp = 30;      /* MHz */
 497
 498         if (sc->sc_flags.status_high_speed)
 499                 temp *= 125;
 500         else
 501                 temp *= 1000;
 502
 503         DPRINTF("HFIR=0x%08x\n", temp);
 504
 505         DWC_OTG_WRITE_4(sc, DOTG_HFIR, temp);
 506 #endif
 507 }
 508
 509 static void
 510 dwc_otg_clocks_on(struct dwc_otg_softc *sc)
 511 {
 512         if (sc->sc_flags.clocks_off &&
 513             sc->sc_flags.port_powered) {
 514
 515                 DPRINTFN(5, "\n");
 516
 517                 /* TODO - platform specific */
 518
 519                 sc->sc_flags.clocks_off = 0;
 520         }
 521 }
 522
 523 static void
 524 dwc_otg_clocks_off(struct dwc_otg_softc *sc)
 525 {
 526         if (!sc->sc_flags.clocks_off) {
 527
 528                 DPRINTFN(5, "\n");
 529
 530                 /* TODO - platform specific */
 531
 532                 sc->sc_flags.clocks_off = 1;
 533         }
 534 }
 535
 536 static void
 537 dwc_otg_pull_up(struct dwc_otg_softc *sc)
 538 {
 539         uint32_t temp;
 540
 541         /* pullup D+, if possible */
 542
 543         if (!sc->sc_flags.d_pulled_up &&
 544             sc->sc_flags.port_powered) {
 545                 sc->sc_flags.d_pulled_up = 1;
 546
 547                 temp = DWC_OTG_READ_4(sc, DOTG_DCTL);
 548                 temp &= ~DCTL_SFTDISCON;
 549                 DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
 550         }
 551 }
 552
 553 static void
 554 dwc_otg_pull_down(struct dwc_otg_softc *sc)
 555 {
 556         uint32_t temp;
 557
 558         /* pulldown D+, if possible */
 559
 560         if (sc->sc_flags.d_pulled_up) {
 561                 sc->sc_flags.d_pulled_up = 0;
 562
 563                 temp = DWC_OTG_READ_4(sc, DOTG_DCTL);
 564                 temp |= DCTL_SFTDISCON;
 565                 DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
 566         }
 567 }
 568
 569 static void
 570 dwc_otg_enable_sof_irq(struct dwc_otg_softc *sc)
 571 {
 572         /* In device mode we don't use the SOF interrupt */
 573         if (sc->sc_flags.status_device_mode != 0)
 574                 return;
 575         /* Ensure the SOF interrupt is not disabled */
 576         sc->sc_needsof = 1;
 577         /* Check if the SOF interrupt is already enabled */
 578         if ((sc->sc_irq_mask & GINTMSK_SOFMSK) != 0)
 579                 return;
 580         sc->sc_irq_mask |= GINTMSK_SOFMSK;
 581         DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 582 }
 583
 584 static void
 585 dwc_otg_resume_irq(struct dwc_otg_softc *sc)
 586 {
 587         if (sc->sc_flags.status_suspend) {
 588                 /* update status bits */
 589                 sc->sc_flags.status_suspend = 0;
 590                 sc->sc_flags.change_suspend = 1;
 591
 592                 if (sc->sc_flags.status_device_mode) {
 593                         /*
 594                          * Disable resume interrupt and enable suspend
 595                          * interrupt:
 596                          */
 597                         sc->sc_irq_mask &= ~GINTMSK_WKUPINTMSK;
 598                         sc->sc_irq_mask |= GINTMSK_USBSUSPMSK;
 599                         DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 600                 }
 601
 602                 /* complete root HUB interrupt endpoint */
 603                 dwc_otg_root_intr(sc);
 604         }
 605 }
 606
 607 static void
 608 dwc_otg_suspend_irq(struct dwc_otg_softc *sc)
 609 {
 610         if (!sc->sc_flags.status_suspend) {
 611                 /* update status bits */
 612                 sc->sc_flags.status_suspend = 1;
 613                 sc->sc_flags.change_suspend = 1;
 614
 615                 if (sc->sc_flags.status_device_mode) {
 616                         /*
 617                          * Disable suspend interrupt and enable resume
 618                          * interrupt:
 619                          */
 620                         sc->sc_irq_mask &= ~GINTMSK_USBSUSPMSK;
 621                         sc->sc_irq_mask |= GINTMSK_WKUPINTMSK;
 622                         DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 623                 }
 624
 625                 /* complete root HUB interrupt endpoint */
 626                 dwc_otg_root_intr(sc);
 627         }
 628 }
 629
 630 static void
 631 dwc_otg_wakeup_peer(struct dwc_otg_softc *sc)
 632 {
 633         if (!sc->sc_flags.status_suspend)
 634                 return;
 635
 636         DPRINTFN(5, "Remote wakeup\n");
 637
 638         if (sc->sc_flags.status_device_mode) {
 639                 uint32_t temp;
 640
 641                 /* enable remote wakeup signalling */
 642                 temp = DWC_OTG_READ_4(sc, DOTG_DCTL);
 643                 temp |= DCTL_RMTWKUPSIG;
 644                 DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
 645
 646                 /* Wait 8ms for remote wakeup to complete. */
 647                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 125);
 648
 649                 temp &= ~DCTL_RMTWKUPSIG;
 650                 DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
 651         } else {
 652                 /* enable USB port */
 653                 DWC_OTG_WRITE_4(sc, DOTG_PCGCCTL, 0);
 654
 655                 /* wait 10ms */
 656                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 100);
 657
 658                 /* resume port */
 659                 sc->sc_hprt_val |= HPRT_PRTRES;
 660                 DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
 661
 662                 /* Wait 100ms for resume signalling to complete. */
 663                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 10);
 664
 665                 /* clear suspend and resume */
 666                 sc->sc_hprt_val &= ~(HPRT_PRTSUSP | HPRT_PRTRES);
 667                 DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
 668
 669                 /* Wait 4ms */
 670                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 250);
 671         }
 672
 673         /* need to fake resume IRQ */
 674         dwc_otg_resume_irq(sc);
 675 }
 676
 677 static void
 678 dwc_otg_set_address(struct dwc_otg_softc *sc, uint8_t addr)
 679 {
 680         uint32_t temp;
 681
 682         DPRINTFN(5, "addr=%d\n", addr);
 683
 684         temp = DWC_OTG_READ_4(sc, DOTG_DCFG);
 685         temp &= ~DCFG_DEVADDR_SET(0x7F);
 686         temp |= DCFG_DEVADDR_SET(addr);
 687         DWC_OTG_WRITE_4(sc, DOTG_DCFG, temp);
 688 }
 689
 690 static void
 691 dwc_otg_common_rx_ack(struct dwc_otg_softc *sc)
 692 {
 693         DPRINTFN(5, "RX status clear\n");
 694
 695         /* enable RX FIFO level interrupt */
 696         sc->sc_irq_mask |= GINTMSK_RXFLVLMSK;
 697         DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 698
 699         if (sc->sc_current_rx_bytes != 0) {
 700                 /* need to dump remaining data */
 701                 bus_space_read_region_4(sc->sc_io_tag, sc->sc_io_hdl,
 702                     sc->sc_current_rx_fifo, sc->sc_bounce_buffer,
 703                     sc->sc_current_rx_bytes / 4);
 704                 /* clear number of active bytes to receive */
 705                 sc->sc_current_rx_bytes = 0;
 706         }
 707         /* clear cached status */
 708         sc->sc_last_rx_status = 0;
 709 }
 710
 711 static void
 712 dwc_otg_clear_hcint(struct dwc_otg_softc *sc, uint8_t x)
 713 {
 714         uint32_t hcint;
 715
 716         /* clear all pending interrupts */
 717         hcint = DWC_OTG_READ_4(sc, DOTG_HCINT(x));
 718         DWC_OTG_WRITE_4(sc, DOTG_HCINT(x), hcint);
 719
 720         /* clear buffered interrupts */
 721         sc->sc_chan_state[x].hcint = 0;
 722 }
 723
 724 static uint8_t
 725 dwc_otg_host_check_tx_fifo_empty(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
 726 {
 727         uint32_t temp;
 728
 729         temp = DWC_OTG_READ_4(sc, DOTG_GINTSTS);
 730
 731         if (td->ep_type == UE_ISOCHRONOUS) {
 732                 /*
 733                  * NOTE: USB INTERRUPT transactions are executed like
 734                  * USB CONTROL transactions! See the setup standard
 735                  * chain function for more information.
 736                  */
 737                 if (!(temp & GINTSTS_PTXFEMP)) {
 738                         DPRINTF("Periodic TX FIFO is not empty\n");
 739                         if (!(sc->sc_irq_mask & GINTMSK_PTXFEMPMSK)) {
 740                                 sc->sc_irq_mask |= GINTMSK_PTXFEMPMSK;
 741                                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 742                         }
 743                         return (1);     /* busy */
 744                 }
 745         } else {
 746                 if (!(temp & GINTSTS_NPTXFEMP)) {
 747                         DPRINTF("Non-periodic TX FIFO is not empty\n");
 748                         if (!(sc->sc_irq_mask & GINTMSK_NPTXFEMPMSK)) {
 749                                 sc->sc_irq_mask |= GINTMSK_NPTXFEMPMSK;
 750                                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 751                         }
 752                         return (1);     /* busy */
 753                 }
 754         }
 755         return (0);     /* ready for transmit */
 756 }
 757
 758 static uint8_t
 759 dwc_otg_host_channel_alloc(struct dwc_otg_softc *sc,
 760     struct dwc_otg_td *td, uint8_t is_out)
 761 {
 762         uint8_t x;
 763         uint8_t y;
 764         uint8_t z;
 765
 766         if (td->channel[0] < DWC_OTG_MAX_CHANNELS)
 767                 return (0);             /* already allocated */
 768
 769         /* check if device is suspended */
 770         if (DWC_OTG_PC2UDEV(td->pc)->flags.self_suspended != 0)
 771                 return (1);             /* busy - cannot transfer data */
 772
 773         /* compute needed TX FIFO size */
 774         if (is_out != 0) {
 775                 if (dwc_otg_host_check_tx_fifo_empty(sc, td) != 0)
 776                         return (1);     /* busy - cannot transfer data */
 777         }
 778         z = td->max_packet_count;
 779         for (x = y = 0; x != sc->sc_host_ch_max; x++) {
 780                 /* check if channel is allocated */
 781                 if (sc->sc_chan_state[x].allocated != 0)
 782                         continue;
 783                 /* check if channel is still enabled */
 784                 if (sc->sc_chan_state[x].wait_halted != 0)
 785                         continue;
 786                 /* store channel number */
 787                 td->channel[y++] = x;
 788                 /* check if we got all channels */
 789                 if (y == z)
 790                         break;
 791         }
 792         if (y != z) {
 793                 /* reset channel variable */
 794                 td->channel[0] = DWC_OTG_MAX_CHANNELS;
 795                 td->channel[1] = DWC_OTG_MAX_CHANNELS;
 796                 td->channel[2] = DWC_OTG_MAX_CHANNELS;
 797                 /* wait a bit */
 798                 dwc_otg_enable_sof_irq(sc);
 799                 return (1);     /* busy - not enough channels */
 800         }
 801
 802         for (y = 0; y != z; y++) {
 803                 x = td->channel[y];
 804
 805                 /* set allocated */
 806                 sc->sc_chan_state[x].allocated = 1;
 807
 808                 /* set wait halted */
 809                 sc->sc_chan_state[x].wait_halted = 1;
 810
 811                 /* clear interrupts */
 812                 dwc_otg_clear_hcint(sc, x);
 813
 814                 DPRINTF("CH=%d HCCHAR=0x%08x "
 815                     "HCSPLT=0x%08x\n", x, td->hcchar, td->hcsplt);
 816
 817                 /* set active channel */
 818                 sc->sc_active_rx_ep |= (1 << x);
 819         }
 820         return (0);     /* allocated */
 821 }
 822
 823 static void
 824 dwc_otg_host_channel_free_sub(struct dwc_otg_softc *sc, struct dwc_otg_td *td, uint8_t index)
 825 {
 826         uint32_t hcchar;
 827         uint8_t x;
 828
 829         if (td->channel[index] >= DWC_OTG_MAX_CHANNELS)
 830                 return;         /* already freed */
 831
 832         /* free channel */
 833         x = td->channel[index];
 834         td->channel[index] = DWC_OTG_MAX_CHANNELS;
 835
 836         DPRINTF("CH=%d\n", x);
 837
 838         /*
 839          * We need to let programmed host channels run till complete
 840          * else the host channel will stop functioning.
 841          */
 842         sc->sc_chan_state[x].allocated = 0;
 843
 844         /* ack any pending messages */
 845         if (sc->sc_last_rx_status != 0 &&
 846             GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) == x) {
 847                 dwc_otg_common_rx_ack(sc);
 848         }
 849
 850         /* clear active channel */
 851         sc->sc_active_rx_ep &= ~(1 << x);
 852
 853         /* check if already halted */
 854         if (sc->sc_chan_state[x].wait_halted == 0)
 855                 return;
 856
 857         /* disable host channel */
 858         hcchar = DWC_OTG_READ_4(sc, DOTG_HCCHAR(x));
 859         if (hcchar & HCCHAR_CHENA) {
 860                 DPRINTF("Halting channel %d\n", x);
 861                 DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(x),
 862                     hcchar | HCCHAR_CHDIS);
 863                 /* don't write HCCHAR until the channel is halted */
 864         } else {
 865                 sc->sc_chan_state[x].wait_halted = 0;
 866         }
 867 }
 868
 869 static void
 870 dwc_otg_host_channel_free(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
 871 {
 872         uint8_t x;
 873         for (x = 0; x != td->max_packet_count; x++)
 874                 dwc_otg_host_channel_free_sub(sc, td, x);
 875 }
 876
 877 static void
 878 dwc_otg_host_dump_rx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
 879 {
 880         uint8_t x;
 881         /* dump any pending messages */
 882         if (sc->sc_last_rx_status == 0)
 883                 return;
 884         for (x = 0; x != td->max_packet_count; x++) {
 885                 if (td->channel[x] >= DWC_OTG_MAX_CHANNELS ||
 886                     td->channel[x] != GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status))
 887                         continue;
 888                 dwc_otg_common_rx_ack(sc);
 889                 break;
 890         }
 891 }
 892
 893 static uint8_t
 894 dwc_otg_host_setup_tx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
 895 {
 896         struct usb_device_request req __aligned(4);
 897         uint32_t hcint;
 898         uint32_t hcchar;
 899         uint8_t delta;
 900
 901         dwc_otg_host_dump_rx(sc, td);
 902
 903         if (td->channel[0] < DWC_OTG_MAX_CHANNELS) {
 904                 hcint = sc->sc_chan_state[td->channel[0]].hcint;
 905
 906                 DPRINTF("CH=%d ST=%d HCINT=0x%08x HCCHAR=0x%08x HCTSIZ=0x%08x\n",
 907                     td->channel[0], td->state, hcint,
 908                     DWC_OTG_READ_4(sc, DOTG_HCCHAR(td->channel[0])),
 909                     DWC_OTG_READ_4(sc, DOTG_HCTSIZ(td->channel[0])));
 910         } else {
 911                 hcint = 0;
 912                 goto check_state;
 913         }
 914
 915         if (hcint & (HCINT_RETRY |
 916             HCINT_ACK | HCINT_NYET)) {
 917                 /* give success bits priority over failure bits */
 918         } else if (hcint & HCINT_STALL) {
 919                 DPRINTF("CH=%d STALL\n", td->channel[0]);
 920                 td->error_stall = 1;
 921                 td->error_any = 1;
 922                 goto complete;
 923         } else if (hcint & HCINT_ERRORS) {
 924                 DPRINTF("CH=%d ERROR\n", td->channel[0]);
 925                 td->errcnt++;
 926                 if (td->hcsplt != 0 || td->errcnt >= 3) {
 927                         td->error_any = 1;
 928                         goto complete;
 929                 }
 930         }
 931
 932         if (hcint & (HCINT_ERRORS | HCINT_RETRY |
 933             HCINT_ACK | HCINT_NYET)) {
 934                 if (!(hcint & HCINT_ERRORS))
 935                         td->errcnt = 0;
 936         }
 937
 938 check_state:
 939         switch (td->state) {
 940         case DWC_CHAN_ST_START:
 941                 goto send_pkt;
 942
 943         case DWC_CHAN_ST_WAIT_ANE:
 944                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
 945                         td->did_nak = 1;
 946                         td->tt_scheduled = 0;
 947                         goto send_pkt;
 948                 } else if (hcint & (HCINT_ACK | HCINT_NYET)) {
 949                         td->offset += td->tx_bytes;
 950                         td->remainder -= td->tx_bytes;
 951                         td->toggle = 1;
 952                         td->tt_scheduled = 0;
 953                         goto complete;
 954                 }
 955                 break;
 956
 957         case DWC_CHAN_ST_WAIT_S_ANE:
 958                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
 959                         td->did_nak = 1;
 960                         td->tt_scheduled = 0;
 961                         goto send_pkt;
 962                 } else if (hcint & (HCINT_ACK | HCINT_NYET)) {
 963                         goto send_cpkt;
 964                 }
 965                 break;
 966
 967         case DWC_CHAN_ST_WAIT_C_ANE:
 968                 if (hcint & HCINT_NYET) {
 969                         goto send_cpkt;
 970                 } else if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
 971                         td->did_nak = 1;
 972                         td->tt_scheduled = 0;
 973                         goto send_pkt;
 974                 } else if (hcint & HCINT_ACK) {
 975                         td->offset += td->tx_bytes;
 976                         td->remainder -= td->tx_bytes;
 977                         td->toggle = 1;
 978                         goto complete;
 979                 }
 980                 break;
 981
 982         case DWC_CHAN_ST_WAIT_C_PKT:
 983                 goto send_cpkt;
 984
 985         default:
 986                 break;
 987         }
 988         goto busy;
 989
 990 send_pkt:
 991         /* free existing channel, if any */
 992         dwc_otg_host_channel_free(sc, td);
 993
 994         if (sizeof(req) != td->remainder) {
 995                 td->error_any = 1;
 996                 goto complete;
 997         }
 998
 999         if (td->hcsplt != 0) {
1000                 delta = td->tt_start_slot - sc->sc_last_frame_num - 1;
1001                 if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
1002                         td->state = DWC_CHAN_ST_START;
1003                         goto busy;
1004                 }
1005                 delta = sc->sc_last_frame_num - td->tt_start_slot;
1006                 if (delta > 5) {
1007                         /* missed it */
1008                         td->tt_scheduled = 0;
1009                         td->state = DWC_CHAN_ST_START;
1010                         goto busy;
1011                 }
1012         }
1013
1014         /* allocate a new channel */
1015         if (dwc_otg_host_channel_alloc(sc, td, 1)) {
1016                 td->state = DWC_CHAN_ST_START;
1017                 goto busy;
1018         }
1019
1020         if (td->hcsplt != 0) {
1021                 td->hcsplt &= ~HCSPLT_COMPSPLT;
1022                 td->state = DWC_CHAN_ST_WAIT_S_ANE;
1023         } else {
1024                 td->state = DWC_CHAN_ST_WAIT_ANE;
1025         }
1026
1027         /* copy out control request */
1028         usbd_copy_out(td->pc, 0, &req, sizeof(req));
1029
1030         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(td->channel[0]),
1031             (sizeof(req) << HCTSIZ_XFERSIZE_SHIFT) |
1032             (1 << HCTSIZ_PKTCNT_SHIFT) |
1033             (HCTSIZ_PID_SETUP << HCTSIZ_PID_SHIFT));
1034
1035         DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(td->channel[0]), td->hcsplt);
1036
1037         hcchar = td->hcchar;
1038         hcchar &= ~(HCCHAR_EPDIR_IN | HCCHAR_EPTYPE_MASK);
1039         hcchar |= UE_CONTROL << HCCHAR_EPTYPE_SHIFT;
1040
1041         /* must enable channel before writing data to FIFO */
1042         DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(td->channel[0]), hcchar);
1043
1044         /* transfer data into FIFO */
1045         bus_space_write_region_4(sc->sc_io_tag, sc->sc_io_hdl,
1046             DOTG_DFIFO(td->channel[0]), (uint32_t *)&req, sizeof(req) / 4);
1047
1048         /* wait until next slot before trying complete split */
1049         td->tt_complete_slot = sc->sc_last_frame_num + 1;
1050
1051         /* store number of bytes transmitted */
1052         td->tx_bytes = sizeof(req);
1053         goto busy;
1054
1055 send_cpkt:
1056         /* free existing channel, if any */
1057         dwc_otg_host_channel_free(sc, td);
1058
1059         delta = td->tt_complete_slot - sc->sc_last_frame_num - 1;
1060         if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
1061                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
1062                 goto busy;
1063         }
1064         delta = sc->sc_last_frame_num - td->tt_start_slot;
1065         if (delta > DWC_OTG_TT_SLOT_MAX) {
1066                 /* we missed the service interval */
1067                 if (td->ep_type != UE_ISOCHRONOUS)
1068                         td->error_any = 1;
1069                 goto complete;
1070         }
1071         /* allocate a new channel */
1072         if (dwc_otg_host_channel_alloc(sc, td, 0)) {
1073                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
1074                 goto busy;
1075         }
1076
1077         /* wait until next slot before trying complete split */
1078         td->tt_complete_slot = sc->sc_last_frame_num + 1;
1079
1080         td->hcsplt |= HCSPLT_COMPSPLT;
1081         td->state = DWC_CHAN_ST_WAIT_C_ANE;
1082
1083         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(td->channel[0]),
1084             (HCTSIZ_PID_SETUP << HCTSIZ_PID_SHIFT));
1085
1086         DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(td->channel[0]), td->hcsplt);
1087
1088         hcchar = td->hcchar;
1089         hcchar &= ~(HCCHAR_EPDIR_IN | HCCHAR_EPTYPE_MASK);
1090         hcchar |= UE_CONTROL << HCCHAR_EPTYPE_SHIFT;
1091
1092         /* must enable channel before writing data to FIFO */
1093         DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(td->channel[0]), hcchar);
1094
1095 busy:
1096         return (1);     /* busy */
1097
1098 complete:
1099         dwc_otg_host_channel_free(sc, td);
1100         return (0);     /* complete */
1101 }
1102
1103 static uint8_t
1104 dwc_otg_setup_rx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1105 {
1106         struct usb_device_request req __aligned(4);
1107         uint32_t temp;
1108         uint16_t count;
1109
1110         /* check endpoint status */
1111
1112         if (sc->sc_last_rx_status == 0)
1113                 goto not_complete;
1114
1115         if (GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) != 0)
1116                 goto not_complete;
1117
1118         if ((sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) !=
1119             GRXSTSRD_STP_DATA) {
1120                 if ((sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) !=
1121                     GRXSTSRD_STP_COMPLETE || td->remainder != 0) {
1122                         /* release FIFO */
1123                         dwc_otg_common_rx_ack(sc);
1124                         goto not_complete;
1125                 }
1126                 /* release FIFO */
1127                 dwc_otg_common_rx_ack(sc);
1128                 return (0);     /* complete */
1129         }
1130
1131         if ((sc->sc_last_rx_status & GRXSTSRD_DPID_MASK) !=
1132             GRXSTSRD_DPID_DATA0) {
1133                 /* release FIFO */
1134                 dwc_otg_common_rx_ack(sc);
1135                 goto not_complete;
1136         }
1137
1138         DPRINTFN(5, "GRXSTSR=0x%08x\n", sc->sc_last_rx_status);
1139
1140         /* clear did stall */
1141         td->did_stall = 0;
1142
1143         /* get the packet byte count */
1144         count = GRXSTSRD_BCNT_GET(sc->sc_last_rx_status);
1145
1146         if (count != sizeof(req)) {
1147                 DPRINTFN(0, "Unsupported SETUP packet "
1148                     "length, %d bytes\n", count);
1149                 /* release FIFO */
1150                 dwc_otg_common_rx_ack(sc);
1151                 goto not_complete;
1152         }
1153
1154         /* read FIFO */
1155         dwc_otg_read_fifo(sc, td->pc, 0, sizeof(req));
1156
1157         /* copy out control request */
1158         usbd_copy_out(td->pc, 0, &req, sizeof(req));
1159
1160         td->offset = sizeof(req);
1161         td->remainder = 0;
1162
1163         /* sneak peek the set address */
1164         if ((req.bmRequestType == UT_WRITE_DEVICE) &&
1165             (req.bRequest == UR_SET_ADDRESS)) {
1166                 /* must write address before ZLP */
1167                 dwc_otg_set_address(sc, req.wValue[0] & 0x7F);
1168         }
1169
1170         /* don't send any data by default */
1171         DWC_OTG_WRITE_4(sc, DOTG_DIEPTSIZ(0), DIEPCTL_EPDIS);
1172         DWC_OTG_WRITE_4(sc, DOTG_DOEPTSIZ(0), DOEPCTL_EPDIS);
1173
1174         /* reset IN endpoint buffer */
1175         dwc_otg_tx_fifo_reset(sc,
1176             GRSTCTL_TXFIFO(0) |
1177             GRSTCTL_TXFFLSH);
1178
1179         /* acknowledge RX status */
1180         dwc_otg_common_rx_ack(sc);
1181         td->did_stall = 1;
1182
1183 not_complete:
1184         /* abort any ongoing transfer, before enabling again */
1185         if (!td->did_stall) {
1186                 td->did_stall = 1;
1187
1188                 DPRINTFN(5, "stalling IN and OUT direction\n");
1189
1190                 temp = sc->sc_out_ctl[0];
1191
1192                 /* set stall after enabling endpoint */
1193                 DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(0),
1194                     temp | DOEPCTL_STALL);
1195
1196                 temp = sc->sc_in_ctl[0];
1197
1198                 /* set stall assuming endpoint is enabled */
1199                 DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(0),
1200                     temp | DIEPCTL_STALL);
1201         }
1202         return (1);                     /* not complete */
1203 }
1204
1205 static uint8_t
1206 dwc_otg_host_rate_check_interrupt(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1207 {
1208         uint8_t delta;
1209
1210         delta = sc->sc_tmr_val - td->tmr_val;
1211         if (delta >= 128)
1212                 return (1);     /* busy */
1213
1214         td->tmr_val = sc->sc_tmr_val + td->tmr_res;
1215
1216         /* set toggle, if any */
1217         if (td->set_toggle) {
1218                 td->set_toggle = 0;
1219                 td->toggle = 1;
1220         }
1221         return (0);
1222 }
1223
1224 static uint8_t
1225 dwc_otg_host_rate_check(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1226 {
1227         uint8_t frame_num = (uint8_t)sc->sc_last_frame_num;
1228
1229         if (td->ep_type == UE_ISOCHRONOUS) {
1230                 /* non TT isochronous traffic */
1231                 if (frame_num & (td->tmr_res - 1))
1232                         goto busy;
1233                 if ((frame_num ^ td->tmr_val) & td->tmr_res)
1234                         goto busy;
1235                 td->tmr_val = td->tmr_res + sc->sc_last_frame_num;
1236                 td->toggle = 0;
1237                 return (0);
1238         } else if (td->ep_type == UE_INTERRUPT) {
1239                 if (!td->tt_scheduled)
1240                         goto busy;
1241                 td->tt_scheduled = 0;
1242                 return (0);
1243         } else if (td->did_nak != 0) {
1244                 /* check if we should pause sending queries for 125us */
1245                 if (td->tmr_res == frame_num) {
1246                         /* wait a bit */
1247                         dwc_otg_enable_sof_irq(sc);
1248                         goto busy;
1249                 }
1250         } else if (td->set_toggle) {
1251                 td->set_toggle = 0;
1252                 td->toggle = 1;
1253         }
1254         /* query for data one more time */
1255         td->tmr_res = frame_num;
1256         td->did_nak = 0;
1257         return (0);
1258 busy:
1259         return (1);
1260 }
1261
1262 static uint8_t
1263 dwc_otg_host_data_rx_sub(struct dwc_otg_softc *sc, struct dwc_otg_td *td,
1264     uint8_t channel)
1265 {
1266         uint32_t count;
1267
1268         /* check endpoint status */
1269         if (sc->sc_last_rx_status == 0)
1270                 goto busy;
1271
1272         if (channel >= DWC_OTG_MAX_CHANNELS)
1273                 goto busy;
1274
1275         if (GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) != channel)
1276                 goto busy;
1277
1278         switch (sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) {
1279         case GRXSTSRH_IN_DATA:
1280
1281                 DPRINTF("DATA ST=%d STATUS=0x%08x\n",
1282                     (int)td->state, (int)sc->sc_last_rx_status);
1283
1284                 if (sc->sc_chan_state[channel].hcint & HCINT_SOFTWARE_ONLY) {
1285                         /*
1286                          * When using SPLIT transactions on interrupt
1287                          * endpoints, sometimes data occurs twice.
1288                          */
1289                         DPRINTF("Data already received\n");
1290                         break;
1291                 }
1292
1293                 /* get the packet byte count */
1294                 count = GRXSTSRD_BCNT_GET(sc->sc_last_rx_status);
1295
1296                 /* check for ISOCHRONOUS endpoint */
1297                 if (td->ep_type == UE_ISOCHRONOUS) {
1298                         if ((sc->sc_last_rx_status & GRXSTSRD_DPID_MASK) !=
1299                             GRXSTSRD_DPID_DATA0) {
1300                                 /* more data to be received */
1301                                 td->tt_xactpos = HCSPLT_XACTPOS_MIDDLE;
1302                         } else {
1303                                 /* all data received */
1304                                 td->tt_xactpos = HCSPLT_XACTPOS_BEGIN;
1305                                 /* verify the packet byte count */
1306                                 if (count != td->remainder) {
1307                                         /* we have a short packet */
1308                                         td->short_pkt = 1;
1309                                         td->got_short = 1;
1310                                 }
1311                         }
1312                 } else {
1313                         /* verify the packet byte count */
1314                         if (count != td->max_packet_size) {
1315                                 if (count < td->max_packet_size) {
1316                                         /* we have a short packet */
1317                                         td->short_pkt = 1;
1318                                         td->got_short = 1;
1319                                 } else {
1320                                         /* invalid USB packet */
1321                                         td->error_any = 1;
1322
1323                                         /* release FIFO */
1324                                         dwc_otg_common_rx_ack(sc);
1325                                         goto complete;
1326                                 }
1327                         }
1328                         td->toggle ^= 1;
1329                         td->tt_scheduled = 0;
1330                 }
1331
1332                 /* verify the packet byte count */
1333                 if (count > td->remainder) {
1334                         /* invalid USB packet */
1335                         td->error_any = 1;
1336
1337                         /* release FIFO */
1338                         dwc_otg_common_rx_ack(sc);
1339                         goto complete;
1340                 }
1341
1342                 /* read data from FIFO */
1343                 dwc_otg_read_fifo(sc, td->pc, td->offset, count);
1344
1345                 td->remainder -= count;
1346                 td->offset += count;
1347                 sc->sc_chan_state[channel].hcint |= HCINT_SOFTWARE_ONLY;
1348                 break;
1349         default:
1350                 break;
1351         }
1352         /* release FIFO */
1353         dwc_otg_common_rx_ack(sc);
1354 busy:
1355         return (0);
1356 complete:
1357         return (1);
1358 }
1359
1360 static uint8_t
1361 dwc_otg_host_data_rx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1362 {
1363         uint32_t hcint = 0;
1364         uint32_t hcchar;
1365         uint8_t delta;
1366         uint8_t channel;
1367         uint8_t x;
1368
1369         for (x = 0; x != td->max_packet_count; x++) {
1370                 channel = td->channel[x];
1371                 if (channel >= DWC_OTG_MAX_CHANNELS)
1372                         continue;
1373                 hcint |= sc->sc_chan_state[channel].hcint;
1374
1375                 DPRINTF("CH=%d ST=%d HCINT=0x%08x HCCHAR=0x%08x HCTSIZ=0x%08x\n",
1376                     channel, td->state, hcint,
1377                     DWC_OTG_READ_4(sc, DOTG_HCCHAR(channel)),
1378                     DWC_OTG_READ_4(sc, DOTG_HCTSIZ(channel)));
1379
1380                 /* check interrupt bits */
1381                 if (hcint & (HCINT_RETRY |
1382                     HCINT_ACK | HCINT_NYET)) {
1383                         /* give success bits priority over failure bits */
1384                 } else if (hcint & HCINT_STALL) {
1385                         DPRINTF("CH=%d STALL\n", channel);
1386                         td->error_stall = 1;
1387                         td->error_any = 1;
1388                         goto complete;
1389                 } else if (hcint & HCINT_ERRORS) {
1390                         DPRINTF("CH=%d ERROR\n", channel);
1391                         td->errcnt++;
1392                         if (td->hcsplt != 0 || td->errcnt >= 3) {
1393                                 if (td->ep_type != UE_ISOCHRONOUS) {
1394                                         td->error_any = 1;
1395                                         goto complete;
1396                                 }
1397                         }
1398                 }
1399
1400                 /* check channels for data, if any */
1401                 if (dwc_otg_host_data_rx_sub(sc, td, channel))
1402                         goto complete;
1403
1404                 /* refresh interrupt status */
1405                 hcint |= sc->sc_chan_state[channel].hcint;
1406
1407                 if (hcint & (HCINT_ERRORS | HCINT_RETRY |
1408                     HCINT_ACK | HCINT_NYET)) {
1409                         if (!(hcint & HCINT_ERRORS))
1410                                 td->errcnt = 0;
1411                 }
1412         }
1413
1414         switch (td->state) {
1415         case DWC_CHAN_ST_START:
1416                 if (td->hcsplt != 0)
1417                         goto receive_spkt;
1418                 else
1419                         goto receive_pkt;
1420
1421         case DWC_CHAN_ST_WAIT_ANE:
1422                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1423                         if (td->ep_type == UE_INTERRUPT) {
1424                                 /*
1425                                  * The USB specification does not
1426                                  * mandate a particular data toggle
1427                                  * value for USB INTERRUPT
1428                                  * transfers. Switch the data toggle
1429                                  * value to receive the packet
1430                                  * correctly:
1431                                  */
1432                                 if (hcint & HCINT_DATATGLERR) {
1433                                         DPRINTF("Retrying packet due to "
1434                                             "data toggle error\n");
1435                                         td->toggle ^= 1;
1436                                         goto receive_pkt;
1437                                 }
1438                         } else if (td->ep_type == UE_ISOCHRONOUS) {
1439                                 goto complete;
1440                         }
1441                         td->did_nak = 1;
1442                         td->tt_scheduled = 0;
1443                         if (td->hcsplt != 0)
1444                                 goto receive_spkt;
1445                         else
1446                                 goto receive_pkt;
1447                 } else if (hcint & HCINT_NYET) {
1448                         if (td->hcsplt != 0) {
1449                                 /* try again */
1450                                 goto receive_pkt;
1451                         } else {
1452                                 /* not a valid token for IN endpoints */
1453                                 td->error_any = 1;
1454                                 goto complete;
1455                         }
1456                 } else if (hcint & HCINT_ACK) {
1457                         /* wait for data - ACK arrived first */
1458                         if (!(hcint & HCINT_SOFTWARE_ONLY))
1459                                 goto busy;
1460
1461                         if (td->ep_type == UE_ISOCHRONOUS) {
1462                                 /* check if we are complete */
1463                                 if (td->tt_xactpos == HCSPLT_XACTPOS_BEGIN) {
1464                                         goto complete;
1465                                 } else {
1466                                         /* get more packets */
1467                                         goto busy;
1468                                 }
1469                         } else {
1470                                 /* check if we are complete */
1471                                 if ((td->remainder == 0) || (td->got_short != 0)) {
1472                                         if (td->short_pkt)
1473                                                 goto complete;
1474
1475                                         /*
1476                                          * Else need to receive a zero length
1477                                          * packet.
1478                                          */
1479                                 }
1480                                 td->tt_scheduled = 0;
1481                                 td->did_nak = 0;
1482                                 if (td->hcsplt != 0)
1483                                         goto receive_spkt;
1484                                 else
1485                                         goto receive_pkt;
1486                         }
1487                 }
1488                 break;
1489
1490         case DWC_CHAN_ST_WAIT_S_ANE:
1491                 /*
1492                  * NOTE: The DWC OTG hardware provides a fake ACK in
1493                  * case of interrupt and isochronous transfers:
1494                  */
1495                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1496                         td->did_nak = 1;
1497                         td->tt_scheduled = 0;
1498                         goto receive_spkt;
1499                 } else if (hcint & HCINT_NYET) {
1500                         td->tt_scheduled = 0;
1501                         goto receive_spkt;
1502                 } else if (hcint & HCINT_ACK) {
1503                         td->did_nak = 0;
1504                         goto receive_pkt;
1505                 }
1506                 break;
1507
1508         case DWC_CHAN_ST_WAIT_C_PKT:
1509                 goto receive_pkt;
1510
1511         default:
1512                 break;
1513         }
1514         goto busy;
1515
1516 receive_pkt:
1517         /* free existing channel, if any */
1518         dwc_otg_host_channel_free(sc, td);
1519
1520         if (td->hcsplt != 0) {
1521                 delta = td->tt_complete_slot - sc->sc_last_frame_num - 1;
1522                 if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
1523                         td->state = DWC_CHAN_ST_WAIT_C_PKT;
1524                         goto busy;
1525                 }
1526                 delta = sc->sc_last_frame_num - td->tt_start_slot;
1527                 if (delta > DWC_OTG_TT_SLOT_MAX) {
1528                         if (td->ep_type != UE_ISOCHRONOUS) {
1529                                 /* we missed the service interval */
1530                                 td->error_any = 1;
1531                         }
1532                         goto complete;
1533                 }
1534                 /* complete split */
1535                 td->hcsplt |= HCSPLT_COMPSPLT;
1536         } else if (dwc_otg_host_rate_check(sc, td)) {
1537                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
1538                 goto busy;
1539         }
1540
1541         /* allocate a new channel */
1542         if (dwc_otg_host_channel_alloc(sc, td, 0)) {
1543                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
1544                 goto busy;
1545         }
1546
1547         /* set toggle, if any */
1548         if (td->set_toggle) {
1549                 td->set_toggle = 0;
1550                 td->toggle = 1;
1551         }
1552
1553         td->state = DWC_CHAN_ST_WAIT_ANE;
1554
1555         for (x = 0; x != td->max_packet_count; x++) {
1556                 channel = td->channel[x];
1557
1558                 /* receive one packet */
1559                 DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1560                     (td->max_packet_size << HCTSIZ_XFERSIZE_SHIFT) |
1561                     (1 << HCTSIZ_PKTCNT_SHIFT) |
1562                     (td->toggle ? (HCTSIZ_PID_DATA1 << HCTSIZ_PID_SHIFT) :
1563                     (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT)));
1564
1565                 DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(channel), td->hcsplt);
1566
1567                 hcchar = td->hcchar;
1568                 hcchar |= HCCHAR_EPDIR_IN;
1569
1570                 /* receive complete split ASAP */
1571                 if ((sc->sc_last_frame_num & 1) != 0 &&
1572                     td->ep_type == UE_ISOCHRONOUS)
1573                         hcchar |= HCCHAR_ODDFRM;
1574                 else
1575                         hcchar &= ~HCCHAR_ODDFRM;
1576
1577                 /* must enable channel before data can be received */
1578                 DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(channel), hcchar);
1579         }
1580         /* wait until next slot before trying complete split */
1581         td->tt_complete_slot = sc->sc_last_frame_num + 1;
1582
1583         goto busy;
1584
1585 receive_spkt:
1586         /* free existing channel(s), if any */
1587         dwc_otg_host_channel_free(sc, td);
1588
1589         delta = td->tt_start_slot - sc->sc_last_frame_num - 1;
1590         if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
1591                 td->state = DWC_CHAN_ST_START;
1592                 goto busy;
1593         }
1594         delta = sc->sc_last_frame_num - td->tt_start_slot;
1595         if (delta > 5) {
1596                 /* missed it */
1597                 td->tt_scheduled = 0;
1598                 td->state = DWC_CHAN_ST_START;
1599                 goto busy;
1600         }
1601
1602         /* allocate a new channel */
1603         if (dwc_otg_host_channel_alloc(sc, td, 0)) {
1604                 td->state = DWC_CHAN_ST_START;
1605                 goto busy;
1606         }
1607
1608         channel = td->channel[0];
1609
1610         td->hcsplt &= ~HCSPLT_COMPSPLT;
1611         td->state = DWC_CHAN_ST_WAIT_S_ANE;
1612
1613         /* receive one packet */
1614         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1615             (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT));
1616
1617         DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(channel), td->hcsplt);
1618
1619         /* send after next SOF event */
1620         if ((sc->sc_last_frame_num & 1) == 0 &&
1621             td->ep_type == UE_ISOCHRONOUS)
1622                 td->hcchar |= HCCHAR_ODDFRM;
1623         else
1624                 td->hcchar &= ~HCCHAR_ODDFRM;
1625
1626         hcchar = td->hcchar;
1627         hcchar |= HCCHAR_EPDIR_IN;
1628
1629         /* wait until next slot before trying complete split */
1630         td->tt_complete_slot = sc->sc_last_frame_num + 1;
1631
1632         /* must enable channel before data can be received */
1633         DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(channel), hcchar);
1634 busy:
1635         return (1);     /* busy */
1636
1637 complete:
1638         dwc_otg_host_channel_free(sc, td);
1639         return (0);     /* complete */
1640 }
1641
1642 static uint8_t
1643 dwc_otg_data_rx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1644 {
1645         uint32_t temp;
1646         uint16_t count;
1647         uint8_t got_short;
1648
1649         got_short = 0;
1650
1651         /* check endpoint status */
1652         if (sc->sc_last_rx_status == 0)
1653                 goto not_complete;
1654
1655         if (GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) != td->ep_no)
1656                 goto not_complete;
1657
1658         /* check for SETUP packet */
1659         if ((sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) ==
1660             GRXSTSRD_STP_DATA ||
1661             (sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) ==
1662             GRXSTSRD_STP_COMPLETE) {
1663                 if (td->remainder == 0) {
1664                         /*
1665                          * We are actually complete and have
1666                          * received the next SETUP
1667                          */
1668                         DPRINTFN(5, "faking complete\n");
1669                         return (0);     /* complete */
1670                 }
1671                 /*
1672                  * USB Host Aborted the transfer.
1673                  */
1674                 td->error_any = 1;
1675                 return (0);             /* complete */
1676         }
1677
1678         if ((sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) !=
1679             GRXSTSRD_OUT_DATA) {
1680                 /* release FIFO */
1681                 dwc_otg_common_rx_ack(sc);
1682                 goto not_complete;
1683         }
1684
1685         /* get the packet byte count */
1686         count = GRXSTSRD_BCNT_GET(sc->sc_last_rx_status);
1687
1688         /* verify the packet byte count */
1689         if (count != td->max_packet_size) {
1690                 if (count < td->max_packet_size) {
1691                         /* we have a short packet */
1692                         td->short_pkt = 1;
1693                         got_short = 1;
1694                 } else {
1695                         /* invalid USB packet */
1696                         td->error_any = 1;
1697
1698                         /* release FIFO */
1699                         dwc_otg_common_rx_ack(sc);
1700                         return (0);     /* we are complete */
1701                 }
1702         }
1703         /* verify the packet byte count */
1704         if (count > td->remainder) {
1705                 /* invalid USB packet */
1706                 td->error_any = 1;
1707
1708                 /* release FIFO */
1709                 dwc_otg_common_rx_ack(sc);
1710                 return (0);             /* we are complete */
1711         }
1712
1713         /* read data from FIFO */
1714         dwc_otg_read_fifo(sc, td->pc, td->offset, count);
1715
1716         td->remainder -= count;
1717         td->offset += count;
1718
1719         /* release FIFO */
1720         dwc_otg_common_rx_ack(sc);
1721
1722         temp = sc->sc_out_ctl[td->ep_no];
1723
1724         /* check for isochronous mode */
1725         if ((temp & DIEPCTL_EPTYPE_MASK) ==
1726             (DIEPCTL_EPTYPE_ISOC << DIEPCTL_EPTYPE_SHIFT)) {
1727                 /* toggle odd or even frame bit */
1728                 if (temp & DIEPCTL_SETD1PID) {
1729                         temp &= ~DIEPCTL_SETD1PID;
1730                         temp |= DIEPCTL_SETD0PID;
1731                 } else {
1732                         temp &= ~DIEPCTL_SETD0PID;
1733                         temp |= DIEPCTL_SETD1PID;
1734                 }
1735                 sc->sc_out_ctl[td->ep_no] = temp;
1736         }
1737
1738         /* check if we are complete */
1739         if ((td->remainder == 0) || got_short) {
1740                 if (td->short_pkt) {
1741                         /* we are complete */
1742                         return (0);
1743                 }
1744                 /* else need to receive a zero length packet */
1745         }
1746
1747 not_complete:
1748
1749         /* enable SETUP and transfer complete interrupt */
1750         if (td->ep_no == 0) {
1751                 DWC_OTG_WRITE_4(sc, DOTG_DOEPTSIZ(0),
1752                     DXEPTSIZ_SET_MULTI(3) |
1753                     DXEPTSIZ_SET_NPKT(1) |
1754                     DXEPTSIZ_SET_NBYTES(td->max_packet_size));
1755         } else {
1756                 /* allow reception of multiple packets */
1757                 DWC_OTG_WRITE_4(sc, DOTG_DOEPTSIZ(td->ep_no),
1758                     DXEPTSIZ_SET_MULTI(1) |
1759                     DXEPTSIZ_SET_NPKT(4) |
1760                     DXEPTSIZ_SET_NBYTES(4 *
1761                         ((td->max_packet_size + 3) & ~3)));
1762         }
1763         temp = sc->sc_out_ctl[td->ep_no];
1764         DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(td->ep_no), temp |
1765             DOEPCTL_EPENA | DOEPCTL_CNAK);
1766
1767         return (1);                     /* not complete */
1768 }
1769
1770 static uint8_t
1771 dwc_otg_host_data_tx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1772 {
1773         uint32_t count;
1774         uint32_t hcint;
1775         uint32_t hcchar;
1776         uint8_t delta;
1777         uint8_t channel;
1778         uint8_t x;
1779
1780         dwc_otg_host_dump_rx(sc, td);
1781
1782         /* check that last channel is complete */
1783         channel = td->channel[td->npkt];
1784
1785         if (channel < DWC_OTG_MAX_CHANNELS) {
1786                 hcint = sc->sc_chan_state[channel].hcint;
1787
1788                 DPRINTF("CH=%d ST=%d HCINT=0x%08x HCCHAR=0x%08x HCTSIZ=0x%08x\n",
1789                     channel, td->state, hcint,
1790                     DWC_OTG_READ_4(sc, DOTG_HCCHAR(channel)),
1791                     DWC_OTG_READ_4(sc, DOTG_HCTSIZ(channel)));
1792
1793                 if (hcint & (HCINT_RETRY |
1794                     HCINT_ACK | HCINT_NYET)) {
1795                         /* give success bits priority over failure bits */
1796                 } else if (hcint & HCINT_STALL) {
1797                         DPRINTF("CH=%d STALL\n", channel);
1798                         td->error_stall = 1;
1799                         td->error_any = 1;
1800                         goto complete;
1801                 } else if (hcint & HCINT_ERRORS) {
1802                         DPRINTF("CH=%d ERROR\n", channel);
1803                         td->errcnt++;
1804                         if (td->hcsplt != 0 || td->errcnt >= 3) {
1805                                 td->error_any = 1;
1806                                 goto complete;
1807                         }
1808                 }
1809
1810                 if (hcint & (HCINT_ERRORS | HCINT_RETRY |
1811                     HCINT_ACK | HCINT_NYET)) {
1812
1813                         if (!(hcint & HCINT_ERRORS))
1814                                 td->errcnt = 0;
1815                 }
1816         } else {
1817                 hcint = 0;
1818         }
1819
1820         switch (td->state) {
1821         case DWC_CHAN_ST_START:
1822                 goto send_pkt;
1823
1824         case DWC_CHAN_ST_WAIT_ANE:
1825                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1826                         td->did_nak = 1;
1827                         td->tt_scheduled = 0;
1828                         goto send_pkt;
1829                 } else if (hcint & (HCINT_ACK | HCINT_NYET)) {
1830                         td->offset += td->tx_bytes;
1831                         td->remainder -= td->tx_bytes;
1832                         td->toggle ^= 1;
1833                         /* check if next response will be a NAK */
1834                         if (hcint & HCINT_NYET)
1835                                 td->did_nak = 1;
1836                         else
1837                                 td->did_nak = 0;
1838                         td->tt_scheduled = 0;
1839
1840                         /* check remainder */
1841                         if (td->remainder == 0) {
1842                                 if (td->short_pkt)
1843                                         goto complete;
1844
1845                                 /*
1846                                  * Else we need to transmit a short
1847                                  * packet:
1848                                  */
1849                         }
1850                         goto send_pkt;
1851                 }
1852                 break;
1853
1854         case DWC_CHAN_ST_WAIT_S_ANE:
1855                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1856                         td->did_nak = 1;
1857                         td->tt_scheduled = 0;
1858                         goto send_pkt;
1859                 } else if (hcint & (HCINT_ACK | HCINT_NYET)) {
1860                         td->did_nak = 0;
1861                         goto send_cpkt;
1862                 }
1863                 break;
1864
1865         case DWC_CHAN_ST_WAIT_C_ANE:
1866                 if (hcint & HCINT_NYET) {
1867                         goto send_cpkt;
1868                 } else if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1869                         td->did_nak = 1;
1870                         td->tt_scheduled = 0;
1871                         goto send_pkt;
1872                 } else if (hcint & HCINT_ACK) {
1873                         td->offset += td->tx_bytes;
1874                         td->remainder -= td->tx_bytes;
1875                         td->toggle ^= 1;
1876                         td->did_nak = 0;
1877                         td->tt_scheduled = 0;
1878
1879                         /* check remainder */
1880                         if (td->remainder == 0) {
1881                                 if (td->short_pkt)
1882                                         goto complete;
1883
1884                                 /* else we need to transmit a short packet */
1885                         }
1886                         goto send_pkt;
1887                 }
1888                 break;
1889
1890         case DWC_CHAN_ST_WAIT_C_PKT:
1891                 goto send_cpkt;
1892
1893         case DWC_CHAN_ST_TX_WAIT_ISOC:
1894                 /* Check if ISOCHRONOUS OUT traffic is complete */
1895                 if ((hcint & HCINT_HCH_DONE_MASK) == 0)
1896                         break;
1897
1898                 td->offset += td->tx_bytes;
1899                 td->remainder -= td->tx_bytes;
1900                 goto complete;
1901         default:
1902                 break;
1903         }
1904         goto busy;
1905
1906 send_pkt:
1907         /* free existing channel(s), if any */
1908         dwc_otg_host_channel_free(sc, td);
1909
1910         if (td->hcsplt != 0) {
1911                 delta = td->tt_start_slot - sc->sc_last_frame_num - 1;
1912                 if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
1913                         td->state = DWC_CHAN_ST_START;
1914                         goto busy;
1915                 }
1916                 delta = sc->sc_last_frame_num - td->tt_start_slot;
1917                 if (delta > 5) {
1918                         /* missed it */
1919                         td->tt_scheduled = 0;
1920                         td->state = DWC_CHAN_ST_START;
1921                         goto busy;
1922                 }
1923         } else if (dwc_otg_host_rate_check(sc, td)) {
1924                 td->state = DWC_CHAN_ST_START;
1925                 goto busy;
1926         }
1927
1928         /* allocate a new channel */
1929         if (dwc_otg_host_channel_alloc(sc, td, 1)) {
1930                 td->state = DWC_CHAN_ST_START;
1931                 goto busy;
1932         }
1933
1934         /* set toggle, if any */
1935         if (td->set_toggle) {
1936                 td->set_toggle = 0;
1937                 td->toggle = 1;
1938         }
1939
1940         if (td->ep_type == UE_ISOCHRONOUS) {
1941                 /* ISOCHRONOUS OUT transfers don't have any ACKs */
1942                 td->state = DWC_CHAN_ST_TX_WAIT_ISOC;
1943                 td->hcsplt &= ~HCSPLT_COMPSPLT;
1944                 if (td->hcsplt != 0) {
1945                         /* get maximum transfer length */
1946                         count = td->remainder;
1947                         if (count > HCSPLT_XACTLEN_BURST) {
1948                                 DPRINTF("TT overflow\n");
1949                                 td->error_any = 1;
1950                                 goto complete;
1951                         }
1952                         /* Update transaction position */
1953                         td->hcsplt &= ~HCSPLT_XACTPOS_MASK;
1954                         td->hcsplt |= (HCSPLT_XACTPOS_ALL << HCSPLT_XACTPOS_SHIFT);
1955                 }
1956         } else if (td->hcsplt != 0) {
1957                 td->hcsplt &= ~HCSPLT_COMPSPLT;
1958                 /* Wait for ACK/NAK/ERR from TT */
1959                 td->state = DWC_CHAN_ST_WAIT_S_ANE;
1960         } else {
1961                 /* Wait for ACK/NAK/STALL from device */
1962                 td->state = DWC_CHAN_ST_WAIT_ANE;
1963         }
1964
1965         td->tx_bytes = 0;
1966
1967         for (x = 0; x != td->max_packet_count; x++) {
1968                 uint32_t rem_bytes;
1969
1970                 channel = td->channel[x];
1971
1972                 /* send one packet at a time */
1973                 count = td->max_packet_size;
1974                 rem_bytes = td->remainder - td->tx_bytes;
1975                 if (rem_bytes < count) {
1976                         /* we have a short packet */
1977                         td->short_pkt = 1;
1978                         count = rem_bytes;
1979                 }
1980                 if (count == rem_bytes) {
1981                         /* last packet */
1982                         switch (x) {
1983                         case 0:
1984                                 DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1985                                     (count << HCTSIZ_XFERSIZE_SHIFT) |
1986                                     (1 << HCTSIZ_PKTCNT_SHIFT) |
1987                                     (td->toggle ? (HCTSIZ_PID_DATA1 << HCTSIZ_PID_SHIFT) :
1988                                     (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT)));
1989                                 break;
1990                         case 1:
1991                                 DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1992                                     (count << HCTSIZ_XFERSIZE_SHIFT) |
1993                                     (1 << HCTSIZ_PKTCNT_SHIFT) |
1994                                     (HCTSIZ_PID_DATA1 << HCTSIZ_PID_SHIFT));
1995                                 break;
1996                         default:
1997                                 DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1998                                     (count << HCTSIZ_XFERSIZE_SHIFT) |
1999                                     (1 << HCTSIZ_PKTCNT_SHIFT) |
2000                                     (HCTSIZ_PID_DATA2 << HCTSIZ_PID_SHIFT));
2001                                 break;
2002                         }
2003                 } else if (td->ep_type == UE_ISOCHRONOUS &&
2004                            td->max_packet_count > 1) {
2005                         /* ISOCHRONOUS multi packet */
2006                         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
2007                             (count << HCTSIZ_XFERSIZE_SHIFT) |
2008                             (1 << HCTSIZ_PKTCNT_SHIFT) |
2009                             (HCTSIZ_PID_MDATA << HCTSIZ_PID_SHIFT));
2010                 } else {
2011                         /* TODO: HCTSIZ_DOPNG */
2012                         /* standard BULK/INTERRUPT/CONTROL packet */
2013                         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
2014                             (count << HCTSIZ_XFERSIZE_SHIFT) |
2015                             (1 << HCTSIZ_PKTCNT_SHIFT) |
2016                             (td->toggle ? (HCTSIZ_PID_DATA1 << HCTSIZ_PID_SHIFT) :
2017                             (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT)));
2018                 }
2019
2020                 DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(channel), td->hcsplt);
2021
2022                 hcchar = td->hcchar;
2023                 hcchar &= ~HCCHAR_EPDIR_IN;
2024
2025                 /* send after next SOF event */
2026                 if ((sc->sc_last_frame_num & 1) == 0 &&
2027                     td->ep_type == UE_ISOCHRONOUS)
2028                         hcchar |= HCCHAR_ODDFRM;
2029                 else
2030                         hcchar &= ~HCCHAR_ODDFRM;
2031
2032                 /* must enable before writing data to FIFO */
2033                 DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(channel), hcchar);
2034
2035                 if (count != 0) {
2036                         /* write data into FIFO */
2037                         dwc_otg_write_fifo(sc, td->pc, td->offset +
2038                             td->tx_bytes, DOTG_DFIFO(channel), count);
2039                 }
2040
2041                 /* store number of bytes transmitted */
2042                 td->tx_bytes += count;
2043
2044                 /* store last packet index */
2045                 td->npkt = x;
2046
2047                 /* check for last packet */
2048                 if (count == rem_bytes)
2049                         break;
2050         }
2051         goto busy;
2052
2053 send_cpkt:
2054         /* free existing channel, if any */
2055         dwc_otg_host_channel_free(sc, td);
2056
2057         delta = td->tt_complete_slot - sc->sc_last_frame_num - 1;
2058         if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
2059                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
2060                 goto busy;
2061         }
2062         delta = sc->sc_last_frame_num - td->tt_start_slot;
2063         if (delta > DWC_OTG_TT_SLOT_MAX) {
2064                 /* we missed the service interval */
2065                 if (td->ep_type != UE_ISOCHRONOUS)
2066                         td->error_any = 1;
2067                 goto complete;
2068         }
2069
2070         /* allocate a new channel */
2071         if (dwc_otg_host_channel_alloc(sc, td, 0)) {
2072                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
2073                 goto busy;
2074         }
2075
2076         channel = td->channel[0];
2077
2078         td->hcsplt |= HCSPLT_COMPSPLT;
2079         td->state = DWC_CHAN_ST_WAIT_C_ANE;
2080
2081         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
2082             (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT));
2083
2084         DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(channel), td->hcsplt);
2085
2086         hcchar = td->hcchar;
2087         hcchar &= ~HCCHAR_EPDIR_IN;
2088
2089         /* receive complete split ASAP */
2090         if ((sc->sc_last_frame_num & 1) != 0 &&
2091             td->ep_type == UE_ISOCHRONOUS)
2092                 hcchar |= HCCHAR_ODDFRM;
2093         else
2094                 hcchar &= ~HCCHAR_ODDFRM;
2095
2096         /* must enable channel before data can be received */
2097         DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(channel), hcchar);
2098
2099         /* wait until next slot before trying complete split */
2100         td->tt_complete_slot = sc->sc_last_frame_num + 1;
2101 busy:
2102         return (1);     /* busy */
2103
2104 complete:
2105         dwc_otg_host_channel_free(sc, td);
2106         return (0);     /* complete */
2107 }
2108
2109 static uint8_t
2110 dwc_otg_data_tx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
2111 {
2112         uint32_t max_buffer;
2113         uint32_t count;
2114         uint32_t fifo_left;
2115         uint32_t mpkt;
2116         uint32_t temp;
2117         uint8_t to;
2118
2119         to = 3;                         /* don't loop forever! */
2120
2121         max_buffer = sc->sc_hw_ep_profile[td->ep_no].max_buffer;
2122
2123 repeat:
2124         /* check for for endpoint 0 data */
2125
2126         temp = sc->sc_last_rx_status;
2127
2128         if ((td->ep_no == 0) && (temp != 0) &&
2129             (GRXSTSRD_CHNUM_GET(temp) == 0)) {
2130
2131                 if ((temp & GRXSTSRD_PKTSTS_MASK) !=
2132                     GRXSTSRD_STP_DATA &&
2133                     (temp & GRXSTSRD_PKTSTS_MASK) !=
2134                     GRXSTSRD_STP_COMPLETE) {
2135
2136                         /* dump data - wrong direction */
2137                         dwc_otg_common_rx_ack(sc);
2138                 } else {
2139                         /*
2140                          * The current transfer was cancelled
2141                          * by the USB Host:
2142                          */
2143                         td->error_any = 1;
2144                         return (0);             /* complete */
2145                 }
2146         }
2147
2148         /* fill in more TX data, if possible */
2149         if (td->tx_bytes != 0) {
2150
2151                 uint16_t cpkt;
2152
2153                 /* check if packets have been transferred */
2154                 temp = DWC_OTG_READ_4(sc, DOTG_DIEPTSIZ(td->ep_no));
2155
2156                 /* get current packet number */
2157                 cpkt = DXEPTSIZ_GET_NPKT(temp);
2158
2159                 if (cpkt >= td->npkt) {
2160                         fifo_left = 0;
2161                 } else {
2162                         if (max_buffer != 0) {
2163                                 fifo_left = (td->npkt - cpkt) *
2164                                     td->max_packet_size;
2165
2166                                 if (fifo_left > max_buffer)
2167                                         fifo_left = max_buffer;
2168                         } else {
2169                                 fifo_left = td->max_packet_size;
2170                         }
2171                 }
2172
2173                 count = td->tx_bytes;
2174                 if (count > fifo_left)
2175                         count = fifo_left;
2176
2177                 if (count != 0) {
2178                         /* write data into FIFO */
2179                         dwc_otg_write_fifo(sc, td->pc, td->offset,
2180                             DOTG_DFIFO(td->ep_no), count);
2181
2182                         td->tx_bytes -= count;
2183                         td->remainder -= count;
2184                         td->offset += count;
2185                         td->npkt = cpkt;
2186                 }
2187                 if (td->tx_bytes != 0)
2188                         goto not_complete;
2189
2190                 /* check remainder */
2191                 if (td->remainder == 0) {
2192                         if (td->short_pkt)
2193                                 return (0);     /* complete */
2194
2195                         /* else we need to transmit a short packet */
2196                 }
2197         }
2198
2199         if (!to--)
2200                 goto not_complete;
2201
2202         /* check if not all packets have been transferred */
2203         temp = DWC_OTG_READ_4(sc, DOTG_DIEPTSIZ(td->ep_no));
2204
2205         if (DXEPTSIZ_GET_NPKT(temp) != 0) {
2206
2207                 DPRINTFN(5, "busy ep=%d npkt=%d DIEPTSIZ=0x%08x "
2208                     "DIEPCTL=0x%08x\n", td->ep_no,
2209                     DXEPTSIZ_GET_NPKT(temp),
2210                     temp, DWC_OTG_READ_4(sc, DOTG_DIEPCTL(td->ep_no)));
2211
2212                 goto not_complete;
2213         }
2214
2215         DPRINTFN(5, "rem=%u ep=%d\n", td->remainder, td->ep_no);
2216
2217         /* try to optimise by sending more data */
2218         if ((max_buffer != 0) && ((td->max_packet_size & 3) == 0)) {
2219
2220                 /* send multiple packets at the same time */
2221                 mpkt = max_buffer / td->max_packet_size;
2222
2223                 if (mpkt > 0x3FE)
2224                         mpkt = 0x3FE;
2225
2226                 count = td->remainder;
2227                 if (count > 0x7FFFFF)
2228                         count = 0x7FFFFF - (0x7FFFFF % td->max_packet_size);
2229
2230                 td->npkt = count / td->max_packet_size;
2231
2232                 /*
2233                  * NOTE: We could use 0x3FE instead of "mpkt" in the
2234                  * check below to get more throughput, but then we
2235                  * have a dependency towards non-generic chip features
2236                  * to disable the TX-FIFO-EMPTY interrupts on a per
2237                  * endpoint basis. Increase the maximum buffer size of
2238                  * the IN endpoint to increase the performance.
2239                  */
2240                 if (td->npkt > mpkt) {
2241                         td->npkt = mpkt;
2242                         count = td->max_packet_size * mpkt;
2243                 } else if ((count == 0) || (count % td->max_packet_size)) {
2244                         /* we are transmitting a short packet */
2245                         td->npkt++;
2246                         td->short_pkt = 1;
2247                 }
2248         } else {
2249                 /* send one packet at a time */
2250                 mpkt = 1;
2251                 count = td->max_packet_size;
2252                 if (td->remainder < count) {
2253                         /* we have a short packet */
2254                         td->short_pkt = 1;
2255                         count = td->remainder;
2256                 }
2257                 td->npkt = 1;
2258         }
2259         DWC_OTG_WRITE_4(sc, DOTG_DIEPTSIZ(td->ep_no),
2260             DXEPTSIZ_SET_MULTI(1) |
2261             DXEPTSIZ_SET_NPKT(td->npkt) |
2262             DXEPTSIZ_SET_NBYTES(count));
2263
2264         /* make room for buffering */
2265         td->npkt += mpkt;
2266
2267         temp = sc->sc_in_ctl[td->ep_no];
2268
2269         /* check for isochronous mode */
2270         if ((temp & DIEPCTL_EPTYPE_MASK) ==
2271             (DIEPCTL_EPTYPE_ISOC << DIEPCTL_EPTYPE_SHIFT)) {
2272                 /* toggle odd or even frame bit */
2273                 if (temp & DIEPCTL_SETD1PID) {
2274                         temp &= ~DIEPCTL_SETD1PID;
2275                         temp |= DIEPCTL_SETD0PID;
2276                 } else {
2277                         temp &= ~DIEPCTL_SETD0PID;
2278                         temp |= DIEPCTL_SETD1PID;
2279                 }
2280                 sc->sc_in_ctl[td->ep_no] = temp;
2281         }
2282
2283         /* must enable before writing data to FIFO */
2284         DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(td->ep_no), temp |
2285             DIEPCTL_EPENA | DIEPCTL_CNAK);
2286
2287         td->tx_bytes = count;
2288
2289         /* check remainder */
2290         if (td->tx_bytes == 0 &&
2291             td->remainder == 0) {
2292                 if (td->short_pkt)
2293                         return (0);     /* complete */
2294
2295                 /* else we need to transmit a short packet */
2296         }
2297         goto repeat;
2298
2299 not_complete:
2300         return (1);                     /* not complete */
2301 }
2302
2303 static uint8_t
2304 dwc_otg_data_tx_sync(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
2305 {
2306         uint32_t temp;
2307
2308         /*
2309          * If all packets are transferred we are complete:
2310          */
2311         temp = DWC_OTG_READ_4(sc, DOTG_DIEPTSIZ(td->ep_no));
2312
2313         /* check that all packets have been transferred */
2314         if (DXEPTSIZ_GET_NPKT(temp) != 0) {
2315                 DPRINTFN(5, "busy ep=%d\n", td->ep_no);
2316                 goto not_complete;
2317         }
2318         return (0);
2319
2320 not_complete:
2321
2322         /* we only want to know if there is a SETUP packet or free IN packet */
2323
2324         temp = sc->sc_last_rx_status;
2325
2326         if ((td->ep_no == 0) && (temp != 0) &&
2327             (GRXSTSRD_CHNUM_GET(temp) == 0)) {
2328
2329                 if ((temp & GRXSTSRD_PKTSTS_MASK) ==
2330                     GRXSTSRD_STP_DATA ||
2331                     (temp & GRXSTSRD_PKTSTS_MASK) ==
2332                     GRXSTSRD_STP_COMPLETE) {
2333                         DPRINTFN(5, "faking complete\n");
2334                         /*
2335                          * Race condition: We are complete!
2336                          */
2337                         return (0);
2338                 } else {
2339                         /* dump data - wrong direction */
2340                         dwc_otg_common_rx_ack(sc);
2341                 }
2342         }
2343         return (1);                     /* not complete */
2344 }
2345
2346 static void
2347 dwc_otg_xfer_do_fifo(struct dwc_otg_softc *sc, struct usb_xfer *xfer)
2348 {
2349         struct dwc_otg_td *td;
2350         uint8_t toggle;
2351         uint8_t tmr_val;
2352         uint8_t tmr_res;
2353
2354         DPRINTFN(9, "\n");
2355
2356         td = xfer->td_transfer_cache;
2357         if (td == NULL)
2358                 return;
2359
2360         while (1) {
2361                 if ((td->func) (sc, td)) {
2362                         /* operation in progress */
2363                         break;
2364                 }
2365                 if (((void *)td) == xfer->td_transfer_last) {
2366                         goto done;
2367                 }
2368                 if (td->error_any) {
2369                         goto done;
2370                 } else if (td->remainder > 0) {
2371                         /*
2372                          * We had a short transfer. If there is no alternate
2373                          * next, stop processing !
2374                          */
2375                         if (!td->alt_next)
2376                                 goto done;
2377                 }
2378
2379                 /*
2380                  * Fetch the next transfer descriptor and transfer
2381                  * some flags to the next transfer descriptor
2382                  */
2383                 tmr_res = td->tmr_res;
2384                 tmr_val = td->tmr_val;
2385                 toggle = td->toggle;
2386                 td = td->obj_next;
2387                 xfer->td_transfer_cache = td;
2388                 td->toggle = toggle;    /* transfer toggle */
2389                 td->tmr_res = tmr_res;
2390                 td->tmr_val = tmr_val;
2391         }
2392         return;
2393
2394 done:
2395         xfer->td_transfer_cache = NULL;
2396         sc->sc_xfer_complete = 1;
2397 }
2398
2399 static uint8_t
2400 dwc_otg_xfer_do_complete_locked(struct dwc_otg_softc *sc, struct usb_xfer *xfer)
2401 {
2402         struct dwc_otg_td *td;
2403
2404         DPRINTFN(9, "\n");
2405
2406         td = xfer->td_transfer_cache;
2407         if (td == NULL) {
2408                 /* compute all actual lengths */
2409                 dwc_otg_standard_done(xfer);
2410                 return (1);
2411         }
2412         return (0);
2413 }
2414
2415 static void
2416 dwc_otg_timer(void *_sc)
2417 {
2418         struct dwc_otg_softc *sc = _sc;
2419
2420         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
2421
2422         DPRINTF("\n");
2423
2424         USB_BUS_SPIN_LOCK(&sc->sc_bus);
2425
2426         /* increment timer value */
2427         sc->sc_tmr_val++;
2428
2429         /* enable SOF interrupt, which will poll jobs */
2430         dwc_otg_enable_sof_irq(sc);
2431
2432         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
2433
2434         if (sc->sc_timer_active) {
2435                 /* restart timer */
2436                 usb_callout_reset(&sc->sc_timer,
2437                     hz / (1000 / DWC_OTG_HOST_TIMER_RATE),
2438                     &dwc_otg_timer, sc);
2439         }
2440 }
2441
2442 static void
2443 dwc_otg_timer_start(struct dwc_otg_softc *sc)
2444 {
2445         if (sc->sc_timer_active != 0)
2446                 return;
2447
2448         sc->sc_timer_active = 1;
2449
2450         /* restart timer */
2451         usb_callout_reset(&sc->sc_timer,
2452             hz / (1000 / DWC_OTG_HOST_TIMER_RATE),
2453             &dwc_otg_timer, sc);
2454 }
2455
2456 static void
2457 dwc_otg_timer_stop(struct dwc_otg_softc *sc)
2458 {
2459         if (sc->sc_timer_active == 0)
2460                 return;
2461
2462         sc->sc_timer_active = 0;
2463
2464         /* stop timer */
2465         usb_callout_stop(&sc->sc_timer);
2466 }
2467
2468 static uint16_t
2469 dwc_otg_compute_isoc_rx_tt_slot(struct dwc_otg_tt_info *pinfo)
2470 {
2471         if (pinfo->slot_index < DWC_OTG_TT_SLOT_MAX)
2472                 pinfo->slot_index++;
2473         return (pinfo->slot_index);
2474 }
2475
2476 static uint8_t
2477 dwc_otg_update_host_transfer_schedule_locked(struct dwc_otg_softc *sc)
2478 {
2479         TAILQ_HEAD(, usb_xfer) head;
2480         struct usb_xfer *xfer;
2481         struct usb_xfer *xfer_next;
2482         struct dwc_otg_td *td;
2483         uint16_t temp;
2484         uint16_t slot;
2485
2486         temp = DWC_OTG_READ_4(sc, DOTG_HFNUM) & DWC_OTG_FRAME_MASK;
2487
2488         if (sc->sc_last_frame_num == temp)
2489                 return (0);
2490
2491         sc->sc_last_frame_num = temp;
2492
2493         TAILQ_INIT(&head);
2494
2495         if ((temp & 7) == 0) {
2496
2497                 /* reset the schedule */
2498                 memset(sc->sc_tt_info, 0, sizeof(sc->sc_tt_info));
2499
2500                 TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2501                         td = xfer->td_transfer_cache;
2502                         if (td == NULL || td->ep_type != UE_ISOCHRONOUS)
2503                                 continue;
2504
2505                         /* check for IN direction */
2506                         if ((td->hcchar & HCCHAR_EPDIR_IN) != 0)
2507                                 continue;
2508
2509                         sc->sc_needsof = 1;
2510
2511                         if (td->hcsplt == 0 || td->tt_scheduled != 0)
2512                                 continue;
2513
2514                         /* compute slot */
2515                         slot = dwc_otg_compute_isoc_rx_tt_slot(
2516                             sc->sc_tt_info + td->tt_index);
2517                         if (slot > 3) {
2518                                 /*
2519                                  * Not enough time to get complete
2520                                  * split executed.
2521                                  */
2522                                 continue;
2523                         }
2524                         /* Delayed start */
2525                         td->tt_start_slot = temp + slot;
2526                         td->tt_scheduled = 1;
2527                         TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2528                         TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2529                 }
2530
2531                 TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2532                         td = xfer->td_transfer_cache;
2533                         if (td == NULL || td->ep_type != UE_ISOCHRONOUS)
2534                                 continue;
2535
2536                         /* check for OUT direction */
2537                         if ((td->hcchar & HCCHAR_EPDIR_IN) == 0)
2538                                 continue;
2539
2540                         sc->sc_needsof = 1;
2541
2542                         if (td->hcsplt == 0 || td->tt_scheduled != 0)
2543                                 continue;
2544
2545                         /* Start ASAP */
2546                         td->tt_start_slot = temp;
2547                         td->tt_scheduled = 1;
2548                         TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2549                         TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2550                 }
2551
2552                 TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2553                         td = xfer->td_transfer_cache;
2554                         if (td == NULL || td->ep_type != UE_INTERRUPT)
2555                                 continue;
2556
2557                         if (td->tt_scheduled != 0) {
2558                                 sc->sc_needsof = 1;
2559                                 continue;
2560                         }
2561
2562                         if (dwc_otg_host_rate_check_interrupt(sc, td))
2563                                 continue;
2564
2565                         if (td->hcsplt == 0) {
2566                                 sc->sc_needsof = 1;
2567                                 td->tt_scheduled = 1;
2568                                 continue;
2569                         }
2570
2571                         /* start ASAP */
2572                         td->tt_start_slot = temp;
2573                         sc->sc_needsof = 1;
2574                         td->tt_scheduled = 1;
2575                         TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2576                         TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2577                 }
2578
2579                 TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2580                         td = xfer->td_transfer_cache;
2581                         if (td == NULL ||
2582                             td->ep_type != UE_CONTROL) {
2583                                 continue;
2584                         }
2585
2586                         sc->sc_needsof = 1;
2587
2588                         if (td->hcsplt == 0 || td->tt_scheduled != 0)
2589                                 continue;
2590
2591                         /* start ASAP */
2592                         td->tt_start_slot = temp;
2593                         td->tt_scheduled = 1;
2594                         TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2595                         TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2596                 }
2597         }
2598         if ((temp & 7) < 6) {
2599                 TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2600                         td = xfer->td_transfer_cache;
2601                         if (td == NULL ||
2602                             td->ep_type != UE_BULK) {
2603                                 continue;
2604                         }
2605
2606                         sc->sc_needsof = 1;
2607
2608                         if (td->hcsplt == 0 || td->tt_scheduled != 0)
2609                                 continue;
2610
2611                         /* start ASAP */
2612                         td->tt_start_slot = temp;
2613                         td->tt_scheduled = 1;
2614                         TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2615                         TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2616                 }
2617         }
2618
2619         /* Put TT transfers in execution order at the end */
2620         TAILQ_CONCAT(&sc->sc_bus.intr_q.head, &head, wait_entry);
2621
2622         /* move all TT transfers in front, keeping the current order */
2623         TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2624                 td = xfer->td_transfer_cache;
2625                 if (td == NULL || td->hcsplt == 0)
2626                         continue;
2627                 TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2628                 TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2629         }
2630         TAILQ_CONCAT(&head, &sc->sc_bus.intr_q.head, wait_entry);
2631         TAILQ_CONCAT(&sc->sc_bus.intr_q.head, &head, wait_entry);
2632
2633         /* put non-TT non-ISOCHRONOUS transfers last */
2634         TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2635                 td = xfer->td_transfer_cache;
2636                 if (td == NULL || td->hcsplt != 0 || td->ep_type == UE_ISOCHRONOUS)
2637                         continue;
2638                 TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2639                 TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2640         }
2641         TAILQ_CONCAT(&sc->sc_bus.intr_q.head, &head, wait_entry);
2642
2643         if ((temp & 7) == 0) {
2644
2645                 DPRINTFN(12, "SOF interrupt #%d, needsof=%d\n",
2646                     (int)temp, (int)sc->sc_needsof);
2647
2648                 /* update SOF IRQ mask */
2649                 if (sc->sc_irq_mask & GINTMSK_SOFMSK) {
2650                         if (sc->sc_needsof == 0) {
2651                                 sc->sc_irq_mask &= ~GINTMSK_SOFMSK;
2652                                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2653                         }
2654                 } else {
2655                         if (sc->sc_needsof != 0) {
2656                                 sc->sc_irq_mask |= GINTMSK_SOFMSK;
2657                                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2658                         }
2659                 }
2660
2661                 /* clear need SOF flag */
2662                 sc->sc_needsof = 0;
2663         }
2664         return (1);
2665 }
2666
2667 static void
2668 dwc_otg_interrupt_poll_locked(struct dwc_otg_softc *sc)
2669 {
2670         struct usb_xfer *xfer;
2671         uint32_t count;
2672         uint32_t temp;
2673         uint32_t haint;
2674         uint8_t got_rx_status;
2675         uint8_t x;
2676
2677         if (sc->sc_flags.status_device_mode == 0) {
2678                 /*
2679                  * Update host transfer schedule, so that new
2680                  * transfers can be issued:
2681                  */
2682                 dwc_otg_update_host_transfer_schedule_locked(sc);
2683         }
2684         count = 0;
2685 repeat:
2686         if (++count == 16) {
2687                 /* give other interrupts a chance */
2688                 DPRINTF("Yield\n");
2689                 return;
2690         }
2691
2692         /* get all host channel interrupts */
2693         haint = DWC_OTG_READ_4(sc, DOTG_HAINT);
2694         while (1) {
2695                 x = ffs(haint) - 1;
2696                 if (x >= sc->sc_host_ch_max)
2697                         break;
2698                 temp = DWC_OTG_READ_4(sc, DOTG_HCINT(x));
2699                 DWC_OTG_WRITE_4(sc, DOTG_HCINT(x), temp);
2700                 temp &= ~HCINT_SOFTWARE_ONLY;
2701                 sc->sc_chan_state[x].hcint |= temp;
2702                 haint &= ~(1U << x);
2703         }
2704
2705         if (sc->sc_last_rx_status == 0) {
2706
2707                 temp = DWC_OTG_READ_4(sc, DOTG_GINTSTS);
2708                 if (temp & GINTSTS_RXFLVL) {
2709                         /* pop current status */
2710                         sc->sc_last_rx_status =
2711                             DWC_OTG_READ_4(sc, DOTG_GRXSTSPD);
2712                 }
2713
2714                 if (sc->sc_last_rx_status != 0) {
2715
2716                         uint8_t ep_no;
2717
2718                         temp = sc->sc_last_rx_status &
2719                             GRXSTSRD_PKTSTS_MASK;
2720
2721                         /* non-data messages we simply skip */
2722                         if (temp != GRXSTSRD_STP_DATA &&
2723                             temp != GRXSTSRD_STP_COMPLETE &&
2724                             temp != GRXSTSRD_OUT_DATA) {
2725                                 /* check for halted channel */
2726                                 if (temp == GRXSTSRH_HALTED) {
2727                                         ep_no = GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status);
2728                                         sc->sc_chan_state[ep_no].wait_halted = 0;
2729                                         DPRINTFN(5, "channel halt complete ch=%u\n", ep_no);
2730                                 }
2731                                 /* store bytes and FIFO offset */
2732                                 sc->sc_current_rx_bytes = 0;
2733                                 sc->sc_current_rx_fifo = 0;
2734
2735                                 /* acknowledge status */
2736                                 dwc_otg_common_rx_ack(sc);
2737                                 goto repeat;
2738                         }
2739
2740                         temp = GRXSTSRD_BCNT_GET(
2741                             sc->sc_last_rx_status);
2742                         ep_no = GRXSTSRD_CHNUM_GET(
2743                             sc->sc_last_rx_status);
2744
2745                         /* store bytes and FIFO offset */
2746                         sc->sc_current_rx_bytes = (temp + 3) & ~3;
2747                         sc->sc_current_rx_fifo = DOTG_DFIFO(ep_no);
2748
2749                         DPRINTF("Reading %d bytes from ep %d\n", temp, ep_no);
2750
2751                         /* check if we should dump the data */
2752                         if (!(sc->sc_active_rx_ep & (1U << ep_no))) {
2753                                 dwc_otg_common_rx_ack(sc);
2754                                 goto repeat;
2755                         }
2756
2757                         got_rx_status = 1;
2758
2759                         DPRINTFN(5, "RX status = 0x%08x: ch=%d pid=%d bytes=%d sts=%d\n",
2760                             sc->sc_last_rx_status, ep_no,
2761                             (sc->sc_last_rx_status >> 15) & 3,
2762                             GRXSTSRD_BCNT_GET(sc->sc_last_rx_status),
2763                             (sc->sc_last_rx_status >> 17) & 15);
2764                 } else {
2765                         got_rx_status = 0;
2766                 }
2767         } else {
2768                 uint8_t ep_no;
2769
2770                 ep_no = GRXSTSRD_CHNUM_GET(
2771                     sc->sc_last_rx_status);
2772
2773                 /* check if we should dump the data */
2774                 if (!(sc->sc_active_rx_ep & (1U << ep_no))) {
2775                         dwc_otg_common_rx_ack(sc);
2776                         goto repeat;
2777                 }
2778
2779                 got_rx_status = 1;
2780         }
2781
2782         /* execute FIFOs */
2783         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry)
2784                 dwc_otg_xfer_do_fifo(sc, xfer);
2785
2786         if (got_rx_status) {
2787                 /* check if data was consumed */
2788                 if (sc->sc_last_rx_status == 0)
2789                         goto repeat;
2790
2791                 /* disable RX FIFO level interrupt */
2792                 sc->sc_irq_mask &= ~GINTMSK_RXFLVLMSK;
2793                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2794         }
2795 }
2796
2797 static void
2798 dwc_otg_interrupt_complete_locked(struct dwc_otg_softc *sc)
2799 {
2800         struct usb_xfer *xfer;
2801 repeat:
2802         /* scan for completion events */
2803         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
2804                 if (dwc_otg_xfer_do_complete_locked(sc, xfer))
2805                         goto repeat;
2806         }
2807 }
2808
2809 static void
2810 dwc_otg_vbus_interrupt(struct dwc_otg_softc *sc, uint8_t is_on)
2811 {
2812         DPRINTFN(5, "vbus = %u\n", is_on);
2813
2814         if (is_on) {
2815                 if (!sc->sc_flags.status_vbus) {
2816                         sc->sc_flags.status_vbus = 1;
2817
2818                         /* complete root HUB interrupt endpoint */
2819
2820                         dwc_otg_root_intr(sc);
2821                 }
2822         } else {
2823                 if (sc->sc_flags.status_vbus) {
2824                         sc->sc_flags.status_vbus = 0;
2825                         sc->sc_flags.status_bus_reset = 0;
2826                         sc->sc_flags.status_suspend = 0;
2827                         sc->sc_flags.change_suspend = 0;
2828                         sc->sc_flags.change_connect = 1;
2829
2830                         /* complete root HUB interrupt endpoint */
2831
2832                         dwc_otg_root_intr(sc);
2833                 }
2834         }
2835 }
2836
2837 int
2838 dwc_otg_filter_interrupt(void *arg)
2839 {
2840         struct dwc_otg_softc *sc = arg;
2841         int retval = FILTER_HANDLED;
2842         uint32_t status;
2843
2844         USB_BUS_SPIN_LOCK(&sc->sc_bus);
2845
2846         /* read and clear interrupt status */
2847         status = DWC_OTG_READ_4(sc, DOTG_GINTSTS);
2848
2849         /* clear interrupts we are handling here */
2850         DWC_OTG_WRITE_4(sc, DOTG_GINTSTS, status & ~DWC_OTG_MSK_GINT_THREAD_IRQ);
2851
2852         /* check for USB state change interrupts */
2853         if ((status & DWC_OTG_MSK_GINT_THREAD_IRQ) != 0)
2854                 retval = FILTER_SCHEDULE_THREAD;
2855
2856         /* clear FIFO empty interrupts */
2857         if (status & sc->sc_irq_mask &
2858             (GINTSTS_PTXFEMP | GINTSTS_NPTXFEMP)) {
2859                 sc->sc_irq_mask &= ~(GINTSTS_PTXFEMP | GINTSTS_NPTXFEMP);
2860                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2861         }
2862         /* clear all IN endpoint interrupts */
2863         if (status & GINTSTS_IEPINT) {
2864                 uint32_t temp;
2865                 uint8_t x;
2866
2867                 for (x = 0; x != sc->sc_dev_in_ep_max; x++) {
2868                         temp = DWC_OTG_READ_4(sc, DOTG_DIEPINT(x));
2869                         if (temp & DIEPMSK_XFERCOMPLMSK) {
2870                                 DWC_OTG_WRITE_4(sc, DOTG_DIEPINT(x),
2871                                     DIEPMSK_XFERCOMPLMSK);
2872                         }
2873                 }
2874         }
2875
2876         /* poll FIFOs, if any */
2877         dwc_otg_interrupt_poll_locked(sc);
2878
2879         if (sc->sc_xfer_complete != 0)
2880                 retval = FILTER_SCHEDULE_THREAD;
2881
2882         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
2883
2884         return (retval);
2885 }
2886
2887 void
2888 dwc_otg_interrupt(void *arg)
2889 {
2890         struct dwc_otg_softc *sc = arg;
2891         uint32_t status;
2892
2893         USB_BUS_LOCK(&sc->sc_bus);
2894         USB_BUS_SPIN_LOCK(&sc->sc_bus);
2895
2896         /* read and clear interrupt status */
2897         status = DWC_OTG_READ_4(sc, DOTG_GINTSTS);
2898
2899         /* clear interrupts we are handling here */
2900         DWC_OTG_WRITE_4(sc, DOTG_GINTSTS, status & DWC_OTG_MSK_GINT_THREAD_IRQ);
2901
2902         DPRINTFN(14, "GINTSTS=0x%08x HAINT=0x%08x HFNUM=0x%08x\n",
2903             status, DWC_OTG_READ_4(sc, DOTG_HAINT),
2904             DWC_OTG_READ_4(sc, DOTG_HFNUM));
2905
2906         if (status & GINTSTS_USBRST) {
2907
2908                 /* set correct state */
2909                 sc->sc_flags.status_device_mode = 1;
2910                 sc->sc_flags.status_bus_reset = 0;
2911                 sc->sc_flags.status_suspend = 0;
2912                 sc->sc_flags.change_suspend = 0;
2913                 sc->sc_flags.change_connect = 1;
2914
2915                 /* Disable SOF interrupt */
2916                 sc->sc_irq_mask &= ~GINTMSK_SOFMSK;
2917                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2918
2919                 /* complete root HUB interrupt endpoint */
2920                 dwc_otg_root_intr(sc);
2921         }
2922
2923         /* check for any bus state change interrupts */
2924         if (status & GINTSTS_ENUMDONE) {
2925
2926                 uint32_t temp;
2927
2928                 DPRINTFN(5, "end of reset\n");
2929
2930                 /* set correct state */
2931                 sc->sc_flags.status_device_mode = 1;
2932                 sc->sc_flags.status_bus_reset = 1;
2933                 sc->sc_flags.status_suspend = 0;
2934                 sc->sc_flags.change_suspend = 0;
2935                 sc->sc_flags.change_connect = 1;
2936                 sc->sc_flags.status_low_speed = 0;
2937                 sc->sc_flags.port_enabled = 1;
2938
2939                 /* reset FIFOs */
2940                 (void) dwc_otg_init_fifo(sc, DWC_MODE_DEVICE);
2941
2942                 /* reset function address */
2943                 dwc_otg_set_address(sc, 0);
2944
2945                 /* figure out enumeration speed */
2946                 temp = DWC_OTG_READ_4(sc, DOTG_DSTS);
2947                 if (DSTS_ENUMSPD_GET(temp) == DSTS_ENUMSPD_HI)
2948                         sc->sc_flags.status_high_speed = 1;
2949                 else
2950                         sc->sc_flags.status_high_speed = 0;
2951
2952                 /*
2953                  * Disable resume and SOF interrupt, and enable
2954                  * suspend and RX frame interrupt:
2955                  */
2956                 sc->sc_irq_mask &= ~(GINTMSK_WKUPINTMSK | GINTMSK_SOFMSK);
2957                 sc->sc_irq_mask |= GINTMSK_USBSUSPMSK;
2958                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2959
2960                 /* complete root HUB interrupt endpoint */
2961                 dwc_otg_root_intr(sc);
2962         }
2963
2964         if (status & GINTSTS_PRTINT) {
2965                 uint32_t hprt;
2966
2967                 hprt = DWC_OTG_READ_4(sc, DOTG_HPRT);
2968
2969                 /* clear change bits */
2970                 DWC_OTG_WRITE_4(sc, DOTG_HPRT, (hprt & (
2971                     HPRT_PRTPWR | HPRT_PRTENCHNG |
2972                     HPRT_PRTCONNDET | HPRT_PRTOVRCURRCHNG)) |
2973                     sc->sc_hprt_val);
2974
2975                 DPRINTFN(12, "GINTSTS=0x%08x, HPRT=0x%08x\n", status, hprt);
2976
2977                 sc->sc_flags.status_device_mode = 0;
2978
2979                 if (hprt & HPRT_PRTCONNSTS)
2980                         sc->sc_flags.status_bus_reset = 1;
2981                 else
2982                         sc->sc_flags.status_bus_reset = 0;
2983
2984                 if (hprt & HPRT_PRTENCHNG)
2985                         sc->sc_flags.change_enabled = 1;
2986
2987                 if (hprt & HPRT_PRTENA)
2988                         sc->sc_flags.port_enabled = 1;
2989                 else
2990                         sc->sc_flags.port_enabled = 0;
2991
2992                 if (hprt & HPRT_PRTOVRCURRCHNG)
2993                         sc->sc_flags.change_over_current = 1;
2994
2995                 if (hprt & HPRT_PRTOVRCURRACT)
2996                         sc->sc_flags.port_over_current = 1;
2997                 else
2998                         sc->sc_flags.port_over_current = 0;
2999
3000                 if (hprt & HPRT_PRTPWR)
3001                         sc->sc_flags.port_powered = 1;
3002                 else
3003                         sc->sc_flags.port_powered = 0;
3004
3005                 if (((hprt & HPRT_PRTSPD_MASK)
3006                     >> HPRT_PRTSPD_SHIFT) == HPRT_PRTSPD_LOW)
3007                         sc->sc_flags.status_low_speed = 1;
3008                 else
3009                         sc->sc_flags.status_low_speed = 0;
3010
3011                 if (((hprt & HPRT_PRTSPD_MASK)
3012                     >> HPRT_PRTSPD_SHIFT) == HPRT_PRTSPD_HIGH)
3013                         sc->sc_flags.status_high_speed = 1;
3014                 else
3015                         sc->sc_flags.status_high_speed = 0;
3016
3017                 if (hprt & HPRT_PRTCONNDET)
3018                         sc->sc_flags.change_connect = 1;
3019
3020                 if (hprt & HPRT_PRTSUSP)
3021                         dwc_otg_suspend_irq(sc);
3022                 else
3023                         dwc_otg_resume_irq(sc);
3024
3025                 /* complete root HUB interrupt endpoint */
3026                 dwc_otg_root_intr(sc);
3027
3028                 /* update host frame interval */
3029                 dwc_otg_update_host_frame_interval(sc);
3030         }
3031
3032         /*
3033          * If resume and suspend is set at the same time we interpret
3034          * that like RESUME. Resume is set when there is at least 3
3035          * milliseconds of inactivity on the USB BUS.
3036          */
3037         if (status & GINTSTS_WKUPINT) {
3038
3039                 DPRINTFN(5, "resume interrupt\n");
3040
3041                 dwc_otg_resume_irq(sc);
3042
3043         } else if (status & GINTSTS_USBSUSP) {
3044
3045                 DPRINTFN(5, "suspend interrupt\n");
3046
3047                 dwc_otg_suspend_irq(sc);
3048         }
3049         /* check VBUS */
3050         if (status & (GINTSTS_USBSUSP |
3051             GINTSTS_USBRST |
3052             GINTMSK_OTGINTMSK |
3053             GINTSTS_SESSREQINT)) {
3054                 uint32_t temp;
3055
3056                 temp = DWC_OTG_READ_4(sc, DOTG_GOTGCTL);
3057
3058                 DPRINTFN(5, "GOTGCTL=0x%08x\n", temp);
3059
3060                 dwc_otg_vbus_interrupt(sc,
3061                     (temp & (GOTGCTL_ASESVLD | GOTGCTL_BSESVLD)) ? 1 : 0);
3062         }
3063
3064         if (sc->sc_xfer_complete != 0) {
3065                 sc->sc_xfer_complete = 0;
3066
3067                 /* complete FIFOs, if any */
3068                 dwc_otg_interrupt_complete_locked(sc);
3069         }
3070         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3071         USB_BUS_UNLOCK(&sc->sc_bus);
3072 }
3073
3074 static void
3075 dwc_otg_setup_standard_chain_sub(struct dwc_otg_std_temp *temp)
3076 {
3077         struct dwc_otg_td *td;
3078
3079         /* get current Transfer Descriptor */
3080         td = temp->td_next;
3081         temp->td = td;
3082
3083         /* prepare for next TD */
3084         temp->td_next = td->obj_next;
3085
3086         /* fill out the Transfer Descriptor */
3087         td->func = temp->func;
3088         td->pc = temp->pc;
3089         td->offset = temp->offset;
3090         td->remainder = temp->len;
3091         td->tx_bytes = 0;
3092         td->error_any = 0;
3093         td->error_stall = 0;
3094         td->npkt = 0;
3095         td->did_stall = temp->did_stall;
3096         td->short_pkt = temp->short_pkt;
3097         td->alt_next = temp->setup_alt_next;
3098         td->set_toggle = 0;
3099         td->got_short = 0;
3100         td->did_nak = 0;
3101         td->channel[0] = DWC_OTG_MAX_CHANNELS;
3102         td->channel[1] = DWC_OTG_MAX_CHANNELS;
3103         td->channel[2] = DWC_OTG_MAX_CHANNELS;
3104         td->state = 0;
3105         td->errcnt = 0;
3106         td->tt_scheduled = 0;
3107         td->tt_xactpos = HCSPLT_XACTPOS_BEGIN;
3108 }
3109
3110 static void
3111 dwc_otg_setup_standard_chain(struct usb_xfer *xfer)
3112 {
3113         struct dwc_otg_std_temp temp;
3114         struct dwc_otg_td *td;
3115         uint32_t x;
3116         uint8_t need_sync;
3117         uint8_t is_host;
3118
3119         DPRINTFN(9, "addr=%d endpt=%d sumlen=%d speed=%d\n",
3120             xfer->address, UE_GET_ADDR(xfer->endpointno),
3121             xfer->sumlen, usbd_get_speed(xfer->xroot->udev));
3122
3123         temp.max_frame_size = xfer->max_frame_size;
3124
3125         td = xfer->td_start[0];
3126         xfer->td_transfer_first = td;
3127         xfer->td_transfer_cache = td;
3128
3129         /* setup temp */
3130
3131         temp.pc = NULL;
3132         temp.td = NULL;
3133         temp.td_next = xfer->td_start[0];
3134         temp.offset = 0;
3135         temp.setup_alt_next = xfer->flags_int.short_frames_ok ||
3136             xfer->flags_int.isochronous_xfr;
3137         temp.did_stall = !xfer->flags_int.control_stall;
3138
3139         is_host = (xfer->xroot->udev->flags.usb_mode == USB_MODE_HOST);
3140
3141         /* check if we should prepend a setup message */
3142
3143         if (xfer->flags_int.control_xfr) {
3144                 if (xfer->flags_int.control_hdr) {
3145
3146                         if (is_host)
3147                                 temp.func = &dwc_otg_host_setup_tx;
3148                         else
3149                                 temp.func = &dwc_otg_setup_rx;
3150
3151                         temp.len = xfer->frlengths[0];
3152                         temp.pc = xfer->frbuffers + 0;
3153                         temp.short_pkt = temp.len ? 1 : 0;
3154
3155                         /* check for last frame */
3156                         if (xfer->nframes == 1) {
3157                                 /* no STATUS stage yet, SETUP is last */
3158                                 if (xfer->flags_int.control_act)
3159                                         temp.setup_alt_next = 0;
3160                         }
3161
3162                         dwc_otg_setup_standard_chain_sub(&temp);
3163                 }
3164                 x = 1;
3165         } else {
3166                 x = 0;
3167         }
3168
3169         if (x != xfer->nframes) {
3170                 if (xfer->endpointno & UE_DIR_IN) {
3171                         if (is_host) {
3172                                 temp.func = &dwc_otg_host_data_rx;
3173                                 need_sync = 0;
3174                         } else {
3175                                 temp.func = &dwc_otg_data_tx;
3176                                 need_sync = 1;
3177                         }
3178                 } else {
3179                         if (is_host) {
3180                                 temp.func = &dwc_otg_host_data_tx;
3181                                 need_sync = 0;
3182                         } else {
3183                                 temp.func = &dwc_otg_data_rx;
3184                                 need_sync = 0;
3185                         }
3186                 }
3187
3188                 /* setup "pc" pointer */
3189                 temp.pc = xfer->frbuffers + x;
3190         } else {
3191                 need_sync = 0;
3192         }
3193         while (x != xfer->nframes) {
3194
3195                 /* DATA0 / DATA1 message */
3196
3197                 temp.len = xfer->frlengths[x];
3198
3199                 x++;
3200
3201                 if (x == xfer->nframes) {
3202                         if (xfer->flags_int.control_xfr) {
3203                                 if (xfer->flags_int.control_act) {
3204                                         temp.setup_alt_next = 0;
3205                                 }
3206                         } else {
3207                                 temp.setup_alt_next = 0;
3208                         }
3209                 }
3210                 if (temp.len == 0) {
3211
3212                         /* make sure that we send an USB packet */
3213
3214                         temp.short_pkt = 0;
3215
3216                 } else {
3217
3218                         /* regular data transfer */
3219
3220                         temp.short_pkt = (xfer->flags.force_short_xfer ? 0 : 1);
3221                 }
3222
3223                 dwc_otg_setup_standard_chain_sub(&temp);
3224
3225                 if (xfer->flags_int.isochronous_xfr) {
3226                         temp.offset += temp.len;
3227                 } else {
3228                         /* get next Page Cache pointer */
3229                         temp.pc = xfer->frbuffers + x;
3230                 }
3231         }
3232
3233         if (xfer->flags_int.control_xfr) {
3234
3235                 /* always setup a valid "pc" pointer for status and sync */
3236                 temp.pc = xfer->frbuffers + 0;
3237                 temp.len = 0;
3238                 temp.short_pkt = 0;
3239                 temp.setup_alt_next = 0;
3240
3241                 /* check if we need to sync */
3242                 if (need_sync) {
3243                         /* we need a SYNC point after TX */
3244                         temp.func = &dwc_otg_data_tx_sync;
3245                         dwc_otg_setup_standard_chain_sub(&temp);
3246                 }
3247
3248                 /* check if we should append a status stage */
3249                 if (!xfer->flags_int.control_act) {
3250
3251                         /*
3252                          * Send a DATA1 message and invert the current
3253                          * endpoint direction.
3254                          */
3255                         if (xfer->endpointno & UE_DIR_IN) {
3256                                 if (is_host) {
3257                                         temp.func = &dwc_otg_host_data_tx;
3258                                         need_sync = 0;
3259                                 } else {
3260                                         temp.func = &dwc_otg_data_rx;
3261                                         need_sync = 0;
3262                                 }
3263                         } else {
3264                                 if (is_host) {
3265                                         temp.func = &dwc_otg_host_data_rx;
3266                                         need_sync = 0;
3267                                 } else {
3268                                         temp.func = &dwc_otg_data_tx;
3269                                         need_sync = 1;
3270                                 }
3271                         }
3272
3273                         dwc_otg_setup_standard_chain_sub(&temp);
3274
3275                         /* data toggle should be DATA1 */
3276                         td = temp.td;
3277                         td->set_toggle = 1;
3278
3279                         if (need_sync) {
3280                                 /* we need a SYNC point after TX */
3281                                 temp.func = &dwc_otg_data_tx_sync;
3282                                 dwc_otg_setup_standard_chain_sub(&temp);
3283                         }
3284                 }
3285         } else {
3286                 /* check if we need to sync */
3287                 if (need_sync) {
3288
3289                         temp.pc = xfer->frbuffers + 0;
3290                         temp.len = 0;
3291                         temp.short_pkt = 0;
3292                         temp.setup_alt_next = 0;
3293
3294                         /* we need a SYNC point after TX */
3295                         temp.func = &dwc_otg_data_tx_sync;
3296                         dwc_otg_setup_standard_chain_sub(&temp);
3297                 }
3298         }
3299
3300         /* must have at least one frame! */
3301         td = temp.td;
3302         xfer->td_transfer_last = td;
3303
3304         if (is_host) {
3305
3306                 struct dwc_otg_softc *sc;
3307                 uint32_t hcchar;
3308                 uint32_t hcsplt;
3309
3310                 sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
3311
3312                 /* get first again */
3313                 td = xfer->td_transfer_first;
3314                 td->toggle = (xfer->endpoint->toggle_next ? 1 : 0);
3315
3316                 hcchar =
3317                         (xfer->address << HCCHAR_DEVADDR_SHIFT) |
3318                         ((xfer->endpointno & UE_ADDR) << HCCHAR_EPNUM_SHIFT) |
3319                         (xfer->max_packet_size << HCCHAR_MPS_SHIFT) |
3320                         HCCHAR_CHENA;
3321
3322                 /*
3323                  * We are not always able to meet the timing
3324                  * requirements of the USB interrupt endpoint's
3325                  * complete split token, when doing transfers going
3326                  * via a transaction translator. Use the CONTROL
3327                  * transfer type instead of the INTERRUPT transfer
3328                  * type in general, as a means to workaround
3329                  * that. This trick should work for both FULL and LOW
3330                  * speed USB traffic going through a TT. For non-TT
3331                  * traffic it works aswell. The reason for using
3332                  * CONTROL type instead of BULK is that some TTs might
3333                  * reject LOW speed BULK traffic.
3334                  */
3335                 if (td->ep_type == UE_INTERRUPT)
3336                         hcchar |= (UE_CONTROL << HCCHAR_EPTYPE_SHIFT);
3337                 else
3338                         hcchar |= (td->ep_type << HCCHAR_EPTYPE_SHIFT);
3339
3340                 if (UE_GET_DIR(xfer->endpointno) == UE_DIR_IN)
3341                         hcchar |= HCCHAR_EPDIR_IN;
3342
3343                 switch (xfer->xroot->udev->speed) {
3344                 case USB_SPEED_LOW:
3345                         hcchar |= HCCHAR_LSPDDEV;
3346                         /* FALLTHROUGH */
3347                 case USB_SPEED_FULL:
3348                         /* check if root HUB port is running High Speed */
3349                         if (dwc_otg_uses_split(xfer->xroot->udev)) {
3350                                 hcsplt = HCSPLT_SPLTENA |
3351                                     (xfer->xroot->udev->hs_port_no <<
3352                                     HCSPLT_PRTADDR_SHIFT) |
3353                                     (xfer->xroot->udev->hs_hub_addr <<
3354                                     HCSPLT_HUBADDR_SHIFT);
3355                         } else {
3356                                 hcsplt = 0;
3357                         }
3358                         if (td->ep_type == UE_INTERRUPT) {
3359                                 uint32_t ival;
3360                                 ival = xfer->interval / DWC_OTG_HOST_TIMER_RATE;
3361                                 if (ival == 0)
3362                                         ival = 1;
3363                                 else if (ival > 127)
3364                                         ival = 127;
3365                                 td->tmr_val = sc->sc_tmr_val + ival;
3366                                 td->tmr_res = ival;
3367                         } else if (td->ep_type == UE_ISOCHRONOUS) {
3368                                 td->tmr_res = 1;
3369                                 td->tmr_val = sc->sc_last_frame_num;
3370                                 if (td->hcchar & HCCHAR_EPDIR_IN)
3371                                         td->tmr_val++;
3372                         } else {
3373                                 td->tmr_val = 0;
3374                                 td->tmr_res = (uint8_t)sc->sc_last_frame_num;
3375                         }
3376                         break;
3377                 case USB_SPEED_HIGH:
3378                         hcsplt = 0;
3379                         if (td->ep_type == UE_INTERRUPT) {
3380                                 uint32_t ival;
3381                                 hcchar |= ((xfer->max_packet_count & 3)
3382                                     << HCCHAR_MC_SHIFT);
3383                                 ival = xfer->interval / DWC_OTG_HOST_TIMER_RATE;
3384                                 if (ival == 0)
3385                                         ival = 1;
3386                                 else if (ival > 127)
3387                                         ival = 127;
3388                                 td->tmr_val = sc->sc_tmr_val + ival;
3389                                 td->tmr_res = ival;
3390                         } else if (td->ep_type == UE_ISOCHRONOUS) {
3391                                 hcchar |= ((xfer->max_packet_count & 3)
3392                                     << HCCHAR_MC_SHIFT);
3393                                 td->tmr_res = 1 << usbd_xfer_get_fps_shift(xfer);
3394                                 td->tmr_val = sc->sc_last_frame_num;
3395                                 if (td->hcchar & HCCHAR_EPDIR_IN)
3396                                         td->tmr_val += td->tmr_res;
3397
3398                         } else {
3399                                 td->tmr_val = 0;
3400                                 td->tmr_res = (uint8_t)sc->sc_last_frame_num;
3401                         }
3402                         break;
3403                 default:
3404                         hcsplt = 0;
3405                         td->tmr_val = 0;
3406                         td->tmr_res = 0;
3407                         break;
3408                 }
3409
3410                 /* store configuration in all TD's */
3411                 while (1) {
3412                         td->hcchar = hcchar;
3413                         td->hcsplt = hcsplt;
3414
3415                         if (((void *)td) == xfer->td_transfer_last)
3416                                 break;
3417
3418                         td = td->obj_next;
3419                 }
3420         }
3421 }
3422
3423 static void
3424 dwc_otg_timeout(void *arg)
3425 {
3426         struct usb_xfer *xfer = arg;
3427
3428         DPRINTF("xfer=%p\n", xfer);
3429
3430         USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
3431
3432         /* transfer is transferred */
3433         dwc_otg_device_done(xfer, USB_ERR_TIMEOUT);
3434 }
3435
3436 static void
3437 dwc_otg_start_standard_chain(struct usb_xfer *xfer)
3438 {
3439         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
3440
3441         DPRINTFN(9, "\n");
3442
3443         /*
3444          * Poll one time in device mode, which will turn on the
3445          * endpoint interrupts. Else wait for SOF interrupt in host
3446          * mode.
3447          */
3448         USB_BUS_SPIN_LOCK(&sc->sc_bus);
3449
3450         if (sc->sc_flags.status_device_mode != 0) {
3451                 dwc_otg_xfer_do_fifo(sc, xfer);
3452                 if (dwc_otg_xfer_do_complete_locked(sc, xfer))
3453                         goto done;
3454         } else {
3455                 struct dwc_otg_td *td = xfer->td_transfer_cache;
3456                 if (td->ep_type == UE_ISOCHRONOUS &&
3457                     (td->hcchar & HCCHAR_EPDIR_IN) == 0) {
3458                         /*
3459                          * Need to start ISOCHRONOUS OUT transfer ASAP
3460                          * because execution is delayed by one 125us
3461                          * microframe:
3462                          */
3463                         dwc_otg_xfer_do_fifo(sc, xfer);
3464                         if (dwc_otg_xfer_do_complete_locked(sc, xfer))
3465                                 goto done;
3466                 }
3467         }
3468
3469         /* put transfer on interrupt queue */
3470         usbd_transfer_enqueue(&xfer->xroot->bus->intr_q, xfer);
3471
3472         /* start timeout, if any */
3473         if (xfer->timeout != 0) {
3474                 usbd_transfer_timeout_ms(xfer,
3475                     &dwc_otg_timeout, xfer->timeout);
3476         }
3477
3478         if (sc->sc_flags.status_device_mode != 0)
3479                 goto done;
3480
3481         /* enable SOF interrupt, if any */
3482         dwc_otg_enable_sof_irq(sc);
3483 done:
3484         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3485 }
3486
3487 static void
3488 dwc_otg_root_intr(struct dwc_otg_softc *sc)
3489 {
3490         DPRINTFN(9, "\n");
3491
3492         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
3493
3494         /* set port bit */
3495         sc->sc_hub_idata[0] = 0x02;     /* we only have one port */
3496
3497         uhub_root_intr(&sc->sc_bus, sc->sc_hub_idata,
3498             sizeof(sc->sc_hub_idata));
3499 }
3500
3501 static usb_error_t
3502 dwc_otg_standard_done_sub(struct usb_xfer *xfer)
3503 {
3504         struct dwc_otg_td *td;
3505         uint32_t len;
3506         usb_error_t error;
3507
3508         DPRINTFN(9, "\n");
3509
3510         td = xfer->td_transfer_cache;
3511
3512         do {
3513                 len = td->remainder;
3514
3515                 /* store last data toggle */
3516                 xfer->endpoint->toggle_next = td->toggle;
3517
3518                 if (xfer->aframes != xfer->nframes) {
3519                         /*
3520                          * Verify the length and subtract
3521                          * the remainder from "frlengths[]":
3522                          */
3523                         if (len > xfer->frlengths[xfer->aframes]) {
3524                                 td->error_any = 1;
3525                         } else {
3526                                 xfer->frlengths[xfer->aframes] -= len;
3527                         }
3528                 }
3529                 /* Check for transfer error */
3530                 if (td->error_any) {
3531                         /* the transfer is finished */
3532                         error = (td->error_stall ?
3533                             USB_ERR_STALLED : USB_ERR_IOERROR);
3534                         td = NULL;
3535                         break;
3536                 }
3537                 /* Check for short transfer */
3538                 if (len > 0) {
3539                         if (xfer->flags_int.short_frames_ok ||
3540                             xfer->flags_int.isochronous_xfr) {
3541                                 /* follow alt next */
3542                                 if (td->alt_next) {
3543                                         td = td->obj_next;
3544                                 } else {
3545                                         td = NULL;
3546                                 }
3547                         } else {
3548                                 /* the transfer is finished */
3549                                 td = NULL;
3550                         }
3551                         error = 0;
3552                         break;
3553                 }
3554                 td = td->obj_next;
3555
3556                 /* this USB frame is complete */
3557                 error = 0;
3558                 break;
3559
3560         } while (0);
3561
3562         /* update transfer cache */
3563
3564         xfer->td_transfer_cache = td;
3565
3566         return (error);
3567 }
3568
3569 static void
3570 dwc_otg_standard_done(struct usb_xfer *xfer)
3571 {
3572         usb_error_t err = 0;
3573
3574         DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
3575             xfer, xfer->endpoint);
3576
3577         /* reset scanner */
3578
3579         xfer->td_transfer_cache = xfer->td_transfer_first;
3580
3581         if (xfer->flags_int.control_xfr) {
3582
3583                 if (xfer->flags_int.control_hdr) {
3584
3585                         err = dwc_otg_standard_done_sub(xfer);
3586                 }
3587                 xfer->aframes = 1;
3588
3589                 if (xfer->td_transfer_cache == NULL) {
3590                         goto done;
3591                 }
3592         }
3593         while (xfer->aframes != xfer->nframes) {
3594
3595                 err = dwc_otg_standard_done_sub(xfer);
3596                 xfer->aframes++;
3597
3598                 if (xfer->td_transfer_cache == NULL) {
3599                         goto done;
3600                 }
3601         }
3602
3603         if (xfer->flags_int.control_xfr &&
3604             !xfer->flags_int.control_act) {
3605
3606                 err = dwc_otg_standard_done_sub(xfer);
3607         }
3608 done:
3609         dwc_otg_device_done(xfer, err);
3610 }
3611
3612 /*------------------------------------------------------------------------*
3613  *      dwc_otg_device_done
3614  *
3615  * NOTE: this function can be called more than one time on the
3616  * same USB transfer!
3617  *------------------------------------------------------------------------*/
3618 static void
3619 dwc_otg_device_done(struct usb_xfer *xfer, usb_error_t error)
3620 {
3621         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
3622
3623         DPRINTFN(9, "xfer=%p, endpoint=%p, error=%d\n",
3624             xfer, xfer->endpoint, error);
3625
3626         USB_BUS_SPIN_LOCK(&sc->sc_bus);
3627
3628         if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) {
3629                 /* Interrupts are cleared by the interrupt handler */
3630         } else {
3631                 struct dwc_otg_td *td;
3632
3633                 td = xfer->td_transfer_cache;
3634                 if (td != NULL)
3635                         dwc_otg_host_channel_free(sc, td);
3636         }
3637         /* dequeue transfer and start next transfer */
3638         usbd_transfer_done(xfer, error);
3639
3640         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3641 }
3642
3643 static void
3644 dwc_otg_xfer_stall(struct usb_xfer *xfer)
3645 {
3646         dwc_otg_device_done(xfer, USB_ERR_STALLED);
3647 }
3648
3649 static void
3650 dwc_otg_set_stall(struct usb_device *udev,
3651     struct usb_endpoint *ep, uint8_t *did_stall)
3652 {
3653         struct dwc_otg_softc *sc;
3654         uint32_t temp;
3655         uint32_t reg;
3656         uint8_t ep_no;
3657
3658         USB_BUS_LOCK_ASSERT(udev->bus, MA_OWNED);
3659
3660         /* check mode */
3661         if (udev->flags.usb_mode != USB_MODE_DEVICE) {
3662                 /* not supported */
3663                 return;
3664         }
3665
3666         sc = DWC_OTG_BUS2SC(udev->bus);
3667
3668         USB_BUS_SPIN_LOCK(&sc->sc_bus);
3669
3670         /* get endpoint address */
3671         ep_no = ep->edesc->bEndpointAddress;
3672
3673         DPRINTFN(5, "endpoint=0x%x\n", ep_no);
3674
3675         if (ep_no & UE_DIR_IN) {
3676                 reg = DOTG_DIEPCTL(ep_no & UE_ADDR);
3677                 temp = sc->sc_in_ctl[ep_no & UE_ADDR];
3678         } else {
3679                 reg = DOTG_DOEPCTL(ep_no & UE_ADDR);
3680                 temp = sc->sc_out_ctl[ep_no & UE_ADDR];
3681         }
3682
3683         /* disable and stall endpoint */
3684         DWC_OTG_WRITE_4(sc, reg, temp | DOEPCTL_EPDIS);
3685         DWC_OTG_WRITE_4(sc, reg, temp | DOEPCTL_STALL);
3686
3687         /* clear active OUT ep */
3688         if (!(ep_no & UE_DIR_IN)) {
3689
3690                 sc->sc_active_rx_ep &= ~(1U << (ep_no & UE_ADDR));
3691
3692                 if (sc->sc_last_rx_status != 0 &&
3693                     (ep_no & UE_ADDR) == GRXSTSRD_CHNUM_GET(
3694                     sc->sc_last_rx_status)) {
3695                         /* dump data */
3696                         dwc_otg_common_rx_ack(sc);
3697                         /* poll interrupt */
3698                         dwc_otg_interrupt_poll_locked(sc);
3699                         dwc_otg_interrupt_complete_locked(sc);
3700                 }
3701         }
3702         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3703 }
3704
3705 static void
3706 dwc_otg_clear_stall_sub_locked(struct dwc_otg_softc *sc, uint32_t mps,
3707     uint8_t ep_no, uint8_t ep_type, uint8_t ep_dir)
3708 {
3709         uint32_t reg;
3710         uint32_t temp;
3711
3712         if (ep_type == UE_CONTROL) {
3713                 /* clearing stall is not needed */
3714                 return;
3715         }
3716
3717         if (ep_dir) {
3718                 reg = DOTG_DIEPCTL(ep_no);
3719         } else {
3720                 reg = DOTG_DOEPCTL(ep_no);
3721                 sc->sc_active_rx_ep |= (1U << ep_no);
3722         }
3723
3724         /* round up and mask away the multiplier count */
3725         mps = (mps + 3) & 0x7FC;
3726
3727         if (ep_type == UE_BULK) {
3728                 temp = DIEPCTL_EPTYPE_SET(
3729                     DIEPCTL_EPTYPE_BULK) |
3730                     DIEPCTL_USBACTEP;
3731         } else if (ep_type == UE_INTERRUPT) {
3732                 temp = DIEPCTL_EPTYPE_SET(
3733                     DIEPCTL_EPTYPE_INTERRUPT) |
3734                     DIEPCTL_USBACTEP;
3735         } else {
3736                 temp = DIEPCTL_EPTYPE_SET(
3737                     DIEPCTL_EPTYPE_ISOC) |
3738                     DIEPCTL_USBACTEP;
3739         }
3740
3741         temp |= DIEPCTL_MPS_SET(mps);
3742         temp |= DIEPCTL_TXFNUM_SET(ep_no);
3743
3744         if (ep_dir)
3745                 sc->sc_in_ctl[ep_no] = temp;
3746         else
3747                 sc->sc_out_ctl[ep_no] = temp;
3748
3749         DWC_OTG_WRITE_4(sc, reg, temp | DOEPCTL_EPDIS);
3750         DWC_OTG_WRITE_4(sc, reg, temp | DOEPCTL_SETD0PID);
3751         DWC_OTG_WRITE_4(sc, reg, temp | DIEPCTL_SNAK);
3752
3753         /* we only reset the transmit FIFO */
3754         if (ep_dir) {
3755                 dwc_otg_tx_fifo_reset(sc,
3756                     GRSTCTL_TXFIFO(ep_no) |
3757                     GRSTCTL_TXFFLSH);
3758
3759                 DWC_OTG_WRITE_4(sc,
3760                     DOTG_DIEPTSIZ(ep_no), 0);
3761         }
3762
3763         /* poll interrupt */
3764         dwc_otg_interrupt_poll_locked(sc);
3765         dwc_otg_interrupt_complete_locked(sc);
3766 }
3767
3768 static void
3769 dwc_otg_clear_stall(struct usb_device *udev, struct usb_endpoint *ep)
3770 {
3771         struct dwc_otg_softc *sc;
3772         struct usb_endpoint_descriptor *ed;
3773
3774         DPRINTFN(5, "endpoint=%p\n", ep);
3775
3776         USB_BUS_LOCK_ASSERT(udev->bus, MA_OWNED);
3777
3778         /* check mode */
3779         if (udev->flags.usb_mode != USB_MODE_DEVICE) {
3780                 /* not supported */
3781                 return;
3782         }
3783         /* get softc */
3784         sc = DWC_OTG_BUS2SC(udev->bus);
3785
3786         USB_BUS_SPIN_LOCK(&sc->sc_bus);
3787
3788         /* get endpoint descriptor */
3789         ed = ep->edesc;
3790
3791         /* reset endpoint */
3792         dwc_otg_clear_stall_sub_locked(sc,
3793             UGETW(ed->wMaxPacketSize),
3794             (ed->bEndpointAddress & UE_ADDR),
3795             (ed->bmAttributes & UE_XFERTYPE),
3796             (ed->bEndpointAddress & (UE_DIR_IN | UE_DIR_OUT)));
3797
3798         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3799 }
3800
3801 static void
3802 dwc_otg_device_state_change(struct usb_device *udev)
3803 {
3804         struct dwc_otg_softc *sc;
3805         uint8_t x;
3806
3807         /* check mode */
3808         if (udev->flags.usb_mode != USB_MODE_DEVICE) {
3809                 /* not supported */
3810                 return;
3811         }
3812
3813         /* get softc */
3814         sc = DWC_OTG_BUS2SC(udev->bus);
3815
3816         /* deactivate all other endpoint but the control endpoint */
3817         if (udev->state == USB_STATE_CONFIGURED ||
3818             udev->state == USB_STATE_ADDRESSED) {
3819
3820                 USB_BUS_LOCK(&sc->sc_bus);
3821
3822                 for (x = 1; x != sc->sc_dev_ep_max; x++) {
3823
3824                         if (x < sc->sc_dev_in_ep_max) {
3825                                 DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(x),
3826                                     DIEPCTL_EPDIS);
3827                                 DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(x), 0);
3828                         }
3829
3830                         DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(x),
3831                             DOEPCTL_EPDIS);
3832                         DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(x), 0);
3833                 }
3834                 USB_BUS_UNLOCK(&sc->sc_bus);
3835         }
3836 }
3837
3838 int
3839 dwc_otg_init(struct dwc_otg_softc *sc)
3840 {
3841         uint32_t temp;
3842
3843         DPRINTF("start\n");
3844
3845         /* set up the bus structure */
3846         sc->sc_bus.usbrev = USB_REV_2_0;
3847         sc->sc_bus.methods = &dwc_otg_bus_methods;
3848
3849         usb_callout_init_mtx(&sc->sc_timer,
3850             &sc->sc_bus.bus_mtx, 0);
3851
3852         USB_BUS_LOCK(&sc->sc_bus);
3853
3854         /* turn on clocks */
3855         dwc_otg_clocks_on(sc);
3856
3857         temp = DWC_OTG_READ_4(sc, DOTG_GSNPSID);
3858         DPRINTF("Version = 0x%08x\n", temp);
3859
3860         /* disconnect */
3861         DWC_OTG_WRITE_4(sc, DOTG_DCTL,
3862             DCTL_SFTDISCON);
3863
3864         /* wait for host to detect disconnect */
3865         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 32);
3866
3867         DWC_OTG_WRITE_4(sc, DOTG_GRSTCTL,
3868             GRSTCTL_CSFTRST);
3869
3870         /* wait a little bit for block to reset */
3871         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 128);
3872
3873         switch (sc->sc_mode) {
3874         case DWC_MODE_DEVICE:
3875                 temp = GUSBCFG_FORCEDEVMODE;
3876                 break;
3877         case DWC_MODE_HOST:
3878                 temp = GUSBCFG_FORCEHOSTMODE;
3879                 break;
3880         default:
3881                 temp = 0;
3882                 break;
3883         }
3884
3885         /* select HSIC, ULPI or internal PHY mode */
3886         switch (dwc_otg_phy_type) {
3887         case DWC_OTG_PHY_HSIC:
3888                 DWC_OTG_WRITE_4(sc, DOTG_GUSBCFG,
3889                     GUSBCFG_PHYIF |
3890                     GUSBCFG_TRD_TIM_SET(5) | temp);
3891                 DWC_OTG_WRITE_4(sc, DOTG_GOTGCTL,
3892                     0x000000EC);
3893
3894                 temp = DWC_OTG_READ_4(sc, DOTG_GLPMCFG);
3895                 DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3896                     temp & ~GLPMCFG_HSIC_CONN);
3897                 DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3898                     temp | GLPMCFG_HSIC_CONN);
3899                 break;
3900         case DWC_OTG_PHY_ULPI:
3901                 DWC_OTG_WRITE_4(sc, DOTG_GUSBCFG,
3902                     GUSBCFG_ULPI_UTMI_SEL |
3903                     GUSBCFG_TRD_TIM_SET(5) | temp);
3904                 DWC_OTG_WRITE_4(sc, DOTG_GOTGCTL, 0);
3905
3906                 temp = DWC_OTG_READ_4(sc, DOTG_GLPMCFG);
3907                 DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3908                     temp & ~GLPMCFG_HSIC_CONN);
3909                 break;
3910         case DWC_OTG_PHY_INTERNAL:
3911                 DWC_OTG_WRITE_4(sc, DOTG_GUSBCFG,
3912                     GUSBCFG_PHYSEL |
3913                     GUSBCFG_TRD_TIM_SET(5) | temp);
3914                 DWC_OTG_WRITE_4(sc, DOTG_GOTGCTL, 0);
3915
3916                 temp = DWC_OTG_READ_4(sc, DOTG_GLPMCFG);
3917                 DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3918                     temp & ~GLPMCFG_HSIC_CONN);
3919
3920                 temp = DWC_OTG_READ_4(sc, DOTG_GGPIO);
3921                 temp &= ~(DOTG_GGPIO_NOVBUSSENS | DOTG_GGPIO_I2CPADEN);
3922                 temp |= (DOTG_GGPIO_VBUSASEN | DOTG_GGPIO_VBUSBSEN |
3923                     DOTG_GGPIO_PWRDWN);
3924                 DWC_OTG_WRITE_4(sc, DOTG_GGPIO, temp);
3925                 break;
3926         default:
3927                 break;
3928         }
3929
3930         /* clear global nak */
3931         DWC_OTG_WRITE_4(sc, DOTG_DCTL,
3932             DCTL_CGOUTNAK |
3933             DCTL_CGNPINNAK);
3934
3935         /* disable USB port */
3936         DWC_OTG_WRITE_4(sc, DOTG_PCGCCTL, 0xFFFFFFFF);
3937
3938         /* wait 10ms */
3939         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 100);
3940
3941         /* enable USB port */
3942         DWC_OTG_WRITE_4(sc, DOTG_PCGCCTL, 0);
3943
3944         /* wait 10ms */
3945         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 100);
3946
3947         temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG3);
3948
3949         sc->sc_fifo_size = 4 * GHWCFG3_DFIFODEPTH_GET(temp);
3950
3951         temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG2);
3952
3953         sc->sc_dev_ep_max = GHWCFG2_NUMDEVEPS_GET(temp);
3954
3955         if (sc->sc_dev_ep_max > DWC_OTG_MAX_ENDPOINTS)
3956                 sc->sc_dev_ep_max = DWC_OTG_MAX_ENDPOINTS;
3957
3958         sc->sc_host_ch_max = GHWCFG2_NUMHSTCHNL_GET(temp);
3959
3960         if (sc->sc_host_ch_max > DWC_OTG_MAX_CHANNELS)
3961                 sc->sc_host_ch_max = DWC_OTG_MAX_CHANNELS;
3962
3963         temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG4);
3964
3965         sc->sc_dev_in_ep_max = GHWCFG4_NUM_IN_EP_GET(temp);
3966
3967         DPRINTF("Total FIFO size = %d bytes, Device EPs = %d/%d Host CHs = %d\n",
3968             sc->sc_fifo_size, sc->sc_dev_ep_max, sc->sc_dev_in_ep_max,
3969             sc->sc_host_ch_max);
3970
3971         /* setup FIFO */
3972         if (dwc_otg_init_fifo(sc, DWC_MODE_OTG)) {
3973                 USB_BUS_UNLOCK(&sc->sc_bus);
3974                 return (EINVAL);
3975         }
3976
3977         /* enable interrupts */
3978         sc->sc_irq_mask = DWC_OTG_MSK_GINT_ENABLED;
3979         DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
3980
3981         if (sc->sc_mode == DWC_MODE_OTG || sc->sc_mode == DWC_MODE_DEVICE) {
3982
3983                 /* enable all endpoint interrupts */
3984                 temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG2);
3985                 if (temp & GHWCFG2_MPI) {
3986                         uint8_t x;
3987
3988                         DPRINTF("Disable Multi Process Interrupts\n");
3989
3990                         for (x = 0; x != sc->sc_dev_in_ep_max; x++) {
3991                                 DWC_OTG_WRITE_4(sc, DOTG_DIEPEACHINTMSK(x), 0);
3992                                 DWC_OTG_WRITE_4(sc, DOTG_DOEPEACHINTMSK(x), 0);
3993                         }
3994                         DWC_OTG_WRITE_4(sc, DOTG_DEACHINTMSK, 0);
3995                 }
3996                 DWC_OTG_WRITE_4(sc, DOTG_DIEPMSK,
3997                     DIEPMSK_XFERCOMPLMSK);
3998                 DWC_OTG_WRITE_4(sc, DOTG_DOEPMSK, 0);
3999                 DWC_OTG_WRITE_4(sc, DOTG_DAINTMSK, 0xFFFF);
4000         }
4001
4002         if (sc->sc_mode == DWC_MODE_OTG || sc->sc_mode == DWC_MODE_HOST) {
4003                 /* setup clocks */
4004                 temp = DWC_OTG_READ_4(sc, DOTG_HCFG);
4005                 temp &= ~(HCFG_FSLSSUPP | HCFG_FSLSPCLKSEL_MASK);
4006                 temp |= (1 << HCFG_FSLSPCLKSEL_SHIFT);
4007                 DWC_OTG_WRITE_4(sc, DOTG_HCFG, temp);
4008         }
4009
4010         /* only enable global IRQ */
4011         DWC_OTG_WRITE_4(sc, DOTG_GAHBCFG,
4012             GAHBCFG_GLBLINTRMSK);
4013
4014         /* turn off clocks */
4015         dwc_otg_clocks_off(sc);
4016
4017         /* read initial VBUS state */
4018
4019         temp = DWC_OTG_READ_4(sc, DOTG_GOTGCTL);
4020
4021         DPRINTFN(5, "GOTCTL=0x%08x\n", temp);
4022
4023         dwc_otg_vbus_interrupt(sc,
4024             (temp & (GOTGCTL_ASESVLD | GOTGCTL_BSESVLD)) ? 1 : 0);
4025
4026         USB_BUS_UNLOCK(&sc->sc_bus);
4027
4028         /* catch any lost interrupts */
4029
4030         dwc_otg_do_poll(&sc->sc_bus);
4031
4032         return (0);                     /* success */
4033 }
4034
4035 void
4036 dwc_otg_uninit(struct dwc_otg_softc *sc)
4037 {
4038         USB_BUS_LOCK(&sc->sc_bus);
4039
4040         /* stop host timer */
4041         dwc_otg_timer_stop(sc);
4042
4043         /* set disconnect */
4044         DWC_OTG_WRITE_4(sc, DOTG_DCTL,
4045             DCTL_SFTDISCON);
4046
4047         /* turn off global IRQ */
4048         DWC_OTG_WRITE_4(sc, DOTG_GAHBCFG, 0);
4049
4050         sc->sc_flags.port_enabled = 0;
4051         sc->sc_flags.port_powered = 0;
4052         sc->sc_flags.status_vbus = 0;
4053         sc->sc_flags.status_bus_reset = 0;
4054         sc->sc_flags.status_suspend = 0;
4055         sc->sc_flags.change_suspend = 0;
4056         sc->sc_flags.change_connect = 1;
4057
4058         dwc_otg_pull_down(sc);
4059         dwc_otg_clocks_off(sc);
4060
4061         USB_BUS_UNLOCK(&sc->sc_bus);
4062
4063         usb_callout_drain(&sc->sc_timer);
4064 }
4065
4066 static void
4067 dwc_otg_suspend(struct dwc_otg_softc *sc)
4068 {
4069         return;
4070 }
4071
4072 static void
4073 dwc_otg_resume(struct dwc_otg_softc *sc)
4074 {
4075         return;
4076 }
4077
4078 static void
4079 dwc_otg_do_poll(struct usb_bus *bus)
4080 {
4081         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(bus);
4082
4083         USB_BUS_LOCK(&sc->sc_bus);
4084         USB_BUS_SPIN_LOCK(&sc->sc_bus);
4085         dwc_otg_interrupt_poll_locked(sc);
4086         dwc_otg_interrupt_complete_locked(sc);
4087         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
4088         USB_BUS_UNLOCK(&sc->sc_bus);
4089 }
4090
4091 /*------------------------------------------------------------------------*
4092  * DWC OTG bulk support
4093  * DWC OTG control support
4094  * DWC OTG interrupt support
4095  *------------------------------------------------------------------------*/
4096 static void
4097 dwc_otg_device_non_isoc_open(struct usb_xfer *xfer)
4098 {
4099 }
4100
4101 static void
4102 dwc_otg_device_non_isoc_close(struct usb_xfer *xfer)
4103 {
4104         dwc_otg_device_done(xfer, USB_ERR_CANCELLED);
4105 }
4106
4107 static void
4108 dwc_otg_device_non_isoc_enter(struct usb_xfer *xfer)
4109 {
4110 }
4111
4112 static void
4113 dwc_otg_device_non_isoc_start(struct usb_xfer *xfer)
4114 {
4115         /* setup TDs */
4116         dwc_otg_setup_standard_chain(xfer);
4117         dwc_otg_start_standard_chain(xfer);
4118 }
4119
4120 struct usb_pipe_methods dwc_otg_device_non_isoc_methods =
4121 {
4122         .open = dwc_otg_device_non_isoc_open,
4123         .close = dwc_otg_device_non_isoc_close,
4124         .enter = dwc_otg_device_non_isoc_enter,
4125         .start = dwc_otg_device_non_isoc_start,
4126 };
4127
4128 /*------------------------------------------------------------------------*
4129  * DWC OTG full speed isochronous support
4130  *------------------------------------------------------------------------*/
4131 static void
4132 dwc_otg_device_isoc_open(struct usb_xfer *xfer)
4133 {
4134 }
4135
4136 static void
4137 dwc_otg_device_isoc_close(struct usb_xfer *xfer)
4138 {
4139         dwc_otg_device_done(xfer, USB_ERR_CANCELLED);
4140 }
4141
4142 static void
4143 dwc_otg_device_isoc_enter(struct usb_xfer *xfer)
4144 {
4145 }
4146
4147 static void
4148 dwc_otg_device_isoc_start(struct usb_xfer *xfer)
4149 {
4150         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
4151         uint32_t temp;
4152         uint32_t msframes;
4153         uint32_t framenum;
4154         uint8_t shift = usbd_xfer_get_fps_shift(xfer);
4155
4156         DPRINTFN(6, "xfer=%p next=%d nframes=%d\n",
4157             xfer, xfer->endpoint->isoc_next, xfer->nframes);
4158
4159         if (xfer->xroot->udev->flags.usb_mode == USB_MODE_HOST) {
4160                 temp = DWC_OTG_READ_4(sc, DOTG_HFNUM);
4161
4162                 /* get the current frame index */
4163                 framenum = (temp & HFNUM_FRNUM_MASK);
4164         } else {
4165                 temp = DWC_OTG_READ_4(sc, DOTG_DSTS);
4166
4167                 /* get the current frame index */
4168                 framenum = DSTS_SOFFN_GET(temp);
4169         }
4170
4171         /*
4172          * Check if port is doing 8000 or 1000 frames per second:
4173          */
4174         if (sc->sc_flags.status_high_speed)
4175                 framenum /= 8;
4176
4177         framenum &= DWC_OTG_FRAME_MASK;
4178
4179         /*
4180          * Compute number of milliseconds worth of data traffic for
4181          * this USB transfer:
4182          */
4183         if (xfer->xroot->udev->speed == USB_SPEED_HIGH)
4184                 msframes = ((xfer->nframes << shift) + 7) / 8;
4185         else
4186                 msframes = xfer->nframes;
4187
4188         /*
4189          * check if the frame index is within the window where the frames
4190          * will be inserted
4191          */
4192         temp = (framenum - xfer->endpoint->isoc_next) & DWC_OTG_FRAME_MASK;
4193
4194         if ((xfer->endpoint->is_synced == 0) || (temp < msframes)) {
4195                 /*
4196                  * If there is data underflow or the pipe queue is
4197                  * empty we schedule the transfer a few frames ahead
4198                  * of the current frame position. Else two isochronous
4199                  * transfers might overlap.
4200                  */
4201                 xfer->endpoint->isoc_next = (framenum + 3) & DWC_OTG_FRAME_MASK;
4202                 xfer->endpoint->is_synced = 1;
4203                 DPRINTFN(3, "start next=%d\n", xfer->endpoint->isoc_next);
4204         }
4205         /*
4206          * compute how many milliseconds the insertion is ahead of the
4207          * current frame position:
4208          */
4209         temp = (xfer->endpoint->isoc_next - framenum) & DWC_OTG_FRAME_MASK;
4210
4211         /*
4212          * pre-compute when the isochronous transfer will be finished:
4213          */
4214         xfer->isoc_time_complete =
4215                 usb_isoc_time_expand(&sc->sc_bus, framenum) + temp + msframes;
4216
4217         /* setup TDs */
4218         dwc_otg_setup_standard_chain(xfer);
4219
4220         /* compute frame number for next insertion */
4221         xfer->endpoint->isoc_next += msframes;
4222
4223         /* start TD chain */
4224         dwc_otg_start_standard_chain(xfer);
4225 }
4226
4227 struct usb_pipe_methods dwc_otg_device_isoc_methods =
4228 {
4229         .open = dwc_otg_device_isoc_open,
4230         .close = dwc_otg_device_isoc_close,
4231         .enter = dwc_otg_device_isoc_enter,
4232         .start = dwc_otg_device_isoc_start,
4233 };
4234
4235 /*------------------------------------------------------------------------*
4236  * DWC OTG root control support
4237  *------------------------------------------------------------------------*
4238  * Simulate a hardware HUB by handling all the necessary requests.
4239  *------------------------------------------------------------------------*/
4240
4241 static const struct usb_device_descriptor dwc_otg_devd = {
4242         .bLength = sizeof(struct usb_device_descriptor),
4243         .bDescriptorType = UDESC_DEVICE,
4244         .bcdUSB = {0x00, 0x02},
4245         .bDeviceClass = UDCLASS_HUB,
4246         .bDeviceSubClass = UDSUBCLASS_HUB,
4247         .bDeviceProtocol = UDPROTO_HSHUBSTT,
4248         .bMaxPacketSize = 64,
4249         .bcdDevice = {0x00, 0x01},
4250         .iManufacturer = 1,
4251         .iProduct = 2,
4252         .bNumConfigurations = 1,
4253 };
4254
4255 static const struct dwc_otg_config_desc dwc_otg_confd = {
4256         .confd = {
4257                 .bLength = sizeof(struct usb_config_descriptor),
4258                 .bDescriptorType = UDESC_CONFIG,
4259                 .wTotalLength[0] = sizeof(dwc_otg_confd),
4260                 .bNumInterface = 1,
4261                 .bConfigurationValue = 1,
4262                 .iConfiguration = 0,
4263                 .bmAttributes = UC_SELF_POWERED,
4264                 .bMaxPower = 0,
4265         },
4266         .ifcd = {
4267                 .bLength = sizeof(struct usb_interface_descriptor),
4268                 .bDescriptorType = UDESC_INTERFACE,
4269                 .bNumEndpoints = 1,
4270                 .bInterfaceClass = UICLASS_HUB,
4271                 .bInterfaceSubClass = UISUBCLASS_HUB,
4272                 .bInterfaceProtocol = 0,
4273         },
4274         .endpd = {
4275                 .bLength = sizeof(struct usb_endpoint_descriptor),
4276                 .bDescriptorType = UDESC_ENDPOINT,
4277                 .bEndpointAddress = (UE_DIR_IN | DWC_OTG_INTR_ENDPT),
4278                 .bmAttributes = UE_INTERRUPT,
4279                 .wMaxPacketSize[0] = 8,
4280                 .bInterval = 255,
4281         },
4282 };
4283
4284 #define HSETW(ptr, val) ptr = { (uint8_t)(val), (uint8_t)((val) >> 8) }
4285
4286 static const struct usb_hub_descriptor_min dwc_otg_hubd = {
4287         .bDescLength = sizeof(dwc_otg_hubd),
4288         .bDescriptorType = UDESC_HUB,
4289         .bNbrPorts = 1,
4290         HSETW(.wHubCharacteristics, (UHD_PWR_NO_SWITCH | UHD_OC_INDIVIDUAL)),
4291         .bPwrOn2PwrGood = 50,
4292         .bHubContrCurrent = 0,
4293         .DeviceRemovable = {0},         /* port is removable */
4294 };
4295
4296 #define STRING_VENDOR \
4297   "D\0W\0C\0O\0T\0G"
4298
4299 #define STRING_PRODUCT \
4300   "O\0T\0G\0 \0R\0o\0o\0t\0 \0H\0U\0B"
4301
4302 USB_MAKE_STRING_DESC(STRING_VENDOR, dwc_otg_vendor);
4303 USB_MAKE_STRING_DESC(STRING_PRODUCT, dwc_otg_product);
4304
4305 static usb_error_t
4306 dwc_otg_roothub_exec(struct usb_device *udev,
4307     struct usb_device_request *req, const void **pptr, uint16_t *plength)
4308 {
4309         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(udev->bus);
4310         const void *ptr;
4311         uint16_t len;
4312         uint16_t value;
4313         uint16_t index;
4314         usb_error_t err;
4315
4316         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
4317
4318         /* buffer reset */
4319         ptr = (const void *)&sc->sc_hub_temp;
4320         len = 0;
4321         err = 0;
4322
4323         value = UGETW(req->wValue);
4324         index = UGETW(req->wIndex);
4325
4326         /* demultiplex the control request */
4327
4328         switch (req->bmRequestType) {
4329         case UT_READ_DEVICE:
4330                 switch (req->bRequest) {
4331                 case UR_GET_DESCRIPTOR:
4332                         goto tr_handle_get_descriptor;
4333                 case UR_GET_CONFIG:
4334                         goto tr_handle_get_config;
4335                 case UR_GET_STATUS:
4336                         goto tr_handle_get_status;
4337                 default:
4338                         goto tr_stalled;
4339                 }
4340                 break;
4341
4342         case UT_WRITE_DEVICE:
4343                 switch (req->bRequest) {
4344                 case UR_SET_ADDRESS:
4345                         goto tr_handle_set_address;
4346                 case UR_SET_CONFIG:
4347                         goto tr_handle_set_config;
4348                 case UR_CLEAR_FEATURE:
4349                         goto tr_valid;  /* nop */
4350                 case UR_SET_DESCRIPTOR:
4351                         goto tr_valid;  /* nop */
4352                 case UR_SET_FEATURE:
4353                 default:
4354                         goto tr_stalled;
4355                 }
4356                 break;
4357
4358         case UT_WRITE_ENDPOINT:
4359                 switch (req->bRequest) {
4360                 case UR_CLEAR_FEATURE:
4361                         switch (UGETW(req->wValue)) {
4362                         case UF_ENDPOINT_HALT:
4363                                 goto tr_handle_clear_halt;
4364                         case UF_DEVICE_REMOTE_WAKEUP:
4365                                 goto tr_handle_clear_wakeup;
4366                         default:
4367                                 goto tr_stalled;
4368                         }
4369                         break;
4370                 case UR_SET_FEATURE:
4371                         switch (UGETW(req->wValue)) {
4372                         case UF_ENDPOINT_HALT:
4373                                 goto tr_handle_set_halt;
4374                         case UF_DEVICE_REMOTE_WAKEUP:
4375                                 goto tr_handle_set_wakeup;
4376                         default:
4377                                 goto tr_stalled;
4378                         }
4379                         break;
4380                 case UR_SYNCH_FRAME:
4381                         goto tr_valid;  /* nop */
4382                 default:
4383                         goto tr_stalled;
4384                 }
4385                 break;
4386
4387         case UT_READ_ENDPOINT:
4388                 switch (req->bRequest) {
4389                 case UR_GET_STATUS:
4390                         goto tr_handle_get_ep_status;
4391                 default:
4392                         goto tr_stalled;
4393                 }
4394                 break;
4395
4396         case UT_WRITE_INTERFACE:
4397                 switch (req->bRequest) {
4398                 case UR_SET_INTERFACE:
4399                         goto tr_handle_set_interface;
4400                 case UR_CLEAR_FEATURE:
4401                         goto tr_valid;  /* nop */
4402                 case UR_SET_FEATURE:
4403                 default:
4404                         goto tr_stalled;
4405                 }
4406                 break;
4407
4408         case UT_READ_INTERFACE:
4409                 switch (req->bRequest) {
4410                 case UR_GET_INTERFACE:
4411                         goto tr_handle_get_interface;
4412                 case UR_GET_STATUS:
4413                         goto tr_handle_get_iface_status;
4414                 default:
4415                         goto tr_stalled;
4416                 }
4417                 break;
4418
4419         case UT_WRITE_CLASS_INTERFACE:
4420         case UT_WRITE_VENDOR_INTERFACE:
4421                 /* XXX forward */
4422                 break;
4423
4424         case UT_READ_CLASS_INTERFACE:
4425         case UT_READ_VENDOR_INTERFACE:
4426                 /* XXX forward */
4427                 break;
4428
4429         case UT_WRITE_CLASS_DEVICE:
4430                 switch (req->bRequest) {
4431                 case UR_CLEAR_FEATURE:
4432                         goto tr_valid;
4433                 case UR_SET_DESCRIPTOR:
4434                 case UR_SET_FEATURE:
4435                         break;
4436                 default:
4437                         goto tr_stalled;
4438                 }
4439                 break;
4440
4441         case UT_WRITE_CLASS_OTHER:
4442                 switch (req->bRequest) {
4443                 case UR_CLEAR_FEATURE:
4444                         goto tr_handle_clear_port_feature;
4445                 case UR_SET_FEATURE:
4446                         goto tr_handle_set_port_feature;
4447                 case UR_CLEAR_TT_BUFFER:
4448                 case UR_RESET_TT:
4449                 case UR_STOP_TT:
4450                         goto tr_valid;
4451
4452                 default:
4453                         goto tr_stalled;
4454                 }
4455                 break;
4456
4457         case UT_READ_CLASS_OTHER:
4458                 switch (req->bRequest) {
4459                 case UR_GET_TT_STATE:
4460                         goto tr_handle_get_tt_state;
4461                 case UR_GET_STATUS:
4462                         goto tr_handle_get_port_status;
4463                 default:
4464                         goto tr_stalled;
4465                 }
4466                 break;
4467
4468         case UT_READ_CLASS_DEVICE:
4469                 switch (req->bRequest) {
4470                 case UR_GET_DESCRIPTOR:
4471                         goto tr_handle_get_class_descriptor;
4472                 case UR_GET_STATUS:
4473                         goto tr_handle_get_class_status;
4474
4475                 default:
4476                         goto tr_stalled;
4477                 }
4478                 break;
4479         default:
4480                 goto tr_stalled;
4481         }
4482         goto tr_valid;
4483
4484 tr_handle_get_descriptor:
4485         switch (value >> 8) {
4486         case UDESC_DEVICE:
4487                 if (value & 0xff) {
4488                         goto tr_stalled;
4489                 }
4490                 len = sizeof(dwc_otg_devd);
4491                 ptr = (const void *)&dwc_otg_devd;
4492                 goto tr_valid;
4493         case UDESC_CONFIG:
4494                 if (value & 0xff) {
4495                         goto tr_stalled;
4496                 }
4497                 len = sizeof(dwc_otg_confd);
4498                 ptr = (const void *)&dwc_otg_confd;
4499                 goto tr_valid;
4500         case UDESC_STRING:
4501                 switch (value & 0xff) {
4502                 case 0:         /* Language table */
4503                         len = sizeof(usb_string_lang_en);
4504                         ptr = (const void *)&usb_string_lang_en;
4505                         goto tr_valid;
4506
4507                 case 1:         /* Vendor */
4508                         len = sizeof(dwc_otg_vendor);
4509                         ptr = (const void *)&dwc_otg_vendor;
4510                         goto tr_valid;
4511
4512                 case 2:         /* Product */
4513                         len = sizeof(dwc_otg_product);
4514                         ptr = (const void *)&dwc_otg_product;
4515                         goto tr_valid;
4516                 default:
4517                         break;
4518                 }
4519                 break;
4520         default:
4521                 goto tr_stalled;
4522         }
4523         goto tr_stalled;
4524
4525 tr_handle_get_config:
4526         len = 1;
4527         sc->sc_hub_temp.wValue[0] = sc->sc_conf;
4528         goto tr_valid;
4529
4530 tr_handle_get_status:
4531         len = 2;
4532         USETW(sc->sc_hub_temp.wValue, UDS_SELF_POWERED);
4533         goto tr_valid;
4534
4535 tr_handle_set_address:
4536         if (value & 0xFF00) {
4537                 goto tr_stalled;
4538         }
4539         sc->sc_rt_addr = value;
4540         goto tr_valid;
4541
4542 tr_handle_set_config:
4543         if (value >= 2) {
4544                 goto tr_stalled;
4545         }
4546         sc->sc_conf = value;
4547         goto tr_valid;
4548
4549 tr_handle_get_interface:
4550         len = 1;
4551         sc->sc_hub_temp.wValue[0] = 0;
4552         goto tr_valid;
4553
4554 tr_handle_get_tt_state:
4555 tr_handle_get_class_status:
4556 tr_handle_get_iface_status:
4557 tr_handle_get_ep_status:
4558         len = 2;
4559         USETW(sc->sc_hub_temp.wValue, 0);
4560         goto tr_valid;
4561
4562 tr_handle_set_halt:
4563 tr_handle_set_interface:
4564 tr_handle_set_wakeup:
4565 tr_handle_clear_wakeup:
4566 tr_handle_clear_halt:
4567         goto tr_valid;
4568
4569 tr_handle_clear_port_feature:
4570         if (index != 1)
4571                 goto tr_stalled;
4572
4573         DPRINTFN(9, "UR_CLEAR_PORT_FEATURE on port %d\n", index);
4574
4575         switch (value) {
4576         case UHF_PORT_SUSPEND:
4577                 dwc_otg_wakeup_peer(sc);
4578                 break;
4579
4580         case UHF_PORT_ENABLE:
4581                 if (sc->sc_flags.status_device_mode == 0) {
4582                         DWC_OTG_WRITE_4(sc, DOTG_HPRT,
4583                             sc->sc_hprt_val | HPRT_PRTENA);
4584                 }
4585                 sc->sc_flags.port_enabled = 0;
4586                 break;
4587
4588         case UHF_C_PORT_RESET:
4589                 sc->sc_flags.change_reset = 0;
4590                 break;
4591
4592         case UHF_C_PORT_ENABLE:
4593                 sc->sc_flags.change_enabled = 0;
4594                 break;
4595
4596         case UHF_C_PORT_OVER_CURRENT:
4597                 sc->sc_flags.change_over_current = 0;
4598                 break;
4599
4600         case UHF_PORT_TEST:
4601         case UHF_PORT_INDICATOR:
4602                 /* nops */
4603                 break;
4604
4605         case UHF_PORT_POWER:
4606                 sc->sc_flags.port_powered = 0;
4607                 if (sc->sc_mode == DWC_MODE_HOST || sc->sc_mode == DWC_MODE_OTG) {
4608                         sc->sc_hprt_val = 0;
4609                         DWC_OTG_WRITE_4(sc, DOTG_HPRT, HPRT_PRTENA);
4610                 }
4611                 dwc_otg_pull_down(sc);
4612                 dwc_otg_clocks_off(sc);
4613                 break;
4614
4615         case UHF_C_PORT_CONNECTION:
4616                 /* clear connect change flag */
4617                 sc->sc_flags.change_connect = 0;
4618                 break;
4619
4620         case UHF_C_PORT_SUSPEND:
4621                 sc->sc_flags.change_suspend = 0;
4622                 break;
4623
4624         default:
4625                 err = USB_ERR_IOERROR;
4626                 goto done;
4627         }
4628         goto tr_valid;
4629
4630 tr_handle_set_port_feature:
4631         if (index != 1) {
4632                 goto tr_stalled;
4633         }
4634         DPRINTFN(9, "UR_SET_PORT_FEATURE\n");
4635
4636         switch (value) {
4637         case UHF_PORT_ENABLE:
4638                 break;
4639
4640         case UHF_PORT_SUSPEND:
4641                 if (sc->sc_flags.status_device_mode == 0) {
4642                         /* set suspend BIT */
4643                         sc->sc_hprt_val |= HPRT_PRTSUSP;
4644                         DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
4645
4646                         /* generate HUB suspend event */
4647                         dwc_otg_suspend_irq(sc);
4648                 }
4649                 break;
4650
4651         case UHF_PORT_RESET:
4652                 if (sc->sc_flags.status_device_mode == 0) {
4653
4654                         DPRINTF("PORT RESET\n");
4655
4656                         /* enable PORT reset */
4657                         DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val | HPRT_PRTRST);
4658
4659                         /* Wait 62.5ms for reset to complete */
4660                         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 16);
4661
4662                         DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
4663
4664                         /* Wait 62.5ms for reset to complete */
4665                         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 16);
4666
4667                         /* reset FIFOs */
4668                         (void) dwc_otg_init_fifo(sc, DWC_MODE_HOST);
4669
4670                         sc->sc_flags.change_reset = 1;
4671                 } else {
4672                         err = USB_ERR_IOERROR;
4673                 }
4674                 break;
4675
4676         case UHF_PORT_TEST:
4677         case UHF_PORT_INDICATOR:
4678                 /* nops */
4679                 break;
4680         case UHF_PORT_POWER:
4681                 sc->sc_flags.port_powered = 1;
4682                 if (sc->sc_mode == DWC_MODE_HOST || sc->sc_mode == DWC_MODE_OTG) {
4683                         sc->sc_hprt_val |= HPRT_PRTPWR;
4684                         DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
4685                 }
4686                 if (sc->sc_mode == DWC_MODE_DEVICE || sc->sc_mode == DWC_MODE_OTG) {
4687                         /* pull up D+, if any */
4688                         dwc_otg_pull_up(sc);
4689                 }
4690                 break;
4691         default:
4692                 err = USB_ERR_IOERROR;
4693                 goto done;
4694         }
4695         goto tr_valid;
4696
4697 tr_handle_get_port_status:
4698
4699         DPRINTFN(9, "UR_GET_PORT_STATUS\n");
4700
4701         if (index != 1)
4702                 goto tr_stalled;
4703
4704         if (sc->sc_flags.status_vbus)
4705                 dwc_otg_clocks_on(sc);
4706         else
4707                 dwc_otg_clocks_off(sc);
4708
4709         /* Select Device Side Mode */
4710
4711         if (sc->sc_flags.status_device_mode) {
4712                 value = UPS_PORT_MODE_DEVICE;
4713                 dwc_otg_timer_stop(sc);
4714         } else {
4715                 value = 0;
4716                 dwc_otg_timer_start(sc);
4717         }
4718
4719         if (sc->sc_flags.status_high_speed)
4720                 value |= UPS_HIGH_SPEED;
4721         else if (sc->sc_flags.status_low_speed)
4722                 value |= UPS_LOW_SPEED;
4723
4724         if (sc->sc_flags.port_powered)
4725                 value |= UPS_PORT_POWER;
4726
4727         if (sc->sc_flags.port_enabled)
4728                 value |= UPS_PORT_ENABLED;
4729
4730         if (sc->sc_flags.port_over_current)
4731                 value |= UPS_OVERCURRENT_INDICATOR;
4732
4733         if (sc->sc_flags.status_vbus &&
4734             sc->sc_flags.status_bus_reset)
4735                 value |= UPS_CURRENT_CONNECT_STATUS;
4736
4737         if (sc->sc_flags.status_suspend)
4738                 value |= UPS_SUSPEND;
4739
4740         USETW(sc->sc_hub_temp.ps.wPortStatus, value);
4741
4742         value = 0;
4743
4744         if (sc->sc_flags.change_connect)
4745                 value |= UPS_C_CONNECT_STATUS;
4746         if (sc->sc_flags.change_suspend)
4747                 value |= UPS_C_SUSPEND;
4748         if (sc->sc_flags.change_reset)
4749                 value |= UPS_C_PORT_RESET;
4750         if (sc->sc_flags.change_over_current)
4751                 value |= UPS_C_OVERCURRENT_INDICATOR;
4752
4753         USETW(sc->sc_hub_temp.ps.wPortChange, value);
4754         len = sizeof(sc->sc_hub_temp.ps);
4755         goto tr_valid;
4756
4757 tr_handle_get_class_descriptor:
4758         if (value & 0xFF) {
4759                 goto tr_stalled;
4760         }
4761         ptr = (const void *)&dwc_otg_hubd;
4762         len = sizeof(dwc_otg_hubd);
4763         goto tr_valid;
4764
4765 tr_stalled:
4766         err = USB_ERR_STALLED;
4767 tr_valid:
4768 done:
4769         *plength = len;
4770         *pptr = ptr;
4771         return (err);
4772 }
4773
4774 static void
4775 dwc_otg_xfer_setup(struct usb_setup_params *parm)
4776 {
4777         struct usb_xfer *xfer;
4778         void *last_obj;
4779         uint32_t ntd;
4780         uint32_t n;
4781         uint8_t ep_no;
4782         uint8_t ep_type;
4783
4784         xfer = parm->curr_xfer;
4785
4786         /*
4787          * NOTE: This driver does not use any of the parameters that
4788          * are computed from the following values. Just set some
4789          * reasonable dummies:
4790          */
4791         parm->hc_max_packet_size = 0x500;
4792         parm->hc_max_packet_count = 3;
4793         parm->hc_max_frame_size = 3 * 0x500;
4794
4795         usbd_transfer_setup_sub(parm);
4796
4797         /*
4798          * compute maximum number of TDs
4799          */
4800         ep_type = (xfer->endpoint->edesc->bmAttributes & UE_XFERTYPE);
4801
4802         if (ep_type == UE_CONTROL) {
4803
4804                 ntd = xfer->nframes + 1 /* STATUS */ + 1 /* SYNC 1 */
4805                     + 1 /* SYNC 2 */ + 1 /* SYNC 3 */;
4806         } else {
4807
4808                 ntd = xfer->nframes + 1 /* SYNC */ ;
4809         }
4810
4811         /*
4812          * check if "usbd_transfer_setup_sub" set an error
4813          */
4814         if (parm->err)
4815                 return;
4816
4817         /*
4818          * allocate transfer descriptors
4819          */
4820         last_obj = NULL;
4821
4822         ep_no = xfer->endpointno & UE_ADDR;
4823
4824         /*
4825          * Check for a valid endpoint profile in USB device mode:
4826          */
4827         if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) {
4828                 const struct usb_hw_ep_profile *pf;
4829
4830                 dwc_otg_get_hw_ep_profile(parm->udev, &pf, ep_no);
4831
4832                 if (pf == NULL) {
4833                         /* should not happen */
4834                         parm->err = USB_ERR_INVAL;
4835                         return;
4836                 }
4837         }
4838
4839         /* align data */
4840         parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
4841
4842         for (n = 0; n != ntd; n++) {
4843
4844                 struct dwc_otg_td *td;
4845
4846                 if (parm->buf) {
4847
4848                         td = USB_ADD_BYTES(parm->buf, parm->size[0]);
4849
4850                         /* compute shared bandwidth resource index for TT */
4851                         if (dwc_otg_uses_split(parm->udev)) {
4852                                 if (parm->udev->parent_hs_hub->ddesc.bDeviceProtocol == UDPROTO_HSHUBMTT)
4853                                         td->tt_index = parm->udev->device_index;
4854                                 else
4855                                         td->tt_index = parm->udev->parent_hs_hub->device_index;
4856                         } else {
4857                                 td->tt_index = parm->udev->device_index;
4858                         }
4859
4860                         /* init TD */
4861                         td->max_packet_size = xfer->max_packet_size;
4862                         td->max_packet_count = xfer->max_packet_count;
4863                         /* range check */
4864                         if (td->max_packet_count == 0 || td->max_packet_count > 3)
4865                                 td->max_packet_count = 1;
4866                         td->ep_no = ep_no;
4867                         td->ep_type = ep_type;
4868                         td->obj_next = last_obj;
4869
4870                         last_obj = td;
4871                 }
4872                 parm->size[0] += sizeof(*td);
4873         }
4874
4875         xfer->td_start[0] = last_obj;
4876 }
4877
4878 static void
4879 dwc_otg_xfer_unsetup(struct usb_xfer *xfer)
4880 {
4881         return;
4882 }
4883
4884 static void
4885 dwc_otg_ep_init(struct usb_device *udev, struct usb_endpoint_descriptor *edesc,
4886     struct usb_endpoint *ep)
4887 {
4888         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(udev->bus);
4889
4890         DPRINTFN(2, "endpoint=%p, addr=%d, endpt=%d, mode=%d (%d,%d)\n",
4891             ep, udev->address,
4892             edesc->bEndpointAddress, udev->flags.usb_mode,
4893             sc->sc_rt_addr, udev->device_index);
4894
4895         if (udev->device_index != sc->sc_rt_addr) {
4896
4897                 if (udev->flags.usb_mode == USB_MODE_DEVICE) {
4898                         if (udev->speed != USB_SPEED_FULL &&
4899                             udev->speed != USB_SPEED_HIGH) {
4900                                 /* not supported */
4901                                 return;
4902                         }
4903                 } else {
4904                         if (udev->speed == USB_SPEED_HIGH &&
4905                             (edesc->wMaxPacketSize[1] & 0x18) != 0 &&
4906                             (edesc->bmAttributes & UE_XFERTYPE) != UE_ISOCHRONOUS) {
4907                                 /* not supported */
4908                                 DPRINTFN(-1, "Non-isochronous high bandwidth "
4909                                     "endpoint not supported\n");
4910                                 return;
4911                         }
4912                 }
4913                 if ((edesc->bmAttributes & UE_XFERTYPE) == UE_ISOCHRONOUS)
4914                         ep->methods = &dwc_otg_device_isoc_methods;
4915                 else
4916                         ep->methods = &dwc_otg_device_non_isoc_methods;
4917         }
4918 }
4919
4920 static void
4921 dwc_otg_set_hw_power_sleep(struct usb_bus *bus, uint32_t state)
4922 {
4923         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(bus);
4924
4925         switch (state) {
4926         case USB_HW_POWER_SUSPEND:
4927                 dwc_otg_suspend(sc);
4928                 break;
4929         case USB_HW_POWER_SHUTDOWN:
4930                 dwc_otg_uninit(sc);
4931                 break;
4932         case USB_HW_POWER_RESUME:
4933                 dwc_otg_resume(sc);
4934                 break;
4935         default:
4936                 break;
4937         }
4938 }
4939
4940 static void
4941 dwc_otg_get_dma_delay(struct usb_device *udev, uint32_t *pus)
4942 {
4943         /* DMA delay - wait until any use of memory is finished */
4944         *pus = (2125);                  /* microseconds */
4945 }
4946
4947 static void
4948 dwc_otg_device_resume(struct usb_device *udev)
4949 {
4950         DPRINTF("\n");
4951
4952         /* poll all transfers again to restart resumed ones */
4953         dwc_otg_do_poll(udev->bus);
4954 }
4955
4956 static void
4957 dwc_otg_device_suspend(struct usb_device *udev)
4958 {
4959         DPRINTF("\n");
4960 }
4961
4962 struct usb_bus_methods dwc_otg_bus_methods =
4963 {
4964         .endpoint_init = &dwc_otg_ep_init,
4965         .xfer_setup = &dwc_otg_xfer_setup,
4966         .xfer_unsetup = &dwc_otg_xfer_unsetup,
4967         .get_hw_ep_profile = &dwc_otg_get_hw_ep_profile,
4968         .xfer_stall = &dwc_otg_xfer_stall,
4969         .set_stall = &dwc_otg_set_stall,
4970         .clear_stall = &dwc_otg_clear_stall,
4971         .roothub_exec = &dwc_otg_roothub_exec,
4972         .xfer_poll = &dwc_otg_do_poll,
4973         .device_state_change = &dwc_otg_device_state_change,
4974         .set_hw_power_sleep = &dwc_otg_set_hw_power_sleep,
4975         .get_dma_delay = &dwc_otg_get_dma_delay,
4976         .device_resume = &dwc_otg_device_resume,
4977         .device_suspend = &dwc_otg_device_suspend,
4978 };