sys/dev/usb/controller/dwc_otg.c

   1 /* $FreeBSD$ */
   2 /*-
   3  * Copyright (c) 2015 Daisuke Aoyama. All rights reserved.
   4  * Copyright (c) 2012-2015 Hans Petter Selasky. All rights reserved.
   5  * Copyright (c) 2010-2011 Aleksandr Rybalko. All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  26  * SUCH DAMAGE.
  27  */
  28
  29 /*
  30  * This file contains the driver for the DesignWare series USB 2.0 OTG
  31  * Controller.
  32  */
  33
  34 /*
  35  * LIMITATION: Drivers must be bound to all OUT endpoints in the
  36  * active configuration for this driver to work properly. Blocking any
  37  * OUT endpoint will block all OUT endpoints including the control
  38  * endpoint. Usually this is not a problem.
  39  */
  40
  41 /*
  42  * NOTE: Writing to non-existing registers appears to cause an
  43  * internal reset.
  44  */
  45
  46 #ifdef USB_GLOBAL_INCLUDE_FILE
  47 #include USB_GLOBAL_INCLUDE_FILE
  48 #else
  49 #include <sys/stdint.h>
  50 #include <sys/stddef.h>
  51 #include <sys/param.h>
  52 #include <sys/queue.h>
  53 #include <sys/types.h>
  54 #include <sys/systm.h>
  55 #include <sys/kernel.h>
  56 #include <sys/bus.h>
  57 #include <sys/module.h>
  58 #include <sys/lock.h>
  59 #include <sys/mutex.h>
  60 #include <sys/condvar.h>
  61 #include <sys/sysctl.h>
  62 #include <sys/sx.h>
  63 #include <sys/unistd.h>
  64 #include <sys/callout.h>
  65 #include <sys/malloc.h>
  66 #include <sys/priv.h>
  67
  68 #include <dev/usb/usb.h>
  69 #include <dev/usb/usbdi.h>
  70
  71 #define USB_DEBUG_VAR dwc_otg_debug
  72
  73 #include <dev/usb/usb_core.h>
  74 #include <dev/usb/usb_debug.h>
  75 #include <dev/usb/usb_busdma.h>
  76 #include <dev/usb/usb_process.h>
  77 #include <dev/usb/usb_transfer.h>
  78 #include <dev/usb/usb_device.h>
  79 #include <dev/usb/usb_hub.h>
  80 #include <dev/usb/usb_util.h>
  81
  82 #include <dev/usb/usb_controller.h>
  83 #include <dev/usb/usb_bus.h>
  84 #endif                  /* USB_GLOBAL_INCLUDE_FILE */
  85
  86 #include <dev/usb/controller/dwc_otg.h>
  87 #include <dev/usb/controller/dwc_otgreg.h>
  88
  89 #define DWC_OTG_BUS2SC(bus) \
  90    ((struct dwc_otg_softc *)(((uint8_t *)(bus)) - \
  91     ((uint8_t *)&(((struct dwc_otg_softc *)0)->sc_bus))))
  92
  93 #define DWC_OTG_PC2UDEV(pc) \
  94    (USB_DMATAG_TO_XROOT((pc)->tag_parent)->udev)
  95
  96 #define DWC_OTG_MSK_GINT_THREAD_IRQ                             \
  97    (GINTSTS_USBRST | GINTSTS_ENUMDONE | GINTSTS_PRTINT |        \
  98    GINTSTS_WKUPINT | GINTSTS_USBSUSP | GINTMSK_OTGINTMSK |      \
  99    GINTSTS_SESSREQINT)
 100
 101 #ifndef DWC_OTG_PHY_DEFAULT
 102 #define DWC_OTG_PHY_DEFAULT DWC_OTG_PHY_ULPI
 103 #endif
 104
 105 static int dwc_otg_phy_type = DWC_OTG_PHY_DEFAULT;
 106
 107 static SYSCTL_NODE(_hw_usb, OID_AUTO, dwc_otg, CTLFLAG_RW, 0, "USB DWC OTG");
 108 SYSCTL_INT(_hw_usb_dwc_otg, OID_AUTO, phy_type, CTLFLAG_RDTUN,
 109     &dwc_otg_phy_type, 0, "DWC OTG PHY TYPE - 0/1/2/3 - ULPI/HSIC/INTERNAL/UTMI+");
 110
 111 #ifdef USB_DEBUG
 112 static int dwc_otg_debug = 0;
 113
 114 SYSCTL_INT(_hw_usb_dwc_otg, OID_AUTO, debug, CTLFLAG_RWTUN,
 115     &dwc_otg_debug, 0, "DWC OTG debug level");
 116 #endif
 117
 118 #define DWC_OTG_INTR_ENDPT 1
 119
 120 /* prototypes */
 121
 122 static const struct usb_bus_methods dwc_otg_bus_methods;
 123 static const struct usb_pipe_methods dwc_otg_device_non_isoc_methods;
 124 static const struct usb_pipe_methods dwc_otg_device_isoc_methods;
 125
 126 static dwc_otg_cmd_t dwc_otg_setup_rx;
 127 static dwc_otg_cmd_t dwc_otg_data_rx;
 128 static dwc_otg_cmd_t dwc_otg_data_tx;
 129 static dwc_otg_cmd_t dwc_otg_data_tx_sync;
 130
 131 static dwc_otg_cmd_t dwc_otg_host_setup_tx;
 132 static dwc_otg_cmd_t dwc_otg_host_data_tx;
 133 static dwc_otg_cmd_t dwc_otg_host_data_rx;
 134
 135 static void dwc_otg_device_done(struct usb_xfer *, usb_error_t);
 136 static void dwc_otg_do_poll(struct usb_bus *);
 137 static void dwc_otg_standard_done(struct usb_xfer *);
 138 static void dwc_otg_root_intr(struct dwc_otg_softc *);
 139 static void dwc_otg_interrupt_poll_locked(struct dwc_otg_softc *);
 140
 141 /*
 142  * Here is a configuration that the chip supports.
 143  */
 144 static const struct usb_hw_ep_profile dwc_otg_ep_profile[1] = {
 145
 146         [0] = {
 147                 .max_in_frame_size = 64,/* fixed */
 148                 .max_out_frame_size = 64,       /* fixed */
 149                 .is_simplex = 1,
 150                 .support_control = 1,
 151         }
 152 };
 153
 154 static void
 155 dwc_otg_get_hw_ep_profile(struct usb_device *udev,
 156     const struct usb_hw_ep_profile **ppf, uint8_t ep_addr)
 157 {
 158         struct dwc_otg_softc *sc;
 159
 160         sc = DWC_OTG_BUS2SC(udev->bus);
 161
 162         if (ep_addr < sc->sc_dev_ep_max)
 163                 *ppf = &sc->sc_hw_ep_profile[ep_addr].usb;
 164         else
 165                 *ppf = NULL;
 166 }
 167
 168 static void
 169 dwc_otg_write_fifo(struct dwc_otg_softc *sc, struct usb_page_cache *pc,
 170     uint32_t offset, uint32_t fifo, uint32_t count)
 171 {
 172         uint32_t temp;
 173
 174         /* round down length to nearest 4-bytes */
 175         temp = count & ~3;
 176
 177         /* check if we can write the data directly */
 178         if (temp != 0 && usb_pc_buffer_is_aligned(pc, offset, temp, 3)) {
 179                 struct usb_page_search buf_res;
 180
 181                 /* pre-subtract length */
 182                 count -= temp;
 183
 184                 /* iterate buffer list */
 185                 do {
 186                         /* get current buffer pointer */
 187                         usbd_get_page(pc, offset, &buf_res);
 188
 189                         if (buf_res.length > temp)
 190                                 buf_res.length = temp;
 191
 192                         /* transfer data into FIFO */
 193                         bus_space_write_region_4(sc->sc_io_tag, sc->sc_io_hdl,
 194                             fifo, buf_res.buffer, buf_res.length / 4);
 195
 196                         offset += buf_res.length;
 197                         fifo += buf_res.length;
 198                         temp -= buf_res.length;
 199                 } while (temp != 0);
 200         }
 201
 202         /* check for remainder */
 203         if (count != 0) {
 204                 /* clear topmost word before copy */
 205                 sc->sc_bounce_buffer[(count - 1) / 4] = 0;
 206
 207                 /* copy out data */
 208                 usbd_copy_out(pc, offset,
 209                     sc->sc_bounce_buffer, count);
 210
 211                 /* transfer data into FIFO */
 212                 bus_space_write_region_4(sc->sc_io_tag,
 213                     sc->sc_io_hdl, fifo, sc->sc_bounce_buffer,
 214                     (count + 3) / 4);
 215         }
 216 }
 217
 218 static void
 219 dwc_otg_read_fifo(struct dwc_otg_softc *sc, struct usb_page_cache *pc,
 220     uint32_t offset, uint32_t count)
 221 {
 222         uint32_t temp;
 223
 224         /* round down length to nearest 4-bytes */
 225         temp = count & ~3;
 226
 227         /* check if we can read the data directly */
 228         if (temp != 0 && usb_pc_buffer_is_aligned(pc, offset, temp, 3)) {
 229                 struct usb_page_search buf_res;
 230
 231                 /* pre-subtract length */
 232                 count -= temp;
 233
 234                 /* iterate buffer list */
 235                 do {
 236                         /* get current buffer pointer */
 237                         usbd_get_page(pc, offset, &buf_res);
 238
 239                         if (buf_res.length > temp)
 240                                 buf_res.length = temp;
 241
 242                         /* transfer data from FIFO */
 243                         bus_space_read_region_4(sc->sc_io_tag, sc->sc_io_hdl,
 244                             sc->sc_current_rx_fifo, buf_res.buffer, buf_res.length / 4);
 245
 246                         offset += buf_res.length;
 247                         sc->sc_current_rx_fifo += buf_res.length;
 248                         sc->sc_current_rx_bytes -= buf_res.length;
 249                         temp -= buf_res.length;
 250                 } while (temp != 0);
 251         }
 252
 253         /* check for remainder */
 254         if (count != 0) {
 255                 /* read data into bounce buffer */
 256                 bus_space_read_region_4(sc->sc_io_tag, sc->sc_io_hdl,
 257                         sc->sc_current_rx_fifo,
 258                         sc->sc_bounce_buffer, (count + 3) / 4);
 259
 260                 /* store data into proper buffer */
 261                 usbd_copy_in(pc, offset, sc->sc_bounce_buffer, count);
 262
 263                 /* round length up to nearest 4 bytes */
 264                 count = (count + 3) & ~3;
 265
 266                 /* update counters */
 267                 sc->sc_current_rx_bytes -= count;
 268                 sc->sc_current_rx_fifo += count;
 269         }
 270 }
 271
 272 static void
 273 dwc_otg_tx_fifo_reset(struct dwc_otg_softc *sc, uint32_t value)
 274 {
 275         uint32_t temp;
 276
 277         /* reset FIFO */
 278         DWC_OTG_WRITE_4(sc, DOTG_GRSTCTL, value);
 279
 280         /* wait for reset to complete */
 281         for (temp = 0; temp != 16; temp++) {
 282                 value = DWC_OTG_READ_4(sc, DOTG_GRSTCTL);
 283                 if (!(value & (GRSTCTL_TXFFLSH | GRSTCTL_RXFFLSH)))
 284                         break;
 285         }
 286 }
 287
 288 static int
 289 dwc_otg_init_fifo(struct dwc_otg_softc *sc, uint8_t mode)
 290 {
 291         struct dwc_otg_profile *pf;
 292         uint32_t fifo_size;
 293         uint32_t fifo_regs;
 294         uint32_t tx_start;
 295         uint8_t x;
 296
 297         fifo_size = sc->sc_fifo_size;
 298
 299         /*
 300          * NOTE: Reserved fixed size area at end of RAM, which must
 301          * not be allocated to the FIFOs:
 302          */
 303         fifo_regs = 4 * 16;
 304
 305         if (fifo_size < fifo_regs) {
 306                 DPRINTF("Too little FIFO\n");
 307                 return (EINVAL);
 308         }
 309
 310         /* subtract FIFO regs from total once */
 311         fifo_size -= fifo_regs;
 312
 313         /* split equally for IN and OUT */
 314         fifo_size /= 2;
 315
 316         /* Align to 4 bytes boundary (refer to PGM) */
 317         fifo_size &= ~3;
 318
 319         /* set global receive FIFO size */
 320         DWC_OTG_WRITE_4(sc, DOTG_GRXFSIZ, fifo_size / 4);
 321
 322         tx_start = fifo_size;
 323
 324         if (fifo_size < 64) {
 325                 DPRINTFN(-1, "Not enough data space for EP0 FIFO.\n");
 326                 return (EINVAL);
 327         }
 328
 329         if (mode == DWC_MODE_HOST) {
 330
 331                 /* reset active endpoints */
 332                 sc->sc_active_rx_ep = 0;
 333
 334                 /* split equally for periodic and non-periodic */
 335                 fifo_size /= 2;
 336
 337                 DPRINTF("PTX/NPTX FIFO=%u\n", fifo_size);
 338
 339                 /* align to 4 bytes boundary */
 340                 fifo_size &= ~3;
 341
 342                 DWC_OTG_WRITE_4(sc, DOTG_GNPTXFSIZ,
 343                     ((fifo_size / 4) << 16) |
 344                     (tx_start / 4));
 345
 346                 tx_start += fifo_size;
 347
 348                 for (x = 0; x != sc->sc_host_ch_max; x++) {
 349                         /* enable all host interrupts */
 350                         DWC_OTG_WRITE_4(sc, DOTG_HCINTMSK(x),
 351                             HCINT_DEFAULT_MASK);
 352                 }
 353
 354                 DWC_OTG_WRITE_4(sc, DOTG_HPTXFSIZ,
 355                     ((fifo_size / 4) << 16) |
 356                     (tx_start / 4));
 357
 358                 /* reset host channel state */
 359                 memset(sc->sc_chan_state, 0, sizeof(sc->sc_chan_state));
 360
 361                 /* enable all host channel interrupts */
 362                 DWC_OTG_WRITE_4(sc, DOTG_HAINTMSK,
 363                     (1U << sc->sc_host_ch_max) - 1U);
 364
 365                 /* enable proper host channel interrupts */
 366                 sc->sc_irq_mask |= GINTMSK_HCHINTMSK;
 367                 sc->sc_irq_mask &= ~GINTMSK_IEPINTMSK;
 368                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 369         }
 370
 371         if (mode == DWC_MODE_DEVICE) {
 372
 373             DWC_OTG_WRITE_4(sc, DOTG_GNPTXFSIZ,
 374                 (0x10 << 16) | (tx_start / 4));
 375             fifo_size -= 0x40;
 376             tx_start += 0x40;
 377
 378             /* setup control endpoint profile */
 379             sc->sc_hw_ep_profile[0].usb = dwc_otg_ep_profile[0];
 380
 381             /* reset active endpoints */
 382             sc->sc_active_rx_ep = 1;
 383
 384             for (x = 1; x != sc->sc_dev_ep_max; x++) {
 385
 386                 pf = sc->sc_hw_ep_profile + x;
 387
 388                 pf->usb.max_out_frame_size = 1024 * 3;
 389                 pf->usb.is_simplex = 0; /* assume duplex */
 390                 pf->usb.support_bulk = 1;
 391                 pf->usb.support_interrupt = 1;
 392                 pf->usb.support_isochronous = 1;
 393                 pf->usb.support_out = 1;
 394
 395                 if (x < sc->sc_dev_in_ep_max) {
 396                         uint32_t limit;
 397
 398                         limit = (x == 1) ? MIN(DWC_OTG_TX_MAX_FIFO_SIZE,
 399                             DWC_OTG_MAX_TXN) : MIN(DWC_OTG_MAX_TXN / 2,
 400                             DWC_OTG_TX_MAX_FIFO_SIZE);
 401
 402                         /* see if there is enough FIFO space */
 403                         if (limit <= fifo_size) {
 404                                 pf->max_buffer = limit;
 405                                 pf->usb.support_in = 1;
 406                         } else {
 407                                 limit = MIN(DWC_OTG_TX_MAX_FIFO_SIZE, 0x40);
 408                                 if (limit <= fifo_size) {
 409                                         pf->usb.support_in = 1;
 410                                 } else {
 411                                         pf->usb.is_simplex = 1;
 412                                         limit = 0;
 413                                 }
 414                         }
 415                         /* set FIFO size */
 416                         DWC_OTG_WRITE_4(sc, DOTG_DIEPTXF(x),
 417                             ((limit / 4) << 16) | (tx_start / 4));
 418                         tx_start += limit;
 419                         fifo_size -= limit;
 420                         pf->usb.max_in_frame_size = limit;
 421                 } else {
 422                         pf->usb.is_simplex = 1;
 423                 }
 424
 425                 DPRINTF("FIFO%d = IN:%d / OUT:%d\n", x,
 426                     pf->usb.max_in_frame_size,
 427                     pf->usb.max_out_frame_size);
 428             }
 429
 430             /* enable proper device channel interrupts */
 431             sc->sc_irq_mask &= ~GINTMSK_HCHINTMSK;
 432             sc->sc_irq_mask |= GINTMSK_IEPINTMSK;
 433             DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 434         }
 435
 436         /* reset RX FIFO */
 437         dwc_otg_tx_fifo_reset(sc, GRSTCTL_RXFFLSH);
 438
 439         if (mode != DWC_MODE_OTG) {
 440                 /* reset all TX FIFOs */
 441                 dwc_otg_tx_fifo_reset(sc,
 442                     GRSTCTL_TXFIFO(0x10) |
 443                     GRSTCTL_TXFFLSH);
 444         } else {
 445                 /* reset active endpoints */
 446                 sc->sc_active_rx_ep = 0;
 447
 448                 /* reset host channel state */
 449                 memset(sc->sc_chan_state, 0, sizeof(sc->sc_chan_state));
 450         }
 451         return (0);
 452 }
 453
 454 static uint8_t
 455 dwc_otg_uses_split(struct usb_device *udev)
 456 {
 457         /*
 458          * When a LOW or FULL speed device is connected directly to
 459          * the USB port we don't use split transactions:
 460          */
 461         return (udev->speed != USB_SPEED_HIGH &&
 462             udev->parent_hs_hub != NULL &&
 463             udev->parent_hs_hub->parent_hub != NULL);
 464 }
 465
 466 static void
 467 dwc_otg_update_host_frame_interval(struct dwc_otg_softc *sc)
 468 {
 469
 470   /*
 471    * Disabled until further. Assuming that the register is already
 472    * programmed correctly by the boot loader.
 473    */
 474 #if 0
 475         uint32_t temp;
 476
 477         /* setup HOST frame interval register, based on existing value */
 478         temp = DWC_OTG_READ_4(sc, DOTG_HFIR) & HFIR_FRINT_MASK;
 479         if (temp >= 10000)
 480                 temp /= 1000;
 481         else
 482                 temp /= 125;
 483
 484         /* figure out nearest X-tal value */
 485         if (temp >= 54)
 486                 temp = 60;      /* MHz */
 487         else if (temp >= 39)
 488                 temp = 48;      /* MHz */
 489         else
 490                 temp = 30;      /* MHz */
 491
 492         if (sc->sc_flags.status_high_speed)
 493                 temp *= 125;
 494         else
 495                 temp *= 1000;
 496
 497         DPRINTF("HFIR=0x%08x\n", temp);
 498
 499         DWC_OTG_WRITE_4(sc, DOTG_HFIR, temp);
 500 #endif
 501 }
 502
 503 static void
 504 dwc_otg_clocks_on(struct dwc_otg_softc *sc)
 505 {
 506         if (sc->sc_flags.clocks_off &&
 507             sc->sc_flags.port_powered) {
 508
 509                 DPRINTFN(5, "\n");
 510
 511                 /* TODO - platform specific */
 512
 513                 sc->sc_flags.clocks_off = 0;
 514         }
 515 }
 516
 517 static void
 518 dwc_otg_clocks_off(struct dwc_otg_softc *sc)
 519 {
 520         if (!sc->sc_flags.clocks_off) {
 521
 522                 DPRINTFN(5, "\n");
 523
 524                 /* TODO - platform specific */
 525
 526                 sc->sc_flags.clocks_off = 1;
 527         }
 528 }
 529
 530 static void
 531 dwc_otg_pull_up(struct dwc_otg_softc *sc)
 532 {
 533         uint32_t temp;
 534
 535         /* pullup D+, if possible */
 536
 537         if (!sc->sc_flags.d_pulled_up &&
 538             sc->sc_flags.port_powered) {
 539                 sc->sc_flags.d_pulled_up = 1;
 540
 541                 temp = DWC_OTG_READ_4(sc, DOTG_DCTL);
 542                 temp &= ~DCTL_SFTDISCON;
 543                 DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
 544         }
 545 }
 546
 547 static void
 548 dwc_otg_pull_down(struct dwc_otg_softc *sc)
 549 {
 550         uint32_t temp;
 551
 552         /* pulldown D+, if possible */
 553
 554         if (sc->sc_flags.d_pulled_up) {
 555                 sc->sc_flags.d_pulled_up = 0;
 556
 557                 temp = DWC_OTG_READ_4(sc, DOTG_DCTL);
 558                 temp |= DCTL_SFTDISCON;
 559                 DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
 560         }
 561 }
 562
 563 static void
 564 dwc_otg_enable_sof_irq(struct dwc_otg_softc *sc)
 565 {
 566         /* In device mode we don't use the SOF interrupt */
 567         if (sc->sc_flags.status_device_mode != 0)
 568                 return;
 569         /* Ensure the SOF interrupt is not disabled */
 570         sc->sc_needsof = 1;
 571         /* Check if the SOF interrupt is already enabled */
 572         if ((sc->sc_irq_mask & GINTMSK_SOFMSK) != 0)
 573                 return;
 574         sc->sc_irq_mask |= GINTMSK_SOFMSK;
 575         DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 576 }
 577
 578 static void
 579 dwc_otg_resume_irq(struct dwc_otg_softc *sc)
 580 {
 581         if (sc->sc_flags.status_suspend) {
 582                 /* update status bits */
 583                 sc->sc_flags.status_suspend = 0;
 584                 sc->sc_flags.change_suspend = 1;
 585
 586                 if (sc->sc_flags.status_device_mode) {
 587                         /*
 588                          * Disable resume interrupt and enable suspend
 589                          * interrupt:
 590                          */
 591                         sc->sc_irq_mask &= ~GINTMSK_WKUPINTMSK;
 592                         sc->sc_irq_mask |= GINTMSK_USBSUSPMSK;
 593                         DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 594                 }
 595
 596                 /* complete root HUB interrupt endpoint */
 597                 dwc_otg_root_intr(sc);
 598         }
 599 }
 600
 601 static void
 602 dwc_otg_suspend_irq(struct dwc_otg_softc *sc)
 603 {
 604         if (!sc->sc_flags.status_suspend) {
 605                 /* update status bits */
 606                 sc->sc_flags.status_suspend = 1;
 607                 sc->sc_flags.change_suspend = 1;
 608
 609                 if (sc->sc_flags.status_device_mode) {
 610                         /*
 611                          * Disable suspend interrupt and enable resume
 612                          * interrupt:
 613                          */
 614                         sc->sc_irq_mask &= ~GINTMSK_USBSUSPMSK;
 615                         sc->sc_irq_mask |= GINTMSK_WKUPINTMSK;
 616                         DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 617                 }
 618
 619                 /* complete root HUB interrupt endpoint */
 620                 dwc_otg_root_intr(sc);
 621         }
 622 }
 623
 624 static void
 625 dwc_otg_wakeup_peer(struct dwc_otg_softc *sc)
 626 {
 627         if (!sc->sc_flags.status_suspend)
 628                 return;
 629
 630         DPRINTFN(5, "Remote wakeup\n");
 631
 632         if (sc->sc_flags.status_device_mode) {
 633                 uint32_t temp;
 634
 635                 /* enable remote wakeup signalling */
 636                 temp = DWC_OTG_READ_4(sc, DOTG_DCTL);
 637                 temp |= DCTL_RMTWKUPSIG;
 638                 DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
 639
 640                 /* Wait 8ms for remote wakeup to complete. */
 641                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 125);
 642
 643                 temp &= ~DCTL_RMTWKUPSIG;
 644                 DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
 645         } else {
 646                 /* enable USB port */
 647                 DWC_OTG_WRITE_4(sc, DOTG_PCGCCTL, 0);
 648
 649                 /* wait 10ms */
 650                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 100);
 651
 652                 /* resume port */
 653                 sc->sc_hprt_val |= HPRT_PRTRES;
 654                 DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
 655
 656                 /* Wait 100ms for resume signalling to complete. */
 657                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 10);
 658
 659                 /* clear suspend and resume */
 660                 sc->sc_hprt_val &= ~(HPRT_PRTSUSP | HPRT_PRTRES);
 661                 DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
 662
 663                 /* Wait 4ms */
 664                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 250);
 665         }
 666
 667         /* need to fake resume IRQ */
 668         dwc_otg_resume_irq(sc);
 669 }
 670
 671 static void
 672 dwc_otg_set_address(struct dwc_otg_softc *sc, uint8_t addr)
 673 {
 674         uint32_t temp;
 675
 676         DPRINTFN(5, "addr=%d\n", addr);
 677
 678         temp = DWC_OTG_READ_4(sc, DOTG_DCFG);
 679         temp &= ~DCFG_DEVADDR_SET(0x7F);
 680         temp |= DCFG_DEVADDR_SET(addr);
 681         DWC_OTG_WRITE_4(sc, DOTG_DCFG, temp);
 682 }
 683
 684 static void
 685 dwc_otg_common_rx_ack(struct dwc_otg_softc *sc)
 686 {
 687         DPRINTFN(5, "RX status clear\n");
 688
 689         /* enable RX FIFO level interrupt */
 690         sc->sc_irq_mask |= GINTMSK_RXFLVLMSK;
 691         DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 692
 693         if (sc->sc_current_rx_bytes != 0) {
 694                 /* need to dump remaining data */
 695                 bus_space_read_region_4(sc->sc_io_tag, sc->sc_io_hdl,
 696                     sc->sc_current_rx_fifo, sc->sc_bounce_buffer,
 697                     sc->sc_current_rx_bytes / 4);
 698                 /* clear number of active bytes to receive */
 699                 sc->sc_current_rx_bytes = 0;
 700         }
 701         /* clear cached status */
 702         sc->sc_last_rx_status = 0;
 703 }
 704
 705 static void
 706 dwc_otg_clear_hcint(struct dwc_otg_softc *sc, uint8_t x)
 707 {
 708         uint32_t hcint;
 709
 710         /* clear all pending interrupts */
 711         hcint = DWC_OTG_READ_4(sc, DOTG_HCINT(x));
 712         DWC_OTG_WRITE_4(sc, DOTG_HCINT(x), hcint);
 713
 714         /* clear buffered interrupts */
 715         sc->sc_chan_state[x].hcint = 0;
 716 }
 717
 718 static uint8_t
 719 dwc_otg_host_check_tx_fifo_empty(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
 720 {
 721         uint32_t temp;
 722
 723         temp = DWC_OTG_READ_4(sc, DOTG_GINTSTS);
 724
 725         if (td->ep_type == UE_ISOCHRONOUS) {
 726                 /*
 727                  * NOTE: USB INTERRUPT transactions are executed like
 728                  * USB CONTROL transactions! See the setup standard
 729                  * chain function for more information.
 730                  */
 731                 if (!(temp & GINTSTS_PTXFEMP)) {
 732                         DPRINTF("Periodic TX FIFO is not empty\n");
 733                         if (!(sc->sc_irq_mask & GINTMSK_PTXFEMPMSK)) {
 734                                 sc->sc_irq_mask |= GINTMSK_PTXFEMPMSK;
 735                                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 736                         }
 737                         return (1);     /* busy */
 738                 }
 739         } else {
 740                 if (!(temp & GINTSTS_NPTXFEMP)) {
 741                         DPRINTF("Non-periodic TX FIFO is not empty\n");
 742                         if (!(sc->sc_irq_mask & GINTMSK_NPTXFEMPMSK)) {
 743                                 sc->sc_irq_mask |= GINTMSK_NPTXFEMPMSK;
 744                                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 745                         }
 746                         return (1);     /* busy */
 747                 }
 748         }
 749         return (0);     /* ready for transmit */
 750 }
 751
 752 static uint8_t
 753 dwc_otg_host_channel_alloc(struct dwc_otg_softc *sc,
 754     struct dwc_otg_td *td, uint8_t is_out)
 755 {
 756         uint8_t x;
 757         uint8_t y;
 758         uint8_t z;
 759
 760         if (td->channel[0] < DWC_OTG_MAX_CHANNELS)
 761                 return (0);             /* already allocated */
 762
 763         /* check if device is suspended */
 764         if (DWC_OTG_PC2UDEV(td->pc)->flags.self_suspended != 0)
 765                 return (1);             /* busy - cannot transfer data */
 766
 767         /* compute needed TX FIFO size */
 768         if (is_out != 0) {
 769                 if (dwc_otg_host_check_tx_fifo_empty(sc, td) != 0)
 770                         return (1);     /* busy - cannot transfer data */
 771         }
 772         z = td->max_packet_count;
 773         for (x = y = 0; x != sc->sc_host_ch_max; x++) {
 774                 /* check if channel is allocated */
 775                 if (sc->sc_chan_state[x].allocated != 0)
 776                         continue;
 777                 /* check if channel is still enabled */
 778                 if (sc->sc_chan_state[x].wait_halted != 0)
 779                         continue;
 780                 /* store channel number */
 781                 td->channel[y++] = x;
 782                 /* check if we got all channels */
 783                 if (y == z)
 784                         break;
 785         }
 786         if (y != z) {
 787                 /* reset channel variable */
 788                 td->channel[0] = DWC_OTG_MAX_CHANNELS;
 789                 td->channel[1] = DWC_OTG_MAX_CHANNELS;
 790                 td->channel[2] = DWC_OTG_MAX_CHANNELS;
 791                 /* wait a bit */
 792                 dwc_otg_enable_sof_irq(sc);
 793                 return (1);     /* busy - not enough channels */
 794         }
 795
 796         for (y = 0; y != z; y++) {
 797                 x = td->channel[y];
 798
 799                 /* set allocated */
 800                 sc->sc_chan_state[x].allocated = 1;
 801
 802                 /* set wait halted */
 803                 sc->sc_chan_state[x].wait_halted = 1;
 804
 805                 /* clear interrupts */
 806                 dwc_otg_clear_hcint(sc, x);
 807
 808                 DPRINTF("CH=%d HCCHAR=0x%08x "
 809                     "HCSPLT=0x%08x\n", x, td->hcchar, td->hcsplt);
 810
 811                 /* set active channel */
 812                 sc->sc_active_rx_ep |= (1 << x);
 813         }
 814         return (0);     /* allocated */
 815 }
 816
 817 static void
 818 dwc_otg_host_channel_free_sub(struct dwc_otg_softc *sc, struct dwc_otg_td *td, uint8_t index)
 819 {
 820         uint32_t hcchar;
 821         uint8_t x;
 822
 823         if (td->channel[index] >= DWC_OTG_MAX_CHANNELS)
 824                 return;         /* already freed */
 825
 826         /* free channel */
 827         x = td->channel[index];
 828         td->channel[index] = DWC_OTG_MAX_CHANNELS;
 829
 830         DPRINTF("CH=%d\n", x);
 831
 832         /*
 833          * We need to let programmed host channels run till complete
 834          * else the host channel will stop functioning.
 835          */
 836         sc->sc_chan_state[x].allocated = 0;
 837
 838         /* ack any pending messages */
 839         if (sc->sc_last_rx_status != 0 &&
 840             GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) == x) {
 841                 dwc_otg_common_rx_ack(sc);
 842         }
 843
 844         /* clear active channel */
 845         sc->sc_active_rx_ep &= ~(1 << x);
 846
 847         /* check if already halted */
 848         if (sc->sc_chan_state[x].wait_halted == 0)
 849                 return;
 850
 851         /* disable host channel */
 852         hcchar = DWC_OTG_READ_4(sc, DOTG_HCCHAR(x));
 853         if (hcchar & HCCHAR_CHENA) {
 854                 DPRINTF("Halting channel %d\n", x);
 855                 DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(x),
 856                     hcchar | HCCHAR_CHDIS);
 857                 /* don't write HCCHAR until the channel is halted */
 858         } else {
 859                 sc->sc_chan_state[x].wait_halted = 0;
 860         }
 861 }
 862
 863 static void
 864 dwc_otg_host_channel_free(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
 865 {
 866         uint8_t x;
 867         for (x = 0; x != td->max_packet_count; x++)
 868                 dwc_otg_host_channel_free_sub(sc, td, x);
 869 }
 870
 871 static void
 872 dwc_otg_host_dump_rx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
 873 {
 874         uint8_t x;
 875         /* dump any pending messages */
 876         if (sc->sc_last_rx_status == 0)
 877                 return;
 878         for (x = 0; x != td->max_packet_count; x++) {
 879                 if (td->channel[x] >= DWC_OTG_MAX_CHANNELS ||
 880                     td->channel[x] != GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status))
 881                         continue;
 882                 dwc_otg_common_rx_ack(sc);
 883                 break;
 884         }
 885 }
 886
 887 static uint8_t
 888 dwc_otg_host_setup_tx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
 889 {
 890         struct usb_device_request req __aligned(4);
 891         uint32_t hcint;
 892         uint32_t hcchar;
 893         uint8_t delta;
 894
 895         dwc_otg_host_dump_rx(sc, td);
 896
 897         if (td->channel[0] < DWC_OTG_MAX_CHANNELS) {
 898                 hcint = sc->sc_chan_state[td->channel[0]].hcint;
 899
 900                 DPRINTF("CH=%d ST=%d HCINT=0x%08x HCCHAR=0x%08x HCTSIZ=0x%08x\n",
 901                     td->channel[0], td->state, hcint,
 902                     DWC_OTG_READ_4(sc, DOTG_HCCHAR(td->channel[0])),
 903                     DWC_OTG_READ_4(sc, DOTG_HCTSIZ(td->channel[0])));
 904         } else {
 905                 hcint = 0;
 906                 goto check_state;
 907         }
 908
 909         if (hcint & (HCINT_RETRY |
 910             HCINT_ACK | HCINT_NYET)) {
 911                 /* give success bits priority over failure bits */
 912         } else if (hcint & HCINT_STALL) {
 913                 DPRINTF("CH=%d STALL\n", td->channel[0]);
 914                 td->error_stall = 1;
 915                 td->error_any = 1;
 916                 goto complete;
 917         } else if (hcint & HCINT_ERRORS) {
 918                 DPRINTF("CH=%d ERROR\n", td->channel[0]);
 919                 td->errcnt++;
 920                 if (td->hcsplt != 0 || td->errcnt >= 3) {
 921                         td->error_any = 1;
 922                         goto complete;
 923                 }
 924         }
 925
 926         if (hcint & (HCINT_ERRORS | HCINT_RETRY |
 927             HCINT_ACK | HCINT_NYET)) {
 928                 if (!(hcint & HCINT_ERRORS))
 929                         td->errcnt = 0;
 930         }
 931
 932 check_state:
 933         switch (td->state) {
 934         case DWC_CHAN_ST_START:
 935                 goto send_pkt;
 936
 937         case DWC_CHAN_ST_WAIT_ANE:
 938                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
 939                         td->did_nak = 1;
 940                         td->tt_scheduled = 0;
 941                         goto send_pkt;
 942                 } else if (hcint & (HCINT_ACK | HCINT_NYET)) {
 943                         td->offset += td->tx_bytes;
 944                         td->remainder -= td->tx_bytes;
 945                         td->toggle = 1;
 946                         td->tt_scheduled = 0;
 947                         goto complete;
 948                 }
 949                 break;
 950
 951         case DWC_CHAN_ST_WAIT_S_ANE:
 952                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
 953                         td->did_nak = 1;
 954                         td->tt_scheduled = 0;
 955                         goto send_pkt;
 956                 } else if (hcint & (HCINT_ACK | HCINT_NYET)) {
 957                         goto send_cpkt;
 958                 }
 959                 break;
 960
 961         case DWC_CHAN_ST_WAIT_C_ANE:
 962                 if (hcint & HCINT_NYET) {
 963                         goto send_cpkt;
 964                 } else if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
 965                         td->did_nak = 1;
 966                         td->tt_scheduled = 0;
 967                         goto send_pkt;
 968                 } else if (hcint & HCINT_ACK) {
 969                         td->offset += td->tx_bytes;
 970                         td->remainder -= td->tx_bytes;
 971                         td->toggle = 1;
 972                         goto complete;
 973                 }
 974                 break;
 975
 976         case DWC_CHAN_ST_WAIT_C_PKT:
 977                 goto send_cpkt;
 978
 979         default:
 980                 break;
 981         }
 982         goto busy;
 983
 984 send_pkt:
 985         /* free existing channel, if any */
 986         dwc_otg_host_channel_free(sc, td);
 987
 988         if (sizeof(req) != td->remainder) {
 989                 td->error_any = 1;
 990                 goto complete;
 991         }
 992
 993         if (td->hcsplt != 0) {
 994                 delta = td->tt_start_slot - sc->sc_last_frame_num - 1;
 995                 if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
 996                         td->state = DWC_CHAN_ST_START;
 997                         goto busy;
 998                 }
 999                 delta = sc->sc_last_frame_num - td->tt_start_slot;
1000                 if (delta > 5) {
1001                         /* missed it */
1002                         td->tt_scheduled = 0;
1003                         td->state = DWC_CHAN_ST_START;
1004                         goto busy;
1005                 }
1006         }
1007
1008         /* allocate a new channel */
1009         if (dwc_otg_host_channel_alloc(sc, td, 1)) {
1010                 td->state = DWC_CHAN_ST_START;
1011                 goto busy;
1012         }
1013
1014         if (td->hcsplt != 0) {
1015                 td->hcsplt &= ~HCSPLT_COMPSPLT;
1016                 td->state = DWC_CHAN_ST_WAIT_S_ANE;
1017         } else {
1018                 td->state = DWC_CHAN_ST_WAIT_ANE;
1019         }
1020
1021         /* copy out control request */
1022         usbd_copy_out(td->pc, 0, &req, sizeof(req));
1023
1024         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(td->channel[0]),
1025             (sizeof(req) << HCTSIZ_XFERSIZE_SHIFT) |
1026             (1 << HCTSIZ_PKTCNT_SHIFT) |
1027             (HCTSIZ_PID_SETUP << HCTSIZ_PID_SHIFT));
1028
1029         DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(td->channel[0]), td->hcsplt);
1030
1031         hcchar = td->hcchar;
1032         hcchar &= ~(HCCHAR_EPDIR_IN | HCCHAR_EPTYPE_MASK);
1033         hcchar |= UE_CONTROL << HCCHAR_EPTYPE_SHIFT;
1034
1035         /* must enable channel before writing data to FIFO */
1036         DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(td->channel[0]), hcchar);
1037
1038         /* transfer data into FIFO */
1039         bus_space_write_region_4(sc->sc_io_tag, sc->sc_io_hdl,
1040             DOTG_DFIFO(td->channel[0]), (uint32_t *)&req, sizeof(req) / 4);
1041
1042         /* wait until next slot before trying complete split */
1043         td->tt_complete_slot = sc->sc_last_frame_num + 1;
1044
1045         /* store number of bytes transmitted */
1046         td->tx_bytes = sizeof(req);
1047         goto busy;
1048
1049 send_cpkt:
1050         /* free existing channel, if any */
1051         dwc_otg_host_channel_free(sc, td);
1052
1053         delta = td->tt_complete_slot - sc->sc_last_frame_num - 1;
1054         if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
1055                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
1056                 goto busy;
1057         }
1058         delta = sc->sc_last_frame_num - td->tt_start_slot;
1059         if (delta > DWC_OTG_TT_SLOT_MAX) {
1060                 /* we missed the service interval */
1061                 if (td->ep_type != UE_ISOCHRONOUS)
1062                         td->error_any = 1;
1063                 goto complete;
1064         }
1065         /* allocate a new channel */
1066         if (dwc_otg_host_channel_alloc(sc, td, 0)) {
1067                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
1068                 goto busy;
1069         }
1070
1071         /* wait until next slot before trying complete split */
1072         td->tt_complete_slot = sc->sc_last_frame_num + 1;
1073
1074         td->hcsplt |= HCSPLT_COMPSPLT;
1075         td->state = DWC_CHAN_ST_WAIT_C_ANE;
1076
1077         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(td->channel[0]),
1078             (HCTSIZ_PID_SETUP << HCTSIZ_PID_SHIFT));
1079
1080         DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(td->channel[0]), td->hcsplt);
1081
1082         hcchar = td->hcchar;
1083         hcchar &= ~(HCCHAR_EPDIR_IN | HCCHAR_EPTYPE_MASK);
1084         hcchar |= UE_CONTROL << HCCHAR_EPTYPE_SHIFT;
1085
1086         /* must enable channel before writing data to FIFO */
1087         DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(td->channel[0]), hcchar);
1088
1089 busy:
1090         return (1);     /* busy */
1091
1092 complete:
1093         dwc_otg_host_channel_free(sc, td);
1094         return (0);     /* complete */
1095 }
1096
1097 static uint8_t
1098 dwc_otg_setup_rx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1099 {
1100         struct usb_device_request req __aligned(4);
1101         uint32_t temp;
1102         uint16_t count;
1103
1104         /* check endpoint status */
1105
1106         if (sc->sc_last_rx_status == 0)
1107                 goto not_complete;
1108
1109         if (GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) != 0)
1110                 goto not_complete;
1111
1112         if ((sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) !=
1113             GRXSTSRD_STP_DATA) {
1114                 if ((sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) !=
1115                     GRXSTSRD_STP_COMPLETE || td->remainder != 0) {
1116                         /* release FIFO */
1117                         dwc_otg_common_rx_ack(sc);
1118                         goto not_complete;
1119                 }
1120                 /* release FIFO */
1121                 dwc_otg_common_rx_ack(sc);
1122                 return (0);     /* complete */
1123         }
1124
1125         if ((sc->sc_last_rx_status & GRXSTSRD_DPID_MASK) !=
1126             GRXSTSRD_DPID_DATA0) {
1127                 /* release FIFO */
1128                 dwc_otg_common_rx_ack(sc);
1129                 goto not_complete;
1130         }
1131
1132         DPRINTFN(5, "GRXSTSR=0x%08x\n", sc->sc_last_rx_status);
1133
1134         /* clear did stall */
1135         td->did_stall = 0;
1136
1137         /* get the packet byte count */
1138         count = GRXSTSRD_BCNT_GET(sc->sc_last_rx_status);
1139
1140         if (count != sizeof(req)) {
1141                 DPRINTFN(0, "Unsupported SETUP packet "
1142                     "length, %d bytes\n", count);
1143                 /* release FIFO */
1144                 dwc_otg_common_rx_ack(sc);
1145                 goto not_complete;
1146         }
1147
1148         /* read FIFO */
1149         dwc_otg_read_fifo(sc, td->pc, 0, sizeof(req));
1150
1151         /* copy out control request */
1152         usbd_copy_out(td->pc, 0, &req, sizeof(req));
1153
1154         td->offset = sizeof(req);
1155         td->remainder = 0;
1156
1157         /* sneak peek the set address */
1158         if ((req.bmRequestType == UT_WRITE_DEVICE) &&
1159             (req.bRequest == UR_SET_ADDRESS)) {
1160                 /* must write address before ZLP */
1161                 dwc_otg_set_address(sc, req.wValue[0] & 0x7F);
1162         }
1163
1164         /* don't send any data by default */
1165         DWC_OTG_WRITE_4(sc, DOTG_DIEPTSIZ(0), DIEPCTL_EPDIS);
1166         DWC_OTG_WRITE_4(sc, DOTG_DOEPTSIZ(0), DOEPCTL_EPDIS);
1167
1168         /* reset IN endpoint buffer */
1169         dwc_otg_tx_fifo_reset(sc,
1170             GRSTCTL_TXFIFO(0) |
1171             GRSTCTL_TXFFLSH);
1172
1173         /* acknowledge RX status */
1174         dwc_otg_common_rx_ack(sc);
1175         td->did_stall = 1;
1176
1177 not_complete:
1178         /* abort any ongoing transfer, before enabling again */
1179         if (!td->did_stall) {
1180                 td->did_stall = 1;
1181
1182                 DPRINTFN(5, "stalling IN and OUT direction\n");
1183
1184                 temp = sc->sc_out_ctl[0];
1185
1186                 /* set stall after enabling endpoint */
1187                 DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(0),
1188                     temp | DOEPCTL_STALL);
1189
1190                 temp = sc->sc_in_ctl[0];
1191
1192                 /* set stall assuming endpoint is enabled */
1193                 DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(0),
1194                     temp | DIEPCTL_STALL);
1195         }
1196         return (1);                     /* not complete */
1197 }
1198
1199 static uint8_t
1200 dwc_otg_host_rate_check_interrupt(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1201 {
1202         uint8_t delta;
1203
1204         delta = sc->sc_tmr_val - td->tmr_val;
1205         if (delta >= 128)
1206                 return (1);     /* busy */
1207
1208         td->tmr_val = sc->sc_tmr_val + td->tmr_res;
1209
1210         /* set toggle, if any */
1211         if (td->set_toggle) {
1212                 td->set_toggle = 0;
1213                 td->toggle = 1;
1214         }
1215         return (0);
1216 }
1217
1218 static uint8_t
1219 dwc_otg_host_rate_check(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1220 {
1221         uint8_t frame_num = (uint8_t)sc->sc_last_frame_num;
1222
1223         if (td->ep_type == UE_ISOCHRONOUS) {
1224                 /* non TT isochronous traffic */
1225                 if (frame_num & (td->tmr_res - 1))
1226                         goto busy;
1227                 if ((frame_num ^ td->tmr_val) & td->tmr_res)
1228                         goto busy;
1229                 td->tmr_val = td->tmr_res + sc->sc_last_frame_num;
1230                 td->toggle = 0;
1231                 return (0);
1232         } else if (td->ep_type == UE_INTERRUPT) {
1233                 if (!td->tt_scheduled)
1234                         goto busy;
1235                 td->tt_scheduled = 0;
1236                 return (0);
1237         } else if (td->did_nak != 0) {
1238                 /* check if we should pause sending queries for 125us */
1239                 if (td->tmr_res == frame_num) {
1240                         /* wait a bit */
1241                         dwc_otg_enable_sof_irq(sc);
1242                         goto busy;
1243                 }
1244         } else if (td->set_toggle) {
1245                 td->set_toggle = 0;
1246                 td->toggle = 1;
1247         }
1248         /* query for data one more time */
1249         td->tmr_res = frame_num;
1250         td->did_nak = 0;
1251         return (0);
1252 busy:
1253         return (1);
1254 }
1255
1256 static uint8_t
1257 dwc_otg_host_data_rx_sub(struct dwc_otg_softc *sc, struct dwc_otg_td *td,
1258     uint8_t channel)
1259 {
1260         uint32_t count;
1261
1262         /* check endpoint status */
1263         if (sc->sc_last_rx_status == 0)
1264                 goto busy;
1265
1266         if (channel >= DWC_OTG_MAX_CHANNELS)
1267                 goto busy;
1268
1269         if (GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) != channel)
1270                 goto busy;
1271
1272         switch (sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) {
1273         case GRXSTSRH_IN_DATA:
1274
1275                 DPRINTF("DATA ST=%d STATUS=0x%08x\n",
1276                     (int)td->state, (int)sc->sc_last_rx_status);
1277
1278                 if (sc->sc_chan_state[channel].hcint & HCINT_SOFTWARE_ONLY) {
1279                         /*
1280                          * When using SPLIT transactions on interrupt
1281                          * endpoints, sometimes data occurs twice.
1282                          */
1283                         DPRINTF("Data already received\n");
1284                         break;
1285                 }
1286
1287                 /* get the packet byte count */
1288                 count = GRXSTSRD_BCNT_GET(sc->sc_last_rx_status);
1289
1290                 /* check for ISOCHRONOUS endpoint */
1291                 if (td->ep_type == UE_ISOCHRONOUS) {
1292                         if ((sc->sc_last_rx_status & GRXSTSRD_DPID_MASK) !=
1293                             GRXSTSRD_DPID_DATA0) {
1294                                 /* more data to be received */
1295                                 td->tt_xactpos = HCSPLT_XACTPOS_MIDDLE;
1296                         } else {
1297                                 /* all data received */
1298                                 td->tt_xactpos = HCSPLT_XACTPOS_BEGIN;
1299                                 /* verify the packet byte count */
1300                                 if (count != td->remainder) {
1301                                         /* we have a short packet */
1302                                         td->short_pkt = 1;
1303                                         td->got_short = 1;
1304                                 }
1305                         }
1306                 } else {
1307                         /* verify the packet byte count */
1308                         if (count != td->max_packet_size) {
1309                                 if (count < td->max_packet_size) {
1310                                         /* we have a short packet */
1311                                         td->short_pkt = 1;
1312                                         td->got_short = 1;
1313                                 } else {
1314                                         /* invalid USB packet */
1315                                         td->error_any = 1;
1316
1317                                         /* release FIFO */
1318                                         dwc_otg_common_rx_ack(sc);
1319                                         goto complete;
1320                                 }
1321                         }
1322                         td->toggle ^= 1;
1323                         td->tt_scheduled = 0;
1324                 }
1325
1326                 /* verify the packet byte count */
1327                 if (count > td->remainder) {
1328                         /* invalid USB packet */
1329                         td->error_any = 1;
1330
1331                         /* release FIFO */
1332                         dwc_otg_common_rx_ack(sc);
1333                         goto complete;
1334                 }
1335
1336                 /* read data from FIFO */
1337                 dwc_otg_read_fifo(sc, td->pc, td->offset, count);
1338
1339                 td->remainder -= count;
1340                 td->offset += count;
1341                 sc->sc_chan_state[channel].hcint |= HCINT_SOFTWARE_ONLY;
1342                 break;
1343         default:
1344                 break;
1345         }
1346         /* release FIFO */
1347         dwc_otg_common_rx_ack(sc);
1348 busy:
1349         return (0);
1350 complete:
1351         return (1);
1352 }
1353
1354 static uint8_t
1355 dwc_otg_host_data_rx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1356 {
1357         uint32_t hcint = 0;
1358         uint32_t hcchar;
1359         uint8_t delta;
1360         uint8_t channel;
1361         uint8_t x;
1362
1363         for (x = 0; x != td->max_packet_count; x++) {
1364                 channel = td->channel[x];
1365                 if (channel >= DWC_OTG_MAX_CHANNELS)
1366                         continue;
1367                 hcint |= sc->sc_chan_state[channel].hcint;
1368
1369                 DPRINTF("CH=%d ST=%d HCINT=0x%08x HCCHAR=0x%08x HCTSIZ=0x%08x\n",
1370                     channel, td->state, hcint,
1371                     DWC_OTG_READ_4(sc, DOTG_HCCHAR(channel)),
1372                     DWC_OTG_READ_4(sc, DOTG_HCTSIZ(channel)));
1373
1374                 /* check interrupt bits */
1375                 if (hcint & (HCINT_RETRY |
1376                     HCINT_ACK | HCINT_NYET)) {
1377                         /* give success bits priority over failure bits */
1378                 } else if (hcint & HCINT_STALL) {
1379                         DPRINTF("CH=%d STALL\n", channel);
1380                         td->error_stall = 1;
1381                         td->error_any = 1;
1382                         goto complete;
1383                 } else if (hcint & HCINT_ERRORS) {
1384                         DPRINTF("CH=%d ERROR\n", channel);
1385                         td->errcnt++;
1386                         if (td->hcsplt != 0 || td->errcnt >= 3) {
1387                                 if (td->ep_type != UE_ISOCHRONOUS) {
1388                                         td->error_any = 1;
1389                                         goto complete;
1390                                 }
1391                         }
1392                 }
1393
1394                 /* check channels for data, if any */
1395                 if (dwc_otg_host_data_rx_sub(sc, td, channel))
1396                         goto complete;
1397
1398                 /* refresh interrupt status */
1399                 hcint |= sc->sc_chan_state[channel].hcint;
1400
1401                 if (hcint & (HCINT_ERRORS | HCINT_RETRY |
1402                     HCINT_ACK | HCINT_NYET)) {
1403                         if (!(hcint & HCINT_ERRORS))
1404                                 td->errcnt = 0;
1405                 }
1406         }
1407
1408         switch (td->state) {
1409         case DWC_CHAN_ST_START:
1410                 if (td->hcsplt != 0)
1411                         goto receive_spkt;
1412                 else
1413                         goto receive_pkt;
1414
1415         case DWC_CHAN_ST_WAIT_ANE:
1416                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1417                         if (td->ep_type == UE_INTERRUPT) {
1418                                 /*
1419                                  * The USB specification does not
1420                                  * mandate a particular data toggle
1421                                  * value for USB INTERRUPT
1422                                  * transfers. Switch the data toggle
1423                                  * value to receive the packet
1424                                  * correctly:
1425                                  */
1426                                 if (hcint & HCINT_DATATGLERR) {
1427                                         DPRINTF("Retrying packet due to "
1428                                             "data toggle error\n");
1429                                         td->toggle ^= 1;
1430                                         goto receive_pkt;
1431                                 }
1432                         } else if (td->ep_type == UE_ISOCHRONOUS) {
1433                                 goto complete;
1434                         }
1435                         td->did_nak = 1;
1436                         td->tt_scheduled = 0;
1437                         if (td->hcsplt != 0)
1438                                 goto receive_spkt;
1439                         else
1440                                 goto receive_pkt;
1441                 } else if (hcint & HCINT_NYET) {
1442                         if (td->hcsplt != 0) {
1443                                 /* try again */
1444                                 goto receive_pkt;
1445                         } else {
1446                                 /* not a valid token for IN endpoints */
1447                                 td->error_any = 1;
1448                                 goto complete;
1449                         }
1450                 } else if (hcint & HCINT_ACK) {
1451                         /* wait for data - ACK arrived first */
1452                         if (!(hcint & HCINT_SOFTWARE_ONLY))
1453                                 goto busy;
1454
1455                         if (td->ep_type == UE_ISOCHRONOUS) {
1456                                 /* check if we are complete */
1457                                 if (td->tt_xactpos == HCSPLT_XACTPOS_BEGIN) {
1458                                         goto complete;
1459                                 } else {
1460                                         /* get more packets */
1461                                         goto busy;
1462                                 }
1463                         } else {
1464                                 /* check if we are complete */
1465                                 if ((td->remainder == 0) || (td->got_short != 0)) {
1466                                         if (td->short_pkt)
1467                                                 goto complete;
1468
1469                                         /*
1470                                          * Else need to receive a zero length
1471                                          * packet.
1472                                          */
1473                                 }
1474                                 td->tt_scheduled = 0;
1475                                 td->did_nak = 0;
1476                                 if (td->hcsplt != 0)
1477                                         goto receive_spkt;
1478                                 else
1479                                         goto receive_pkt;
1480                         }
1481                 }
1482                 break;
1483
1484         case DWC_CHAN_ST_WAIT_S_ANE:
1485                 /*
1486                  * NOTE: The DWC OTG hardware provides a fake ACK in
1487                  * case of interrupt and isochronous transfers:
1488                  */
1489                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1490                         td->did_nak = 1;
1491                         td->tt_scheduled = 0;
1492                         goto receive_spkt;
1493                 } else if (hcint & HCINT_NYET) {
1494                         td->tt_scheduled = 0;
1495                         goto receive_spkt;
1496                 } else if (hcint & HCINT_ACK) {
1497                         td->did_nak = 0;
1498                         goto receive_pkt;
1499                 }
1500                 break;
1501
1502         case DWC_CHAN_ST_WAIT_C_PKT:
1503                 goto receive_pkt;
1504
1505         default:
1506                 break;
1507         }
1508         goto busy;
1509
1510 receive_pkt:
1511         /* free existing channel, if any */
1512         dwc_otg_host_channel_free(sc, td);
1513
1514         if (td->hcsplt != 0) {
1515                 delta = td->tt_complete_slot - sc->sc_last_frame_num - 1;
1516                 if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
1517                         td->state = DWC_CHAN_ST_WAIT_C_PKT;
1518                         goto busy;
1519                 }
1520                 delta = sc->sc_last_frame_num - td->tt_start_slot;
1521                 if (delta > DWC_OTG_TT_SLOT_MAX) {
1522                         if (td->ep_type != UE_ISOCHRONOUS) {
1523                                 /* we missed the service interval */
1524                                 td->error_any = 1;
1525                         }
1526                         goto complete;
1527                 }
1528                 /* complete split */
1529                 td->hcsplt |= HCSPLT_COMPSPLT;
1530         } else if (dwc_otg_host_rate_check(sc, td)) {
1531                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
1532                 goto busy;
1533         }
1534
1535         /* allocate a new channel */
1536         if (dwc_otg_host_channel_alloc(sc, td, 0)) {
1537                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
1538                 goto busy;
1539         }
1540
1541         /* set toggle, if any */
1542         if (td->set_toggle) {
1543                 td->set_toggle = 0;
1544                 td->toggle = 1;
1545         }
1546
1547         td->state = DWC_CHAN_ST_WAIT_ANE;
1548
1549         for (x = 0; x != td->max_packet_count; x++) {
1550                 channel = td->channel[x];
1551
1552                 /* receive one packet */
1553                 DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1554                     (td->max_packet_size << HCTSIZ_XFERSIZE_SHIFT) |
1555                     (1 << HCTSIZ_PKTCNT_SHIFT) |
1556                     (td->toggle ? (HCTSIZ_PID_DATA1 << HCTSIZ_PID_SHIFT) :
1557                     (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT)));
1558
1559                 DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(channel), td->hcsplt);
1560
1561                 hcchar = td->hcchar;
1562                 hcchar |= HCCHAR_EPDIR_IN;
1563
1564                 /* receive complete split ASAP */
1565                 if ((sc->sc_last_frame_num & 1) != 0 &&
1566                     td->ep_type == UE_ISOCHRONOUS)
1567                         hcchar |= HCCHAR_ODDFRM;
1568                 else
1569                         hcchar &= ~HCCHAR_ODDFRM;
1570
1571                 /* must enable channel before data can be received */
1572                 DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(channel), hcchar);
1573         }
1574         /* wait until next slot before trying complete split */
1575         td->tt_complete_slot = sc->sc_last_frame_num + 1;
1576
1577         goto busy;
1578
1579 receive_spkt:
1580         /* free existing channel(s), if any */
1581         dwc_otg_host_channel_free(sc, td);
1582
1583         delta = td->tt_start_slot - sc->sc_last_frame_num - 1;
1584         if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
1585                 td->state = DWC_CHAN_ST_START;
1586                 goto busy;
1587         }
1588         delta = sc->sc_last_frame_num - td->tt_start_slot;
1589         if (delta > 5) {
1590                 /* missed it */
1591                 td->tt_scheduled = 0;
1592                 td->state = DWC_CHAN_ST_START;
1593                 goto busy;
1594         }
1595
1596         /* allocate a new channel */
1597         if (dwc_otg_host_channel_alloc(sc, td, 0)) {
1598                 td->state = DWC_CHAN_ST_START;
1599                 goto busy;
1600         }
1601
1602         channel = td->channel[0];
1603
1604         td->hcsplt &= ~HCSPLT_COMPSPLT;
1605         td->state = DWC_CHAN_ST_WAIT_S_ANE;
1606
1607         /* receive one packet */
1608         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1609             (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT));
1610
1611         DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(channel), td->hcsplt);
1612
1613         /* send after next SOF event */
1614         if ((sc->sc_last_frame_num & 1) == 0 &&
1615             td->ep_type == UE_ISOCHRONOUS)
1616                 td->hcchar |= HCCHAR_ODDFRM;
1617         else
1618                 td->hcchar &= ~HCCHAR_ODDFRM;
1619
1620         hcchar = td->hcchar;
1621         hcchar |= HCCHAR_EPDIR_IN;
1622
1623         /* wait until next slot before trying complete split */
1624         td->tt_complete_slot = sc->sc_last_frame_num + 1;
1625
1626         /* must enable channel before data can be received */
1627         DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(channel), hcchar);
1628 busy:
1629         return (1);     /* busy */
1630
1631 complete:
1632         dwc_otg_host_channel_free(sc, td);
1633         return (0);     /* complete */
1634 }
1635
1636 static uint8_t
1637 dwc_otg_data_rx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1638 {
1639         uint32_t temp;
1640         uint16_t count;
1641         uint8_t got_short;
1642
1643         got_short = 0;
1644
1645         /* check endpoint status */
1646         if (sc->sc_last_rx_status == 0)
1647                 goto not_complete;
1648
1649         if (GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) != td->ep_no)
1650                 goto not_complete;
1651
1652         /* check for SETUP packet */
1653         if ((sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) ==
1654             GRXSTSRD_STP_DATA ||
1655             (sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) ==
1656             GRXSTSRD_STP_COMPLETE) {
1657                 if (td->remainder == 0) {
1658                         /*
1659                          * We are actually complete and have
1660                          * received the next SETUP
1661                          */
1662                         DPRINTFN(5, "faking complete\n");
1663                         return (0);     /* complete */
1664                 }
1665                 /*
1666                  * USB Host Aborted the transfer.
1667                  */
1668                 td->error_any = 1;
1669                 return (0);             /* complete */
1670         }
1671
1672         if ((sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) !=
1673             GRXSTSRD_OUT_DATA) {
1674                 /* release FIFO */
1675                 dwc_otg_common_rx_ack(sc);
1676                 goto not_complete;
1677         }
1678
1679         /* get the packet byte count */
1680         count = GRXSTSRD_BCNT_GET(sc->sc_last_rx_status);
1681
1682         /* verify the packet byte count */
1683         if (count != td->max_packet_size) {
1684                 if (count < td->max_packet_size) {
1685                         /* we have a short packet */
1686                         td->short_pkt = 1;
1687                         got_short = 1;
1688                 } else {
1689                         /* invalid USB packet */
1690                         td->error_any = 1;
1691
1692                         /* release FIFO */
1693                         dwc_otg_common_rx_ack(sc);
1694                         return (0);     /* we are complete */
1695                 }
1696         }
1697         /* verify the packet byte count */
1698         if (count > td->remainder) {
1699                 /* invalid USB packet */
1700                 td->error_any = 1;
1701
1702                 /* release FIFO */
1703                 dwc_otg_common_rx_ack(sc);
1704                 return (0);             /* we are complete */
1705         }
1706
1707         /* read data from FIFO */
1708         dwc_otg_read_fifo(sc, td->pc, td->offset, count);
1709
1710         td->remainder -= count;
1711         td->offset += count;
1712
1713         /* release FIFO */
1714         dwc_otg_common_rx_ack(sc);
1715
1716         temp = sc->sc_out_ctl[td->ep_no];
1717
1718         /* check for isochronous mode */
1719         if ((temp & DIEPCTL_EPTYPE_MASK) ==
1720             (DIEPCTL_EPTYPE_ISOC << DIEPCTL_EPTYPE_SHIFT)) {
1721                 /* toggle odd or even frame bit */
1722                 if (temp & DIEPCTL_SETD1PID) {
1723                         temp &= ~DIEPCTL_SETD1PID;
1724                         temp |= DIEPCTL_SETD0PID;
1725                 } else {
1726                         temp &= ~DIEPCTL_SETD0PID;
1727                         temp |= DIEPCTL_SETD1PID;
1728                 }
1729                 sc->sc_out_ctl[td->ep_no] = temp;
1730         }
1731
1732         /* check if we are complete */
1733         if ((td->remainder == 0) || got_short) {
1734                 if (td->short_pkt) {
1735                         /* we are complete */
1736                         return (0);
1737                 }
1738                 /* else need to receive a zero length packet */
1739         }
1740
1741 not_complete:
1742
1743         /* enable SETUP and transfer complete interrupt */
1744         if (td->ep_no == 0) {
1745                 DWC_OTG_WRITE_4(sc, DOTG_DOEPTSIZ(0),
1746                     DXEPTSIZ_SET_MULTI(3) |
1747                     DXEPTSIZ_SET_NPKT(1) |
1748                     DXEPTSIZ_SET_NBYTES(td->max_packet_size));
1749         } else {
1750                 /* allow reception of multiple packets */
1751                 DWC_OTG_WRITE_4(sc, DOTG_DOEPTSIZ(td->ep_no),
1752                     DXEPTSIZ_SET_MULTI(1) |
1753                     DXEPTSIZ_SET_NPKT(4) |
1754                     DXEPTSIZ_SET_NBYTES(4 *
1755                         ((td->max_packet_size + 3) & ~3)));
1756         }
1757         temp = sc->sc_out_ctl[td->ep_no];
1758         DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(td->ep_no), temp |
1759             DOEPCTL_EPENA | DOEPCTL_CNAK);
1760
1761         return (1);                     /* not complete */
1762 }
1763
1764 static uint8_t
1765 dwc_otg_host_data_tx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1766 {
1767         uint32_t count;
1768         uint32_t hcint;
1769         uint32_t hcchar;
1770         uint8_t delta;
1771         uint8_t channel;
1772         uint8_t x;
1773
1774         dwc_otg_host_dump_rx(sc, td);
1775
1776         /* check that last channel is complete */
1777         channel = td->channel[td->npkt];
1778
1779         if (channel < DWC_OTG_MAX_CHANNELS) {
1780                 hcint = sc->sc_chan_state[channel].hcint;
1781
1782                 DPRINTF("CH=%d ST=%d HCINT=0x%08x HCCHAR=0x%08x HCTSIZ=0x%08x\n",
1783                     channel, td->state, hcint,
1784                     DWC_OTG_READ_4(sc, DOTG_HCCHAR(channel)),
1785                     DWC_OTG_READ_4(sc, DOTG_HCTSIZ(channel)));
1786
1787                 if (hcint & (HCINT_RETRY |
1788                     HCINT_ACK | HCINT_NYET)) {
1789                         /* give success bits priority over failure bits */
1790                 } else if (hcint & HCINT_STALL) {
1791                         DPRINTF("CH=%d STALL\n", channel);
1792                         td->error_stall = 1;
1793                         td->error_any = 1;
1794                         goto complete;
1795                 } else if (hcint & HCINT_ERRORS) {
1796                         DPRINTF("CH=%d ERROR\n", channel);
1797                         td->errcnt++;
1798                         if (td->hcsplt != 0 || td->errcnt >= 3) {
1799                                 td->error_any = 1;
1800                                 goto complete;
1801                         }
1802                 }
1803
1804                 if (hcint & (HCINT_ERRORS | HCINT_RETRY |
1805                     HCINT_ACK | HCINT_NYET)) {
1806
1807                         if (!(hcint & HCINT_ERRORS))
1808                                 td->errcnt = 0;
1809                 }
1810         } else {
1811                 hcint = 0;
1812         }
1813
1814         switch (td->state) {
1815         case DWC_CHAN_ST_START:
1816                 goto send_pkt;
1817
1818         case DWC_CHAN_ST_WAIT_ANE:
1819                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1820                         td->did_nak = 1;
1821                         td->tt_scheduled = 0;
1822                         goto send_pkt;
1823                 } else if (hcint & (HCINT_ACK | HCINT_NYET)) {
1824                         td->offset += td->tx_bytes;
1825                         td->remainder -= td->tx_bytes;
1826                         td->toggle ^= 1;
1827                         /* check if next response will be a NAK */
1828                         if (hcint & HCINT_NYET)
1829                                 td->did_nak = 1;
1830                         else
1831                                 td->did_nak = 0;
1832                         td->tt_scheduled = 0;
1833
1834                         /* check remainder */
1835                         if (td->remainder == 0) {
1836                                 if (td->short_pkt)
1837                                         goto complete;
1838
1839                                 /*
1840                                  * Else we need to transmit a short
1841                                  * packet:
1842                                  */
1843                         }
1844                         goto send_pkt;
1845                 }
1846                 break;
1847
1848         case DWC_CHAN_ST_WAIT_S_ANE:
1849                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1850                         td->did_nak = 1;
1851                         td->tt_scheduled = 0;
1852                         goto send_pkt;
1853                 } else if (hcint & (HCINT_ACK | HCINT_NYET)) {
1854                         td->did_nak = 0;
1855                         goto send_cpkt;
1856                 }
1857                 break;
1858
1859         case DWC_CHAN_ST_WAIT_C_ANE:
1860                 if (hcint & HCINT_NYET) {
1861                         goto send_cpkt;
1862                 } else if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1863                         td->did_nak = 1;
1864                         td->tt_scheduled = 0;
1865                         goto send_pkt;
1866                 } else if (hcint & HCINT_ACK) {
1867                         td->offset += td->tx_bytes;
1868                         td->remainder -= td->tx_bytes;
1869                         td->toggle ^= 1;
1870                         td->did_nak = 0;
1871                         td->tt_scheduled = 0;
1872
1873                         /* check remainder */
1874                         if (td->remainder == 0) {
1875                                 if (td->short_pkt)
1876                                         goto complete;
1877
1878                                 /* else we need to transmit a short packet */
1879                         }
1880                         goto send_pkt;
1881                 }
1882                 break;
1883
1884         case DWC_CHAN_ST_WAIT_C_PKT:
1885                 goto send_cpkt;
1886
1887         case DWC_CHAN_ST_TX_WAIT_ISOC:
1888                 /* Check if ISOCHRONOUS OUT traffic is complete */
1889                 if ((hcint & HCINT_HCH_DONE_MASK) == 0)
1890                         break;
1891
1892                 td->offset += td->tx_bytes;
1893                 td->remainder -= td->tx_bytes;
1894                 goto complete;
1895         default:
1896                 break;
1897         }
1898         goto busy;
1899
1900 send_pkt:
1901         /* free existing channel(s), if any */
1902         dwc_otg_host_channel_free(sc, td);
1903
1904         if (td->hcsplt != 0) {
1905                 delta = td->tt_start_slot - sc->sc_last_frame_num - 1;
1906                 if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
1907                         td->state = DWC_CHAN_ST_START;
1908                         goto busy;
1909                 }
1910                 delta = sc->sc_last_frame_num - td->tt_start_slot;
1911                 if (delta > 5) {
1912                         /* missed it */
1913                         td->tt_scheduled = 0;
1914                         td->state = DWC_CHAN_ST_START;
1915                         goto busy;
1916                 }
1917         } else if (dwc_otg_host_rate_check(sc, td)) {
1918                 td->state = DWC_CHAN_ST_START;
1919                 goto busy;
1920         }
1921
1922         /* allocate a new channel */
1923         if (dwc_otg_host_channel_alloc(sc, td, 1)) {
1924                 td->state = DWC_CHAN_ST_START;
1925                 goto busy;
1926         }
1927
1928         /* set toggle, if any */
1929         if (td->set_toggle) {
1930                 td->set_toggle = 0;
1931                 td->toggle = 1;
1932         }
1933
1934         if (td->ep_type == UE_ISOCHRONOUS) {
1935                 /* ISOCHRONOUS OUT transfers don't have any ACKs */
1936                 td->state = DWC_CHAN_ST_TX_WAIT_ISOC;
1937                 td->hcsplt &= ~HCSPLT_COMPSPLT;
1938                 if (td->hcsplt != 0) {
1939                         /* get maximum transfer length */
1940                         count = td->remainder;
1941                         if (count > HCSPLT_XACTLEN_BURST) {
1942                                 DPRINTF("TT overflow\n");
1943                                 td->error_any = 1;
1944                                 goto complete;
1945                         }
1946                         /* Update transaction position */
1947                         td->hcsplt &= ~HCSPLT_XACTPOS_MASK;
1948                         td->hcsplt |= (HCSPLT_XACTPOS_ALL << HCSPLT_XACTPOS_SHIFT);
1949                 }
1950         } else if (td->hcsplt != 0) {
1951                 td->hcsplt &= ~HCSPLT_COMPSPLT;
1952                 /* Wait for ACK/NAK/ERR from TT */
1953                 td->state = DWC_CHAN_ST_WAIT_S_ANE;
1954         } else {
1955                 /* Wait for ACK/NAK/STALL from device */
1956                 td->state = DWC_CHAN_ST_WAIT_ANE;
1957         }
1958
1959         td->tx_bytes = 0;
1960
1961         for (x = 0; x != td->max_packet_count; x++) {
1962                 uint32_t rem_bytes;
1963
1964                 channel = td->channel[x];
1965
1966                 /* send one packet at a time */
1967                 count = td->max_packet_size;
1968                 rem_bytes = td->remainder - td->tx_bytes;
1969                 if (rem_bytes < count) {
1970                         /* we have a short packet */
1971                         td->short_pkt = 1;
1972                         count = rem_bytes;
1973                 }
1974                 if (count == rem_bytes) {
1975                         /* last packet */
1976                         switch (x) {
1977                         case 0:
1978                                 DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1979                                     (count << HCTSIZ_XFERSIZE_SHIFT) |
1980                                     (1 << HCTSIZ_PKTCNT_SHIFT) |
1981                                     (td->toggle ? (HCTSIZ_PID_DATA1 << HCTSIZ_PID_SHIFT) :
1982                                     (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT)));
1983                                 break;
1984                         case 1:
1985                                 DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1986                                     (count << HCTSIZ_XFERSIZE_SHIFT) |
1987                                     (1 << HCTSIZ_PKTCNT_SHIFT) |
1988                                     (HCTSIZ_PID_DATA1 << HCTSIZ_PID_SHIFT));
1989                                 break;
1990                         default:
1991                                 DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1992                                     (count << HCTSIZ_XFERSIZE_SHIFT) |
1993                                     (1 << HCTSIZ_PKTCNT_SHIFT) |
1994                                     (HCTSIZ_PID_DATA2 << HCTSIZ_PID_SHIFT));
1995                                 break;
1996                         }
1997                 } else if (td->ep_type == UE_ISOCHRONOUS &&
1998                            td->max_packet_count > 1) {
1999                         /* ISOCHRONOUS multi packet */
2000                         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
2001                             (count << HCTSIZ_XFERSIZE_SHIFT) |
2002                             (1 << HCTSIZ_PKTCNT_SHIFT) |
2003                             (HCTSIZ_PID_MDATA << HCTSIZ_PID_SHIFT));
2004                 } else {
2005                         /* TODO: HCTSIZ_DOPNG */
2006                         /* standard BULK/INTERRUPT/CONTROL packet */
2007                         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
2008                             (count << HCTSIZ_XFERSIZE_SHIFT) |
2009                             (1 << HCTSIZ_PKTCNT_SHIFT) |
2010                             (td->toggle ? (HCTSIZ_PID_DATA1 << HCTSIZ_PID_SHIFT) :
2011                             (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT)));
2012                 }
2013
2014                 DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(channel), td->hcsplt);
2015
2016                 hcchar = td->hcchar;
2017                 hcchar &= ~HCCHAR_EPDIR_IN;
2018
2019                 /* send after next SOF event */
2020                 if ((sc->sc_last_frame_num & 1) == 0 &&
2021                     td->ep_type == UE_ISOCHRONOUS)
2022                         hcchar |= HCCHAR_ODDFRM;
2023                 else
2024                         hcchar &= ~HCCHAR_ODDFRM;
2025
2026                 /* must enable before writing data to FIFO */
2027                 DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(channel), hcchar);
2028
2029                 if (count != 0) {
2030                         /* write data into FIFO */
2031                         dwc_otg_write_fifo(sc, td->pc, td->offset +
2032                             td->tx_bytes, DOTG_DFIFO(channel), count);
2033                 }
2034
2035                 /* store number of bytes transmitted */
2036                 td->tx_bytes += count;
2037
2038                 /* store last packet index */
2039                 td->npkt = x;
2040
2041                 /* check for last packet */
2042                 if (count == rem_bytes)
2043                         break;
2044         }
2045         goto busy;
2046
2047 send_cpkt:
2048         /* free existing channel, if any */
2049         dwc_otg_host_channel_free(sc, td);
2050
2051         delta = td->tt_complete_slot - sc->sc_last_frame_num - 1;
2052         if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
2053                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
2054                 goto busy;
2055         }
2056         delta = sc->sc_last_frame_num - td->tt_start_slot;
2057         if (delta > DWC_OTG_TT_SLOT_MAX) {
2058                 /* we missed the service interval */
2059                 if (td->ep_type != UE_ISOCHRONOUS)
2060                         td->error_any = 1;
2061                 goto complete;
2062         }
2063
2064         /* allocate a new channel */
2065         if (dwc_otg_host_channel_alloc(sc, td, 0)) {
2066                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
2067                 goto busy;
2068         }
2069
2070         channel = td->channel[0];
2071
2072         td->hcsplt |= HCSPLT_COMPSPLT;
2073         td->state = DWC_CHAN_ST_WAIT_C_ANE;
2074
2075         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
2076             (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT));
2077
2078         DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(channel), td->hcsplt);
2079
2080         hcchar = td->hcchar;
2081         hcchar &= ~HCCHAR_EPDIR_IN;
2082
2083         /* receive complete split ASAP */
2084         if ((sc->sc_last_frame_num & 1) != 0 &&
2085             td->ep_type == UE_ISOCHRONOUS)
2086                 hcchar |= HCCHAR_ODDFRM;
2087         else
2088                 hcchar &= ~HCCHAR_ODDFRM;
2089
2090         /* must enable channel before data can be received */
2091         DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(channel), hcchar);
2092
2093         /* wait until next slot before trying complete split */
2094         td->tt_complete_slot = sc->sc_last_frame_num + 1;
2095 busy:
2096         return (1);     /* busy */
2097
2098 complete:
2099         dwc_otg_host_channel_free(sc, td);
2100         return (0);     /* complete */
2101 }
2102
2103 static uint8_t
2104 dwc_otg_data_tx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
2105 {
2106         uint32_t max_buffer;
2107         uint32_t count;
2108         uint32_t fifo_left;
2109         uint32_t mpkt;
2110         uint32_t temp;
2111         uint8_t to;
2112
2113         to = 3;                         /* don't loop forever! */
2114
2115         max_buffer = sc->sc_hw_ep_profile[td->ep_no].max_buffer;
2116
2117 repeat:
2118         /* check for for endpoint 0 data */
2119
2120         temp = sc->sc_last_rx_status;
2121
2122         if ((td->ep_no == 0) && (temp != 0) &&
2123             (GRXSTSRD_CHNUM_GET(temp) == 0)) {
2124
2125                 if ((temp & GRXSTSRD_PKTSTS_MASK) !=
2126                     GRXSTSRD_STP_DATA &&
2127                     (temp & GRXSTSRD_PKTSTS_MASK) !=
2128                     GRXSTSRD_STP_COMPLETE) {
2129
2130                         /* dump data - wrong direction */
2131                         dwc_otg_common_rx_ack(sc);
2132                 } else {
2133                         /*
2134                          * The current transfer was cancelled
2135                          * by the USB Host:
2136                          */
2137                         td->error_any = 1;
2138                         return (0);             /* complete */
2139                 }
2140         }
2141
2142         /* fill in more TX data, if possible */
2143         if (td->tx_bytes != 0) {
2144
2145                 uint16_t cpkt;
2146
2147                 /* check if packets have been transferred */
2148                 temp = DWC_OTG_READ_4(sc, DOTG_DIEPTSIZ(td->ep_no));
2149
2150                 /* get current packet number */
2151                 cpkt = DXEPTSIZ_GET_NPKT(temp);
2152
2153                 if (cpkt >= td->npkt) {
2154                         fifo_left = 0;
2155                 } else {
2156                         if (max_buffer != 0) {
2157                                 fifo_left = (td->npkt - cpkt) *
2158                                     td->max_packet_size;
2159
2160                                 if (fifo_left > max_buffer)
2161                                         fifo_left = max_buffer;
2162                         } else {
2163                                 fifo_left = td->max_packet_size;
2164                         }
2165                 }
2166
2167                 count = td->tx_bytes;
2168                 if (count > fifo_left)
2169                         count = fifo_left;
2170
2171                 if (count != 0) {
2172                         /* write data into FIFO */
2173                         dwc_otg_write_fifo(sc, td->pc, td->offset,
2174                             DOTG_DFIFO(td->ep_no), count);
2175
2176                         td->tx_bytes -= count;
2177                         td->remainder -= count;
2178                         td->offset += count;
2179                         td->npkt = cpkt;
2180                 }
2181                 if (td->tx_bytes != 0)
2182                         goto not_complete;
2183
2184                 /* check remainder */
2185                 if (td->remainder == 0) {
2186                         if (td->short_pkt)
2187                                 return (0);     /* complete */
2188
2189                         /* else we need to transmit a short packet */
2190                 }
2191         }
2192
2193         if (!to--)
2194                 goto not_complete;
2195
2196         /* check if not all packets have been transferred */
2197         temp = DWC_OTG_READ_4(sc, DOTG_DIEPTSIZ(td->ep_no));
2198
2199         if (DXEPTSIZ_GET_NPKT(temp) != 0) {
2200
2201                 DPRINTFN(5, "busy ep=%d npkt=%d DIEPTSIZ=0x%08x "
2202                     "DIEPCTL=0x%08x\n", td->ep_no,
2203                     DXEPTSIZ_GET_NPKT(temp),
2204                     temp, DWC_OTG_READ_4(sc, DOTG_DIEPCTL(td->ep_no)));
2205
2206                 goto not_complete;
2207         }
2208
2209         DPRINTFN(5, "rem=%u ep=%d\n", td->remainder, td->ep_no);
2210
2211         /* try to optimise by sending more data */
2212         if ((max_buffer != 0) && ((td->max_packet_size & 3) == 0)) {
2213
2214                 /* send multiple packets at the same time */
2215                 mpkt = max_buffer / td->max_packet_size;
2216
2217                 if (mpkt > 0x3FE)
2218                         mpkt = 0x3FE;
2219
2220                 count = td->remainder;
2221                 if (count > 0x7FFFFF)
2222                         count = 0x7FFFFF - (0x7FFFFF % td->max_packet_size);
2223
2224                 td->npkt = count / td->max_packet_size;
2225
2226                 /*
2227                  * NOTE: We could use 0x3FE instead of "mpkt" in the
2228                  * check below to get more throughput, but then we
2229                  * have a dependency towards non-generic chip features
2230                  * to disable the TX-FIFO-EMPTY interrupts on a per
2231                  * endpoint basis. Increase the maximum buffer size of
2232                  * the IN endpoint to increase the performance.
2233                  */
2234                 if (td->npkt > mpkt) {
2235                         td->npkt = mpkt;
2236                         count = td->max_packet_size * mpkt;
2237                 } else if ((count == 0) || (count % td->max_packet_size)) {
2238                         /* we are transmitting a short packet */
2239                         td->npkt++;
2240                         td->short_pkt = 1;
2241                 }
2242         } else {
2243                 /* send one packet at a time */
2244                 mpkt = 1;
2245                 count = td->max_packet_size;
2246                 if (td->remainder < count) {
2247                         /* we have a short packet */
2248                         td->short_pkt = 1;
2249                         count = td->remainder;
2250                 }
2251                 td->npkt = 1;
2252         }
2253         DWC_OTG_WRITE_4(sc, DOTG_DIEPTSIZ(td->ep_no),
2254             DXEPTSIZ_SET_MULTI(1) |
2255             DXEPTSIZ_SET_NPKT(td->npkt) |
2256             DXEPTSIZ_SET_NBYTES(count));
2257
2258         /* make room for buffering */
2259         td->npkt += mpkt;
2260
2261         temp = sc->sc_in_ctl[td->ep_no];
2262
2263         /* check for isochronous mode */
2264         if ((temp & DIEPCTL_EPTYPE_MASK) ==
2265             (DIEPCTL_EPTYPE_ISOC << DIEPCTL_EPTYPE_SHIFT)) {
2266                 /* toggle odd or even frame bit */
2267                 if (temp & DIEPCTL_SETD1PID) {
2268                         temp &= ~DIEPCTL_SETD1PID;
2269                         temp |= DIEPCTL_SETD0PID;
2270                 } else {
2271                         temp &= ~DIEPCTL_SETD0PID;
2272                         temp |= DIEPCTL_SETD1PID;
2273                 }
2274                 sc->sc_in_ctl[td->ep_no] = temp;
2275         }
2276
2277         /* must enable before writing data to FIFO */
2278         DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(td->ep_no), temp |
2279             DIEPCTL_EPENA | DIEPCTL_CNAK);
2280
2281         td->tx_bytes = count;
2282
2283         /* check remainder */
2284         if (td->tx_bytes == 0 &&
2285             td->remainder == 0) {
2286                 if (td->short_pkt)
2287                         return (0);     /* complete */
2288
2289                 /* else we need to transmit a short packet */
2290         }
2291         goto repeat;
2292
2293 not_complete:
2294         return (1);                     /* not complete */
2295 }
2296
2297 static uint8_t
2298 dwc_otg_data_tx_sync(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
2299 {
2300         uint32_t temp;
2301
2302         /*
2303          * If all packets are transferred we are complete:
2304          */
2305         temp = DWC_OTG_READ_4(sc, DOTG_DIEPTSIZ(td->ep_no));
2306
2307         /* check that all packets have been transferred */
2308         if (DXEPTSIZ_GET_NPKT(temp) != 0) {
2309                 DPRINTFN(5, "busy ep=%d\n", td->ep_no);
2310                 goto not_complete;
2311         }
2312         return (0);
2313
2314 not_complete:
2315
2316         /* we only want to know if there is a SETUP packet or free IN packet */
2317
2318         temp = sc->sc_last_rx_status;
2319
2320         if ((td->ep_no == 0) && (temp != 0) &&
2321             (GRXSTSRD_CHNUM_GET(temp) == 0)) {
2322
2323                 if ((temp & GRXSTSRD_PKTSTS_MASK) ==
2324                     GRXSTSRD_STP_DATA ||
2325                     (temp & GRXSTSRD_PKTSTS_MASK) ==
2326                     GRXSTSRD_STP_COMPLETE) {
2327                         DPRINTFN(5, "faking complete\n");
2328                         /*
2329                          * Race condition: We are complete!
2330                          */
2331                         return (0);
2332                 } else {
2333                         /* dump data - wrong direction */
2334                         dwc_otg_common_rx_ack(sc);
2335                 }
2336         }
2337         return (1);                     /* not complete */
2338 }
2339
2340 static void
2341 dwc_otg_xfer_do_fifo(struct dwc_otg_softc *sc, struct usb_xfer *xfer)
2342 {
2343         struct dwc_otg_td *td;
2344         uint8_t toggle;
2345         uint8_t tmr_val;
2346         uint8_t tmr_res;
2347
2348         DPRINTFN(9, "\n");
2349
2350         td = xfer->td_transfer_cache;
2351         if (td == NULL)
2352                 return;
2353
2354         while (1) {
2355                 if ((td->func) (sc, td)) {
2356                         /* operation in progress */
2357                         break;
2358                 }
2359                 if (((void *)td) == xfer->td_transfer_last) {
2360                         goto done;
2361                 }
2362                 if (td->error_any) {
2363                         goto done;
2364                 } else if (td->remainder > 0) {
2365                         /*
2366                          * We had a short transfer. If there is no alternate
2367                          * next, stop processing !
2368                          */
2369                         if (!td->alt_next)
2370                                 goto done;
2371                 }
2372
2373                 /*
2374                  * Fetch the next transfer descriptor and transfer
2375                  * some flags to the next transfer descriptor
2376                  */
2377                 tmr_res = td->tmr_res;
2378                 tmr_val = td->tmr_val;
2379                 toggle = td->toggle;
2380                 td = td->obj_next;
2381                 xfer->td_transfer_cache = td;
2382                 td->toggle = toggle;    /* transfer toggle */
2383                 td->tmr_res = tmr_res;
2384                 td->tmr_val = tmr_val;
2385         }
2386         return;
2387
2388 done:
2389         xfer->td_transfer_cache = NULL;
2390         sc->sc_xfer_complete = 1;
2391 }
2392
2393 static uint8_t
2394 dwc_otg_xfer_do_complete_locked(struct dwc_otg_softc *sc, struct usb_xfer *xfer)
2395 {
2396         struct dwc_otg_td *td;
2397
2398         DPRINTFN(9, "\n");
2399
2400         td = xfer->td_transfer_cache;
2401         if (td == NULL) {
2402                 /* compute all actual lengths */
2403                 dwc_otg_standard_done(xfer);
2404                 return (1);
2405         }
2406         return (0);
2407 }
2408
2409 static void
2410 dwc_otg_timer(void *_sc)
2411 {
2412         struct dwc_otg_softc *sc = _sc;
2413
2414         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
2415
2416         DPRINTF("\n");
2417
2418         USB_BUS_SPIN_LOCK(&sc->sc_bus);
2419
2420         /* increment timer value */
2421         sc->sc_tmr_val++;
2422
2423         /* enable SOF interrupt, which will poll jobs */
2424         dwc_otg_enable_sof_irq(sc);
2425
2426         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
2427
2428         if (sc->sc_timer_active) {
2429                 /* restart timer */
2430                 usb_callout_reset(&sc->sc_timer,
2431                     hz / (1000 / DWC_OTG_HOST_TIMER_RATE),
2432                     &dwc_otg_timer, sc);
2433         }
2434 }
2435
2436 static void
2437 dwc_otg_timer_start(struct dwc_otg_softc *sc)
2438 {
2439         if (sc->sc_timer_active != 0)
2440                 return;
2441
2442         sc->sc_timer_active = 1;
2443
2444         /* restart timer */
2445         usb_callout_reset(&sc->sc_timer,
2446             hz / (1000 / DWC_OTG_HOST_TIMER_RATE),
2447             &dwc_otg_timer, sc);
2448 }
2449
2450 static void
2451 dwc_otg_timer_stop(struct dwc_otg_softc *sc)
2452 {
2453         if (sc->sc_timer_active == 0)
2454                 return;
2455
2456         sc->sc_timer_active = 0;
2457
2458         /* stop timer */
2459         usb_callout_stop(&sc->sc_timer);
2460 }
2461
2462 static uint16_t
2463 dwc_otg_compute_isoc_rx_tt_slot(struct dwc_otg_tt_info *pinfo)
2464 {
2465         if (pinfo->slot_index < DWC_OTG_TT_SLOT_MAX)
2466                 pinfo->slot_index++;
2467         return (pinfo->slot_index);
2468 }
2469
2470 static uint8_t
2471 dwc_otg_update_host_transfer_schedule_locked(struct dwc_otg_softc *sc)
2472 {
2473         TAILQ_HEAD(, usb_xfer) head;
2474         struct usb_xfer *xfer;
2475         struct usb_xfer *xfer_next;
2476         struct dwc_otg_td *td;
2477         uint16_t temp;
2478         uint16_t slot;
2479
2480         temp = DWC_OTG_READ_4(sc, DOTG_HFNUM) & DWC_OTG_FRAME_MASK;
2481
2482         if (sc->sc_last_frame_num == temp)
2483                 return (0);
2484
2485         sc->sc_last_frame_num = temp;
2486
2487         TAILQ_INIT(&head);
2488
2489         if ((temp & 7) == 0) {
2490
2491                 /* reset the schedule */
2492                 memset(sc->sc_tt_info, 0, sizeof(sc->sc_tt_info));
2493
2494                 TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2495                         td = xfer->td_transfer_cache;
2496                         if (td == NULL || td->ep_type != UE_ISOCHRONOUS)
2497                                 continue;
2498
2499                         /* check for IN direction */
2500                         if ((td->hcchar & HCCHAR_EPDIR_IN) != 0)
2501                                 continue;
2502
2503                         sc->sc_needsof = 1;
2504
2505                         if (td->hcsplt == 0 || td->tt_scheduled != 0)
2506                                 continue;
2507
2508                         /* compute slot */
2509                         slot = dwc_otg_compute_isoc_rx_tt_slot(
2510                             sc->sc_tt_info + td->tt_index);
2511                         if (slot > 3) {
2512                                 /*
2513                                  * Not enough time to get complete
2514                                  * split executed.
2515                                  */
2516                                 continue;
2517                         }
2518                         /* Delayed start */
2519                         td->tt_start_slot = temp + slot;
2520                         td->tt_scheduled = 1;
2521                         TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2522                         TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2523                 }
2524
2525                 TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2526                         td = xfer->td_transfer_cache;
2527                         if (td == NULL || td->ep_type != UE_ISOCHRONOUS)
2528                                 continue;
2529
2530                         /* check for OUT direction */
2531                         if ((td->hcchar & HCCHAR_EPDIR_IN) == 0)
2532                                 continue;
2533
2534                         sc->sc_needsof = 1;
2535
2536                         if (td->hcsplt == 0 || td->tt_scheduled != 0)
2537                                 continue;
2538
2539                         /* Start ASAP */
2540                         td->tt_start_slot = temp;
2541                         td->tt_scheduled = 1;
2542                         TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2543                         TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2544                 }
2545
2546                 TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2547                         td = xfer->td_transfer_cache;
2548                         if (td == NULL || td->ep_type != UE_INTERRUPT)
2549                                 continue;
2550
2551                         if (td->tt_scheduled != 0) {
2552                                 sc->sc_needsof = 1;
2553                                 continue;
2554                         }
2555
2556                         if (dwc_otg_host_rate_check_interrupt(sc, td))
2557                                 continue;
2558
2559                         if (td->hcsplt == 0) {
2560                                 sc->sc_needsof = 1;
2561                                 td->tt_scheduled = 1;
2562                                 continue;
2563                         }
2564
2565                         /* start ASAP */
2566                         td->tt_start_slot = temp;
2567                         sc->sc_needsof = 1;
2568                         td->tt_scheduled = 1;
2569                         TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2570                         TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2571                 }
2572
2573                 TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2574                         td = xfer->td_transfer_cache;
2575                         if (td == NULL ||
2576                             td->ep_type != UE_CONTROL) {
2577                                 continue;
2578                         }
2579
2580                         sc->sc_needsof = 1;
2581
2582                         if (td->hcsplt == 0 || td->tt_scheduled != 0)
2583                                 continue;
2584
2585                         /* start ASAP */
2586                         td->tt_start_slot = temp;
2587                         td->tt_scheduled = 1;
2588                         TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2589                         TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2590                 }
2591         }
2592         if ((temp & 7) < 6) {
2593                 TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2594                         td = xfer->td_transfer_cache;
2595                         if (td == NULL ||
2596                             td->ep_type != UE_BULK) {
2597                                 continue;
2598                         }
2599
2600                         sc->sc_needsof = 1;
2601
2602                         if (td->hcsplt == 0 || td->tt_scheduled != 0)
2603                                 continue;
2604
2605                         /* start ASAP */
2606                         td->tt_start_slot = temp;
2607                         td->tt_scheduled = 1;
2608                         TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2609                         TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2610                 }
2611         }
2612
2613         /* Put TT transfers in execution order at the end */
2614         TAILQ_CONCAT(&sc->sc_bus.intr_q.head, &head, wait_entry);
2615
2616         /* move all TT transfers in front, keeping the current order */
2617         TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2618                 td = xfer->td_transfer_cache;
2619                 if (td == NULL || td->hcsplt == 0)
2620                         continue;
2621                 TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2622                 TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2623         }
2624         TAILQ_CONCAT(&head, &sc->sc_bus.intr_q.head, wait_entry);
2625         TAILQ_CONCAT(&sc->sc_bus.intr_q.head, &head, wait_entry);
2626
2627         /* put non-TT non-ISOCHRONOUS transfers last */
2628         TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2629                 td = xfer->td_transfer_cache;
2630                 if (td == NULL || td->hcsplt != 0 || td->ep_type == UE_ISOCHRONOUS)
2631                         continue;
2632                 TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2633                 TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2634         }
2635         TAILQ_CONCAT(&sc->sc_bus.intr_q.head, &head, wait_entry);
2636
2637         if ((temp & 7) == 0) {
2638
2639                 DPRINTFN(12, "SOF interrupt #%d, needsof=%d\n",
2640                     (int)temp, (int)sc->sc_needsof);
2641
2642                 /* update SOF IRQ mask */
2643                 if (sc->sc_irq_mask & GINTMSK_SOFMSK) {
2644                         if (sc->sc_needsof == 0) {
2645                                 sc->sc_irq_mask &= ~GINTMSK_SOFMSK;
2646                                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2647                         }
2648                 } else {
2649                         if (sc->sc_needsof != 0) {
2650                                 sc->sc_irq_mask |= GINTMSK_SOFMSK;
2651                                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2652                         }
2653                 }
2654
2655                 /* clear need SOF flag */
2656                 sc->sc_needsof = 0;
2657         }
2658         return (1);
2659 }
2660
2661 static void
2662 dwc_otg_interrupt_poll_locked(struct dwc_otg_softc *sc)
2663 {
2664         struct usb_xfer *xfer;
2665         uint32_t count;
2666         uint32_t temp;
2667         uint32_t haint;
2668         uint8_t got_rx_status;
2669         uint8_t x;
2670
2671         if (sc->sc_flags.status_device_mode == 0) {
2672                 /*
2673                  * Update host transfer schedule, so that new
2674                  * transfers can be issued:
2675                  */
2676                 dwc_otg_update_host_transfer_schedule_locked(sc);
2677         }
2678         count = 0;
2679 repeat:
2680         if (++count == 16) {
2681                 /* give other interrupts a chance */
2682                 DPRINTF("Yield\n");
2683                 return;
2684         }
2685
2686         /* get all host channel interrupts */
2687         haint = DWC_OTG_READ_4(sc, DOTG_HAINT);
2688         while (1) {
2689                 x = ffs(haint) - 1;
2690                 if (x >= sc->sc_host_ch_max)
2691                         break;
2692                 temp = DWC_OTG_READ_4(sc, DOTG_HCINT(x));
2693                 DWC_OTG_WRITE_4(sc, DOTG_HCINT(x), temp);
2694                 temp &= ~HCINT_SOFTWARE_ONLY;
2695                 sc->sc_chan_state[x].hcint |= temp;
2696                 haint &= ~(1U << x);
2697         }
2698
2699         if (sc->sc_last_rx_status == 0) {
2700
2701                 temp = DWC_OTG_READ_4(sc, DOTG_GINTSTS);
2702                 if (temp & GINTSTS_RXFLVL) {
2703                         /* pop current status */
2704                         sc->sc_last_rx_status =
2705                             DWC_OTG_READ_4(sc, DOTG_GRXSTSPD);
2706                 }
2707
2708                 if (sc->sc_last_rx_status != 0) {
2709
2710                         uint8_t ep_no;
2711
2712                         temp = sc->sc_last_rx_status &
2713                             GRXSTSRD_PKTSTS_MASK;
2714
2715                         /* non-data messages we simply skip */
2716                         if (temp != GRXSTSRD_STP_DATA &&
2717                             temp != GRXSTSRD_STP_COMPLETE &&
2718                             temp != GRXSTSRD_OUT_DATA) {
2719                                 /* check for halted channel */
2720                                 if (temp == GRXSTSRH_HALTED) {
2721                                         ep_no = GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status);
2722                                         sc->sc_chan_state[ep_no].wait_halted = 0;
2723                                         DPRINTFN(5, "channel halt complete ch=%u\n", ep_no);
2724                                 }
2725                                 /* store bytes and FIFO offset */
2726                                 sc->sc_current_rx_bytes = 0;
2727                                 sc->sc_current_rx_fifo = 0;
2728
2729                                 /* acknowledge status */
2730                                 dwc_otg_common_rx_ack(sc);
2731                                 goto repeat;
2732                         }
2733
2734                         temp = GRXSTSRD_BCNT_GET(
2735                             sc->sc_last_rx_status);
2736                         ep_no = GRXSTSRD_CHNUM_GET(
2737                             sc->sc_last_rx_status);
2738
2739                         /* store bytes and FIFO offset */
2740                         sc->sc_current_rx_bytes = (temp + 3) & ~3;
2741                         sc->sc_current_rx_fifo = DOTG_DFIFO(ep_no);
2742
2743                         DPRINTF("Reading %d bytes from ep %d\n", temp, ep_no);
2744
2745                         /* check if we should dump the data */
2746                         if (!(sc->sc_active_rx_ep & (1U << ep_no))) {
2747                                 dwc_otg_common_rx_ack(sc);
2748                                 goto repeat;
2749                         }
2750
2751                         got_rx_status = 1;
2752
2753                         DPRINTFN(5, "RX status = 0x%08x: ch=%d pid=%d bytes=%d sts=%d\n",
2754                             sc->sc_last_rx_status, ep_no,
2755                             (sc->sc_last_rx_status >> 15) & 3,
2756                             GRXSTSRD_BCNT_GET(sc->sc_last_rx_status),
2757                             (sc->sc_last_rx_status >> 17) & 15);
2758                 } else {
2759                         got_rx_status = 0;
2760                 }
2761         } else {
2762                 uint8_t ep_no;
2763
2764                 ep_no = GRXSTSRD_CHNUM_GET(
2765                     sc->sc_last_rx_status);
2766
2767                 /* check if we should dump the data */
2768                 if (!(sc->sc_active_rx_ep & (1U << ep_no))) {
2769                         dwc_otg_common_rx_ack(sc);
2770                         goto repeat;
2771                 }
2772
2773                 got_rx_status = 1;
2774         }
2775
2776         /* execute FIFOs */
2777         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry)
2778                 dwc_otg_xfer_do_fifo(sc, xfer);
2779
2780         if (got_rx_status) {
2781                 /* check if data was consumed */
2782                 if (sc->sc_last_rx_status == 0)
2783                         goto repeat;
2784
2785                 /* disable RX FIFO level interrupt */
2786                 sc->sc_irq_mask &= ~GINTMSK_RXFLVLMSK;
2787                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2788         }
2789 }
2790
2791 static void
2792 dwc_otg_interrupt_complete_locked(struct dwc_otg_softc *sc)
2793 {
2794         struct usb_xfer *xfer;
2795 repeat:
2796         /* scan for completion events */
2797         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
2798                 if (dwc_otg_xfer_do_complete_locked(sc, xfer))
2799                         goto repeat;
2800         }
2801 }
2802
2803 static void
2804 dwc_otg_vbus_interrupt(struct dwc_otg_softc *sc, uint8_t is_on)
2805 {
2806         DPRINTFN(5, "vbus = %u\n", is_on);
2807
2808         /*
2809          * If the USB host mode is forced, then assume VBUS is always
2810          * present else rely on the input to this function:
2811          */
2812         if ((is_on != 0) || (sc->sc_mode == DWC_MODE_HOST)) {
2813
2814                 if (!sc->sc_flags.status_vbus) {
2815                         sc->sc_flags.status_vbus = 1;
2816
2817                         /* complete root HUB interrupt endpoint */
2818
2819                         dwc_otg_root_intr(sc);
2820                 }
2821         } else {
2822                 if (sc->sc_flags.status_vbus) {
2823                         sc->sc_flags.status_vbus = 0;
2824                         sc->sc_flags.status_bus_reset = 0;
2825                         sc->sc_flags.status_suspend = 0;
2826                         sc->sc_flags.change_suspend = 0;
2827                         sc->sc_flags.change_connect = 1;
2828
2829                         /* complete root HUB interrupt endpoint */
2830
2831                         dwc_otg_root_intr(sc);
2832                 }
2833         }
2834 }
2835
2836 int
2837 dwc_otg_filter_interrupt(void *arg)
2838 {
2839         struct dwc_otg_softc *sc = arg;
2840         int retval = FILTER_HANDLED;
2841         uint32_t status;
2842
2843         USB_BUS_SPIN_LOCK(&sc->sc_bus);
2844
2845         /* read and clear interrupt status */
2846         status = DWC_OTG_READ_4(sc, DOTG_GINTSTS);
2847
2848         /* clear interrupts we are handling here */
2849         DWC_OTG_WRITE_4(sc, DOTG_GINTSTS, status & ~DWC_OTG_MSK_GINT_THREAD_IRQ);
2850
2851         /* check for USB state change interrupts */
2852         if ((status & DWC_OTG_MSK_GINT_THREAD_IRQ) != 0)
2853                 retval = FILTER_SCHEDULE_THREAD;
2854
2855         /* clear FIFO empty interrupts */
2856         if (status & sc->sc_irq_mask &
2857             (GINTSTS_PTXFEMP | GINTSTS_NPTXFEMP)) {
2858                 sc->sc_irq_mask &= ~(GINTSTS_PTXFEMP | GINTSTS_NPTXFEMP);
2859                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2860         }
2861         /* clear all IN endpoint interrupts */
2862         if (status & GINTSTS_IEPINT) {
2863                 uint32_t temp;
2864                 uint8_t x;
2865
2866                 for (x = 0; x != sc->sc_dev_in_ep_max; x++) {
2867                         temp = DWC_OTG_READ_4(sc, DOTG_DIEPINT(x));
2868                         /*
2869                          * NOTE: Need to clear all interrupt bits,
2870                          * because some appears to be unmaskable and
2871                          * can cause an interrupt loop:
2872                          */
2873                         if (temp != 0)
2874                                 DWC_OTG_WRITE_4(sc, DOTG_DIEPINT(x), temp);
2875                 }
2876         }
2877
2878         /* poll FIFOs, if any */
2879         dwc_otg_interrupt_poll_locked(sc);
2880
2881         if (sc->sc_xfer_complete != 0)
2882                 retval = FILTER_SCHEDULE_THREAD;
2883
2884         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
2885
2886         return (retval);
2887 }
2888
2889 void
2890 dwc_otg_interrupt(void *arg)
2891 {
2892         struct dwc_otg_softc *sc = arg;
2893         uint32_t status;
2894
2895         USB_BUS_LOCK(&sc->sc_bus);
2896         USB_BUS_SPIN_LOCK(&sc->sc_bus);
2897
2898         /* read and clear interrupt status */
2899         status = DWC_OTG_READ_4(sc, DOTG_GINTSTS);
2900
2901         /* clear interrupts we are handling here */
2902         DWC_OTG_WRITE_4(sc, DOTG_GINTSTS, status & DWC_OTG_MSK_GINT_THREAD_IRQ);
2903
2904         DPRINTFN(14, "GINTSTS=0x%08x HAINT=0x%08x HFNUM=0x%08x\n",
2905             status, DWC_OTG_READ_4(sc, DOTG_HAINT),
2906             DWC_OTG_READ_4(sc, DOTG_HFNUM));
2907
2908         if (status & GINTSTS_USBRST) {
2909
2910                 /* set correct state */
2911                 sc->sc_flags.status_device_mode = 1;
2912                 sc->sc_flags.status_bus_reset = 0;
2913                 sc->sc_flags.status_suspend = 0;
2914                 sc->sc_flags.change_suspend = 0;
2915                 sc->sc_flags.change_connect = 1;
2916
2917                 /* Disable SOF interrupt */
2918                 sc->sc_irq_mask &= ~GINTMSK_SOFMSK;
2919                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2920
2921                 /* complete root HUB interrupt endpoint */
2922                 dwc_otg_root_intr(sc);
2923         }
2924
2925         /* check for any bus state change interrupts */
2926         if (status & GINTSTS_ENUMDONE) {
2927
2928                 uint32_t temp;
2929
2930                 DPRINTFN(5, "end of reset\n");
2931
2932                 /* set correct state */
2933                 sc->sc_flags.status_device_mode = 1;
2934                 sc->sc_flags.status_bus_reset = 1;
2935                 sc->sc_flags.status_suspend = 0;
2936                 sc->sc_flags.change_suspend = 0;
2937                 sc->sc_flags.change_connect = 1;
2938                 sc->sc_flags.status_low_speed = 0;
2939                 sc->sc_flags.port_enabled = 1;
2940
2941                 /* reset FIFOs */
2942                 (void) dwc_otg_init_fifo(sc, DWC_MODE_DEVICE);
2943
2944                 /* reset function address */
2945                 dwc_otg_set_address(sc, 0);
2946
2947                 /* figure out enumeration speed */
2948                 temp = DWC_OTG_READ_4(sc, DOTG_DSTS);
2949                 if (DSTS_ENUMSPD_GET(temp) == DSTS_ENUMSPD_HI)
2950                         sc->sc_flags.status_high_speed = 1;
2951                 else
2952                         sc->sc_flags.status_high_speed = 0;
2953
2954                 /*
2955                  * Disable resume and SOF interrupt, and enable
2956                  * suspend and RX frame interrupt:
2957                  */
2958                 sc->sc_irq_mask &= ~(GINTMSK_WKUPINTMSK | GINTMSK_SOFMSK);
2959                 sc->sc_irq_mask |= GINTMSK_USBSUSPMSK;
2960                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2961
2962                 /* complete root HUB interrupt endpoint */
2963                 dwc_otg_root_intr(sc);
2964         }
2965
2966         if (status & GINTSTS_PRTINT) {
2967                 uint32_t hprt;
2968
2969                 hprt = DWC_OTG_READ_4(sc, DOTG_HPRT);
2970
2971                 /* clear change bits */
2972                 DWC_OTG_WRITE_4(sc, DOTG_HPRT, (hprt & (
2973                     HPRT_PRTPWR | HPRT_PRTENCHNG |
2974                     HPRT_PRTCONNDET | HPRT_PRTOVRCURRCHNG)) |
2975                     sc->sc_hprt_val);
2976
2977                 DPRINTFN(12, "GINTSTS=0x%08x, HPRT=0x%08x\n", status, hprt);
2978
2979                 sc->sc_flags.status_device_mode = 0;
2980
2981                 if (hprt & HPRT_PRTCONNSTS)
2982                         sc->sc_flags.status_bus_reset = 1;
2983                 else
2984                         sc->sc_flags.status_bus_reset = 0;
2985
2986                 if ((hprt & HPRT_PRTENCHNG) &&
2987                     (hprt & HPRT_PRTENA) == 0)
2988                         sc->sc_flags.change_enabled = 1;
2989
2990                 if (hprt & HPRT_PRTENA)
2991                         sc->sc_flags.port_enabled = 1;
2992                 else
2993                         sc->sc_flags.port_enabled = 0;
2994
2995                 if (hprt & HPRT_PRTOVRCURRCHNG)
2996                         sc->sc_flags.change_over_current = 1;
2997
2998                 if (hprt & HPRT_PRTOVRCURRACT)
2999                         sc->sc_flags.port_over_current = 1;
3000                 else
3001                         sc->sc_flags.port_over_current = 0;
3002
3003                 if (hprt & HPRT_PRTPWR)
3004                         sc->sc_flags.port_powered = 1;
3005                 else
3006                         sc->sc_flags.port_powered = 0;
3007
3008                 if (((hprt & HPRT_PRTSPD_MASK)
3009                     >> HPRT_PRTSPD_SHIFT) == HPRT_PRTSPD_LOW)
3010                         sc->sc_flags.status_low_speed = 1;
3011                 else
3012                         sc->sc_flags.status_low_speed = 0;
3013
3014                 if (((hprt & HPRT_PRTSPD_MASK)
3015                     >> HPRT_PRTSPD_SHIFT) == HPRT_PRTSPD_HIGH)
3016                         sc->sc_flags.status_high_speed = 1;
3017                 else
3018                         sc->sc_flags.status_high_speed = 0;
3019
3020                 if (hprt & HPRT_PRTCONNDET)
3021                         sc->sc_flags.change_connect = 1;
3022
3023                 if (hprt & HPRT_PRTSUSP)
3024                         dwc_otg_suspend_irq(sc);
3025                 else
3026                         dwc_otg_resume_irq(sc);
3027
3028                 /* complete root HUB interrupt endpoint */
3029                 dwc_otg_root_intr(sc);
3030
3031                 /* update host frame interval */
3032                 dwc_otg_update_host_frame_interval(sc);
3033         }
3034
3035         /*
3036          * If resume and suspend is set at the same time we interpret
3037          * that like RESUME. Resume is set when there is at least 3
3038          * milliseconds of inactivity on the USB BUS.
3039          */
3040         if (status & GINTSTS_WKUPINT) {
3041
3042                 DPRINTFN(5, "resume interrupt\n");
3043
3044                 dwc_otg_resume_irq(sc);
3045
3046         } else if (status & GINTSTS_USBSUSP) {
3047
3048                 DPRINTFN(5, "suspend interrupt\n");
3049
3050                 dwc_otg_suspend_irq(sc);
3051         }
3052         /* check VBUS */
3053         if (status & (GINTSTS_USBSUSP |
3054             GINTSTS_USBRST |
3055             GINTMSK_OTGINTMSK |
3056             GINTSTS_SESSREQINT)) {
3057                 uint32_t temp;
3058
3059                 temp = DWC_OTG_READ_4(sc, DOTG_GOTGCTL);
3060
3061                 DPRINTFN(5, "GOTGCTL=0x%08x\n", temp);
3062
3063                 dwc_otg_vbus_interrupt(sc,
3064                     (temp & (GOTGCTL_ASESVLD | GOTGCTL_BSESVLD)) ? 1 : 0);
3065         }
3066
3067         if (sc->sc_xfer_complete != 0) {
3068                 sc->sc_xfer_complete = 0;
3069
3070                 /* complete FIFOs, if any */
3071                 dwc_otg_interrupt_complete_locked(sc);
3072         }
3073         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3074         USB_BUS_UNLOCK(&sc->sc_bus);
3075 }
3076
3077 static void
3078 dwc_otg_setup_standard_chain_sub(struct dwc_otg_std_temp *temp)
3079 {
3080         struct dwc_otg_td *td;
3081
3082         /* get current Transfer Descriptor */
3083         td = temp->td_next;
3084         temp->td = td;
3085
3086         /* prepare for next TD */
3087         temp->td_next = td->obj_next;
3088
3089         /* fill out the Transfer Descriptor */
3090         td->func = temp->func;
3091         td->pc = temp->pc;
3092         td->offset = temp->offset;
3093         td->remainder = temp->len;
3094         td->tx_bytes = 0;
3095         td->error_any = 0;
3096         td->error_stall = 0;
3097         td->npkt = 0;
3098         td->did_stall = temp->did_stall;
3099         td->short_pkt = temp->short_pkt;
3100         td->alt_next = temp->setup_alt_next;
3101         td->set_toggle = 0;
3102         td->got_short = 0;
3103         td->did_nak = 0;
3104         td->channel[0] = DWC_OTG_MAX_CHANNELS;
3105         td->channel[1] = DWC_OTG_MAX_CHANNELS;
3106         td->channel[2] = DWC_OTG_MAX_CHANNELS;
3107         td->state = 0;
3108         td->errcnt = 0;
3109         td->tt_scheduled = 0;
3110         td->tt_xactpos = HCSPLT_XACTPOS_BEGIN;
3111 }
3112
3113 static void
3114 dwc_otg_setup_standard_chain(struct usb_xfer *xfer)
3115 {
3116         struct dwc_otg_std_temp temp;
3117         struct dwc_otg_td *td;
3118         uint32_t x;
3119         uint8_t need_sync;
3120         uint8_t is_host;
3121
3122         DPRINTFN(9, "addr=%d endpt=%d sumlen=%d speed=%d\n",
3123             xfer->address, UE_GET_ADDR(xfer->endpointno),
3124             xfer->sumlen, usbd_get_speed(xfer->xroot->udev));
3125
3126         temp.max_frame_size = xfer->max_frame_size;
3127
3128         td = xfer->td_start[0];
3129         xfer->td_transfer_first = td;
3130         xfer->td_transfer_cache = td;
3131
3132         /* setup temp */
3133
3134         temp.pc = NULL;
3135         temp.td = NULL;
3136         temp.td_next = xfer->td_start[0];
3137         temp.offset = 0;
3138         temp.setup_alt_next = xfer->flags_int.short_frames_ok ||
3139             xfer->flags_int.isochronous_xfr;
3140         temp.did_stall = !xfer->flags_int.control_stall;
3141
3142         is_host = (xfer->xroot->udev->flags.usb_mode == USB_MODE_HOST);
3143
3144         /* check if we should prepend a setup message */
3145
3146         if (xfer->flags_int.control_xfr) {
3147                 if (xfer->flags_int.control_hdr) {
3148
3149                         if (is_host)
3150                                 temp.func = &dwc_otg_host_setup_tx;
3151                         else
3152                                 temp.func = &dwc_otg_setup_rx;
3153
3154                         temp.len = xfer->frlengths[0];
3155                         temp.pc = xfer->frbuffers + 0;
3156                         temp.short_pkt = temp.len ? 1 : 0;
3157
3158                         /* check for last frame */
3159                         if (xfer->nframes == 1) {
3160                                 /* no STATUS stage yet, SETUP is last */
3161                                 if (xfer->flags_int.control_act)
3162                                         temp.setup_alt_next = 0;
3163                         }
3164
3165                         dwc_otg_setup_standard_chain_sub(&temp);
3166                 }
3167                 x = 1;
3168         } else {
3169                 x = 0;
3170         }
3171
3172         if (x != xfer->nframes) {
3173                 if (xfer->endpointno & UE_DIR_IN) {
3174                         if (is_host) {
3175                                 temp.func = &dwc_otg_host_data_rx;
3176                                 need_sync = 0;
3177                         } else {
3178                                 temp.func = &dwc_otg_data_tx;
3179                                 need_sync = 1;
3180                         }
3181                 } else {
3182                         if (is_host) {
3183                                 temp.func = &dwc_otg_host_data_tx;
3184                                 need_sync = 0;
3185                         } else {
3186                                 temp.func = &dwc_otg_data_rx;
3187                                 need_sync = 0;
3188                         }
3189                 }
3190
3191                 /* setup "pc" pointer */
3192                 temp.pc = xfer->frbuffers + x;
3193         } else {
3194                 need_sync = 0;
3195         }
3196         while (x != xfer->nframes) {
3197
3198                 /* DATA0 / DATA1 message */
3199
3200                 temp.len = xfer->frlengths[x];
3201
3202                 x++;
3203
3204                 if (x == xfer->nframes) {
3205                         if (xfer->flags_int.control_xfr) {
3206                                 if (xfer->flags_int.control_act) {
3207                                         temp.setup_alt_next = 0;
3208                                 }
3209                         } else {
3210                                 temp.setup_alt_next = 0;
3211                         }
3212                 }
3213                 if (temp.len == 0) {
3214
3215                         /* make sure that we send an USB packet */
3216
3217                         temp.short_pkt = 0;
3218
3219                 } else {
3220
3221                         /* regular data transfer */
3222
3223                         temp.short_pkt = (xfer->flags.force_short_xfer ? 0 : 1);
3224                 }
3225
3226                 dwc_otg_setup_standard_chain_sub(&temp);
3227
3228                 if (xfer->flags_int.isochronous_xfr) {
3229                         temp.offset += temp.len;
3230                 } else {
3231                         /* get next Page Cache pointer */
3232                         temp.pc = xfer->frbuffers + x;
3233                 }
3234         }
3235
3236         if (xfer->flags_int.control_xfr) {
3237
3238                 /* always setup a valid "pc" pointer for status and sync */
3239                 temp.pc = xfer->frbuffers + 0;
3240                 temp.len = 0;
3241                 temp.short_pkt = 0;
3242                 temp.setup_alt_next = 0;
3243
3244                 /* check if we need to sync */
3245                 if (need_sync) {
3246                         /* we need a SYNC point after TX */
3247                         temp.func = &dwc_otg_data_tx_sync;
3248                         dwc_otg_setup_standard_chain_sub(&temp);
3249                 }
3250
3251                 /* check if we should append a status stage */
3252                 if (!xfer->flags_int.control_act) {
3253
3254                         /*
3255                          * Send a DATA1 message and invert the current
3256                          * endpoint direction.
3257                          */
3258                         if (xfer->endpointno & UE_DIR_IN) {
3259                                 if (is_host) {
3260                                         temp.func = &dwc_otg_host_data_tx;
3261                                         need_sync = 0;
3262                                 } else {
3263                                         temp.func = &dwc_otg_data_rx;
3264                                         need_sync = 0;
3265                                 }
3266                         } else {
3267                                 if (is_host) {
3268                                         temp.func = &dwc_otg_host_data_rx;
3269                                         need_sync = 0;
3270                                 } else {
3271                                         temp.func = &dwc_otg_data_tx;
3272                                         need_sync = 1;
3273                                 }
3274                         }
3275
3276                         dwc_otg_setup_standard_chain_sub(&temp);
3277
3278                         /* data toggle should be DATA1 */
3279                         td = temp.td;
3280                         td->set_toggle = 1;
3281
3282                         if (need_sync) {
3283                                 /* we need a SYNC point after TX */
3284                                 temp.func = &dwc_otg_data_tx_sync;
3285                                 dwc_otg_setup_standard_chain_sub(&temp);
3286                         }
3287                 }
3288         } else {
3289                 /* check if we need to sync */
3290                 if (need_sync) {
3291
3292                         temp.pc = xfer->frbuffers + 0;
3293                         temp.len = 0;
3294                         temp.short_pkt = 0;
3295                         temp.setup_alt_next = 0;
3296
3297                         /* we need a SYNC point after TX */
3298                         temp.func = &dwc_otg_data_tx_sync;
3299                         dwc_otg_setup_standard_chain_sub(&temp);
3300                 }
3301         }
3302
3303         /* must have at least one frame! */
3304         td = temp.td;
3305         xfer->td_transfer_last = td;
3306
3307         if (is_host) {
3308
3309                 struct dwc_otg_softc *sc;
3310                 uint32_t hcchar;
3311                 uint32_t hcsplt;
3312
3313                 sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
3314
3315                 /* get first again */
3316                 td = xfer->td_transfer_first;
3317                 td->toggle = (xfer->endpoint->toggle_next ? 1 : 0);
3318
3319                 hcchar =
3320                         (xfer->address << HCCHAR_DEVADDR_SHIFT) |
3321                         ((xfer->endpointno & UE_ADDR) << HCCHAR_EPNUM_SHIFT) |
3322                         (xfer->max_packet_size << HCCHAR_MPS_SHIFT) |
3323                         HCCHAR_CHENA;
3324
3325                 /*
3326                  * We are not always able to meet the timing
3327                  * requirements of the USB interrupt endpoint's
3328                  * complete split token, when doing transfers going
3329                  * via a transaction translator. Use the CONTROL
3330                  * transfer type instead of the INTERRUPT transfer
3331                  * type in general, as a means to workaround
3332                  * that. This trick should work for both FULL and LOW
3333                  * speed USB traffic going through a TT. For non-TT
3334                  * traffic it works as well. The reason for using
3335                  * CONTROL type instead of BULK is that some TTs might
3336                  * reject LOW speed BULK traffic.
3337                  */
3338                 if (td->ep_type == UE_INTERRUPT)
3339                         hcchar |= (UE_CONTROL << HCCHAR_EPTYPE_SHIFT);
3340                 else
3341                         hcchar |= (td->ep_type << HCCHAR_EPTYPE_SHIFT);
3342
3343                 if (UE_GET_DIR(xfer->endpointno) == UE_DIR_IN)
3344                         hcchar |= HCCHAR_EPDIR_IN;
3345
3346                 switch (xfer->xroot->udev->speed) {
3347                 case USB_SPEED_LOW:
3348                         hcchar |= HCCHAR_LSPDDEV;
3349                         /* FALLTHROUGH */
3350                 case USB_SPEED_FULL:
3351                         /* check if root HUB port is running High Speed */
3352                         if (dwc_otg_uses_split(xfer->xroot->udev)) {
3353                                 hcsplt = HCSPLT_SPLTENA |
3354                                     (xfer->xroot->udev->hs_port_no <<
3355                                     HCSPLT_PRTADDR_SHIFT) |
3356                                     (xfer->xroot->udev->hs_hub_addr <<
3357                                     HCSPLT_HUBADDR_SHIFT);
3358                         } else {
3359                                 hcsplt = 0;
3360                         }
3361                         if (td->ep_type == UE_INTERRUPT) {
3362                                 uint32_t ival;
3363                                 ival = xfer->interval / DWC_OTG_HOST_TIMER_RATE;
3364                                 if (ival == 0)
3365                                         ival = 1;
3366                                 else if (ival > 127)
3367                                         ival = 127;
3368                                 td->tmr_val = sc->sc_tmr_val + ival;
3369                                 td->tmr_res = ival;
3370                         } else if (td->ep_type == UE_ISOCHRONOUS) {
3371                                 td->tmr_res = 1;
3372                                 td->tmr_val = sc->sc_last_frame_num;
3373                                 if (td->hcchar & HCCHAR_EPDIR_IN)
3374                                         td->tmr_val++;
3375                         } else {
3376                                 td->tmr_val = 0;
3377                                 td->tmr_res = (uint8_t)sc->sc_last_frame_num;
3378                         }
3379                         break;
3380                 case USB_SPEED_HIGH:
3381                         hcsplt = 0;
3382                         if (td->ep_type == UE_INTERRUPT) {
3383                                 uint32_t ival;
3384                                 hcchar |= ((xfer->max_packet_count & 3)
3385                                     << HCCHAR_MC_SHIFT);
3386                                 ival = xfer->interval / DWC_OTG_HOST_TIMER_RATE;
3387                                 if (ival == 0)
3388                                         ival = 1;
3389                                 else if (ival > 127)
3390                                         ival = 127;
3391                                 td->tmr_val = sc->sc_tmr_val + ival;
3392                                 td->tmr_res = ival;
3393                         } else if (td->ep_type == UE_ISOCHRONOUS) {
3394                                 hcchar |= ((xfer->max_packet_count & 3)
3395                                     << HCCHAR_MC_SHIFT);
3396                                 td->tmr_res = 1 << usbd_xfer_get_fps_shift(xfer);
3397                                 td->tmr_val = sc->sc_last_frame_num;
3398                                 if (td->hcchar & HCCHAR_EPDIR_IN)
3399                                         td->tmr_val += td->tmr_res;
3400
3401                         } else {
3402                                 td->tmr_val = 0;
3403                                 td->tmr_res = (uint8_t)sc->sc_last_frame_num;
3404                         }
3405                         break;
3406                 default:
3407                         hcsplt = 0;
3408                         td->tmr_val = 0;
3409                         td->tmr_res = 0;
3410                         break;
3411                 }
3412
3413                 /* store configuration in all TD's */
3414                 while (1) {
3415                         td->hcchar = hcchar;
3416                         td->hcsplt = hcsplt;
3417
3418                         if (((void *)td) == xfer->td_transfer_last)
3419                                 break;
3420
3421                         td = td->obj_next;
3422                 }
3423         }
3424 }
3425
3426 static void
3427 dwc_otg_timeout(void *arg)
3428 {
3429         struct usb_xfer *xfer = arg;
3430
3431         DPRINTF("xfer=%p\n", xfer);
3432
3433         USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
3434
3435         /* transfer is transferred */
3436         dwc_otg_device_done(xfer, USB_ERR_TIMEOUT);
3437 }
3438
3439 static void
3440 dwc_otg_start_standard_chain(struct usb_xfer *xfer)
3441 {
3442         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
3443
3444         DPRINTFN(9, "\n");
3445
3446         /*
3447          * Poll one time in device mode, which will turn on the
3448          * endpoint interrupts. Else wait for SOF interrupt in host
3449          * mode.
3450          */
3451         USB_BUS_SPIN_LOCK(&sc->sc_bus);
3452
3453         if (sc->sc_flags.status_device_mode != 0) {
3454                 dwc_otg_xfer_do_fifo(sc, xfer);
3455                 if (dwc_otg_xfer_do_complete_locked(sc, xfer))
3456                         goto done;
3457         } else {
3458                 struct dwc_otg_td *td = xfer->td_transfer_cache;
3459                 if (td->ep_type == UE_ISOCHRONOUS &&
3460                     (td->hcchar & HCCHAR_EPDIR_IN) == 0) {
3461                         /*
3462                          * Need to start ISOCHRONOUS OUT transfer ASAP
3463                          * because execution is delayed by one 125us
3464                          * microframe:
3465                          */
3466                         dwc_otg_xfer_do_fifo(sc, xfer);
3467                         if (dwc_otg_xfer_do_complete_locked(sc, xfer))
3468                                 goto done;
3469                 }
3470         }
3471
3472         /* put transfer on interrupt queue */
3473         usbd_transfer_enqueue(&xfer->xroot->bus->intr_q, xfer);
3474
3475         /* start timeout, if any */
3476         if (xfer->timeout != 0) {
3477                 usbd_transfer_timeout_ms(xfer,
3478                     &dwc_otg_timeout, xfer->timeout);
3479         }
3480
3481         if (sc->sc_flags.status_device_mode != 0)
3482                 goto done;
3483
3484         /* enable SOF interrupt, if any */
3485         dwc_otg_enable_sof_irq(sc);
3486 done:
3487         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3488 }
3489
3490 static void
3491 dwc_otg_root_intr(struct dwc_otg_softc *sc)
3492 {
3493         DPRINTFN(9, "\n");
3494
3495         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
3496
3497         /* set port bit */
3498         sc->sc_hub_idata[0] = 0x02;     /* we only have one port */
3499
3500         uhub_root_intr(&sc->sc_bus, sc->sc_hub_idata,
3501             sizeof(sc->sc_hub_idata));
3502 }
3503
3504 static usb_error_t
3505 dwc_otg_standard_done_sub(struct usb_xfer *xfer)
3506 {
3507         struct dwc_otg_td *td;
3508         uint32_t len;
3509         usb_error_t error;
3510
3511         DPRINTFN(9, "\n");
3512
3513         td = xfer->td_transfer_cache;
3514
3515         do {
3516                 len = td->remainder;
3517
3518                 /* store last data toggle */
3519                 xfer->endpoint->toggle_next = td->toggle;
3520
3521                 if (xfer->aframes != xfer->nframes) {
3522                         /*
3523                          * Verify the length and subtract
3524                          * the remainder from "frlengths[]":
3525                          */
3526                         if (len > xfer->frlengths[xfer->aframes]) {
3527                                 td->error_any = 1;
3528                         } else {
3529                                 xfer->frlengths[xfer->aframes] -= len;
3530                         }
3531                 }
3532                 /* Check for transfer error */
3533                 if (td->error_any) {
3534                         /* the transfer is finished */
3535                         error = (td->error_stall ?
3536                             USB_ERR_STALLED : USB_ERR_IOERROR);
3537                         td = NULL;
3538                         break;
3539                 }
3540                 /* Check for short transfer */
3541                 if (len > 0) {
3542                         if (xfer->flags_int.short_frames_ok ||
3543                             xfer->flags_int.isochronous_xfr) {
3544                                 /* follow alt next */
3545                                 if (td->alt_next) {
3546                                         td = td->obj_next;
3547                                 } else {
3548                                         td = NULL;
3549                                 }
3550                         } else {
3551                                 /* the transfer is finished */
3552                                 td = NULL;
3553                         }
3554                         error = 0;
3555                         break;
3556                 }
3557                 td = td->obj_next;
3558
3559                 /* this USB frame is complete */
3560                 error = 0;
3561                 break;
3562
3563         } while (0);
3564
3565         /* update transfer cache */
3566
3567         xfer->td_transfer_cache = td;
3568
3569         return (error);
3570 }
3571
3572 static void
3573 dwc_otg_standard_done(struct usb_xfer *xfer)
3574 {
3575         usb_error_t err = 0;
3576
3577         DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
3578             xfer, xfer->endpoint);
3579
3580         /* reset scanner */
3581
3582         xfer->td_transfer_cache = xfer->td_transfer_first;
3583
3584         if (xfer->flags_int.control_xfr) {
3585
3586                 if (xfer->flags_int.control_hdr) {
3587
3588                         err = dwc_otg_standard_done_sub(xfer);
3589                 }
3590                 xfer->aframes = 1;
3591
3592                 if (xfer->td_transfer_cache == NULL) {
3593                         goto done;
3594                 }
3595         }
3596         while (xfer->aframes != xfer->nframes) {
3597
3598                 err = dwc_otg_standard_done_sub(xfer);
3599                 xfer->aframes++;
3600
3601                 if (xfer->td_transfer_cache == NULL) {
3602                         goto done;
3603                 }
3604         }
3605
3606         if (xfer->flags_int.control_xfr &&
3607             !xfer->flags_int.control_act) {
3608
3609                 err = dwc_otg_standard_done_sub(xfer);
3610         }
3611 done:
3612         dwc_otg_device_done(xfer, err);
3613 }
3614
3615 /*------------------------------------------------------------------------*
3616  *      dwc_otg_device_done
3617  *
3618  * NOTE: this function can be called more than one time on the
3619  * same USB transfer!
3620  *------------------------------------------------------------------------*/
3621 static void
3622 dwc_otg_device_done(struct usb_xfer *xfer, usb_error_t error)
3623 {
3624         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
3625
3626         DPRINTFN(9, "xfer=%p, endpoint=%p, error=%d\n",
3627             xfer, xfer->endpoint, error);
3628
3629         USB_BUS_SPIN_LOCK(&sc->sc_bus);
3630
3631         if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) {
3632                 /* Interrupts are cleared by the interrupt handler */
3633         } else {
3634                 struct dwc_otg_td *td;
3635
3636                 td = xfer->td_transfer_cache;
3637                 if (td != NULL)
3638                         dwc_otg_host_channel_free(sc, td);
3639         }
3640         /* dequeue transfer and start next transfer */
3641         usbd_transfer_done(xfer, error);
3642
3643         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3644 }
3645
3646 static void
3647 dwc_otg_xfer_stall(struct usb_xfer *xfer)
3648 {
3649         dwc_otg_device_done(xfer, USB_ERR_STALLED);
3650 }
3651
3652 static void
3653 dwc_otg_set_stall(struct usb_device *udev,
3654     struct usb_endpoint *ep, uint8_t *did_stall)
3655 {
3656         struct dwc_otg_softc *sc;
3657         uint32_t temp;
3658         uint32_t reg;
3659         uint8_t ep_no;
3660
3661         USB_BUS_LOCK_ASSERT(udev->bus, MA_OWNED);
3662
3663         /* check mode */
3664         if (udev->flags.usb_mode != USB_MODE_DEVICE) {
3665                 /* not supported */
3666                 return;
3667         }
3668
3669         sc = DWC_OTG_BUS2SC(udev->bus);
3670
3671         USB_BUS_SPIN_LOCK(&sc->sc_bus);
3672
3673         /* get endpoint address */
3674         ep_no = ep->edesc->bEndpointAddress;
3675
3676         DPRINTFN(5, "endpoint=0x%x\n", ep_no);
3677
3678         if (ep_no & UE_DIR_IN) {
3679                 reg = DOTG_DIEPCTL(ep_no & UE_ADDR);
3680                 temp = sc->sc_in_ctl[ep_no & UE_ADDR];
3681         } else {
3682                 reg = DOTG_DOEPCTL(ep_no & UE_ADDR);
3683                 temp = sc->sc_out_ctl[ep_no & UE_ADDR];
3684         }
3685
3686         /* disable and stall endpoint */
3687         DWC_OTG_WRITE_4(sc, reg, temp | DOEPCTL_EPDIS);
3688         DWC_OTG_WRITE_4(sc, reg, temp | DOEPCTL_STALL);
3689
3690         /* clear active OUT ep */
3691         if (!(ep_no & UE_DIR_IN)) {
3692
3693                 sc->sc_active_rx_ep &= ~(1U << (ep_no & UE_ADDR));
3694
3695                 if (sc->sc_last_rx_status != 0 &&
3696                     (ep_no & UE_ADDR) == GRXSTSRD_CHNUM_GET(
3697                     sc->sc_last_rx_status)) {
3698                         /* dump data */
3699                         dwc_otg_common_rx_ack(sc);
3700                         /* poll interrupt */
3701                         dwc_otg_interrupt_poll_locked(sc);
3702                         dwc_otg_interrupt_complete_locked(sc);
3703                 }
3704         }
3705         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3706 }
3707
3708 static void
3709 dwc_otg_clear_stall_sub_locked(struct dwc_otg_softc *sc, uint32_t mps,
3710     uint8_t ep_no, uint8_t ep_type, uint8_t ep_dir)
3711 {
3712         uint32_t reg;
3713         uint32_t temp;
3714
3715         if (ep_type == UE_CONTROL) {
3716                 /* clearing stall is not needed */
3717                 return;
3718         }
3719
3720         if (ep_dir) {
3721                 reg = DOTG_DIEPCTL(ep_no);
3722         } else {
3723                 reg = DOTG_DOEPCTL(ep_no);
3724                 sc->sc_active_rx_ep |= (1U << ep_no);
3725         }
3726
3727         /* round up and mask away the multiplier count */
3728         mps = (mps + 3) & 0x7FC;
3729
3730         if (ep_type == UE_BULK) {
3731                 temp = DIEPCTL_EPTYPE_SET(
3732                     DIEPCTL_EPTYPE_BULK) |
3733                     DIEPCTL_USBACTEP;
3734         } else if (ep_type == UE_INTERRUPT) {
3735                 temp = DIEPCTL_EPTYPE_SET(
3736                     DIEPCTL_EPTYPE_INTERRUPT) |
3737                     DIEPCTL_USBACTEP;
3738         } else {
3739                 temp = DIEPCTL_EPTYPE_SET(
3740                     DIEPCTL_EPTYPE_ISOC) |
3741                     DIEPCTL_USBACTEP;
3742         }
3743
3744         temp |= DIEPCTL_MPS_SET(mps);
3745         temp |= DIEPCTL_TXFNUM_SET(ep_no);
3746
3747         if (ep_dir)
3748                 sc->sc_in_ctl[ep_no] = temp;
3749         else
3750                 sc->sc_out_ctl[ep_no] = temp;
3751
3752         DWC_OTG_WRITE_4(sc, reg, temp | DOEPCTL_EPDIS);
3753         DWC_OTG_WRITE_4(sc, reg, temp | DOEPCTL_SETD0PID);
3754         DWC_OTG_WRITE_4(sc, reg, temp | DIEPCTL_SNAK);
3755
3756         /* we only reset the transmit FIFO */
3757         if (ep_dir) {
3758                 dwc_otg_tx_fifo_reset(sc,
3759                     GRSTCTL_TXFIFO(ep_no) |
3760                     GRSTCTL_TXFFLSH);
3761
3762                 DWC_OTG_WRITE_4(sc,
3763                     DOTG_DIEPTSIZ(ep_no), 0);
3764         }
3765
3766         /* poll interrupt */
3767         dwc_otg_interrupt_poll_locked(sc);
3768         dwc_otg_interrupt_complete_locked(sc);
3769 }
3770
3771 static void
3772 dwc_otg_clear_stall(struct usb_device *udev, struct usb_endpoint *ep)
3773 {
3774         struct dwc_otg_softc *sc;
3775         struct usb_endpoint_descriptor *ed;
3776
3777         DPRINTFN(5, "endpoint=%p\n", ep);
3778
3779         USB_BUS_LOCK_ASSERT(udev->bus, MA_OWNED);
3780
3781         /* check mode */
3782         if (udev->flags.usb_mode != USB_MODE_DEVICE) {
3783                 /* not supported */
3784                 return;
3785         }
3786         /* get softc */
3787         sc = DWC_OTG_BUS2SC(udev->bus);
3788
3789         USB_BUS_SPIN_LOCK(&sc->sc_bus);
3790
3791         /* get endpoint descriptor */
3792         ed = ep->edesc;
3793
3794         /* reset endpoint */
3795         dwc_otg_clear_stall_sub_locked(sc,
3796             UGETW(ed->wMaxPacketSize),
3797             (ed->bEndpointAddress & UE_ADDR),
3798             (ed->bmAttributes & UE_XFERTYPE),
3799             (ed->bEndpointAddress & (UE_DIR_IN | UE_DIR_OUT)));
3800
3801         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3802 }
3803
3804 static void
3805 dwc_otg_device_state_change(struct usb_device *udev)
3806 {
3807         struct dwc_otg_softc *sc;
3808         uint8_t x;
3809
3810         /* check mode */
3811         if (udev->flags.usb_mode != USB_MODE_DEVICE) {
3812                 /* not supported */
3813                 return;
3814         }
3815
3816         /* get softc */
3817         sc = DWC_OTG_BUS2SC(udev->bus);
3818
3819         /* deactivate all other endpoint but the control endpoint */
3820         if (udev->state == USB_STATE_CONFIGURED ||
3821             udev->state == USB_STATE_ADDRESSED) {
3822
3823                 USB_BUS_LOCK(&sc->sc_bus);
3824
3825                 for (x = 1; x != sc->sc_dev_ep_max; x++) {
3826
3827                         if (x < sc->sc_dev_in_ep_max) {
3828                                 DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(x),
3829                                     DIEPCTL_EPDIS);
3830                                 DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(x), 0);
3831                         }
3832
3833                         DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(x),
3834                             DOEPCTL_EPDIS);
3835                         DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(x), 0);
3836                 }
3837                 USB_BUS_UNLOCK(&sc->sc_bus);
3838         }
3839 }
3840
3841 int
3842 dwc_otg_init(struct dwc_otg_softc *sc)
3843 {
3844         uint32_t temp;
3845
3846         DPRINTF("start\n");
3847
3848         /* set up the bus structure */
3849         sc->sc_bus.usbrev = USB_REV_2_0;
3850         sc->sc_bus.methods = &dwc_otg_bus_methods;
3851
3852         usb_callout_init_mtx(&sc->sc_timer,
3853             &sc->sc_bus.bus_mtx, 0);
3854
3855         USB_BUS_LOCK(&sc->sc_bus);
3856
3857         /* turn on clocks */
3858         dwc_otg_clocks_on(sc);
3859
3860         temp = DWC_OTG_READ_4(sc, DOTG_GSNPSID);
3861         DPRINTF("Version = 0x%08x\n", temp);
3862
3863         /* disconnect */
3864         DWC_OTG_WRITE_4(sc, DOTG_DCTL,
3865             DCTL_SFTDISCON);
3866
3867         /* wait for host to detect disconnect */
3868         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 32);
3869
3870         DWC_OTG_WRITE_4(sc, DOTG_GRSTCTL,
3871             GRSTCTL_CSFTRST);
3872
3873         /* wait a little bit for block to reset */
3874         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 128);
3875
3876         switch (sc->sc_mode) {
3877         case DWC_MODE_DEVICE:
3878                 temp = GUSBCFG_FORCEDEVMODE;
3879                 break;
3880         case DWC_MODE_HOST:
3881                 temp = GUSBCFG_FORCEHOSTMODE;
3882                 break;
3883         default:
3884                 temp = 0;
3885                 break;
3886         }
3887
3888         if (sc->sc_phy_type == 0)
3889                 sc->sc_phy_type = dwc_otg_phy_type + 1;
3890         if (sc->sc_phy_bits == 0)
3891                 sc->sc_phy_bits = 16;
3892
3893         /* select HSIC, ULPI, UTMI+ or internal PHY mode */
3894         switch (sc->sc_phy_type) {
3895         case DWC_OTG_PHY_HSIC:
3896                 DWC_OTG_WRITE_4(sc, DOTG_GUSBCFG,
3897                     GUSBCFG_PHYIF |
3898                     GUSBCFG_TRD_TIM_SET(5) | temp);
3899                 DWC_OTG_WRITE_4(sc, DOTG_GOTGCTL,
3900                     0x000000EC);
3901
3902                 temp = DWC_OTG_READ_4(sc, DOTG_GLPMCFG);
3903                 DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3904                     temp & ~GLPMCFG_HSIC_CONN);
3905                 DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3906                     temp | GLPMCFG_HSIC_CONN);
3907                 break;
3908         case DWC_OTG_PHY_ULPI:
3909                 DWC_OTG_WRITE_4(sc, DOTG_GUSBCFG,
3910                     GUSBCFG_ULPI_UTMI_SEL |
3911                     GUSBCFG_TRD_TIM_SET(5) | temp);
3912                 DWC_OTG_WRITE_4(sc, DOTG_GOTGCTL, 0);
3913
3914                 temp = DWC_OTG_READ_4(sc, DOTG_GLPMCFG);
3915                 DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3916                     temp & ~GLPMCFG_HSIC_CONN);
3917                 break;
3918         case DWC_OTG_PHY_UTMI:
3919                 DWC_OTG_WRITE_4(sc, DOTG_GUSBCFG,
3920                     (sc->sc_phy_bits == 16 ? GUSBCFG_PHYIF : 0) |
3921                     GUSBCFG_TRD_TIM_SET(5) | temp);
3922                 DWC_OTG_WRITE_4(sc, DOTG_GOTGCTL, 0);
3923
3924                 temp = DWC_OTG_READ_4(sc, DOTG_GLPMCFG);
3925                 DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3926                     temp & ~GLPMCFG_HSIC_CONN);
3927                 break;
3928         case DWC_OTG_PHY_INTERNAL:
3929                 DWC_OTG_WRITE_4(sc, DOTG_GUSBCFG,
3930                     GUSBCFG_PHYSEL |
3931                     GUSBCFG_TRD_TIM_SET(5) | temp);
3932                 DWC_OTG_WRITE_4(sc, DOTG_GOTGCTL, 0);
3933
3934                 temp = DWC_OTG_READ_4(sc, DOTG_GLPMCFG);
3935                 DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3936                     temp & ~GLPMCFG_HSIC_CONN);
3937
3938                 temp = DWC_OTG_READ_4(sc, DOTG_GGPIO);
3939                 temp &= ~(DOTG_GGPIO_NOVBUSSENS | DOTG_GGPIO_I2CPADEN);
3940                 temp |= (DOTG_GGPIO_VBUSASEN | DOTG_GGPIO_VBUSBSEN |
3941                     DOTG_GGPIO_PWRDWN);
3942                 DWC_OTG_WRITE_4(sc, DOTG_GGPIO, temp);
3943                 break;
3944         default:
3945                 break;
3946         }
3947
3948         /* clear global nak */
3949         DWC_OTG_WRITE_4(sc, DOTG_DCTL,
3950             DCTL_CGOUTNAK |
3951             DCTL_CGNPINNAK);
3952
3953         /* disable USB port */
3954         DWC_OTG_WRITE_4(sc, DOTG_PCGCCTL, 0xFFFFFFFF);
3955
3956         /* wait 10ms */
3957         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 100);
3958
3959         /* enable USB port */
3960         DWC_OTG_WRITE_4(sc, DOTG_PCGCCTL, 0);
3961
3962         /* wait 10ms */
3963         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 100);
3964
3965         temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG3);
3966
3967         sc->sc_fifo_size = 4 * GHWCFG3_DFIFODEPTH_GET(temp);
3968
3969         temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG2);
3970
3971         sc->sc_dev_ep_max = GHWCFG2_NUMDEVEPS_GET(temp);
3972
3973         if (sc->sc_dev_ep_max > DWC_OTG_MAX_ENDPOINTS)
3974                 sc->sc_dev_ep_max = DWC_OTG_MAX_ENDPOINTS;
3975
3976         sc->sc_host_ch_max = GHWCFG2_NUMHSTCHNL_GET(temp);
3977
3978         if (sc->sc_host_ch_max > DWC_OTG_MAX_CHANNELS)
3979                 sc->sc_host_ch_max = DWC_OTG_MAX_CHANNELS;
3980
3981         temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG4);
3982
3983         sc->sc_dev_in_ep_max = GHWCFG4_NUM_IN_EP_GET(temp);
3984
3985         DPRINTF("Total FIFO size = %d bytes, Device EPs = %d/%d Host CHs = %d\n",
3986             sc->sc_fifo_size, sc->sc_dev_ep_max, sc->sc_dev_in_ep_max,
3987             sc->sc_host_ch_max);
3988
3989         /* setup FIFO */
3990         if (dwc_otg_init_fifo(sc, sc->sc_mode)) {
3991                 USB_BUS_UNLOCK(&sc->sc_bus);
3992                 return (EINVAL);
3993         }
3994
3995         /* enable interrupts */
3996         sc->sc_irq_mask |= DWC_OTG_MSK_GINT_THREAD_IRQ;
3997         DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
3998
3999         if (sc->sc_mode == DWC_MODE_OTG || sc->sc_mode == DWC_MODE_DEVICE) {
4000
4001                 /* enable all endpoint interrupts */
4002                 temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG2);
4003                 if (temp & GHWCFG2_MPI) {
4004                         uint8_t x;
4005
4006                         DPRINTF("Disable Multi Process Interrupts\n");
4007
4008                         for (x = 0; x != sc->sc_dev_in_ep_max; x++) {
4009                                 DWC_OTG_WRITE_4(sc, DOTG_DIEPEACHINTMSK(x), 0);
4010                                 DWC_OTG_WRITE_4(sc, DOTG_DOEPEACHINTMSK(x), 0);
4011                         }
4012                         DWC_OTG_WRITE_4(sc, DOTG_DEACHINTMSK, 0);
4013                 }
4014                 DWC_OTG_WRITE_4(sc, DOTG_DIEPMSK,
4015                     DIEPMSK_XFERCOMPLMSK);
4016                 DWC_OTG_WRITE_4(sc, DOTG_DOEPMSK, 0);
4017                 DWC_OTG_WRITE_4(sc, DOTG_DAINTMSK, 0xFFFF);
4018         }
4019
4020         if (sc->sc_mode == DWC_MODE_OTG || sc->sc_mode == DWC_MODE_HOST) {
4021                 /* setup clocks */
4022                 temp = DWC_OTG_READ_4(sc, DOTG_HCFG);
4023                 temp &= ~(HCFG_FSLSSUPP | HCFG_FSLSPCLKSEL_MASK);
4024                 temp |= (1 << HCFG_FSLSPCLKSEL_SHIFT);
4025                 DWC_OTG_WRITE_4(sc, DOTG_HCFG, temp);
4026         }
4027
4028         /* only enable global IRQ */
4029         DWC_OTG_WRITE_4(sc, DOTG_GAHBCFG,
4030             GAHBCFG_GLBLINTRMSK);
4031
4032         /* turn off clocks */
4033         dwc_otg_clocks_off(sc);
4034
4035         /* read initial VBUS state */
4036
4037         temp = DWC_OTG_READ_4(sc, DOTG_GOTGCTL);
4038
4039         DPRINTFN(5, "GOTCTL=0x%08x\n", temp);
4040
4041         dwc_otg_vbus_interrupt(sc,
4042             (temp & (GOTGCTL_ASESVLD | GOTGCTL_BSESVLD)) ? 1 : 0);
4043
4044         USB_BUS_UNLOCK(&sc->sc_bus);
4045
4046         /* catch any lost interrupts */
4047
4048         dwc_otg_do_poll(&sc->sc_bus);
4049
4050         return (0);                     /* success */
4051 }
4052
4053 void
4054 dwc_otg_uninit(struct dwc_otg_softc *sc)
4055 {
4056         USB_BUS_LOCK(&sc->sc_bus);
4057
4058         /* stop host timer */
4059         dwc_otg_timer_stop(sc);
4060
4061         /* set disconnect */
4062         DWC_OTG_WRITE_4(sc, DOTG_DCTL,
4063             DCTL_SFTDISCON);
4064
4065         /* turn off global IRQ */
4066         DWC_OTG_WRITE_4(sc, DOTG_GAHBCFG, 0);
4067
4068         sc->sc_flags.port_enabled = 0;
4069         sc->sc_flags.port_powered = 0;
4070         sc->sc_flags.status_vbus = 0;
4071         sc->sc_flags.status_bus_reset = 0;
4072         sc->sc_flags.status_suspend = 0;
4073         sc->sc_flags.change_suspend = 0;
4074         sc->sc_flags.change_connect = 1;
4075
4076         dwc_otg_pull_down(sc);
4077         dwc_otg_clocks_off(sc);
4078
4079         USB_BUS_UNLOCK(&sc->sc_bus);
4080
4081         usb_callout_drain(&sc->sc_timer);
4082 }
4083
4084 static void
4085 dwc_otg_suspend(struct dwc_otg_softc *sc)
4086 {
4087         return;
4088 }
4089
4090 static void
4091 dwc_otg_resume(struct dwc_otg_softc *sc)
4092 {
4093         return;
4094 }
4095
4096 static void
4097 dwc_otg_do_poll(struct usb_bus *bus)
4098 {
4099         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(bus);
4100
4101         USB_BUS_LOCK(&sc->sc_bus);
4102         USB_BUS_SPIN_LOCK(&sc->sc_bus);
4103         dwc_otg_interrupt_poll_locked(sc);
4104         dwc_otg_interrupt_complete_locked(sc);
4105         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
4106         USB_BUS_UNLOCK(&sc->sc_bus);
4107 }
4108
4109 /*------------------------------------------------------------------------*
4110  * DWC OTG bulk support
4111  * DWC OTG control support
4112  * DWC OTG interrupt support
4113  *------------------------------------------------------------------------*/
4114 static void
4115 dwc_otg_device_non_isoc_open(struct usb_xfer *xfer)
4116 {
4117 }
4118
4119 static void
4120 dwc_otg_device_non_isoc_close(struct usb_xfer *xfer)
4121 {
4122         dwc_otg_device_done(xfer, USB_ERR_CANCELLED);
4123 }
4124
4125 static void
4126 dwc_otg_device_non_isoc_enter(struct usb_xfer *xfer)
4127 {
4128 }
4129
4130 static void
4131 dwc_otg_device_non_isoc_start(struct usb_xfer *xfer)
4132 {
4133         /* setup TDs */
4134         dwc_otg_setup_standard_chain(xfer);
4135         dwc_otg_start_standard_chain(xfer);
4136 }
4137
4138 static const struct usb_pipe_methods dwc_otg_device_non_isoc_methods =
4139 {
4140         .open = dwc_otg_device_non_isoc_open,
4141         .close = dwc_otg_device_non_isoc_close,
4142         .enter = dwc_otg_device_non_isoc_enter,
4143         .start = dwc_otg_device_non_isoc_start,
4144 };
4145
4146 /*------------------------------------------------------------------------*
4147  * DWC OTG full speed isochronous support
4148  *------------------------------------------------------------------------*/
4149 static void
4150 dwc_otg_device_isoc_open(struct usb_xfer *xfer)
4151 {
4152 }
4153
4154 static void
4155 dwc_otg_device_isoc_close(struct usb_xfer *xfer)
4156 {
4157         dwc_otg_device_done(xfer, USB_ERR_CANCELLED);
4158 }
4159
4160 static void
4161 dwc_otg_device_isoc_enter(struct usb_xfer *xfer)
4162 {
4163 }
4164
4165 static void
4166 dwc_otg_device_isoc_start(struct usb_xfer *xfer)
4167 {
4168         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
4169         uint32_t temp;
4170         uint32_t msframes;
4171         uint32_t framenum;
4172         uint8_t shift = usbd_xfer_get_fps_shift(xfer);
4173
4174         DPRINTFN(6, "xfer=%p next=%d nframes=%d\n",
4175             xfer, xfer->endpoint->isoc_next, xfer->nframes);
4176
4177         if (xfer->xroot->udev->flags.usb_mode == USB_MODE_HOST) {
4178                 temp = DWC_OTG_READ_4(sc, DOTG_HFNUM);
4179
4180                 /* get the current frame index */
4181                 framenum = (temp & HFNUM_FRNUM_MASK);
4182         } else {
4183                 temp = DWC_OTG_READ_4(sc, DOTG_DSTS);
4184
4185                 /* get the current frame index */
4186                 framenum = DSTS_SOFFN_GET(temp);
4187         }
4188
4189         /*
4190          * Check if port is doing 8000 or 1000 frames per second:
4191          */
4192         if (sc->sc_flags.status_high_speed)
4193                 framenum /= 8;
4194
4195         framenum &= DWC_OTG_FRAME_MASK;
4196
4197         /*
4198          * Compute number of milliseconds worth of data traffic for
4199          * this USB transfer:
4200          */
4201         if (xfer->xroot->udev->speed == USB_SPEED_HIGH)
4202                 msframes = ((xfer->nframes << shift) + 7) / 8;
4203         else
4204                 msframes = xfer->nframes;
4205
4206         /*
4207          * check if the frame index is within the window where the frames
4208          * will be inserted
4209          */
4210         temp = (framenum - xfer->endpoint->isoc_next) & DWC_OTG_FRAME_MASK;
4211
4212         if ((xfer->endpoint->is_synced == 0) || (temp < msframes)) {
4213                 /*
4214                  * If there is data underflow or the pipe queue is
4215                  * empty we schedule the transfer a few frames ahead
4216                  * of the current frame position. Else two isochronous
4217                  * transfers might overlap.
4218                  */
4219                 xfer->endpoint->isoc_next = (framenum + 3) & DWC_OTG_FRAME_MASK;
4220                 xfer->endpoint->is_synced = 1;
4221                 DPRINTFN(3, "start next=%d\n", xfer->endpoint->isoc_next);
4222         }
4223         /*
4224          * compute how many milliseconds the insertion is ahead of the
4225          * current frame position:
4226          */
4227         temp = (xfer->endpoint->isoc_next - framenum) & DWC_OTG_FRAME_MASK;
4228
4229         /*
4230          * pre-compute when the isochronous transfer will be finished:
4231          */
4232         xfer->isoc_time_complete =
4233                 usb_isoc_time_expand(&sc->sc_bus, framenum) + temp + msframes;
4234
4235         /* setup TDs */
4236         dwc_otg_setup_standard_chain(xfer);
4237
4238         /* compute frame number for next insertion */
4239         xfer->endpoint->isoc_next += msframes;
4240
4241         /* start TD chain */
4242         dwc_otg_start_standard_chain(xfer);
4243 }
4244
4245 static const struct usb_pipe_methods dwc_otg_device_isoc_methods =
4246 {
4247         .open = dwc_otg_device_isoc_open,
4248         .close = dwc_otg_device_isoc_close,
4249         .enter = dwc_otg_device_isoc_enter,
4250         .start = dwc_otg_device_isoc_start,
4251 };
4252
4253 /*------------------------------------------------------------------------*
4254  * DWC OTG root control support
4255  *------------------------------------------------------------------------*
4256  * Simulate a hardware HUB by handling all the necessary requests.
4257  *------------------------------------------------------------------------*/
4258
4259 static const struct usb_device_descriptor dwc_otg_devd = {
4260         .bLength = sizeof(struct usb_device_descriptor),
4261         .bDescriptorType = UDESC_DEVICE,
4262         .bcdUSB = {0x00, 0x02},
4263         .bDeviceClass = UDCLASS_HUB,
4264         .bDeviceSubClass = UDSUBCLASS_HUB,
4265         .bDeviceProtocol = UDPROTO_HSHUBSTT,
4266         .bMaxPacketSize = 64,
4267         .bcdDevice = {0x00, 0x01},
4268         .iManufacturer = 1,
4269         .iProduct = 2,
4270         .bNumConfigurations = 1,
4271 };
4272
4273 static const struct dwc_otg_config_desc dwc_otg_confd = {
4274         .confd = {
4275                 .bLength = sizeof(struct usb_config_descriptor),
4276                 .bDescriptorType = UDESC_CONFIG,
4277                 .wTotalLength[0] = sizeof(dwc_otg_confd),
4278                 .bNumInterface = 1,
4279                 .bConfigurationValue = 1,
4280                 .iConfiguration = 0,
4281                 .bmAttributes = UC_SELF_POWERED,
4282                 .bMaxPower = 0,
4283         },
4284         .ifcd = {
4285                 .bLength = sizeof(struct usb_interface_descriptor),
4286                 .bDescriptorType = UDESC_INTERFACE,
4287                 .bNumEndpoints = 1,
4288                 .bInterfaceClass = UICLASS_HUB,
4289                 .bInterfaceSubClass = UISUBCLASS_HUB,
4290                 .bInterfaceProtocol = 0,
4291         },
4292         .endpd = {
4293                 .bLength = sizeof(struct usb_endpoint_descriptor),
4294                 .bDescriptorType = UDESC_ENDPOINT,
4295                 .bEndpointAddress = (UE_DIR_IN | DWC_OTG_INTR_ENDPT),
4296                 .bmAttributes = UE_INTERRUPT,
4297                 .wMaxPacketSize[0] = 8,
4298                 .bInterval = 255,
4299         },
4300 };
4301
4302 #define HSETW(ptr, val) ptr = { (uint8_t)(val), (uint8_t)((val) >> 8) }
4303
4304 static const struct usb_hub_descriptor_min dwc_otg_hubd = {
4305         .bDescLength = sizeof(dwc_otg_hubd),
4306         .bDescriptorType = UDESC_HUB,
4307         .bNbrPorts = 1,
4308         HSETW(.wHubCharacteristics, (UHD_PWR_NO_SWITCH | UHD_OC_INDIVIDUAL)),
4309         .bPwrOn2PwrGood = 50,
4310         .bHubContrCurrent = 0,
4311         .DeviceRemovable = {0},         /* port is removable */
4312 };
4313
4314 #define STRING_VENDOR \
4315   "D\0W\0C\0O\0T\0G"
4316
4317 #define STRING_PRODUCT \
4318   "O\0T\0G\0 \0R\0o\0o\0t\0 \0H\0U\0B"
4319
4320 USB_MAKE_STRING_DESC(STRING_VENDOR, dwc_otg_vendor);
4321 USB_MAKE_STRING_DESC(STRING_PRODUCT, dwc_otg_product);
4322
4323 static usb_error_t
4324 dwc_otg_roothub_exec(struct usb_device *udev,
4325     struct usb_device_request *req, const void **pptr, uint16_t *plength)
4326 {
4327         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(udev->bus);
4328         const void *ptr;
4329         uint16_t len;
4330         uint16_t value;
4331         uint16_t index;
4332         usb_error_t err;
4333
4334         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
4335
4336         /* buffer reset */
4337         ptr = (const void *)&sc->sc_hub_temp;
4338         len = 0;
4339         err = 0;
4340
4341         value = UGETW(req->wValue);
4342         index = UGETW(req->wIndex);
4343
4344         /* demultiplex the control request */
4345
4346         switch (req->bmRequestType) {
4347         case UT_READ_DEVICE:
4348                 switch (req->bRequest) {
4349                 case UR_GET_DESCRIPTOR:
4350                         goto tr_handle_get_descriptor;
4351                 case UR_GET_CONFIG:
4352                         goto tr_handle_get_config;
4353                 case UR_GET_STATUS:
4354                         goto tr_handle_get_status;
4355                 default:
4356                         goto tr_stalled;
4357                 }
4358                 break;
4359
4360         case UT_WRITE_DEVICE:
4361                 switch (req->bRequest) {
4362                 case UR_SET_ADDRESS:
4363                         goto tr_handle_set_address;
4364                 case UR_SET_CONFIG:
4365                         goto tr_handle_set_config;
4366                 case UR_CLEAR_FEATURE:
4367                         goto tr_valid;  /* nop */
4368                 case UR_SET_DESCRIPTOR:
4369                         goto tr_valid;  /* nop */
4370                 case UR_SET_FEATURE:
4371                 default:
4372                         goto tr_stalled;
4373                 }
4374                 break;
4375
4376         case UT_WRITE_ENDPOINT:
4377                 switch (req->bRequest) {
4378                 case UR_CLEAR_FEATURE:
4379                         switch (UGETW(req->wValue)) {
4380                         case UF_ENDPOINT_HALT:
4381                                 goto tr_handle_clear_halt;
4382                         case UF_DEVICE_REMOTE_WAKEUP:
4383                                 goto tr_handle_clear_wakeup;
4384                         default:
4385                                 goto tr_stalled;
4386                         }
4387                         break;
4388                 case UR_SET_FEATURE:
4389                         switch (UGETW(req->wValue)) {
4390                         case UF_ENDPOINT_HALT:
4391                                 goto tr_handle_set_halt;
4392                         case UF_DEVICE_REMOTE_WAKEUP:
4393                                 goto tr_handle_set_wakeup;
4394                         default:
4395                                 goto tr_stalled;
4396                         }
4397                         break;
4398                 case UR_SYNCH_FRAME:
4399                         goto tr_valid;  /* nop */
4400                 default:
4401                         goto tr_stalled;
4402                 }
4403                 break;
4404
4405         case UT_READ_ENDPOINT:
4406                 switch (req->bRequest) {
4407                 case UR_GET_STATUS:
4408                         goto tr_handle_get_ep_status;
4409                 default:
4410                         goto tr_stalled;
4411                 }
4412                 break;
4413
4414         case UT_WRITE_INTERFACE:
4415                 switch (req->bRequest) {
4416                 case UR_SET_INTERFACE:
4417                         goto tr_handle_set_interface;
4418                 case UR_CLEAR_FEATURE:
4419                         goto tr_valid;  /* nop */
4420                 case UR_SET_FEATURE:
4421                 default:
4422                         goto tr_stalled;
4423                 }
4424                 break;
4425
4426         case UT_READ_INTERFACE:
4427                 switch (req->bRequest) {
4428                 case UR_GET_INTERFACE:
4429                         goto tr_handle_get_interface;
4430                 case UR_GET_STATUS:
4431                         goto tr_handle_get_iface_status;
4432                 default:
4433                         goto tr_stalled;
4434                 }
4435                 break;
4436
4437         case UT_WRITE_CLASS_INTERFACE:
4438         case UT_WRITE_VENDOR_INTERFACE:
4439                 /* XXX forward */
4440                 break;
4441
4442         case UT_READ_CLASS_INTERFACE:
4443         case UT_READ_VENDOR_INTERFACE:
4444                 /* XXX forward */
4445                 break;
4446
4447         case UT_WRITE_CLASS_DEVICE:
4448                 switch (req->bRequest) {
4449                 case UR_CLEAR_FEATURE:
4450                         goto tr_valid;
4451                 case UR_SET_DESCRIPTOR:
4452                 case UR_SET_FEATURE:
4453                         break;
4454                 default:
4455                         goto tr_stalled;
4456                 }
4457                 break;
4458
4459         case UT_WRITE_CLASS_OTHER:
4460                 switch (req->bRequest) {
4461                 case UR_CLEAR_FEATURE:
4462                         goto tr_handle_clear_port_feature;
4463                 case UR_SET_FEATURE:
4464                         goto tr_handle_set_port_feature;
4465                 case UR_CLEAR_TT_BUFFER:
4466                 case UR_RESET_TT:
4467                 case UR_STOP_TT:
4468                         goto tr_valid;
4469
4470                 default:
4471                         goto tr_stalled;
4472                 }
4473                 break;
4474
4475         case UT_READ_CLASS_OTHER:
4476                 switch (req->bRequest) {
4477                 case UR_GET_TT_STATE:
4478                         goto tr_handle_get_tt_state;
4479                 case UR_GET_STATUS:
4480                         goto tr_handle_get_port_status;
4481                 default:
4482                         goto tr_stalled;
4483                 }
4484                 break;
4485
4486         case UT_READ_CLASS_DEVICE:
4487                 switch (req->bRequest) {
4488                 case UR_GET_DESCRIPTOR:
4489                         goto tr_handle_get_class_descriptor;
4490                 case UR_GET_STATUS:
4491                         goto tr_handle_get_class_status;
4492
4493                 default:
4494                         goto tr_stalled;
4495                 }
4496                 break;
4497         default:
4498                 goto tr_stalled;
4499         }
4500         goto tr_valid;
4501
4502 tr_handle_get_descriptor:
4503         switch (value >> 8) {
4504         case UDESC_DEVICE:
4505                 if (value & 0xff) {
4506                         goto tr_stalled;
4507                 }
4508                 len = sizeof(dwc_otg_devd);
4509                 ptr = (const void *)&dwc_otg_devd;
4510                 goto tr_valid;
4511         case UDESC_CONFIG:
4512                 if (value & 0xff) {
4513                         goto tr_stalled;
4514                 }
4515                 len = sizeof(dwc_otg_confd);
4516                 ptr = (const void *)&dwc_otg_confd;
4517                 goto tr_valid;
4518         case UDESC_STRING:
4519                 switch (value & 0xff) {
4520                 case 0:         /* Language table */
4521                         len = sizeof(usb_string_lang_en);
4522                         ptr = (const void *)&usb_string_lang_en;
4523                         goto tr_valid;
4524
4525                 case 1:         /* Vendor */
4526                         len = sizeof(dwc_otg_vendor);
4527                         ptr = (const void *)&dwc_otg_vendor;
4528                         goto tr_valid;
4529
4530                 case 2:         /* Product */
4531                         len = sizeof(dwc_otg_product);
4532                         ptr = (const void *)&dwc_otg_product;
4533                         goto tr_valid;
4534                 default:
4535                         break;
4536                 }
4537                 break;
4538         default:
4539                 goto tr_stalled;
4540         }
4541         goto tr_stalled;
4542
4543 tr_handle_get_config:
4544         len = 1;
4545         sc->sc_hub_temp.wValue[0] = sc->sc_conf;
4546         goto tr_valid;
4547
4548 tr_handle_get_status:
4549         len = 2;
4550         USETW(sc->sc_hub_temp.wValue, UDS_SELF_POWERED);
4551         goto tr_valid;
4552
4553 tr_handle_set_address:
4554         if (value & 0xFF00) {
4555                 goto tr_stalled;
4556         }
4557         sc->sc_rt_addr = value;
4558         goto tr_valid;
4559
4560 tr_handle_set_config:
4561         if (value >= 2) {
4562                 goto tr_stalled;
4563         }
4564         sc->sc_conf = value;
4565         goto tr_valid;
4566
4567 tr_handle_get_interface:
4568         len = 1;
4569         sc->sc_hub_temp.wValue[0] = 0;
4570         goto tr_valid;
4571
4572 tr_handle_get_tt_state:
4573 tr_handle_get_class_status:
4574 tr_handle_get_iface_status:
4575 tr_handle_get_ep_status:
4576         len = 2;
4577         USETW(sc->sc_hub_temp.wValue, 0);
4578         goto tr_valid;
4579
4580 tr_handle_set_halt:
4581 tr_handle_set_interface:
4582 tr_handle_set_wakeup:
4583 tr_handle_clear_wakeup:
4584 tr_handle_clear_halt:
4585         goto tr_valid;
4586
4587 tr_handle_clear_port_feature:
4588         if (index != 1)
4589                 goto tr_stalled;
4590
4591         DPRINTFN(9, "UR_CLEAR_PORT_FEATURE on port %d\n", index);
4592
4593         switch (value) {
4594         case UHF_PORT_SUSPEND:
4595                 dwc_otg_wakeup_peer(sc);
4596                 break;
4597
4598         case UHF_PORT_ENABLE:
4599                 if (sc->sc_flags.status_device_mode == 0) {
4600                         DWC_OTG_WRITE_4(sc, DOTG_HPRT,
4601                             sc->sc_hprt_val | HPRT_PRTENA);
4602                 }
4603                 sc->sc_flags.port_enabled = 0;
4604                 break;
4605
4606         case UHF_C_PORT_RESET:
4607                 sc->sc_flags.change_reset = 0;
4608                 break;
4609
4610         case UHF_C_PORT_ENABLE:
4611                 sc->sc_flags.change_enabled = 0;
4612                 break;
4613
4614         case UHF_C_PORT_OVER_CURRENT:
4615                 sc->sc_flags.change_over_current = 0;
4616                 break;
4617
4618         case UHF_PORT_TEST:
4619         case UHF_PORT_INDICATOR:
4620                 /* nops */
4621                 break;
4622
4623         case UHF_PORT_POWER:
4624                 sc->sc_flags.port_powered = 0;
4625                 if (sc->sc_mode == DWC_MODE_HOST || sc->sc_mode == DWC_MODE_OTG) {
4626                         sc->sc_hprt_val = 0;
4627                         DWC_OTG_WRITE_4(sc, DOTG_HPRT, HPRT_PRTENA);
4628                 }
4629                 dwc_otg_pull_down(sc);
4630                 dwc_otg_clocks_off(sc);
4631                 break;
4632
4633         case UHF_C_PORT_CONNECTION:
4634                 /* clear connect change flag */
4635                 sc->sc_flags.change_connect = 0;
4636                 break;
4637
4638         case UHF_C_PORT_SUSPEND:
4639                 sc->sc_flags.change_suspend = 0;
4640                 break;
4641
4642         default:
4643                 err = USB_ERR_IOERROR;
4644                 goto done;
4645         }
4646         goto tr_valid;
4647
4648 tr_handle_set_port_feature:
4649         if (index != 1) {
4650                 goto tr_stalled;
4651         }
4652         DPRINTFN(9, "UR_SET_PORT_FEATURE\n");
4653
4654         switch (value) {
4655         case UHF_PORT_ENABLE:
4656                 break;
4657
4658         case UHF_PORT_SUSPEND:
4659                 if (sc->sc_flags.status_device_mode == 0) {
4660                         /* set suspend BIT */
4661                         sc->sc_hprt_val |= HPRT_PRTSUSP;
4662                         DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
4663
4664                         /* generate HUB suspend event */
4665                         dwc_otg_suspend_irq(sc);
4666                 }
4667                 break;
4668
4669         case UHF_PORT_RESET:
4670                 if (sc->sc_flags.status_device_mode == 0) {
4671
4672                         DPRINTF("PORT RESET\n");
4673
4674                         /* enable PORT reset */
4675                         DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val | HPRT_PRTRST);
4676
4677                         /* Wait 62.5ms for reset to complete */
4678                         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 16);
4679
4680                         DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
4681
4682                         /* Wait 62.5ms for reset to complete */
4683                         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 16);
4684
4685                         /* reset FIFOs */
4686                         (void) dwc_otg_init_fifo(sc, DWC_MODE_HOST);
4687
4688                         sc->sc_flags.change_reset = 1;
4689                 } else {
4690                         err = USB_ERR_IOERROR;
4691                 }
4692                 break;
4693
4694         case UHF_PORT_TEST:
4695         case UHF_PORT_INDICATOR:
4696                 /* nops */
4697                 break;
4698         case UHF_PORT_POWER:
4699                 sc->sc_flags.port_powered = 1;
4700                 if (sc->sc_mode == DWC_MODE_HOST || sc->sc_mode == DWC_MODE_OTG) {
4701                         sc->sc_hprt_val |= HPRT_PRTPWR;
4702                         DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
4703                 }
4704                 if (sc->sc_mode == DWC_MODE_DEVICE || sc->sc_mode == DWC_MODE_OTG) {
4705                         /* pull up D+, if any */
4706                         dwc_otg_pull_up(sc);
4707                 }
4708                 break;
4709         default:
4710                 err = USB_ERR_IOERROR;
4711                 goto done;
4712         }
4713         goto tr_valid;
4714
4715 tr_handle_get_port_status:
4716
4717         DPRINTFN(9, "UR_GET_PORT_STATUS\n");
4718
4719         if (index != 1)
4720                 goto tr_stalled;
4721
4722         if (sc->sc_flags.status_vbus)
4723                 dwc_otg_clocks_on(sc);
4724         else
4725                 dwc_otg_clocks_off(sc);
4726
4727         /* Select Device Side Mode */
4728
4729         if (sc->sc_flags.status_device_mode) {
4730                 value = UPS_PORT_MODE_DEVICE;
4731                 dwc_otg_timer_stop(sc);
4732         } else {
4733                 value = 0;
4734                 dwc_otg_timer_start(sc);
4735         }
4736
4737         if (sc->sc_flags.status_high_speed)
4738                 value |= UPS_HIGH_SPEED;
4739         else if (sc->sc_flags.status_low_speed)
4740                 value |= UPS_LOW_SPEED;
4741
4742         if (sc->sc_flags.port_powered)
4743                 value |= UPS_PORT_POWER;
4744
4745         if (sc->sc_flags.port_enabled)
4746                 value |= UPS_PORT_ENABLED;
4747
4748         if (sc->sc_flags.port_over_current)
4749                 value |= UPS_OVERCURRENT_INDICATOR;
4750
4751         if (sc->sc_flags.status_vbus &&
4752             sc->sc_flags.status_bus_reset)
4753                 value |= UPS_CURRENT_CONNECT_STATUS;
4754
4755         if (sc->sc_flags.status_suspend)
4756                 value |= UPS_SUSPEND;
4757
4758         USETW(sc->sc_hub_temp.ps.wPortStatus, value);
4759
4760         value = 0;
4761
4762         if (sc->sc_flags.change_enabled)
4763                 value |= UPS_C_PORT_ENABLED;
4764         if (sc->sc_flags.change_connect)
4765                 value |= UPS_C_CONNECT_STATUS;
4766         if (sc->sc_flags.change_suspend)
4767                 value |= UPS_C_SUSPEND;
4768         if (sc->sc_flags.change_reset)
4769                 value |= UPS_C_PORT_RESET;
4770         if (sc->sc_flags.change_over_current)
4771                 value |= UPS_C_OVERCURRENT_INDICATOR;
4772
4773         USETW(sc->sc_hub_temp.ps.wPortChange, value);
4774         len = sizeof(sc->sc_hub_temp.ps);
4775         goto tr_valid;
4776
4777 tr_handle_get_class_descriptor:
4778         if (value & 0xFF) {
4779                 goto tr_stalled;
4780         }
4781         ptr = (const void *)&dwc_otg_hubd;
4782         len = sizeof(dwc_otg_hubd);
4783         goto tr_valid;
4784
4785 tr_stalled:
4786         err = USB_ERR_STALLED;
4787 tr_valid:
4788 done:
4789         *plength = len;
4790         *pptr = ptr;
4791         return (err);
4792 }
4793
4794 static void
4795 dwc_otg_xfer_setup(struct usb_setup_params *parm)
4796 {
4797         struct usb_xfer *xfer;
4798         void *last_obj;
4799         uint32_t ntd;
4800         uint32_t n;
4801         uint8_t ep_no;
4802         uint8_t ep_type;
4803
4804         xfer = parm->curr_xfer;
4805
4806         /*
4807          * NOTE: This driver does not use any of the parameters that
4808          * are computed from the following values. Just set some
4809          * reasonable dummies:
4810          */
4811         parm->hc_max_packet_size = 0x500;
4812         parm->hc_max_packet_count = 3;
4813         parm->hc_max_frame_size = 3 * 0x500;
4814
4815         usbd_transfer_setup_sub(parm);
4816
4817         /*
4818          * compute maximum number of TDs
4819          */
4820         ep_type = (xfer->endpoint->edesc->bmAttributes & UE_XFERTYPE);
4821
4822         if (ep_type == UE_CONTROL) {
4823
4824                 ntd = xfer->nframes + 1 /* STATUS */ + 1 /* SYNC 1 */
4825                     + 1 /* SYNC 2 */ + 1 /* SYNC 3 */;
4826         } else {
4827
4828                 ntd = xfer->nframes + 1 /* SYNC */ ;
4829         }
4830
4831         /*
4832          * check if "usbd_transfer_setup_sub" set an error
4833          */
4834         if (parm->err)
4835                 return;
4836
4837         /*
4838          * allocate transfer descriptors
4839          */
4840         last_obj = NULL;
4841
4842         ep_no = xfer->endpointno & UE_ADDR;
4843
4844         /*
4845          * Check for a valid endpoint profile in USB device mode:
4846          */
4847         if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) {
4848                 const struct usb_hw_ep_profile *pf;
4849
4850                 dwc_otg_get_hw_ep_profile(parm->udev, &pf, ep_no);
4851
4852                 if (pf == NULL) {
4853                         /* should not happen */
4854                         parm->err = USB_ERR_INVAL;
4855                         return;
4856                 }
4857         }
4858
4859         /* align data */
4860         parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
4861
4862         for (n = 0; n != ntd; n++) {
4863
4864                 struct dwc_otg_td *td;
4865
4866                 if (parm->buf) {
4867
4868                         td = USB_ADD_BYTES(parm->buf, parm->size[0]);
4869
4870                         /* compute shared bandwidth resource index for TT */
4871                         if (dwc_otg_uses_split(parm->udev)) {
4872                                 if (parm->udev->parent_hs_hub->ddesc.bDeviceProtocol == UDPROTO_HSHUBMTT)
4873                                         td->tt_index = parm->udev->device_index;
4874                                 else
4875                                         td->tt_index = parm->udev->parent_hs_hub->device_index;
4876                         } else {
4877                                 td->tt_index = parm->udev->device_index;
4878                         }
4879
4880                         /* init TD */
4881                         td->max_packet_size = xfer->max_packet_size;
4882                         td->max_packet_count = xfer->max_packet_count;
4883                         /* range check */
4884                         if (td->max_packet_count == 0 || td->max_packet_count > 3)
4885                                 td->max_packet_count = 1;
4886                         td->ep_no = ep_no;
4887                         td->ep_type = ep_type;
4888                         td->obj_next = last_obj;
4889
4890                         last_obj = td;
4891                 }
4892                 parm->size[0] += sizeof(*td);
4893         }
4894
4895         xfer->td_start[0] = last_obj;
4896 }
4897
4898 static void
4899 dwc_otg_xfer_unsetup(struct usb_xfer *xfer)
4900 {
4901         return;
4902 }
4903
4904 static void
4905 dwc_otg_ep_init(struct usb_device *udev, struct usb_endpoint_descriptor *edesc,
4906     struct usb_endpoint *ep)
4907 {
4908         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(udev->bus);
4909
4910         DPRINTFN(2, "endpoint=%p, addr=%d, endpt=%d, mode=%d (%d,%d)\n",
4911             ep, udev->address,
4912             edesc->bEndpointAddress, udev->flags.usb_mode,
4913             sc->sc_rt_addr, udev->device_index);
4914
4915         if (udev->device_index != sc->sc_rt_addr) {
4916
4917                 if (udev->flags.usb_mode == USB_MODE_DEVICE) {
4918                         if (udev->speed != USB_SPEED_FULL &&
4919                             udev->speed != USB_SPEED_HIGH) {
4920                                 /* not supported */
4921                                 return;
4922                         }
4923                 } else {
4924                         if (udev->speed == USB_SPEED_HIGH &&
4925                             (edesc->wMaxPacketSize[1] & 0x18) != 0 &&
4926                             (edesc->bmAttributes & UE_XFERTYPE) != UE_ISOCHRONOUS) {
4927                                 /* not supported */
4928                                 DPRINTFN(-1, "Non-isochronous high bandwidth "
4929                                     "endpoint not supported\n");
4930                                 return;
4931                         }
4932                 }
4933                 if ((edesc->bmAttributes & UE_XFERTYPE) == UE_ISOCHRONOUS)
4934                         ep->methods = &dwc_otg_device_isoc_methods;
4935                 else
4936                         ep->methods = &dwc_otg_device_non_isoc_methods;
4937         }
4938 }
4939
4940 static void
4941 dwc_otg_set_hw_power_sleep(struct usb_bus *bus, uint32_t state)
4942 {
4943         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(bus);
4944
4945         switch (state) {
4946         case USB_HW_POWER_SUSPEND:
4947                 dwc_otg_suspend(sc);
4948                 break;
4949         case USB_HW_POWER_SHUTDOWN:
4950                 dwc_otg_uninit(sc);
4951                 break;
4952         case USB_HW_POWER_RESUME:
4953                 dwc_otg_resume(sc);
4954                 break;
4955         default:
4956                 break;
4957         }
4958 }
4959
4960 static void
4961 dwc_otg_get_dma_delay(struct usb_device *udev, uint32_t *pus)
4962 {
4963         /* DMA delay - wait until any use of memory is finished */
4964         *pus = (2125);                  /* microseconds */
4965 }
4966
4967 static void
4968 dwc_otg_device_resume(struct usb_device *udev)
4969 {
4970         DPRINTF("\n");
4971
4972         /* poll all transfers again to restart resumed ones */
4973         dwc_otg_do_poll(udev->bus);
4974 }
4975
4976 static void
4977 dwc_otg_device_suspend(struct usb_device *udev)
4978 {
4979         DPRINTF("\n");
4980 }
4981
4982 static const struct usb_bus_methods dwc_otg_bus_methods =
4983 {
4984         .endpoint_init = &dwc_otg_ep_init,
4985         .xfer_setup = &dwc_otg_xfer_setup,
4986         .xfer_unsetup = &dwc_otg_xfer_unsetup,
4987         .get_hw_ep_profile = &dwc_otg_get_hw_ep_profile,
4988         .xfer_stall = &dwc_otg_xfer_stall,
4989         .set_stall = &dwc_otg_set_stall,
4990         .clear_stall = &dwc_otg_clear_stall,
4991         .roothub_exec = &dwc_otg_roothub_exec,
4992         .xfer_poll = &dwc_otg_do_poll,
4993         .device_state_change = &dwc_otg_device_state_change,
4994         .set_hw_power_sleep = &dwc_otg_set_hw_power_sleep,
4995         .get_dma_delay = &dwc_otg_get_dma_delay,
4996         .device_resume = &dwc_otg_device_resume,
4997         .device_suspend = &dwc_otg_device_suspend,
4998 };