sys/dev/usb/controller/dwc_otg.c

   1 /* $FreeBSD$ */
   2 /*-
   3  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   4  *
   5  * Copyright (c) 2015 Daisuke Aoyama. All rights reserved.
   6  * Copyright (c) 2012-2015 Hans Petter Selasky. All rights reserved.
   7  * Copyright (c) 2010-2011 Aleksandr Rybalko. All rights reserved.
   8  *
   9  * Redistribution and use in source and binary forms, with or without
  10  * modification, are permitted provided that the following conditions
  11  * are met:
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions and the following disclaimer.
  14  * 2. Redistributions in binary form must reproduce the above copyright
  15  *    notice, this list of conditions and the following disclaimer in the
  16  *    documentation and/or other materials provided with the distribution.
  17  *
  18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  21  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  28  * SUCH DAMAGE.
  29  */
  30
  31 /*
  32  * This file contains the driver for the DesignWare series USB 2.0 OTG
  33  * Controller.
  34  */
  35
  36 /*
  37  * LIMITATION: Drivers must be bound to all OUT endpoints in the
  38  * active configuration for this driver to work properly. Blocking any
  39  * OUT endpoint will block all OUT endpoints including the control
  40  * endpoint. Usually this is not a problem.
  41  */
  42
  43 /*
  44  * NOTE: Writing to non-existing registers appears to cause an
  45  * internal reset.
  46  */
  47
  48 #ifdef USB_GLOBAL_INCLUDE_FILE
  49 #include USB_GLOBAL_INCLUDE_FILE
  50 #else
  51 #include <sys/stdint.h>
  52 #include <sys/stddef.h>
  53 #include <sys/param.h>
  54 #include <sys/queue.h>
  55 #include <sys/types.h>
  56 #include <sys/systm.h>
  57 #include <sys/kernel.h>
  58 #include <sys/bus.h>
  59 #include <sys/module.h>
  60 #include <sys/lock.h>
  61 #include <sys/mutex.h>
  62 #include <sys/condvar.h>
  63 #include <sys/sysctl.h>
  64 #include <sys/sx.h>
  65 #include <sys/unistd.h>
  66 #include <sys/callout.h>
  67 #include <sys/malloc.h>
  68 #include <sys/priv.h>
  69
  70 #include <dev/usb/usb.h>
  71 #include <dev/usb/usbdi.h>
  72
  73 #define USB_DEBUG_VAR dwc_otg_debug
  74
  75 #include <dev/usb/usb_core.h>
  76 #include <dev/usb/usb_debug.h>
  77 #include <dev/usb/usb_busdma.h>
  78 #include <dev/usb/usb_process.h>
  79 #include <dev/usb/usb_transfer.h>
  80 #include <dev/usb/usb_device.h>
  81 #include <dev/usb/usb_hub.h>
  82 #include <dev/usb/usb_util.h>
  83
  84 #include <dev/usb/usb_controller.h>
  85 #include <dev/usb/usb_bus.h>
  86 #endif                  /* USB_GLOBAL_INCLUDE_FILE */
  87
  88 #include <dev/usb/controller/dwc_otg.h>
  89 #include <dev/usb/controller/dwc_otgreg.h>
  90
  91 #define DWC_OTG_BUS2SC(bus) \
  92    ((struct dwc_otg_softc *)(((uint8_t *)(bus)) - \
  93     ((uint8_t *)&(((struct dwc_otg_softc *)0)->sc_bus))))
  94
  95 #define DWC_OTG_PC2UDEV(pc) \
  96    (USB_DMATAG_TO_XROOT((pc)->tag_parent)->udev)
  97
  98 #define DWC_OTG_MSK_GINT_THREAD_IRQ                             \
  99    (GINTSTS_USBRST | GINTSTS_ENUMDONE | GINTSTS_PRTINT |        \
 100    GINTSTS_WKUPINT | GINTSTS_USBSUSP | GINTMSK_OTGINTMSK |      \
 101    GINTSTS_SESSREQINT)
 102
 103 #ifndef DWC_OTG_PHY_DEFAULT
 104 #define DWC_OTG_PHY_DEFAULT DWC_OTG_PHY_ULPI
 105 #endif
 106
 107 static int dwc_otg_phy_type = DWC_OTG_PHY_DEFAULT;
 108
 109 static SYSCTL_NODE(_hw_usb, OID_AUTO, dwc_otg, CTLFLAG_RW, 0, "USB DWC OTG");
 110 SYSCTL_INT(_hw_usb_dwc_otg, OID_AUTO, phy_type, CTLFLAG_RDTUN,
 111     &dwc_otg_phy_type, 0, "DWC OTG PHY TYPE - 0/1/2/3 - ULPI/HSIC/INTERNAL/UTMI+");
 112
 113 #ifdef USB_DEBUG
 114 static int dwc_otg_debug = 0;
 115
 116 SYSCTL_INT(_hw_usb_dwc_otg, OID_AUTO, debug, CTLFLAG_RWTUN,
 117     &dwc_otg_debug, 0, "DWC OTG debug level");
 118 #endif
 119
 120 #define DWC_OTG_INTR_ENDPT 1
 121
 122 /* prototypes */
 123
 124 static const struct usb_bus_methods dwc_otg_bus_methods;
 125 static const struct usb_pipe_methods dwc_otg_device_non_isoc_methods;
 126 static const struct usb_pipe_methods dwc_otg_device_isoc_methods;
 127
 128 static dwc_otg_cmd_t dwc_otg_setup_rx;
 129 static dwc_otg_cmd_t dwc_otg_data_rx;
 130 static dwc_otg_cmd_t dwc_otg_data_tx;
 131 static dwc_otg_cmd_t dwc_otg_data_tx_sync;
 132
 133 static dwc_otg_cmd_t dwc_otg_host_setup_tx;
 134 static dwc_otg_cmd_t dwc_otg_host_data_tx;
 135 static dwc_otg_cmd_t dwc_otg_host_data_rx;
 136
 137 static void dwc_otg_device_done(struct usb_xfer *, usb_error_t);
 138 static void dwc_otg_do_poll(struct usb_bus *);
 139 static void dwc_otg_standard_done(struct usb_xfer *);
 140 static void dwc_otg_root_intr(struct dwc_otg_softc *);
 141 static void dwc_otg_interrupt_poll_locked(struct dwc_otg_softc *);
 142
 143 /*
 144  * Here is a configuration that the chip supports.
 145  */
 146 static const struct usb_hw_ep_profile dwc_otg_ep_profile[1] = {
 147
 148         [0] = {
 149                 .max_in_frame_size = 64,/* fixed */
 150                 .max_out_frame_size = 64,       /* fixed */
 151                 .is_simplex = 1,
 152                 .support_control = 1,
 153         }
 154 };
 155
 156 static void
 157 dwc_otg_get_hw_ep_profile(struct usb_device *udev,
 158     const struct usb_hw_ep_profile **ppf, uint8_t ep_addr)
 159 {
 160         struct dwc_otg_softc *sc;
 161
 162         sc = DWC_OTG_BUS2SC(udev->bus);
 163
 164         if (ep_addr < sc->sc_dev_ep_max)
 165                 *ppf = &sc->sc_hw_ep_profile[ep_addr].usb;
 166         else
 167                 *ppf = NULL;
 168 }
 169
 170 static void
 171 dwc_otg_write_fifo(struct dwc_otg_softc *sc, struct usb_page_cache *pc,
 172     uint32_t offset, uint32_t fifo, uint32_t count)
 173 {
 174         uint32_t temp;
 175
 176         /* round down length to nearest 4-bytes */
 177         temp = count & ~3;
 178
 179         /* check if we can write the data directly */
 180         if (temp != 0 && usb_pc_buffer_is_aligned(pc, offset, temp, 3)) {
 181                 struct usb_page_search buf_res;
 182
 183                 /* pre-subtract length */
 184                 count -= temp;
 185
 186                 /* iterate buffer list */
 187                 do {
 188                         /* get current buffer pointer */
 189                         usbd_get_page(pc, offset, &buf_res);
 190
 191                         if (buf_res.length > temp)
 192                                 buf_res.length = temp;
 193
 194                         /* transfer data into FIFO */
 195                         bus_space_write_region_4(sc->sc_io_tag, sc->sc_io_hdl,
 196                             fifo, buf_res.buffer, buf_res.length / 4);
 197
 198                         offset += buf_res.length;
 199                         fifo += buf_res.length;
 200                         temp -= buf_res.length;
 201                 } while (temp != 0);
 202         }
 203
 204         /* check for remainder */
 205         if (count != 0) {
 206                 /* clear topmost word before copy */
 207                 sc->sc_bounce_buffer[(count - 1) / 4] = 0;
 208
 209                 /* copy out data */
 210                 usbd_copy_out(pc, offset,
 211                     sc->sc_bounce_buffer, count);
 212
 213                 /* transfer data into FIFO */
 214                 bus_space_write_region_4(sc->sc_io_tag,
 215                     sc->sc_io_hdl, fifo, sc->sc_bounce_buffer,
 216                     (count + 3) / 4);
 217         }
 218 }
 219
 220 static void
 221 dwc_otg_read_fifo(struct dwc_otg_softc *sc, struct usb_page_cache *pc,
 222     uint32_t offset, uint32_t count)
 223 {
 224         uint32_t temp;
 225
 226         /* round down length to nearest 4-bytes */
 227         temp = count & ~3;
 228
 229         /* check if we can read the data directly */
 230         if (temp != 0 && usb_pc_buffer_is_aligned(pc, offset, temp, 3)) {
 231                 struct usb_page_search buf_res;
 232
 233                 /* pre-subtract length */
 234                 count -= temp;
 235
 236                 /* iterate buffer list */
 237                 do {
 238                         /* get current buffer pointer */
 239                         usbd_get_page(pc, offset, &buf_res);
 240
 241                         if (buf_res.length > temp)
 242                                 buf_res.length = temp;
 243
 244                         /* transfer data from FIFO */
 245                         bus_space_read_region_4(sc->sc_io_tag, sc->sc_io_hdl,
 246                             sc->sc_current_rx_fifo, buf_res.buffer, buf_res.length / 4);
 247
 248                         offset += buf_res.length;
 249                         sc->sc_current_rx_fifo += buf_res.length;
 250                         sc->sc_current_rx_bytes -= buf_res.length;
 251                         temp -= buf_res.length;
 252                 } while (temp != 0);
 253         }
 254
 255         /* check for remainder */
 256         if (count != 0) {
 257                 /* read data into bounce buffer */
 258                 bus_space_read_region_4(sc->sc_io_tag, sc->sc_io_hdl,
 259                         sc->sc_current_rx_fifo,
 260                         sc->sc_bounce_buffer, (count + 3) / 4);
 261
 262                 /* store data into proper buffer */
 263                 usbd_copy_in(pc, offset, sc->sc_bounce_buffer, count);
 264
 265                 /* round length up to nearest 4 bytes */
 266                 count = (count + 3) & ~3;
 267
 268                 /* update counters */
 269                 sc->sc_current_rx_bytes -= count;
 270                 sc->sc_current_rx_fifo += count;
 271         }
 272 }
 273
 274 static void
 275 dwc_otg_tx_fifo_reset(struct dwc_otg_softc *sc, uint32_t value)
 276 {
 277         uint32_t temp;
 278
 279         /* reset FIFO */
 280         DWC_OTG_WRITE_4(sc, DOTG_GRSTCTL, value);
 281
 282         /* wait for reset to complete */
 283         for (temp = 0; temp != 16; temp++) {
 284                 value = DWC_OTG_READ_4(sc, DOTG_GRSTCTL);
 285                 if (!(value & (GRSTCTL_TXFFLSH | GRSTCTL_RXFFLSH)))
 286                         break;
 287         }
 288 }
 289
 290 static int
 291 dwc_otg_init_fifo(struct dwc_otg_softc *sc, uint8_t mode)
 292 {
 293         struct dwc_otg_profile *pf;
 294         uint32_t fifo_size;
 295         uint32_t fifo_regs;
 296         uint32_t tx_start;
 297         uint8_t x;
 298
 299         fifo_size = sc->sc_fifo_size;
 300
 301         /*
 302          * NOTE: Reserved fixed size area at end of RAM, which must
 303          * not be allocated to the FIFOs:
 304          */
 305         fifo_regs = 4 * 16;
 306
 307         if (fifo_size < fifo_regs) {
 308                 DPRINTF("Too little FIFO\n");
 309                 return (EINVAL);
 310         }
 311
 312         /* subtract FIFO regs from total once */
 313         fifo_size -= fifo_regs;
 314
 315         /* split equally for IN and OUT */
 316         fifo_size /= 2;
 317
 318         /* Align to 4 bytes boundary (refer to PGM) */
 319         fifo_size &= ~3;
 320
 321         /* set global receive FIFO size */
 322         DWC_OTG_WRITE_4(sc, DOTG_GRXFSIZ, fifo_size / 4);
 323
 324         tx_start = fifo_size;
 325
 326         if (fifo_size < 64) {
 327                 DPRINTFN(-1, "Not enough data space for EP0 FIFO.\n");
 328                 return (EINVAL);
 329         }
 330
 331         if (mode == DWC_MODE_HOST) {
 332
 333                 /* reset active endpoints */
 334                 sc->sc_active_rx_ep = 0;
 335
 336                 /* split equally for periodic and non-periodic */
 337                 fifo_size /= 2;
 338
 339                 DPRINTF("PTX/NPTX FIFO=%u\n", fifo_size);
 340
 341                 /* align to 4 bytes boundary */
 342                 fifo_size &= ~3;
 343
 344                 DWC_OTG_WRITE_4(sc, DOTG_GNPTXFSIZ,
 345                     ((fifo_size / 4) << 16) |
 346                     (tx_start / 4));
 347
 348                 tx_start += fifo_size;
 349
 350                 for (x = 0; x != sc->sc_host_ch_max; x++) {
 351                         /* enable all host interrupts */
 352                         DWC_OTG_WRITE_4(sc, DOTG_HCINTMSK(x),
 353                             HCINT_DEFAULT_MASK);
 354                 }
 355
 356                 DWC_OTG_WRITE_4(sc, DOTG_HPTXFSIZ,
 357                     ((fifo_size / 4) << 16) |
 358                     (tx_start / 4));
 359
 360                 /* reset host channel state */
 361                 memset(sc->sc_chan_state, 0, sizeof(sc->sc_chan_state));
 362
 363                 /* enable all host channel interrupts */
 364                 DWC_OTG_WRITE_4(sc, DOTG_HAINTMSK,
 365                     (1U << sc->sc_host_ch_max) - 1U);
 366
 367                 /* enable proper host channel interrupts */
 368                 sc->sc_irq_mask |= GINTMSK_HCHINTMSK;
 369                 sc->sc_irq_mask &= ~GINTMSK_IEPINTMSK;
 370                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 371         }
 372
 373         if (mode == DWC_MODE_DEVICE) {
 374
 375             DWC_OTG_WRITE_4(sc, DOTG_GNPTXFSIZ,
 376                 (0x10 << 16) | (tx_start / 4));
 377             fifo_size -= 0x40;
 378             tx_start += 0x40;
 379
 380             /* setup control endpoint profile */
 381             sc->sc_hw_ep_profile[0].usb = dwc_otg_ep_profile[0];
 382
 383             /* reset active endpoints */
 384             sc->sc_active_rx_ep = 1;
 385
 386             for (x = 1; x != sc->sc_dev_ep_max; x++) {
 387
 388                 pf = sc->sc_hw_ep_profile + x;
 389
 390                 pf->usb.max_out_frame_size = 1024 * 3;
 391                 pf->usb.is_simplex = 0; /* assume duplex */
 392                 pf->usb.support_bulk = 1;
 393                 pf->usb.support_interrupt = 1;
 394                 pf->usb.support_isochronous = 1;
 395                 pf->usb.support_out = 1;
 396
 397                 if (x < sc->sc_dev_in_ep_max) {
 398                         uint32_t limit;
 399
 400                         limit = (x == 1) ? MIN(DWC_OTG_TX_MAX_FIFO_SIZE,
 401                             DWC_OTG_MAX_TXN) : MIN(DWC_OTG_MAX_TXN / 2,
 402                             DWC_OTG_TX_MAX_FIFO_SIZE);
 403
 404                         /* see if there is enough FIFO space */
 405                         if (limit <= fifo_size) {
 406                                 pf->max_buffer = limit;
 407                                 pf->usb.support_in = 1;
 408                         } else {
 409                                 limit = MIN(DWC_OTG_TX_MAX_FIFO_SIZE, 0x40);
 410                                 if (limit <= fifo_size) {
 411                                         pf->usb.support_in = 1;
 412                                 } else {
 413                                         pf->usb.is_simplex = 1;
 414                                         limit = 0;
 415                                 }
 416                         }
 417                         /* set FIFO size */
 418                         DWC_OTG_WRITE_4(sc, DOTG_DIEPTXF(x),
 419                             ((limit / 4) << 16) | (tx_start / 4));
 420                         tx_start += limit;
 421                         fifo_size -= limit;
 422                         pf->usb.max_in_frame_size = limit;
 423                 } else {
 424                         pf->usb.is_simplex = 1;
 425                 }
 426
 427                 DPRINTF("FIFO%d = IN:%d / OUT:%d\n", x,
 428                     pf->usb.max_in_frame_size,
 429                     pf->usb.max_out_frame_size);
 430             }
 431
 432             /* enable proper device channel interrupts */
 433             sc->sc_irq_mask &= ~GINTMSK_HCHINTMSK;
 434             sc->sc_irq_mask |= GINTMSK_IEPINTMSK;
 435             DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 436         }
 437
 438         /* reset RX FIFO */
 439         dwc_otg_tx_fifo_reset(sc, GRSTCTL_RXFFLSH);
 440
 441         if (mode != DWC_MODE_OTG) {
 442                 /* reset all TX FIFOs */
 443                 dwc_otg_tx_fifo_reset(sc,
 444                     GRSTCTL_TXFIFO(0x10) |
 445                     GRSTCTL_TXFFLSH);
 446         } else {
 447                 /* reset active endpoints */
 448                 sc->sc_active_rx_ep = 0;
 449
 450                 /* reset host channel state */
 451                 memset(sc->sc_chan_state, 0, sizeof(sc->sc_chan_state));
 452         }
 453         return (0);
 454 }
 455
 456 static uint8_t
 457 dwc_otg_uses_split(struct usb_device *udev)
 458 {
 459         /*
 460          * When a LOW or FULL speed device is connected directly to
 461          * the USB port we don't use split transactions:
 462          */
 463         return (udev->speed != USB_SPEED_HIGH &&
 464             udev->parent_hs_hub != NULL &&
 465             udev->parent_hs_hub->parent_hub != NULL);
 466 }
 467
 468 static void
 469 dwc_otg_update_host_frame_interval(struct dwc_otg_softc *sc)
 470 {
 471
 472   /*
 473    * Disabled until further. Assuming that the register is already
 474    * programmed correctly by the boot loader.
 475    */
 476 #if 0
 477         uint32_t temp;
 478
 479         /* setup HOST frame interval register, based on existing value */
 480         temp = DWC_OTG_READ_4(sc, DOTG_HFIR) & HFIR_FRINT_MASK;
 481         if (temp >= 10000)
 482                 temp /= 1000;
 483         else
 484                 temp /= 125;
 485
 486         /* figure out nearest X-tal value */
 487         if (temp >= 54)
 488                 temp = 60;      /* MHz */
 489         else if (temp >= 39)
 490                 temp = 48;      /* MHz */
 491         else
 492                 temp = 30;      /* MHz */
 493
 494         if (sc->sc_flags.status_high_speed)
 495                 temp *= 125;
 496         else
 497                 temp *= 1000;
 498
 499         DPRINTF("HFIR=0x%08x\n", temp);
 500
 501         DWC_OTG_WRITE_4(sc, DOTG_HFIR, temp);
 502 #endif
 503 }
 504
 505 static void
 506 dwc_otg_clocks_on(struct dwc_otg_softc *sc)
 507 {
 508         if (sc->sc_flags.clocks_off &&
 509             sc->sc_flags.port_powered) {
 510
 511                 DPRINTFN(5, "\n");
 512
 513                 /* TODO - platform specific */
 514
 515                 sc->sc_flags.clocks_off = 0;
 516         }
 517 }
 518
 519 static void
 520 dwc_otg_clocks_off(struct dwc_otg_softc *sc)
 521 {
 522         if (!sc->sc_flags.clocks_off) {
 523
 524                 DPRINTFN(5, "\n");
 525
 526                 /* TODO - platform specific */
 527
 528                 sc->sc_flags.clocks_off = 1;
 529         }
 530 }
 531
 532 static void
 533 dwc_otg_pull_up(struct dwc_otg_softc *sc)
 534 {
 535         uint32_t temp;
 536
 537         /* pullup D+, if possible */
 538
 539         if (!sc->sc_flags.d_pulled_up &&
 540             sc->sc_flags.port_powered) {
 541                 sc->sc_flags.d_pulled_up = 1;
 542
 543                 temp = DWC_OTG_READ_4(sc, DOTG_DCTL);
 544                 temp &= ~DCTL_SFTDISCON;
 545                 DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
 546         }
 547 }
 548
 549 static void
 550 dwc_otg_pull_down(struct dwc_otg_softc *sc)
 551 {
 552         uint32_t temp;
 553
 554         /* pulldown D+, if possible */
 555
 556         if (sc->sc_flags.d_pulled_up) {
 557                 sc->sc_flags.d_pulled_up = 0;
 558
 559                 temp = DWC_OTG_READ_4(sc, DOTG_DCTL);
 560                 temp |= DCTL_SFTDISCON;
 561                 DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
 562         }
 563 }
 564
 565 static void
 566 dwc_otg_enable_sof_irq(struct dwc_otg_softc *sc)
 567 {
 568         /* In device mode we don't use the SOF interrupt */
 569         if (sc->sc_flags.status_device_mode != 0)
 570                 return;
 571         /* Ensure the SOF interrupt is not disabled */
 572         sc->sc_needsof = 1;
 573         /* Check if the SOF interrupt is already enabled */
 574         if ((sc->sc_irq_mask & GINTMSK_SOFMSK) != 0)
 575                 return;
 576         sc->sc_irq_mask |= GINTMSK_SOFMSK;
 577         DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 578 }
 579
 580 static void
 581 dwc_otg_resume_irq(struct dwc_otg_softc *sc)
 582 {
 583         if (sc->sc_flags.status_suspend) {
 584                 /* update status bits */
 585                 sc->sc_flags.status_suspend = 0;
 586                 sc->sc_flags.change_suspend = 1;
 587
 588                 if (sc->sc_flags.status_device_mode) {
 589                         /*
 590                          * Disable resume interrupt and enable suspend
 591                          * interrupt:
 592                          */
 593                         sc->sc_irq_mask &= ~GINTMSK_WKUPINTMSK;
 594                         sc->sc_irq_mask |= GINTMSK_USBSUSPMSK;
 595                         DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 596                 }
 597
 598                 /* complete root HUB interrupt endpoint */
 599                 dwc_otg_root_intr(sc);
 600         }
 601 }
 602
 603 static void
 604 dwc_otg_suspend_irq(struct dwc_otg_softc *sc)
 605 {
 606         if (!sc->sc_flags.status_suspend) {
 607                 /* update status bits */
 608                 sc->sc_flags.status_suspend = 1;
 609                 sc->sc_flags.change_suspend = 1;
 610
 611                 if (sc->sc_flags.status_device_mode) {
 612                         /*
 613                          * Disable suspend interrupt and enable resume
 614                          * interrupt:
 615                          */
 616                         sc->sc_irq_mask &= ~GINTMSK_USBSUSPMSK;
 617                         sc->sc_irq_mask |= GINTMSK_WKUPINTMSK;
 618                         DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 619                 }
 620
 621                 /* complete root HUB interrupt endpoint */
 622                 dwc_otg_root_intr(sc);
 623         }
 624 }
 625
 626 static void
 627 dwc_otg_wakeup_peer(struct dwc_otg_softc *sc)
 628 {
 629         if (!sc->sc_flags.status_suspend)
 630                 return;
 631
 632         DPRINTFN(5, "Remote wakeup\n");
 633
 634         if (sc->sc_flags.status_device_mode) {
 635                 uint32_t temp;
 636
 637                 /* enable remote wakeup signalling */
 638                 temp = DWC_OTG_READ_4(sc, DOTG_DCTL);
 639                 temp |= DCTL_RMTWKUPSIG;
 640                 DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
 641
 642                 /* Wait 8ms for remote wakeup to complete. */
 643                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 125);
 644
 645                 temp &= ~DCTL_RMTWKUPSIG;
 646                 DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
 647         } else {
 648                 /* enable USB port */
 649                 DWC_OTG_WRITE_4(sc, DOTG_PCGCCTL, 0);
 650
 651                 /* wait 10ms */
 652                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 100);
 653
 654                 /* resume port */
 655                 sc->sc_hprt_val |= HPRT_PRTRES;
 656                 DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
 657
 658                 /* Wait 100ms for resume signalling to complete. */
 659                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 10);
 660
 661                 /* clear suspend and resume */
 662                 sc->sc_hprt_val &= ~(HPRT_PRTSUSP | HPRT_PRTRES);
 663                 DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
 664
 665                 /* Wait 4ms */
 666                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 250);
 667         }
 668
 669         /* need to fake resume IRQ */
 670         dwc_otg_resume_irq(sc);
 671 }
 672
 673 static void
 674 dwc_otg_set_address(struct dwc_otg_softc *sc, uint8_t addr)
 675 {
 676         uint32_t temp;
 677
 678         DPRINTFN(5, "addr=%d\n", addr);
 679
 680         temp = DWC_OTG_READ_4(sc, DOTG_DCFG);
 681         temp &= ~DCFG_DEVADDR_SET(0x7F);
 682         temp |= DCFG_DEVADDR_SET(addr);
 683         DWC_OTG_WRITE_4(sc, DOTG_DCFG, temp);
 684 }
 685
 686 static void
 687 dwc_otg_common_rx_ack(struct dwc_otg_softc *sc)
 688 {
 689         DPRINTFN(5, "RX status clear\n");
 690
 691         /* enable RX FIFO level interrupt */
 692         sc->sc_irq_mask |= GINTMSK_RXFLVLMSK;
 693         DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 694
 695         if (sc->sc_current_rx_bytes != 0) {
 696                 /* need to dump remaining data */
 697                 bus_space_read_region_4(sc->sc_io_tag, sc->sc_io_hdl,
 698                     sc->sc_current_rx_fifo, sc->sc_bounce_buffer,
 699                     sc->sc_current_rx_bytes / 4);
 700                 /* clear number of active bytes to receive */
 701                 sc->sc_current_rx_bytes = 0;
 702         }
 703         /* clear cached status */
 704         sc->sc_last_rx_status = 0;
 705 }
 706
 707 static void
 708 dwc_otg_clear_hcint(struct dwc_otg_softc *sc, uint8_t x)
 709 {
 710         uint32_t hcint;
 711
 712         /* clear all pending interrupts */
 713         hcint = DWC_OTG_READ_4(sc, DOTG_HCINT(x));
 714         DWC_OTG_WRITE_4(sc, DOTG_HCINT(x), hcint);
 715
 716         /* clear buffered interrupts */
 717         sc->sc_chan_state[x].hcint = 0;
 718 }
 719
 720 static uint8_t
 721 dwc_otg_host_check_tx_fifo_empty(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
 722 {
 723         uint32_t temp;
 724
 725         temp = DWC_OTG_READ_4(sc, DOTG_GINTSTS);
 726
 727         if (td->ep_type == UE_ISOCHRONOUS) {
 728                 /*
 729                  * NOTE: USB INTERRUPT transactions are executed like
 730                  * USB CONTROL transactions! See the setup standard
 731                  * chain function for more information.
 732                  */
 733                 if (!(temp & GINTSTS_PTXFEMP)) {
 734                         DPRINTF("Periodic TX FIFO is not empty\n");
 735                         if (!(sc->sc_irq_mask & GINTMSK_PTXFEMPMSK)) {
 736                                 sc->sc_irq_mask |= GINTMSK_PTXFEMPMSK;
 737                                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 738                         }
 739                         return (1);     /* busy */
 740                 }
 741         } else {
 742                 if (!(temp & GINTSTS_NPTXFEMP)) {
 743                         DPRINTF("Non-periodic TX FIFO is not empty\n");
 744                         if (!(sc->sc_irq_mask & GINTMSK_NPTXFEMPMSK)) {
 745                                 sc->sc_irq_mask |= GINTMSK_NPTXFEMPMSK;
 746                                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 747                         }
 748                         return (1);     /* busy */
 749                 }
 750         }
 751         return (0);     /* ready for transmit */
 752 }
 753
 754 static uint8_t
 755 dwc_otg_host_channel_alloc(struct dwc_otg_softc *sc,
 756     struct dwc_otg_td *td, uint8_t is_out)
 757 {
 758         uint8_t x;
 759         uint8_t y;
 760         uint8_t z;
 761
 762         if (td->channel[0] < DWC_OTG_MAX_CHANNELS)
 763                 return (0);             /* already allocated */
 764
 765         /* check if device is suspended */
 766         if (DWC_OTG_PC2UDEV(td->pc)->flags.self_suspended != 0)
 767                 return (1);             /* busy - cannot transfer data */
 768
 769         /* compute needed TX FIFO size */
 770         if (is_out != 0) {
 771                 if (dwc_otg_host_check_tx_fifo_empty(sc, td) != 0)
 772                         return (1);     /* busy - cannot transfer data */
 773         }
 774         z = td->max_packet_count;
 775         for (x = y = 0; x != sc->sc_host_ch_max; x++) {
 776                 /* check if channel is allocated */
 777                 if (sc->sc_chan_state[x].allocated != 0)
 778                         continue;
 779                 /* check if channel is still enabled */
 780                 if (sc->sc_chan_state[x].wait_halted != 0)
 781                         continue;
 782                 /* store channel number */
 783                 td->channel[y++] = x;
 784                 /* check if we got all channels */
 785                 if (y == z)
 786                         break;
 787         }
 788         if (y != z) {
 789                 /* reset channel variable */
 790                 td->channel[0] = DWC_OTG_MAX_CHANNELS;
 791                 td->channel[1] = DWC_OTG_MAX_CHANNELS;
 792                 td->channel[2] = DWC_OTG_MAX_CHANNELS;
 793                 /* wait a bit */
 794                 dwc_otg_enable_sof_irq(sc);
 795                 return (1);     /* busy - not enough channels */
 796         }
 797
 798         for (y = 0; y != z; y++) {
 799                 x = td->channel[y];
 800
 801                 /* set allocated */
 802                 sc->sc_chan_state[x].allocated = 1;
 803
 804                 /* set wait halted */
 805                 sc->sc_chan_state[x].wait_halted = 1;
 806
 807                 /* clear interrupts */
 808                 dwc_otg_clear_hcint(sc, x);
 809
 810                 DPRINTF("CH=%d HCCHAR=0x%08x "
 811                     "HCSPLT=0x%08x\n", x, td->hcchar, td->hcsplt);
 812
 813                 /* set active channel */
 814                 sc->sc_active_rx_ep |= (1 << x);
 815         }
 816         return (0);     /* allocated */
 817 }
 818
 819 static void
 820 dwc_otg_host_channel_free_sub(struct dwc_otg_softc *sc, struct dwc_otg_td *td, uint8_t index)
 821 {
 822         uint32_t hcchar;
 823         uint8_t x;
 824
 825         if (td->channel[index] >= DWC_OTG_MAX_CHANNELS)
 826                 return;         /* already freed */
 827
 828         /* free channel */
 829         x = td->channel[index];
 830         td->channel[index] = DWC_OTG_MAX_CHANNELS;
 831
 832         DPRINTF("CH=%d\n", x);
 833
 834         /*
 835          * We need to let programmed host channels run till complete
 836          * else the host channel will stop functioning.
 837          */
 838         sc->sc_chan_state[x].allocated = 0;
 839
 840         /* ack any pending messages */
 841         if (sc->sc_last_rx_status != 0 &&
 842             GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) == x) {
 843                 dwc_otg_common_rx_ack(sc);
 844         }
 845
 846         /* clear active channel */
 847         sc->sc_active_rx_ep &= ~(1 << x);
 848
 849         /* check if already halted */
 850         if (sc->sc_chan_state[x].wait_halted == 0)
 851                 return;
 852
 853         /* disable host channel */
 854         hcchar = DWC_OTG_READ_4(sc, DOTG_HCCHAR(x));
 855         if (hcchar & HCCHAR_CHENA) {
 856                 DPRINTF("Halting channel %d\n", x);
 857                 DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(x),
 858                     hcchar | HCCHAR_CHDIS);
 859                 /* don't write HCCHAR until the channel is halted */
 860         } else {
 861                 sc->sc_chan_state[x].wait_halted = 0;
 862         }
 863 }
 864
 865 static void
 866 dwc_otg_host_channel_free(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
 867 {
 868         uint8_t x;
 869         for (x = 0; x != td->max_packet_count; x++)
 870                 dwc_otg_host_channel_free_sub(sc, td, x);
 871 }
 872
 873 static void
 874 dwc_otg_host_dump_rx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
 875 {
 876         uint8_t x;
 877         /* dump any pending messages */
 878         if (sc->sc_last_rx_status == 0)
 879                 return;
 880         for (x = 0; x != td->max_packet_count; x++) {
 881                 if (td->channel[x] >= DWC_OTG_MAX_CHANNELS ||
 882                     td->channel[x] != GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status))
 883                         continue;
 884                 dwc_otg_common_rx_ack(sc);
 885                 break;
 886         }
 887 }
 888
 889 static uint8_t
 890 dwc_otg_host_setup_tx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
 891 {
 892         struct usb_device_request req __aligned(4);
 893         uint32_t hcint;
 894         uint32_t hcchar;
 895         uint8_t delta;
 896
 897         dwc_otg_host_dump_rx(sc, td);
 898
 899         if (td->channel[0] < DWC_OTG_MAX_CHANNELS) {
 900                 hcint = sc->sc_chan_state[td->channel[0]].hcint;
 901
 902                 DPRINTF("CH=%d ST=%d HCINT=0x%08x HCCHAR=0x%08x HCTSIZ=0x%08x\n",
 903                     td->channel[0], td->state, hcint,
 904                     DWC_OTG_READ_4(sc, DOTG_HCCHAR(td->channel[0])),
 905                     DWC_OTG_READ_4(sc, DOTG_HCTSIZ(td->channel[0])));
 906         } else {
 907                 hcint = 0;
 908                 goto check_state;
 909         }
 910
 911         if (hcint & (HCINT_RETRY |
 912             HCINT_ACK | HCINT_NYET)) {
 913                 /* give success bits priority over failure bits */
 914         } else if (hcint & HCINT_STALL) {
 915                 DPRINTF("CH=%d STALL\n", td->channel[0]);
 916                 td->error_stall = 1;
 917                 td->error_any = 1;
 918                 goto complete;
 919         } else if (hcint & HCINT_ERRORS) {
 920                 DPRINTF("CH=%d ERROR\n", td->channel[0]);
 921                 td->errcnt++;
 922                 if (td->hcsplt != 0 || td->errcnt >= 3) {
 923                         td->error_any = 1;
 924                         goto complete;
 925                 }
 926         }
 927
 928         if (hcint & (HCINT_ERRORS | HCINT_RETRY |
 929             HCINT_ACK | HCINT_NYET)) {
 930                 if (!(hcint & HCINT_ERRORS))
 931                         td->errcnt = 0;
 932         }
 933
 934 check_state:
 935         switch (td->state) {
 936         case DWC_CHAN_ST_START:
 937                 goto send_pkt;
 938
 939         case DWC_CHAN_ST_WAIT_ANE:
 940                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
 941                         td->did_nak = 1;
 942                         td->tt_scheduled = 0;
 943                         goto send_pkt;
 944                 } else if (hcint & (HCINT_ACK | HCINT_NYET)) {
 945                         td->offset += td->tx_bytes;
 946                         td->remainder -= td->tx_bytes;
 947                         td->toggle = 1;
 948                         td->tt_scheduled = 0;
 949                         goto complete;
 950                 }
 951                 break;
 952
 953         case DWC_CHAN_ST_WAIT_S_ANE:
 954                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
 955                         td->did_nak = 1;
 956                         td->tt_scheduled = 0;
 957                         goto send_pkt;
 958                 } else if (hcint & (HCINT_ACK | HCINT_NYET)) {
 959                         goto send_cpkt;
 960                 }
 961                 break;
 962
 963         case DWC_CHAN_ST_WAIT_C_ANE:
 964                 if (hcint & HCINT_NYET) {
 965                         goto send_cpkt;
 966                 } else if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
 967                         td->did_nak = 1;
 968                         td->tt_scheduled = 0;
 969                         goto send_pkt;
 970                 } else if (hcint & HCINT_ACK) {
 971                         td->offset += td->tx_bytes;
 972                         td->remainder -= td->tx_bytes;
 973                         td->toggle = 1;
 974                         goto complete;
 975                 }
 976                 break;
 977
 978         case DWC_CHAN_ST_WAIT_C_PKT:
 979                 goto send_cpkt;
 980
 981         default:
 982                 break;
 983         }
 984         goto busy;
 985
 986 send_pkt:
 987         /* free existing channel, if any */
 988         dwc_otg_host_channel_free(sc, td);
 989
 990         if (sizeof(req) != td->remainder) {
 991                 td->error_any = 1;
 992                 goto complete;
 993         }
 994
 995         if (td->hcsplt != 0) {
 996                 delta = td->tt_start_slot - sc->sc_last_frame_num - 1;
 997                 if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
 998                         td->state = DWC_CHAN_ST_START;
 999                         goto busy;
1000                 }
1001                 delta = sc->sc_last_frame_num - td->tt_start_slot;
1002                 if (delta > 5) {
1003                         /* missed it */
1004                         td->tt_scheduled = 0;
1005                         td->state = DWC_CHAN_ST_START;
1006                         goto busy;
1007                 }
1008         }
1009
1010         /* allocate a new channel */
1011         if (dwc_otg_host_channel_alloc(sc, td, 1)) {
1012                 td->state = DWC_CHAN_ST_START;
1013                 goto busy;
1014         }
1015
1016         if (td->hcsplt != 0) {
1017                 td->hcsplt &= ~HCSPLT_COMPSPLT;
1018                 td->state = DWC_CHAN_ST_WAIT_S_ANE;
1019         } else {
1020                 td->state = DWC_CHAN_ST_WAIT_ANE;
1021         }
1022
1023         /* copy out control request */
1024         usbd_copy_out(td->pc, 0, &req, sizeof(req));
1025
1026         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(td->channel[0]),
1027             (sizeof(req) << HCTSIZ_XFERSIZE_SHIFT) |
1028             (1 << HCTSIZ_PKTCNT_SHIFT) |
1029             (HCTSIZ_PID_SETUP << HCTSIZ_PID_SHIFT));
1030
1031         DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(td->channel[0]), td->hcsplt);
1032
1033         hcchar = td->hcchar;
1034         hcchar &= ~(HCCHAR_EPDIR_IN | HCCHAR_EPTYPE_MASK);
1035         hcchar |= UE_CONTROL << HCCHAR_EPTYPE_SHIFT;
1036
1037         /* must enable channel before writing data to FIFO */
1038         DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(td->channel[0]), hcchar);
1039
1040         /* transfer data into FIFO */
1041         bus_space_write_region_4(sc->sc_io_tag, sc->sc_io_hdl,
1042             DOTG_DFIFO(td->channel[0]), (uint32_t *)&req, sizeof(req) / 4);
1043
1044         /* wait until next slot before trying complete split */
1045         td->tt_complete_slot = sc->sc_last_frame_num + 1;
1046
1047         /* store number of bytes transmitted */
1048         td->tx_bytes = sizeof(req);
1049         goto busy;
1050
1051 send_cpkt:
1052         /* free existing channel, if any */
1053         dwc_otg_host_channel_free(sc, td);
1054
1055         delta = td->tt_complete_slot - sc->sc_last_frame_num - 1;
1056         if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
1057                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
1058                 goto busy;
1059         }
1060         delta = sc->sc_last_frame_num - td->tt_start_slot;
1061         if (delta > DWC_OTG_TT_SLOT_MAX) {
1062                 /* we missed the service interval */
1063                 if (td->ep_type != UE_ISOCHRONOUS)
1064                         td->error_any = 1;
1065                 goto complete;
1066         }
1067         /* allocate a new channel */
1068         if (dwc_otg_host_channel_alloc(sc, td, 0)) {
1069                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
1070                 goto busy;
1071         }
1072
1073         /* wait until next slot before trying complete split */
1074         td->tt_complete_slot = sc->sc_last_frame_num + 1;
1075
1076         td->hcsplt |= HCSPLT_COMPSPLT;
1077         td->state = DWC_CHAN_ST_WAIT_C_ANE;
1078
1079         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(td->channel[0]),
1080             (HCTSIZ_PID_SETUP << HCTSIZ_PID_SHIFT));
1081
1082         DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(td->channel[0]), td->hcsplt);
1083
1084         hcchar = td->hcchar;
1085         hcchar &= ~(HCCHAR_EPDIR_IN | HCCHAR_EPTYPE_MASK);
1086         hcchar |= UE_CONTROL << HCCHAR_EPTYPE_SHIFT;
1087
1088         /* must enable channel before writing data to FIFO */
1089         DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(td->channel[0]), hcchar);
1090
1091 busy:
1092         return (1);     /* busy */
1093
1094 complete:
1095         dwc_otg_host_channel_free(sc, td);
1096         return (0);     /* complete */
1097 }
1098
1099 static uint8_t
1100 dwc_otg_setup_rx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1101 {
1102         struct usb_device_request req __aligned(4);
1103         uint32_t temp;
1104         uint16_t count;
1105
1106         /* check endpoint status */
1107
1108         if (sc->sc_last_rx_status == 0)
1109                 goto not_complete;
1110
1111         if (GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) != 0)
1112                 goto not_complete;
1113
1114         if ((sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) !=
1115             GRXSTSRD_STP_DATA) {
1116                 if ((sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) !=
1117                     GRXSTSRD_STP_COMPLETE || td->remainder != 0) {
1118                         /* release FIFO */
1119                         dwc_otg_common_rx_ack(sc);
1120                         goto not_complete;
1121                 }
1122                 /* release FIFO */
1123                 dwc_otg_common_rx_ack(sc);
1124                 return (0);     /* complete */
1125         }
1126
1127         if ((sc->sc_last_rx_status & GRXSTSRD_DPID_MASK) !=
1128             GRXSTSRD_DPID_DATA0) {
1129                 /* release FIFO */
1130                 dwc_otg_common_rx_ack(sc);
1131                 goto not_complete;
1132         }
1133
1134         DPRINTFN(5, "GRXSTSR=0x%08x\n", sc->sc_last_rx_status);
1135
1136         /* clear did stall */
1137         td->did_stall = 0;
1138
1139         /* get the packet byte count */
1140         count = GRXSTSRD_BCNT_GET(sc->sc_last_rx_status);
1141
1142         if (count != sizeof(req)) {
1143                 DPRINTFN(0, "Unsupported SETUP packet "
1144                     "length, %d bytes\n", count);
1145                 /* release FIFO */
1146                 dwc_otg_common_rx_ack(sc);
1147                 goto not_complete;
1148         }
1149
1150         /* read FIFO */
1151         dwc_otg_read_fifo(sc, td->pc, 0, sizeof(req));
1152
1153         /* copy out control request */
1154         usbd_copy_out(td->pc, 0, &req, sizeof(req));
1155
1156         td->offset = sizeof(req);
1157         td->remainder = 0;
1158
1159         /* sneak peek the set address */
1160         if ((req.bmRequestType == UT_WRITE_DEVICE) &&
1161             (req.bRequest == UR_SET_ADDRESS)) {
1162                 /* must write address before ZLP */
1163                 dwc_otg_set_address(sc, req.wValue[0] & 0x7F);
1164         }
1165
1166         /* don't send any data by default */
1167         DWC_OTG_WRITE_4(sc, DOTG_DIEPTSIZ(0), DIEPCTL_EPDIS);
1168         DWC_OTG_WRITE_4(sc, DOTG_DOEPTSIZ(0), DOEPCTL_EPDIS);
1169
1170         /* reset IN endpoint buffer */
1171         dwc_otg_tx_fifo_reset(sc,
1172             GRSTCTL_TXFIFO(0) |
1173             GRSTCTL_TXFFLSH);
1174
1175         /* acknowledge RX status */
1176         dwc_otg_common_rx_ack(sc);
1177         td->did_stall = 1;
1178
1179 not_complete:
1180         /* abort any ongoing transfer, before enabling again */
1181         if (!td->did_stall) {
1182                 td->did_stall = 1;
1183
1184                 DPRINTFN(5, "stalling IN and OUT direction\n");
1185
1186                 temp = sc->sc_out_ctl[0];
1187
1188                 /* set stall after enabling endpoint */
1189                 DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(0),
1190                     temp | DOEPCTL_STALL);
1191
1192                 temp = sc->sc_in_ctl[0];
1193
1194                 /* set stall assuming endpoint is enabled */
1195                 DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(0),
1196                     temp | DIEPCTL_STALL);
1197         }
1198         return (1);                     /* not complete */
1199 }
1200
1201 static uint8_t
1202 dwc_otg_host_rate_check_interrupt(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1203 {
1204         uint8_t delta;
1205
1206         delta = sc->sc_tmr_val - td->tmr_val;
1207         if (delta >= 128)
1208                 return (1);     /* busy */
1209
1210         td->tmr_val = sc->sc_tmr_val + td->tmr_res;
1211
1212         /* set toggle, if any */
1213         if (td->set_toggle) {
1214                 td->set_toggle = 0;
1215                 td->toggle = 1;
1216         }
1217         return (0);
1218 }
1219
1220 static uint8_t
1221 dwc_otg_host_rate_check(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1222 {
1223         uint8_t frame_num = (uint8_t)sc->sc_last_frame_num;
1224
1225         if (td->ep_type == UE_ISOCHRONOUS) {
1226                 /* non TT isochronous traffic */
1227                 if (frame_num & (td->tmr_res - 1))
1228                         goto busy;
1229                 if ((frame_num ^ td->tmr_val) & td->tmr_res)
1230                         goto busy;
1231                 td->tmr_val = td->tmr_res + sc->sc_last_frame_num;
1232                 td->toggle = 0;
1233                 return (0);
1234         } else if (td->ep_type == UE_INTERRUPT) {
1235                 if (!td->tt_scheduled)
1236                         goto busy;
1237                 td->tt_scheduled = 0;
1238                 return (0);
1239         } else if (td->did_nak != 0) {
1240                 /* check if we should pause sending queries for 125us */
1241                 if (td->tmr_res == frame_num) {
1242                         /* wait a bit */
1243                         dwc_otg_enable_sof_irq(sc);
1244                         goto busy;
1245                 }
1246         } else if (td->set_toggle) {
1247                 td->set_toggle = 0;
1248                 td->toggle = 1;
1249         }
1250         /* query for data one more time */
1251         td->tmr_res = frame_num;
1252         td->did_nak = 0;
1253         return (0);
1254 busy:
1255         return (1);
1256 }
1257
1258 static uint8_t
1259 dwc_otg_host_data_rx_sub(struct dwc_otg_softc *sc, struct dwc_otg_td *td,
1260     uint8_t channel)
1261 {
1262         uint32_t count;
1263
1264         /* check endpoint status */
1265         if (sc->sc_last_rx_status == 0)
1266                 goto busy;
1267
1268         if (channel >= DWC_OTG_MAX_CHANNELS)
1269                 goto busy;
1270
1271         if (GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) != channel)
1272                 goto busy;
1273
1274         switch (sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) {
1275         case GRXSTSRH_IN_DATA:
1276
1277                 DPRINTF("DATA ST=%d STATUS=0x%08x\n",
1278                     (int)td->state, (int)sc->sc_last_rx_status);
1279
1280                 if (sc->sc_chan_state[channel].hcint & HCINT_SOFTWARE_ONLY) {
1281                         /*
1282                          * When using SPLIT transactions on interrupt
1283                          * endpoints, sometimes data occurs twice.
1284                          */
1285                         DPRINTF("Data already received\n");
1286                         break;
1287                 }
1288
1289                 /* get the packet byte count */
1290                 count = GRXSTSRD_BCNT_GET(sc->sc_last_rx_status);
1291
1292                 /* check for ISOCHRONOUS endpoint */
1293                 if (td->ep_type == UE_ISOCHRONOUS) {
1294                         if ((sc->sc_last_rx_status & GRXSTSRD_DPID_MASK) !=
1295                             GRXSTSRD_DPID_DATA0) {
1296                                 /* more data to be received */
1297                                 td->tt_xactpos = HCSPLT_XACTPOS_MIDDLE;
1298                         } else {
1299                                 /* all data received */
1300                                 td->tt_xactpos = HCSPLT_XACTPOS_BEGIN;
1301                                 /* verify the packet byte count */
1302                                 if (count != td->remainder) {
1303                                         /* we have a short packet */
1304                                         td->short_pkt = 1;
1305                                         td->got_short = 1;
1306                                 }
1307                         }
1308                 } else {
1309                         /* verify the packet byte count */
1310                         if (count != td->max_packet_size) {
1311                                 if (count < td->max_packet_size) {
1312                                         /* we have a short packet */
1313                                         td->short_pkt = 1;
1314                                         td->got_short = 1;
1315                                 } else {
1316                                         /* invalid USB packet */
1317                                         td->error_any = 1;
1318
1319                                         /* release FIFO */
1320                                         dwc_otg_common_rx_ack(sc);
1321                                         goto complete;
1322                                 }
1323                         }
1324                         td->toggle ^= 1;
1325                         td->tt_scheduled = 0;
1326                 }
1327
1328                 /* verify the packet byte count */
1329                 if (count > td->remainder) {
1330                         /* invalid USB packet */
1331                         td->error_any = 1;
1332
1333                         /* release FIFO */
1334                         dwc_otg_common_rx_ack(sc);
1335                         goto complete;
1336                 }
1337
1338                 /* read data from FIFO */
1339                 dwc_otg_read_fifo(sc, td->pc, td->offset, count);
1340
1341                 td->remainder -= count;
1342                 td->offset += count;
1343                 sc->sc_chan_state[channel].hcint |= HCINT_SOFTWARE_ONLY;
1344                 break;
1345         default:
1346                 break;
1347         }
1348         /* release FIFO */
1349         dwc_otg_common_rx_ack(sc);
1350 busy:
1351         return (0);
1352 complete:
1353         return (1);
1354 }
1355
1356 static uint8_t
1357 dwc_otg_host_data_rx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1358 {
1359         uint32_t hcint = 0;
1360         uint32_t hcchar;
1361         uint8_t delta;
1362         uint8_t channel;
1363         uint8_t x;
1364
1365         for (x = 0; x != td->max_packet_count; x++) {
1366                 channel = td->channel[x];
1367                 if (channel >= DWC_OTG_MAX_CHANNELS)
1368                         continue;
1369                 hcint |= sc->sc_chan_state[channel].hcint;
1370
1371                 DPRINTF("CH=%d ST=%d HCINT=0x%08x HCCHAR=0x%08x HCTSIZ=0x%08x\n",
1372                     channel, td->state, hcint,
1373                     DWC_OTG_READ_4(sc, DOTG_HCCHAR(channel)),
1374                     DWC_OTG_READ_4(sc, DOTG_HCTSIZ(channel)));
1375
1376                 /* check interrupt bits */
1377                 if (hcint & (HCINT_RETRY |
1378                     HCINT_ACK | HCINT_NYET)) {
1379                         /* give success bits priority over failure bits */
1380                 } else if (hcint & HCINT_STALL) {
1381                         DPRINTF("CH=%d STALL\n", channel);
1382                         td->error_stall = 1;
1383                         td->error_any = 1;
1384                         goto complete;
1385                 } else if (hcint & HCINT_ERRORS) {
1386                         DPRINTF("CH=%d ERROR\n", channel);
1387                         td->errcnt++;
1388                         if (td->hcsplt != 0 || td->errcnt >= 3) {
1389                                 if (td->ep_type != UE_ISOCHRONOUS) {
1390                                         td->error_any = 1;
1391                                         goto complete;
1392                                 }
1393                         }
1394                 }
1395
1396                 /* check channels for data, if any */
1397                 if (dwc_otg_host_data_rx_sub(sc, td, channel))
1398                         goto complete;
1399
1400                 /* refresh interrupt status */
1401                 hcint |= sc->sc_chan_state[channel].hcint;
1402
1403                 if (hcint & (HCINT_ERRORS | HCINT_RETRY |
1404                     HCINT_ACK | HCINT_NYET)) {
1405                         if (!(hcint & HCINT_ERRORS))
1406                                 td->errcnt = 0;
1407                 }
1408         }
1409
1410         switch (td->state) {
1411         case DWC_CHAN_ST_START:
1412                 if (td->hcsplt != 0)
1413                         goto receive_spkt;
1414                 else
1415                         goto receive_pkt;
1416
1417         case DWC_CHAN_ST_WAIT_ANE:
1418                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1419                         if (td->ep_type == UE_INTERRUPT) {
1420                                 /*
1421                                  * The USB specification does not
1422                                  * mandate a particular data toggle
1423                                  * value for USB INTERRUPT
1424                                  * transfers. Switch the data toggle
1425                                  * value to receive the packet
1426                                  * correctly:
1427                                  */
1428                                 if (hcint & HCINT_DATATGLERR) {
1429                                         DPRINTF("Retrying packet due to "
1430                                             "data toggle error\n");
1431                                         td->toggle ^= 1;
1432                                         goto receive_pkt;
1433                                 }
1434                         } else if (td->ep_type == UE_ISOCHRONOUS) {
1435                                 if (td->hcsplt != 0) {
1436                                         /*
1437                                          * Sometimes the complete
1438                                          * split packet may be queued
1439                                          * too early and the
1440                                          * transaction translator will
1441                                          * return a NAK. Ignore
1442                                          * this message and retry the
1443                                          * complete split instead.
1444                                          */
1445                                         DPRINTF("Retrying complete split\n");
1446                                         goto receive_pkt;
1447                                 }
1448                                 goto complete;
1449                         }
1450                         td->did_nak = 1;
1451                         td->tt_scheduled = 0;
1452                         if (td->hcsplt != 0)
1453                                 goto receive_spkt;
1454                         else
1455                                 goto receive_pkt;
1456                 } else if (hcint & HCINT_NYET) {
1457                         if (td->hcsplt != 0) {
1458                                 /* try again */
1459                                 goto receive_pkt;
1460                         } else {
1461                                 /* not a valid token for IN endpoints */
1462                                 td->error_any = 1;
1463                                 goto complete;
1464                         }
1465                 } else if (hcint & HCINT_ACK) {
1466                         /* wait for data - ACK arrived first */
1467                         if (!(hcint & HCINT_SOFTWARE_ONLY))
1468                                 goto busy;
1469
1470                         if (td->ep_type == UE_ISOCHRONOUS) {
1471                                 /* check if we are complete */
1472                                 if (td->tt_xactpos == HCSPLT_XACTPOS_BEGIN) {
1473                                         goto complete;
1474                                 } else if (td->hcsplt != 0) {
1475                                         goto receive_pkt;
1476                                 } else {
1477                                         /* get more packets */
1478                                         goto busy;
1479                                 }
1480                         } else {
1481                                 /* check if we are complete */
1482                                 if ((td->remainder == 0) || (td->got_short != 0)) {
1483                                         if (td->short_pkt)
1484                                                 goto complete;
1485
1486                                         /*
1487                                          * Else need to receive a zero length
1488                                          * packet.
1489                                          */
1490                                 }
1491                                 td->tt_scheduled = 0;
1492                                 td->did_nak = 0;
1493                                 if (td->hcsplt != 0)
1494                                         goto receive_spkt;
1495                                 else
1496                                         goto receive_pkt;
1497                         }
1498                 }
1499                 break;
1500
1501         case DWC_CHAN_ST_WAIT_S_ANE:
1502                 /*
1503                  * NOTE: The DWC OTG hardware provides a fake ACK in
1504                  * case of interrupt and isochronous transfers:
1505                  */
1506                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1507                         td->did_nak = 1;
1508                         td->tt_scheduled = 0;
1509                         goto receive_spkt;
1510                 } else if (hcint & HCINT_NYET) {
1511                         td->tt_scheduled = 0;
1512                         goto receive_spkt;
1513                 } else if (hcint & HCINT_ACK) {
1514                         td->did_nak = 0;
1515                         goto receive_pkt;
1516                 }
1517                 break;
1518
1519         case DWC_CHAN_ST_WAIT_C_PKT:
1520                 goto receive_pkt;
1521
1522         default:
1523                 break;
1524         }
1525         goto busy;
1526
1527 receive_pkt:
1528         /* free existing channel, if any */
1529         dwc_otg_host_channel_free(sc, td);
1530
1531         if (td->hcsplt != 0) {
1532                 delta = td->tt_complete_slot - sc->sc_last_frame_num - 1;
1533                 if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
1534                         if (td->ep_type != UE_ISOCHRONOUS) {
1535                                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
1536                                 goto busy;
1537                         }
1538                 }
1539                 delta = sc->sc_last_frame_num - td->tt_start_slot;
1540                 if (delta > DWC_OTG_TT_SLOT_MAX) {
1541                         if (td->ep_type != UE_ISOCHRONOUS) {
1542                                 /* we missed the service interval */
1543                                 td->error_any = 1;
1544                         }
1545                         goto complete;
1546                 }
1547                 /* complete split */
1548                 td->hcsplt |= HCSPLT_COMPSPLT;
1549         } else if (dwc_otg_host_rate_check(sc, td)) {
1550                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
1551                 goto busy;
1552         }
1553
1554         /* allocate a new channel */
1555         if (dwc_otg_host_channel_alloc(sc, td, 0)) {
1556                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
1557                 goto busy;
1558         }
1559
1560         /* set toggle, if any */
1561         if (td->set_toggle) {
1562                 td->set_toggle = 0;
1563                 td->toggle = 1;
1564         }
1565
1566         td->state = DWC_CHAN_ST_WAIT_ANE;
1567
1568         for (x = 0; x != td->max_packet_count; x++) {
1569                 channel = td->channel[x];
1570
1571                 /* receive one packet */
1572                 DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1573                     (td->max_packet_size << HCTSIZ_XFERSIZE_SHIFT) |
1574                     (1 << HCTSIZ_PKTCNT_SHIFT) |
1575                     (td->toggle ? (HCTSIZ_PID_DATA1 << HCTSIZ_PID_SHIFT) :
1576                     (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT)));
1577
1578                 DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(channel), td->hcsplt);
1579
1580                 hcchar = td->hcchar;
1581                 hcchar |= HCCHAR_EPDIR_IN;
1582
1583                 if (td->ep_type == UE_ISOCHRONOUS) {
1584                         if (td->hcsplt != 0) {
1585                                 /* continously buffer */
1586                                 if (sc->sc_last_frame_num & 1)
1587                                         hcchar &= ~HCCHAR_ODDFRM;
1588                                 else
1589                                         hcchar |= HCCHAR_ODDFRM;
1590                         } else {
1591                                 /* multi buffer, if any */
1592                                 if (sc->sc_last_frame_num & 1)
1593                                         hcchar |= HCCHAR_ODDFRM;
1594                                 else
1595                                         hcchar &= ~HCCHAR_ODDFRM;
1596                         }
1597                 } else {
1598                         hcchar &= ~HCCHAR_ODDFRM;
1599                 }
1600
1601                 /* must enable channel before data can be received */
1602                 DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(channel), hcchar);
1603         }
1604         /* wait until next slot before trying complete split */
1605         td->tt_complete_slot = sc->sc_last_frame_num + 1;
1606
1607         goto busy;
1608
1609 receive_spkt:
1610         /* free existing channel(s), if any */
1611         dwc_otg_host_channel_free(sc, td);
1612
1613         delta = td->tt_start_slot - sc->sc_last_frame_num - 1;
1614         if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
1615                 td->state = DWC_CHAN_ST_START;
1616                 goto busy;
1617         }
1618         delta = sc->sc_last_frame_num - td->tt_start_slot;
1619         if (delta > 5) {
1620                 /* missed it */
1621                 td->tt_scheduled = 0;
1622                 td->state = DWC_CHAN_ST_START;
1623                 goto busy;
1624         }
1625
1626         /* allocate a new channel */
1627         if (dwc_otg_host_channel_alloc(sc, td, 0)) {
1628                 td->state = DWC_CHAN_ST_START;
1629                 goto busy;
1630         }
1631
1632         channel = td->channel[0];
1633
1634         td->hcsplt &= ~HCSPLT_COMPSPLT;
1635         td->state = DWC_CHAN_ST_WAIT_S_ANE;
1636
1637         /* receive one packet */
1638         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1639             (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT));
1640
1641         DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(channel), td->hcsplt);
1642
1643         /* send after next SOF event */
1644         if ((sc->sc_last_frame_num & 1) == 0 &&
1645             td->ep_type == UE_ISOCHRONOUS)
1646                 td->hcchar |= HCCHAR_ODDFRM;
1647         else
1648                 td->hcchar &= ~HCCHAR_ODDFRM;
1649
1650         hcchar = td->hcchar;
1651         hcchar |= HCCHAR_EPDIR_IN;
1652
1653         /* wait until next slot before trying complete split */
1654         td->tt_complete_slot = sc->sc_last_frame_num + 1;
1655
1656         /* must enable channel before data can be received */
1657         DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(channel), hcchar);
1658 busy:
1659         return (1);     /* busy */
1660
1661 complete:
1662         dwc_otg_host_channel_free(sc, td);
1663         return (0);     /* complete */
1664 }
1665
1666 static uint8_t
1667 dwc_otg_data_rx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1668 {
1669         uint32_t temp;
1670         uint16_t count;
1671         uint8_t got_short;
1672
1673         got_short = 0;
1674
1675         /* check endpoint status */
1676         if (sc->sc_last_rx_status == 0)
1677                 goto not_complete;
1678
1679         if (GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) != td->ep_no)
1680                 goto not_complete;
1681
1682         /* check for SETUP packet */
1683         if ((sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) ==
1684             GRXSTSRD_STP_DATA ||
1685             (sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) ==
1686             GRXSTSRD_STP_COMPLETE) {
1687                 if (td->remainder == 0) {
1688                         /*
1689                          * We are actually complete and have
1690                          * received the next SETUP
1691                          */
1692                         DPRINTFN(5, "faking complete\n");
1693                         return (0);     /* complete */
1694                 }
1695                 /*
1696                  * USB Host Aborted the transfer.
1697                  */
1698                 td->error_any = 1;
1699                 return (0);             /* complete */
1700         }
1701
1702         if ((sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) !=
1703             GRXSTSRD_OUT_DATA) {
1704                 /* release FIFO */
1705                 dwc_otg_common_rx_ack(sc);
1706                 goto not_complete;
1707         }
1708
1709         /* get the packet byte count */
1710         count = GRXSTSRD_BCNT_GET(sc->sc_last_rx_status);
1711
1712         /* verify the packet byte count */
1713         if (count != td->max_packet_size) {
1714                 if (count < td->max_packet_size) {
1715                         /* we have a short packet */
1716                         td->short_pkt = 1;
1717                         got_short = 1;
1718                 } else {
1719                         /* invalid USB packet */
1720                         td->error_any = 1;
1721
1722                         /* release FIFO */
1723                         dwc_otg_common_rx_ack(sc);
1724                         return (0);     /* we are complete */
1725                 }
1726         }
1727         /* verify the packet byte count */
1728         if (count > td->remainder) {
1729                 /* invalid USB packet */
1730                 td->error_any = 1;
1731
1732                 /* release FIFO */
1733                 dwc_otg_common_rx_ack(sc);
1734                 return (0);             /* we are complete */
1735         }
1736
1737         /* read data from FIFO */
1738         dwc_otg_read_fifo(sc, td->pc, td->offset, count);
1739
1740         td->remainder -= count;
1741         td->offset += count;
1742
1743         /* release FIFO */
1744         dwc_otg_common_rx_ack(sc);
1745
1746         temp = sc->sc_out_ctl[td->ep_no];
1747
1748         /* check for isochronous mode */
1749         if ((temp & DIEPCTL_EPTYPE_MASK) ==
1750             (DIEPCTL_EPTYPE_ISOC << DIEPCTL_EPTYPE_SHIFT)) {
1751                 /* toggle odd or even frame bit */
1752                 if (temp & DIEPCTL_SETD1PID) {
1753                         temp &= ~DIEPCTL_SETD1PID;
1754                         temp |= DIEPCTL_SETD0PID;
1755                 } else {
1756                         temp &= ~DIEPCTL_SETD0PID;
1757                         temp |= DIEPCTL_SETD1PID;
1758                 }
1759                 sc->sc_out_ctl[td->ep_no] = temp;
1760         }
1761
1762         /* check if we are complete */
1763         if ((td->remainder == 0) || got_short) {
1764                 if (td->short_pkt) {
1765                         /* we are complete */
1766                         return (0);
1767                 }
1768                 /* else need to receive a zero length packet */
1769         }
1770
1771 not_complete:
1772
1773         /* enable SETUP and transfer complete interrupt */
1774         if (td->ep_no == 0) {
1775                 DWC_OTG_WRITE_4(sc, DOTG_DOEPTSIZ(0),
1776                     DXEPTSIZ_SET_MULTI(3) |
1777                     DXEPTSIZ_SET_NPKT(1) |
1778                     DXEPTSIZ_SET_NBYTES(td->max_packet_size));
1779         } else {
1780                 /* allow reception of multiple packets */
1781                 DWC_OTG_WRITE_4(sc, DOTG_DOEPTSIZ(td->ep_no),
1782                     DXEPTSIZ_SET_MULTI(1) |
1783                     DXEPTSIZ_SET_NPKT(4) |
1784                     DXEPTSIZ_SET_NBYTES(4 *
1785                         ((td->max_packet_size + 3) & ~3)));
1786         }
1787         temp = sc->sc_out_ctl[td->ep_no];
1788         DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(td->ep_no), temp |
1789             DOEPCTL_EPENA | DOEPCTL_CNAK);
1790
1791         return (1);                     /* not complete */
1792 }
1793
1794 static uint8_t
1795 dwc_otg_host_data_tx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1796 {
1797         uint32_t count;
1798         uint32_t hcint;
1799         uint32_t hcchar;
1800         uint8_t delta;
1801         uint8_t channel;
1802         uint8_t x;
1803
1804         dwc_otg_host_dump_rx(sc, td);
1805
1806         /* check that last channel is complete */
1807         channel = td->channel[td->npkt];
1808
1809         if (channel < DWC_OTG_MAX_CHANNELS) {
1810                 hcint = sc->sc_chan_state[channel].hcint;
1811
1812                 DPRINTF("CH=%d ST=%d HCINT=0x%08x HCCHAR=0x%08x HCTSIZ=0x%08x\n",
1813                     channel, td->state, hcint,
1814                     DWC_OTG_READ_4(sc, DOTG_HCCHAR(channel)),
1815                     DWC_OTG_READ_4(sc, DOTG_HCTSIZ(channel)));
1816
1817                 if (hcint & (HCINT_RETRY |
1818                     HCINT_ACK | HCINT_NYET)) {
1819                         /* give success bits priority over failure bits */
1820                 } else if (hcint & HCINT_STALL) {
1821                         DPRINTF("CH=%d STALL\n", channel);
1822                         td->error_stall = 1;
1823                         td->error_any = 1;
1824                         goto complete;
1825                 } else if (hcint & HCINT_ERRORS) {
1826                         DPRINTF("CH=%d ERROR\n", channel);
1827                         td->errcnt++;
1828                         if (td->hcsplt != 0 || td->errcnt >= 3) {
1829                                 td->error_any = 1;
1830                                 goto complete;
1831                         }
1832                 }
1833
1834                 if (hcint & (HCINT_ERRORS | HCINT_RETRY |
1835                     HCINT_ACK | HCINT_NYET)) {
1836
1837                         if (!(hcint & HCINT_ERRORS))
1838                                 td->errcnt = 0;
1839                 }
1840         } else {
1841                 hcint = 0;
1842         }
1843
1844         switch (td->state) {
1845         case DWC_CHAN_ST_START:
1846                 goto send_pkt;
1847
1848         case DWC_CHAN_ST_WAIT_ANE:
1849                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1850                         td->did_nak = 1;
1851                         td->tt_scheduled = 0;
1852                         goto send_pkt;
1853                 } else if (hcint & (HCINT_ACK | HCINT_NYET)) {
1854                         td->offset += td->tx_bytes;
1855                         td->remainder -= td->tx_bytes;
1856                         td->toggle ^= 1;
1857                         /* check if next response will be a NAK */
1858                         if (hcint & HCINT_NYET)
1859                                 td->did_nak = 1;
1860                         else
1861                                 td->did_nak = 0;
1862                         td->tt_scheduled = 0;
1863
1864                         /* check remainder */
1865                         if (td->remainder == 0) {
1866                                 if (td->short_pkt)
1867                                         goto complete;
1868
1869                                 /*
1870                                  * Else we need to transmit a short
1871                                  * packet:
1872                                  */
1873                         }
1874                         goto send_pkt;
1875                 }
1876                 break;
1877
1878         case DWC_CHAN_ST_WAIT_S_ANE:
1879                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1880                         td->did_nak = 1;
1881                         td->tt_scheduled = 0;
1882                         goto send_pkt;
1883                 } else if (hcint & (HCINT_ACK | HCINT_NYET)) {
1884                         td->did_nak = 0;
1885                         goto send_cpkt;
1886                 }
1887                 break;
1888
1889         case DWC_CHAN_ST_WAIT_C_ANE:
1890                 if (hcint & HCINT_NYET) {
1891                         goto send_cpkt;
1892                 } else if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1893                         td->did_nak = 1;
1894                         td->tt_scheduled = 0;
1895                         goto send_pkt;
1896                 } else if (hcint & HCINT_ACK) {
1897                         td->offset += td->tx_bytes;
1898                         td->remainder -= td->tx_bytes;
1899                         td->toggle ^= 1;
1900                         td->did_nak = 0;
1901                         td->tt_scheduled = 0;
1902
1903                         /* check remainder */
1904                         if (td->remainder == 0) {
1905                                 if (td->short_pkt)
1906                                         goto complete;
1907
1908                                 /* else we need to transmit a short packet */
1909                         }
1910                         goto send_pkt;
1911                 }
1912                 break;
1913
1914         case DWC_CHAN_ST_WAIT_C_PKT:
1915                 goto send_cpkt;
1916
1917         case DWC_CHAN_ST_TX_WAIT_ISOC:
1918                 /* Check if ISOCHRONOUS OUT traffic is complete */
1919                 if ((hcint & HCINT_HCH_DONE_MASK) == 0)
1920                         break;
1921
1922                 td->offset += td->tx_bytes;
1923                 td->remainder -= td->tx_bytes;
1924                 goto complete;
1925         default:
1926                 break;
1927         }
1928         goto busy;
1929
1930 send_pkt:
1931         /* free existing channel(s), if any */
1932         dwc_otg_host_channel_free(sc, td);
1933
1934         if (td->hcsplt != 0) {
1935                 delta = td->tt_start_slot - sc->sc_last_frame_num - 1;
1936                 if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
1937                         td->state = DWC_CHAN_ST_START;
1938                         goto busy;
1939                 }
1940                 delta = sc->sc_last_frame_num - td->tt_start_slot;
1941                 if (delta > 5) {
1942                         /* missed it */
1943                         td->tt_scheduled = 0;
1944                         td->state = DWC_CHAN_ST_START;
1945                         goto busy;
1946                 }
1947         } else if (dwc_otg_host_rate_check(sc, td)) {
1948                 td->state = DWC_CHAN_ST_START;
1949                 goto busy;
1950         }
1951
1952         /* allocate a new channel */
1953         if (dwc_otg_host_channel_alloc(sc, td, 1)) {
1954                 td->state = DWC_CHAN_ST_START;
1955                 goto busy;
1956         }
1957
1958         /* set toggle, if any */
1959         if (td->set_toggle) {
1960                 td->set_toggle = 0;
1961                 td->toggle = 1;
1962         }
1963
1964         if (td->ep_type == UE_ISOCHRONOUS) {
1965                 /* ISOCHRONOUS OUT transfers don't have any ACKs */
1966                 td->state = DWC_CHAN_ST_TX_WAIT_ISOC;
1967                 td->hcsplt &= ~HCSPLT_COMPSPLT;
1968                 if (td->hcsplt != 0) {
1969                         /* get maximum transfer length */
1970                         count = td->remainder;
1971                         if (count > HCSPLT_XACTLEN_BURST) {
1972                                 DPRINTF("TT overflow\n");
1973                                 td->error_any = 1;
1974                                 goto complete;
1975                         }
1976                         /* Update transaction position */
1977                         td->hcsplt &= ~HCSPLT_XACTPOS_MASK;
1978                         td->hcsplt |= (HCSPLT_XACTPOS_ALL << HCSPLT_XACTPOS_SHIFT);
1979                 }
1980         } else if (td->hcsplt != 0) {
1981                 td->hcsplt &= ~HCSPLT_COMPSPLT;
1982                 /* Wait for ACK/NAK/ERR from TT */
1983                 td->state = DWC_CHAN_ST_WAIT_S_ANE;
1984         } else {
1985                 /* Wait for ACK/NAK/STALL from device */
1986                 td->state = DWC_CHAN_ST_WAIT_ANE;
1987         }
1988
1989         td->tx_bytes = 0;
1990
1991         for (x = 0; x != td->max_packet_count; x++) {
1992                 uint32_t rem_bytes;
1993
1994                 channel = td->channel[x];
1995
1996                 /* send one packet at a time */
1997                 count = td->max_packet_size;
1998                 rem_bytes = td->remainder - td->tx_bytes;
1999                 if (rem_bytes < count) {
2000                         /* we have a short packet */
2001                         td->short_pkt = 1;
2002                         count = rem_bytes;
2003                 }
2004                 if (count == rem_bytes) {
2005                         /* last packet */
2006                         switch (x) {
2007                         case 0:
2008                                 DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
2009                                     (count << HCTSIZ_XFERSIZE_SHIFT) |
2010                                     (1 << HCTSIZ_PKTCNT_SHIFT) |
2011                                     (td->toggle ? (HCTSIZ_PID_DATA1 << HCTSIZ_PID_SHIFT) :
2012                                     (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT)));
2013                                 break;
2014                         case 1:
2015                                 DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
2016                                     (count << HCTSIZ_XFERSIZE_SHIFT) |
2017                                     (1 << HCTSIZ_PKTCNT_SHIFT) |
2018                                     (HCTSIZ_PID_DATA1 << HCTSIZ_PID_SHIFT));
2019                                 break;
2020                         default:
2021                                 DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
2022                                     (count << HCTSIZ_XFERSIZE_SHIFT) |
2023                                     (1 << HCTSIZ_PKTCNT_SHIFT) |
2024                                     (HCTSIZ_PID_DATA2 << HCTSIZ_PID_SHIFT));
2025                                 break;
2026                         }
2027                 } else if (td->ep_type == UE_ISOCHRONOUS &&
2028                            td->max_packet_count > 1) {
2029                         /* ISOCHRONOUS multi packet */
2030                         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
2031                             (count << HCTSIZ_XFERSIZE_SHIFT) |
2032                             (1 << HCTSIZ_PKTCNT_SHIFT) |
2033                             (HCTSIZ_PID_MDATA << HCTSIZ_PID_SHIFT));
2034                 } else {
2035                         /* TODO: HCTSIZ_DOPNG */
2036                         /* standard BULK/INTERRUPT/CONTROL packet */
2037                         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
2038                             (count << HCTSIZ_XFERSIZE_SHIFT) |
2039                             (1 << HCTSIZ_PKTCNT_SHIFT) |
2040                             (td->toggle ? (HCTSIZ_PID_DATA1 << HCTSIZ_PID_SHIFT) :
2041                             (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT)));
2042                 }
2043
2044                 DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(channel), td->hcsplt);
2045
2046                 hcchar = td->hcchar;
2047                 hcchar &= ~HCCHAR_EPDIR_IN;
2048
2049                 /* send after next SOF event */
2050                 if ((sc->sc_last_frame_num & 1) == 0 &&
2051                     td->ep_type == UE_ISOCHRONOUS)
2052                         hcchar |= HCCHAR_ODDFRM;
2053                 else
2054                         hcchar &= ~HCCHAR_ODDFRM;
2055
2056                 /* must enable before writing data to FIFO */
2057                 DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(channel), hcchar);
2058
2059                 if (count != 0) {
2060                         /* write data into FIFO */
2061                         dwc_otg_write_fifo(sc, td->pc, td->offset +
2062                             td->tx_bytes, DOTG_DFIFO(channel), count);
2063                 }
2064
2065                 /* store number of bytes transmitted */
2066                 td->tx_bytes += count;
2067
2068                 /* store last packet index */
2069                 td->npkt = x;
2070
2071                 /* check for last packet */
2072                 if (count == rem_bytes)
2073                         break;
2074         }
2075         goto busy;
2076
2077 send_cpkt:
2078         /* free existing channel, if any */
2079         dwc_otg_host_channel_free(sc, td);
2080
2081         delta = td->tt_complete_slot - sc->sc_last_frame_num - 1;
2082         if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
2083                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
2084                 goto busy;
2085         }
2086         delta = sc->sc_last_frame_num - td->tt_start_slot;
2087         if (delta > DWC_OTG_TT_SLOT_MAX) {
2088                 /* we missed the service interval */
2089                 if (td->ep_type != UE_ISOCHRONOUS)
2090                         td->error_any = 1;
2091                 goto complete;
2092         }
2093
2094         /* allocate a new channel */
2095         if (dwc_otg_host_channel_alloc(sc, td, 0)) {
2096                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
2097                 goto busy;
2098         }
2099
2100         channel = td->channel[0];
2101
2102         td->hcsplt |= HCSPLT_COMPSPLT;
2103         td->state = DWC_CHAN_ST_WAIT_C_ANE;
2104
2105         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
2106             (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT));
2107
2108         DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(channel), td->hcsplt);
2109
2110         hcchar = td->hcchar;
2111         hcchar &= ~HCCHAR_EPDIR_IN;
2112
2113         /* receive complete split ASAP */
2114         if ((sc->sc_last_frame_num & 1) != 0 &&
2115             td->ep_type == UE_ISOCHRONOUS)
2116                 hcchar |= HCCHAR_ODDFRM;
2117         else
2118                 hcchar &= ~HCCHAR_ODDFRM;
2119
2120         /* must enable channel before data can be received */
2121         DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(channel), hcchar);
2122
2123         /* wait until next slot before trying complete split */
2124         td->tt_complete_slot = sc->sc_last_frame_num + 1;
2125 busy:
2126         return (1);     /* busy */
2127
2128 complete:
2129         dwc_otg_host_channel_free(sc, td);
2130         return (0);     /* complete */
2131 }
2132
2133 static uint8_t
2134 dwc_otg_data_tx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
2135 {
2136         uint32_t max_buffer;
2137         uint32_t count;
2138         uint32_t fifo_left;
2139         uint32_t mpkt;
2140         uint32_t temp;
2141         uint8_t to;
2142
2143         to = 3;                         /* don't loop forever! */
2144
2145         max_buffer = sc->sc_hw_ep_profile[td->ep_no].max_buffer;
2146
2147 repeat:
2148         /* check for for endpoint 0 data */
2149
2150         temp = sc->sc_last_rx_status;
2151
2152         if ((td->ep_no == 0) && (temp != 0) &&
2153             (GRXSTSRD_CHNUM_GET(temp) == 0)) {
2154
2155                 if ((temp & GRXSTSRD_PKTSTS_MASK) !=
2156                     GRXSTSRD_STP_DATA &&
2157                     (temp & GRXSTSRD_PKTSTS_MASK) !=
2158                     GRXSTSRD_STP_COMPLETE) {
2159
2160                         /* dump data - wrong direction */
2161                         dwc_otg_common_rx_ack(sc);
2162                 } else {
2163                         /*
2164                          * The current transfer was cancelled
2165                          * by the USB Host:
2166                          */
2167                         td->error_any = 1;
2168                         return (0);             /* complete */
2169                 }
2170         }
2171
2172         /* fill in more TX data, if possible */
2173         if (td->tx_bytes != 0) {
2174
2175                 uint16_t cpkt;
2176
2177                 /* check if packets have been transferred */
2178                 temp = DWC_OTG_READ_4(sc, DOTG_DIEPTSIZ(td->ep_no));
2179
2180                 /* get current packet number */
2181                 cpkt = DXEPTSIZ_GET_NPKT(temp);
2182
2183                 if (cpkt >= td->npkt) {
2184                         fifo_left = 0;
2185                 } else {
2186                         if (max_buffer != 0) {
2187                                 fifo_left = (td->npkt - cpkt) *
2188                                     td->max_packet_size;
2189
2190                                 if (fifo_left > max_buffer)
2191                                         fifo_left = max_buffer;
2192                         } else {
2193                                 fifo_left = td->max_packet_size;
2194                         }
2195                 }
2196
2197                 count = td->tx_bytes;
2198                 if (count > fifo_left)
2199                         count = fifo_left;
2200
2201                 if (count != 0) {
2202                         /* write data into FIFO */
2203                         dwc_otg_write_fifo(sc, td->pc, td->offset,
2204                             DOTG_DFIFO(td->ep_no), count);
2205
2206                         td->tx_bytes -= count;
2207                         td->remainder -= count;
2208                         td->offset += count;
2209                         td->npkt = cpkt;
2210                 }
2211                 if (td->tx_bytes != 0)
2212                         goto not_complete;
2213
2214                 /* check remainder */
2215                 if (td->remainder == 0) {
2216                         if (td->short_pkt)
2217                                 return (0);     /* complete */
2218
2219                         /* else we need to transmit a short packet */
2220                 }
2221         }
2222
2223         if (!to--)
2224                 goto not_complete;
2225
2226         /* check if not all packets have been transferred */
2227         temp = DWC_OTG_READ_4(sc, DOTG_DIEPTSIZ(td->ep_no));
2228
2229         if (DXEPTSIZ_GET_NPKT(temp) != 0) {
2230
2231                 DPRINTFN(5, "busy ep=%d npkt=%d DIEPTSIZ=0x%08x "
2232                     "DIEPCTL=0x%08x\n", td->ep_no,
2233                     DXEPTSIZ_GET_NPKT(temp),
2234                     temp, DWC_OTG_READ_4(sc, DOTG_DIEPCTL(td->ep_no)));
2235
2236                 goto not_complete;
2237         }
2238
2239         DPRINTFN(5, "rem=%u ep=%d\n", td->remainder, td->ep_no);
2240
2241         /* try to optimise by sending more data */
2242         if ((max_buffer != 0) && ((td->max_packet_size & 3) == 0)) {
2243
2244                 /* send multiple packets at the same time */
2245                 mpkt = max_buffer / td->max_packet_size;
2246
2247                 if (mpkt > 0x3FE)
2248                         mpkt = 0x3FE;
2249
2250                 count = td->remainder;
2251                 if (count > 0x7FFFFF)
2252                         count = 0x7FFFFF - (0x7FFFFF % td->max_packet_size);
2253
2254                 td->npkt = count / td->max_packet_size;
2255
2256                 /*
2257                  * NOTE: We could use 0x3FE instead of "mpkt" in the
2258                  * check below to get more throughput, but then we
2259                  * have a dependency towards non-generic chip features
2260                  * to disable the TX-FIFO-EMPTY interrupts on a per
2261                  * endpoint basis. Increase the maximum buffer size of
2262                  * the IN endpoint to increase the performance.
2263                  */
2264                 if (td->npkt > mpkt) {
2265                         td->npkt = mpkt;
2266                         count = td->max_packet_size * mpkt;
2267                 } else if ((count == 0) || (count % td->max_packet_size)) {
2268                         /* we are transmitting a short packet */
2269                         td->npkt++;
2270                         td->short_pkt = 1;
2271                 }
2272         } else {
2273                 /* send one packet at a time */
2274                 mpkt = 1;
2275                 count = td->max_packet_size;
2276                 if (td->remainder < count) {
2277                         /* we have a short packet */
2278                         td->short_pkt = 1;
2279                         count = td->remainder;
2280                 }
2281                 td->npkt = 1;
2282         }
2283         DWC_OTG_WRITE_4(sc, DOTG_DIEPTSIZ(td->ep_no),
2284             DXEPTSIZ_SET_MULTI(1) |
2285             DXEPTSIZ_SET_NPKT(td->npkt) |
2286             DXEPTSIZ_SET_NBYTES(count));
2287
2288         /* make room for buffering */
2289         td->npkt += mpkt;
2290
2291         temp = sc->sc_in_ctl[td->ep_no];
2292
2293         /* check for isochronous mode */
2294         if ((temp & DIEPCTL_EPTYPE_MASK) ==
2295             (DIEPCTL_EPTYPE_ISOC << DIEPCTL_EPTYPE_SHIFT)) {
2296                 /* toggle odd or even frame bit */
2297                 if (temp & DIEPCTL_SETD1PID) {
2298                         temp &= ~DIEPCTL_SETD1PID;
2299                         temp |= DIEPCTL_SETD0PID;
2300                 } else {
2301                         temp &= ~DIEPCTL_SETD0PID;
2302                         temp |= DIEPCTL_SETD1PID;
2303                 }
2304                 sc->sc_in_ctl[td->ep_no] = temp;
2305         }
2306
2307         /* must enable before writing data to FIFO */
2308         DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(td->ep_no), temp |
2309             DIEPCTL_EPENA | DIEPCTL_CNAK);
2310
2311         td->tx_bytes = count;
2312
2313         /* check remainder */
2314         if (td->tx_bytes == 0 &&
2315             td->remainder == 0) {
2316                 if (td->short_pkt)
2317                         return (0);     /* complete */
2318
2319                 /* else we need to transmit a short packet */
2320         }
2321         goto repeat;
2322
2323 not_complete:
2324         return (1);                     /* not complete */
2325 }
2326
2327 static uint8_t
2328 dwc_otg_data_tx_sync(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
2329 {
2330         uint32_t temp;
2331
2332         /*
2333          * If all packets are transferred we are complete:
2334          */
2335         temp = DWC_OTG_READ_4(sc, DOTG_DIEPTSIZ(td->ep_no));
2336
2337         /* check that all packets have been transferred */
2338         if (DXEPTSIZ_GET_NPKT(temp) != 0) {
2339                 DPRINTFN(5, "busy ep=%d\n", td->ep_no);
2340                 goto not_complete;
2341         }
2342         return (0);
2343
2344 not_complete:
2345
2346         /* we only want to know if there is a SETUP packet or free IN packet */
2347
2348         temp = sc->sc_last_rx_status;
2349
2350         if ((td->ep_no == 0) && (temp != 0) &&
2351             (GRXSTSRD_CHNUM_GET(temp) == 0)) {
2352
2353                 if ((temp & GRXSTSRD_PKTSTS_MASK) ==
2354                     GRXSTSRD_STP_DATA ||
2355                     (temp & GRXSTSRD_PKTSTS_MASK) ==
2356                     GRXSTSRD_STP_COMPLETE) {
2357                         DPRINTFN(5, "faking complete\n");
2358                         /*
2359                          * Race condition: We are complete!
2360                          */
2361                         return (0);
2362                 } else {
2363                         /* dump data - wrong direction */
2364                         dwc_otg_common_rx_ack(sc);
2365                 }
2366         }
2367         return (1);                     /* not complete */
2368 }
2369
2370 static void
2371 dwc_otg_xfer_do_fifo(struct dwc_otg_softc *sc, struct usb_xfer *xfer)
2372 {
2373         struct dwc_otg_td *td;
2374         uint8_t toggle;
2375         uint8_t tmr_val;
2376         uint8_t tmr_res;
2377
2378         DPRINTFN(9, "\n");
2379
2380         td = xfer->td_transfer_cache;
2381         if (td == NULL)
2382                 return;
2383
2384         while (1) {
2385                 if ((td->func) (sc, td)) {
2386                         /* operation in progress */
2387                         break;
2388                 }
2389                 if (((void *)td) == xfer->td_transfer_last) {
2390                         goto done;
2391                 }
2392                 if (td->error_any) {
2393                         goto done;
2394                 } else if (td->remainder > 0) {
2395                         /*
2396                          * We had a short transfer. If there is no alternate
2397                          * next, stop processing !
2398                          */
2399                         if (!td->alt_next)
2400                                 goto done;
2401                 }
2402
2403                 /*
2404                  * Fetch the next transfer descriptor and transfer
2405                  * some flags to the next transfer descriptor
2406                  */
2407                 tmr_res = td->tmr_res;
2408                 tmr_val = td->tmr_val;
2409                 toggle = td->toggle;
2410                 td = td->obj_next;
2411                 xfer->td_transfer_cache = td;
2412                 td->toggle = toggle;    /* transfer toggle */
2413                 td->tmr_res = tmr_res;
2414                 td->tmr_val = tmr_val;
2415         }
2416         return;
2417
2418 done:
2419         xfer->td_transfer_cache = NULL;
2420         sc->sc_xfer_complete = 1;
2421 }
2422
2423 static uint8_t
2424 dwc_otg_xfer_do_complete_locked(struct dwc_otg_softc *sc, struct usb_xfer *xfer)
2425 {
2426         struct dwc_otg_td *td;
2427
2428         DPRINTFN(9, "\n");
2429
2430         td = xfer->td_transfer_cache;
2431         if (td == NULL) {
2432                 /* compute all actual lengths */
2433                 dwc_otg_standard_done(xfer);
2434                 return (1);
2435         }
2436         return (0);
2437 }
2438
2439 static void
2440 dwc_otg_timer(void *_sc)
2441 {
2442         struct dwc_otg_softc *sc = _sc;
2443
2444         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
2445
2446         DPRINTF("\n");
2447
2448         USB_BUS_SPIN_LOCK(&sc->sc_bus);
2449
2450         /* increment timer value */
2451         sc->sc_tmr_val++;
2452
2453         /* enable SOF interrupt, which will poll jobs */
2454         dwc_otg_enable_sof_irq(sc);
2455
2456         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
2457
2458         if (sc->sc_timer_active) {
2459                 /* restart timer */
2460                 usb_callout_reset(&sc->sc_timer,
2461                     hz / (1000 / DWC_OTG_HOST_TIMER_RATE),
2462                     &dwc_otg_timer, sc);
2463         }
2464 }
2465
2466 static void
2467 dwc_otg_timer_start(struct dwc_otg_softc *sc)
2468 {
2469         if (sc->sc_timer_active != 0)
2470                 return;
2471
2472         sc->sc_timer_active = 1;
2473
2474         /* restart timer */
2475         usb_callout_reset(&sc->sc_timer,
2476             hz / (1000 / DWC_OTG_HOST_TIMER_RATE),
2477             &dwc_otg_timer, sc);
2478 }
2479
2480 static void
2481 dwc_otg_timer_stop(struct dwc_otg_softc *sc)
2482 {
2483         if (sc->sc_timer_active == 0)
2484                 return;
2485
2486         sc->sc_timer_active = 0;
2487
2488         /* stop timer */
2489         usb_callout_stop(&sc->sc_timer);
2490 }
2491
2492 static uint16_t
2493 dwc_otg_compute_isoc_rx_tt_slot(struct dwc_otg_tt_info *pinfo)
2494 {
2495         if (pinfo->slot_index < DWC_OTG_TT_SLOT_MAX)
2496                 pinfo->slot_index++;
2497         return (pinfo->slot_index);
2498 }
2499
2500 static uint8_t
2501 dwc_otg_update_host_transfer_schedule_locked(struct dwc_otg_softc *sc)
2502 {
2503         TAILQ_HEAD(, usb_xfer) head;
2504         struct usb_xfer *xfer;
2505         struct usb_xfer *xfer_next;
2506         struct dwc_otg_td *td;
2507         uint16_t temp;
2508         uint16_t slot;
2509
2510         temp = DWC_OTG_READ_4(sc, DOTG_HFNUM) & DWC_OTG_FRAME_MASK;
2511
2512         if (sc->sc_last_frame_num == temp)
2513                 return (0);
2514
2515         sc->sc_last_frame_num = temp;
2516
2517         TAILQ_INIT(&head);
2518
2519         if ((temp & 7) == 0) {
2520
2521                 /* reset the schedule */
2522                 memset(sc->sc_tt_info, 0, sizeof(sc->sc_tt_info));
2523
2524                 TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2525                         td = xfer->td_transfer_cache;
2526                         if (td == NULL || td->ep_type != UE_ISOCHRONOUS)
2527                                 continue;
2528
2529                         /* check for IN direction */
2530                         if ((td->hcchar & HCCHAR_EPDIR_IN) != 0)
2531                                 continue;
2532
2533                         sc->sc_needsof = 1;
2534
2535                         if (td->hcsplt == 0 || td->tt_scheduled != 0)
2536                                 continue;
2537
2538                         /* compute slot */
2539                         slot = dwc_otg_compute_isoc_rx_tt_slot(
2540                             sc->sc_tt_info + td->tt_index);
2541                         if (slot > 3) {
2542                                 /*
2543                                  * Not enough time to get complete
2544                                  * split executed.
2545                                  */
2546                                 continue;
2547                         }
2548                         /* Delayed start */
2549                         td->tt_start_slot = temp + slot;
2550                         td->tt_scheduled = 1;
2551                         TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2552                         TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2553                 }
2554
2555                 TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2556                         td = xfer->td_transfer_cache;
2557                         if (td == NULL || td->ep_type != UE_ISOCHRONOUS)
2558                                 continue;
2559
2560                         /* check for OUT direction */
2561                         if ((td->hcchar & HCCHAR_EPDIR_IN) == 0)
2562                                 continue;
2563
2564                         sc->sc_needsof = 1;
2565
2566                         if (td->hcsplt == 0 || td->tt_scheduled != 0)
2567                                 continue;
2568
2569                         /* Start ASAP */
2570                         td->tt_start_slot = temp;
2571                         td->tt_scheduled = 1;
2572                         TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2573                         TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2574                 }
2575
2576                 TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2577                         td = xfer->td_transfer_cache;
2578                         if (td == NULL || td->ep_type != UE_INTERRUPT)
2579                                 continue;
2580
2581                         if (td->tt_scheduled != 0) {
2582                                 sc->sc_needsof = 1;
2583                                 continue;
2584                         }
2585
2586                         if (dwc_otg_host_rate_check_interrupt(sc, td))
2587                                 continue;
2588
2589                         if (td->hcsplt == 0) {
2590                                 sc->sc_needsof = 1;
2591                                 td->tt_scheduled = 1;
2592                                 continue;
2593                         }
2594
2595                         /* start ASAP */
2596                         td->tt_start_slot = temp;
2597                         sc->sc_needsof = 1;
2598                         td->tt_scheduled = 1;
2599                         TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2600                         TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2601                 }
2602
2603                 TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2604                         td = xfer->td_transfer_cache;
2605                         if (td == NULL ||
2606                             td->ep_type != UE_CONTROL) {
2607                                 continue;
2608                         }
2609
2610                         sc->sc_needsof = 1;
2611
2612                         if (td->hcsplt == 0 || td->tt_scheduled != 0)
2613                                 continue;
2614
2615                         /* start ASAP */
2616                         td->tt_start_slot = temp;
2617                         td->tt_scheduled = 1;
2618                         TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2619                         TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2620                 }
2621         }
2622         if ((temp & 7) < 6) {
2623                 TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2624                         td = xfer->td_transfer_cache;
2625                         if (td == NULL ||
2626                             td->ep_type != UE_BULK) {
2627                                 continue;
2628                         }
2629
2630                         sc->sc_needsof = 1;
2631
2632                         if (td->hcsplt == 0 || td->tt_scheduled != 0)
2633                                 continue;
2634
2635                         /* start ASAP */
2636                         td->tt_start_slot = temp;
2637                         td->tt_scheduled = 1;
2638                         TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2639                         TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2640                 }
2641         }
2642
2643         /* Put TT transfers in execution order at the end */
2644         TAILQ_CONCAT(&sc->sc_bus.intr_q.head, &head, wait_entry);
2645
2646         /* move all TT transfers in front, keeping the current order */
2647         TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2648                 td = xfer->td_transfer_cache;
2649                 if (td == NULL || td->hcsplt == 0)
2650                         continue;
2651                 TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2652                 TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2653         }
2654         TAILQ_CONCAT(&head, &sc->sc_bus.intr_q.head, wait_entry);
2655         TAILQ_CONCAT(&sc->sc_bus.intr_q.head, &head, wait_entry);
2656
2657         /* put non-TT non-ISOCHRONOUS transfers last */
2658         TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2659                 td = xfer->td_transfer_cache;
2660                 if (td == NULL || td->hcsplt != 0 || td->ep_type == UE_ISOCHRONOUS)
2661                         continue;
2662                 TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2663                 TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2664         }
2665         TAILQ_CONCAT(&sc->sc_bus.intr_q.head, &head, wait_entry);
2666
2667         if ((temp & 7) == 0) {
2668
2669                 DPRINTFN(12, "SOF interrupt #%d, needsof=%d\n",
2670                     (int)temp, (int)sc->sc_needsof);
2671
2672                 /* update SOF IRQ mask */
2673                 if (sc->sc_irq_mask & GINTMSK_SOFMSK) {
2674                         if (sc->sc_needsof == 0) {
2675                                 sc->sc_irq_mask &= ~GINTMSK_SOFMSK;
2676                                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2677                         }
2678                 } else {
2679                         if (sc->sc_needsof != 0) {
2680                                 sc->sc_irq_mask |= GINTMSK_SOFMSK;
2681                                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2682                         }
2683                 }
2684
2685                 /* clear need SOF flag */
2686                 sc->sc_needsof = 0;
2687         }
2688         return (1);
2689 }
2690
2691 static void
2692 dwc_otg_interrupt_poll_locked(struct dwc_otg_softc *sc)
2693 {
2694         struct usb_xfer *xfer;
2695         uint32_t count;
2696         uint32_t temp;
2697         uint32_t haint;
2698         uint8_t got_rx_status;
2699         uint8_t x;
2700
2701         if (sc->sc_flags.status_device_mode == 0) {
2702                 /*
2703                  * Update host transfer schedule, so that new
2704                  * transfers can be issued:
2705                  */
2706                 dwc_otg_update_host_transfer_schedule_locked(sc);
2707         }
2708         count = 0;
2709 repeat:
2710         if (++count == 16) {
2711                 /* give other interrupts a chance */
2712                 DPRINTF("Yield\n");
2713                 return;
2714         }
2715
2716         /* get all host channel interrupts */
2717         haint = DWC_OTG_READ_4(sc, DOTG_HAINT);
2718         while (1) {
2719                 x = ffs(haint) - 1;
2720                 if (x >= sc->sc_host_ch_max)
2721                         break;
2722                 temp = DWC_OTG_READ_4(sc, DOTG_HCINT(x));
2723                 DWC_OTG_WRITE_4(sc, DOTG_HCINT(x), temp);
2724                 temp &= ~HCINT_SOFTWARE_ONLY;
2725                 sc->sc_chan_state[x].hcint |= temp;
2726                 haint &= ~(1U << x);
2727         }
2728
2729         if (sc->sc_last_rx_status == 0) {
2730
2731                 temp = DWC_OTG_READ_4(sc, DOTG_GINTSTS);
2732                 if (temp & GINTSTS_RXFLVL) {
2733                         /* pop current status */
2734                         sc->sc_last_rx_status =
2735                             DWC_OTG_READ_4(sc, DOTG_GRXSTSPD);
2736                 }
2737
2738                 if (sc->sc_last_rx_status != 0) {
2739
2740                         uint8_t ep_no;
2741
2742                         temp = sc->sc_last_rx_status &
2743                             GRXSTSRD_PKTSTS_MASK;
2744
2745                         /* non-data messages we simply skip */
2746                         if (temp != GRXSTSRD_STP_DATA &&
2747                             temp != GRXSTSRD_STP_COMPLETE &&
2748                             temp != GRXSTSRD_OUT_DATA) {
2749                                 /* check for halted channel */
2750                                 if (temp == GRXSTSRH_HALTED) {
2751                                         ep_no = GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status);
2752                                         sc->sc_chan_state[ep_no].wait_halted = 0;
2753                                         DPRINTFN(5, "channel halt complete ch=%u\n", ep_no);
2754                                 }
2755                                 /* store bytes and FIFO offset */
2756                                 sc->sc_current_rx_bytes = 0;
2757                                 sc->sc_current_rx_fifo = 0;
2758
2759                                 /* acknowledge status */
2760                                 dwc_otg_common_rx_ack(sc);
2761                                 goto repeat;
2762                         }
2763
2764                         temp = GRXSTSRD_BCNT_GET(
2765                             sc->sc_last_rx_status);
2766                         ep_no = GRXSTSRD_CHNUM_GET(
2767                             sc->sc_last_rx_status);
2768
2769                         /* store bytes and FIFO offset */
2770                         sc->sc_current_rx_bytes = (temp + 3) & ~3;
2771                         sc->sc_current_rx_fifo = DOTG_DFIFO(ep_no);
2772
2773                         DPRINTF("Reading %d bytes from ep %d\n", temp, ep_no);
2774
2775                         /* check if we should dump the data */
2776                         if (!(sc->sc_active_rx_ep & (1U << ep_no))) {
2777                                 dwc_otg_common_rx_ack(sc);
2778                                 goto repeat;
2779                         }
2780
2781                         got_rx_status = 1;
2782
2783                         DPRINTFN(5, "RX status = 0x%08x: ch=%d pid=%d bytes=%d sts=%d\n",
2784                             sc->sc_last_rx_status, ep_no,
2785                             (sc->sc_last_rx_status >> 15) & 3,
2786                             GRXSTSRD_BCNT_GET(sc->sc_last_rx_status),
2787                             (sc->sc_last_rx_status >> 17) & 15);
2788                 } else {
2789                         got_rx_status = 0;
2790                 }
2791         } else {
2792                 uint8_t ep_no;
2793
2794                 ep_no = GRXSTSRD_CHNUM_GET(
2795                     sc->sc_last_rx_status);
2796
2797                 /* check if we should dump the data */
2798                 if (!(sc->sc_active_rx_ep & (1U << ep_no))) {
2799                         dwc_otg_common_rx_ack(sc);
2800                         goto repeat;
2801                 }
2802
2803                 got_rx_status = 1;
2804         }
2805
2806         /* execute FIFOs */
2807         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry)
2808                 dwc_otg_xfer_do_fifo(sc, xfer);
2809
2810         if (got_rx_status) {
2811                 /* check if data was consumed */
2812                 if (sc->sc_last_rx_status == 0)
2813                         goto repeat;
2814
2815                 /* disable RX FIFO level interrupt */
2816                 sc->sc_irq_mask &= ~GINTMSK_RXFLVLMSK;
2817                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2818         }
2819 }
2820
2821 static void
2822 dwc_otg_interrupt_complete_locked(struct dwc_otg_softc *sc)
2823 {
2824         struct usb_xfer *xfer;
2825 repeat:
2826         /* scan for completion events */
2827         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
2828                 if (dwc_otg_xfer_do_complete_locked(sc, xfer))
2829                         goto repeat;
2830         }
2831 }
2832
2833 static void
2834 dwc_otg_vbus_interrupt(struct dwc_otg_softc *sc, uint8_t is_on)
2835 {
2836         DPRINTFN(5, "vbus = %u\n", is_on);
2837
2838         /*
2839          * If the USB host mode is forced, then assume VBUS is always
2840          * present else rely on the input to this function:
2841          */
2842         if ((is_on != 0) || (sc->sc_mode == DWC_MODE_HOST)) {
2843
2844                 if (!sc->sc_flags.status_vbus) {
2845                         sc->sc_flags.status_vbus = 1;
2846
2847                         /* complete root HUB interrupt endpoint */
2848
2849                         dwc_otg_root_intr(sc);
2850                 }
2851         } else {
2852                 if (sc->sc_flags.status_vbus) {
2853                         sc->sc_flags.status_vbus = 0;
2854                         sc->sc_flags.status_bus_reset = 0;
2855                         sc->sc_flags.status_suspend = 0;
2856                         sc->sc_flags.change_suspend = 0;
2857                         sc->sc_flags.change_connect = 1;
2858
2859                         /* complete root HUB interrupt endpoint */
2860
2861                         dwc_otg_root_intr(sc);
2862                 }
2863         }
2864 }
2865
2866 int
2867 dwc_otg_filter_interrupt(void *arg)
2868 {
2869         struct dwc_otg_softc *sc = arg;
2870         int retval = FILTER_HANDLED;
2871         uint32_t status;
2872
2873         USB_BUS_SPIN_LOCK(&sc->sc_bus);
2874
2875         /* read and clear interrupt status */
2876         status = DWC_OTG_READ_4(sc, DOTG_GINTSTS);
2877
2878         /* clear interrupts we are handling here */
2879         DWC_OTG_WRITE_4(sc, DOTG_GINTSTS, status & ~DWC_OTG_MSK_GINT_THREAD_IRQ);
2880
2881         /* check for USB state change interrupts */
2882         if ((status & DWC_OTG_MSK_GINT_THREAD_IRQ) != 0)
2883                 retval = FILTER_SCHEDULE_THREAD;
2884
2885         /* clear FIFO empty interrupts */
2886         if (status & sc->sc_irq_mask &
2887             (GINTSTS_PTXFEMP | GINTSTS_NPTXFEMP)) {
2888                 sc->sc_irq_mask &= ~(GINTSTS_PTXFEMP | GINTSTS_NPTXFEMP);
2889                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2890         }
2891         /* clear all IN endpoint interrupts */
2892         if (status & GINTSTS_IEPINT) {
2893                 uint32_t temp;
2894                 uint8_t x;
2895
2896                 for (x = 0; x != sc->sc_dev_in_ep_max; x++) {
2897                         temp = DWC_OTG_READ_4(sc, DOTG_DIEPINT(x));
2898                         /*
2899                          * NOTE: Need to clear all interrupt bits,
2900                          * because some appears to be unmaskable and
2901                          * can cause an interrupt loop:
2902                          */
2903                         if (temp != 0)
2904                                 DWC_OTG_WRITE_4(sc, DOTG_DIEPINT(x), temp);
2905                 }
2906         }
2907
2908         /* poll FIFOs, if any */
2909         dwc_otg_interrupt_poll_locked(sc);
2910
2911         if (sc->sc_xfer_complete != 0)
2912                 retval = FILTER_SCHEDULE_THREAD;
2913
2914         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
2915
2916         return (retval);
2917 }
2918
2919 void
2920 dwc_otg_interrupt(void *arg)
2921 {
2922         struct dwc_otg_softc *sc = arg;
2923         uint32_t status;
2924
2925         USB_BUS_LOCK(&sc->sc_bus);
2926         USB_BUS_SPIN_LOCK(&sc->sc_bus);
2927
2928         /* read and clear interrupt status */
2929         status = DWC_OTG_READ_4(sc, DOTG_GINTSTS);
2930
2931         /* clear interrupts we are handling here */
2932         DWC_OTG_WRITE_4(sc, DOTG_GINTSTS, status & DWC_OTG_MSK_GINT_THREAD_IRQ);
2933
2934         DPRINTFN(14, "GINTSTS=0x%08x HAINT=0x%08x HFNUM=0x%08x\n",
2935             status, DWC_OTG_READ_4(sc, DOTG_HAINT),
2936             DWC_OTG_READ_4(sc, DOTG_HFNUM));
2937
2938         if (status & GINTSTS_USBRST) {
2939
2940                 /* set correct state */
2941                 sc->sc_flags.status_device_mode = 1;
2942                 sc->sc_flags.status_bus_reset = 0;
2943                 sc->sc_flags.status_suspend = 0;
2944                 sc->sc_flags.change_suspend = 0;
2945                 sc->sc_flags.change_connect = 1;
2946
2947                 /* Disable SOF interrupt */
2948                 sc->sc_irq_mask &= ~GINTMSK_SOFMSK;
2949                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2950
2951                 /* complete root HUB interrupt endpoint */
2952                 dwc_otg_root_intr(sc);
2953         }
2954
2955         /* check for any bus state change interrupts */
2956         if (status & GINTSTS_ENUMDONE) {
2957
2958                 uint32_t temp;
2959
2960                 DPRINTFN(5, "end of reset\n");
2961
2962                 /* set correct state */
2963                 sc->sc_flags.status_device_mode = 1;
2964                 sc->sc_flags.status_bus_reset = 1;
2965                 sc->sc_flags.status_suspend = 0;
2966                 sc->sc_flags.change_suspend = 0;
2967                 sc->sc_flags.change_connect = 1;
2968                 sc->sc_flags.status_low_speed = 0;
2969                 sc->sc_flags.port_enabled = 1;
2970
2971                 /* reset FIFOs */
2972                 (void) dwc_otg_init_fifo(sc, DWC_MODE_DEVICE);
2973
2974                 /* reset function address */
2975                 dwc_otg_set_address(sc, 0);
2976
2977                 /* figure out enumeration speed */
2978                 temp = DWC_OTG_READ_4(sc, DOTG_DSTS);
2979                 if (DSTS_ENUMSPD_GET(temp) == DSTS_ENUMSPD_HI)
2980                         sc->sc_flags.status_high_speed = 1;
2981                 else
2982                         sc->sc_flags.status_high_speed = 0;
2983
2984                 /*
2985                  * Disable resume and SOF interrupt, and enable
2986                  * suspend and RX frame interrupt:
2987                  */
2988                 sc->sc_irq_mask &= ~(GINTMSK_WKUPINTMSK | GINTMSK_SOFMSK);
2989                 sc->sc_irq_mask |= GINTMSK_USBSUSPMSK;
2990                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2991
2992                 /* complete root HUB interrupt endpoint */
2993                 dwc_otg_root_intr(sc);
2994         }
2995
2996         if (status & GINTSTS_PRTINT) {
2997                 uint32_t hprt;
2998
2999                 hprt = DWC_OTG_READ_4(sc, DOTG_HPRT);
3000
3001                 /* clear change bits */
3002                 DWC_OTG_WRITE_4(sc, DOTG_HPRT, (hprt & (
3003                     HPRT_PRTPWR | HPRT_PRTENCHNG |
3004                     HPRT_PRTCONNDET | HPRT_PRTOVRCURRCHNG)) |
3005                     sc->sc_hprt_val);
3006
3007                 DPRINTFN(12, "GINTSTS=0x%08x, HPRT=0x%08x\n", status, hprt);
3008
3009                 sc->sc_flags.status_device_mode = 0;
3010
3011                 if (hprt & HPRT_PRTCONNSTS)
3012                         sc->sc_flags.status_bus_reset = 1;
3013                 else
3014                         sc->sc_flags.status_bus_reset = 0;
3015
3016                 if ((hprt & HPRT_PRTENCHNG) &&
3017                     (hprt & HPRT_PRTENA) == 0)
3018                         sc->sc_flags.change_enabled = 1;
3019
3020                 if (hprt & HPRT_PRTENA)
3021                         sc->sc_flags.port_enabled = 1;
3022                 else
3023                         sc->sc_flags.port_enabled = 0;
3024
3025                 if (hprt & HPRT_PRTOVRCURRCHNG)
3026                         sc->sc_flags.change_over_current = 1;
3027
3028                 if (hprt & HPRT_PRTOVRCURRACT)
3029                         sc->sc_flags.port_over_current = 1;
3030                 else
3031                         sc->sc_flags.port_over_current = 0;
3032
3033                 if (hprt & HPRT_PRTPWR)
3034                         sc->sc_flags.port_powered = 1;
3035                 else
3036                         sc->sc_flags.port_powered = 0;
3037
3038                 if (((hprt & HPRT_PRTSPD_MASK)
3039                     >> HPRT_PRTSPD_SHIFT) == HPRT_PRTSPD_LOW)
3040                         sc->sc_flags.status_low_speed = 1;
3041                 else
3042                         sc->sc_flags.status_low_speed = 0;
3043
3044                 if (((hprt & HPRT_PRTSPD_MASK)
3045                     >> HPRT_PRTSPD_SHIFT) == HPRT_PRTSPD_HIGH)
3046                         sc->sc_flags.status_high_speed = 1;
3047                 else
3048                         sc->sc_flags.status_high_speed = 0;
3049
3050                 if (hprt & HPRT_PRTCONNDET)
3051                         sc->sc_flags.change_connect = 1;
3052
3053                 if (hprt & HPRT_PRTSUSP)
3054                         dwc_otg_suspend_irq(sc);
3055                 else
3056                         dwc_otg_resume_irq(sc);
3057
3058                 /* complete root HUB interrupt endpoint */
3059                 dwc_otg_root_intr(sc);
3060
3061                 /* update host frame interval */
3062                 dwc_otg_update_host_frame_interval(sc);
3063         }
3064
3065         /*
3066          * If resume and suspend is set at the same time we interpret
3067          * that like RESUME. Resume is set when there is at least 3
3068          * milliseconds of inactivity on the USB BUS.
3069          */
3070         if (status & GINTSTS_WKUPINT) {
3071
3072                 DPRINTFN(5, "resume interrupt\n");
3073
3074                 dwc_otg_resume_irq(sc);
3075
3076         } else if (status & GINTSTS_USBSUSP) {
3077
3078                 DPRINTFN(5, "suspend interrupt\n");
3079
3080                 dwc_otg_suspend_irq(sc);
3081         }
3082         /* check VBUS */
3083         if (status & (GINTSTS_USBSUSP |
3084             GINTSTS_USBRST |
3085             GINTMSK_OTGINTMSK |
3086             GINTSTS_SESSREQINT)) {
3087                 uint32_t temp;
3088
3089                 temp = DWC_OTG_READ_4(sc, DOTG_GOTGCTL);
3090
3091                 DPRINTFN(5, "GOTGCTL=0x%08x\n", temp);
3092
3093                 dwc_otg_vbus_interrupt(sc,
3094                     (temp & (GOTGCTL_ASESVLD | GOTGCTL_BSESVLD)) ? 1 : 0);
3095         }
3096
3097         if (sc->sc_xfer_complete != 0) {
3098                 sc->sc_xfer_complete = 0;
3099
3100                 /* complete FIFOs, if any */
3101                 dwc_otg_interrupt_complete_locked(sc);
3102         }
3103         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3104         USB_BUS_UNLOCK(&sc->sc_bus);
3105 }
3106
3107 static void
3108 dwc_otg_setup_standard_chain_sub(struct dwc_otg_std_temp *temp)
3109 {
3110         struct dwc_otg_td *td;
3111
3112         /* get current Transfer Descriptor */
3113         td = temp->td_next;
3114         temp->td = td;
3115
3116         /* prepare for next TD */
3117         temp->td_next = td->obj_next;
3118
3119         /* fill out the Transfer Descriptor */
3120         td->func = temp->func;
3121         td->pc = temp->pc;
3122         td->offset = temp->offset;
3123         td->remainder = temp->len;
3124         td->tx_bytes = 0;
3125         td->error_any = 0;
3126         td->error_stall = 0;
3127         td->npkt = 0;
3128         td->did_stall = temp->did_stall;
3129         td->short_pkt = temp->short_pkt;
3130         td->alt_next = temp->setup_alt_next;
3131         td->set_toggle = 0;
3132         td->got_short = 0;
3133         td->did_nak = 0;
3134         td->channel[0] = DWC_OTG_MAX_CHANNELS;
3135         td->channel[1] = DWC_OTG_MAX_CHANNELS;
3136         td->channel[2] = DWC_OTG_MAX_CHANNELS;
3137         td->state = 0;
3138         td->errcnt = 0;
3139         td->tt_scheduled = 0;
3140         td->tt_xactpos = HCSPLT_XACTPOS_BEGIN;
3141 }
3142
3143 static void
3144 dwc_otg_setup_standard_chain(struct usb_xfer *xfer)
3145 {
3146         struct dwc_otg_std_temp temp;
3147         struct dwc_otg_td *td;
3148         uint32_t x;
3149         uint8_t need_sync;
3150         uint8_t is_host;
3151
3152         DPRINTFN(9, "addr=%d endpt=%d sumlen=%d speed=%d\n",
3153             xfer->address, UE_GET_ADDR(xfer->endpointno),
3154             xfer->sumlen, usbd_get_speed(xfer->xroot->udev));
3155
3156         temp.max_frame_size = xfer->max_frame_size;
3157
3158         td = xfer->td_start[0];
3159         xfer->td_transfer_first = td;
3160         xfer->td_transfer_cache = td;
3161
3162         /* setup temp */
3163
3164         temp.pc = NULL;
3165         temp.td = NULL;
3166         temp.td_next = xfer->td_start[0];
3167         temp.offset = 0;
3168         temp.setup_alt_next = xfer->flags_int.short_frames_ok ||
3169             xfer->flags_int.isochronous_xfr;
3170         temp.did_stall = !xfer->flags_int.control_stall;
3171
3172         is_host = (xfer->xroot->udev->flags.usb_mode == USB_MODE_HOST);
3173
3174         /* check if we should prepend a setup message */
3175
3176         if (xfer->flags_int.control_xfr) {
3177                 if (xfer->flags_int.control_hdr) {
3178
3179                         if (is_host)
3180                                 temp.func = &dwc_otg_host_setup_tx;
3181                         else
3182                                 temp.func = &dwc_otg_setup_rx;
3183
3184                         temp.len = xfer->frlengths[0];
3185                         temp.pc = xfer->frbuffers + 0;
3186                         temp.short_pkt = temp.len ? 1 : 0;
3187
3188                         /* check for last frame */
3189                         if (xfer->nframes == 1) {
3190                                 /* no STATUS stage yet, SETUP is last */
3191                                 if (xfer->flags_int.control_act)
3192                                         temp.setup_alt_next = 0;
3193                         }
3194
3195                         dwc_otg_setup_standard_chain_sub(&temp);
3196                 }
3197                 x = 1;
3198         } else {
3199                 x = 0;
3200         }
3201
3202         if (x != xfer->nframes) {
3203                 if (xfer->endpointno & UE_DIR_IN) {
3204                         if (is_host) {
3205                                 temp.func = &dwc_otg_host_data_rx;
3206                                 need_sync = 0;
3207                         } else {
3208                                 temp.func = &dwc_otg_data_tx;
3209                                 need_sync = 1;
3210                         }
3211                 } else {
3212                         if (is_host) {
3213                                 temp.func = &dwc_otg_host_data_tx;
3214                                 need_sync = 0;
3215                         } else {
3216                                 temp.func = &dwc_otg_data_rx;
3217                                 need_sync = 0;
3218                         }
3219                 }
3220
3221                 /* setup "pc" pointer */
3222                 temp.pc = xfer->frbuffers + x;
3223         } else {
3224                 need_sync = 0;
3225         }
3226         while (x != xfer->nframes) {
3227
3228                 /* DATA0 / DATA1 message */
3229
3230                 temp.len = xfer->frlengths[x];
3231
3232                 x++;
3233
3234                 if (x == xfer->nframes) {
3235                         if (xfer->flags_int.control_xfr) {
3236                                 if (xfer->flags_int.control_act) {
3237                                         temp.setup_alt_next = 0;
3238                                 }
3239                         } else {
3240                                 temp.setup_alt_next = 0;
3241                         }
3242                 }
3243                 if (temp.len == 0) {
3244
3245                         /* make sure that we send an USB packet */
3246
3247                         temp.short_pkt = 0;
3248
3249                 } else {
3250
3251                         /* regular data transfer */
3252
3253                         temp.short_pkt = (xfer->flags.force_short_xfer ? 0 : 1);
3254                 }
3255
3256                 dwc_otg_setup_standard_chain_sub(&temp);
3257
3258                 if (xfer->flags_int.isochronous_xfr) {
3259                         temp.offset += temp.len;
3260                 } else {
3261                         /* get next Page Cache pointer */
3262                         temp.pc = xfer->frbuffers + x;
3263                 }
3264         }
3265
3266         if (xfer->flags_int.control_xfr) {
3267
3268                 /* always setup a valid "pc" pointer for status and sync */
3269                 temp.pc = xfer->frbuffers + 0;
3270                 temp.len = 0;
3271                 temp.short_pkt = 0;
3272                 temp.setup_alt_next = 0;
3273
3274                 /* check if we need to sync */
3275                 if (need_sync) {
3276                         /* we need a SYNC point after TX */
3277                         temp.func = &dwc_otg_data_tx_sync;
3278                         dwc_otg_setup_standard_chain_sub(&temp);
3279                 }
3280
3281                 /* check if we should append a status stage */
3282                 if (!xfer->flags_int.control_act) {
3283
3284                         /*
3285                          * Send a DATA1 message and invert the current
3286                          * endpoint direction.
3287                          */
3288                         if (xfer->endpointno & UE_DIR_IN) {
3289                                 if (is_host) {
3290                                         temp.func = &dwc_otg_host_data_tx;
3291                                         need_sync = 0;
3292                                 } else {
3293                                         temp.func = &dwc_otg_data_rx;
3294                                         need_sync = 0;
3295                                 }
3296                         } else {
3297                                 if (is_host) {
3298                                         temp.func = &dwc_otg_host_data_rx;
3299                                         need_sync = 0;
3300                                 } else {
3301                                         temp.func = &dwc_otg_data_tx;
3302                                         need_sync = 1;
3303                                 }
3304                         }
3305
3306                         dwc_otg_setup_standard_chain_sub(&temp);
3307
3308                         /* data toggle should be DATA1 */
3309                         td = temp.td;
3310                         td->set_toggle = 1;
3311
3312                         if (need_sync) {
3313                                 /* we need a SYNC point after TX */
3314                                 temp.func = &dwc_otg_data_tx_sync;
3315                                 dwc_otg_setup_standard_chain_sub(&temp);
3316                         }
3317                 }
3318         } else {
3319                 /* check if we need to sync */
3320                 if (need_sync) {
3321
3322                         temp.pc = xfer->frbuffers + 0;
3323                         temp.len = 0;
3324                         temp.short_pkt = 0;
3325                         temp.setup_alt_next = 0;
3326
3327                         /* we need a SYNC point after TX */
3328                         temp.func = &dwc_otg_data_tx_sync;
3329                         dwc_otg_setup_standard_chain_sub(&temp);
3330                 }
3331         }
3332
3333         /* must have at least one frame! */
3334         td = temp.td;
3335         xfer->td_transfer_last = td;
3336
3337         if (is_host) {
3338
3339                 struct dwc_otg_softc *sc;
3340                 uint32_t hcchar;
3341                 uint32_t hcsplt;
3342
3343                 sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
3344
3345                 /* get first again */
3346                 td = xfer->td_transfer_first;
3347                 td->toggle = (xfer->endpoint->toggle_next ? 1 : 0);
3348
3349                 hcchar =
3350                         (xfer->address << HCCHAR_DEVADDR_SHIFT) |
3351                         ((xfer->endpointno & UE_ADDR) << HCCHAR_EPNUM_SHIFT) |
3352                         (xfer->max_packet_size << HCCHAR_MPS_SHIFT) |
3353                         HCCHAR_CHENA;
3354
3355                 /*
3356                  * We are not always able to meet the timing
3357                  * requirements of the USB interrupt endpoint's
3358                  * complete split token, when doing transfers going
3359                  * via a transaction translator. Use the CONTROL
3360                  * transfer type instead of the INTERRUPT transfer
3361                  * type in general, as a means to workaround
3362                  * that. This trick should work for both FULL and LOW
3363                  * speed USB traffic going through a TT. For non-TT
3364                  * traffic it works as well. The reason for using
3365                  * CONTROL type instead of BULK is that some TTs might
3366                  * reject LOW speed BULK traffic.
3367                  */
3368                 if (td->ep_type == UE_INTERRUPT)
3369                         hcchar |= (UE_CONTROL << HCCHAR_EPTYPE_SHIFT);
3370                 else
3371                         hcchar |= (td->ep_type << HCCHAR_EPTYPE_SHIFT);
3372
3373                 if (UE_GET_DIR(xfer->endpointno) == UE_DIR_IN)
3374                         hcchar |= HCCHAR_EPDIR_IN;
3375
3376                 switch (xfer->xroot->udev->speed) {
3377                 case USB_SPEED_LOW:
3378                         hcchar |= HCCHAR_LSPDDEV;
3379                         /* FALLTHROUGH */
3380                 case USB_SPEED_FULL:
3381                         /* check if root HUB port is running High Speed */
3382                         if (dwc_otg_uses_split(xfer->xroot->udev)) {
3383                                 hcsplt = HCSPLT_SPLTENA |
3384                                     (xfer->xroot->udev->hs_port_no <<
3385                                     HCSPLT_PRTADDR_SHIFT) |
3386                                     (xfer->xroot->udev->hs_hub_addr <<
3387                                     HCSPLT_HUBADDR_SHIFT);
3388                         } else {
3389                                 hcsplt = 0;
3390                         }
3391                         if (td->ep_type == UE_INTERRUPT) {
3392                                 uint32_t ival;
3393                                 ival = xfer->interval / DWC_OTG_HOST_TIMER_RATE;
3394                                 if (ival == 0)
3395                                         ival = 1;
3396                                 else if (ival > 127)
3397                                         ival = 127;
3398                                 td->tmr_val = sc->sc_tmr_val + ival;
3399                                 td->tmr_res = ival;
3400                         } else if (td->ep_type == UE_ISOCHRONOUS) {
3401                                 td->tmr_res = 1;
3402                                 td->tmr_val = sc->sc_last_frame_num;
3403                                 if (td->hcchar & HCCHAR_EPDIR_IN)
3404                                         td->tmr_val++;
3405                         } else {
3406                                 td->tmr_val = 0;
3407                                 td->tmr_res = (uint8_t)sc->sc_last_frame_num;
3408                         }
3409                         break;
3410                 case USB_SPEED_HIGH:
3411                         hcsplt = 0;
3412                         if (td->ep_type == UE_INTERRUPT) {
3413                                 uint32_t ival;
3414                                 hcchar |= ((xfer->max_packet_count & 3)
3415                                     << HCCHAR_MC_SHIFT);
3416                                 ival = xfer->interval / DWC_OTG_HOST_TIMER_RATE;
3417                                 if (ival == 0)
3418                                         ival = 1;
3419                                 else if (ival > 127)
3420                                         ival = 127;
3421                                 td->tmr_val = sc->sc_tmr_val + ival;
3422                                 td->tmr_res = ival;
3423                         } else if (td->ep_type == UE_ISOCHRONOUS) {
3424                                 hcchar |= ((xfer->max_packet_count & 3)
3425                                     << HCCHAR_MC_SHIFT);
3426                                 td->tmr_res = 1 << usbd_xfer_get_fps_shift(xfer);
3427                                 td->tmr_val = sc->sc_last_frame_num;
3428                                 if (td->hcchar & HCCHAR_EPDIR_IN)
3429                                         td->tmr_val += td->tmr_res;
3430
3431                         } else {
3432                                 td->tmr_val = 0;
3433                                 td->tmr_res = (uint8_t)sc->sc_last_frame_num;
3434                         }
3435                         break;
3436                 default:
3437                         hcsplt = 0;
3438                         td->tmr_val = 0;
3439                         td->tmr_res = 0;
3440                         break;
3441                 }
3442
3443                 /* store configuration in all TD's */
3444                 while (1) {
3445                         td->hcchar = hcchar;
3446                         td->hcsplt = hcsplt;
3447
3448                         if (((void *)td) == xfer->td_transfer_last)
3449                                 break;
3450
3451                         td = td->obj_next;
3452                 }
3453         }
3454 }
3455
3456 static void
3457 dwc_otg_timeout(void *arg)
3458 {
3459         struct usb_xfer *xfer = arg;
3460
3461         DPRINTF("xfer=%p\n", xfer);
3462
3463         USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
3464
3465         /* transfer is transferred */
3466         dwc_otg_device_done(xfer, USB_ERR_TIMEOUT);
3467 }
3468
3469 static void
3470 dwc_otg_start_standard_chain(struct usb_xfer *xfer)
3471 {
3472         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
3473
3474         DPRINTFN(9, "\n");
3475
3476         /*
3477          * Poll one time in device mode, which will turn on the
3478          * endpoint interrupts. Else wait for SOF interrupt in host
3479          * mode.
3480          */
3481         USB_BUS_SPIN_LOCK(&sc->sc_bus);
3482
3483         if (sc->sc_flags.status_device_mode != 0) {
3484                 dwc_otg_xfer_do_fifo(sc, xfer);
3485                 if (dwc_otg_xfer_do_complete_locked(sc, xfer))
3486                         goto done;
3487         } else {
3488                 struct dwc_otg_td *td = xfer->td_transfer_cache;
3489                 if (td->ep_type == UE_ISOCHRONOUS &&
3490                     (td->hcchar & HCCHAR_EPDIR_IN) == 0) {
3491                         /*
3492                          * Need to start ISOCHRONOUS OUT transfer ASAP
3493                          * because execution is delayed by one 125us
3494                          * microframe:
3495                          */
3496                         dwc_otg_xfer_do_fifo(sc, xfer);
3497                         if (dwc_otg_xfer_do_complete_locked(sc, xfer))
3498                                 goto done;
3499                 }
3500         }
3501
3502         /* put transfer on interrupt queue */
3503         usbd_transfer_enqueue(&xfer->xroot->bus->intr_q, xfer);
3504
3505         /* start timeout, if any */
3506         if (xfer->timeout != 0) {
3507                 usbd_transfer_timeout_ms(xfer,
3508                     &dwc_otg_timeout, xfer->timeout);
3509         }
3510
3511         if (sc->sc_flags.status_device_mode != 0)
3512                 goto done;
3513
3514         /* enable SOF interrupt, if any */
3515         dwc_otg_enable_sof_irq(sc);
3516 done:
3517         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3518 }
3519
3520 static void
3521 dwc_otg_root_intr(struct dwc_otg_softc *sc)
3522 {
3523         DPRINTFN(9, "\n");
3524
3525         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
3526
3527         /* set port bit */
3528         sc->sc_hub_idata[0] = 0x02;     /* we only have one port */
3529
3530         uhub_root_intr(&sc->sc_bus, sc->sc_hub_idata,
3531             sizeof(sc->sc_hub_idata));
3532 }
3533
3534 static usb_error_t
3535 dwc_otg_standard_done_sub(struct usb_xfer *xfer)
3536 {
3537         struct dwc_otg_td *td;
3538         uint32_t len;
3539         usb_error_t error;
3540
3541         DPRINTFN(9, "\n");
3542
3543         td = xfer->td_transfer_cache;
3544
3545         do {
3546                 len = td->remainder;
3547
3548                 /* store last data toggle */
3549                 xfer->endpoint->toggle_next = td->toggle;
3550
3551                 if (xfer->aframes != xfer->nframes) {
3552                         /*
3553                          * Verify the length and subtract
3554                          * the remainder from "frlengths[]":
3555                          */
3556                         if (len > xfer->frlengths[xfer->aframes]) {
3557                                 td->error_any = 1;
3558                         } else {
3559                                 xfer->frlengths[xfer->aframes] -= len;
3560                         }
3561                 }
3562                 /* Check for transfer error */
3563                 if (td->error_any) {
3564                         /* the transfer is finished */
3565                         error = (td->error_stall ?
3566                             USB_ERR_STALLED : USB_ERR_IOERROR);
3567                         td = NULL;
3568                         break;
3569                 }
3570                 /* Check for short transfer */
3571                 if (len > 0) {
3572                         if (xfer->flags_int.short_frames_ok ||
3573                             xfer->flags_int.isochronous_xfr) {
3574                                 /* follow alt next */
3575                                 if (td->alt_next) {
3576                                         td = td->obj_next;
3577                                 } else {
3578                                         td = NULL;
3579                                 }
3580                         } else {
3581                                 /* the transfer is finished */
3582                                 td = NULL;
3583                         }
3584                         error = 0;
3585                         break;
3586                 }
3587                 td = td->obj_next;
3588
3589                 /* this USB frame is complete */
3590                 error = 0;
3591                 break;
3592
3593         } while (0);
3594
3595         /* update transfer cache */
3596
3597         xfer->td_transfer_cache = td;
3598
3599         return (error);
3600 }
3601
3602 static void
3603 dwc_otg_standard_done(struct usb_xfer *xfer)
3604 {
3605         usb_error_t err = 0;
3606
3607         DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
3608             xfer, xfer->endpoint);
3609
3610         /* reset scanner */
3611
3612         xfer->td_transfer_cache = xfer->td_transfer_first;
3613
3614         if (xfer->flags_int.control_xfr) {
3615
3616                 if (xfer->flags_int.control_hdr) {
3617
3618                         err = dwc_otg_standard_done_sub(xfer);
3619                 }
3620                 xfer->aframes = 1;
3621
3622                 if (xfer->td_transfer_cache == NULL) {
3623                         goto done;
3624                 }
3625         }
3626         while (xfer->aframes != xfer->nframes) {
3627
3628                 err = dwc_otg_standard_done_sub(xfer);
3629                 xfer->aframes++;
3630
3631                 if (xfer->td_transfer_cache == NULL) {
3632                         goto done;
3633                 }
3634         }
3635
3636         if (xfer->flags_int.control_xfr &&
3637             !xfer->flags_int.control_act) {
3638
3639                 err = dwc_otg_standard_done_sub(xfer);
3640         }
3641 done:
3642         dwc_otg_device_done(xfer, err);
3643 }
3644
3645 /*------------------------------------------------------------------------*
3646  *      dwc_otg_device_done
3647  *
3648  * NOTE: this function can be called more than one time on the
3649  * same USB transfer!
3650  *------------------------------------------------------------------------*/
3651 static void
3652 dwc_otg_device_done(struct usb_xfer *xfer, usb_error_t error)
3653 {
3654         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
3655
3656         DPRINTFN(9, "xfer=%p, endpoint=%p, error=%d\n",
3657             xfer, xfer->endpoint, error);
3658
3659         USB_BUS_SPIN_LOCK(&sc->sc_bus);
3660
3661         if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) {
3662                 /* Interrupts are cleared by the interrupt handler */
3663         } else {
3664                 struct dwc_otg_td *td;
3665
3666                 td = xfer->td_transfer_cache;
3667                 if (td != NULL)
3668                         dwc_otg_host_channel_free(sc, td);
3669         }
3670         /* dequeue transfer and start next transfer */
3671         usbd_transfer_done(xfer, error);
3672
3673         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3674 }
3675
3676 static void
3677 dwc_otg_xfer_stall(struct usb_xfer *xfer)
3678 {
3679         dwc_otg_device_done(xfer, USB_ERR_STALLED);
3680 }
3681
3682 static void
3683 dwc_otg_set_stall(struct usb_device *udev,
3684     struct usb_endpoint *ep, uint8_t *did_stall)
3685 {
3686         struct dwc_otg_softc *sc;
3687         uint32_t temp;
3688         uint32_t reg;
3689         uint8_t ep_no;
3690
3691         USB_BUS_LOCK_ASSERT(udev->bus, MA_OWNED);
3692
3693         /* check mode */
3694         if (udev->flags.usb_mode != USB_MODE_DEVICE) {
3695                 /* not supported */
3696                 return;
3697         }
3698
3699         sc = DWC_OTG_BUS2SC(udev->bus);
3700
3701         USB_BUS_SPIN_LOCK(&sc->sc_bus);
3702
3703         /* get endpoint address */
3704         ep_no = ep->edesc->bEndpointAddress;
3705
3706         DPRINTFN(5, "endpoint=0x%x\n", ep_no);
3707
3708         if (ep_no & UE_DIR_IN) {
3709                 reg = DOTG_DIEPCTL(ep_no & UE_ADDR);
3710                 temp = sc->sc_in_ctl[ep_no & UE_ADDR];
3711         } else {
3712                 reg = DOTG_DOEPCTL(ep_no & UE_ADDR);
3713                 temp = sc->sc_out_ctl[ep_no & UE_ADDR];
3714         }
3715
3716         /* disable and stall endpoint */
3717         DWC_OTG_WRITE_4(sc, reg, temp | DOEPCTL_EPDIS);
3718         DWC_OTG_WRITE_4(sc, reg, temp | DOEPCTL_STALL);
3719
3720         /* clear active OUT ep */
3721         if (!(ep_no & UE_DIR_IN)) {
3722
3723                 sc->sc_active_rx_ep &= ~(1U << (ep_no & UE_ADDR));
3724
3725                 if (sc->sc_last_rx_status != 0 &&
3726                     (ep_no & UE_ADDR) == GRXSTSRD_CHNUM_GET(
3727                     sc->sc_last_rx_status)) {
3728                         /* dump data */
3729                         dwc_otg_common_rx_ack(sc);
3730                         /* poll interrupt */
3731                         dwc_otg_interrupt_poll_locked(sc);
3732                         dwc_otg_interrupt_complete_locked(sc);
3733                 }
3734         }
3735         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3736 }
3737
3738 static void
3739 dwc_otg_clear_stall_sub_locked(struct dwc_otg_softc *sc, uint32_t mps,
3740     uint8_t ep_no, uint8_t ep_type, uint8_t ep_dir)
3741 {
3742         uint32_t reg;
3743         uint32_t temp;
3744
3745         if (ep_type == UE_CONTROL) {
3746                 /* clearing stall is not needed */
3747                 return;
3748         }
3749
3750         if (ep_dir) {
3751                 reg = DOTG_DIEPCTL(ep_no);
3752         } else {
3753                 reg = DOTG_DOEPCTL(ep_no);
3754                 sc->sc_active_rx_ep |= (1U << ep_no);
3755         }
3756
3757         /* round up and mask away the multiplier count */
3758         mps = (mps + 3) & 0x7FC;
3759
3760         if (ep_type == UE_BULK) {
3761                 temp = DIEPCTL_EPTYPE_SET(
3762                     DIEPCTL_EPTYPE_BULK) |
3763                     DIEPCTL_USBACTEP;
3764         } else if (ep_type == UE_INTERRUPT) {
3765                 temp = DIEPCTL_EPTYPE_SET(
3766                     DIEPCTL_EPTYPE_INTERRUPT) |
3767                     DIEPCTL_USBACTEP;
3768         } else {
3769                 temp = DIEPCTL_EPTYPE_SET(
3770                     DIEPCTL_EPTYPE_ISOC) |
3771                     DIEPCTL_USBACTEP;
3772         }
3773
3774         temp |= DIEPCTL_MPS_SET(mps);
3775         temp |= DIEPCTL_TXFNUM_SET(ep_no);
3776
3777         if (ep_dir)
3778                 sc->sc_in_ctl[ep_no] = temp;
3779         else
3780                 sc->sc_out_ctl[ep_no] = temp;
3781
3782         DWC_OTG_WRITE_4(sc, reg, temp | DOEPCTL_EPDIS);
3783         DWC_OTG_WRITE_4(sc, reg, temp | DOEPCTL_SETD0PID);
3784         DWC_OTG_WRITE_4(sc, reg, temp | DIEPCTL_SNAK);
3785
3786         /* we only reset the transmit FIFO */
3787         if (ep_dir) {
3788                 dwc_otg_tx_fifo_reset(sc,
3789                     GRSTCTL_TXFIFO(ep_no) |
3790                     GRSTCTL_TXFFLSH);
3791
3792                 DWC_OTG_WRITE_4(sc,
3793                     DOTG_DIEPTSIZ(ep_no), 0);
3794         }
3795
3796         /* poll interrupt */
3797         dwc_otg_interrupt_poll_locked(sc);
3798         dwc_otg_interrupt_complete_locked(sc);
3799 }
3800
3801 static void
3802 dwc_otg_clear_stall(struct usb_device *udev, struct usb_endpoint *ep)
3803 {
3804         struct dwc_otg_softc *sc;
3805         struct usb_endpoint_descriptor *ed;
3806
3807         DPRINTFN(5, "endpoint=%p\n", ep);
3808
3809         USB_BUS_LOCK_ASSERT(udev->bus, MA_OWNED);
3810
3811         /* check mode */
3812         if (udev->flags.usb_mode != USB_MODE_DEVICE) {
3813                 /* not supported */
3814                 return;
3815         }
3816         /* get softc */
3817         sc = DWC_OTG_BUS2SC(udev->bus);
3818
3819         USB_BUS_SPIN_LOCK(&sc->sc_bus);
3820
3821         /* get endpoint descriptor */
3822         ed = ep->edesc;
3823
3824         /* reset endpoint */
3825         dwc_otg_clear_stall_sub_locked(sc,
3826             UGETW(ed->wMaxPacketSize),
3827             (ed->bEndpointAddress & UE_ADDR),
3828             (ed->bmAttributes & UE_XFERTYPE),
3829             (ed->bEndpointAddress & (UE_DIR_IN | UE_DIR_OUT)));
3830
3831         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3832 }
3833
3834 static void
3835 dwc_otg_device_state_change(struct usb_device *udev)
3836 {
3837         struct dwc_otg_softc *sc;
3838         uint8_t x;
3839
3840         /* check mode */
3841         if (udev->flags.usb_mode != USB_MODE_DEVICE) {
3842                 /* not supported */
3843                 return;
3844         }
3845
3846         /* get softc */
3847         sc = DWC_OTG_BUS2SC(udev->bus);
3848
3849         /* deactivate all other endpoint but the control endpoint */
3850         if (udev->state == USB_STATE_CONFIGURED ||
3851             udev->state == USB_STATE_ADDRESSED) {
3852
3853                 USB_BUS_LOCK(&sc->sc_bus);
3854
3855                 for (x = 1; x != sc->sc_dev_ep_max; x++) {
3856
3857                         if (x < sc->sc_dev_in_ep_max) {
3858                                 DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(x),
3859                                     DIEPCTL_EPDIS);
3860                                 DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(x), 0);
3861                         }
3862
3863                         DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(x),
3864                             DOEPCTL_EPDIS);
3865                         DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(x), 0);
3866                 }
3867                 USB_BUS_UNLOCK(&sc->sc_bus);
3868         }
3869 }
3870
3871 int
3872 dwc_otg_init(struct dwc_otg_softc *sc)
3873 {
3874         uint32_t temp;
3875
3876         DPRINTF("start\n");
3877
3878         /* set up the bus structure */
3879         sc->sc_bus.usbrev = USB_REV_2_0;
3880         sc->sc_bus.methods = &dwc_otg_bus_methods;
3881
3882         usb_callout_init_mtx(&sc->sc_timer,
3883             &sc->sc_bus.bus_mtx, 0);
3884
3885         USB_BUS_LOCK(&sc->sc_bus);
3886
3887         /* turn on clocks */
3888         dwc_otg_clocks_on(sc);
3889
3890         temp = DWC_OTG_READ_4(sc, DOTG_GSNPSID);
3891         DPRINTF("Version = 0x%08x\n", temp);
3892
3893         /* disconnect */
3894         DWC_OTG_WRITE_4(sc, DOTG_DCTL,
3895             DCTL_SFTDISCON);
3896
3897         /* wait for host to detect disconnect */
3898         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 32);
3899
3900         DWC_OTG_WRITE_4(sc, DOTG_GRSTCTL,
3901             GRSTCTL_CSFTRST);
3902
3903         /* wait a little bit for block to reset */
3904         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 128);
3905
3906         switch (sc->sc_mode) {
3907         case DWC_MODE_DEVICE:
3908                 temp = GUSBCFG_FORCEDEVMODE;
3909                 break;
3910         case DWC_MODE_HOST:
3911                 temp = GUSBCFG_FORCEHOSTMODE;
3912                 break;
3913         default:
3914                 temp = 0;
3915                 break;
3916         }
3917
3918         if (sc->sc_phy_type == 0)
3919                 sc->sc_phy_type = dwc_otg_phy_type + 1;
3920         if (sc->sc_phy_bits == 0)
3921                 sc->sc_phy_bits = 16;
3922
3923         /* select HSIC, ULPI, UTMI+ or internal PHY mode */
3924         switch (sc->sc_phy_type) {
3925         case DWC_OTG_PHY_HSIC:
3926                 DWC_OTG_WRITE_4(sc, DOTG_GUSBCFG,
3927                     GUSBCFG_PHYIF |
3928                     GUSBCFG_TRD_TIM_SET(5) | temp);
3929                 DWC_OTG_WRITE_4(sc, DOTG_GOTGCTL,
3930                     0x000000EC);
3931
3932                 temp = DWC_OTG_READ_4(sc, DOTG_GLPMCFG);
3933                 DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3934                     temp & ~GLPMCFG_HSIC_CONN);
3935                 DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3936                     temp | GLPMCFG_HSIC_CONN);
3937                 break;
3938         case DWC_OTG_PHY_ULPI:
3939                 DWC_OTG_WRITE_4(sc, DOTG_GUSBCFG,
3940                     GUSBCFG_ULPI_UTMI_SEL |
3941                     GUSBCFG_TRD_TIM_SET(5) | temp);
3942                 DWC_OTG_WRITE_4(sc, DOTG_GOTGCTL, 0);
3943
3944                 temp = DWC_OTG_READ_4(sc, DOTG_GLPMCFG);
3945                 DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3946                     temp & ~GLPMCFG_HSIC_CONN);
3947                 break;
3948         case DWC_OTG_PHY_UTMI:
3949                 DWC_OTG_WRITE_4(sc, DOTG_GUSBCFG,
3950                     (sc->sc_phy_bits == 16 ? GUSBCFG_PHYIF : 0) |
3951                     GUSBCFG_TRD_TIM_SET(5) | temp);
3952                 DWC_OTG_WRITE_4(sc, DOTG_GOTGCTL, 0);
3953
3954                 temp = DWC_OTG_READ_4(sc, DOTG_GLPMCFG);
3955                 DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3956                     temp & ~GLPMCFG_HSIC_CONN);
3957                 break;
3958         case DWC_OTG_PHY_INTERNAL:
3959                 DWC_OTG_WRITE_4(sc, DOTG_GUSBCFG,
3960                     GUSBCFG_PHYSEL |
3961                     GUSBCFG_TRD_TIM_SET(5) | temp);
3962                 DWC_OTG_WRITE_4(sc, DOTG_GOTGCTL, 0);
3963
3964                 temp = DWC_OTG_READ_4(sc, DOTG_GLPMCFG);
3965                 DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3966                     temp & ~GLPMCFG_HSIC_CONN);
3967
3968                 temp = DWC_OTG_READ_4(sc, DOTG_GGPIO);
3969                 temp &= ~(DOTG_GGPIO_NOVBUSSENS | DOTG_GGPIO_I2CPADEN);
3970                 temp |= (DOTG_GGPIO_VBUSASEN | DOTG_GGPIO_VBUSBSEN |
3971                     DOTG_GGPIO_PWRDWN);
3972                 DWC_OTG_WRITE_4(sc, DOTG_GGPIO, temp);
3973                 break;
3974         default:
3975                 break;
3976         }
3977
3978         /* clear global nak */
3979         DWC_OTG_WRITE_4(sc, DOTG_DCTL,
3980             DCTL_CGOUTNAK |
3981             DCTL_CGNPINNAK);
3982
3983         /* disable USB port */
3984         DWC_OTG_WRITE_4(sc, DOTG_PCGCCTL, 0xFFFFFFFF);
3985
3986         /* wait 10ms */
3987         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 100);
3988
3989         /* enable USB port */
3990         DWC_OTG_WRITE_4(sc, DOTG_PCGCCTL, 0);
3991
3992         /* wait 10ms */
3993         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 100);
3994
3995         temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG3);
3996
3997         sc->sc_fifo_size = 4 * GHWCFG3_DFIFODEPTH_GET(temp);
3998
3999         temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG2);
4000
4001         sc->sc_dev_ep_max = GHWCFG2_NUMDEVEPS_GET(temp);
4002
4003         if (sc->sc_dev_ep_max > DWC_OTG_MAX_ENDPOINTS)
4004                 sc->sc_dev_ep_max = DWC_OTG_MAX_ENDPOINTS;
4005
4006         sc->sc_host_ch_max = GHWCFG2_NUMHSTCHNL_GET(temp);
4007
4008         if (sc->sc_host_ch_max > DWC_OTG_MAX_CHANNELS)
4009                 sc->sc_host_ch_max = DWC_OTG_MAX_CHANNELS;
4010
4011         temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG4);
4012
4013         sc->sc_dev_in_ep_max = GHWCFG4_NUM_IN_EP_GET(temp);
4014
4015         DPRINTF("Total FIFO size = %d bytes, Device EPs = %d/%d Host CHs = %d\n",
4016             sc->sc_fifo_size, sc->sc_dev_ep_max, sc->sc_dev_in_ep_max,
4017             sc->sc_host_ch_max);
4018
4019         /* setup FIFO */
4020         if (dwc_otg_init_fifo(sc, sc->sc_mode)) {
4021                 USB_BUS_UNLOCK(&sc->sc_bus);
4022                 return (EINVAL);
4023         }
4024
4025         /* enable interrupts */
4026         sc->sc_irq_mask |= DWC_OTG_MSK_GINT_THREAD_IRQ;
4027         DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
4028
4029         if (sc->sc_mode == DWC_MODE_OTG || sc->sc_mode == DWC_MODE_DEVICE) {
4030
4031                 /* enable all endpoint interrupts */
4032                 temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG2);
4033                 if (temp & GHWCFG2_MPI) {
4034                         uint8_t x;
4035
4036                         DPRINTF("Disable Multi Process Interrupts\n");
4037
4038                         for (x = 0; x != sc->sc_dev_in_ep_max; x++) {
4039                                 DWC_OTG_WRITE_4(sc, DOTG_DIEPEACHINTMSK(x), 0);
4040                                 DWC_OTG_WRITE_4(sc, DOTG_DOEPEACHINTMSK(x), 0);
4041                         }
4042                         DWC_OTG_WRITE_4(sc, DOTG_DEACHINTMSK, 0);
4043                 }
4044                 DWC_OTG_WRITE_4(sc, DOTG_DIEPMSK,
4045                     DIEPMSK_XFERCOMPLMSK);
4046                 DWC_OTG_WRITE_4(sc, DOTG_DOEPMSK, 0);
4047                 DWC_OTG_WRITE_4(sc, DOTG_DAINTMSK, 0xFFFF);
4048         }
4049
4050         if (sc->sc_mode == DWC_MODE_OTG || sc->sc_mode == DWC_MODE_HOST) {
4051                 /* setup clocks */
4052                 temp = DWC_OTG_READ_4(sc, DOTG_HCFG);
4053                 temp &= ~(HCFG_FSLSSUPP | HCFG_FSLSPCLKSEL_MASK);
4054                 temp |= (1 << HCFG_FSLSPCLKSEL_SHIFT);
4055                 DWC_OTG_WRITE_4(sc, DOTG_HCFG, temp);
4056         }
4057
4058         /* only enable global IRQ */
4059         DWC_OTG_WRITE_4(sc, DOTG_GAHBCFG,
4060             GAHBCFG_GLBLINTRMSK);
4061
4062         /* turn off clocks */
4063         dwc_otg_clocks_off(sc);
4064
4065         /* read initial VBUS state */
4066
4067         temp = DWC_OTG_READ_4(sc, DOTG_GOTGCTL);
4068
4069         DPRINTFN(5, "GOTCTL=0x%08x\n", temp);
4070
4071         dwc_otg_vbus_interrupt(sc,
4072             (temp & (GOTGCTL_ASESVLD | GOTGCTL_BSESVLD)) ? 1 : 0);
4073
4074         USB_BUS_UNLOCK(&sc->sc_bus);
4075
4076         /* catch any lost interrupts */
4077
4078         dwc_otg_do_poll(&sc->sc_bus);
4079
4080         return (0);                     /* success */
4081 }
4082
4083 void
4084 dwc_otg_uninit(struct dwc_otg_softc *sc)
4085 {
4086         USB_BUS_LOCK(&sc->sc_bus);
4087
4088         /* stop host timer */
4089         dwc_otg_timer_stop(sc);
4090
4091         /* set disconnect */
4092         DWC_OTG_WRITE_4(sc, DOTG_DCTL,
4093             DCTL_SFTDISCON);
4094
4095         /* turn off global IRQ */
4096         DWC_OTG_WRITE_4(sc, DOTG_GAHBCFG, 0);
4097
4098         sc->sc_flags.port_enabled = 0;
4099         sc->sc_flags.port_powered = 0;
4100         sc->sc_flags.status_vbus = 0;
4101         sc->sc_flags.status_bus_reset = 0;
4102         sc->sc_flags.status_suspend = 0;
4103         sc->sc_flags.change_suspend = 0;
4104         sc->sc_flags.change_connect = 1;
4105
4106         dwc_otg_pull_down(sc);
4107         dwc_otg_clocks_off(sc);
4108
4109         USB_BUS_UNLOCK(&sc->sc_bus);
4110
4111         usb_callout_drain(&sc->sc_timer);
4112 }
4113
4114 static void
4115 dwc_otg_suspend(struct dwc_otg_softc *sc)
4116 {
4117         return;
4118 }
4119
4120 static void
4121 dwc_otg_resume(struct dwc_otg_softc *sc)
4122 {
4123         return;
4124 }
4125
4126 static void
4127 dwc_otg_do_poll(struct usb_bus *bus)
4128 {
4129         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(bus);
4130
4131         USB_BUS_LOCK(&sc->sc_bus);
4132         USB_BUS_SPIN_LOCK(&sc->sc_bus);
4133         dwc_otg_interrupt_poll_locked(sc);
4134         dwc_otg_interrupt_complete_locked(sc);
4135         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
4136         USB_BUS_UNLOCK(&sc->sc_bus);
4137 }
4138
4139 /*------------------------------------------------------------------------*
4140  * DWC OTG bulk support
4141  * DWC OTG control support
4142  * DWC OTG interrupt support
4143  *------------------------------------------------------------------------*/
4144 static void
4145 dwc_otg_device_non_isoc_open(struct usb_xfer *xfer)
4146 {
4147 }
4148
4149 static void
4150 dwc_otg_device_non_isoc_close(struct usb_xfer *xfer)
4151 {
4152         dwc_otg_device_done(xfer, USB_ERR_CANCELLED);
4153 }
4154
4155 static void
4156 dwc_otg_device_non_isoc_enter(struct usb_xfer *xfer)
4157 {
4158 }
4159
4160 static void
4161 dwc_otg_device_non_isoc_start(struct usb_xfer *xfer)
4162 {
4163         /* setup TDs */
4164         dwc_otg_setup_standard_chain(xfer);
4165         dwc_otg_start_standard_chain(xfer);
4166 }
4167
4168 static const struct usb_pipe_methods dwc_otg_device_non_isoc_methods =
4169 {
4170         .open = dwc_otg_device_non_isoc_open,
4171         .close = dwc_otg_device_non_isoc_close,
4172         .enter = dwc_otg_device_non_isoc_enter,
4173         .start = dwc_otg_device_non_isoc_start,
4174 };
4175
4176 /*------------------------------------------------------------------------*
4177  * DWC OTG full speed isochronous support
4178  *------------------------------------------------------------------------*/
4179 static void
4180 dwc_otg_device_isoc_open(struct usb_xfer *xfer)
4181 {
4182 }
4183
4184 static void
4185 dwc_otg_device_isoc_close(struct usb_xfer *xfer)
4186 {
4187         dwc_otg_device_done(xfer, USB_ERR_CANCELLED);
4188 }
4189
4190 static void
4191 dwc_otg_device_isoc_enter(struct usb_xfer *xfer)
4192 {
4193 }
4194
4195 static void
4196 dwc_otg_device_isoc_start(struct usb_xfer *xfer)
4197 {
4198         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
4199         uint32_t temp;
4200         uint32_t msframes;
4201         uint32_t framenum;
4202         uint8_t shift = usbd_xfer_get_fps_shift(xfer);
4203
4204         DPRINTFN(6, "xfer=%p next=%d nframes=%d\n",
4205             xfer, xfer->endpoint->isoc_next, xfer->nframes);
4206
4207         if (xfer->xroot->udev->flags.usb_mode == USB_MODE_HOST) {
4208                 temp = DWC_OTG_READ_4(sc, DOTG_HFNUM);
4209
4210                 /* get the current frame index */
4211                 framenum = (temp & HFNUM_FRNUM_MASK);
4212         } else {
4213                 temp = DWC_OTG_READ_4(sc, DOTG_DSTS);
4214
4215                 /* get the current frame index */
4216                 framenum = DSTS_SOFFN_GET(temp);
4217         }
4218
4219         /*
4220          * Check if port is doing 8000 or 1000 frames per second:
4221          */
4222         if (sc->sc_flags.status_high_speed)
4223                 framenum /= 8;
4224
4225         framenum &= DWC_OTG_FRAME_MASK;
4226
4227         /*
4228          * Compute number of milliseconds worth of data traffic for
4229          * this USB transfer:
4230          */
4231         if (xfer->xroot->udev->speed == USB_SPEED_HIGH)
4232                 msframes = ((xfer->nframes << shift) + 7) / 8;
4233         else
4234                 msframes = xfer->nframes;
4235
4236         /*
4237          * check if the frame index is within the window where the frames
4238          * will be inserted
4239          */
4240         temp = (framenum - xfer->endpoint->isoc_next) & DWC_OTG_FRAME_MASK;
4241
4242         if ((xfer->endpoint->is_synced == 0) || (temp < msframes)) {
4243                 /*
4244                  * If there is data underflow or the pipe queue is
4245                  * empty we schedule the transfer a few frames ahead
4246                  * of the current frame position. Else two isochronous
4247                  * transfers might overlap.
4248                  */
4249                 xfer->endpoint->isoc_next = (framenum + 3) & DWC_OTG_FRAME_MASK;
4250                 xfer->endpoint->is_synced = 1;
4251                 DPRINTFN(3, "start next=%d\n", xfer->endpoint->isoc_next);
4252         }
4253         /*
4254          * compute how many milliseconds the insertion is ahead of the
4255          * current frame position:
4256          */
4257         temp = (xfer->endpoint->isoc_next - framenum) & DWC_OTG_FRAME_MASK;
4258
4259         /*
4260          * pre-compute when the isochronous transfer will be finished:
4261          */
4262         xfer->isoc_time_complete =
4263                 usb_isoc_time_expand(&sc->sc_bus, framenum) + temp + msframes;
4264
4265         /* setup TDs */
4266         dwc_otg_setup_standard_chain(xfer);
4267
4268         /* compute frame number for next insertion */
4269         xfer->endpoint->isoc_next += msframes;
4270
4271         /* start TD chain */
4272         dwc_otg_start_standard_chain(xfer);
4273 }
4274
4275 static const struct usb_pipe_methods dwc_otg_device_isoc_methods =
4276 {
4277         .open = dwc_otg_device_isoc_open,
4278         .close = dwc_otg_device_isoc_close,
4279         .enter = dwc_otg_device_isoc_enter,
4280         .start = dwc_otg_device_isoc_start,
4281 };
4282
4283 /*------------------------------------------------------------------------*
4284  * DWC OTG root control support
4285  *------------------------------------------------------------------------*
4286  * Simulate a hardware HUB by handling all the necessary requests.
4287  *------------------------------------------------------------------------*/
4288
4289 static const struct usb_device_descriptor dwc_otg_devd = {
4290         .bLength = sizeof(struct usb_device_descriptor),
4291         .bDescriptorType = UDESC_DEVICE,
4292         .bcdUSB = {0x00, 0x02},
4293         .bDeviceClass = UDCLASS_HUB,
4294         .bDeviceSubClass = UDSUBCLASS_HUB,
4295         .bDeviceProtocol = UDPROTO_HSHUBSTT,
4296         .bMaxPacketSize = 64,
4297         .bcdDevice = {0x00, 0x01},
4298         .iManufacturer = 1,
4299         .iProduct = 2,
4300         .bNumConfigurations = 1,
4301 };
4302
4303 static const struct dwc_otg_config_desc dwc_otg_confd = {
4304         .confd = {
4305                 .bLength = sizeof(struct usb_config_descriptor),
4306                 .bDescriptorType = UDESC_CONFIG,
4307                 .wTotalLength[0] = sizeof(dwc_otg_confd),
4308                 .bNumInterface = 1,
4309                 .bConfigurationValue = 1,
4310                 .iConfiguration = 0,
4311                 .bmAttributes = UC_SELF_POWERED,
4312                 .bMaxPower = 0,
4313         },
4314         .ifcd = {
4315                 .bLength = sizeof(struct usb_interface_descriptor),
4316                 .bDescriptorType = UDESC_INTERFACE,
4317                 .bNumEndpoints = 1,
4318                 .bInterfaceClass = UICLASS_HUB,
4319                 .bInterfaceSubClass = UISUBCLASS_HUB,
4320                 .bInterfaceProtocol = 0,
4321         },
4322         .endpd = {
4323                 .bLength = sizeof(struct usb_endpoint_descriptor),
4324                 .bDescriptorType = UDESC_ENDPOINT,
4325                 .bEndpointAddress = (UE_DIR_IN | DWC_OTG_INTR_ENDPT),
4326                 .bmAttributes = UE_INTERRUPT,
4327                 .wMaxPacketSize[0] = 8,
4328                 .bInterval = 255,
4329         },
4330 };
4331
4332 #define HSETW(ptr, val) ptr = { (uint8_t)(val), (uint8_t)((val) >> 8) }
4333
4334 static const struct usb_hub_descriptor_min dwc_otg_hubd = {
4335         .bDescLength = sizeof(dwc_otg_hubd),
4336         .bDescriptorType = UDESC_HUB,
4337         .bNbrPorts = 1,
4338         HSETW(.wHubCharacteristics, (UHD_PWR_NO_SWITCH | UHD_OC_INDIVIDUAL)),
4339         .bPwrOn2PwrGood = 50,
4340         .bHubContrCurrent = 0,
4341         .DeviceRemovable = {0},         /* port is removable */
4342 };
4343
4344 #define STRING_VENDOR \
4345   "D\0W\0C\0O\0T\0G"
4346
4347 #define STRING_PRODUCT \
4348   "O\0T\0G\0 \0R\0o\0o\0t\0 \0H\0U\0B"
4349
4350 USB_MAKE_STRING_DESC(STRING_VENDOR, dwc_otg_vendor);
4351 USB_MAKE_STRING_DESC(STRING_PRODUCT, dwc_otg_product);
4352
4353 static usb_error_t
4354 dwc_otg_roothub_exec(struct usb_device *udev,
4355     struct usb_device_request *req, const void **pptr, uint16_t *plength)
4356 {
4357         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(udev->bus);
4358         const void *ptr;
4359         uint16_t len;
4360         uint16_t value;
4361         uint16_t index;
4362         usb_error_t err;
4363
4364         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
4365
4366         /* buffer reset */
4367         ptr = (const void *)&sc->sc_hub_temp;
4368         len = 0;
4369         err = 0;
4370
4371         value = UGETW(req->wValue);
4372         index = UGETW(req->wIndex);
4373
4374         /* demultiplex the control request */
4375
4376         switch (req->bmRequestType) {
4377         case UT_READ_DEVICE:
4378                 switch (req->bRequest) {
4379                 case UR_GET_DESCRIPTOR:
4380                         goto tr_handle_get_descriptor;
4381                 case UR_GET_CONFIG:
4382                         goto tr_handle_get_config;
4383                 case UR_GET_STATUS:
4384                         goto tr_handle_get_status;
4385                 default:
4386                         goto tr_stalled;
4387                 }
4388                 break;
4389
4390         case UT_WRITE_DEVICE:
4391                 switch (req->bRequest) {
4392                 case UR_SET_ADDRESS:
4393                         goto tr_handle_set_address;
4394                 case UR_SET_CONFIG:
4395                         goto tr_handle_set_config;
4396                 case UR_CLEAR_FEATURE:
4397                         goto tr_valid;  /* nop */
4398                 case UR_SET_DESCRIPTOR:
4399                         goto tr_valid;  /* nop */
4400                 case UR_SET_FEATURE:
4401                 default:
4402                         goto tr_stalled;
4403                 }
4404                 break;
4405
4406         case UT_WRITE_ENDPOINT:
4407                 switch (req->bRequest) {
4408                 case UR_CLEAR_FEATURE:
4409                         switch (UGETW(req->wValue)) {
4410                         case UF_ENDPOINT_HALT:
4411                                 goto tr_handle_clear_halt;
4412                         case UF_DEVICE_REMOTE_WAKEUP:
4413                                 goto tr_handle_clear_wakeup;
4414                         default:
4415                                 goto tr_stalled;
4416                         }
4417                         break;
4418                 case UR_SET_FEATURE:
4419                         switch (UGETW(req->wValue)) {
4420                         case UF_ENDPOINT_HALT:
4421                                 goto tr_handle_set_halt;
4422                         case UF_DEVICE_REMOTE_WAKEUP:
4423                                 goto tr_handle_set_wakeup;
4424                         default:
4425                                 goto tr_stalled;
4426                         }
4427                         break;
4428                 case UR_SYNCH_FRAME:
4429                         goto tr_valid;  /* nop */
4430                 default:
4431                         goto tr_stalled;
4432                 }
4433                 break;
4434
4435         case UT_READ_ENDPOINT:
4436                 switch (req->bRequest) {
4437                 case UR_GET_STATUS:
4438                         goto tr_handle_get_ep_status;
4439                 default:
4440                         goto tr_stalled;
4441                 }
4442                 break;
4443
4444         case UT_WRITE_INTERFACE:
4445                 switch (req->bRequest) {
4446                 case UR_SET_INTERFACE:
4447                         goto tr_handle_set_interface;
4448                 case UR_CLEAR_FEATURE:
4449                         goto tr_valid;  /* nop */
4450                 case UR_SET_FEATURE:
4451                 default:
4452                         goto tr_stalled;
4453                 }
4454                 break;
4455
4456         case UT_READ_INTERFACE:
4457                 switch (req->bRequest) {
4458                 case UR_GET_INTERFACE:
4459                         goto tr_handle_get_interface;
4460                 case UR_GET_STATUS:
4461                         goto tr_handle_get_iface_status;
4462                 default:
4463                         goto tr_stalled;
4464                 }
4465                 break;
4466
4467         case UT_WRITE_CLASS_INTERFACE:
4468         case UT_WRITE_VENDOR_INTERFACE:
4469                 /* XXX forward */
4470                 break;
4471
4472         case UT_READ_CLASS_INTERFACE:
4473         case UT_READ_VENDOR_INTERFACE:
4474                 /* XXX forward */
4475                 break;
4476
4477         case UT_WRITE_CLASS_DEVICE:
4478                 switch (req->bRequest) {
4479                 case UR_CLEAR_FEATURE:
4480                         goto tr_valid;
4481                 case UR_SET_DESCRIPTOR:
4482                 case UR_SET_FEATURE:
4483                         break;
4484                 default:
4485                         goto tr_stalled;
4486                 }
4487                 break;
4488
4489         case UT_WRITE_CLASS_OTHER:
4490                 switch (req->bRequest) {
4491                 case UR_CLEAR_FEATURE:
4492                         goto tr_handle_clear_port_feature;
4493                 case UR_SET_FEATURE:
4494                         goto tr_handle_set_port_feature;
4495                 case UR_CLEAR_TT_BUFFER:
4496                 case UR_RESET_TT:
4497                 case UR_STOP_TT:
4498                         goto tr_valid;
4499
4500                 default:
4501                         goto tr_stalled;
4502                 }
4503                 break;
4504
4505         case UT_READ_CLASS_OTHER:
4506                 switch (req->bRequest) {
4507                 case UR_GET_TT_STATE:
4508                         goto tr_handle_get_tt_state;
4509                 case UR_GET_STATUS:
4510                         goto tr_handle_get_port_status;
4511                 default:
4512                         goto tr_stalled;
4513                 }
4514                 break;
4515
4516         case UT_READ_CLASS_DEVICE:
4517                 switch (req->bRequest) {
4518                 case UR_GET_DESCRIPTOR:
4519                         goto tr_handle_get_class_descriptor;
4520                 case UR_GET_STATUS:
4521                         goto tr_handle_get_class_status;
4522
4523                 default:
4524                         goto tr_stalled;
4525                 }
4526                 break;
4527         default:
4528                 goto tr_stalled;
4529         }
4530         goto tr_valid;
4531
4532 tr_handle_get_descriptor:
4533         switch (value >> 8) {
4534         case UDESC_DEVICE:
4535                 if (value & 0xff) {
4536                         goto tr_stalled;
4537                 }
4538                 len = sizeof(dwc_otg_devd);
4539                 ptr = (const void *)&dwc_otg_devd;
4540                 goto tr_valid;
4541         case UDESC_CONFIG:
4542                 if (value & 0xff) {
4543                         goto tr_stalled;
4544                 }
4545                 len = sizeof(dwc_otg_confd);
4546                 ptr = (const void *)&dwc_otg_confd;
4547                 goto tr_valid;
4548         case UDESC_STRING:
4549                 switch (value & 0xff) {
4550                 case 0:         /* Language table */
4551                         len = sizeof(usb_string_lang_en);
4552                         ptr = (const void *)&usb_string_lang_en;
4553                         goto tr_valid;
4554
4555                 case 1:         /* Vendor */
4556                         len = sizeof(dwc_otg_vendor);
4557                         ptr = (const void *)&dwc_otg_vendor;
4558                         goto tr_valid;
4559
4560                 case 2:         /* Product */
4561                         len = sizeof(dwc_otg_product);
4562                         ptr = (const void *)&dwc_otg_product;
4563                         goto tr_valid;
4564                 default:
4565                         break;
4566                 }
4567                 break;
4568         default:
4569                 goto tr_stalled;
4570         }
4571         goto tr_stalled;
4572
4573 tr_handle_get_config:
4574         len = 1;
4575         sc->sc_hub_temp.wValue[0] = sc->sc_conf;
4576         goto tr_valid;
4577
4578 tr_handle_get_status:
4579         len = 2;
4580         USETW(sc->sc_hub_temp.wValue, UDS_SELF_POWERED);
4581         goto tr_valid;
4582
4583 tr_handle_set_address:
4584         if (value & 0xFF00) {
4585                 goto tr_stalled;
4586         }
4587         sc->sc_rt_addr = value;
4588         goto tr_valid;
4589
4590 tr_handle_set_config:
4591         if (value >= 2) {
4592                 goto tr_stalled;
4593         }
4594         sc->sc_conf = value;
4595         goto tr_valid;
4596
4597 tr_handle_get_interface:
4598         len = 1;
4599         sc->sc_hub_temp.wValue[0] = 0;
4600         goto tr_valid;
4601
4602 tr_handle_get_tt_state:
4603 tr_handle_get_class_status:
4604 tr_handle_get_iface_status:
4605 tr_handle_get_ep_status:
4606         len = 2;
4607         USETW(sc->sc_hub_temp.wValue, 0);
4608         goto tr_valid;
4609
4610 tr_handle_set_halt:
4611 tr_handle_set_interface:
4612 tr_handle_set_wakeup:
4613 tr_handle_clear_wakeup:
4614 tr_handle_clear_halt:
4615         goto tr_valid;
4616
4617 tr_handle_clear_port_feature:
4618         if (index != 1)
4619                 goto tr_stalled;
4620
4621         DPRINTFN(9, "UR_CLEAR_PORT_FEATURE on port %d\n", index);
4622
4623         switch (value) {
4624         case UHF_PORT_SUSPEND:
4625                 dwc_otg_wakeup_peer(sc);
4626                 break;
4627
4628         case UHF_PORT_ENABLE:
4629                 if (sc->sc_flags.status_device_mode == 0) {
4630                         DWC_OTG_WRITE_4(sc, DOTG_HPRT,
4631                             sc->sc_hprt_val | HPRT_PRTENA);
4632                 }
4633                 sc->sc_flags.port_enabled = 0;
4634                 break;
4635
4636         case UHF_C_PORT_RESET:
4637                 sc->sc_flags.change_reset = 0;
4638                 break;
4639
4640         case UHF_C_PORT_ENABLE:
4641                 sc->sc_flags.change_enabled = 0;
4642                 break;
4643
4644         case UHF_C_PORT_OVER_CURRENT:
4645                 sc->sc_flags.change_over_current = 0;
4646                 break;
4647
4648         case UHF_PORT_TEST:
4649         case UHF_PORT_INDICATOR:
4650                 /* nops */
4651                 break;
4652
4653         case UHF_PORT_POWER:
4654                 sc->sc_flags.port_powered = 0;
4655                 if (sc->sc_mode == DWC_MODE_HOST || sc->sc_mode == DWC_MODE_OTG) {
4656                         sc->sc_hprt_val = 0;
4657                         DWC_OTG_WRITE_4(sc, DOTG_HPRT, HPRT_PRTENA);
4658                 }
4659                 dwc_otg_pull_down(sc);
4660                 dwc_otg_clocks_off(sc);
4661                 break;
4662
4663         case UHF_C_PORT_CONNECTION:
4664                 /* clear connect change flag */
4665                 sc->sc_flags.change_connect = 0;
4666                 break;
4667
4668         case UHF_C_PORT_SUSPEND:
4669                 sc->sc_flags.change_suspend = 0;
4670                 break;
4671
4672         default:
4673                 err = USB_ERR_IOERROR;
4674                 goto done;
4675         }
4676         goto tr_valid;
4677
4678 tr_handle_set_port_feature:
4679         if (index != 1) {
4680                 goto tr_stalled;
4681         }
4682         DPRINTFN(9, "UR_SET_PORT_FEATURE\n");
4683
4684         switch (value) {
4685         case UHF_PORT_ENABLE:
4686                 break;
4687
4688         case UHF_PORT_SUSPEND:
4689                 if (sc->sc_flags.status_device_mode == 0) {
4690                         /* set suspend BIT */
4691                         sc->sc_hprt_val |= HPRT_PRTSUSP;
4692                         DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
4693
4694                         /* generate HUB suspend event */
4695                         dwc_otg_suspend_irq(sc);
4696                 }
4697                 break;
4698
4699         case UHF_PORT_RESET:
4700                 if (sc->sc_flags.status_device_mode == 0) {
4701
4702                         DPRINTF("PORT RESET\n");
4703
4704                         /* enable PORT reset */
4705                         DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val | HPRT_PRTRST);
4706
4707                         /* Wait 62.5ms for reset to complete */
4708                         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 16);
4709
4710                         DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
4711
4712                         /* Wait 62.5ms for reset to complete */
4713                         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 16);
4714
4715                         /* reset FIFOs */
4716                         (void) dwc_otg_init_fifo(sc, DWC_MODE_HOST);
4717
4718                         sc->sc_flags.change_reset = 1;
4719                 } else {
4720                         err = USB_ERR_IOERROR;
4721                 }
4722                 break;
4723
4724         case UHF_PORT_TEST:
4725         case UHF_PORT_INDICATOR:
4726                 /* nops */
4727                 break;
4728         case UHF_PORT_POWER:
4729                 sc->sc_flags.port_powered = 1;
4730                 if (sc->sc_mode == DWC_MODE_HOST || sc->sc_mode == DWC_MODE_OTG) {
4731                         sc->sc_hprt_val |= HPRT_PRTPWR;
4732                         DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
4733                 }
4734                 if (sc->sc_mode == DWC_MODE_DEVICE || sc->sc_mode == DWC_MODE_OTG) {
4735                         /* pull up D+, if any */
4736                         dwc_otg_pull_up(sc);
4737                 }
4738                 break;
4739         default:
4740                 err = USB_ERR_IOERROR;
4741                 goto done;
4742         }
4743         goto tr_valid;
4744
4745 tr_handle_get_port_status:
4746
4747         DPRINTFN(9, "UR_GET_PORT_STATUS\n");
4748
4749         if (index != 1)
4750                 goto tr_stalled;
4751
4752         if (sc->sc_flags.status_vbus)
4753                 dwc_otg_clocks_on(sc);
4754         else
4755                 dwc_otg_clocks_off(sc);
4756
4757         /* Select Device Side Mode */
4758
4759         if (sc->sc_flags.status_device_mode) {
4760                 value = UPS_PORT_MODE_DEVICE;
4761                 dwc_otg_timer_stop(sc);
4762         } else {
4763                 value = 0;
4764                 dwc_otg_timer_start(sc);
4765         }
4766
4767         if (sc->sc_flags.status_high_speed)
4768                 value |= UPS_HIGH_SPEED;
4769         else if (sc->sc_flags.status_low_speed)
4770                 value |= UPS_LOW_SPEED;
4771
4772         if (sc->sc_flags.port_powered)
4773                 value |= UPS_PORT_POWER;
4774
4775         if (sc->sc_flags.port_enabled)
4776                 value |= UPS_PORT_ENABLED;
4777
4778         if (sc->sc_flags.port_over_current)
4779                 value |= UPS_OVERCURRENT_INDICATOR;
4780
4781         if (sc->sc_flags.status_vbus &&
4782             sc->sc_flags.status_bus_reset)
4783                 value |= UPS_CURRENT_CONNECT_STATUS;
4784
4785         if (sc->sc_flags.status_suspend)
4786                 value |= UPS_SUSPEND;
4787
4788         USETW(sc->sc_hub_temp.ps.wPortStatus, value);
4789
4790         value = 0;
4791
4792         if (sc->sc_flags.change_enabled)
4793                 value |= UPS_C_PORT_ENABLED;
4794         if (sc->sc_flags.change_connect)
4795                 value |= UPS_C_CONNECT_STATUS;
4796         if (sc->sc_flags.change_suspend)
4797                 value |= UPS_C_SUSPEND;
4798         if (sc->sc_flags.change_reset)
4799                 value |= UPS_C_PORT_RESET;
4800         if (sc->sc_flags.change_over_current)
4801                 value |= UPS_C_OVERCURRENT_INDICATOR;
4802
4803         USETW(sc->sc_hub_temp.ps.wPortChange, value);
4804         len = sizeof(sc->sc_hub_temp.ps);
4805         goto tr_valid;
4806
4807 tr_handle_get_class_descriptor:
4808         if (value & 0xFF) {
4809                 goto tr_stalled;
4810         }
4811         ptr = (const void *)&dwc_otg_hubd;
4812         len = sizeof(dwc_otg_hubd);
4813         goto tr_valid;
4814
4815 tr_stalled:
4816         err = USB_ERR_STALLED;
4817 tr_valid:
4818 done:
4819         *plength = len;
4820         *pptr = ptr;
4821         return (err);
4822 }
4823
4824 static void
4825 dwc_otg_xfer_setup(struct usb_setup_params *parm)
4826 {
4827         struct usb_xfer *xfer;
4828         void *last_obj;
4829         uint32_t ntd;
4830         uint32_t n;
4831         uint8_t ep_no;
4832         uint8_t ep_type;
4833
4834         xfer = parm->curr_xfer;
4835
4836         /*
4837          * NOTE: This driver does not use any of the parameters that
4838          * are computed from the following values. Just set some
4839          * reasonable dummies:
4840          */
4841         parm->hc_max_packet_size = 0x500;
4842         parm->hc_max_packet_count = 3;
4843         parm->hc_max_frame_size = 3 * 0x500;
4844
4845         usbd_transfer_setup_sub(parm);
4846
4847         /*
4848          * compute maximum number of TDs
4849          */
4850         ep_type = (xfer->endpoint->edesc->bmAttributes & UE_XFERTYPE);
4851
4852         if (ep_type == UE_CONTROL) {
4853
4854                 ntd = xfer->nframes + 1 /* STATUS */ + 1 /* SYNC 1 */
4855                     + 1 /* SYNC 2 */ + 1 /* SYNC 3 */;
4856         } else {
4857
4858                 ntd = xfer->nframes + 1 /* SYNC */ ;
4859         }
4860
4861         /*
4862          * check if "usbd_transfer_setup_sub" set an error
4863          */
4864         if (parm->err)
4865                 return;
4866
4867         /*
4868          * allocate transfer descriptors
4869          */
4870         last_obj = NULL;
4871
4872         ep_no = xfer->endpointno & UE_ADDR;
4873
4874         /*
4875          * Check for a valid endpoint profile in USB device mode:
4876          */
4877         if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) {
4878                 const struct usb_hw_ep_profile *pf;
4879
4880                 dwc_otg_get_hw_ep_profile(parm->udev, &pf, ep_no);
4881
4882                 if (pf == NULL) {
4883                         /* should not happen */
4884                         parm->err = USB_ERR_INVAL;
4885                         return;
4886                 }
4887         }
4888
4889         /* align data */
4890         parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
4891
4892         for (n = 0; n != ntd; n++) {
4893
4894                 struct dwc_otg_td *td;
4895
4896                 if (parm->buf) {
4897
4898                         td = USB_ADD_BYTES(parm->buf, parm->size[0]);
4899
4900                         /* compute shared bandwidth resource index for TT */
4901                         if (dwc_otg_uses_split(parm->udev)) {
4902                                 if (parm->udev->parent_hs_hub->ddesc.bDeviceProtocol == UDPROTO_HSHUBMTT)
4903                                         td->tt_index = parm->udev->device_index;
4904                                 else
4905                                         td->tt_index = parm->udev->parent_hs_hub->device_index;
4906                         } else {
4907                                 td->tt_index = parm->udev->device_index;
4908                         }
4909
4910                         /* init TD */
4911                         td->max_packet_size = xfer->max_packet_size;
4912                         td->max_packet_count = xfer->max_packet_count;
4913                         /* range check */
4914                         if (td->max_packet_count == 0 || td->max_packet_count > 3)
4915                                 td->max_packet_count = 1;
4916                         td->ep_no = ep_no;
4917                         td->ep_type = ep_type;
4918                         td->obj_next = last_obj;
4919
4920                         last_obj = td;
4921                 }
4922                 parm->size[0] += sizeof(*td);
4923         }
4924
4925         xfer->td_start[0] = last_obj;
4926 }
4927
4928 static void
4929 dwc_otg_xfer_unsetup(struct usb_xfer *xfer)
4930 {
4931         return;
4932 }
4933
4934 static void
4935 dwc_otg_ep_init(struct usb_device *udev, struct usb_endpoint_descriptor *edesc,
4936     struct usb_endpoint *ep)
4937 {
4938         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(udev->bus);
4939
4940         DPRINTFN(2, "endpoint=%p, addr=%d, endpt=%d, mode=%d (%d,%d)\n",
4941             ep, udev->address,
4942             edesc->bEndpointAddress, udev->flags.usb_mode,
4943             sc->sc_rt_addr, udev->device_index);
4944
4945         if (udev->device_index != sc->sc_rt_addr) {
4946
4947                 if (udev->flags.usb_mode == USB_MODE_DEVICE) {
4948                         if (udev->speed != USB_SPEED_FULL &&
4949                             udev->speed != USB_SPEED_HIGH) {
4950                                 /* not supported */
4951                                 return;
4952                         }
4953                 } else {
4954                         if (udev->speed == USB_SPEED_HIGH &&
4955                             (edesc->wMaxPacketSize[1] & 0x18) != 0 &&
4956                             (edesc->bmAttributes & UE_XFERTYPE) != UE_ISOCHRONOUS) {
4957                                 /* not supported */
4958                                 DPRINTFN(-1, "Non-isochronous high bandwidth "
4959                                     "endpoint not supported\n");
4960                                 return;
4961                         }
4962                 }
4963                 if ((edesc->bmAttributes & UE_XFERTYPE) == UE_ISOCHRONOUS)
4964                         ep->methods = &dwc_otg_device_isoc_methods;
4965                 else
4966                         ep->methods = &dwc_otg_device_non_isoc_methods;
4967         }
4968 }
4969
4970 static void
4971 dwc_otg_set_hw_power_sleep(struct usb_bus *bus, uint32_t state)
4972 {
4973         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(bus);
4974
4975         switch (state) {
4976         case USB_HW_POWER_SUSPEND:
4977                 dwc_otg_suspend(sc);
4978                 break;
4979         case USB_HW_POWER_SHUTDOWN:
4980                 dwc_otg_uninit(sc);
4981                 break;
4982         case USB_HW_POWER_RESUME:
4983                 dwc_otg_resume(sc);
4984                 break;
4985         default:
4986                 break;
4987         }
4988 }
4989
4990 static void
4991 dwc_otg_get_dma_delay(struct usb_device *udev, uint32_t *pus)
4992 {
4993         /* DMA delay - wait until any use of memory is finished */
4994         *pus = (2125);                  /* microseconds */
4995 }
4996
4997 static void
4998 dwc_otg_device_resume(struct usb_device *udev)
4999 {
5000         DPRINTF("\n");
5001
5002         /* poll all transfers again to restart resumed ones */
5003         dwc_otg_do_poll(udev->bus);
5004 }
5005
5006 static void
5007 dwc_otg_device_suspend(struct usb_device *udev)
5008 {
5009         DPRINTF("\n");
5010 }
5011
5012 static const struct usb_bus_methods dwc_otg_bus_methods =
5013 {
5014         .endpoint_init = &dwc_otg_ep_init,
5015         .xfer_setup = &dwc_otg_xfer_setup,
5016         .xfer_unsetup = &dwc_otg_xfer_unsetup,
5017         .get_hw_ep_profile = &dwc_otg_get_hw_ep_profile,
5018         .xfer_stall = &dwc_otg_xfer_stall,
5019         .set_stall = &dwc_otg_set_stall,
5020         .clear_stall = &dwc_otg_clear_stall,
5021         .roothub_exec = &dwc_otg_roothub_exec,
5022         .xfer_poll = &dwc_otg_do_poll,
5023         .device_state_change = &dwc_otg_device_state_change,
5024         .set_hw_power_sleep = &dwc_otg_set_hw_power_sleep,
5025         .get_dma_delay = &dwc_otg_get_dma_delay,
5026         .device_resume = &dwc_otg_device_resume,
5027         .device_suspend = &dwc_otg_device_suspend,
5028 };