sys/dev/usb/controller/dwc_otg.c

   1 /* $FreeBSD$ */
   2 /*-
   3  * Copyright (c) 2012 Hans Petter Selasky. All rights reserved.
   4  * Copyright (c) 2010-2011 Aleksandr Rybalko. All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  */
  27
  28 /*
  29  * This file contains the driver for the DesignWare series USB 2.0 OTG
  30  * Controller.
  31  */
  32
  33 /*
  34  * LIMITATION: Drivers must be bound to all OUT endpoints in the
  35  * active configuration for this driver to work properly. Blocking any
  36  * OUT endpoint will block all OUT endpoints including the control
  37  * endpoint. Usually this is not a problem.
  38  */
  39
  40 /*
  41  * NOTE: Writing to non-existing registers appears to cause an
  42  * internal reset.
  43  */
  44
  45 #ifdef USB_GLOBAL_INCLUDE_FILE
  46 #include USB_GLOBAL_INCLUDE_FILE
  47 #else
  48 #include <sys/stdint.h>
  49 #include <sys/stddef.h>
  50 #include <sys/param.h>
  51 #include <sys/queue.h>
  52 #include <sys/types.h>
  53 #include <sys/systm.h>
  54 #include <sys/kernel.h>
  55 #include <sys/bus.h>
  56 #include <sys/module.h>
  57 #include <sys/lock.h>
  58 #include <sys/mutex.h>
  59 #include <sys/condvar.h>
  60 #include <sys/sysctl.h>
  61 #include <sys/sx.h>
  62 #include <sys/unistd.h>
  63 #include <sys/callout.h>
  64 #include <sys/malloc.h>
  65 #include <sys/priv.h>
  66
  67 #include <dev/usb/usb.h>
  68 #include <dev/usb/usbdi.h>
  69
  70 #define USB_DEBUG_VAR dwc_otg_debug
  71
  72 #include <dev/usb/usb_core.h>
  73 #include <dev/usb/usb_debug.h>
  74 #include <dev/usb/usb_busdma.h>
  75 #include <dev/usb/usb_process.h>
  76 #include <dev/usb/usb_transfer.h>
  77 #include <dev/usb/usb_device.h>
  78 #include <dev/usb/usb_hub.h>
  79 #include <dev/usb/usb_util.h>
  80
  81 #include <dev/usb/usb_controller.h>
  82 #include <dev/usb/usb_bus.h>
  83 #endif                  /* USB_GLOBAL_INCLUDE_FILE */
  84
  85 #include <dev/usb/controller/dwc_otg.h>
  86 #include <dev/usb/controller/dwc_otgreg.h>
  87
  88 #define DWC_OTG_BUS2SC(bus) \
  89    ((struct dwc_otg_softc *)(((uint8_t *)(bus)) - \
  90     ((uint8_t *)&(((struct dwc_otg_softc *)0)->sc_bus))))
  91
  92 #define DWC_OTG_PC2UDEV(pc) \
  93    (USB_DMATAG_TO_XROOT((pc)->tag_parent)->udev)
  94
  95 #define DWC_OTG_MSK_GINT_ENABLED        \
  96    (GINTMSK_ENUMDONEMSK |               \
  97    GINTMSK_USBRSTMSK |                  \
  98    GINTMSK_USBSUSPMSK |                 \
  99    GINTMSK_IEPINTMSK |                  \
 100    GINTMSK_SESSREQINTMSK |              \
 101    GINTMSK_RXFLVLMSK |                  \
 102    GINTMSK_HCHINTMSK |                  \
 103    GINTMSK_OTGINTMSK |                  \
 104    GINTMSK_PRTINTMSK)
 105
 106 #define DWC_OTG_MSK_GINT_THREAD_IRQ                             \
 107    (GINTSTS_USBRST | GINTSTS_ENUMDONE | GINTSTS_PRTINT |        \
 108    GINTSTS_WKUPINT | GINTSTS_USBSUSP | GINTMSK_OTGINTMSK |      \
 109    GINTSTS_SESSREQINT)
 110
 111 #define DWC_OTG_PHY_ULPI 0
 112 #define DWC_OTG_PHY_HSIC 1
 113 #define DWC_OTG_PHY_INTERNAL 2
 114
 115 #ifndef DWC_OTG_PHY_DEFAULT
 116 #define DWC_OTG_PHY_DEFAULT DWC_OTG_PHY_ULPI
 117 #endif
 118
 119 static int dwc_otg_phy_type = DWC_OTG_PHY_DEFAULT;
 120
 121 static SYSCTL_NODE(_hw_usb, OID_AUTO, dwc_otg, CTLFLAG_RW, 0, "USB DWC OTG");
 122 SYSCTL_INT(_hw_usb_dwc_otg, OID_AUTO, phy_type, CTLFLAG_RDTUN,
 123     &dwc_otg_phy_type, 0, "DWC OTG PHY TYPE - 0/1/2 - ULPI/HSIC/INTERNAL");
 124 TUNABLE_INT("hw.usb.dwc_otg.phy_type", &dwc_otg_phy_type);
 125
 126 #ifdef USB_DEBUG
 127 static int dwc_otg_debug;
 128
 129 SYSCTL_INT(_hw_usb_dwc_otg, OID_AUTO, debug, CTLFLAG_RW,
 130     &dwc_otg_debug, 0, "DWC OTG debug level");
 131 #endif
 132
 133 #define DWC_OTG_INTR_ENDPT 1
 134
 135 /* prototypes */
 136
 137 struct usb_bus_methods dwc_otg_bus_methods;
 138 struct usb_pipe_methods dwc_otg_device_non_isoc_methods;
 139 struct usb_pipe_methods dwc_otg_device_isoc_methods;
 140
 141 static dwc_otg_cmd_t dwc_otg_setup_rx;
 142 static dwc_otg_cmd_t dwc_otg_data_rx;
 143 static dwc_otg_cmd_t dwc_otg_data_tx;
 144 static dwc_otg_cmd_t dwc_otg_data_tx_sync;
 145
 146 static dwc_otg_cmd_t dwc_otg_host_setup_tx;
 147 static dwc_otg_cmd_t dwc_otg_host_data_tx;
 148 static dwc_otg_cmd_t dwc_otg_host_data_rx;
 149
 150 static void dwc_otg_device_done(struct usb_xfer *, usb_error_t);
 151 static void dwc_otg_do_poll(struct usb_bus *);
 152 static void dwc_otg_standard_done(struct usb_xfer *);
 153 static void dwc_otg_root_intr(struct dwc_otg_softc *);
 154 static void dwc_otg_interrupt_poll_locked(struct dwc_otg_softc *);
 155 static void dwc_otg_host_channel_disable(struct dwc_otg_softc *, uint8_t);
 156
 157 /*
 158  * Here is a configuration that the chip supports.
 159  */
 160 static const struct usb_hw_ep_profile dwc_otg_ep_profile[1] = {
 161
 162         [0] = {
 163                 .max_in_frame_size = 64,/* fixed */
 164                 .max_out_frame_size = 64,       /* fixed */
 165                 .is_simplex = 1,
 166                 .support_control = 1,
 167         }
 168 };
 169
 170 static void
 171 dwc_otg_get_hw_ep_profile(struct usb_device *udev,
 172     const struct usb_hw_ep_profile **ppf, uint8_t ep_addr)
 173 {
 174         struct dwc_otg_softc *sc;
 175
 176         sc = DWC_OTG_BUS2SC(udev->bus);
 177
 178         if (ep_addr < sc->sc_dev_ep_max)
 179                 *ppf = &sc->sc_hw_ep_profile[ep_addr].usb;
 180         else
 181                 *ppf = NULL;
 182 }
 183
 184 static int
 185 dwc_otg_init_fifo(struct dwc_otg_softc *sc, uint8_t mode)
 186 {
 187         struct dwc_otg_profile *pf;
 188         uint32_t fifo_size;
 189         uint32_t fifo_regs;
 190         uint32_t tx_start;
 191         uint8_t x;
 192
 193         fifo_size = sc->sc_fifo_size;
 194
 195         /*
 196          * NOTE: Reserved fixed size area at end of RAM, which must
 197          * not be allocated to the FIFOs:
 198          */
 199         fifo_regs = 4 * 16;
 200
 201         if (fifo_size < fifo_regs) {
 202                 DPRINTF("Too little FIFO\n");
 203                 return (EINVAL);
 204         }
 205
 206         /* subtract FIFO regs from total once */
 207         fifo_size -= fifo_regs;
 208
 209         /* split equally for IN and OUT */
 210         fifo_size /= 2;
 211
 212         /* align to 4 bytes boundary */
 213         fifo_size &= ~3;
 214
 215         /* set global receive FIFO size */
 216         DWC_OTG_WRITE_4(sc, DOTG_GRXFSIZ, fifo_size / 4);
 217
 218         tx_start = fifo_size;
 219
 220         if (fifo_size < 64) {
 221                 DPRINTFN(-1, "Not enough data space for EP0 FIFO.\n");
 222                 return (EINVAL);
 223         }
 224
 225         /* disable any leftover host channels */
 226         for (x = 0; x != sc->sc_host_ch_max; x++) {
 227                 if (sc->sc_chan_state[x].wait_sof == 0)
 228                         continue;
 229                 dwc_otg_host_channel_disable(sc, x);
 230         }
 231
 232         if (mode == DWC_MODE_HOST) {
 233
 234                 /* reset active endpoints */
 235                 sc->sc_active_rx_ep = 0;
 236
 237                 /* split equally for periodic and non-periodic */
 238                 fifo_size /= 2;
 239
 240                 /* align to 4 bytes boundary */
 241                 fifo_size &= ~3;
 242
 243                 DWC_OTG_WRITE_4(sc, DOTG_GNPTXFSIZ,
 244                     ((fifo_size / 4) << 16) |
 245                     (tx_start / 4));
 246
 247                 tx_start += fifo_size;
 248
 249                 for (x = 0; x != sc->sc_host_ch_max; x++) {
 250                         /* disable all host interrupts */
 251                         DWC_OTG_WRITE_4(sc, DOTG_HCINTMSK(x),
 252                             HCINT_DEFAULT_MASK);
 253                 }
 254
 255                 DWC_OTG_WRITE_4(sc, DOTG_HPTXFSIZ,
 256                     ((fifo_size / 4) << 16) |
 257                     (tx_start / 4));
 258
 259                 /* reset host channel state */
 260                 memset(sc->sc_chan_state, 0, sizeof(sc->sc_chan_state));
 261
 262                 /* reset FIFO TX levels */
 263                 sc->sc_tx_cur_p_level = 0;
 264                 sc->sc_tx_cur_np_level = 0;
 265
 266                 /* store maximum periodic and non-periodic FIFO TX size */
 267                 sc->sc_tx_max_size = fifo_size;
 268
 269                 /* enable all host channel interrupts */
 270                 DWC_OTG_WRITE_4(sc, DOTG_HAINTMSK,
 271                     (1U << sc->sc_host_ch_max) - 1U);
 272         }
 273
 274         if (mode == DWC_MODE_DEVICE) {
 275
 276             DWC_OTG_WRITE_4(sc, DOTG_GNPTXFSIZ,
 277                 (0x10 << 16) | (tx_start / 4));
 278             fifo_size -= 0x40;
 279             tx_start += 0x40;
 280
 281             /* setup control endpoint profile */
 282             sc->sc_hw_ep_profile[0].usb = dwc_otg_ep_profile[0];
 283
 284             /* reset active endpoints */
 285             sc->sc_active_rx_ep = 1;
 286
 287             for (x = 1; x != sc->sc_dev_ep_max; x++) {
 288
 289                 pf = sc->sc_hw_ep_profile + x;
 290
 291                 pf->usb.max_out_frame_size = 1024 * 3;
 292                 pf->usb.is_simplex = 0; /* assume duplex */
 293                 pf->usb.support_bulk = 1;
 294                 pf->usb.support_interrupt = 1;
 295                 pf->usb.support_isochronous = 1;
 296                 pf->usb.support_out = 1;
 297
 298                 if (x < sc->sc_dev_in_ep_max) {
 299                         uint32_t limit;
 300
 301                         limit = (x == 1) ? DWC_OTG_MAX_TXN :
 302                             (DWC_OTG_MAX_TXN / 2);
 303
 304                         if (fifo_size >= limit) {
 305                                 DWC_OTG_WRITE_4(sc, DOTG_DIEPTXF(x),
 306                                     ((limit / 4) << 16) |
 307                                     (tx_start / 4));
 308                                 tx_start += limit;
 309                                 fifo_size -= limit;
 310                                 pf->usb.max_in_frame_size = 0x200;
 311                                 pf->usb.support_in = 1;
 312                                 pf->max_buffer = limit;
 313
 314                         } else if (fifo_size >= 0x80) {
 315                                 DWC_OTG_WRITE_4(sc, DOTG_DIEPTXF(x),
 316                                     ((0x80 / 4) << 16) | (tx_start / 4));
 317                                 tx_start += 0x80;
 318                                 fifo_size -= 0x80;
 319                                 pf->usb.max_in_frame_size = 0x40;
 320                                 pf->usb.support_in = 1;
 321
 322                         } else {
 323                                 pf->usb.is_simplex = 1;
 324                                 DWC_OTG_WRITE_4(sc, DOTG_DIEPTXF(x),
 325                                     (0x0 << 16) | (tx_start / 4));
 326                         }
 327                 } else {
 328                         pf->usb.is_simplex = 1;
 329                 }
 330
 331                 DPRINTF("FIFO%d = IN:%d / OUT:%d\n", x,
 332                     pf->usb.max_in_frame_size,
 333                     pf->usb.max_out_frame_size);
 334             }
 335         }
 336
 337         /* reset RX FIFO */
 338         DWC_OTG_WRITE_4(sc, DOTG_GRSTCTL,
 339             GRSTCTL_RXFFLSH);
 340
 341         if (mode != DWC_MODE_OTG) {
 342                 /* reset all TX FIFOs */
 343                 DWC_OTG_WRITE_4(sc, DOTG_GRSTCTL,
 344                     GRSTCTL_TXFIFO(0x10) |
 345                     GRSTCTL_TXFFLSH);
 346         } else {
 347                 /* reset active endpoints */
 348                 sc->sc_active_rx_ep = 0;
 349
 350                 /* reset periodic and non-periodic FIFO TX size */
 351                 sc->sc_tx_max_size = fifo_size;
 352
 353                 /* reset host channel state */
 354                 memset(sc->sc_chan_state, 0, sizeof(sc->sc_chan_state));
 355
 356                 /* reset FIFO TX levels */
 357                 sc->sc_tx_cur_p_level = 0;
 358                 sc->sc_tx_cur_np_level = 0;
 359         }
 360         return (0);
 361 }
 362
 363 static void
 364 dwc_otg_update_host_frame_interval(struct dwc_otg_softc *sc)
 365 {
 366
 367   /*
 368    * Disabled until further. Assuming that the register is already
 369    * programmed correctly by the boot loader.
 370    */
 371 #if 0
 372         uint32_t temp;
 373
 374         /* setup HOST frame interval register, based on existing value */
 375         temp = DWC_OTG_READ_4(sc, DOTG_HFIR) & HFIR_FRINT_MASK;
 376         if (temp >= 10000)
 377                 temp /= 1000;
 378         else
 379                 temp /= 125;
 380
 381         /* figure out nearest X-tal value */
 382         if (temp >= 54)
 383                 temp = 60;      /* MHz */
 384         else if (temp >= 39)
 385                 temp = 48;      /* MHz */
 386         else
 387                 temp = 30;      /* MHz */
 388
 389         if (sc->sc_flags.status_high_speed)
 390                 temp *= 125;
 391         else
 392                 temp *= 1000;
 393
 394         DPRINTF("HFIR=0x%08x\n", temp);
 395
 396         DWC_OTG_WRITE_4(sc, DOTG_HFIR, temp);
 397 #endif
 398 }
 399
 400 static void
 401 dwc_otg_clocks_on(struct dwc_otg_softc *sc)
 402 {
 403         if (sc->sc_flags.clocks_off &&
 404             sc->sc_flags.port_powered) {
 405
 406                 DPRINTFN(5, "\n");
 407
 408                 /* TODO - platform specific */
 409
 410                 sc->sc_flags.clocks_off = 0;
 411         }
 412 }
 413
 414 static void
 415 dwc_otg_clocks_off(struct dwc_otg_softc *sc)
 416 {
 417         if (!sc->sc_flags.clocks_off) {
 418
 419                 DPRINTFN(5, "\n");
 420
 421                 /* TODO - platform specific */
 422
 423                 sc->sc_flags.clocks_off = 1;
 424         }
 425 }
 426
 427 static void
 428 dwc_otg_pull_up(struct dwc_otg_softc *sc)
 429 {
 430         uint32_t temp;
 431
 432         /* pullup D+, if possible */
 433
 434         if (!sc->sc_flags.d_pulled_up &&
 435             sc->sc_flags.port_powered) {
 436                 sc->sc_flags.d_pulled_up = 1;
 437
 438                 temp = DWC_OTG_READ_4(sc, DOTG_DCTL);
 439                 temp &= ~DCTL_SFTDISCON;
 440                 DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
 441         }
 442 }
 443
 444 static void
 445 dwc_otg_pull_down(struct dwc_otg_softc *sc)
 446 {
 447         uint32_t temp;
 448
 449         /* pulldown D+, if possible */
 450
 451         if (sc->sc_flags.d_pulled_up) {
 452                 sc->sc_flags.d_pulled_up = 0;
 453
 454                 temp = DWC_OTG_READ_4(sc, DOTG_DCTL);
 455                 temp |= DCTL_SFTDISCON;
 456                 DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
 457         }
 458 }
 459
 460 static void
 461 dwc_otg_enable_sof_irq(struct dwc_otg_softc *sc)
 462 {
 463         /* In device mode we don't use the SOF interrupt */
 464         if (sc->sc_flags.status_device_mode != 0 ||
 465             (sc->sc_irq_mask & GINTMSK_SOFMSK) != 0)
 466                 return;
 467         sc->sc_irq_mask |= GINTMSK_SOFMSK;
 468         DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 469 }
 470
 471 static void
 472 dwc_otg_resume_irq(struct dwc_otg_softc *sc)
 473 {
 474         if (sc->sc_flags.status_suspend) {
 475                 /* update status bits */
 476                 sc->sc_flags.status_suspend = 0;
 477                 sc->sc_flags.change_suspend = 1;
 478
 479                 if (sc->sc_flags.status_device_mode) {
 480                         /*
 481                          * Disable resume interrupt and enable suspend
 482                          * interrupt:
 483                          */
 484                         sc->sc_irq_mask &= ~GINTMSK_WKUPINTMSK;
 485                         sc->sc_irq_mask |= GINTMSK_USBSUSPMSK;
 486                         DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 487                 }
 488
 489                 /* complete root HUB interrupt endpoint */
 490                 dwc_otg_root_intr(sc);
 491         }
 492 }
 493
 494 static void
 495 dwc_otg_suspend_irq(struct dwc_otg_softc *sc)
 496 {
 497         if (!sc->sc_flags.status_suspend) {
 498                 /* update status bits */
 499                 sc->sc_flags.status_suspend = 1;
 500                 sc->sc_flags.change_suspend = 1;
 501
 502                 if (sc->sc_flags.status_device_mode) {
 503                         /*
 504                          * Disable suspend interrupt and enable resume
 505                          * interrupt:
 506                          */
 507                         sc->sc_irq_mask &= ~GINTMSK_USBSUSPMSK;
 508                         sc->sc_irq_mask |= GINTMSK_WKUPINTMSK;
 509                         DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 510                 }
 511
 512                 /* complete root HUB interrupt endpoint */
 513                 dwc_otg_root_intr(sc);
 514         }
 515 }
 516
 517 static void
 518 dwc_otg_wakeup_peer(struct dwc_otg_softc *sc)
 519 {
 520         if (!sc->sc_flags.status_suspend)
 521                 return;
 522
 523         DPRINTFN(5, "Remote wakeup\n");
 524
 525         if (sc->sc_flags.status_device_mode) {
 526                 uint32_t temp;
 527
 528                 /* enable remote wakeup signalling */
 529                 temp = DWC_OTG_READ_4(sc, DOTG_DCTL);
 530                 temp |= DCTL_RMTWKUPSIG;
 531                 DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
 532
 533                 /* Wait 8ms for remote wakeup to complete. */
 534                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 125);
 535
 536                 temp &= ~DCTL_RMTWKUPSIG;
 537                 DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
 538         } else {
 539                 /* enable USB port */
 540                 DWC_OTG_WRITE_4(sc, DOTG_PCGCCTL, 0);
 541
 542                 /* wait 10ms */
 543                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 100);
 544
 545                 /* resume port */
 546                 sc->sc_hprt_val |= HPRT_PRTRES;
 547                 DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
 548
 549                 /* Wait 100ms for resume signalling to complete. */
 550                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 10);
 551
 552                 /* clear suspend and resume */
 553                 sc->sc_hprt_val &= ~(HPRT_PRTSUSP | HPRT_PRTRES);
 554                 DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
 555
 556                 /* Wait 4ms */
 557                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 250);
 558         }
 559
 560         /* need to fake resume IRQ */
 561         dwc_otg_resume_irq(sc);
 562 }
 563
 564 static void
 565 dwc_otg_set_address(struct dwc_otg_softc *sc, uint8_t addr)
 566 {
 567         uint32_t temp;
 568
 569         DPRINTFN(5, "addr=%d\n", addr);
 570
 571         temp = DWC_OTG_READ_4(sc, DOTG_DCFG);
 572         temp &= ~DCFG_DEVADDR_SET(0x7F);
 573         temp |= DCFG_DEVADDR_SET(addr);
 574         DWC_OTG_WRITE_4(sc, DOTG_DCFG, temp);
 575 }
 576
 577 static void
 578 dwc_otg_common_rx_ack(struct dwc_otg_softc *sc)
 579 {
 580         DPRINTFN(5, "RX status clear\n");
 581
 582         /* enable RX FIFO level interrupt */
 583         sc->sc_irq_mask |= GINTMSK_RXFLVLMSK;
 584         DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 585
 586         /* clear cached status */
 587         sc->sc_last_rx_status = 0;
 588 }
 589
 590 static void
 591 dwc_otg_clear_hcint(struct dwc_otg_softc *sc, uint8_t x)
 592 {
 593         uint32_t hcint;
 594
 595         /* clear all pending interrupts */
 596         hcint = DWC_OTG_READ_4(sc, DOTG_HCINT(x));
 597         DWC_OTG_WRITE_4(sc, DOTG_HCINT(x), hcint);
 598
 599         /* clear buffered interrupts */
 600         sc->sc_chan_state[x].hcint = 0;
 601 }
 602
 603 static uint8_t
 604 dwc_otg_host_channel_alloc(struct dwc_otg_softc *sc, struct dwc_otg_td *td, uint8_t is_out)
 605 {
 606         uint32_t tx_p_size;
 607         uint32_t tx_np_size;
 608         uint8_t x;
 609
 610         if (td->channel < DWC_OTG_MAX_CHANNELS)
 611                 return (0);             /* already allocated */
 612
 613         /* check if device is suspended */
 614         if (DWC_OTG_PC2UDEV(td->pc)->flags.self_suspended != 0)
 615                 return (1);             /* busy - cannot transfer data */
 616
 617         /* compute needed TX FIFO size */
 618         if (is_out != 0) {
 619                 if (td->ep_type == UE_ISOCHRONOUS) {
 620                         tx_p_size = td->max_packet_size;
 621                         tx_np_size = 0;
 622                         if (td->hcsplt != 0 && tx_p_size > HCSPLT_XACTLEN_BURST)
 623                                 tx_p_size = HCSPLT_XACTLEN_BURST;
 624                         if ((sc->sc_tx_cur_p_level + tx_p_size) > sc->sc_tx_max_size) {
 625                                 DPRINTF("Too little FIFO space\n");
 626                                 return (1);     /* too little FIFO */
 627                         }
 628                 } else {
 629                         tx_p_size = 0;
 630                         tx_np_size = td->max_packet_size;
 631                         if (td->hcsplt != 0 && tx_np_size > HCSPLT_XACTLEN_BURST)
 632                                 tx_np_size = HCSPLT_XACTLEN_BURST;
 633                         if ((sc->sc_tx_cur_np_level + tx_np_size) > sc->sc_tx_max_size) {
 634                                 DPRINTF("Too little FIFO space\n");
 635                                 return (1);     /* too little FIFO */
 636                         }
 637                 }
 638         } else {
 639                 /* not a TX transaction */
 640                 tx_p_size = 0;
 641                 tx_np_size = 0;
 642         }
 643
 644         for (x = 0; x != sc->sc_host_ch_max; x++) {
 645                 if (sc->sc_chan_state[x].allocated != 0)
 646                         continue;
 647                 /* check if channel is still enabled */
 648                 if (sc->sc_chan_state[x].wait_sof != 0)
 649                         continue;
 650
 651                 sc->sc_chan_state[x].allocated = 1;
 652                 sc->sc_chan_state[x].tx_p_size = tx_p_size;
 653                 sc->sc_chan_state[x].tx_np_size = tx_np_size;
 654
 655                 /* keep track of used TX FIFO, if any */
 656                 sc->sc_tx_cur_p_level += tx_p_size;
 657                 sc->sc_tx_cur_np_level += tx_np_size;
 658
 659                 /* clear interrupts */
 660                 dwc_otg_clear_hcint(sc, x);
 661
 662                 DPRINTF("CH=%d HCCHAR=0x%08x "
 663                     "HCSPLT=0x%08x\n", x, td->hcchar, td->hcsplt);
 664
 665                 /* set active channel */
 666                 sc->sc_active_rx_ep |= (1 << x);
 667
 668                 /* set channel */
 669                 td->channel = x;
 670
 671                 return (0);     /* allocated */
 672         }
 673         /* wait a bit */
 674         dwc_otg_enable_sof_irq(sc);
 675         return (1);     /* busy */
 676 }
 677
 678 static void
 679 dwc_otg_host_channel_free(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
 680 {
 681         uint8_t x;
 682
 683         if (td->channel >= DWC_OTG_MAX_CHANNELS)
 684                 return;         /* already freed */
 685
 686         /* free channel */
 687         x = td->channel;
 688         td->channel = DWC_OTG_MAX_CHANNELS;
 689
 690         DPRINTF("CH=%d\n", x);
 691
 692         /*
 693          * We need to let programmed host channels run till complete
 694          * else the host channel will stop functioning. Assume that
 695          * after a fixed given amount of time the host channel is no
 696          * longer doing any USB traffic:
 697          */
 698         if (td->ep_type == UE_ISOCHRONOUS) {
 699                 /* double buffered */
 700                 sc->sc_chan_state[x].wait_sof = DWC_OTG_SLOT_IDLE_MAX;
 701         } else {
 702                 /* single buffered */
 703                 sc->sc_chan_state[x].wait_sof = DWC_OTG_SLOT_IDLE_MIN;
 704         }
 705
 706         sc->sc_chan_state[x].allocated = 0;
 707
 708         /* ack any pending messages */
 709         if (sc->sc_last_rx_status != 0 &&
 710             GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) == x) {
 711                 dwc_otg_common_rx_ack(sc);
 712         }
 713
 714         /* clear active channel */
 715         sc->sc_active_rx_ep &= ~(1 << x);
 716 }
 717
 718 static void
 719 dwc_otg_host_dump_rx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
 720 {
 721         /* dump any pending messages */
 722         if (sc->sc_last_rx_status != 0) {
 723                 if (td->channel < DWC_OTG_MAX_CHANNELS &&
 724                     td->channel == GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status)) {
 725                         dwc_otg_common_rx_ack(sc);
 726                 }
 727         }
 728 }
 729
 730 static uint8_t
 731 dwc_otg_host_setup_tx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
 732 {
 733         struct usb_device_request req __aligned(4);
 734         uint32_t hcint;
 735         uint32_t hcchar;
 736         uint8_t delta;
 737
 738         dwc_otg_host_dump_rx(sc, td);
 739
 740         if (td->channel < DWC_OTG_MAX_CHANNELS) {
 741                 hcint = sc->sc_chan_state[td->channel].hcint;
 742
 743                 DPRINTF("CH=%d ST=%d HCINT=0x%08x HCCHAR=0x%08x HCTSIZ=0x%08x\n",
 744                     td->channel, td->state, hcint,
 745                     DWC_OTG_READ_4(sc, DOTG_HCCHAR(td->channel)),
 746                     DWC_OTG_READ_4(sc, DOTG_HCTSIZ(td->channel)));
 747         } else {
 748                 hcint = 0;
 749                 goto check_state;
 750         }
 751
 752         if (hcint & (HCINT_RETRY |
 753             HCINT_ACK | HCINT_NYET)) {
 754                 /* give success bits priority over failure bits */
 755         } else if (hcint & HCINT_STALL) {
 756                 DPRINTF("CH=%d STALL\n", td->channel);
 757                 td->error_stall = 1;
 758                 td->error_any = 1;
 759                 goto complete;
 760         } else if (hcint & HCINT_ERRORS) {
 761                 DPRINTF("CH=%d ERROR\n", td->channel);
 762                 td->errcnt++;
 763                 if (td->hcsplt != 0 || td->errcnt >= 3) {
 764                         td->error_any = 1;
 765                         goto complete;
 766                 }
 767         }
 768
 769         if (hcint & (HCINT_ERRORS | HCINT_RETRY |
 770             HCINT_ACK | HCINT_NYET)) {
 771                 if (!(hcint & HCINT_ERRORS))
 772                         td->errcnt = 0;
 773         }
 774
 775 check_state:
 776         switch (td->state) {
 777         case DWC_CHAN_ST_START:
 778                 goto send_pkt;
 779
 780         case DWC_CHAN_ST_WAIT_ANE:
 781                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
 782                         td->did_nak++;
 783                         td->tt_scheduled = 0;
 784                         goto send_pkt;
 785                 } else if (hcint & (HCINT_ACK | HCINT_NYET)) {
 786                         td->offset += td->tx_bytes;
 787                         td->remainder -= td->tx_bytes;
 788                         td->toggle = 1;
 789                         td->tt_scheduled = 0;
 790                         goto complete;
 791                 }
 792                 break;
 793
 794         case DWC_CHAN_ST_WAIT_S_ANE:
 795                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
 796                         td->did_nak++;
 797                         td->tt_scheduled = 0;
 798                         goto send_pkt;
 799                 } else if (hcint & (HCINT_ACK | HCINT_NYET)) {
 800                         goto send_cpkt;
 801                 }
 802                 break;
 803
 804         case DWC_CHAN_ST_WAIT_C_ANE:
 805                 if (hcint & HCINT_NYET) {
 806                         goto send_cpkt;
 807                 } else if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
 808                         td->did_nak++;
 809                         td->tt_scheduled = 0;
 810                         goto send_pkt;
 811                 } else if (hcint & HCINT_ACK) {
 812                         td->offset += td->tx_bytes;
 813                         td->remainder -= td->tx_bytes;
 814                         td->toggle = 1;
 815                         goto complete;
 816                 }
 817                 break;
 818
 819         case DWC_CHAN_ST_WAIT_C_PKT:
 820                 goto send_cpkt;
 821
 822         default:
 823                 break;
 824         }
 825         goto busy;
 826
 827 send_pkt:
 828         /* free existing channel, if any */
 829         dwc_otg_host_channel_free(sc, td);
 830
 831         if (sizeof(req) != td->remainder) {
 832                 td->error_any = 1;
 833                 goto complete;
 834         }
 835
 836         if (td->hcsplt != 0) {
 837                 delta = td->tt_start_slot - sc->sc_last_frame_num - 1;
 838                 if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
 839                         td->state = DWC_CHAN_ST_START;
 840                         goto busy;
 841                 }
 842                 delta = sc->sc_last_frame_num - td->tt_start_slot;
 843                 if (delta > 5) {
 844                         /* missed it */
 845                         td->tt_scheduled = 0;
 846                         td->state = DWC_CHAN_ST_START;
 847                         goto busy;
 848                 }
 849         }
 850
 851         /* allocate a new channel */
 852         if (dwc_otg_host_channel_alloc(sc, td, 1)) {
 853                 td->state = DWC_CHAN_ST_START;
 854                 goto busy;
 855         }
 856
 857         if (td->hcsplt != 0) {
 858                 td->hcsplt &= ~HCSPLT_COMPSPLT;
 859                 td->state = DWC_CHAN_ST_WAIT_S_ANE;
 860         } else {
 861                 td->state = DWC_CHAN_ST_WAIT_ANE;
 862         }
 863
 864         usbd_copy_out(td->pc, 0, &req, sizeof(req));
 865
 866         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(td->channel),
 867             (sizeof(req) << HCTSIZ_XFERSIZE_SHIFT) |
 868             (1 << HCTSIZ_PKTCNT_SHIFT) |
 869             (HCTSIZ_PID_SETUP << HCTSIZ_PID_SHIFT));
 870
 871         DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(td->channel), td->hcsplt);
 872
 873         hcchar = td->hcchar;
 874         hcchar &= ~(HCCHAR_EPDIR_IN | HCCHAR_EPTYPE_MASK);
 875         hcchar |= UE_CONTROL << HCCHAR_EPTYPE_SHIFT;
 876
 877         /* must enable channel before writing data to FIFO */
 878         DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(td->channel), hcchar);
 879
 880         /* transfer data into FIFO */
 881         bus_space_write_region_4(sc->sc_io_tag, sc->sc_io_hdl,
 882             DOTG_DFIFO(td->channel), (uint32_t *)&req, sizeof(req) / 4);
 883
 884         /* wait until next slot before trying complete split */
 885         td->tt_complete_slot = sc->sc_last_frame_num + 1;
 886
 887         /* store number of bytes transmitted */
 888         td->tx_bytes = sizeof(req);
 889         goto busy;
 890
 891 send_cpkt:
 892         /* free existing channel, if any */
 893         dwc_otg_host_channel_free(sc, td);
 894
 895         delta = td->tt_complete_slot - sc->sc_last_frame_num - 1;
 896         if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
 897                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
 898                 goto busy;
 899         }
 900         delta = sc->sc_last_frame_num - td->tt_start_slot;
 901         if (delta > DWC_OTG_TT_SLOT_MAX) {
 902                 /* we missed the service interval */
 903                 if (td->ep_type != UE_ISOCHRONOUS)
 904                         td->error_any = 1;
 905                 goto complete;
 906         }
 907         /* allocate a new channel */
 908         if (dwc_otg_host_channel_alloc(sc, td, 0)) {
 909                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
 910                 goto busy;
 911         }
 912
 913         /* wait until next slot before trying complete split */
 914         td->tt_complete_slot = sc->sc_last_frame_num + 1;
 915
 916         td->hcsplt |= HCSPLT_COMPSPLT;
 917         td->state = DWC_CHAN_ST_WAIT_C_ANE;
 918
 919         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(td->channel),
 920             (HCTSIZ_PID_SETUP << HCTSIZ_PID_SHIFT));
 921
 922         DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(td->channel), td->hcsplt);
 923
 924         hcchar = td->hcchar;
 925         hcchar &= ~(HCCHAR_EPDIR_IN | HCCHAR_EPTYPE_MASK);
 926         hcchar |= UE_CONTROL << HCCHAR_EPTYPE_SHIFT;
 927
 928         /* must enable channel before writing data to FIFO */
 929         DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(td->channel), hcchar);
 930
 931 busy:
 932         return (1);     /* busy */
 933
 934 complete:
 935         dwc_otg_host_channel_free(sc, td);
 936         return (0);     /* complete */
 937 }
 938
 939 static uint8_t
 940 dwc_otg_setup_rx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
 941 {
 942         struct usb_device_request req __aligned(4);
 943         uint32_t temp;
 944         uint16_t count;
 945
 946         /* check endpoint status */
 947
 948         if (sc->sc_last_rx_status == 0)
 949                 goto not_complete;
 950
 951         if (GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) != 0)
 952                 goto not_complete;
 953
 954         if ((sc->sc_last_rx_status & GRXSTSRD_DPID_MASK) !=
 955             GRXSTSRD_DPID_DATA0) {
 956                 /* release FIFO */
 957                 dwc_otg_common_rx_ack(sc);
 958                 goto not_complete;
 959         }
 960
 961         if ((sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) !=
 962             GRXSTSRD_STP_DATA) {
 963                 /* release FIFO */
 964                 dwc_otg_common_rx_ack(sc);
 965                 goto not_complete;
 966         }
 967
 968         DPRINTFN(5, "GRXSTSR=0x%08x\n", sc->sc_last_rx_status);
 969
 970         /* clear did stall */
 971         td->did_stall = 0;
 972
 973         /* get the packet byte count */
 974         count = GRXSTSRD_BCNT_GET(sc->sc_last_rx_status);
 975
 976         /* verify data length */
 977         if (count != td->remainder) {
 978                 DPRINTFN(0, "Invalid SETUP packet "
 979                     "length, %d bytes\n", count);
 980                 /* release FIFO */
 981                 dwc_otg_common_rx_ack(sc);
 982                 goto not_complete;
 983         }
 984         if (count != sizeof(req)) {
 985                 DPRINTFN(0, "Unsupported SETUP packet "
 986                     "length, %d bytes\n", count);
 987                 /* release FIFO */
 988                 dwc_otg_common_rx_ack(sc);
 989                 goto not_complete;
 990         }
 991
 992         /* copy in control request */
 993         memcpy(&req, sc->sc_rx_bounce_buffer, sizeof(req));
 994
 995         /* copy data into real buffer */
 996         usbd_copy_in(td->pc, 0, &req, sizeof(req));
 997
 998         td->offset = sizeof(req);
 999         td->remainder = 0;
1000
1001         /* sneak peek the set address */
1002         if ((req.bmRequestType == UT_WRITE_DEVICE) &&
1003             (req.bRequest == UR_SET_ADDRESS)) {
1004                 /* must write address before ZLP */
1005                 dwc_otg_set_address(sc, req.wValue[0] & 0x7F);
1006         }
1007
1008         /* don't send any data by default */
1009         DWC_OTG_WRITE_4(sc, DOTG_DIEPTSIZ(0),
1010             DXEPTSIZ_SET_NPKT(0) |
1011             DXEPTSIZ_SET_NBYTES(0));
1012
1013         temp = sc->sc_in_ctl[0];
1014
1015         /* enable IN endpoint */
1016         DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(0),
1017             temp | DIEPCTL_EPENA);
1018         DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(0),
1019             temp | DIEPCTL_SNAK);
1020
1021         /* reset IN endpoint buffer */
1022         DWC_OTG_WRITE_4(sc, DOTG_GRSTCTL,
1023             GRSTCTL_TXFIFO(0) |
1024             GRSTCTL_TXFFLSH);
1025
1026         /* acknowledge RX status */
1027         dwc_otg_common_rx_ack(sc);
1028         return (0);                     /* complete */
1029
1030 not_complete:
1031         /* abort any ongoing transfer, before enabling again */
1032
1033         temp = sc->sc_out_ctl[0];
1034
1035         temp |= DOEPCTL_EPENA |
1036             DOEPCTL_SNAK;
1037
1038         /* enable OUT endpoint */
1039         DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(0), temp);
1040
1041         if (!td->did_stall) {
1042                 td->did_stall = 1;
1043
1044                 DPRINTFN(5, "stalling IN and OUT direction\n");
1045
1046                 /* set stall after enabling endpoint */
1047                 DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(0),
1048                     temp | DOEPCTL_STALL);
1049
1050                 temp = sc->sc_in_ctl[0];
1051
1052                 /* set stall assuming endpoint is enabled */
1053                 DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(0),
1054                     temp | DIEPCTL_STALL);
1055         }
1056
1057         /* setup number of buffers to receive */
1058         DWC_OTG_WRITE_4(sc, DOTG_DOEPTSIZ(0),
1059             DXEPTSIZ_SET_MULTI(3) |
1060             DXEPTSIZ_SET_NPKT(1) |
1061             DXEPTSIZ_SET_NBYTES(sizeof(req)));
1062
1063         return (1);                     /* not complete */
1064 }
1065
1066 static uint8_t
1067 dwc_otg_host_rate_check_interrupt(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1068 {
1069         uint8_t delta;
1070
1071         delta = sc->sc_tmr_val - td->tmr_val;
1072         if (delta >= 128)
1073                 return (1);     /* busy */
1074
1075         td->tmr_val = sc->sc_tmr_val + td->tmr_res;
1076
1077         /* set toggle, if any */
1078         if (td->set_toggle) {
1079                 td->set_toggle = 0;
1080                 td->toggle = 1;
1081         }
1082         return (0);
1083 }
1084
1085 static uint8_t
1086 dwc_otg_host_rate_check(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1087 {
1088         if (td->ep_type == UE_ISOCHRONOUS) {
1089                 /* non TT isochronous traffic */
1090                 if ((td->tmr_val != 0) ||
1091                     (sc->sc_last_frame_num & (td->tmr_res - 1))) {
1092                         goto busy;
1093                 }
1094                 td->tmr_val = 1;        /* executed */
1095                 td->toggle = 0;
1096
1097         } else if (td->ep_type == UE_INTERRUPT) {
1098                 if (!td->tt_scheduled)
1099                         goto busy;
1100                 td->tt_scheduled = 0;
1101         } else if (td->did_nak >= DWC_OTG_NAK_MAX) {
1102                 goto busy;
1103         } else if (td->set_toggle) {
1104                 td->set_toggle = 0;
1105                 td->toggle = 1;
1106         }
1107         return (0);
1108 busy:
1109         return (1);
1110 }
1111
1112 static uint8_t
1113 dwc_otg_host_data_rx_sub(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1114 {
1115         uint32_t count;
1116         uint8_t channel;
1117
1118         /* check endpoint status */
1119         if (sc->sc_last_rx_status == 0)
1120                 goto busy;
1121
1122         channel = td->channel;
1123         if (channel >= DWC_OTG_MAX_CHANNELS)
1124                 goto busy;
1125
1126         if (GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) != channel)
1127                 goto busy;
1128
1129         switch (sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) {
1130         case GRXSTSRH_IN_DATA:
1131
1132                 DPRINTF("DATA ST=%d STATUS=0x%08x\n",
1133                     (int)td->state, (int)sc->sc_last_rx_status);
1134
1135                 if (sc->sc_chan_state[channel].hcint & HCINT_SOFTWARE_ONLY) {
1136                         /*
1137                          * When using SPLIT transactions on interrupt
1138                          * endpoints, sometimes data occurs twice.
1139                          */
1140                         DPRINTF("Data already received\n");
1141                         break;
1142                 }
1143
1144                 /* get the packet byte count */
1145                 count = GRXSTSRD_BCNT_GET(sc->sc_last_rx_status);
1146
1147                 /* check for isochronous transfer or high-speed bandwidth endpoint */
1148                 if (td->ep_type == UE_ISOCHRONOUS || td->max_packet_count > 1) {
1149                         if ((sc->sc_last_rx_status & GRXSTSRD_DPID_MASK) != GRXSTSRD_DPID_DATA0) {
1150                                 td->tt_xactpos = HCSPLT_XACTPOS_MIDDLE;
1151                         } else {
1152                                 td->tt_xactpos = HCSPLT_XACTPOS_BEGIN;
1153
1154                                 /* verify the packet byte count */
1155                                 if (count < td->max_packet_size) {
1156                                         /* we have a short packet */
1157                                         td->short_pkt = 1;
1158                                         td->got_short = 1;
1159                                 }
1160                         }
1161                         td->toggle = 0;
1162                 } else {
1163                         /* verify the packet byte count */
1164                         if (count != td->max_packet_size) {
1165                                 if (count < td->max_packet_size) {
1166                                         /* we have a short packet */
1167                                         td->short_pkt = 1;
1168                                         td->got_short = 1;
1169                                 } else {
1170                                         /* invalid USB packet */
1171                                         td->error_any = 1;
1172
1173                                         /* release FIFO */
1174                                         dwc_otg_common_rx_ack(sc);
1175                                         goto complete;
1176                                 }
1177                         }
1178                         td->toggle ^= 1;
1179                         td->tt_scheduled = 0;
1180                 }
1181
1182                 /* verify the packet byte count */
1183                 if (count > td->remainder) {
1184                         /* invalid USB packet */
1185                         td->error_any = 1;
1186
1187                         /* release FIFO */
1188                         dwc_otg_common_rx_ack(sc);
1189                         goto complete;
1190                 }
1191
1192                 usbd_copy_in(td->pc, td->offset,
1193                     sc->sc_rx_bounce_buffer, count);
1194
1195                 td->remainder -= count;
1196                 td->offset += count;
1197                 sc->sc_chan_state[channel].hcint |= HCINT_SOFTWARE_ONLY;
1198                 break;
1199         default:
1200                 break;
1201         }
1202         /* release FIFO */
1203         dwc_otg_common_rx_ack(sc);
1204 busy:
1205         return (0);
1206 complete:
1207         return (1);
1208 }
1209
1210 static uint8_t
1211 dwc_otg_host_data_rx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1212 {
1213         uint32_t hcint;
1214         uint32_t hcchar;
1215         uint8_t delta;
1216         uint8_t channel;
1217
1218         channel = td->channel;
1219
1220         if (channel < DWC_OTG_MAX_CHANNELS) {
1221                 hcint = sc->sc_chan_state[channel].hcint;
1222
1223                 DPRINTF("CH=%d ST=%d HCINT=0x%08x HCCHAR=0x%08x HCTSIZ=0x%08x\n",
1224                     channel, td->state, hcint,
1225                     DWC_OTG_READ_4(sc, DOTG_HCCHAR(channel)),
1226                     DWC_OTG_READ_4(sc, DOTG_HCTSIZ(channel)));
1227
1228                 /* check interrupt bits */
1229                 if (hcint & (HCINT_RETRY |
1230                     HCINT_ACK | HCINT_NYET)) {
1231                         /* give success bits priority over failure bits */
1232                 } else if (hcint & HCINT_STALL) {
1233                         DPRINTF("CH=%d STALL\n", channel);
1234                         td->error_stall = 1;
1235                         td->error_any = 1;
1236                         goto complete;
1237                 } else if (hcint & HCINT_ERRORS) {
1238                         DPRINTF("CH=%d ERROR\n", channel);
1239                         td->errcnt++;
1240                         if (td->hcsplt != 0 || td->errcnt >= 3) {
1241                                 if (td->ep_type != UE_ISOCHRONOUS) {
1242                                         td->error_any = 1;
1243                                         goto complete;
1244                                 }
1245                         }
1246                 }
1247
1248                 /* check channels for data, if any */
1249                 if (dwc_otg_host_data_rx_sub(sc, td))
1250                         goto complete;
1251
1252                 /* refresh interrupt status */
1253                 hcint = sc->sc_chan_state[channel].hcint;
1254
1255                 if (hcint & (HCINT_ERRORS | HCINT_RETRY |
1256                     HCINT_ACK | HCINT_NYET)) {
1257                         if (!(hcint & HCINT_ERRORS))
1258                                 td->errcnt = 0;
1259                 }
1260         } else {
1261                 hcint = 0;
1262         }
1263
1264         switch (td->state) {
1265         case DWC_CHAN_ST_START:
1266                 if (td->hcsplt != 0)
1267                         goto receive_spkt;
1268                 else
1269                         goto receive_pkt;
1270
1271         case DWC_CHAN_ST_WAIT_ANE:
1272                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1273                         if (td->ep_type == UE_INTERRUPT) {
1274                                 /*
1275                                  * The USB specification does not
1276                                  * mandate a particular data toggle
1277                                  * value for USB INTERRUPT
1278                                  * transfers. Switch the data toggle
1279                                  * value to receive the packet
1280                                  * correctly:
1281                                  */
1282                                 if (hcint & HCINT_DATATGLERR) {
1283                                         DPRINTF("Retrying packet due to "
1284                                             "data toggle error\n");
1285                                         td->toggle ^= 1;
1286                                         goto receive_pkt;
1287                                 }
1288                         }
1289                         td->did_nak++;
1290                         td->tt_scheduled = 0;
1291                         if (td->hcsplt != 0)
1292                                 goto receive_spkt;
1293                         else
1294                                 goto receive_pkt;
1295                 } else if (hcint & HCINT_NYET) {
1296                         if (td->hcsplt != 0) {
1297                                 /* try again */
1298                                 goto receive_pkt;
1299                         } else {
1300                                 /* not a valid token for IN endpoints */
1301                                 td->error_any = 1;
1302                                 goto complete;
1303                         }
1304                 } else if (hcint & HCINT_ACK) {
1305                         /* wait for data - ACK arrived first */
1306                         if (!(hcint & HCINT_SOFTWARE_ONLY))
1307                                 goto busy;
1308
1309                         if (td->ep_type == UE_ISOCHRONOUS) {
1310                                 /* check if we are complete */
1311                                 if ((td->remainder == 0) ||
1312                                     (td->tt_xactpos == HCSPLT_XACTPOS_BEGIN)) {
1313                                         goto complete;
1314                                 }
1315                                 /* get another packet */
1316                                 goto receive_pkt;
1317                         } else {
1318                                 /* check if we are complete */
1319                                 if ((td->remainder == 0) || (td->got_short != 0)) {
1320                                         if (td->short_pkt)
1321                                                 goto complete;
1322
1323                                         /*
1324                                          * Else need to receive a zero length
1325                                          * packet.
1326                                          */
1327                                 }
1328                                 td->tt_scheduled = 0;
1329                                 td->did_nak = 0;
1330                                 if (td->hcsplt != 0)
1331                                         goto receive_spkt;
1332                                 else
1333                                         goto receive_pkt;
1334                         }
1335                 }
1336                 break;
1337
1338         case DWC_CHAN_ST_WAIT_S_ANE:
1339                 /*
1340                  * NOTE: The DWC OTG hardware provides a fake ACK in
1341                  * case of interrupt and isochronous transfers:
1342                  */
1343                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1344                         td->did_nak++;
1345                         td->tt_scheduled = 0;
1346                         goto receive_spkt;
1347                 } else if (hcint & HCINT_NYET) {
1348                         td->tt_scheduled = 0;
1349                         goto receive_spkt;
1350                 } else if (hcint & HCINT_ACK) {
1351                         td->did_nak = 0;
1352                         goto receive_pkt;
1353                 }
1354                 break;
1355
1356         case DWC_CHAN_ST_WAIT_C_PKT:
1357                 goto receive_pkt;
1358
1359         default:
1360                 break;
1361         }
1362         goto busy;
1363
1364 receive_pkt:
1365         /* free existing channel, if any */
1366         dwc_otg_host_channel_free(sc, td);
1367
1368         if (td->hcsplt != 0) {
1369                 delta = td->tt_complete_slot - sc->sc_last_frame_num - 1;
1370                 if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
1371                         td->state = DWC_CHAN_ST_WAIT_C_PKT;
1372                         goto busy;
1373                 }
1374                 delta = sc->sc_last_frame_num - td->tt_start_slot;
1375                 if (delta > DWC_OTG_TT_SLOT_MAX) {
1376                         if (td->ep_type != UE_ISOCHRONOUS) {
1377                                 /* we missed the service interval */
1378                                 td->error_any = 1;
1379                         }
1380                         goto complete;
1381                 }
1382                 /* complete split */
1383                 td->hcsplt |= HCSPLT_COMPSPLT;
1384         } else if (td->tt_xactpos == HCSPLT_XACTPOS_BEGIN &&
1385             dwc_otg_host_rate_check(sc, td)) {
1386                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
1387                 goto busy;
1388         }
1389
1390         /* allocate a new channel */
1391         if (dwc_otg_host_channel_alloc(sc, td, 0)) {
1392                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
1393                 goto busy;
1394         }
1395
1396         channel = td->channel;
1397
1398         /* set toggle, if any */
1399         if (td->set_toggle) {
1400                 td->set_toggle = 0;
1401                 td->toggle = 1;
1402         }
1403
1404         td->state = DWC_CHAN_ST_WAIT_ANE;
1405
1406         /* receive one packet */
1407         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1408             (td->max_packet_size << HCTSIZ_XFERSIZE_SHIFT) |
1409             (1 << HCTSIZ_PKTCNT_SHIFT) |
1410             (td->toggle ? (HCTSIZ_PID_DATA1 << HCTSIZ_PID_SHIFT) :
1411             (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT)));
1412
1413         DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(channel), td->hcsplt);
1414
1415         hcchar = td->hcchar;
1416         hcchar |= HCCHAR_EPDIR_IN;
1417
1418         /* receive complete split ASAP */
1419         if ((sc->sc_last_frame_num & 1) != 0)
1420                 hcchar |= HCCHAR_ODDFRM;
1421         else
1422                 hcchar &= ~HCCHAR_ODDFRM;
1423
1424         /* must enable channel before data can be received */
1425         DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(channel), hcchar);
1426
1427         /* wait until next slot before trying complete split */
1428         td->tt_complete_slot = sc->sc_last_frame_num + 1;
1429
1430         goto busy;
1431
1432 receive_spkt:
1433         /* free existing channel(s), if any */
1434         dwc_otg_host_channel_free(sc, td);
1435
1436         delta = td->tt_start_slot - sc->sc_last_frame_num - 1;
1437         if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
1438                 td->state = DWC_CHAN_ST_START;
1439                 goto busy;
1440         }
1441         delta = sc->sc_last_frame_num - td->tt_start_slot;
1442         if (delta > 5) {
1443                 /* missed it */
1444                 td->tt_scheduled = 0;
1445                 td->state = DWC_CHAN_ST_START;
1446                 goto busy;
1447         }
1448
1449         /* allocate a new channel */
1450         if (dwc_otg_host_channel_alloc(sc, td, 0)) {
1451                 td->state = DWC_CHAN_ST_START;
1452                 goto busy;
1453         }
1454
1455         channel = td->channel;
1456
1457         td->hcsplt &= ~HCSPLT_COMPSPLT;
1458         td->state = DWC_CHAN_ST_WAIT_S_ANE;
1459
1460         /* receive one packet */
1461         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1462             (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT));
1463
1464         DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(channel), td->hcsplt);
1465
1466         /* send after next SOF event */
1467         if ((sc->sc_last_frame_num & 1) == 0)
1468                 td->hcchar |= HCCHAR_ODDFRM;
1469         else
1470                 td->hcchar &= ~HCCHAR_ODDFRM;
1471
1472         hcchar = td->hcchar;
1473         hcchar |= HCCHAR_EPDIR_IN;
1474
1475         /* wait until next slot before trying complete split */
1476         td->tt_complete_slot = sc->sc_last_frame_num + 1;
1477
1478         /* must enable channel before data can be received */
1479         DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(channel), hcchar);
1480 busy:
1481         return (1);     /* busy */
1482
1483 complete:
1484         dwc_otg_host_channel_free(sc, td);
1485         return (0);     /* complete */
1486 }
1487
1488 static uint8_t
1489 dwc_otg_data_rx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1490 {
1491         uint32_t temp;
1492         uint16_t count;
1493         uint8_t got_short;
1494
1495         got_short = 0;
1496
1497         /* check endpoint status */
1498         if (sc->sc_last_rx_status == 0)
1499                 goto not_complete;
1500
1501         if (GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) != td->ep_no)
1502                 goto not_complete;
1503
1504         /* check for SETUP packet */
1505         if ((sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) ==
1506             GRXSTSRD_STP_DATA) {
1507                 if (td->remainder == 0) {
1508                         /*
1509                          * We are actually complete and have
1510                          * received the next SETUP
1511                          */
1512                         DPRINTFN(5, "faking complete\n");
1513                         return (0);     /* complete */
1514                 }
1515                 /*
1516                  * USB Host Aborted the transfer.
1517                  */
1518                 td->error_any = 1;
1519                 return (0);             /* complete */
1520         }
1521
1522         if ((sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) !=
1523             GRXSTSRD_OUT_DATA) {
1524                 /* release FIFO */
1525                 dwc_otg_common_rx_ack(sc);
1526                 goto not_complete;
1527         }
1528
1529         /* get the packet byte count */
1530         count = GRXSTSRD_BCNT_GET(sc->sc_last_rx_status);
1531
1532         /* verify the packet byte count */
1533         if (count != td->max_packet_size) {
1534                 if (count < td->max_packet_size) {
1535                         /* we have a short packet */
1536                         td->short_pkt = 1;
1537                         got_short = 1;
1538                 } else {
1539                         /* invalid USB packet */
1540                         td->error_any = 1;
1541
1542                         /* release FIFO */
1543                         dwc_otg_common_rx_ack(sc);
1544                         return (0);     /* we are complete */
1545                 }
1546         }
1547         /* verify the packet byte count */
1548         if (count > td->remainder) {
1549                 /* invalid USB packet */
1550                 td->error_any = 1;
1551
1552                 /* release FIFO */
1553                 dwc_otg_common_rx_ack(sc);
1554                 return (0);             /* we are complete */
1555         }
1556
1557         usbd_copy_in(td->pc, td->offset, sc->sc_rx_bounce_buffer, count);
1558         td->remainder -= count;
1559         td->offset += count;
1560
1561         /* release FIFO */
1562         dwc_otg_common_rx_ack(sc);
1563
1564         temp = sc->sc_out_ctl[td->ep_no];
1565
1566         /* check for isochronous mode */
1567         if ((temp & DIEPCTL_EPTYPE_MASK) ==
1568             (DIEPCTL_EPTYPE_ISOC << DIEPCTL_EPTYPE_SHIFT)) {
1569                 /* toggle odd or even frame bit */
1570                 if (temp & DIEPCTL_SETD1PID) {
1571                         temp &= ~DIEPCTL_SETD1PID;
1572                         temp |= DIEPCTL_SETD0PID;
1573                 } else {
1574                         temp &= ~DIEPCTL_SETD0PID;
1575                         temp |= DIEPCTL_SETD1PID;
1576                 }
1577                 sc->sc_out_ctl[td->ep_no] = temp;
1578         }
1579
1580         /* check if we are complete */
1581         if ((td->remainder == 0) || got_short) {
1582                 if (td->short_pkt) {
1583                         /* we are complete */
1584                         return (0);
1585                 }
1586                 /* else need to receive a zero length packet */
1587         }
1588
1589 not_complete:
1590
1591         temp = sc->sc_out_ctl[td->ep_no];
1592
1593         temp |= DOEPCTL_EPENA | DOEPCTL_CNAK;
1594
1595         DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(td->ep_no), temp);
1596
1597         /* enable SETUP and transfer complete interrupt */
1598         if (td->ep_no == 0) {
1599                 DWC_OTG_WRITE_4(sc, DOTG_DOEPTSIZ(0),
1600                     DXEPTSIZ_SET_NPKT(1) |
1601                     DXEPTSIZ_SET_NBYTES(td->max_packet_size));
1602         } else {
1603                 /* allow reception of multiple packets */
1604                 DWC_OTG_WRITE_4(sc, DOTG_DOEPTSIZ(td->ep_no),
1605                     DXEPTSIZ_SET_MULTI(1) |
1606                     DXEPTSIZ_SET_NPKT(4) |
1607                     DXEPTSIZ_SET_NBYTES(4 *
1608                     ((td->max_packet_size + 3) & ~3)));
1609         }
1610         return (1);                     /* not complete */
1611 }
1612
1613 static uint8_t
1614 dwc_otg_host_data_tx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1615 {
1616         uint32_t count;
1617         uint32_t hcint;
1618         uint32_t hcchar;
1619         uint8_t delta;
1620         uint8_t channel;
1621
1622         dwc_otg_host_dump_rx(sc, td);
1623
1624         channel = td->channel;
1625
1626         if (channel < DWC_OTG_MAX_CHANNELS) {
1627                 hcint = sc->sc_chan_state[channel].hcint;
1628
1629                 DPRINTF("CH=%d ST=%d HCINT=0x%08x HCCHAR=0x%08x HCTSIZ=0x%08x\n",
1630                     channel, td->state, hcint,
1631                     DWC_OTG_READ_4(sc, DOTG_HCCHAR(channel)),
1632                     DWC_OTG_READ_4(sc, DOTG_HCTSIZ(channel)));
1633
1634                 if (hcint & (HCINT_RETRY |
1635                     HCINT_ACK | HCINT_NYET)) {
1636                         /* give success bits priority over failure bits */
1637                 } else if (hcint & HCINT_STALL) {
1638                         DPRINTF("CH=%d STALL\n", channel);
1639                         td->error_stall = 1;
1640                         td->error_any = 1;
1641                         goto complete;
1642                 } else if (hcint & HCINT_ERRORS) {
1643                         DPRINTF("CH=%d ERROR\n", channel);
1644                         td->errcnt++;
1645                         if (td->hcsplt != 0 || td->errcnt >= 3) {
1646                                 td->error_any = 1;
1647                                 goto complete;
1648                         }
1649                 }
1650
1651                 if (hcint & (HCINT_ERRORS | HCINT_RETRY |
1652                     HCINT_ACK | HCINT_NYET)) {
1653
1654                         if (!(hcint & HCINT_ERRORS))
1655                                 td->errcnt = 0;
1656                 }
1657         } else {
1658                 hcint = 0;
1659         }
1660
1661         switch (td->state) {
1662         case DWC_CHAN_ST_START:
1663                 goto send_pkt;
1664
1665         case DWC_CHAN_ST_WAIT_ANE:
1666                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1667                         td->did_nak++;
1668                         td->tt_scheduled = 0;
1669                         goto send_pkt;
1670                 } else if (hcint & (HCINT_ACK | HCINT_NYET)) {
1671                         td->offset += td->tx_bytes;
1672                         td->remainder -= td->tx_bytes;
1673                         td->toggle ^= 1;
1674                         td->did_nak = 0;
1675                         td->tt_scheduled = 0;
1676
1677                         /* check remainder */
1678                         if (td->remainder == 0) {
1679                                 if (td->short_pkt)
1680                                         goto complete;
1681
1682                                 /*
1683                                  * Else we need to transmit a short
1684                                  * packet:
1685                                  */
1686                         }
1687                         goto send_pkt;
1688                 }
1689                 break;
1690
1691         case DWC_CHAN_ST_WAIT_S_ANE:
1692                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1693                         td->did_nak++;
1694                         td->tt_scheduled = 0;
1695                         goto send_pkt;
1696                 } else if (hcint & (HCINT_ACK | HCINT_NYET)) {
1697                         td->did_nak = 0;
1698                         goto send_cpkt;
1699                 }
1700                 break;
1701
1702         case DWC_CHAN_ST_WAIT_C_ANE:
1703                 if (hcint & HCINT_NYET) {
1704                         goto send_cpkt;
1705                 } else if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1706                         td->did_nak++;
1707                         td->tt_scheduled = 0;
1708                         goto send_pkt;
1709                 } else if (hcint & HCINT_ACK) {
1710                         td->offset += td->tx_bytes;
1711                         td->remainder -= td->tx_bytes;
1712                         td->toggle ^= 1;
1713                         td->did_nak = 0;
1714                         td->tt_scheduled = 0;
1715
1716                         /* check remainder */
1717                         if (td->remainder == 0) {
1718                                 if (td->short_pkt)
1719                                         goto complete;
1720
1721                                 /* else we need to transmit a short packet */
1722                         }
1723                         goto send_pkt;
1724                 }
1725                 break;
1726
1727         case DWC_CHAN_ST_WAIT_C_PKT:
1728                 goto send_cpkt;
1729
1730         case DWC_CHAN_ST_TX_WAIT_ISOC:
1731
1732                 /* Check if isochronous OUT traffic is complete */
1733                 if ((hcint & HCINT_HCH_DONE_MASK) == 0)
1734                         break;
1735
1736                 td->offset += td->tx_bytes;
1737                 td->remainder -= td->tx_bytes;
1738
1739                 if (td->hcsplt != 0 || td->remainder == 0)
1740                         goto complete;
1741
1742                 /* check for next packet */
1743                 if (td->max_packet_count > 1)
1744                         td->tt_xactpos++;
1745
1746                 /* free existing channel, if any */
1747                 dwc_otg_host_channel_free(sc, td);
1748
1749                 td->state = DWC_CHAN_ST_TX_PKT_ISOC;
1750
1751                 /* FALLTHROUGH */
1752
1753         case DWC_CHAN_ST_TX_PKT_ISOC:
1754                 if (dwc_otg_host_channel_alloc(sc, td, 1))
1755                         break;
1756                 channel = td->channel;
1757                 goto send_isoc_pkt;
1758         default:
1759                 break;
1760         }
1761         goto busy;
1762
1763 send_pkt:
1764         /* free existing channel(s), if any */
1765         dwc_otg_host_channel_free(sc, td);
1766
1767         if (td->hcsplt != 0) {
1768                 delta = td->tt_start_slot - sc->sc_last_frame_num - 1;
1769                 if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
1770                         td->state = DWC_CHAN_ST_START;
1771                         goto busy;
1772                 }
1773                 delta = sc->sc_last_frame_num - td->tt_start_slot;
1774                 if (delta > 5) {
1775                         /* missed it */
1776                         td->tt_scheduled = 0;
1777                         td->state = DWC_CHAN_ST_START;
1778                         goto busy;
1779                 }
1780         } else if (dwc_otg_host_rate_check(sc, td)) {
1781                 td->state = DWC_CHAN_ST_START;
1782                 goto busy;
1783         }
1784
1785         /* allocate a new channel */
1786         if (dwc_otg_host_channel_alloc(sc, td, 1)) {
1787                 td->state = DWC_CHAN_ST_START;
1788                 goto busy;
1789         }
1790
1791         channel = td->channel;
1792
1793         /* set toggle, if any */
1794         if (td->set_toggle) {
1795                 td->set_toggle = 0;
1796                 td->toggle = 1;
1797         }
1798
1799         if (td->ep_type == UE_ISOCHRONOUS) {
1800 send_isoc_pkt:
1801                 /* Isochronous OUT transfers don't have any ACKs */
1802                 td->state = DWC_CHAN_ST_TX_WAIT_ISOC;
1803                 td->hcsplt &= ~HCSPLT_COMPSPLT;
1804                 if (td->hcsplt != 0) {
1805                         /* get maximum transfer length */
1806                         count = td->remainder;
1807                         if (count > HCSPLT_XACTLEN_BURST) {
1808                                 DPRINTF("TT overflow\n");
1809                                 td->error_any = 1;
1810                                 goto complete;
1811                         }
1812                         /* Update transaction position */
1813                         td->hcsplt &= ~HCSPLT_XACTPOS_MASK;
1814                         td->hcsplt |= (HCSPLT_XACTPOS_ALL << HCSPLT_XACTPOS_SHIFT);
1815                 } else {
1816                         /* send one packet at a time */
1817                         count = td->max_packet_size;
1818                         if (td->remainder < count) {
1819                                 /* we have a short packet */
1820                                 td->short_pkt = 1;
1821                                 count = td->remainder;
1822                         }
1823                 }
1824         } else if (td->hcsplt != 0) {
1825
1826                 td->hcsplt &= ~HCSPLT_COMPSPLT;
1827
1828                 /* Wait for ACK/NAK/ERR from TT */
1829                 td->state = DWC_CHAN_ST_WAIT_S_ANE;
1830
1831                 /* send one packet at a time */
1832                 count = td->max_packet_size;
1833                 if (td->remainder < count) {
1834                         /* we have a short packet */
1835                         td->short_pkt = 1;
1836                         count = td->remainder;
1837                 }
1838         } else {
1839                 /* Wait for ACK/NAK/STALL from device */
1840                 td->state = DWC_CHAN_ST_WAIT_ANE;
1841
1842                 /* send one packet at a time */
1843                 count = td->max_packet_size;
1844                 if (td->remainder < count) {
1845                         /* we have a short packet */
1846                         td->short_pkt = 1;
1847                         count = td->remainder;
1848                 }
1849         }
1850
1851         /* check for High-Speed multi-packets */
1852         if ((td->hcsplt == 0) && (td->max_packet_count > 1)) {
1853                 if (td->npkt == 0) {
1854                         if (td->remainder >= (3 * td->max_packet_size))
1855                                 td->npkt = 3;
1856                         else if (td->remainder >= (2 * td->max_packet_size))
1857                                 td->npkt = 2;
1858                         else
1859                                 td->npkt = 1;
1860
1861                         if (td->npkt > td->max_packet_count)
1862                                 td->npkt = td->max_packet_count;
1863
1864                         td->tt_xactpos = 1;     /* overload */
1865                 }
1866                 if (td->tt_xactpos == td->npkt) {
1867                         if (td->npkt == 1) {
1868                                 DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1869                                     (count << HCTSIZ_XFERSIZE_SHIFT) |
1870                                     (1 << HCTSIZ_PKTCNT_SHIFT) |
1871                                     (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT));
1872                         } else if (td->npkt == 2) {
1873                                 DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1874                                     (count << HCTSIZ_XFERSIZE_SHIFT) |
1875                                     (1 << HCTSIZ_PKTCNT_SHIFT) |
1876                                     (HCTSIZ_PID_DATA1 << HCTSIZ_PID_SHIFT));
1877                         } else {
1878                                 DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1879                                     (count << HCTSIZ_XFERSIZE_SHIFT) |
1880                                     (1 << HCTSIZ_PKTCNT_SHIFT) |
1881                                     (HCTSIZ_PID_DATA2 << HCTSIZ_PID_SHIFT));
1882                         }
1883                         td->npkt = 0;
1884                 } else {
1885                         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1886                             (count << HCTSIZ_XFERSIZE_SHIFT) |
1887                             (1 << HCTSIZ_PKTCNT_SHIFT) |
1888                             (HCTSIZ_PID_MDATA << HCTSIZ_PID_SHIFT));
1889                 }
1890         } else {
1891                 /* TODO: HCTSIZ_DOPNG */
1892
1893                 DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1894                     (count << HCTSIZ_XFERSIZE_SHIFT) |
1895                     (1 << HCTSIZ_PKTCNT_SHIFT) |
1896                     (td->toggle ? (HCTSIZ_PID_DATA1 << HCTSIZ_PID_SHIFT) :
1897                     (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT)));
1898         }
1899
1900         DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(channel), td->hcsplt);
1901
1902         hcchar = td->hcchar;
1903         hcchar &= ~HCCHAR_EPDIR_IN;
1904
1905         /* send after next SOF event */
1906         if ((sc->sc_last_frame_num & 1) == 0)
1907                 hcchar |= HCCHAR_ODDFRM;
1908         else
1909                 hcchar &= ~HCCHAR_ODDFRM;
1910
1911         /* must enable before writing data to FIFO */
1912         DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(channel), hcchar);
1913
1914         if (count != 0) {
1915
1916                 /* clear topmost word before copy */
1917                 sc->sc_tx_bounce_buffer[(count - 1) / 4] = 0;
1918
1919                 /* copy out data */
1920                 usbd_copy_out(td->pc, td->offset,
1921                     sc->sc_tx_bounce_buffer, count);
1922
1923                 /* transfer data into FIFO */
1924                 bus_space_write_region_4(sc->sc_io_tag, sc->sc_io_hdl,
1925                     DOTG_DFIFO(channel),
1926                     sc->sc_tx_bounce_buffer, (count + 3) / 4);
1927         }
1928
1929         /* store number of bytes transmitted */
1930         td->tx_bytes = count;
1931         goto busy;
1932
1933 send_cpkt:
1934         /* free existing channel, if any */
1935         dwc_otg_host_channel_free(sc, td);
1936
1937         delta = td->tt_complete_slot - sc->sc_last_frame_num - 1;
1938         if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
1939                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
1940                 goto busy;
1941         }
1942         delta = sc->sc_last_frame_num - td->tt_start_slot;
1943         if (delta > DWC_OTG_TT_SLOT_MAX) {
1944                 /* we missed the service interval */
1945                 if (td->ep_type != UE_ISOCHRONOUS)
1946                         td->error_any = 1;
1947                 goto complete;
1948         }
1949
1950         /* allocate a new channel */
1951         if (dwc_otg_host_channel_alloc(sc, td, 0)) {
1952                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
1953                 goto busy;
1954         }
1955
1956         channel = td->channel;
1957
1958         td->hcsplt |= HCSPLT_COMPSPLT;
1959         td->state = DWC_CHAN_ST_WAIT_C_ANE;
1960
1961         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1962             (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT));
1963
1964         DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(channel), td->hcsplt);
1965
1966         hcchar = td->hcchar;
1967         hcchar &= ~HCCHAR_EPDIR_IN;
1968
1969         /* receive complete split ASAP */
1970         if ((sc->sc_last_frame_num & 1) != 0)
1971                 hcchar |= HCCHAR_ODDFRM;
1972         else
1973                 hcchar &= ~HCCHAR_ODDFRM;
1974
1975         /* must enable channel before data can be received */
1976         DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(channel), hcchar);
1977
1978         /* wait until next slot before trying complete split */
1979         td->tt_complete_slot = sc->sc_last_frame_num + 1;
1980 busy:
1981         return (1);     /* busy */
1982
1983 complete:
1984         dwc_otg_host_channel_free(sc, td);
1985         return (0);     /* complete */
1986 }
1987
1988 static uint8_t
1989 dwc_otg_data_tx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1990 {
1991         uint32_t max_buffer;
1992         uint32_t count;
1993         uint32_t fifo_left;
1994         uint32_t mpkt;
1995         uint32_t temp;
1996         uint8_t to;
1997
1998         to = 3;                         /* don't loop forever! */
1999
2000         max_buffer = sc->sc_hw_ep_profile[td->ep_no].max_buffer;
2001
2002 repeat:
2003         /* check for for endpoint 0 data */
2004
2005         temp = sc->sc_last_rx_status;
2006
2007         if ((td->ep_no == 0) && (temp != 0) &&
2008             (GRXSTSRD_CHNUM_GET(temp) == 0)) {
2009
2010                 if ((temp & GRXSTSRD_PKTSTS_MASK) !=
2011                     GRXSTSRD_STP_DATA) {
2012
2013                         /* dump data - wrong direction */
2014                         dwc_otg_common_rx_ack(sc);
2015                 } else {
2016                         /*
2017                          * The current transfer was cancelled
2018                          * by the USB Host:
2019                          */
2020                         td->error_any = 1;
2021                         return (0);             /* complete */
2022                 }
2023         }
2024
2025         /* fill in more TX data, if possible */
2026         if (td->tx_bytes != 0) {
2027
2028                 uint16_t cpkt;
2029
2030                 /* check if packets have been transferred */
2031                 temp = DWC_OTG_READ_4(sc, DOTG_DIEPTSIZ(td->ep_no));
2032
2033                 /* get current packet number */
2034                 cpkt = DXEPTSIZ_GET_NPKT(temp);
2035
2036                 if (cpkt >= td->npkt) {
2037                         fifo_left = 0;
2038                 } else {
2039                         if (max_buffer != 0) {
2040                                 fifo_left = (td->npkt - cpkt) *
2041                                     td->max_packet_size;
2042
2043                                 if (fifo_left > max_buffer)
2044                                         fifo_left = max_buffer;
2045                         } else {
2046                                 fifo_left = td->max_packet_size;
2047                         }
2048                 }
2049
2050                 count = td->tx_bytes;
2051                 if (count > fifo_left)
2052                         count = fifo_left;
2053
2054                 if (count != 0) {
2055
2056                         /* clear topmost word before copy */
2057                         sc->sc_tx_bounce_buffer[(count - 1) / 4] = 0;
2058
2059                         /* copy out data */
2060                         usbd_copy_out(td->pc, td->offset,
2061                             sc->sc_tx_bounce_buffer, count);
2062
2063                         /* transfer data into FIFO */
2064                         bus_space_write_region_4(sc->sc_io_tag, sc->sc_io_hdl,
2065                             DOTG_DFIFO(td->ep_no),
2066                             sc->sc_tx_bounce_buffer, (count + 3) / 4);
2067
2068                         td->tx_bytes -= count;
2069                         td->remainder -= count;
2070                         td->offset += count;
2071                         td->npkt = cpkt;
2072                 }
2073                 if (td->tx_bytes != 0)
2074                         goto not_complete;
2075
2076                 /* check remainder */
2077                 if (td->remainder == 0) {
2078                         if (td->short_pkt)
2079                                 return (0);     /* complete */
2080
2081                         /* else we need to transmit a short packet */
2082                 }
2083         }
2084
2085         if (!to--)
2086                 goto not_complete;
2087
2088         /* check if not all packets have been transferred */
2089         temp = DWC_OTG_READ_4(sc, DOTG_DIEPTSIZ(td->ep_no));
2090
2091         if (DXEPTSIZ_GET_NPKT(temp) != 0) {
2092
2093                 DPRINTFN(5, "busy ep=%d npkt=%d DIEPTSIZ=0x%08x "
2094                     "DIEPCTL=0x%08x\n", td->ep_no,
2095                     DXEPTSIZ_GET_NPKT(temp),
2096                     temp, DWC_OTG_READ_4(sc, DOTG_DIEPCTL(td->ep_no)));
2097
2098                 goto not_complete;
2099         }
2100
2101         DPRINTFN(5, "rem=%u ep=%d\n", td->remainder, td->ep_no);
2102
2103         /* try to optimise by sending more data */
2104         if ((max_buffer != 0) && ((td->max_packet_size & 3) == 0)) {
2105
2106                 /* send multiple packets at the same time */
2107                 mpkt = max_buffer / td->max_packet_size;
2108
2109                 if (mpkt > 0x3FE)
2110                         mpkt = 0x3FE;
2111
2112                 count = td->remainder;
2113                 if (count > 0x7FFFFF)
2114                         count = 0x7FFFFF - (0x7FFFFF % td->max_packet_size);
2115
2116                 td->npkt = count / td->max_packet_size;
2117
2118                 /*
2119                  * NOTE: We could use 0x3FE instead of "mpkt" in the
2120                  * check below to get more throughput, but then we
2121                  * have a dependency towards non-generic chip features
2122                  * to disable the TX-FIFO-EMPTY interrupts on a per
2123                  * endpoint basis. Increase the maximum buffer size of
2124                  * the IN endpoint to increase the performance.
2125                  */
2126                 if (td->npkt > mpkt) {
2127                         td->npkt = mpkt;
2128                         count = td->max_packet_size * mpkt;
2129                 } else if ((count == 0) || (count % td->max_packet_size)) {
2130                         /* we are transmitting a short packet */
2131                         td->npkt++;
2132                         td->short_pkt = 1;
2133                 }
2134         } else {
2135                 /* send one packet at a time */
2136                 mpkt = 1;
2137                 count = td->max_packet_size;
2138                 if (td->remainder < count) {
2139                         /* we have a short packet */
2140                         td->short_pkt = 1;
2141                         count = td->remainder;
2142                 }
2143                 td->npkt = 1;
2144         }
2145         DWC_OTG_WRITE_4(sc, DOTG_DIEPTSIZ(td->ep_no),
2146             DXEPTSIZ_SET_MULTI(1) |
2147             DXEPTSIZ_SET_NPKT(td->npkt) |
2148             DXEPTSIZ_SET_NBYTES(count));
2149
2150         /* make room for buffering */
2151         td->npkt += mpkt;
2152
2153         temp = sc->sc_in_ctl[td->ep_no];
2154
2155         /* check for isochronous mode */
2156         if ((temp & DIEPCTL_EPTYPE_MASK) ==
2157             (DIEPCTL_EPTYPE_ISOC << DIEPCTL_EPTYPE_SHIFT)) {
2158                 /* toggle odd or even frame bit */
2159                 if (temp & DIEPCTL_SETD1PID) {
2160                         temp &= ~DIEPCTL_SETD1PID;
2161                         temp |= DIEPCTL_SETD0PID;
2162                 } else {
2163                         temp &= ~DIEPCTL_SETD0PID;
2164                         temp |= DIEPCTL_SETD1PID;
2165                 }
2166                 sc->sc_in_ctl[td->ep_no] = temp;
2167         }
2168
2169         /* must enable before writing data to FIFO */
2170         DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(td->ep_no), temp |
2171             DIEPCTL_EPENA | DIEPCTL_CNAK);
2172
2173         td->tx_bytes = count;
2174
2175         /* check remainder */
2176         if (td->tx_bytes == 0 &&
2177             td->remainder == 0) {
2178                 if (td->short_pkt)
2179                         return (0);     /* complete */
2180
2181                 /* else we need to transmit a short packet */
2182         }
2183         goto repeat;
2184
2185 not_complete:
2186         return (1);                     /* not complete */
2187 }
2188
2189 static uint8_t
2190 dwc_otg_data_tx_sync(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
2191 {
2192         uint32_t temp;
2193
2194         /*
2195          * If all packets are transferred we are complete:
2196          */
2197         temp = DWC_OTG_READ_4(sc, DOTG_DIEPTSIZ(td->ep_no));
2198
2199         /* check that all packets have been transferred */
2200         if (DXEPTSIZ_GET_NPKT(temp) != 0) {
2201                 DPRINTFN(5, "busy ep=%d\n", td->ep_no);
2202                 goto not_complete;
2203         }
2204         return (0);
2205
2206 not_complete:
2207
2208         /* we only want to know if there is a SETUP packet or free IN packet */
2209
2210         temp = sc->sc_last_rx_status;
2211
2212         if ((td->ep_no == 0) && (temp != 0) &&
2213             (GRXSTSRD_CHNUM_GET(temp) == 0)) {
2214
2215                 if ((temp & GRXSTSRD_PKTSTS_MASK) ==
2216                     GRXSTSRD_STP_DATA) {
2217                         DPRINTFN(5, "faking complete\n");
2218                         /*
2219                          * Race condition: We are complete!
2220                          */
2221                         return (0);
2222                 } else {
2223                         /* dump data - wrong direction */
2224                         dwc_otg_common_rx_ack(sc);
2225                 }
2226         }
2227         return (1);                     /* not complete */
2228 }
2229
2230 static void
2231 dwc_otg_xfer_do_fifo(struct dwc_otg_softc *sc, struct usb_xfer *xfer)
2232 {
2233         struct dwc_otg_td *td;
2234         uint8_t toggle;
2235         uint8_t tmr_val;
2236         uint8_t tmr_res;
2237
2238         DPRINTFN(9, "\n");
2239
2240         td = xfer->td_transfer_cache;
2241         if (td == NULL)
2242                 return;
2243
2244         while (1) {
2245                 if ((td->func) (sc, td)) {
2246                         /* operation in progress */
2247                         break;
2248                 }
2249                 if (((void *)td) == xfer->td_transfer_last) {
2250                         goto done;
2251                 }
2252                 if (td->error_any) {
2253                         goto done;
2254                 } else if (td->remainder > 0) {
2255                         /*
2256                          * We had a short transfer. If there is no alternate
2257                          * next, stop processing !
2258                          */
2259                         if (!td->alt_next)
2260                                 goto done;
2261                 }
2262
2263                 /*
2264                  * Fetch the next transfer descriptor and transfer
2265                  * some flags to the next transfer descriptor
2266                  */
2267                 tmr_res = td->tmr_res;
2268                 tmr_val = td->tmr_val;
2269                 toggle = td->toggle;
2270                 td = td->obj_next;
2271                 xfer->td_transfer_cache = td;
2272                 td->toggle = toggle;    /* transfer toggle */
2273                 td->tmr_res = tmr_res;
2274                 td->tmr_val = tmr_val;
2275         }
2276         return;
2277
2278 done:
2279         xfer->td_transfer_cache = NULL;
2280         sc->sc_xfer_complete = 1;
2281 }
2282
2283 static uint8_t
2284 dwc_otg_xfer_do_complete_locked(struct dwc_otg_softc *sc, struct usb_xfer *xfer)
2285 {
2286         struct dwc_otg_td *td;
2287
2288         DPRINTFN(9, "\n");
2289
2290         td = xfer->td_transfer_cache;
2291         if (td == NULL) {
2292                 /* compute all actual lengths */
2293                 dwc_otg_standard_done(xfer);
2294                 return (1);
2295         }
2296         return (0);
2297 }
2298
2299 static void
2300 dwc_otg_timer(void *_sc)
2301 {
2302         struct dwc_otg_softc *sc = _sc;
2303         struct usb_xfer *xfer;
2304         struct dwc_otg_td *td;
2305
2306         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
2307
2308         DPRINTF("\n");
2309
2310         USB_BUS_SPIN_LOCK(&sc->sc_bus);
2311
2312         /* increment timer value */
2313         sc->sc_tmr_val++;
2314
2315         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
2316                 td = xfer->td_transfer_cache;
2317                 if (td != NULL) {
2318                         /* reset NAK counter */
2319                         td->did_nak = 0;
2320                 }
2321         }
2322
2323         /* enable SOF interrupt, which will poll jobs */
2324         dwc_otg_enable_sof_irq(sc);
2325
2326         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
2327
2328         if (sc->sc_timer_active) {
2329                 /* restart timer */
2330                 usb_callout_reset(&sc->sc_timer,
2331                     hz / (1000 / DWC_OTG_HOST_TIMER_RATE),
2332                     &dwc_otg_timer, sc);
2333         }
2334 }
2335
2336 static void
2337 dwc_otg_timer_start(struct dwc_otg_softc *sc)
2338 {
2339         if (sc->sc_timer_active != 0)
2340                 return;
2341
2342         sc->sc_timer_active = 1;
2343
2344         /* restart timer */
2345         usb_callout_reset(&sc->sc_timer,
2346             hz / (1000 / DWC_OTG_HOST_TIMER_RATE),
2347             &dwc_otg_timer, sc);
2348 }
2349
2350 static void
2351 dwc_otg_timer_stop(struct dwc_otg_softc *sc)
2352 {
2353         if (sc->sc_timer_active == 0)
2354                 return;
2355
2356         sc->sc_timer_active = 0;
2357
2358         /* stop timer */
2359         usb_callout_stop(&sc->sc_timer);
2360 }
2361
2362 static void
2363 dwc_otg_host_channel_disable(struct dwc_otg_softc *sc, uint8_t x)
2364 {
2365         uint32_t hcchar;
2366
2367         hcchar = DWC_OTG_READ_4(sc, DOTG_HCCHAR(x));
2368
2369         /* disable host channel, if any */
2370         if (hcchar & (HCCHAR_CHENA | HCCHAR_CHDIS)) {
2371                 /* disable channel */
2372                 DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(x),
2373                     HCCHAR_CHENA | HCCHAR_CHDIS);
2374                 /* wait for chip to get its brains in order */
2375                 sc->sc_chan_state[x].wait_sof = 2;
2376         }
2377
2378         /* release TX FIFO usage, if any */
2379         sc->sc_tx_cur_p_level -= sc->sc_chan_state[x].tx_p_size;
2380         sc->sc_tx_cur_np_level -= sc->sc_chan_state[x].tx_np_size;
2381
2382         /* don't release TX FIFO usage twice */
2383         sc->sc_chan_state[x].tx_p_size = 0;
2384         sc->sc_chan_state[x].tx_np_size = 0;
2385 }
2386
2387 static uint16_t
2388 dwc_otg_compute_isoc_rx_tt_slot(struct dwc_otg_tt_info *pinfo)
2389 {
2390         if (pinfo->slot_index < DWC_OTG_TT_SLOT_MAX)
2391                 pinfo->slot_index++;
2392         return (pinfo->slot_index);
2393 }
2394
2395 static uint8_t
2396 dwc_otg_update_host_transfer_schedule_locked(struct dwc_otg_softc *sc)
2397 {
2398         TAILQ_HEAD(, usb_xfer) head;
2399         struct usb_xfer *xfer;
2400         struct usb_xfer *xfer_next;
2401         struct dwc_otg_td *td;
2402         uint16_t temp;
2403         uint16_t slot;
2404         uint8_t x;
2405
2406         temp = DWC_OTG_READ_4(sc, DOTG_HFNUM) & DWC_OTG_FRAME_MASK;
2407
2408         if (sc->sc_last_frame_num == temp)
2409                 return (0);
2410
2411         sc->sc_last_frame_num = temp;
2412
2413         TAILQ_INIT(&head);
2414
2415         for (x = 0; x != sc->sc_host_ch_max; x++) {
2416                 if (sc->sc_chan_state[x].wait_sof == 0)
2417                         continue;
2418
2419                 sc->sc_needsof = 1;
2420                 if (--(sc->sc_chan_state[x].wait_sof) == 0)
2421                         dwc_otg_host_channel_disable(sc, x);
2422         }
2423
2424         if ((temp & 7) == 0) {
2425
2426                 /* reset the schedule */
2427                 memset(sc->sc_tt_info, 0, sizeof(sc->sc_tt_info));
2428
2429                 TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2430                         td = xfer->td_transfer_cache;
2431                         if (td == NULL || td->ep_type != UE_ISOCHRONOUS)
2432                                 continue;
2433
2434                         /* check for IN direction */
2435                         if ((td->hcchar & HCCHAR_EPDIR_IN) != 0)
2436                                 continue;
2437
2438                         /* execute more frames */
2439                         td->tmr_val = 0;
2440
2441                         sc->sc_needsof = 1;
2442
2443                         if (td->hcsplt == 0 || td->tt_scheduled != 0)
2444                                 continue;
2445
2446                         /* compute slot */
2447                         slot = dwc_otg_compute_isoc_rx_tt_slot(
2448                             sc->sc_tt_info + td->tt_index);
2449                         if (slot > 3) {
2450                                 /*
2451                                  * Not enough time to get complete
2452                                  * split executed.
2453                                  */
2454                                 continue;
2455                         }
2456                         /* Delayed start */
2457                         td->tt_start_slot = temp + slot;
2458                         td->tt_scheduled = 1;
2459                         TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2460                         TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2461                 }
2462
2463                 TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2464                         td = xfer->td_transfer_cache;
2465                         if (td == NULL || td->ep_type != UE_ISOCHRONOUS)
2466                                 continue;
2467
2468                         /* check for OUT direction */
2469                         if ((td->hcchar & HCCHAR_EPDIR_IN) == 0)
2470                                 continue;
2471
2472                         /* execute more frames */
2473                         td->tmr_val = 0;
2474
2475                         sc->sc_needsof = 1;
2476
2477                         if (td->hcsplt == 0 || td->tt_scheduled != 0)
2478                                 continue;
2479
2480                         /* Start ASAP */
2481                         td->tt_start_slot = temp;
2482                         td->tt_scheduled = 1;
2483                         TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2484                         TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2485                 }
2486
2487                 TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2488                         td = xfer->td_transfer_cache;
2489                         if (td == NULL || td->ep_type != UE_INTERRUPT)
2490                                 continue;
2491
2492                         if (td->tt_scheduled != 0) {
2493                                 sc->sc_needsof = 1;
2494                                 continue;
2495                         }
2496
2497                         if (dwc_otg_host_rate_check_interrupt(sc, td))
2498                                 continue;
2499
2500                         if (td->hcsplt == 0) {
2501                                 sc->sc_needsof = 1;
2502                                 td->tt_scheduled = 1;
2503                                 continue;
2504                         }
2505
2506                         /* start ASAP */
2507                         td->tt_start_slot = temp;
2508                         sc->sc_needsof = 1;
2509                         td->tt_scheduled = 1;
2510                         TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2511                         TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2512                 }
2513
2514                 TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2515                         td = xfer->td_transfer_cache;
2516                         if (td == NULL ||
2517                             td->ep_type != UE_CONTROL ||
2518                             td->did_nak >= DWC_OTG_NAK_MAX) {
2519                                 continue;
2520                         }
2521
2522                         sc->sc_needsof = 1;
2523
2524                         if (td->hcsplt == 0 || td->tt_scheduled != 0)
2525                                 continue;
2526
2527                         /* start ASAP */
2528                         td->tt_start_slot = temp;
2529                         td->tt_scheduled = 1;
2530                         TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2531                         TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2532                 }
2533         }
2534         if ((temp & 7) < 6) {
2535                 TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2536                         td = xfer->td_transfer_cache;
2537                         if (td == NULL ||
2538                             td->ep_type != UE_BULK ||
2539                             td->did_nak >= DWC_OTG_NAK_MAX) {
2540                                 continue;
2541                         }
2542
2543                         sc->sc_needsof = 1;
2544
2545                         if (td->hcsplt == 0 || td->tt_scheduled != 0)
2546                                 continue;
2547
2548                         /* start ASAP */
2549                         td->tt_start_slot = temp;
2550                         td->tt_scheduled = 1;
2551                         TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2552                         TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2553                 }
2554         }
2555
2556         /* Put TT transfers in execution order at the end */
2557         TAILQ_CONCAT(&sc->sc_bus.intr_q.head, &head, wait_entry);
2558
2559         /* move all TT transfers in front, keeping the current order */
2560         TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2561                 td = xfer->td_transfer_cache;
2562                 if (td == NULL || td->hcsplt == 0)
2563                         continue;
2564                 TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2565                 TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2566         }
2567         TAILQ_CONCAT(&head, &sc->sc_bus.intr_q.head, wait_entry);
2568         TAILQ_CONCAT(&sc->sc_bus.intr_q.head, &head, wait_entry);
2569
2570         /* put non-TT BULK transfers last */
2571         TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2572                 td = xfer->td_transfer_cache;
2573                 if (td == NULL || td->hcsplt != 0 || td->ep_type != UE_BULK)
2574                         continue;
2575                 TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2576                 TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2577         }
2578         TAILQ_CONCAT(&sc->sc_bus.intr_q.head, &head, wait_entry);
2579
2580         if ((temp & 7) == 0) {
2581
2582                 DPRINTFN(12, "SOF interrupt #%d, needsof=%d\n",
2583                     (int)temp, (int)sc->sc_needsof);
2584
2585                 /* update SOF IRQ mask */
2586                 if (sc->sc_irq_mask & GINTMSK_SOFMSK) {
2587                         if (sc->sc_needsof == 0) {
2588                                 sc->sc_irq_mask &= ~GINTMSK_SOFMSK;
2589                                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2590                         }
2591                 } else {
2592                         if (sc->sc_needsof != 0) {
2593                                 sc->sc_irq_mask |= GINTMSK_SOFMSK;
2594                                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2595                         }
2596                 }
2597
2598                 /* clear need SOF flag */
2599                 sc->sc_needsof = 0;
2600         }
2601         return (1);
2602 }
2603
2604 static void
2605 dwc_otg_interrupt_poll_locked(struct dwc_otg_softc *sc)
2606 {
2607         struct usb_xfer *xfer;
2608         uint32_t temp;
2609         uint8_t got_rx_status;
2610         uint8_t x;
2611
2612 repeat:
2613         /* get all channel interrupts */
2614         for (x = 0; x != sc->sc_host_ch_max; x++) {
2615                 temp = DWC_OTG_READ_4(sc, DOTG_HCINT(x));
2616                 if (temp != 0) {
2617                         DWC_OTG_WRITE_4(sc, DOTG_HCINT(x), temp);
2618                         temp &= ~HCINT_SOFTWARE_ONLY;
2619                         sc->sc_chan_state[x].hcint |= temp;
2620                 }
2621         }
2622
2623         if (sc->sc_last_rx_status == 0) {
2624
2625                 temp = DWC_OTG_READ_4(sc, DOTG_GINTSTS);
2626                 if (temp & GINTSTS_RXFLVL) {
2627                         /* pop current status */
2628                         sc->sc_last_rx_status =
2629                             DWC_OTG_READ_4(sc, DOTG_GRXSTSPD);
2630                 }
2631
2632                 if (sc->sc_last_rx_status != 0) {
2633
2634                         uint8_t ep_no;
2635
2636                         temp = sc->sc_last_rx_status &
2637                             GRXSTSRD_PKTSTS_MASK;
2638
2639                         /* non-data messages we simply skip */
2640                         if (temp != GRXSTSRD_STP_DATA &&
2641                             temp != GRXSTSRD_OUT_DATA) {
2642                                 dwc_otg_common_rx_ack(sc);
2643                                 goto repeat;
2644                         }
2645
2646                         temp = GRXSTSRD_BCNT_GET(
2647                             sc->sc_last_rx_status);
2648                         ep_no = GRXSTSRD_CHNUM_GET(
2649                             sc->sc_last_rx_status);
2650
2651                         /* receive data, if any */
2652                         if (temp != 0) {
2653                                 DPRINTF("Reading %d bytes from ep %d\n", temp, ep_no);
2654                                 bus_space_read_region_4(sc->sc_io_tag, sc->sc_io_hdl,
2655                                     DOTG_DFIFO(ep_no),
2656                                     sc->sc_rx_bounce_buffer, (temp + 3) / 4);
2657                         }
2658
2659                         /* check if we should dump the data */
2660                         if (!(sc->sc_active_rx_ep & (1U << ep_no))) {
2661                                 dwc_otg_common_rx_ack(sc);
2662                                 goto repeat;
2663                         }
2664
2665                         got_rx_status = 1;
2666
2667                         DPRINTFN(5, "RX status = 0x%08x: ch=%d pid=%d bytes=%d sts=%d\n",
2668                             sc->sc_last_rx_status, ep_no,
2669                             (sc->sc_last_rx_status >> 15) & 3,
2670                             GRXSTSRD_BCNT_GET(sc->sc_last_rx_status),
2671                             (sc->sc_last_rx_status >> 17) & 15);
2672                 } else {
2673                         got_rx_status = 0;
2674                 }
2675         } else {
2676                 uint8_t ep_no;
2677
2678                 ep_no = GRXSTSRD_CHNUM_GET(
2679                     sc->sc_last_rx_status);
2680
2681                 /* check if we should dump the data */
2682                 if (!(sc->sc_active_rx_ep & (1U << ep_no))) {
2683                         dwc_otg_common_rx_ack(sc);
2684                         goto repeat;
2685                 }
2686
2687                 got_rx_status = 1;
2688         }
2689
2690         /* execute FIFOs */
2691         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry)
2692                 dwc_otg_xfer_do_fifo(sc, xfer);
2693
2694         if (got_rx_status) {
2695                 /* check if data was consumed */
2696                 if (sc->sc_last_rx_status == 0)
2697                         goto repeat;
2698
2699                 /* disable RX FIFO level interrupt */
2700                 sc->sc_irq_mask &= ~GINTMSK_RXFLVLMSK;
2701                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2702         }
2703
2704         if (sc->sc_flags.status_device_mode == 0 && sc->sc_xfer_complete == 0) {
2705                 /* update host transfer schedule, so that new transfers can be issued */
2706                 if (dwc_otg_update_host_transfer_schedule_locked(sc))
2707                         goto repeat;
2708         }
2709 }
2710
2711 static void
2712 dwc_otg_interrupt_complete_locked(struct dwc_otg_softc *sc)
2713 {
2714         struct usb_xfer *xfer;
2715 repeat:
2716         /* scan for completion events */
2717         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
2718                 if (dwc_otg_xfer_do_complete_locked(sc, xfer))
2719                         goto repeat;
2720         }
2721 }
2722
2723 static void
2724 dwc_otg_vbus_interrupt(struct dwc_otg_softc *sc, uint8_t is_on)
2725 {
2726         DPRINTFN(5, "vbus = %u\n", is_on);
2727
2728         if (is_on) {
2729                 if (!sc->sc_flags.status_vbus) {
2730                         sc->sc_flags.status_vbus = 1;
2731
2732                         /* complete root HUB interrupt endpoint */
2733
2734                         dwc_otg_root_intr(sc);
2735                 }
2736         } else {
2737                 if (sc->sc_flags.status_vbus) {
2738                         sc->sc_flags.status_vbus = 0;
2739                         sc->sc_flags.status_bus_reset = 0;
2740                         sc->sc_flags.status_suspend = 0;
2741                         sc->sc_flags.change_suspend = 0;
2742                         sc->sc_flags.change_connect = 1;
2743
2744                         /* complete root HUB interrupt endpoint */
2745
2746                         dwc_otg_root_intr(sc);
2747                 }
2748         }
2749 }
2750
2751 int
2752 dwc_otg_filter_interrupt(void *arg)
2753 {
2754         struct dwc_otg_softc *sc = arg;
2755         int retval = FILTER_HANDLED;
2756         uint32_t status;
2757
2758         USB_BUS_SPIN_LOCK(&sc->sc_bus);
2759
2760         /* read and clear interrupt status */
2761         status = DWC_OTG_READ_4(sc, DOTG_GINTSTS);
2762
2763         /* clear interrupts we are handling here */
2764         DWC_OTG_WRITE_4(sc, DOTG_GINTSTS, status & ~DWC_OTG_MSK_GINT_THREAD_IRQ);
2765
2766         /* check for USB state change interrupts */
2767         if ((status & DWC_OTG_MSK_GINT_THREAD_IRQ) != 0)
2768                 retval = FILTER_SCHEDULE_THREAD;
2769
2770         /* clear all IN endpoint interrupts */
2771         if (status & GINTSTS_IEPINT) {
2772                 uint32_t temp;
2773                 uint8_t x;
2774
2775                 for (x = 0; x != sc->sc_dev_in_ep_max; x++) {
2776                         temp = DWC_OTG_READ_4(sc, DOTG_DIEPINT(x));
2777                         if (temp & DIEPMSK_XFERCOMPLMSK) {
2778                                 DWC_OTG_WRITE_4(sc, DOTG_DIEPINT(x),
2779                                     DIEPMSK_XFERCOMPLMSK);
2780                         }
2781                 }
2782         }
2783
2784         /* poll FIFOs, if any */
2785         dwc_otg_interrupt_poll_locked(sc);
2786
2787         if (sc->sc_xfer_complete != 0)
2788                 retval = FILTER_SCHEDULE_THREAD;
2789
2790         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
2791
2792         return (retval);
2793 }
2794
2795 void
2796 dwc_otg_interrupt(void *arg)
2797 {
2798         struct dwc_otg_softc *sc = arg;
2799         uint32_t status;
2800
2801         USB_BUS_LOCK(&sc->sc_bus);
2802         USB_BUS_SPIN_LOCK(&sc->sc_bus);
2803
2804         /* read and clear interrupt status */
2805         status = DWC_OTG_READ_4(sc, DOTG_GINTSTS);
2806
2807         /* clear interrupts we are handling here */
2808         DWC_OTG_WRITE_4(sc, DOTG_GINTSTS, status & DWC_OTG_MSK_GINT_THREAD_IRQ);
2809
2810         DPRINTFN(14, "GINTSTS=0x%08x HAINT=0x%08x HFNUM=0x%08x\n",
2811             status, DWC_OTG_READ_4(sc, DOTG_HAINT),
2812             DWC_OTG_READ_4(sc, DOTG_HFNUM));
2813
2814         if (status & GINTSTS_USBRST) {
2815
2816                 /* set correct state */
2817                 sc->sc_flags.status_device_mode = 1;
2818                 sc->sc_flags.status_bus_reset = 0;
2819                 sc->sc_flags.status_suspend = 0;
2820                 sc->sc_flags.change_suspend = 0;
2821                 sc->sc_flags.change_connect = 1;
2822
2823                 /* Disable SOF interrupt */
2824                 sc->sc_irq_mask &= ~GINTMSK_SOFMSK;
2825                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2826
2827                 /* complete root HUB interrupt endpoint */
2828                 dwc_otg_root_intr(sc);
2829         }
2830
2831         /* check for any bus state change interrupts */
2832         if (status & GINTSTS_ENUMDONE) {
2833
2834                 uint32_t temp;
2835
2836                 DPRINTFN(5, "end of reset\n");
2837
2838                 /* set correct state */
2839                 sc->sc_flags.status_device_mode = 1;
2840                 sc->sc_flags.status_bus_reset = 1;
2841                 sc->sc_flags.status_suspend = 0;
2842                 sc->sc_flags.change_suspend = 0;
2843                 sc->sc_flags.change_connect = 1;
2844                 sc->sc_flags.status_low_speed = 0;
2845                 sc->sc_flags.port_enabled = 1;
2846
2847                 /* reset FIFOs */
2848                 (void) dwc_otg_init_fifo(sc, DWC_MODE_DEVICE);
2849
2850                 /* reset function address */
2851                 dwc_otg_set_address(sc, 0);
2852
2853                 /* figure out enumeration speed */
2854                 temp = DWC_OTG_READ_4(sc, DOTG_DSTS);
2855                 if (DSTS_ENUMSPD_GET(temp) == DSTS_ENUMSPD_HI)
2856                         sc->sc_flags.status_high_speed = 1;
2857                 else
2858                         sc->sc_flags.status_high_speed = 0;
2859
2860                 /*
2861                  * Disable resume and SOF interrupt, and enable
2862                  * suspend and RX frame interrupt:
2863                  */
2864                 sc->sc_irq_mask &= ~(GINTMSK_WKUPINTMSK | GINTMSK_SOFMSK);
2865                 sc->sc_irq_mask |= GINTMSK_USBSUSPMSK;
2866                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2867
2868                 /* complete root HUB interrupt endpoint */
2869                 dwc_otg_root_intr(sc);
2870         }
2871
2872         if (status & GINTSTS_PRTINT) {
2873                 uint32_t hprt;
2874
2875                 hprt = DWC_OTG_READ_4(sc, DOTG_HPRT);
2876
2877                 /* clear change bits */
2878                 DWC_OTG_WRITE_4(sc, DOTG_HPRT, (hprt & (
2879                     HPRT_PRTPWR | HPRT_PRTENCHNG |
2880                     HPRT_PRTCONNDET | HPRT_PRTOVRCURRCHNG)) |
2881                     sc->sc_hprt_val);
2882
2883                 DPRINTFN(12, "GINTSTS=0x%08x, HPRT=0x%08x\n", status, hprt);
2884
2885                 sc->sc_flags.status_device_mode = 0;
2886
2887                 if (hprt & HPRT_PRTCONNSTS)
2888                         sc->sc_flags.status_bus_reset = 1;
2889                 else
2890                         sc->sc_flags.status_bus_reset = 0;
2891
2892                 if (hprt & HPRT_PRTENCHNG)
2893                         sc->sc_flags.change_enabled = 1;
2894
2895                 if (hprt & HPRT_PRTENA)
2896                         sc->sc_flags.port_enabled = 1;
2897                 else
2898                         sc->sc_flags.port_enabled = 0;
2899
2900                 if (hprt & HPRT_PRTOVRCURRCHNG)
2901                         sc->sc_flags.change_over_current = 1;
2902
2903                 if (hprt & HPRT_PRTOVRCURRACT)
2904                         sc->sc_flags.port_over_current = 1;
2905                 else
2906                         sc->sc_flags.port_over_current = 0;
2907
2908                 if (hprt & HPRT_PRTPWR)
2909                         sc->sc_flags.port_powered = 1;
2910                 else
2911                         sc->sc_flags.port_powered = 0;
2912
2913                 if (((hprt & HPRT_PRTSPD_MASK)
2914                     >> HPRT_PRTSPD_SHIFT) == HPRT_PRTSPD_LOW)
2915                         sc->sc_flags.status_low_speed = 1;
2916                 else
2917                         sc->sc_flags.status_low_speed = 0;
2918
2919                 if (((hprt & HPRT_PRTSPD_MASK)
2920                     >> HPRT_PRTSPD_SHIFT) == HPRT_PRTSPD_HIGH)
2921                         sc->sc_flags.status_high_speed = 1;
2922                 else
2923                         sc->sc_flags.status_high_speed = 0;
2924
2925                 if (hprt & HPRT_PRTCONNDET)
2926                         sc->sc_flags.change_connect = 1;
2927
2928                 if (hprt & HPRT_PRTSUSP)
2929                         dwc_otg_suspend_irq(sc);
2930                 else
2931                         dwc_otg_resume_irq(sc);
2932
2933                 /* complete root HUB interrupt endpoint */
2934                 dwc_otg_root_intr(sc);
2935
2936                 /* update host frame interval */
2937                 dwc_otg_update_host_frame_interval(sc);
2938         }
2939
2940         /*
2941          * If resume and suspend is set at the same time we interpret
2942          * that like RESUME. Resume is set when there is at least 3
2943          * milliseconds of inactivity on the USB BUS.
2944          */
2945         if (status & GINTSTS_WKUPINT) {
2946
2947                 DPRINTFN(5, "resume interrupt\n");
2948
2949                 dwc_otg_resume_irq(sc);
2950
2951         } else if (status & GINTSTS_USBSUSP) {
2952
2953                 DPRINTFN(5, "suspend interrupt\n");
2954
2955                 dwc_otg_suspend_irq(sc);
2956         }
2957         /* check VBUS */
2958         if (status & (GINTSTS_USBSUSP |
2959             GINTSTS_USBRST |
2960             GINTMSK_OTGINTMSK |
2961             GINTSTS_SESSREQINT)) {
2962                 uint32_t temp;
2963
2964                 temp = DWC_OTG_READ_4(sc, DOTG_GOTGCTL);
2965
2966                 DPRINTFN(5, "GOTGCTL=0x%08x\n", temp);
2967
2968                 dwc_otg_vbus_interrupt(sc,
2969                     (temp & (GOTGCTL_ASESVLD | GOTGCTL_BSESVLD)) ? 1 : 0);
2970         }
2971
2972         if (sc->sc_xfer_complete != 0) {
2973                 sc->sc_xfer_complete = 0;
2974
2975                 /* complete FIFOs, if any */
2976                 dwc_otg_interrupt_complete_locked(sc);
2977
2978                 if (sc->sc_flags.status_device_mode == 0) {
2979                         /* update host transfer schedule, so that new transfers can be issued */
2980                         if (dwc_otg_update_host_transfer_schedule_locked(sc))
2981                                 dwc_otg_interrupt_poll_locked(sc);
2982                 }
2983         }
2984         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
2985         USB_BUS_UNLOCK(&sc->sc_bus);
2986 }
2987
2988 static void
2989 dwc_otg_setup_standard_chain_sub(struct dwc_otg_std_temp *temp)
2990 {
2991         struct dwc_otg_td *td;
2992
2993         /* get current Transfer Descriptor */
2994         td = temp->td_next;
2995         temp->td = td;
2996
2997         /* prepare for next TD */
2998         temp->td_next = td->obj_next;
2999
3000         /* fill out the Transfer Descriptor */
3001         td->func = temp->func;
3002         td->pc = temp->pc;
3003         td->offset = temp->offset;
3004         td->remainder = temp->len;
3005         td->tx_bytes = 0;
3006         td->error_any = 0;
3007         td->error_stall = 0;
3008         td->npkt = 0;
3009         td->did_stall = temp->did_stall;
3010         td->short_pkt = temp->short_pkt;
3011         td->alt_next = temp->setup_alt_next;
3012         td->set_toggle = 0;
3013         td->got_short = 0;
3014         td->did_nak = 0;
3015         td->channel = DWC_OTG_MAX_CHANNELS;
3016         td->state = 0;
3017         td->errcnt = 0;
3018         td->tt_scheduled = 0;
3019         td->tt_xactpos = HCSPLT_XACTPOS_BEGIN;
3020 }
3021
3022 static void
3023 dwc_otg_setup_standard_chain(struct usb_xfer *xfer)
3024 {
3025         struct dwc_otg_std_temp temp;
3026         struct dwc_otg_td *td;
3027         uint32_t x;
3028         uint8_t need_sync;
3029         uint8_t is_host;
3030
3031         DPRINTFN(9, "addr=%d endpt=%d sumlen=%d speed=%d\n",
3032             xfer->address, UE_GET_ADDR(xfer->endpointno),
3033             xfer->sumlen, usbd_get_speed(xfer->xroot->udev));
3034
3035         temp.max_frame_size = xfer->max_frame_size;
3036
3037         td = xfer->td_start[0];
3038         xfer->td_transfer_first = td;
3039         xfer->td_transfer_cache = td;
3040
3041         /* setup temp */
3042
3043         temp.pc = NULL;
3044         temp.td = NULL;
3045         temp.td_next = xfer->td_start[0];
3046         temp.offset = 0;
3047         temp.setup_alt_next = xfer->flags_int.short_frames_ok ||
3048             xfer->flags_int.isochronous_xfr;
3049         temp.did_stall = !xfer->flags_int.control_stall;
3050
3051         is_host = (xfer->xroot->udev->flags.usb_mode == USB_MODE_HOST);
3052
3053         /* check if we should prepend a setup message */
3054
3055         if (xfer->flags_int.control_xfr) {
3056                 if (xfer->flags_int.control_hdr) {
3057
3058                         if (is_host)
3059                                 temp.func = &dwc_otg_host_setup_tx;
3060                         else
3061                                 temp.func = &dwc_otg_setup_rx;
3062
3063                         temp.len = xfer->frlengths[0];
3064                         temp.pc = xfer->frbuffers + 0;
3065                         temp.short_pkt = temp.len ? 1 : 0;
3066
3067                         /* check for last frame */
3068                         if (xfer->nframes == 1) {
3069                                 /* no STATUS stage yet, SETUP is last */
3070                                 if (xfer->flags_int.control_act)
3071                                         temp.setup_alt_next = 0;
3072                         }
3073
3074                         dwc_otg_setup_standard_chain_sub(&temp);
3075                 }
3076                 x = 1;
3077         } else {
3078                 x = 0;
3079         }
3080
3081         if (x != xfer->nframes) {
3082                 if (xfer->endpointno & UE_DIR_IN) {
3083                         if (is_host) {
3084                                 temp.func = &dwc_otg_host_data_rx;
3085                                 need_sync = 0;
3086                         } else {
3087                                 temp.func = &dwc_otg_data_tx;
3088                                 need_sync = 1;
3089                         }
3090                 } else {
3091                         if (is_host) {
3092                                 temp.func = &dwc_otg_host_data_tx;
3093                                 need_sync = 0;
3094                         } else {
3095                                 temp.func = &dwc_otg_data_rx;
3096                                 need_sync = 0;
3097                         }
3098                 }
3099
3100                 /* setup "pc" pointer */
3101                 temp.pc = xfer->frbuffers + x;
3102         } else {
3103                 need_sync = 0;
3104         }
3105         while (x != xfer->nframes) {
3106
3107                 /* DATA0 / DATA1 message */
3108
3109                 temp.len = xfer->frlengths[x];
3110
3111                 x++;
3112
3113                 if (x == xfer->nframes) {
3114                         if (xfer->flags_int.control_xfr) {
3115                                 if (xfer->flags_int.control_act) {
3116                                         temp.setup_alt_next = 0;
3117                                 }
3118                         } else {
3119                                 temp.setup_alt_next = 0;
3120                         }
3121                 }
3122                 if (temp.len == 0) {
3123
3124                         /* make sure that we send an USB packet */
3125
3126                         temp.short_pkt = 0;
3127
3128                 } else {
3129
3130                         /* regular data transfer */
3131
3132                         temp.short_pkt = (xfer->flags.force_short_xfer ? 0 : 1);
3133                 }
3134
3135                 dwc_otg_setup_standard_chain_sub(&temp);
3136
3137                 if (xfer->flags_int.isochronous_xfr) {
3138                         temp.offset += temp.len;
3139                 } else {
3140                         /* get next Page Cache pointer */
3141                         temp.pc = xfer->frbuffers + x;
3142                 }
3143         }
3144
3145         if (xfer->flags_int.control_xfr) {
3146
3147                 /* always setup a valid "pc" pointer for status and sync */
3148                 temp.pc = xfer->frbuffers + 0;
3149                 temp.len = 0;
3150                 temp.short_pkt = 0;
3151                 temp.setup_alt_next = 0;
3152
3153                 /* check if we need to sync */
3154                 if (need_sync) {
3155                         /* we need a SYNC point after TX */
3156                         temp.func = &dwc_otg_data_tx_sync;
3157                         dwc_otg_setup_standard_chain_sub(&temp);
3158                 }
3159
3160                 /* check if we should append a status stage */
3161                 if (!xfer->flags_int.control_act) {
3162
3163                         /*
3164                          * Send a DATA1 message and invert the current
3165                          * endpoint direction.
3166                          */
3167                         if (xfer->endpointno & UE_DIR_IN) {
3168                                 if (is_host) {
3169                                         temp.func = &dwc_otg_host_data_tx;
3170                                         need_sync = 0;
3171                                 } else {
3172                                         temp.func = &dwc_otg_data_rx;
3173                                         need_sync = 0;
3174                                 }
3175                         } else {
3176                                 if (is_host) {
3177                                         temp.func = &dwc_otg_host_data_rx;
3178                                         need_sync = 0;
3179                                 } else {
3180                                         temp.func = &dwc_otg_data_tx;
3181                                         need_sync = 1;
3182                                 }
3183                         }
3184
3185                         dwc_otg_setup_standard_chain_sub(&temp);
3186
3187                         /* data toggle should be DATA1 */
3188                         td = temp.td;
3189                         td->set_toggle = 1;
3190
3191                         if (need_sync) {
3192                                 /* we need a SYNC point after TX */
3193                                 temp.func = &dwc_otg_data_tx_sync;
3194                                 dwc_otg_setup_standard_chain_sub(&temp);
3195                         }
3196                 }
3197         } else {
3198                 /* check if we need to sync */
3199                 if (need_sync) {
3200
3201                         temp.pc = xfer->frbuffers + 0;
3202                         temp.len = 0;
3203                         temp.short_pkt = 0;
3204                         temp.setup_alt_next = 0;
3205
3206                         /* we need a SYNC point after TX */
3207                         temp.func = &dwc_otg_data_tx_sync;
3208                         dwc_otg_setup_standard_chain_sub(&temp);
3209                 }
3210         }
3211
3212         /* must have at least one frame! */
3213         td = temp.td;
3214         xfer->td_transfer_last = td;
3215
3216         if (is_host) {
3217
3218                 struct dwc_otg_softc *sc;
3219                 uint32_t hcchar;
3220                 uint32_t hcsplt;
3221
3222                 sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
3223
3224                 /* get first again */
3225                 td = xfer->td_transfer_first;
3226                 td->toggle = (xfer->endpoint->toggle_next ? 1 : 0);
3227
3228                 hcchar =
3229                         (xfer->address << HCCHAR_DEVADDR_SHIFT) |
3230                         ((xfer->endpointno & UE_ADDR) << HCCHAR_EPNUM_SHIFT) |
3231                         (xfer->max_packet_size << HCCHAR_MPS_SHIFT) |
3232                         HCCHAR_CHENA;
3233
3234                 /*
3235                  * We are not always able to meet the timing
3236                  * requirements of the USB interrupt endpoint's
3237                  * complete split token, when doing transfers going
3238                  * via a transaction translator. Use the CONTROL
3239                  * transfer type instead of the INTERRUPT transfer
3240                  * type in general, as a means to workaround
3241                  * that. This trick should work for both FULL and LOW
3242                  * speed USB traffic going through a TT. For non-TT
3243                  * traffic it works aswell. The reason for using
3244                  * CONTROL type instead of BULK is that some TTs might
3245                  * reject LOW speed BULK traffic.
3246                  */
3247                 if (td->ep_type == UE_INTERRUPT)
3248                         hcchar |= (UE_CONTROL << HCCHAR_EPTYPE_SHIFT);
3249                 else
3250                         hcchar |= (td->ep_type << HCCHAR_EPTYPE_SHIFT);
3251
3252                 if (usbd_get_speed(xfer->xroot->udev) == USB_SPEED_LOW)
3253                         hcchar |= HCCHAR_LSPDDEV;
3254                 if (UE_GET_DIR(xfer->endpointno) == UE_DIR_IN)
3255                         hcchar |= HCCHAR_EPDIR_IN;
3256
3257                 switch (xfer->xroot->udev->speed) {
3258                 case USB_SPEED_FULL:
3259                 case USB_SPEED_LOW:
3260                         /* check if root HUB port is running High Speed */
3261                         if (xfer->xroot->udev->parent_hs_hub != NULL) {
3262                                 hcsplt = HCSPLT_SPLTENA |
3263                                     (xfer->xroot->udev->hs_port_no <<
3264                                     HCSPLT_PRTADDR_SHIFT) |
3265                                     (xfer->xroot->udev->hs_hub_addr <<
3266                                     HCSPLT_HUBADDR_SHIFT);
3267                         } else {
3268                                 hcsplt = 0;
3269                         }
3270                         if (td->ep_type == UE_INTERRUPT) {
3271                                 uint32_t ival;
3272                                 ival = xfer->interval / DWC_OTG_HOST_TIMER_RATE;
3273                                 if (ival == 0)
3274                                         ival = 1;
3275                                 else if (ival > 127)
3276                                         ival = 127;
3277                                 td->tmr_val = sc->sc_tmr_val + ival;
3278                                 td->tmr_res = ival;
3279                         } else if (td->ep_type == UE_ISOCHRONOUS) {
3280                                 td->tmr_val = 0;
3281                                 td->tmr_res = 1;
3282                         } else {
3283                                 td->tmr_val = 0;
3284                                 td->tmr_res = 0;
3285                         }
3286                         break;
3287                 case USB_SPEED_HIGH:
3288                         hcsplt = 0;
3289                         if (td->ep_type == UE_INTERRUPT) {
3290                                 uint32_t ival;
3291 #if 0
3292                                 hcchar |= ((xfer->max_packet_count & 3)
3293                                     << HCCHAR_MC_SHIFT);
3294 #endif
3295                                 ival = xfer->interval / DWC_OTG_HOST_TIMER_RATE;
3296                                 if (ival == 0)
3297                                         ival = 1;
3298                                 else if (ival > 127)
3299                                         ival = 127;
3300                                 td->tmr_val = sc->sc_tmr_val + ival;
3301                                 td->tmr_res = ival;
3302                         } else if (td->ep_type == UE_ISOCHRONOUS) {
3303                                 hcchar |= ((xfer->max_packet_count & 3)
3304                                     << HCCHAR_MC_SHIFT);
3305                                 td->tmr_val = 0;
3306                                 td->tmr_res = 1 << usbd_xfer_get_fps_shift(xfer);
3307                         } else {
3308                                 td->tmr_val = 0;
3309                                 td->tmr_res = 0;
3310                         }
3311                         break;
3312                 default:
3313                         hcsplt = 0;
3314                         td->tmr_val = 0;
3315                         td->tmr_res = 0;
3316                         break;
3317                 }
3318
3319                 /* store configuration in all TD's */
3320                 while (1) {
3321                         td->hcchar = hcchar;
3322                         td->hcsplt = hcsplt;
3323
3324                         if (((void *)td) == xfer->td_transfer_last)
3325                                 break;
3326
3327                         td = td->obj_next;
3328                 }
3329         }
3330 }
3331
3332 static void
3333 dwc_otg_timeout(void *arg)
3334 {
3335         struct usb_xfer *xfer = arg;
3336
3337         DPRINTF("xfer=%p\n", xfer);
3338
3339         USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
3340
3341         /* transfer is transferred */
3342         dwc_otg_device_done(xfer, USB_ERR_TIMEOUT);
3343 }
3344
3345 static void
3346 dwc_otg_start_standard_chain(struct usb_xfer *xfer)
3347 {
3348         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
3349         struct usb_xfer_root *xroot;
3350         struct dwc_otg_td *td;
3351
3352         DPRINTFN(9, "\n");
3353
3354         /*
3355          * Poll one time in device mode, which will turn on the
3356          * endpoint interrupts. Else wait for SOF interrupt in host
3357          * mode.
3358          */
3359         USB_BUS_SPIN_LOCK(&sc->sc_bus);
3360
3361         if (sc->sc_flags.status_device_mode != 0) {
3362                 dwc_otg_xfer_do_fifo(sc, xfer);
3363                 if (dwc_otg_xfer_do_complete_locked(sc, xfer))
3364                         goto done;
3365         }
3366
3367         /* put transfer on interrupt queue */
3368         usbd_transfer_enqueue(&xfer->xroot->bus->intr_q, xfer);
3369
3370         /* start timeout, if any */
3371         if (xfer->timeout != 0) {
3372                 usbd_transfer_timeout_ms(xfer,
3373                     &dwc_otg_timeout, xfer->timeout);
3374         }
3375
3376         if (sc->sc_flags.status_device_mode != 0)
3377                 goto done;
3378
3379         /* enable SOF interrupt, if any */
3380         dwc_otg_enable_sof_irq(sc);
3381
3382         td = xfer->td_transfer_cache;
3383         if (td->ep_type != UE_BULK)
3384                 goto done;
3385
3386         xroot = xfer->xroot;
3387
3388         /*
3389          * Optimise the ping-pong effect by waking up other BULK
3390          * transfers belonging to the same device group:
3391          */
3392         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
3393                 td = xfer->td_transfer_cache;
3394                 if (td == NULL || td->ep_type != UE_BULK || xfer->xroot != xroot)
3395                         continue;
3396                 /* reset NAK counter */
3397                 td->did_nak = 0;
3398         }
3399 done:
3400         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3401 }
3402
3403 static void
3404 dwc_otg_root_intr(struct dwc_otg_softc *sc)
3405 {
3406         DPRINTFN(9, "\n");
3407
3408         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
3409
3410         /* set port bit */
3411         sc->sc_hub_idata[0] = 0x02;     /* we only have one port */
3412
3413         uhub_root_intr(&sc->sc_bus, sc->sc_hub_idata,
3414             sizeof(sc->sc_hub_idata));
3415 }
3416
3417 static usb_error_t
3418 dwc_otg_standard_done_sub(struct usb_xfer *xfer)
3419 {
3420         struct dwc_otg_td *td;
3421         uint32_t len;
3422         usb_error_t error;
3423
3424         DPRINTFN(9, "\n");
3425
3426         td = xfer->td_transfer_cache;
3427
3428         do {
3429                 len = td->remainder;
3430
3431                 /* store last data toggle */
3432                 xfer->endpoint->toggle_next = td->toggle;
3433
3434                 if (xfer->aframes != xfer->nframes) {
3435                         /*
3436                          * Verify the length and subtract
3437                          * the remainder from "frlengths[]":
3438                          */
3439                         if (len > xfer->frlengths[xfer->aframes]) {
3440                                 td->error_any = 1;
3441                         } else {
3442                                 xfer->frlengths[xfer->aframes] -= len;
3443                         }
3444                 }
3445                 /* Check for transfer error */
3446                 if (td->error_any) {
3447                         /* the transfer is finished */
3448                         error = (td->error_stall ?
3449                             USB_ERR_STALLED : USB_ERR_IOERROR);
3450                         td = NULL;
3451                         break;
3452                 }
3453                 /* Check for short transfer */
3454                 if (len > 0) {
3455                         if (xfer->flags_int.short_frames_ok ||
3456                             xfer->flags_int.isochronous_xfr) {
3457                                 /* follow alt next */
3458                                 if (td->alt_next) {
3459                                         td = td->obj_next;
3460                                 } else {
3461                                         td = NULL;
3462                                 }
3463                         } else {
3464                                 /* the transfer is finished */
3465                                 td = NULL;
3466                         }
3467                         error = 0;
3468                         break;
3469                 }
3470                 td = td->obj_next;
3471
3472                 /* this USB frame is complete */
3473                 error = 0;
3474                 break;
3475
3476         } while (0);
3477
3478         /* update transfer cache */
3479
3480         xfer->td_transfer_cache = td;
3481
3482         return (error);
3483 }
3484
3485 static void
3486 dwc_otg_standard_done(struct usb_xfer *xfer)
3487 {
3488         usb_error_t err = 0;
3489
3490         DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
3491             xfer, xfer->endpoint);
3492
3493         /* reset scanner */
3494
3495         xfer->td_transfer_cache = xfer->td_transfer_first;
3496
3497         if (xfer->flags_int.control_xfr) {
3498
3499                 if (xfer->flags_int.control_hdr) {
3500
3501                         err = dwc_otg_standard_done_sub(xfer);
3502                 }
3503                 xfer->aframes = 1;
3504
3505                 if (xfer->td_transfer_cache == NULL) {
3506                         goto done;
3507                 }
3508         }
3509         while (xfer->aframes != xfer->nframes) {
3510
3511                 err = dwc_otg_standard_done_sub(xfer);
3512                 xfer->aframes++;
3513
3514                 if (xfer->td_transfer_cache == NULL) {
3515                         goto done;
3516                 }
3517         }
3518
3519         if (xfer->flags_int.control_xfr &&
3520             !xfer->flags_int.control_act) {
3521
3522                 err = dwc_otg_standard_done_sub(xfer);
3523         }
3524 done:
3525         dwc_otg_device_done(xfer, err);
3526 }
3527
3528 /*------------------------------------------------------------------------*
3529  *      dwc_otg_device_done
3530  *
3531  * NOTE: this function can be called more than one time on the
3532  * same USB transfer!
3533  *------------------------------------------------------------------------*/
3534 static void
3535 dwc_otg_device_done(struct usb_xfer *xfer, usb_error_t error)
3536 {
3537         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
3538
3539         DPRINTFN(9, "xfer=%p, endpoint=%p, error=%d\n",
3540             xfer, xfer->endpoint, error);
3541
3542         USB_BUS_SPIN_LOCK(&sc->sc_bus);
3543
3544         if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) {
3545                 /* Interrupts are cleared by the interrupt handler */
3546         } else {
3547                 struct dwc_otg_td *td;
3548
3549                 td = xfer->td_transfer_cache;
3550                 if (td != NULL)
3551                         dwc_otg_host_channel_free(sc, td);
3552         }
3553         /* dequeue transfer and start next transfer */
3554         usbd_transfer_done(xfer, error);
3555
3556         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3557 }
3558
3559 static void
3560 dwc_otg_xfer_stall(struct usb_xfer *xfer)
3561 {
3562         dwc_otg_device_done(xfer, USB_ERR_STALLED);
3563 }
3564
3565 static void
3566 dwc_otg_set_stall(struct usb_device *udev,
3567     struct usb_endpoint *ep, uint8_t *did_stall)
3568 {
3569         struct dwc_otg_softc *sc;
3570         uint32_t temp;
3571         uint32_t reg;
3572         uint8_t ep_no;
3573
3574         USB_BUS_LOCK_ASSERT(udev->bus, MA_OWNED);
3575
3576         /* check mode */
3577         if (udev->flags.usb_mode != USB_MODE_DEVICE) {
3578                 /* not supported */
3579                 return;
3580         }
3581
3582         sc = DWC_OTG_BUS2SC(udev->bus);
3583
3584         USB_BUS_SPIN_LOCK(&sc->sc_bus);
3585
3586         /* get endpoint address */
3587         ep_no = ep->edesc->bEndpointAddress;
3588
3589         DPRINTFN(5, "endpoint=0x%x\n", ep_no);
3590
3591         if (ep_no & UE_DIR_IN) {
3592                 reg = DOTG_DIEPCTL(ep_no & UE_ADDR);
3593                 temp = sc->sc_in_ctl[ep_no & UE_ADDR];
3594         } else {
3595                 reg = DOTG_DOEPCTL(ep_no & UE_ADDR);
3596                 temp = sc->sc_out_ctl[ep_no & UE_ADDR];
3597         }
3598
3599         /* disable and stall endpoint */
3600         DWC_OTG_WRITE_4(sc, reg, temp | DOEPCTL_EPDIS);
3601         DWC_OTG_WRITE_4(sc, reg, temp | DOEPCTL_STALL);
3602
3603         /* clear active OUT ep */
3604         if (!(ep_no & UE_DIR_IN)) {
3605
3606                 sc->sc_active_rx_ep &= ~(1U << (ep_no & UE_ADDR));
3607
3608                 if (sc->sc_last_rx_status != 0 &&
3609                     (ep_no & UE_ADDR) == GRXSTSRD_CHNUM_GET(
3610                     sc->sc_last_rx_status)) {
3611                         /* dump data */
3612                         dwc_otg_common_rx_ack(sc);
3613                         /* poll interrupt */
3614                         dwc_otg_interrupt_poll_locked(sc);
3615                         dwc_otg_interrupt_complete_locked(sc);
3616                 }
3617         }
3618         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3619 }
3620
3621 static void
3622 dwc_otg_clear_stall_sub_locked(struct dwc_otg_softc *sc, uint32_t mps,
3623     uint8_t ep_no, uint8_t ep_type, uint8_t ep_dir)
3624 {
3625         uint32_t reg;
3626         uint32_t temp;
3627
3628         if (ep_type == UE_CONTROL) {
3629                 /* clearing stall is not needed */
3630                 return;
3631         }
3632
3633         if (ep_dir) {
3634                 reg = DOTG_DIEPCTL(ep_no);
3635         } else {
3636                 reg = DOTG_DOEPCTL(ep_no);
3637                 sc->sc_active_rx_ep |= (1U << ep_no);
3638         }
3639
3640         /* round up and mask away the multiplier count */
3641         mps = (mps + 3) & 0x7FC;
3642
3643         if (ep_type == UE_BULK) {
3644                 temp = DIEPCTL_EPTYPE_SET(
3645                     DIEPCTL_EPTYPE_BULK) |
3646                     DIEPCTL_USBACTEP;
3647         } else if (ep_type == UE_INTERRUPT) {
3648                 temp = DIEPCTL_EPTYPE_SET(
3649                     DIEPCTL_EPTYPE_INTERRUPT) |
3650                     DIEPCTL_USBACTEP;
3651         } else {
3652                 temp = DIEPCTL_EPTYPE_SET(
3653                     DIEPCTL_EPTYPE_ISOC) |
3654                     DIEPCTL_USBACTEP;
3655         }
3656
3657         temp |= DIEPCTL_MPS_SET(mps);
3658         temp |= DIEPCTL_TXFNUM_SET(ep_no);
3659
3660         if (ep_dir)
3661                 sc->sc_in_ctl[ep_no] = temp;
3662         else
3663                 sc->sc_out_ctl[ep_no] = temp;
3664
3665         DWC_OTG_WRITE_4(sc, reg, temp | DOEPCTL_EPDIS);
3666         DWC_OTG_WRITE_4(sc, reg, temp | DOEPCTL_SETD0PID);
3667         DWC_OTG_WRITE_4(sc, reg, temp | DIEPCTL_SNAK);
3668
3669         /* we only reset the transmit FIFO */
3670         if (ep_dir) {
3671                 DWC_OTG_WRITE_4(sc, DOTG_GRSTCTL,
3672                     GRSTCTL_TXFIFO(ep_no) |
3673                     GRSTCTL_TXFFLSH);
3674
3675                 DWC_OTG_WRITE_4(sc,
3676                     DOTG_DIEPTSIZ(ep_no), 0);
3677         }
3678
3679         /* poll interrupt */
3680         dwc_otg_interrupt_poll_locked(sc);
3681         dwc_otg_interrupt_complete_locked(sc);
3682 }
3683
3684 static void
3685 dwc_otg_clear_stall(struct usb_device *udev, struct usb_endpoint *ep)
3686 {
3687         struct dwc_otg_softc *sc;
3688         struct usb_endpoint_descriptor *ed;
3689
3690         DPRINTFN(5, "endpoint=%p\n", ep);
3691
3692         USB_BUS_LOCK_ASSERT(udev->bus, MA_OWNED);
3693
3694         /* check mode */
3695         if (udev->flags.usb_mode != USB_MODE_DEVICE) {
3696                 /* not supported */
3697                 return;
3698         }
3699         /* get softc */
3700         sc = DWC_OTG_BUS2SC(udev->bus);
3701
3702         USB_BUS_SPIN_LOCK(&sc->sc_bus);
3703
3704         /* get endpoint descriptor */
3705         ed = ep->edesc;
3706
3707         /* reset endpoint */
3708         dwc_otg_clear_stall_sub_locked(sc,
3709             UGETW(ed->wMaxPacketSize),
3710             (ed->bEndpointAddress & UE_ADDR),
3711             (ed->bmAttributes & UE_XFERTYPE),
3712             (ed->bEndpointAddress & (UE_DIR_IN | UE_DIR_OUT)));
3713
3714         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3715 }
3716
3717 static void
3718 dwc_otg_device_state_change(struct usb_device *udev)
3719 {
3720         struct dwc_otg_softc *sc;
3721         uint8_t x;
3722
3723         /* check mode */
3724         if (udev->flags.usb_mode != USB_MODE_DEVICE) {
3725                 /* not supported */
3726                 return;
3727         }
3728
3729         /* get softc */
3730         sc = DWC_OTG_BUS2SC(udev->bus);
3731
3732         /* deactivate all other endpoint but the control endpoint */
3733         if (udev->state == USB_STATE_CONFIGURED ||
3734             udev->state == USB_STATE_ADDRESSED) {
3735
3736                 USB_BUS_LOCK(&sc->sc_bus);
3737
3738                 for (x = 1; x != sc->sc_dev_ep_max; x++) {
3739
3740                         if (x < sc->sc_dev_in_ep_max) {
3741                                 DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(x),
3742                                     DIEPCTL_EPDIS);
3743                                 DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(x), 0);
3744                         }
3745
3746                         DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(x),
3747                             DOEPCTL_EPDIS);
3748                         DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(x), 0);
3749                 }
3750                 USB_BUS_UNLOCK(&sc->sc_bus);
3751         }
3752 }
3753
3754 int
3755 dwc_otg_init(struct dwc_otg_softc *sc)
3756 {
3757         uint32_t temp;
3758
3759         DPRINTF("start\n");
3760
3761         /* set up the bus structure */
3762         sc->sc_bus.usbrev = USB_REV_2_0;
3763         sc->sc_bus.methods = &dwc_otg_bus_methods;
3764
3765         usb_callout_init_mtx(&sc->sc_timer,
3766             &sc->sc_bus.bus_mtx, 0);
3767
3768         USB_BUS_LOCK(&sc->sc_bus);
3769
3770         /* turn on clocks */
3771         dwc_otg_clocks_on(sc);
3772
3773         temp = DWC_OTG_READ_4(sc, DOTG_GSNPSID);
3774         DPRINTF("Version = 0x%08x\n", temp);
3775
3776         /* disconnect */
3777         DWC_OTG_WRITE_4(sc, DOTG_DCTL,
3778             DCTL_SFTDISCON);
3779
3780         /* wait for host to detect disconnect */
3781         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 32);
3782
3783         DWC_OTG_WRITE_4(sc, DOTG_GRSTCTL,
3784             GRSTCTL_CSFTRST);
3785
3786         /* wait a little bit for block to reset */
3787         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 128);
3788
3789         switch (sc->sc_mode) {
3790         case DWC_MODE_DEVICE:
3791                 temp = GUSBCFG_FORCEDEVMODE;
3792                 break;
3793         case DWC_MODE_HOST:
3794                 temp = GUSBCFG_FORCEHOSTMODE;
3795                 break;
3796         default:
3797                 temp = 0;
3798                 break;
3799         }
3800
3801         /* select HSIC, ULPI or internal PHY mode */
3802         switch (dwc_otg_phy_type) {
3803         case DWC_OTG_PHY_HSIC:
3804                 DWC_OTG_WRITE_4(sc, DOTG_GUSBCFG,
3805                     GUSBCFG_PHYIF |
3806                     GUSBCFG_TRD_TIM_SET(5) | temp);
3807                 DWC_OTG_WRITE_4(sc, DOTG_GOTGCTL,
3808                     0x000000EC);
3809
3810                 temp = DWC_OTG_READ_4(sc, DOTG_GLPMCFG);
3811                 DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3812                     temp & ~GLPMCFG_HSIC_CONN);
3813                 DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3814                     temp | GLPMCFG_HSIC_CONN);
3815                 break;
3816         case DWC_OTG_PHY_ULPI:
3817                 DWC_OTG_WRITE_4(sc, DOTG_GUSBCFG,
3818                     GUSBCFG_ULPI_UTMI_SEL |
3819                     GUSBCFG_TRD_TIM_SET(5) | temp);
3820                 DWC_OTG_WRITE_4(sc, DOTG_GOTGCTL, 0);
3821
3822                 temp = DWC_OTG_READ_4(sc, DOTG_GLPMCFG);
3823                 DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3824                     temp & ~GLPMCFG_HSIC_CONN);
3825                 break;
3826         case DWC_OTG_PHY_INTERNAL:
3827                 DWC_OTG_WRITE_4(sc, DOTG_GUSBCFG,
3828                     GUSBCFG_PHYSEL |
3829                     GUSBCFG_TRD_TIM_SET(5) | temp);
3830                 DWC_OTG_WRITE_4(sc, DOTG_GOTGCTL, 0);
3831
3832                 temp = DWC_OTG_READ_4(sc, DOTG_GLPMCFG);
3833                 DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3834                     temp & ~GLPMCFG_HSIC_CONN);
3835
3836                 temp = DWC_OTG_READ_4(sc, DOTG_GGPIO);
3837                 temp &= ~(DOTG_GGPIO_NOVBUSSENS | DOTG_GGPIO_I2CPADEN);
3838                 temp |= (DOTG_GGPIO_VBUSASEN | DOTG_GGPIO_VBUSBSEN |
3839                     DOTG_GGPIO_PWRDWN);
3840                 DWC_OTG_WRITE_4(sc, DOTG_GGPIO, temp);
3841                 break;
3842         default:
3843                 break;
3844         }
3845
3846         /* clear global nak */
3847         DWC_OTG_WRITE_4(sc, DOTG_DCTL,
3848             DCTL_CGOUTNAK |
3849             DCTL_CGNPINNAK);
3850
3851         /* disable USB port */
3852         DWC_OTG_WRITE_4(sc, DOTG_PCGCCTL, 0xFFFFFFFF);
3853
3854         /* wait 10ms */
3855         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 100);
3856
3857         /* enable USB port */
3858         DWC_OTG_WRITE_4(sc, DOTG_PCGCCTL, 0);
3859
3860         /* wait 10ms */
3861         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 100);
3862
3863         temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG3);
3864
3865         sc->sc_fifo_size = 4 * GHWCFG3_DFIFODEPTH_GET(temp);
3866
3867         temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG2);
3868
3869         sc->sc_dev_ep_max = GHWCFG2_NUMDEVEPS_GET(temp);
3870
3871         if (sc->sc_dev_ep_max > DWC_OTG_MAX_ENDPOINTS)
3872                 sc->sc_dev_ep_max = DWC_OTG_MAX_ENDPOINTS;
3873
3874         sc->sc_host_ch_max = GHWCFG2_NUMHSTCHNL_GET(temp);
3875
3876         if (sc->sc_host_ch_max > DWC_OTG_MAX_CHANNELS)
3877                 sc->sc_host_ch_max = DWC_OTG_MAX_CHANNELS;
3878
3879         temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG4);
3880
3881         sc->sc_dev_in_ep_max = GHWCFG4_NUM_IN_EP_GET(temp);
3882
3883         DPRINTF("Total FIFO size = %d bytes, Device EPs = %d/%d Host CHs = %d\n",
3884             sc->sc_fifo_size, sc->sc_dev_ep_max, sc->sc_dev_in_ep_max,
3885             sc->sc_host_ch_max);
3886
3887         /* setup FIFO */
3888         if (dwc_otg_init_fifo(sc, DWC_MODE_OTG)) {
3889                 USB_BUS_UNLOCK(&sc->sc_bus);
3890                 return (EINVAL);
3891         }
3892
3893         /* enable interrupts */
3894         sc->sc_irq_mask = DWC_OTG_MSK_GINT_ENABLED;
3895         DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
3896
3897         if (sc->sc_mode == DWC_MODE_OTG || sc->sc_mode == DWC_MODE_DEVICE) {
3898
3899                 /* enable all endpoint interrupts */
3900                 temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG2);
3901                 if (temp & GHWCFG2_MPI) {
3902                         uint8_t x;
3903
3904                         DPRINTF("Disable Multi Process Interrupts\n");
3905
3906                         for (x = 0; x != sc->sc_dev_in_ep_max; x++) {
3907                                 DWC_OTG_WRITE_4(sc, DOTG_DIEPEACHINTMSK(x), 0);
3908                                 DWC_OTG_WRITE_4(sc, DOTG_DOEPEACHINTMSK(x), 0);
3909                         }
3910                         DWC_OTG_WRITE_4(sc, DOTG_DEACHINTMSK, 0);
3911                 }
3912                 DWC_OTG_WRITE_4(sc, DOTG_DIEPMSK,
3913                     DIEPMSK_XFERCOMPLMSK);
3914                 DWC_OTG_WRITE_4(sc, DOTG_DOEPMSK, 0);
3915                 DWC_OTG_WRITE_4(sc, DOTG_DAINTMSK, 0xFFFF);
3916         }
3917
3918         if (sc->sc_mode == DWC_MODE_OTG || sc->sc_mode == DWC_MODE_HOST) {
3919                 /* setup clocks */
3920                 temp = DWC_OTG_READ_4(sc, DOTG_HCFG);
3921                 temp &= ~(HCFG_FSLSSUPP | HCFG_FSLSPCLKSEL_MASK);
3922                 temp |= (1 << HCFG_FSLSPCLKSEL_SHIFT);
3923                 DWC_OTG_WRITE_4(sc, DOTG_HCFG, temp);
3924         }
3925
3926         /* only enable global IRQ */
3927         DWC_OTG_WRITE_4(sc, DOTG_GAHBCFG,
3928             GAHBCFG_GLBLINTRMSK);
3929
3930         /* turn off clocks */
3931         dwc_otg_clocks_off(sc);
3932
3933         /* read initial VBUS state */
3934
3935         temp = DWC_OTG_READ_4(sc, DOTG_GOTGCTL);
3936
3937         DPRINTFN(5, "GOTCTL=0x%08x\n", temp);
3938
3939         dwc_otg_vbus_interrupt(sc,
3940             (temp & (GOTGCTL_ASESVLD | GOTGCTL_BSESVLD)) ? 1 : 0);
3941
3942         USB_BUS_UNLOCK(&sc->sc_bus);
3943
3944         /* catch any lost interrupts */
3945
3946         dwc_otg_do_poll(&sc->sc_bus);
3947
3948         return (0);                     /* success */
3949 }
3950
3951 void
3952 dwc_otg_uninit(struct dwc_otg_softc *sc)
3953 {
3954         USB_BUS_LOCK(&sc->sc_bus);
3955
3956         /* stop host timer */
3957         dwc_otg_timer_stop(sc);
3958
3959         /* set disconnect */
3960         DWC_OTG_WRITE_4(sc, DOTG_DCTL,
3961             DCTL_SFTDISCON);
3962
3963         /* turn off global IRQ */
3964         DWC_OTG_WRITE_4(sc, DOTG_GAHBCFG, 0);
3965
3966         sc->sc_flags.port_enabled = 0;
3967         sc->sc_flags.port_powered = 0;
3968         sc->sc_flags.status_vbus = 0;
3969         sc->sc_flags.status_bus_reset = 0;
3970         sc->sc_flags.status_suspend = 0;
3971         sc->sc_flags.change_suspend = 0;
3972         sc->sc_flags.change_connect = 1;
3973
3974         dwc_otg_pull_down(sc);
3975         dwc_otg_clocks_off(sc);
3976
3977         USB_BUS_UNLOCK(&sc->sc_bus);
3978
3979         usb_callout_drain(&sc->sc_timer);
3980 }
3981
3982 static void
3983 dwc_otg_suspend(struct dwc_otg_softc *sc)
3984 {
3985         return;
3986 }
3987
3988 static void
3989 dwc_otg_resume(struct dwc_otg_softc *sc)
3990 {
3991         return;
3992 }
3993
3994 static void
3995 dwc_otg_do_poll(struct usb_bus *bus)
3996 {
3997         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(bus);
3998
3999         USB_BUS_LOCK(&sc->sc_bus);
4000         USB_BUS_SPIN_LOCK(&sc->sc_bus);
4001         dwc_otg_interrupt_poll_locked(sc);
4002         dwc_otg_interrupt_complete_locked(sc);
4003         if (sc->sc_flags.status_device_mode == 0) {
4004                 /* update host transfer schedule, so that new transfers can be issued */
4005                 if (dwc_otg_update_host_transfer_schedule_locked(sc))
4006                         dwc_otg_interrupt_poll_locked(sc);
4007         }
4008         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
4009         USB_BUS_UNLOCK(&sc->sc_bus);
4010 }
4011
4012 /*------------------------------------------------------------------------*
4013  * DWC OTG bulk support
4014  * DWC OTG control support
4015  * DWC OTG interrupt support
4016  *------------------------------------------------------------------------*/
4017 static void
4018 dwc_otg_device_non_isoc_open(struct usb_xfer *xfer)
4019 {
4020 }
4021
4022 static void
4023 dwc_otg_device_non_isoc_close(struct usb_xfer *xfer)
4024 {
4025         dwc_otg_device_done(xfer, USB_ERR_CANCELLED);
4026 }
4027
4028 static void
4029 dwc_otg_device_non_isoc_enter(struct usb_xfer *xfer)
4030 {
4031 }
4032
4033 static void
4034 dwc_otg_device_non_isoc_start(struct usb_xfer *xfer)
4035 {
4036         /* setup TDs */
4037         dwc_otg_setup_standard_chain(xfer);
4038         dwc_otg_start_standard_chain(xfer);
4039 }
4040
4041 struct usb_pipe_methods dwc_otg_device_non_isoc_methods =
4042 {
4043         .open = dwc_otg_device_non_isoc_open,
4044         .close = dwc_otg_device_non_isoc_close,
4045         .enter = dwc_otg_device_non_isoc_enter,
4046         .start = dwc_otg_device_non_isoc_start,
4047 };
4048
4049 /*------------------------------------------------------------------------*
4050  * DWC OTG full speed isochronous support
4051  *------------------------------------------------------------------------*/
4052 static void
4053 dwc_otg_device_isoc_open(struct usb_xfer *xfer)
4054 {
4055 }
4056
4057 static void
4058 dwc_otg_device_isoc_close(struct usb_xfer *xfer)
4059 {
4060         dwc_otg_device_done(xfer, USB_ERR_CANCELLED);
4061 }
4062
4063 static void
4064 dwc_otg_device_isoc_enter(struct usb_xfer *xfer)
4065 {
4066 }
4067
4068 static void
4069 dwc_otg_device_isoc_start(struct usb_xfer *xfer)
4070 {
4071         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
4072         uint32_t temp;
4073         uint32_t msframes;
4074         uint32_t framenum;
4075         uint8_t shift = usbd_xfer_get_fps_shift(xfer);
4076
4077         DPRINTFN(6, "xfer=%p next=%d nframes=%d\n",
4078             xfer, xfer->endpoint->isoc_next, xfer->nframes);
4079
4080         if (xfer->xroot->udev->flags.usb_mode == USB_MODE_HOST) {
4081                 temp = DWC_OTG_READ_4(sc, DOTG_HFNUM);
4082
4083                 /* get the current frame index */
4084                 framenum = (temp & HFNUM_FRNUM_MASK);
4085         } else {
4086                 temp = DWC_OTG_READ_4(sc, DOTG_DSTS);
4087
4088                 /* get the current frame index */
4089                 framenum = DSTS_SOFFN_GET(temp);
4090         }
4091
4092         if (xfer->xroot->udev->parent_hs_hub != NULL)
4093                 framenum /= 8;
4094
4095         framenum &= DWC_OTG_FRAME_MASK;
4096
4097         /*
4098          * Compute number of milliseconds worth of data traffic for
4099          * this USB transfer:
4100          */
4101         if (xfer->xroot->udev->speed == USB_SPEED_HIGH)
4102                 msframes = ((xfer->nframes << shift) + 7) / 8;
4103         else
4104                 msframes = xfer->nframes;
4105
4106         /*
4107          * check if the frame index is within the window where the frames
4108          * will be inserted
4109          */
4110         temp = (framenum - xfer->endpoint->isoc_next) & DWC_OTG_FRAME_MASK;
4111
4112         if ((xfer->endpoint->is_synced == 0) || (temp < msframes)) {
4113                 /*
4114                  * If there is data underflow or the pipe queue is
4115                  * empty we schedule the transfer a few frames ahead
4116                  * of the current frame position. Else two isochronous
4117                  * transfers might overlap.
4118                  */
4119                 xfer->endpoint->isoc_next = (framenum + 3) & DWC_OTG_FRAME_MASK;
4120                 xfer->endpoint->is_synced = 1;
4121                 DPRINTFN(3, "start next=%d\n", xfer->endpoint->isoc_next);
4122         }
4123         /*
4124          * compute how many milliseconds the insertion is ahead of the
4125          * current frame position:
4126          */
4127         temp = (xfer->endpoint->isoc_next - framenum) & DWC_OTG_FRAME_MASK;
4128
4129         /*
4130          * pre-compute when the isochronous transfer will be finished:
4131          */
4132         xfer->isoc_time_complete =
4133                 usb_isoc_time_expand(&sc->sc_bus, framenum) + temp + msframes;
4134
4135         /* setup TDs */
4136         dwc_otg_setup_standard_chain(xfer);
4137
4138         /* compute frame number for next insertion */
4139         xfer->endpoint->isoc_next += msframes;
4140
4141         /* start TD chain */
4142         dwc_otg_start_standard_chain(xfer);
4143 }
4144
4145 struct usb_pipe_methods dwc_otg_device_isoc_methods =
4146 {
4147         .open = dwc_otg_device_isoc_open,
4148         .close = dwc_otg_device_isoc_close,
4149         .enter = dwc_otg_device_isoc_enter,
4150         .start = dwc_otg_device_isoc_start,
4151 };
4152
4153 /*------------------------------------------------------------------------*
4154  * DWC OTG root control support
4155  *------------------------------------------------------------------------*
4156  * Simulate a hardware HUB by handling all the necessary requests.
4157  *------------------------------------------------------------------------*/
4158
4159 static const struct usb_device_descriptor dwc_otg_devd = {
4160         .bLength = sizeof(struct usb_device_descriptor),
4161         .bDescriptorType = UDESC_DEVICE,
4162         .bcdUSB = {0x00, 0x02},
4163         .bDeviceClass = UDCLASS_HUB,
4164         .bDeviceSubClass = UDSUBCLASS_HUB,
4165         .bDeviceProtocol = UDPROTO_HSHUBSTT,
4166         .bMaxPacketSize = 64,
4167         .bcdDevice = {0x00, 0x01},
4168         .iManufacturer = 1,
4169         .iProduct = 2,
4170         .bNumConfigurations = 1,
4171 };
4172
4173 static const struct dwc_otg_config_desc dwc_otg_confd = {
4174         .confd = {
4175                 .bLength = sizeof(struct usb_config_descriptor),
4176                 .bDescriptorType = UDESC_CONFIG,
4177                 .wTotalLength[0] = sizeof(dwc_otg_confd),
4178                 .bNumInterface = 1,
4179                 .bConfigurationValue = 1,
4180                 .iConfiguration = 0,
4181                 .bmAttributes = UC_SELF_POWERED,
4182                 .bMaxPower = 0,
4183         },
4184         .ifcd = {
4185                 .bLength = sizeof(struct usb_interface_descriptor),
4186                 .bDescriptorType = UDESC_INTERFACE,
4187                 .bNumEndpoints = 1,
4188                 .bInterfaceClass = UICLASS_HUB,
4189                 .bInterfaceSubClass = UISUBCLASS_HUB,
4190                 .bInterfaceProtocol = 0,
4191         },
4192         .endpd = {
4193                 .bLength = sizeof(struct usb_endpoint_descriptor),
4194                 .bDescriptorType = UDESC_ENDPOINT,
4195                 .bEndpointAddress = (UE_DIR_IN | DWC_OTG_INTR_ENDPT),
4196                 .bmAttributes = UE_INTERRUPT,
4197                 .wMaxPacketSize[0] = 8,
4198                 .bInterval = 255,
4199         },
4200 };
4201
4202 #define HSETW(ptr, val) ptr = { (uint8_t)(val), (uint8_t)((val) >> 8) }
4203
4204 static const struct usb_hub_descriptor_min dwc_otg_hubd = {
4205         .bDescLength = sizeof(dwc_otg_hubd),
4206         .bDescriptorType = UDESC_HUB,
4207         .bNbrPorts = 1,
4208         HSETW(.wHubCharacteristics, (UHD_PWR_NO_SWITCH | UHD_OC_INDIVIDUAL)),
4209         .bPwrOn2PwrGood = 50,
4210         .bHubContrCurrent = 0,
4211         .DeviceRemovable = {0},         /* port is removable */
4212 };
4213
4214 #define STRING_VENDOR \
4215   "D\0W\0C\0O\0T\0G"
4216
4217 #define STRING_PRODUCT \
4218   "O\0T\0G\0 \0R\0o\0o\0t\0 \0H\0U\0B"
4219
4220 USB_MAKE_STRING_DESC(STRING_VENDOR, dwc_otg_vendor);
4221 USB_MAKE_STRING_DESC(STRING_PRODUCT, dwc_otg_product);
4222
4223 static usb_error_t
4224 dwc_otg_roothub_exec(struct usb_device *udev,
4225     struct usb_device_request *req, const void **pptr, uint16_t *plength)
4226 {
4227         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(udev->bus);
4228         const void *ptr;
4229         uint16_t len;
4230         uint16_t value;
4231         uint16_t index;
4232         usb_error_t err;
4233
4234         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
4235
4236         /* buffer reset */
4237         ptr = (const void *)&sc->sc_hub_temp;
4238         len = 0;
4239         err = 0;
4240
4241         value = UGETW(req->wValue);
4242         index = UGETW(req->wIndex);
4243
4244         /* demultiplex the control request */
4245
4246         switch (req->bmRequestType) {
4247         case UT_READ_DEVICE:
4248                 switch (req->bRequest) {
4249                 case UR_GET_DESCRIPTOR:
4250                         goto tr_handle_get_descriptor;
4251                 case UR_GET_CONFIG:
4252                         goto tr_handle_get_config;
4253                 case UR_GET_STATUS:
4254                         goto tr_handle_get_status;
4255                 default:
4256                         goto tr_stalled;
4257                 }
4258                 break;
4259
4260         case UT_WRITE_DEVICE:
4261                 switch (req->bRequest) {
4262                 case UR_SET_ADDRESS:
4263                         goto tr_handle_set_address;
4264                 case UR_SET_CONFIG:
4265                         goto tr_handle_set_config;
4266                 case UR_CLEAR_FEATURE:
4267                         goto tr_valid;  /* nop */
4268                 case UR_SET_DESCRIPTOR:
4269                         goto tr_valid;  /* nop */
4270                 case UR_SET_FEATURE:
4271                 default:
4272                         goto tr_stalled;
4273                 }
4274                 break;
4275
4276         case UT_WRITE_ENDPOINT:
4277                 switch (req->bRequest) {
4278                 case UR_CLEAR_FEATURE:
4279                         switch (UGETW(req->wValue)) {
4280                         case UF_ENDPOINT_HALT:
4281                                 goto tr_handle_clear_halt;
4282                         case UF_DEVICE_REMOTE_WAKEUP:
4283                                 goto tr_handle_clear_wakeup;
4284                         default:
4285                                 goto tr_stalled;
4286                         }
4287                         break;
4288                 case UR_SET_FEATURE:
4289                         switch (UGETW(req->wValue)) {
4290                         case UF_ENDPOINT_HALT:
4291                                 goto tr_handle_set_halt;
4292                         case UF_DEVICE_REMOTE_WAKEUP:
4293                                 goto tr_handle_set_wakeup;
4294                         default:
4295                                 goto tr_stalled;
4296                         }
4297                         break;
4298                 case UR_SYNCH_FRAME:
4299                         goto tr_valid;  /* nop */
4300                 default:
4301                         goto tr_stalled;
4302                 }
4303                 break;
4304
4305         case UT_READ_ENDPOINT:
4306                 switch (req->bRequest) {
4307                 case UR_GET_STATUS:
4308                         goto tr_handle_get_ep_status;
4309                 default:
4310                         goto tr_stalled;
4311                 }
4312                 break;
4313
4314         case UT_WRITE_INTERFACE:
4315                 switch (req->bRequest) {
4316                 case UR_SET_INTERFACE:
4317                         goto tr_handle_set_interface;
4318                 case UR_CLEAR_FEATURE:
4319                         goto tr_valid;  /* nop */
4320                 case UR_SET_FEATURE:
4321                 default:
4322                         goto tr_stalled;
4323                 }
4324                 break;
4325
4326         case UT_READ_INTERFACE:
4327                 switch (req->bRequest) {
4328                 case UR_GET_INTERFACE:
4329                         goto tr_handle_get_interface;
4330                 case UR_GET_STATUS:
4331                         goto tr_handle_get_iface_status;
4332                 default:
4333                         goto tr_stalled;
4334                 }
4335                 break;
4336
4337         case UT_WRITE_CLASS_INTERFACE:
4338         case UT_WRITE_VENDOR_INTERFACE:
4339                 /* XXX forward */
4340                 break;
4341
4342         case UT_READ_CLASS_INTERFACE:
4343         case UT_READ_VENDOR_INTERFACE:
4344                 /* XXX forward */
4345                 break;
4346
4347         case UT_WRITE_CLASS_DEVICE:
4348                 switch (req->bRequest) {
4349                 case UR_CLEAR_FEATURE:
4350                         goto tr_valid;
4351                 case UR_SET_DESCRIPTOR:
4352                 case UR_SET_FEATURE:
4353                         break;
4354                 default:
4355                         goto tr_stalled;
4356                 }
4357                 break;
4358
4359         case UT_WRITE_CLASS_OTHER:
4360                 switch (req->bRequest) {
4361                 case UR_CLEAR_FEATURE:
4362                         goto tr_handle_clear_port_feature;
4363                 case UR_SET_FEATURE:
4364                         goto tr_handle_set_port_feature;
4365                 case UR_CLEAR_TT_BUFFER:
4366                 case UR_RESET_TT:
4367                 case UR_STOP_TT:
4368                         goto tr_valid;
4369
4370                 default:
4371                         goto tr_stalled;
4372                 }
4373                 break;
4374
4375         case UT_READ_CLASS_OTHER:
4376                 switch (req->bRequest) {
4377                 case UR_GET_TT_STATE:
4378                         goto tr_handle_get_tt_state;
4379                 case UR_GET_STATUS:
4380                         goto tr_handle_get_port_status;
4381                 default:
4382                         goto tr_stalled;
4383                 }
4384                 break;
4385
4386         case UT_READ_CLASS_DEVICE:
4387                 switch (req->bRequest) {
4388                 case UR_GET_DESCRIPTOR:
4389                         goto tr_handle_get_class_descriptor;
4390                 case UR_GET_STATUS:
4391                         goto tr_handle_get_class_status;
4392
4393                 default:
4394                         goto tr_stalled;
4395                 }
4396                 break;
4397         default:
4398                 goto tr_stalled;
4399         }
4400         goto tr_valid;
4401
4402 tr_handle_get_descriptor:
4403         switch (value >> 8) {
4404         case UDESC_DEVICE:
4405                 if (value & 0xff) {
4406                         goto tr_stalled;
4407                 }
4408                 len = sizeof(dwc_otg_devd);
4409                 ptr = (const void *)&dwc_otg_devd;
4410                 goto tr_valid;
4411         case UDESC_CONFIG:
4412                 if (value & 0xff) {
4413                         goto tr_stalled;
4414                 }
4415                 len = sizeof(dwc_otg_confd);
4416                 ptr = (const void *)&dwc_otg_confd;
4417                 goto tr_valid;
4418         case UDESC_STRING:
4419                 switch (value & 0xff) {
4420                 case 0:         /* Language table */
4421                         len = sizeof(usb_string_lang_en);
4422                         ptr = (const void *)&usb_string_lang_en;
4423                         goto tr_valid;
4424
4425                 case 1:         /* Vendor */
4426                         len = sizeof(dwc_otg_vendor);
4427                         ptr = (const void *)&dwc_otg_vendor;
4428                         goto tr_valid;
4429
4430                 case 2:         /* Product */
4431                         len = sizeof(dwc_otg_product);
4432                         ptr = (const void *)&dwc_otg_product;
4433                         goto tr_valid;
4434                 default:
4435                         break;
4436                 }
4437                 break;
4438         default:
4439                 goto tr_stalled;
4440         }
4441         goto tr_stalled;
4442
4443 tr_handle_get_config:
4444         len = 1;
4445         sc->sc_hub_temp.wValue[0] = sc->sc_conf;
4446         goto tr_valid;
4447
4448 tr_handle_get_status:
4449         len = 2;
4450         USETW(sc->sc_hub_temp.wValue, UDS_SELF_POWERED);
4451         goto tr_valid;
4452
4453 tr_handle_set_address:
4454         if (value & 0xFF00) {
4455                 goto tr_stalled;
4456         }
4457         sc->sc_rt_addr = value;
4458         goto tr_valid;
4459
4460 tr_handle_set_config:
4461         if (value >= 2) {
4462                 goto tr_stalled;
4463         }
4464         sc->sc_conf = value;
4465         goto tr_valid;
4466
4467 tr_handle_get_interface:
4468         len = 1;
4469         sc->sc_hub_temp.wValue[0] = 0;
4470         goto tr_valid;
4471
4472 tr_handle_get_tt_state:
4473 tr_handle_get_class_status:
4474 tr_handle_get_iface_status:
4475 tr_handle_get_ep_status:
4476         len = 2;
4477         USETW(sc->sc_hub_temp.wValue, 0);
4478         goto tr_valid;
4479
4480 tr_handle_set_halt:
4481 tr_handle_set_interface:
4482 tr_handle_set_wakeup:
4483 tr_handle_clear_wakeup:
4484 tr_handle_clear_halt:
4485         goto tr_valid;
4486
4487 tr_handle_clear_port_feature:
4488         if (index != 1)
4489                 goto tr_stalled;
4490
4491         DPRINTFN(9, "UR_CLEAR_PORT_FEATURE on port %d\n", index);
4492
4493         switch (value) {
4494         case UHF_PORT_SUSPEND:
4495                 dwc_otg_wakeup_peer(sc);
4496                 break;
4497
4498         case UHF_PORT_ENABLE:
4499                 if (sc->sc_flags.status_device_mode == 0) {
4500                         DWC_OTG_WRITE_4(sc, DOTG_HPRT,
4501                             sc->sc_hprt_val | HPRT_PRTENA);
4502                 }
4503                 sc->sc_flags.port_enabled = 0;
4504                 break;
4505
4506         case UHF_C_PORT_RESET:
4507                 sc->sc_flags.change_reset = 0;
4508                 break;
4509
4510         case UHF_C_PORT_ENABLE:
4511                 sc->sc_flags.change_enabled = 0;
4512                 break;
4513
4514         case UHF_C_PORT_OVER_CURRENT:
4515                 sc->sc_flags.change_over_current = 0;
4516                 break;
4517
4518         case UHF_PORT_TEST:
4519         case UHF_PORT_INDICATOR:
4520                 /* nops */
4521                 break;
4522
4523         case UHF_PORT_POWER:
4524                 sc->sc_flags.port_powered = 0;
4525                 if (sc->sc_mode == DWC_MODE_HOST || sc->sc_mode == DWC_MODE_OTG) {
4526                         sc->sc_hprt_val = 0;
4527                         DWC_OTG_WRITE_4(sc, DOTG_HPRT, HPRT_PRTENA);
4528                 }
4529                 dwc_otg_pull_down(sc);
4530                 dwc_otg_clocks_off(sc);
4531                 break;
4532
4533         case UHF_C_PORT_CONNECTION:
4534                 /* clear connect change flag */
4535                 sc->sc_flags.change_connect = 0;
4536                 break;
4537
4538         case UHF_C_PORT_SUSPEND:
4539                 sc->sc_flags.change_suspend = 0;
4540                 break;
4541
4542         default:
4543                 err = USB_ERR_IOERROR;
4544                 goto done;
4545         }
4546         goto tr_valid;
4547
4548 tr_handle_set_port_feature:
4549         if (index != 1) {
4550                 goto tr_stalled;
4551         }
4552         DPRINTFN(9, "UR_SET_PORT_FEATURE\n");
4553
4554         switch (value) {
4555         case UHF_PORT_ENABLE:
4556                 break;
4557
4558         case UHF_PORT_SUSPEND:
4559                 if (sc->sc_flags.status_device_mode == 0) {
4560                         /* set suspend BIT */
4561                         sc->sc_hprt_val |= HPRT_PRTSUSP;
4562                         DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
4563
4564                         /* generate HUB suspend event */
4565                         dwc_otg_suspend_irq(sc);
4566                 }
4567                 break;
4568
4569         case UHF_PORT_RESET:
4570                 if (sc->sc_flags.status_device_mode == 0) {
4571
4572                         DPRINTF("PORT RESET\n");
4573
4574                         /* enable PORT reset */
4575                         DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val | HPRT_PRTRST);
4576
4577                         /* Wait 62.5ms for reset to complete */
4578                         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 16);
4579
4580                         DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
4581
4582                         /* Wait 62.5ms for reset to complete */
4583                         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 16);
4584
4585                         /* reset FIFOs */
4586                         (void) dwc_otg_init_fifo(sc, DWC_MODE_HOST);
4587
4588                         sc->sc_flags.change_reset = 1;
4589                 } else {
4590                         err = USB_ERR_IOERROR;
4591                 }
4592                 break;
4593
4594         case UHF_PORT_TEST:
4595         case UHF_PORT_INDICATOR:
4596                 /* nops */
4597                 break;
4598         case UHF_PORT_POWER:
4599                 sc->sc_flags.port_powered = 1;
4600                 if (sc->sc_mode == DWC_MODE_HOST || sc->sc_mode == DWC_MODE_OTG) {
4601                         sc->sc_hprt_val |= HPRT_PRTPWR;
4602                         DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
4603                 }
4604                 if (sc->sc_mode == DWC_MODE_DEVICE || sc->sc_mode == DWC_MODE_OTG) {
4605                         /* pull up D+, if any */
4606                         dwc_otg_pull_up(sc);
4607                 }
4608                 break;
4609         default:
4610                 err = USB_ERR_IOERROR;
4611                 goto done;
4612         }
4613         goto tr_valid;
4614
4615 tr_handle_get_port_status:
4616
4617         DPRINTFN(9, "UR_GET_PORT_STATUS\n");
4618
4619         if (index != 1)
4620                 goto tr_stalled;
4621
4622         if (sc->sc_flags.status_vbus)
4623                 dwc_otg_clocks_on(sc);
4624         else
4625                 dwc_otg_clocks_off(sc);
4626
4627         /* Select Device Side Mode */
4628
4629         if (sc->sc_flags.status_device_mode) {
4630                 value = UPS_PORT_MODE_DEVICE;
4631                 dwc_otg_timer_stop(sc);
4632         } else {
4633                 value = 0;
4634                 dwc_otg_timer_start(sc);
4635         }
4636
4637         if (sc->sc_flags.status_high_speed)
4638                 value |= UPS_HIGH_SPEED;
4639         else if (sc->sc_flags.status_low_speed)
4640                 value |= UPS_LOW_SPEED;
4641
4642         if (sc->sc_flags.port_powered)
4643                 value |= UPS_PORT_POWER;
4644
4645         if (sc->sc_flags.port_enabled)
4646                 value |= UPS_PORT_ENABLED;
4647
4648         if (sc->sc_flags.port_over_current)
4649                 value |= UPS_OVERCURRENT_INDICATOR;
4650
4651         if (sc->sc_flags.status_vbus &&
4652             sc->sc_flags.status_bus_reset)
4653                 value |= UPS_CURRENT_CONNECT_STATUS;
4654
4655         if (sc->sc_flags.status_suspend)
4656                 value |= UPS_SUSPEND;
4657
4658         USETW(sc->sc_hub_temp.ps.wPortStatus, value);
4659
4660         value = 0;
4661
4662         if (sc->sc_flags.change_connect)
4663                 value |= UPS_C_CONNECT_STATUS;
4664         if (sc->sc_flags.change_suspend)
4665                 value |= UPS_C_SUSPEND;
4666         if (sc->sc_flags.change_reset)
4667                 value |= UPS_C_PORT_RESET;
4668         if (sc->sc_flags.change_over_current)
4669                 value |= UPS_C_OVERCURRENT_INDICATOR;
4670
4671         USETW(sc->sc_hub_temp.ps.wPortChange, value);
4672         len = sizeof(sc->sc_hub_temp.ps);
4673         goto tr_valid;
4674
4675 tr_handle_get_class_descriptor:
4676         if (value & 0xFF) {
4677                 goto tr_stalled;
4678         }
4679         ptr = (const void *)&dwc_otg_hubd;
4680         len = sizeof(dwc_otg_hubd);
4681         goto tr_valid;
4682
4683 tr_stalled:
4684         err = USB_ERR_STALLED;
4685 tr_valid:
4686 done:
4687         *plength = len;
4688         *pptr = ptr;
4689         return (err);
4690 }
4691
4692 static void
4693 dwc_otg_xfer_setup(struct usb_setup_params *parm)
4694 {
4695         struct usb_xfer *xfer;
4696         void *last_obj;
4697         uint32_t ntd;
4698         uint32_t n;
4699         uint8_t ep_no;
4700         uint8_t ep_type;
4701
4702         xfer = parm->curr_xfer;
4703
4704         /*
4705          * NOTE: This driver does not use any of the parameters that
4706          * are computed from the following values. Just set some
4707          * reasonable dummies:
4708          */
4709         parm->hc_max_packet_size = 0x500;
4710         parm->hc_max_packet_count = 3;
4711         parm->hc_max_frame_size = 3 * 0x500;
4712
4713         usbd_transfer_setup_sub(parm);
4714
4715         /*
4716          * compute maximum number of TDs
4717          */
4718         ep_type = (xfer->endpoint->edesc->bmAttributes & UE_XFERTYPE);
4719
4720         if (ep_type == UE_CONTROL) {
4721
4722                 ntd = xfer->nframes + 1 /* STATUS */ + 1 /* SYNC 1 */
4723                     + 1 /* SYNC 2 */ + 1 /* SYNC 3 */;
4724         } else {
4725
4726                 ntd = xfer->nframes + 1 /* SYNC */ ;
4727         }
4728
4729         /*
4730          * check if "usbd_transfer_setup_sub" set an error
4731          */
4732         if (parm->err)
4733                 return;
4734
4735         /*
4736          * allocate transfer descriptors
4737          */
4738         last_obj = NULL;
4739
4740         ep_no = xfer->endpointno & UE_ADDR;
4741
4742         /*
4743          * Check for a valid endpoint profile in USB device mode:
4744          */
4745         if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) {
4746                 const struct usb_hw_ep_profile *pf;
4747
4748                 dwc_otg_get_hw_ep_profile(parm->udev, &pf, ep_no);
4749
4750                 if (pf == NULL) {
4751                         /* should not happen */
4752                         parm->err = USB_ERR_INVAL;
4753                         return;
4754                 }
4755         }
4756
4757         /* align data */
4758         parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
4759
4760         for (n = 0; n != ntd; n++) {
4761
4762                 struct dwc_otg_td *td;
4763
4764                 if (parm->buf) {
4765
4766                         td = USB_ADD_BYTES(parm->buf, parm->size[0]);
4767
4768                         /* compute shared bandwidth resource index for TT */
4769                         if (parm->udev->parent_hs_hub != NULL && parm->udev->speed != USB_SPEED_HIGH) {
4770                                 if (parm->udev->parent_hs_hub->ddesc.bDeviceProtocol == UDPROTO_HSHUBMTT)
4771                                         td->tt_index = parm->udev->device_index;
4772                                 else
4773                                         td->tt_index = parm->udev->parent_hs_hub->device_index;
4774                         } else {
4775                                 td->tt_index = parm->udev->device_index;
4776                         }
4777
4778                         /* init TD */
4779                         td->max_packet_size = xfer->max_packet_size;
4780                         td->max_packet_count = xfer->max_packet_count;
4781                         td->ep_no = ep_no;
4782                         td->ep_type = ep_type;
4783                         td->obj_next = last_obj;
4784
4785                         last_obj = td;
4786                 }
4787                 parm->size[0] += sizeof(*td);
4788         }
4789
4790         xfer->td_start[0] = last_obj;
4791 }
4792
4793 static void
4794 dwc_otg_xfer_unsetup(struct usb_xfer *xfer)
4795 {
4796         return;
4797 }
4798
4799 static void
4800 dwc_otg_ep_init(struct usb_device *udev, struct usb_endpoint_descriptor *edesc,
4801     struct usb_endpoint *ep)
4802 {
4803         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(udev->bus);
4804
4805         DPRINTFN(2, "endpoint=%p, addr=%d, endpt=%d, mode=%d (%d,%d)\n",
4806             ep, udev->address,
4807             edesc->bEndpointAddress, udev->flags.usb_mode,
4808             sc->sc_rt_addr, udev->device_index);
4809
4810         if (udev->device_index != sc->sc_rt_addr) {
4811
4812                 if (udev->flags.usb_mode == USB_MODE_DEVICE) {
4813                         if (udev->speed != USB_SPEED_FULL &&
4814                             udev->speed != USB_SPEED_HIGH) {
4815                                 /* not supported */
4816                                 return;
4817                         }
4818                 } else {
4819                         if (udev->speed == USB_SPEED_HIGH) {
4820                                 if ((UGETW(edesc->wMaxPacketSize) >> 11) & 3) {
4821                                         /* high bandwidth endpoint - not tested */
4822                                         DPRINTF("High Bandwidth Endpoint - not tested\n");
4823                                         return;
4824                                 }
4825                         }
4826                 }
4827                 if ((edesc->bmAttributes & UE_XFERTYPE) == UE_ISOCHRONOUS)
4828                         ep->methods = &dwc_otg_device_isoc_methods;
4829                 else
4830                         ep->methods = &dwc_otg_device_non_isoc_methods;
4831         }
4832 }
4833
4834 static void
4835 dwc_otg_set_hw_power_sleep(struct usb_bus *bus, uint32_t state)
4836 {
4837         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(bus);
4838
4839         switch (state) {
4840         case USB_HW_POWER_SUSPEND:
4841                 dwc_otg_suspend(sc);
4842                 break;
4843         case USB_HW_POWER_SHUTDOWN:
4844                 dwc_otg_uninit(sc);
4845                 break;
4846         case USB_HW_POWER_RESUME:
4847                 dwc_otg_resume(sc);
4848                 break;
4849         default:
4850                 break;
4851         }
4852 }
4853
4854 static void
4855 dwc_otg_get_dma_delay(struct usb_device *udev, uint32_t *pus)
4856 {
4857         /* DMA delay - wait until any use of memory is finished */
4858         *pus = (2125);                  /* microseconds */
4859 }
4860
4861 static void
4862 dwc_otg_device_resume(struct usb_device *udev)
4863 {
4864         DPRINTF("\n");
4865
4866         /* poll all transfers again to restart resumed ones */
4867         dwc_otg_do_poll(udev->bus);
4868 }
4869
4870 static void
4871 dwc_otg_device_suspend(struct usb_device *udev)
4872 {
4873         DPRINTF("\n");
4874 }
4875
4876 struct usb_bus_methods dwc_otg_bus_methods =
4877 {
4878         .endpoint_init = &dwc_otg_ep_init,
4879         .xfer_setup = &dwc_otg_xfer_setup,
4880         .xfer_unsetup = &dwc_otg_xfer_unsetup,
4881         .get_hw_ep_profile = &dwc_otg_get_hw_ep_profile,
4882         .xfer_stall = &dwc_otg_xfer_stall,
4883         .set_stall = &dwc_otg_set_stall,
4884         .clear_stall = &dwc_otg_clear_stall,
4885         .roothub_exec = &dwc_otg_roothub_exec,
4886         .xfer_poll = &dwc_otg_do_poll,
4887         .device_state_change = &dwc_otg_device_state_change,
4888         .set_hw_power_sleep = &dwc_otg_set_hw_power_sleep,
4889         .get_dma_delay = &dwc_otg_get_dma_delay,
4890         .device_resume = &dwc_otg_device_resume,
4891         .device_suspend = &dwc_otg_device_suspend,
4892 };