sys/dev/usb/controller/dwc_otg.c

   1 /* $FreeBSD$ */
   2 /*-
   3  * Copyright (c) 2012 Hans Petter Selasky. All rights reserved.
   4  * Copyright (c) 2010-2011 Aleksandr Rybalko. All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  */
  27
  28 /*
  29  * This file contains the driver for the DesignWare series USB 2.0 OTG
  30  * Controller.
  31  */
  32
  33 /*
  34  * LIMITATION: Drivers must be bound to all OUT endpoints in the
  35  * active configuration for this driver to work properly. Blocking any
  36  * OUT endpoint will block all OUT endpoints including the control
  37  * endpoint. Usually this is not a problem.
  38  */
  39
  40 /*
  41  * NOTE: Writing to non-existing registers appears to cause an
  42  * internal reset.
  43  */
  44
  45 #ifdef USB_GLOBAL_INCLUDE_FILE
  46 #include USB_GLOBAL_INCLUDE_FILE
  47 #else
  48 #include <sys/stdint.h>
  49 #include <sys/stddef.h>
  50 #include <sys/param.h>
  51 #include <sys/queue.h>
  52 #include <sys/types.h>
  53 #include <sys/systm.h>
  54 #include <sys/kernel.h>
  55 #include <sys/bus.h>
  56 #include <sys/module.h>
  57 #include <sys/lock.h>
  58 #include <sys/mutex.h>
  59 #include <sys/condvar.h>
  60 #include <sys/sysctl.h>
  61 #include <sys/sx.h>
  62 #include <sys/unistd.h>
  63 #include <sys/callout.h>
  64 #include <sys/malloc.h>
  65 #include <sys/priv.h>
  66
  67 #include <dev/usb/usb.h>
  68 #include <dev/usb/usbdi.h>
  69
  70 #define USB_DEBUG_VAR dwc_otg_debug
  71
  72 #include <dev/usb/usb_core.h>
  73 #include <dev/usb/usb_debug.h>
  74 #include <dev/usb/usb_busdma.h>
  75 #include <dev/usb/usb_process.h>
  76 #include <dev/usb/usb_transfer.h>
  77 #include <dev/usb/usb_device.h>
  78 #include <dev/usb/usb_hub.h>
  79 #include <dev/usb/usb_util.h>
  80
  81 #include <dev/usb/usb_controller.h>
  82 #include <dev/usb/usb_bus.h>
  83 #endif                  /* USB_GLOBAL_INCLUDE_FILE */
  84
  85 #include <dev/usb/controller/dwc_otg.h>
  86 #include <dev/usb/controller/dwc_otgreg.h>
  87
  88 #define DWC_OTG_BUS2SC(bus) \
  89    ((struct dwc_otg_softc *)(((uint8_t *)(bus)) - \
  90     ((uint8_t *)&(((struct dwc_otg_softc *)0)->sc_bus))))
  91
  92 #define DWC_OTG_PC2UDEV(pc) \
  93    (USB_DMATAG_TO_XROOT((pc)->tag_parent)->udev)
  94
  95 #define DWC_OTG_MSK_GINT_ENABLED        \
  96    (GINTMSK_ENUMDONEMSK |               \
  97    GINTMSK_USBRSTMSK |                  \
  98    GINTMSK_USBSUSPMSK |                 \
  99    GINTMSK_IEPINTMSK |                  \
 100    GINTMSK_SESSREQINTMSK |              \
 101    GINTMSK_RXFLVLMSK |                  \
 102    GINTMSK_HCHINTMSK |                  \
 103    GINTMSK_OTGINTMSK |                  \
 104    GINTMSK_PRTINTMSK)
 105
 106 #define DWC_OTG_MSK_GINT_THREAD_IRQ                             \
 107    (GINTSTS_USBRST | GINTSTS_ENUMDONE | GINTSTS_PRTINT |        \
 108    GINTSTS_WKUPINT | GINTSTS_USBSUSP | GINTMSK_OTGINTMSK |      \
 109    GINTSTS_SESSREQINT)
 110
 111 #define DWC_OTG_PHY_ULPI 0
 112 #define DWC_OTG_PHY_HSIC 1
 113 #define DWC_OTG_PHY_INTERNAL 2
 114
 115 #ifndef DWC_OTG_PHY_DEFAULT
 116 #define DWC_OTG_PHY_DEFAULT DWC_OTG_PHY_ULPI
 117 #endif
 118
 119 static int dwc_otg_phy_type = DWC_OTG_PHY_DEFAULT;
 120
 121 static SYSCTL_NODE(_hw_usb, OID_AUTO, dwc_otg, CTLFLAG_RW, 0, "USB DWC OTG");
 122 SYSCTL_INT(_hw_usb_dwc_otg, OID_AUTO, phy_type, CTLFLAG_RDTUN,
 123     &dwc_otg_phy_type, 0, "DWC OTG PHY TYPE - 0/1/2 - ULPI/HSIC/INTERNAL");
 124
 125 #ifdef USB_DEBUG
 126 static int dwc_otg_debug;
 127
 128 SYSCTL_INT(_hw_usb_dwc_otg, OID_AUTO, debug, CTLFLAG_RWTUN,
 129     &dwc_otg_debug, 0, "DWC OTG debug level");
 130 #endif
 131
 132 #define DWC_OTG_INTR_ENDPT 1
 133
 134 /* prototypes */
 135
 136 static const struct usb_bus_methods dwc_otg_bus_methods;
 137 static const struct usb_pipe_methods dwc_otg_device_non_isoc_methods;
 138 static const struct usb_pipe_methods dwc_otg_device_isoc_methods;
 139
 140 static dwc_otg_cmd_t dwc_otg_setup_rx;
 141 static dwc_otg_cmd_t dwc_otg_data_rx;
 142 static dwc_otg_cmd_t dwc_otg_data_tx;
 143 static dwc_otg_cmd_t dwc_otg_data_tx_sync;
 144
 145 static dwc_otg_cmd_t dwc_otg_host_setup_tx;
 146 static dwc_otg_cmd_t dwc_otg_host_data_tx;
 147 static dwc_otg_cmd_t dwc_otg_host_data_rx;
 148
 149 static void dwc_otg_device_done(struct usb_xfer *, usb_error_t);
 150 static void dwc_otg_do_poll(struct usb_bus *);
 151 static void dwc_otg_standard_done(struct usb_xfer *);
 152 static void dwc_otg_root_intr(struct dwc_otg_softc *);
 153 static void dwc_otg_interrupt_poll_locked(struct dwc_otg_softc *);
 154 static void dwc_otg_host_channel_disable(struct dwc_otg_softc *, uint8_t);
 155
 156 /*
 157  * Here is a configuration that the chip supports.
 158  */
 159 static const struct usb_hw_ep_profile dwc_otg_ep_profile[1] = {
 160
 161         [0] = {
 162                 .max_in_frame_size = 64,/* fixed */
 163                 .max_out_frame_size = 64,       /* fixed */
 164                 .is_simplex = 1,
 165                 .support_control = 1,
 166         }
 167 };
 168
 169 static void
 170 dwc_otg_get_hw_ep_profile(struct usb_device *udev,
 171     const struct usb_hw_ep_profile **ppf, uint8_t ep_addr)
 172 {
 173         struct dwc_otg_softc *sc;
 174
 175         sc = DWC_OTG_BUS2SC(udev->bus);
 176
 177         if (ep_addr < sc->sc_dev_ep_max)
 178                 *ppf = &sc->sc_hw_ep_profile[ep_addr].usb;
 179         else
 180                 *ppf = NULL;
 181 }
 182
 183 static void
 184 dwc_otg_tx_fifo_reset(struct dwc_otg_softc *sc, uint32_t value)
 185 {
 186         uint32_t temp;
 187
 188         /* reset FIFO */
 189         DWC_OTG_WRITE_4(sc, DOTG_GRSTCTL, value);
 190
 191         /* wait for reset to complete */
 192         for (temp = 0; temp != 16; temp++) {
 193                 value = DWC_OTG_READ_4(sc, DOTG_GRSTCTL);
 194                 if (!(value & (GRSTCTL_TXFFLSH | GRSTCTL_RXFFLSH)))
 195                         break;
 196         }
 197 }
 198
 199 static int
 200 dwc_otg_init_fifo(struct dwc_otg_softc *sc, uint8_t mode)
 201 {
 202         struct dwc_otg_profile *pf;
 203         uint32_t fifo_size;
 204         uint32_t fifo_regs;
 205         uint32_t tx_start;
 206         uint8_t x;
 207
 208         fifo_size = sc->sc_fifo_size;
 209
 210         /*
 211          * NOTE: Reserved fixed size area at end of RAM, which must
 212          * not be allocated to the FIFOs:
 213          */
 214         fifo_regs = 4 * 16;
 215
 216         if (fifo_size < fifo_regs) {
 217                 DPRINTF("Too little FIFO\n");
 218                 return (EINVAL);
 219         }
 220
 221         /* subtract FIFO regs from total once */
 222         fifo_size -= fifo_regs;
 223
 224         /* split equally for IN and OUT */
 225         fifo_size /= 2;
 226
 227         /* align to 4 bytes boundary */
 228         fifo_size &= ~3;
 229
 230         /* set global receive FIFO size */
 231         DWC_OTG_WRITE_4(sc, DOTG_GRXFSIZ, fifo_size / 4);
 232
 233         tx_start = fifo_size;
 234
 235         if (fifo_size < 64) {
 236                 DPRINTFN(-1, "Not enough data space for EP0 FIFO.\n");
 237                 return (EINVAL);
 238         }
 239
 240         /* disable any leftover host channels */
 241         for (x = 0; x != sc->sc_host_ch_max; x++) {
 242                 if (sc->sc_chan_state[x].wait_sof == 0)
 243                         continue;
 244                 dwc_otg_host_channel_disable(sc, x);
 245         }
 246
 247         if (mode == DWC_MODE_HOST) {
 248
 249                 /* reset active endpoints */
 250                 sc->sc_active_rx_ep = 0;
 251
 252                 /* split equally for periodic and non-periodic */
 253                 fifo_size /= 2;
 254
 255                 /* align to 4 bytes boundary */
 256                 fifo_size &= ~3;
 257
 258                 DWC_OTG_WRITE_4(sc, DOTG_GNPTXFSIZ,
 259                     ((fifo_size / 4) << 16) |
 260                     (tx_start / 4));
 261
 262                 tx_start += fifo_size;
 263
 264                 for (x = 0; x != sc->sc_host_ch_max; x++) {
 265                         /* disable all host interrupts */
 266                         DWC_OTG_WRITE_4(sc, DOTG_HCINTMSK(x),
 267                             HCINT_DEFAULT_MASK);
 268                 }
 269
 270                 DWC_OTG_WRITE_4(sc, DOTG_HPTXFSIZ,
 271                     ((fifo_size / 4) << 16) |
 272                     (tx_start / 4));
 273
 274                 /* reset host channel state */
 275                 memset(sc->sc_chan_state, 0, sizeof(sc->sc_chan_state));
 276
 277                 /* reset FIFO TX levels */
 278                 sc->sc_tx_cur_p_level = 0;
 279                 sc->sc_tx_cur_np_level = 0;
 280
 281                 /* store maximum periodic and non-periodic FIFO TX size */
 282                 sc->sc_tx_max_size = fifo_size;
 283
 284                 /* enable all host channel interrupts */
 285                 DWC_OTG_WRITE_4(sc, DOTG_HAINTMSK,
 286                     (1U << sc->sc_host_ch_max) - 1U);
 287         }
 288
 289         if (mode == DWC_MODE_DEVICE) {
 290
 291             DWC_OTG_WRITE_4(sc, DOTG_GNPTXFSIZ,
 292                 (0x10 << 16) | (tx_start / 4));
 293             fifo_size -= 0x40;
 294             tx_start += 0x40;
 295
 296             /* setup control endpoint profile */
 297             sc->sc_hw_ep_profile[0].usb = dwc_otg_ep_profile[0];
 298
 299             /* reset active endpoints */
 300             sc->sc_active_rx_ep = 1;
 301
 302             for (x = 1; x != sc->sc_dev_ep_max; x++) {
 303
 304                 pf = sc->sc_hw_ep_profile + x;
 305
 306                 pf->usb.max_out_frame_size = 1024 * 3;
 307                 pf->usb.is_simplex = 0; /* assume duplex */
 308                 pf->usb.support_bulk = 1;
 309                 pf->usb.support_interrupt = 1;
 310                 pf->usb.support_isochronous = 1;
 311                 pf->usb.support_out = 1;
 312
 313                 if (x < sc->sc_dev_in_ep_max) {
 314                         uint32_t limit;
 315
 316                         limit = (x == 1) ? MIN(DWC_OTG_TX_MAX_FIFO_SIZE,
 317                             DWC_OTG_MAX_TXN) : MIN(DWC_OTG_MAX_TXN / 2,
 318                             DWC_OTG_TX_MAX_FIFO_SIZE);
 319
 320                         /* see if there is enough FIFO space */
 321                         if (limit <= fifo_size) {
 322                                 pf->max_buffer = limit;
 323                                 pf->usb.support_in = 1;
 324                         } else {
 325                                 limit = MIN(DWC_OTG_TX_MAX_FIFO_SIZE, 0x40);
 326                                 if (limit <= fifo_size) {
 327                                         pf->usb.support_in = 1;
 328                                 } else {
 329                                         pf->usb.is_simplex = 1;
 330                                         limit = 0;
 331                                 }
 332                         }
 333                         /* set FIFO size */
 334                         DWC_OTG_WRITE_4(sc, DOTG_DIEPTXF(x),
 335                             ((limit / 4) << 16) | (tx_start / 4));
 336                         tx_start += limit;
 337                         fifo_size -= limit;
 338                         pf->usb.max_in_frame_size = limit;
 339                 } else {
 340                         pf->usb.is_simplex = 1;
 341                 }
 342
 343                 DPRINTF("FIFO%d = IN:%d / OUT:%d\n", x,
 344                     pf->usb.max_in_frame_size,
 345                     pf->usb.max_out_frame_size);
 346             }
 347         }
 348
 349         /* reset RX FIFO */
 350         dwc_otg_tx_fifo_reset(sc, GRSTCTL_RXFFLSH);
 351
 352         if (mode != DWC_MODE_OTG) {
 353                 /* reset all TX FIFOs */
 354                 dwc_otg_tx_fifo_reset(sc,
 355                     GRSTCTL_TXFIFO(0x10) |
 356                     GRSTCTL_TXFFLSH);
 357         } else {
 358                 /* reset active endpoints */
 359                 sc->sc_active_rx_ep = 0;
 360
 361                 /* reset periodic and non-periodic FIFO TX size */
 362                 sc->sc_tx_max_size = fifo_size;
 363
 364                 /* reset host channel state */
 365                 memset(sc->sc_chan_state, 0, sizeof(sc->sc_chan_state));
 366
 367                 /* reset FIFO TX levels */
 368                 sc->sc_tx_cur_p_level = 0;
 369                 sc->sc_tx_cur_np_level = 0;
 370         }
 371         return (0);
 372 }
 373
 374 static void
 375 dwc_otg_update_host_frame_interval(struct dwc_otg_softc *sc)
 376 {
 377
 378   /*
 379    * Disabled until further. Assuming that the register is already
 380    * programmed correctly by the boot loader.
 381    */
 382 #if 0
 383         uint32_t temp;
 384
 385         /* setup HOST frame interval register, based on existing value */
 386         temp = DWC_OTG_READ_4(sc, DOTG_HFIR) & HFIR_FRINT_MASK;
 387         if (temp >= 10000)
 388                 temp /= 1000;
 389         else
 390                 temp /= 125;
 391
 392         /* figure out nearest X-tal value */
 393         if (temp >= 54)
 394                 temp = 60;      /* MHz */
 395         else if (temp >= 39)
 396                 temp = 48;      /* MHz */
 397         else
 398                 temp = 30;      /* MHz */
 399
 400         if (sc->sc_flags.status_high_speed)
 401                 temp *= 125;
 402         else
 403                 temp *= 1000;
 404
 405         DPRINTF("HFIR=0x%08x\n", temp);
 406
 407         DWC_OTG_WRITE_4(sc, DOTG_HFIR, temp);
 408 #endif
 409 }
 410
 411 static void
 412 dwc_otg_clocks_on(struct dwc_otg_softc *sc)
 413 {
 414         if (sc->sc_flags.clocks_off &&
 415             sc->sc_flags.port_powered) {
 416
 417                 DPRINTFN(5, "\n");
 418
 419                 /* TODO - platform specific */
 420
 421                 sc->sc_flags.clocks_off = 0;
 422         }
 423 }
 424
 425 static void
 426 dwc_otg_clocks_off(struct dwc_otg_softc *sc)
 427 {
 428         if (!sc->sc_flags.clocks_off) {
 429
 430                 DPRINTFN(5, "\n");
 431
 432                 /* TODO - platform specific */
 433
 434                 sc->sc_flags.clocks_off = 1;
 435         }
 436 }
 437
 438 static void
 439 dwc_otg_pull_up(struct dwc_otg_softc *sc)
 440 {
 441         uint32_t temp;
 442
 443         /* pullup D+, if possible */
 444
 445         if (!sc->sc_flags.d_pulled_up &&
 446             sc->sc_flags.port_powered) {
 447                 sc->sc_flags.d_pulled_up = 1;
 448
 449                 temp = DWC_OTG_READ_4(sc, DOTG_DCTL);
 450                 temp &= ~DCTL_SFTDISCON;
 451                 DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
 452         }
 453 }
 454
 455 static void
 456 dwc_otg_pull_down(struct dwc_otg_softc *sc)
 457 {
 458         uint32_t temp;
 459
 460         /* pulldown D+, if possible */
 461
 462         if (sc->sc_flags.d_pulled_up) {
 463                 sc->sc_flags.d_pulled_up = 0;
 464
 465                 temp = DWC_OTG_READ_4(sc, DOTG_DCTL);
 466                 temp |= DCTL_SFTDISCON;
 467                 DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
 468         }
 469 }
 470
 471 static void
 472 dwc_otg_enable_sof_irq(struct dwc_otg_softc *sc)
 473 {
 474         /* In device mode we don't use the SOF interrupt */
 475         if (sc->sc_flags.status_device_mode != 0 ||
 476             (sc->sc_irq_mask & GINTMSK_SOFMSK) != 0)
 477                 return;
 478         sc->sc_irq_mask |= GINTMSK_SOFMSK;
 479         DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 480 }
 481
 482 static void
 483 dwc_otg_resume_irq(struct dwc_otg_softc *sc)
 484 {
 485         if (sc->sc_flags.status_suspend) {
 486                 /* update status bits */
 487                 sc->sc_flags.status_suspend = 0;
 488                 sc->sc_flags.change_suspend = 1;
 489
 490                 if (sc->sc_flags.status_device_mode) {
 491                         /*
 492                          * Disable resume interrupt and enable suspend
 493                          * interrupt:
 494                          */
 495                         sc->sc_irq_mask &= ~GINTMSK_WKUPINTMSK;
 496                         sc->sc_irq_mask |= GINTMSK_USBSUSPMSK;
 497                         DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 498                 }
 499
 500                 /* complete root HUB interrupt endpoint */
 501                 dwc_otg_root_intr(sc);
 502         }
 503 }
 504
 505 static void
 506 dwc_otg_suspend_irq(struct dwc_otg_softc *sc)
 507 {
 508         if (!sc->sc_flags.status_suspend) {
 509                 /* update status bits */
 510                 sc->sc_flags.status_suspend = 1;
 511                 sc->sc_flags.change_suspend = 1;
 512
 513                 if (sc->sc_flags.status_device_mode) {
 514                         /*
 515                          * Disable suspend interrupt and enable resume
 516                          * interrupt:
 517                          */
 518                         sc->sc_irq_mask &= ~GINTMSK_USBSUSPMSK;
 519                         sc->sc_irq_mask |= GINTMSK_WKUPINTMSK;
 520                         DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 521                 }
 522
 523                 /* complete root HUB interrupt endpoint */
 524                 dwc_otg_root_intr(sc);
 525         }
 526 }
 527
 528 static void
 529 dwc_otg_wakeup_peer(struct dwc_otg_softc *sc)
 530 {
 531         if (!sc->sc_flags.status_suspend)
 532                 return;
 533
 534         DPRINTFN(5, "Remote wakeup\n");
 535
 536         if (sc->sc_flags.status_device_mode) {
 537                 uint32_t temp;
 538
 539                 /* enable remote wakeup signalling */
 540                 temp = DWC_OTG_READ_4(sc, DOTG_DCTL);
 541                 temp |= DCTL_RMTWKUPSIG;
 542                 DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
 543
 544                 /* Wait 8ms for remote wakeup to complete. */
 545                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 125);
 546
 547                 temp &= ~DCTL_RMTWKUPSIG;
 548                 DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
 549         } else {
 550                 /* enable USB port */
 551                 DWC_OTG_WRITE_4(sc, DOTG_PCGCCTL, 0);
 552
 553                 /* wait 10ms */
 554                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 100);
 555
 556                 /* resume port */
 557                 sc->sc_hprt_val |= HPRT_PRTRES;
 558                 DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
 559
 560                 /* Wait 100ms for resume signalling to complete. */
 561                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 10);
 562
 563                 /* clear suspend and resume */
 564                 sc->sc_hprt_val &= ~(HPRT_PRTSUSP | HPRT_PRTRES);
 565                 DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
 566
 567                 /* Wait 4ms */
 568                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 250);
 569         }
 570
 571         /* need to fake resume IRQ */
 572         dwc_otg_resume_irq(sc);
 573 }
 574
 575 static void
 576 dwc_otg_set_address(struct dwc_otg_softc *sc, uint8_t addr)
 577 {
 578         uint32_t temp;
 579
 580         DPRINTFN(5, "addr=%d\n", addr);
 581
 582         temp = DWC_OTG_READ_4(sc, DOTG_DCFG);
 583         temp &= ~DCFG_DEVADDR_SET(0x7F);
 584         temp |= DCFG_DEVADDR_SET(addr);
 585         DWC_OTG_WRITE_4(sc, DOTG_DCFG, temp);
 586 }
 587
 588 static void
 589 dwc_otg_common_rx_ack(struct dwc_otg_softc *sc)
 590 {
 591         DPRINTFN(5, "RX status clear\n");
 592
 593         /* enable RX FIFO level interrupt */
 594         sc->sc_irq_mask |= GINTMSK_RXFLVLMSK;
 595         DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 596
 597         /* clear cached status */
 598         sc->sc_last_rx_status = 0;
 599 }
 600
 601 static void
 602 dwc_otg_clear_hcint(struct dwc_otg_softc *sc, uint8_t x)
 603 {
 604         uint32_t hcint;
 605
 606         /* clear all pending interrupts */
 607         hcint = DWC_OTG_READ_4(sc, DOTG_HCINT(x));
 608         DWC_OTG_WRITE_4(sc, DOTG_HCINT(x), hcint);
 609
 610         /* clear buffered interrupts */
 611         sc->sc_chan_state[x].hcint = 0;
 612 }
 613
 614 static uint8_t
 615 dwc_otg_host_channel_alloc(struct dwc_otg_softc *sc, struct dwc_otg_td *td, uint8_t is_out)
 616 {
 617         uint32_t tx_p_size;
 618         uint32_t tx_np_size;
 619         uint8_t x;
 620
 621         if (td->channel < DWC_OTG_MAX_CHANNELS)
 622                 return (0);             /* already allocated */
 623
 624         /* check if device is suspended */
 625         if (DWC_OTG_PC2UDEV(td->pc)->flags.self_suspended != 0)
 626                 return (1);             /* busy - cannot transfer data */
 627
 628         /* compute needed TX FIFO size */
 629         if (is_out != 0) {
 630                 if (td->ep_type == UE_ISOCHRONOUS) {
 631                         tx_p_size = td->max_packet_size;
 632                         tx_np_size = 0;
 633                         if (td->hcsplt != 0 && tx_p_size > HCSPLT_XACTLEN_BURST)
 634                                 tx_p_size = HCSPLT_XACTLEN_BURST;
 635                         if ((sc->sc_tx_cur_p_level + tx_p_size) > sc->sc_tx_max_size) {
 636                                 DPRINTF("Too little FIFO space\n");
 637                                 return (1);     /* too little FIFO */
 638                         }
 639                 } else {
 640                         tx_p_size = 0;
 641                         tx_np_size = td->max_packet_size;
 642                         if (td->hcsplt != 0 && tx_np_size > HCSPLT_XACTLEN_BURST)
 643                                 tx_np_size = HCSPLT_XACTLEN_BURST;
 644                         if ((sc->sc_tx_cur_np_level + tx_np_size) > sc->sc_tx_max_size) {
 645                                 DPRINTF("Too little FIFO space\n");
 646                                 return (1);     /* too little FIFO */
 647                         }
 648                 }
 649         } else {
 650                 /* not a TX transaction */
 651                 tx_p_size = 0;
 652                 tx_np_size = 0;
 653         }
 654
 655         for (x = 0; x != sc->sc_host_ch_max; x++) {
 656                 if (sc->sc_chan_state[x].allocated != 0)
 657                         continue;
 658                 /* check if channel is still enabled */
 659                 if (sc->sc_chan_state[x].wait_sof != 0)
 660                         continue;
 661
 662                 sc->sc_chan_state[x].allocated = 1;
 663                 sc->sc_chan_state[x].tx_p_size = tx_p_size;
 664                 sc->sc_chan_state[x].tx_np_size = tx_np_size;
 665
 666                 /* keep track of used TX FIFO, if any */
 667                 sc->sc_tx_cur_p_level += tx_p_size;
 668                 sc->sc_tx_cur_np_level += tx_np_size;
 669
 670                 /* clear interrupts */
 671                 dwc_otg_clear_hcint(sc, x);
 672
 673                 DPRINTF("CH=%d HCCHAR=0x%08x "
 674                     "HCSPLT=0x%08x\n", x, td->hcchar, td->hcsplt);
 675
 676                 /* set active channel */
 677                 sc->sc_active_rx_ep |= (1 << x);
 678
 679                 /* set channel */
 680                 td->channel = x;
 681
 682                 return (0);     /* allocated */
 683         }
 684         /* wait a bit */
 685         dwc_otg_enable_sof_irq(sc);
 686         return (1);     /* busy */
 687 }
 688
 689 static void
 690 dwc_otg_host_channel_free(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
 691 {
 692         uint8_t x;
 693
 694         if (td->channel >= DWC_OTG_MAX_CHANNELS)
 695                 return;         /* already freed */
 696
 697         /* free channel */
 698         x = td->channel;
 699         td->channel = DWC_OTG_MAX_CHANNELS;
 700
 701         DPRINTF("CH=%d\n", x);
 702
 703         /*
 704          * We need to let programmed host channels run till complete
 705          * else the host channel will stop functioning. Assume that
 706          * after a fixed given amount of time the host channel is no
 707          * longer doing any USB traffic:
 708          */
 709         if (td->ep_type == UE_ISOCHRONOUS) {
 710                 /* double buffered */
 711                 sc->sc_chan_state[x].wait_sof = DWC_OTG_SLOT_IDLE_MAX;
 712         } else {
 713                 /* single buffered */
 714                 sc->sc_chan_state[x].wait_sof = DWC_OTG_SLOT_IDLE_MIN;
 715         }
 716
 717         sc->sc_chan_state[x].allocated = 0;
 718
 719         /* ack any pending messages */
 720         if (sc->sc_last_rx_status != 0 &&
 721             GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) == x) {
 722                 dwc_otg_common_rx_ack(sc);
 723         }
 724
 725         /* clear active channel */
 726         sc->sc_active_rx_ep &= ~(1 << x);
 727 }
 728
 729 static void
 730 dwc_otg_host_dump_rx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
 731 {
 732         /* dump any pending messages */
 733         if (sc->sc_last_rx_status != 0) {
 734                 if (td->channel < DWC_OTG_MAX_CHANNELS &&
 735                     td->channel == GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status)) {
 736                         dwc_otg_common_rx_ack(sc);
 737                 }
 738         }
 739 }
 740
 741 static uint8_t
 742 dwc_otg_host_setup_tx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
 743 {
 744         struct usb_device_request req __aligned(4);
 745         uint32_t hcint;
 746         uint32_t hcchar;
 747         uint8_t delta;
 748
 749         dwc_otg_host_dump_rx(sc, td);
 750
 751         if (td->channel < DWC_OTG_MAX_CHANNELS) {
 752                 hcint = sc->sc_chan_state[td->channel].hcint;
 753
 754                 DPRINTF("CH=%d ST=%d HCINT=0x%08x HCCHAR=0x%08x HCTSIZ=0x%08x\n",
 755                     td->channel, td->state, hcint,
 756                     DWC_OTG_READ_4(sc, DOTG_HCCHAR(td->channel)),
 757                     DWC_OTG_READ_4(sc, DOTG_HCTSIZ(td->channel)));
 758         } else {
 759                 hcint = 0;
 760                 goto check_state;
 761         }
 762
 763         if (hcint & (HCINT_RETRY |
 764             HCINT_ACK | HCINT_NYET)) {
 765                 /* give success bits priority over failure bits */
 766         } else if (hcint & HCINT_STALL) {
 767                 DPRINTF("CH=%d STALL\n", td->channel);
 768                 td->error_stall = 1;
 769                 td->error_any = 1;
 770                 goto complete;
 771         } else if (hcint & HCINT_ERRORS) {
 772                 DPRINTF("CH=%d ERROR\n", td->channel);
 773                 td->errcnt++;
 774                 if (td->hcsplt != 0 || td->errcnt >= 3) {
 775                         td->error_any = 1;
 776                         goto complete;
 777                 }
 778         }
 779
 780         if (hcint & (HCINT_ERRORS | HCINT_RETRY |
 781             HCINT_ACK | HCINT_NYET)) {
 782                 if (!(hcint & HCINT_ERRORS))
 783                         td->errcnt = 0;
 784         }
 785
 786 check_state:
 787         switch (td->state) {
 788         case DWC_CHAN_ST_START:
 789                 goto send_pkt;
 790
 791         case DWC_CHAN_ST_WAIT_ANE:
 792                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
 793                         td->did_nak++;
 794                         td->tt_scheduled = 0;
 795                         goto send_pkt;
 796                 } else if (hcint & (HCINT_ACK | HCINT_NYET)) {
 797                         td->offset += td->tx_bytes;
 798                         td->remainder -= td->tx_bytes;
 799                         td->toggle = 1;
 800                         td->tt_scheduled = 0;
 801                         goto complete;
 802                 }
 803                 break;
 804
 805         case DWC_CHAN_ST_WAIT_S_ANE:
 806                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
 807                         td->did_nak++;
 808                         td->tt_scheduled = 0;
 809                         goto send_pkt;
 810                 } else if (hcint & (HCINT_ACK | HCINT_NYET)) {
 811                         goto send_cpkt;
 812                 }
 813                 break;
 814
 815         case DWC_CHAN_ST_WAIT_C_ANE:
 816                 if (hcint & HCINT_NYET) {
 817                         goto send_cpkt;
 818                 } else if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
 819                         td->did_nak++;
 820                         td->tt_scheduled = 0;
 821                         goto send_pkt;
 822                 } else if (hcint & HCINT_ACK) {
 823                         td->offset += td->tx_bytes;
 824                         td->remainder -= td->tx_bytes;
 825                         td->toggle = 1;
 826                         goto complete;
 827                 }
 828                 break;
 829
 830         case DWC_CHAN_ST_WAIT_C_PKT:
 831                 goto send_cpkt;
 832
 833         default:
 834                 break;
 835         }
 836         goto busy;
 837
 838 send_pkt:
 839         /* free existing channel, if any */
 840         dwc_otg_host_channel_free(sc, td);
 841
 842         if (sizeof(req) != td->remainder) {
 843                 td->error_any = 1;
 844                 goto complete;
 845         }
 846
 847         if (td->hcsplt != 0) {
 848                 delta = td->tt_start_slot - sc->sc_last_frame_num - 1;
 849                 if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
 850                         td->state = DWC_CHAN_ST_START;
 851                         goto busy;
 852                 }
 853                 delta = sc->sc_last_frame_num - td->tt_start_slot;
 854                 if (delta > 5) {
 855                         /* missed it */
 856                         td->tt_scheduled = 0;
 857                         td->state = DWC_CHAN_ST_START;
 858                         goto busy;
 859                 }
 860         }
 861
 862         /* allocate a new channel */
 863         if (dwc_otg_host_channel_alloc(sc, td, 1)) {
 864                 td->state = DWC_CHAN_ST_START;
 865                 goto busy;
 866         }
 867
 868         if (td->hcsplt != 0) {
 869                 td->hcsplt &= ~HCSPLT_COMPSPLT;
 870                 td->state = DWC_CHAN_ST_WAIT_S_ANE;
 871         } else {
 872                 td->state = DWC_CHAN_ST_WAIT_ANE;
 873         }
 874
 875         usbd_copy_out(td->pc, 0, &req, sizeof(req));
 876
 877         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(td->channel),
 878             (sizeof(req) << HCTSIZ_XFERSIZE_SHIFT) |
 879             (1 << HCTSIZ_PKTCNT_SHIFT) |
 880             (HCTSIZ_PID_SETUP << HCTSIZ_PID_SHIFT));
 881
 882         DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(td->channel), td->hcsplt);
 883
 884         hcchar = td->hcchar;
 885         hcchar &= ~(HCCHAR_EPDIR_IN | HCCHAR_EPTYPE_MASK);
 886         hcchar |= UE_CONTROL << HCCHAR_EPTYPE_SHIFT;
 887
 888         /* must enable channel before writing data to FIFO */
 889         DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(td->channel), hcchar);
 890
 891         /* transfer data into FIFO */
 892         bus_space_write_region_4(sc->sc_io_tag, sc->sc_io_hdl,
 893             DOTG_DFIFO(td->channel), (uint32_t *)&req, sizeof(req) / 4);
 894
 895         /* wait until next slot before trying complete split */
 896         td->tt_complete_slot = sc->sc_last_frame_num + 1;
 897
 898         /* store number of bytes transmitted */
 899         td->tx_bytes = sizeof(req);
 900         goto busy;
 901
 902 send_cpkt:
 903         /* free existing channel, if any */
 904         dwc_otg_host_channel_free(sc, td);
 905
 906         delta = td->tt_complete_slot - sc->sc_last_frame_num - 1;
 907         if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
 908                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
 909                 goto busy;
 910         }
 911         delta = sc->sc_last_frame_num - td->tt_start_slot;
 912         if (delta > DWC_OTG_TT_SLOT_MAX) {
 913                 /* we missed the service interval */
 914                 if (td->ep_type != UE_ISOCHRONOUS)
 915                         td->error_any = 1;
 916                 goto complete;
 917         }
 918         /* allocate a new channel */
 919         if (dwc_otg_host_channel_alloc(sc, td, 0)) {
 920                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
 921                 goto busy;
 922         }
 923
 924         /* wait until next slot before trying complete split */
 925         td->tt_complete_slot = sc->sc_last_frame_num + 1;
 926
 927         td->hcsplt |= HCSPLT_COMPSPLT;
 928         td->state = DWC_CHAN_ST_WAIT_C_ANE;
 929
 930         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(td->channel),
 931             (HCTSIZ_PID_SETUP << HCTSIZ_PID_SHIFT));
 932
 933         DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(td->channel), td->hcsplt);
 934
 935         hcchar = td->hcchar;
 936         hcchar &= ~(HCCHAR_EPDIR_IN | HCCHAR_EPTYPE_MASK);
 937         hcchar |= UE_CONTROL << HCCHAR_EPTYPE_SHIFT;
 938
 939         /* must enable channel before writing data to FIFO */
 940         DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(td->channel), hcchar);
 941
 942 busy:
 943         return (1);     /* busy */
 944
 945 complete:
 946         dwc_otg_host_channel_free(sc, td);
 947         return (0);     /* complete */
 948 }
 949
 950 static uint8_t
 951 dwc_otg_setup_rx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
 952 {
 953         struct usb_device_request req __aligned(4);
 954         uint32_t temp;
 955         uint16_t count;
 956
 957         /* check endpoint status */
 958
 959         if (sc->sc_last_rx_status == 0)
 960                 goto not_complete;
 961
 962         if (GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) != 0)
 963                 goto not_complete;
 964
 965         if ((sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) !=
 966             GRXSTSRD_STP_DATA) {
 967                 if ((sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) !=
 968                     GRXSTSRD_STP_COMPLETE || td->remainder != 0) {
 969                         /* release FIFO */
 970                         dwc_otg_common_rx_ack(sc);
 971                         goto not_complete;
 972                 }
 973                 /* release FIFO */
 974                 dwc_otg_common_rx_ack(sc);
 975                 return (0);     /* complete */
 976         }
 977
 978         if ((sc->sc_last_rx_status & GRXSTSRD_DPID_MASK) !=
 979             GRXSTSRD_DPID_DATA0) {
 980                 /* release FIFO */
 981                 dwc_otg_common_rx_ack(sc);
 982                 goto not_complete;
 983         }
 984
 985         DPRINTFN(5, "GRXSTSR=0x%08x\n", sc->sc_last_rx_status);
 986
 987         /* clear did stall */
 988         td->did_stall = 0;
 989
 990         /* get the packet byte count */
 991         count = GRXSTSRD_BCNT_GET(sc->sc_last_rx_status);
 992
 993         if (count != sizeof(req)) {
 994                 DPRINTFN(0, "Unsupported SETUP packet "
 995                     "length, %d bytes\n", count);
 996                 /* release FIFO */
 997                 dwc_otg_common_rx_ack(sc);
 998                 goto not_complete;
 999         }
1000
1001         /* copy in control request */
1002         memcpy(&req, sc->sc_rx_bounce_buffer, sizeof(req));
1003
1004         /* copy data into real buffer */
1005         usbd_copy_in(td->pc, 0, &req, sizeof(req));
1006
1007         td->offset = sizeof(req);
1008         td->remainder = 0;
1009
1010         /* sneak peek the set address */
1011         if ((req.bmRequestType == UT_WRITE_DEVICE) &&
1012             (req.bRequest == UR_SET_ADDRESS)) {
1013                 /* must write address before ZLP */
1014                 dwc_otg_set_address(sc, req.wValue[0] & 0x7F);
1015         }
1016
1017         /* don't send any data by default */
1018         DWC_OTG_WRITE_4(sc, DOTG_DIEPTSIZ(0), DIEPCTL_EPDIS);
1019         DWC_OTG_WRITE_4(sc, DOTG_DOEPTSIZ(0), DOEPCTL_EPDIS);
1020
1021         /* reset IN endpoint buffer */
1022         dwc_otg_tx_fifo_reset(sc,
1023             GRSTCTL_TXFIFO(0) |
1024             GRSTCTL_TXFFLSH);
1025
1026         /* acknowledge RX status */
1027         dwc_otg_common_rx_ack(sc);
1028         td->did_stall = 1;
1029
1030 not_complete:
1031         /* abort any ongoing transfer, before enabling again */
1032         if (!td->did_stall) {
1033                 td->did_stall = 1;
1034
1035                 DPRINTFN(5, "stalling IN and OUT direction\n");
1036
1037                 temp = sc->sc_out_ctl[0];
1038
1039                 /* set stall after enabling endpoint */
1040                 DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(0),
1041                     temp | DOEPCTL_STALL);
1042
1043                 temp = sc->sc_in_ctl[0];
1044
1045                 /* set stall assuming endpoint is enabled */
1046                 DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(0),
1047                     temp | DIEPCTL_STALL);
1048         }
1049         return (1);                     /* not complete */
1050 }
1051
1052 static uint8_t
1053 dwc_otg_host_rate_check_interrupt(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1054 {
1055         uint8_t delta;
1056
1057         delta = sc->sc_tmr_val - td->tmr_val;
1058         if (delta >= 128)
1059                 return (1);     /* busy */
1060
1061         td->tmr_val = sc->sc_tmr_val + td->tmr_res;
1062
1063         /* set toggle, if any */
1064         if (td->set_toggle) {
1065                 td->set_toggle = 0;
1066                 td->toggle = 1;
1067         }
1068         return (0);
1069 }
1070
1071 static uint8_t
1072 dwc_otg_host_rate_check(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1073 {
1074         if (td->ep_type == UE_ISOCHRONOUS) {
1075                 /* non TT isochronous traffic */
1076                 if ((td->tmr_val != 0) ||
1077                     (sc->sc_last_frame_num & (td->tmr_res - 1))) {
1078                         goto busy;
1079                 }
1080                 td->tmr_val = 1;        /* executed */
1081                 td->toggle = 0;
1082
1083         } else if (td->ep_type == UE_INTERRUPT) {
1084                 if (!td->tt_scheduled)
1085                         goto busy;
1086                 td->tt_scheduled = 0;
1087         } else if (td->did_nak >= DWC_OTG_NAK_MAX) {
1088                 goto busy;
1089         } else if (td->set_toggle) {
1090                 td->set_toggle = 0;
1091                 td->toggle = 1;
1092         }
1093         return (0);
1094 busy:
1095         return (1);
1096 }
1097
1098 static uint8_t
1099 dwc_otg_host_data_rx_sub(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1100 {
1101         uint32_t count;
1102         uint8_t channel;
1103
1104         /* check endpoint status */
1105         if (sc->sc_last_rx_status == 0)
1106                 goto busy;
1107
1108         channel = td->channel;
1109         if (channel >= DWC_OTG_MAX_CHANNELS)
1110                 goto busy;
1111
1112         if (GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) != channel)
1113                 goto busy;
1114
1115         switch (sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) {
1116         case GRXSTSRH_IN_DATA:
1117
1118                 DPRINTF("DATA ST=%d STATUS=0x%08x\n",
1119                     (int)td->state, (int)sc->sc_last_rx_status);
1120
1121                 if (sc->sc_chan_state[channel].hcint & HCINT_SOFTWARE_ONLY) {
1122                         /*
1123                          * When using SPLIT transactions on interrupt
1124                          * endpoints, sometimes data occurs twice.
1125                          */
1126                         DPRINTF("Data already received\n");
1127                         break;
1128                 }
1129
1130                 /* get the packet byte count */
1131                 count = GRXSTSRD_BCNT_GET(sc->sc_last_rx_status);
1132
1133                 /* check for isochronous transfer or high-speed bandwidth endpoint */
1134                 if (td->ep_type == UE_ISOCHRONOUS || td->max_packet_count > 1) {
1135                         if ((sc->sc_last_rx_status & GRXSTSRD_DPID_MASK) != GRXSTSRD_DPID_DATA0) {
1136                                 td->tt_xactpos = HCSPLT_XACTPOS_MIDDLE;
1137                         } else {
1138                                 td->tt_xactpos = HCSPLT_XACTPOS_BEGIN;
1139
1140                                 /* verify the packet byte count */
1141                                 if (count < td->max_packet_size) {
1142                                         /* we have a short packet */
1143                                         td->short_pkt = 1;
1144                                         td->got_short = 1;
1145                                 }
1146                         }
1147                         td->toggle = 0;
1148                 } else {
1149                         /* verify the packet byte count */
1150                         if (count != td->max_packet_size) {
1151                                 if (count < td->max_packet_size) {
1152                                         /* we have a short packet */
1153                                         td->short_pkt = 1;
1154                                         td->got_short = 1;
1155                                 } else {
1156                                         /* invalid USB packet */
1157                                         td->error_any = 1;
1158
1159                                         /* release FIFO */
1160                                         dwc_otg_common_rx_ack(sc);
1161                                         goto complete;
1162                                 }
1163                         }
1164                         td->toggle ^= 1;
1165                         td->tt_scheduled = 0;
1166                 }
1167
1168                 /* verify the packet byte count */
1169                 if (count > td->remainder) {
1170                         /* invalid USB packet */
1171                         td->error_any = 1;
1172
1173                         /* release FIFO */
1174                         dwc_otg_common_rx_ack(sc);
1175                         goto complete;
1176                 }
1177
1178                 usbd_copy_in(td->pc, td->offset,
1179                     sc->sc_rx_bounce_buffer, count);
1180
1181                 td->remainder -= count;
1182                 td->offset += count;
1183                 sc->sc_chan_state[channel].hcint |= HCINT_SOFTWARE_ONLY;
1184                 break;
1185         default:
1186                 break;
1187         }
1188         /* release FIFO */
1189         dwc_otg_common_rx_ack(sc);
1190 busy:
1191         return (0);
1192 complete:
1193         return (1);
1194 }
1195
1196 static uint8_t
1197 dwc_otg_host_data_rx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1198 {
1199         uint32_t hcint;
1200         uint32_t hcchar;
1201         uint8_t delta;
1202         uint8_t channel;
1203
1204         channel = td->channel;
1205
1206         if (channel < DWC_OTG_MAX_CHANNELS) {
1207                 hcint = sc->sc_chan_state[channel].hcint;
1208
1209                 DPRINTF("CH=%d ST=%d HCINT=0x%08x HCCHAR=0x%08x HCTSIZ=0x%08x\n",
1210                     channel, td->state, hcint,
1211                     DWC_OTG_READ_4(sc, DOTG_HCCHAR(channel)),
1212                     DWC_OTG_READ_4(sc, DOTG_HCTSIZ(channel)));
1213
1214                 /* check interrupt bits */
1215                 if (hcint & (HCINT_RETRY |
1216                     HCINT_ACK | HCINT_NYET)) {
1217                         /* give success bits priority over failure bits */
1218                 } else if (hcint & HCINT_STALL) {
1219                         DPRINTF("CH=%d STALL\n", channel);
1220                         td->error_stall = 1;
1221                         td->error_any = 1;
1222                         goto complete;
1223                 } else if (hcint & HCINT_ERRORS) {
1224                         DPRINTF("CH=%d ERROR\n", channel);
1225                         td->errcnt++;
1226                         if (td->hcsplt != 0 || td->errcnt >= 3) {
1227                                 if (td->ep_type != UE_ISOCHRONOUS) {
1228                                         td->error_any = 1;
1229                                         goto complete;
1230                                 }
1231                         }
1232                 }
1233
1234                 /* check channels for data, if any */
1235                 if (dwc_otg_host_data_rx_sub(sc, td))
1236                         goto complete;
1237
1238                 /* refresh interrupt status */
1239                 hcint = sc->sc_chan_state[channel].hcint;
1240
1241                 if (hcint & (HCINT_ERRORS | HCINT_RETRY |
1242                     HCINT_ACK | HCINT_NYET)) {
1243                         if (!(hcint & HCINT_ERRORS))
1244                                 td->errcnt = 0;
1245                 }
1246         } else {
1247                 hcint = 0;
1248         }
1249
1250         switch (td->state) {
1251         case DWC_CHAN_ST_START:
1252                 if (td->hcsplt != 0)
1253                         goto receive_spkt;
1254                 else
1255                         goto receive_pkt;
1256
1257         case DWC_CHAN_ST_WAIT_ANE:
1258                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1259                         if (td->ep_type == UE_INTERRUPT) {
1260                                 /*
1261                                  * The USB specification does not
1262                                  * mandate a particular data toggle
1263                                  * value for USB INTERRUPT
1264                                  * transfers. Switch the data toggle
1265                                  * value to receive the packet
1266                                  * correctly:
1267                                  */
1268                                 if (hcint & HCINT_DATATGLERR) {
1269                                         DPRINTF("Retrying packet due to "
1270                                             "data toggle error\n");
1271                                         td->toggle ^= 1;
1272                                         goto receive_pkt;
1273                                 }
1274                         }
1275                         td->did_nak++;
1276                         td->tt_scheduled = 0;
1277                         if (td->hcsplt != 0)
1278                                 goto receive_spkt;
1279                         else
1280                                 goto receive_pkt;
1281                 } else if (hcint & HCINT_NYET) {
1282                         if (td->hcsplt != 0) {
1283                                 /* try again */
1284                                 goto receive_pkt;
1285                         } else {
1286                                 /* not a valid token for IN endpoints */
1287                                 td->error_any = 1;
1288                                 goto complete;
1289                         }
1290                 } else if (hcint & HCINT_ACK) {
1291                         /* wait for data - ACK arrived first */
1292                         if (!(hcint & HCINT_SOFTWARE_ONLY))
1293                                 goto busy;
1294
1295                         if (td->ep_type == UE_ISOCHRONOUS) {
1296                                 /* check if we are complete */
1297                                 if ((td->remainder == 0) ||
1298                                     (td->tt_xactpos == HCSPLT_XACTPOS_BEGIN)) {
1299                                         goto complete;
1300                                 }
1301                                 /* get another packet */
1302                                 goto receive_pkt;
1303                         } else {
1304                                 /* check if we are complete */
1305                                 if ((td->remainder == 0) || (td->got_short != 0)) {
1306                                         if (td->short_pkt)
1307                                                 goto complete;
1308
1309                                         /*
1310                                          * Else need to receive a zero length
1311                                          * packet.
1312                                          */
1313                                 }
1314                                 td->tt_scheduled = 0;
1315                                 td->did_nak = 0;
1316                                 if (td->hcsplt != 0)
1317                                         goto receive_spkt;
1318                                 else
1319                                         goto receive_pkt;
1320                         }
1321                 }
1322                 break;
1323
1324         case DWC_CHAN_ST_WAIT_S_ANE:
1325                 /*
1326                  * NOTE: The DWC OTG hardware provides a fake ACK in
1327                  * case of interrupt and isochronous transfers:
1328                  */
1329                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1330                         td->did_nak++;
1331                         td->tt_scheduled = 0;
1332                         goto receive_spkt;
1333                 } else if (hcint & HCINT_NYET) {
1334                         td->tt_scheduled = 0;
1335                         goto receive_spkt;
1336                 } else if (hcint & HCINT_ACK) {
1337                         td->did_nak = 0;
1338                         goto receive_pkt;
1339                 }
1340                 break;
1341
1342         case DWC_CHAN_ST_WAIT_C_PKT:
1343                 goto receive_pkt;
1344
1345         default:
1346                 break;
1347         }
1348         goto busy;
1349
1350 receive_pkt:
1351         /* free existing channel, if any */
1352         dwc_otg_host_channel_free(sc, td);
1353
1354         if (td->hcsplt != 0) {
1355                 delta = td->tt_complete_slot - sc->sc_last_frame_num - 1;
1356                 if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
1357                         td->state = DWC_CHAN_ST_WAIT_C_PKT;
1358                         goto busy;
1359                 }
1360                 delta = sc->sc_last_frame_num - td->tt_start_slot;
1361                 if (delta > DWC_OTG_TT_SLOT_MAX) {
1362                         if (td->ep_type != UE_ISOCHRONOUS) {
1363                                 /* we missed the service interval */
1364                                 td->error_any = 1;
1365                         }
1366                         goto complete;
1367                 }
1368                 /* complete split */
1369                 td->hcsplt |= HCSPLT_COMPSPLT;
1370         } else if (td->tt_xactpos == HCSPLT_XACTPOS_BEGIN &&
1371             dwc_otg_host_rate_check(sc, td)) {
1372                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
1373                 goto busy;
1374         }
1375
1376         /* allocate a new channel */
1377         if (dwc_otg_host_channel_alloc(sc, td, 0)) {
1378                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
1379                 goto busy;
1380         }
1381
1382         channel = td->channel;
1383
1384         /* set toggle, if any */
1385         if (td->set_toggle) {
1386                 td->set_toggle = 0;
1387                 td->toggle = 1;
1388         }
1389
1390         td->state = DWC_CHAN_ST_WAIT_ANE;
1391
1392         /* receive one packet */
1393         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1394             (td->max_packet_size << HCTSIZ_XFERSIZE_SHIFT) |
1395             (1 << HCTSIZ_PKTCNT_SHIFT) |
1396             (td->toggle ? (HCTSIZ_PID_DATA1 << HCTSIZ_PID_SHIFT) :
1397             (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT)));
1398
1399         DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(channel), td->hcsplt);
1400
1401         hcchar = td->hcchar;
1402         hcchar |= HCCHAR_EPDIR_IN;
1403
1404         /* receive complete split ASAP */
1405         if ((sc->sc_last_frame_num & 1) != 0)
1406                 hcchar |= HCCHAR_ODDFRM;
1407         else
1408                 hcchar &= ~HCCHAR_ODDFRM;
1409
1410         /* must enable channel before data can be received */
1411         DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(channel), hcchar);
1412
1413         /* wait until next slot before trying complete split */
1414         td->tt_complete_slot = sc->sc_last_frame_num + 1;
1415
1416         goto busy;
1417
1418 receive_spkt:
1419         /* free existing channel(s), if any */
1420         dwc_otg_host_channel_free(sc, td);
1421
1422         delta = td->tt_start_slot - sc->sc_last_frame_num - 1;
1423         if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
1424                 td->state = DWC_CHAN_ST_START;
1425                 goto busy;
1426         }
1427         delta = sc->sc_last_frame_num - td->tt_start_slot;
1428         if (delta > 5) {
1429                 /* missed it */
1430                 td->tt_scheduled = 0;
1431                 td->state = DWC_CHAN_ST_START;
1432                 goto busy;
1433         }
1434
1435         /* allocate a new channel */
1436         if (dwc_otg_host_channel_alloc(sc, td, 0)) {
1437                 td->state = DWC_CHAN_ST_START;
1438                 goto busy;
1439         }
1440
1441         channel = td->channel;
1442
1443         td->hcsplt &= ~HCSPLT_COMPSPLT;
1444         td->state = DWC_CHAN_ST_WAIT_S_ANE;
1445
1446         /* receive one packet */
1447         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1448             (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT));
1449
1450         DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(channel), td->hcsplt);
1451
1452         /* send after next SOF event */
1453         if ((sc->sc_last_frame_num & 1) == 0)
1454                 td->hcchar |= HCCHAR_ODDFRM;
1455         else
1456                 td->hcchar &= ~HCCHAR_ODDFRM;
1457
1458         hcchar = td->hcchar;
1459         hcchar |= HCCHAR_EPDIR_IN;
1460
1461         /* wait until next slot before trying complete split */
1462         td->tt_complete_slot = sc->sc_last_frame_num + 1;
1463
1464         /* must enable channel before data can be received */
1465         DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(channel), hcchar);
1466 busy:
1467         return (1);     /* busy */
1468
1469 complete:
1470         dwc_otg_host_channel_free(sc, td);
1471         return (0);     /* complete */
1472 }
1473
1474 static uint8_t
1475 dwc_otg_data_rx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1476 {
1477         uint32_t temp;
1478         uint16_t count;
1479         uint8_t got_short;
1480
1481         got_short = 0;
1482
1483         /* check endpoint status */
1484         if (sc->sc_last_rx_status == 0)
1485                 goto not_complete;
1486
1487         if (GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) != td->ep_no)
1488                 goto not_complete;
1489
1490         /* check for SETUP packet */
1491         if ((sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) ==
1492             GRXSTSRD_STP_DATA ||
1493             (sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) ==
1494             GRXSTSRD_STP_COMPLETE) {
1495                 if (td->remainder == 0) {
1496                         /*
1497                          * We are actually complete and have
1498                          * received the next SETUP
1499                          */
1500                         DPRINTFN(5, "faking complete\n");
1501                         return (0);     /* complete */
1502                 }
1503                 /*
1504                  * USB Host Aborted the transfer.
1505                  */
1506                 td->error_any = 1;
1507                 return (0);             /* complete */
1508         }
1509
1510         if ((sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) !=
1511             GRXSTSRD_OUT_DATA) {
1512                 /* release FIFO */
1513                 dwc_otg_common_rx_ack(sc);
1514                 goto not_complete;
1515         }
1516
1517         /* get the packet byte count */
1518         count = GRXSTSRD_BCNT_GET(sc->sc_last_rx_status);
1519
1520         /* verify the packet byte count */
1521         if (count != td->max_packet_size) {
1522                 if (count < td->max_packet_size) {
1523                         /* we have a short packet */
1524                         td->short_pkt = 1;
1525                         got_short = 1;
1526                 } else {
1527                         /* invalid USB packet */
1528                         td->error_any = 1;
1529
1530                         /* release FIFO */
1531                         dwc_otg_common_rx_ack(sc);
1532                         return (0);     /* we are complete */
1533                 }
1534         }
1535         /* verify the packet byte count */
1536         if (count > td->remainder) {
1537                 /* invalid USB packet */
1538                 td->error_any = 1;
1539
1540                 /* release FIFO */
1541                 dwc_otg_common_rx_ack(sc);
1542                 return (0);             /* we are complete */
1543         }
1544
1545         usbd_copy_in(td->pc, td->offset, sc->sc_rx_bounce_buffer, count);
1546         td->remainder -= count;
1547         td->offset += count;
1548
1549         /* release FIFO */
1550         dwc_otg_common_rx_ack(sc);
1551
1552         temp = sc->sc_out_ctl[td->ep_no];
1553
1554         /* check for isochronous mode */
1555         if ((temp & DIEPCTL_EPTYPE_MASK) ==
1556             (DIEPCTL_EPTYPE_ISOC << DIEPCTL_EPTYPE_SHIFT)) {
1557                 /* toggle odd or even frame bit */
1558                 if (temp & DIEPCTL_SETD1PID) {
1559                         temp &= ~DIEPCTL_SETD1PID;
1560                         temp |= DIEPCTL_SETD0PID;
1561                 } else {
1562                         temp &= ~DIEPCTL_SETD0PID;
1563                         temp |= DIEPCTL_SETD1PID;
1564                 }
1565                 sc->sc_out_ctl[td->ep_no] = temp;
1566         }
1567
1568         /* check if we are complete */
1569         if ((td->remainder == 0) || got_short) {
1570                 if (td->short_pkt) {
1571                         /* we are complete */
1572                         return (0);
1573                 }
1574                 /* else need to receive a zero length packet */
1575         }
1576
1577 not_complete:
1578
1579         /* enable SETUP and transfer complete interrupt */
1580         if (td->ep_no == 0) {
1581                 DWC_OTG_WRITE_4(sc, DOTG_DOEPTSIZ(0),
1582                     DXEPTSIZ_SET_MULTI(3) |
1583                     DXEPTSIZ_SET_NPKT(1) |
1584                     DXEPTSIZ_SET_NBYTES(td->max_packet_size));
1585         } else {
1586                 /* allow reception of multiple packets */
1587                 DWC_OTG_WRITE_4(sc, DOTG_DOEPTSIZ(td->ep_no),
1588                     DXEPTSIZ_SET_MULTI(1) |
1589                     DXEPTSIZ_SET_NPKT(4) |
1590                     DXEPTSIZ_SET_NBYTES(4 *
1591                         ((td->max_packet_size + 3) & ~3)));
1592         }
1593         temp = sc->sc_out_ctl[td->ep_no];
1594         DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(td->ep_no), temp |
1595             DOEPCTL_EPENA | DOEPCTL_CNAK);
1596
1597         return (1);                     /* not complete */
1598 }
1599
1600 static uint8_t
1601 dwc_otg_host_data_tx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1602 {
1603         uint32_t count;
1604         uint32_t hcint;
1605         uint32_t hcchar;
1606         uint8_t delta;
1607         uint8_t channel;
1608
1609         dwc_otg_host_dump_rx(sc, td);
1610
1611         channel = td->channel;
1612
1613         if (channel < DWC_OTG_MAX_CHANNELS) {
1614                 hcint = sc->sc_chan_state[channel].hcint;
1615
1616                 DPRINTF("CH=%d ST=%d HCINT=0x%08x HCCHAR=0x%08x HCTSIZ=0x%08x\n",
1617                     channel, td->state, hcint,
1618                     DWC_OTG_READ_4(sc, DOTG_HCCHAR(channel)),
1619                     DWC_OTG_READ_4(sc, DOTG_HCTSIZ(channel)));
1620
1621                 if (hcint & (HCINT_RETRY |
1622                     HCINT_ACK | HCINT_NYET)) {
1623                         /* give success bits priority over failure bits */
1624                 } else if (hcint & HCINT_STALL) {
1625                         DPRINTF("CH=%d STALL\n", channel);
1626                         td->error_stall = 1;
1627                         td->error_any = 1;
1628                         goto complete;
1629                 } else if (hcint & HCINT_ERRORS) {
1630                         DPRINTF("CH=%d ERROR\n", channel);
1631                         td->errcnt++;
1632                         if (td->hcsplt != 0 || td->errcnt >= 3) {
1633                                 td->error_any = 1;
1634                                 goto complete;
1635                         }
1636                 }
1637
1638                 if (hcint & (HCINT_ERRORS | HCINT_RETRY |
1639                     HCINT_ACK | HCINT_NYET)) {
1640
1641                         if (!(hcint & HCINT_ERRORS))
1642                                 td->errcnt = 0;
1643                 }
1644         } else {
1645                 hcint = 0;
1646         }
1647
1648         switch (td->state) {
1649         case DWC_CHAN_ST_START:
1650                 goto send_pkt;
1651
1652         case DWC_CHAN_ST_WAIT_ANE:
1653                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1654                         td->did_nak++;
1655                         td->tt_scheduled = 0;
1656                         goto send_pkt;
1657                 } else if (hcint & (HCINT_ACK | HCINT_NYET)) {
1658                         td->offset += td->tx_bytes;
1659                         td->remainder -= td->tx_bytes;
1660                         td->toggle ^= 1;
1661                         td->did_nak = 0;
1662                         td->tt_scheduled = 0;
1663
1664                         /* check remainder */
1665                         if (td->remainder == 0) {
1666                                 if (td->short_pkt)
1667                                         goto complete;
1668
1669                                 /*
1670                                  * Else we need to transmit a short
1671                                  * packet:
1672                                  */
1673                         }
1674                         goto send_pkt;
1675                 }
1676                 break;
1677
1678         case DWC_CHAN_ST_WAIT_S_ANE:
1679                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1680                         td->did_nak++;
1681                         td->tt_scheduled = 0;
1682                         goto send_pkt;
1683                 } else if (hcint & (HCINT_ACK | HCINT_NYET)) {
1684                         td->did_nak = 0;
1685                         goto send_cpkt;
1686                 }
1687                 break;
1688
1689         case DWC_CHAN_ST_WAIT_C_ANE:
1690                 if (hcint & HCINT_NYET) {
1691                         goto send_cpkt;
1692                 } else if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1693                         td->did_nak++;
1694                         td->tt_scheduled = 0;
1695                         goto send_pkt;
1696                 } else if (hcint & HCINT_ACK) {
1697                         td->offset += td->tx_bytes;
1698                         td->remainder -= td->tx_bytes;
1699                         td->toggle ^= 1;
1700                         td->did_nak = 0;
1701                         td->tt_scheduled = 0;
1702
1703                         /* check remainder */
1704                         if (td->remainder == 0) {
1705                                 if (td->short_pkt)
1706                                         goto complete;
1707
1708                                 /* else we need to transmit a short packet */
1709                         }
1710                         goto send_pkt;
1711                 }
1712                 break;
1713
1714         case DWC_CHAN_ST_WAIT_C_PKT:
1715                 goto send_cpkt;
1716
1717         case DWC_CHAN_ST_TX_WAIT_ISOC:
1718
1719                 /* Check if isochronous OUT traffic is complete */
1720                 if ((hcint & HCINT_HCH_DONE_MASK) == 0)
1721                         break;
1722
1723                 td->offset += td->tx_bytes;
1724                 td->remainder -= td->tx_bytes;
1725
1726                 if (td->hcsplt != 0 || td->remainder == 0)
1727                         goto complete;
1728
1729                 /* check for next packet */
1730                 if (td->max_packet_count > 1)
1731                         td->tt_xactpos++;
1732
1733                 /* free existing channel, if any */
1734                 dwc_otg_host_channel_free(sc, td);
1735
1736                 td->state = DWC_CHAN_ST_TX_PKT_ISOC;
1737
1738                 /* FALLTHROUGH */
1739
1740         case DWC_CHAN_ST_TX_PKT_ISOC:
1741                 if (dwc_otg_host_channel_alloc(sc, td, 1))
1742                         break;
1743                 channel = td->channel;
1744                 goto send_isoc_pkt;
1745         default:
1746                 break;
1747         }
1748         goto busy;
1749
1750 send_pkt:
1751         /* free existing channel(s), if any */
1752         dwc_otg_host_channel_free(sc, td);
1753
1754         if (td->hcsplt != 0) {
1755                 delta = td->tt_start_slot - sc->sc_last_frame_num - 1;
1756                 if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
1757                         td->state = DWC_CHAN_ST_START;
1758                         goto busy;
1759                 }
1760                 delta = sc->sc_last_frame_num - td->tt_start_slot;
1761                 if (delta > 5) {
1762                         /* missed it */
1763                         td->tt_scheduled = 0;
1764                         td->state = DWC_CHAN_ST_START;
1765                         goto busy;
1766                 }
1767         } else if (dwc_otg_host_rate_check(sc, td)) {
1768                 td->state = DWC_CHAN_ST_START;
1769                 goto busy;
1770         }
1771
1772         /* allocate a new channel */
1773         if (dwc_otg_host_channel_alloc(sc, td, 1)) {
1774                 td->state = DWC_CHAN_ST_START;
1775                 goto busy;
1776         }
1777
1778         channel = td->channel;
1779
1780         /* set toggle, if any */
1781         if (td->set_toggle) {
1782                 td->set_toggle = 0;
1783                 td->toggle = 1;
1784         }
1785
1786         if (td->ep_type == UE_ISOCHRONOUS) {
1787 send_isoc_pkt:
1788                 /* Isochronous OUT transfers don't have any ACKs */
1789                 td->state = DWC_CHAN_ST_TX_WAIT_ISOC;
1790                 td->hcsplt &= ~HCSPLT_COMPSPLT;
1791                 if (td->hcsplt != 0) {
1792                         /* get maximum transfer length */
1793                         count = td->remainder;
1794                         if (count > HCSPLT_XACTLEN_BURST) {
1795                                 DPRINTF("TT overflow\n");
1796                                 td->error_any = 1;
1797                                 goto complete;
1798                         }
1799                         /* Update transaction position */
1800                         td->hcsplt &= ~HCSPLT_XACTPOS_MASK;
1801                         td->hcsplt |= (HCSPLT_XACTPOS_ALL << HCSPLT_XACTPOS_SHIFT);
1802                 } else {
1803                         /* send one packet at a time */
1804                         count = td->max_packet_size;
1805                         if (td->remainder < count) {
1806                                 /* we have a short packet */
1807                                 td->short_pkt = 1;
1808                                 count = td->remainder;
1809                         }
1810                 }
1811         } else if (td->hcsplt != 0) {
1812
1813                 td->hcsplt &= ~HCSPLT_COMPSPLT;
1814
1815                 /* Wait for ACK/NAK/ERR from TT */
1816                 td->state = DWC_CHAN_ST_WAIT_S_ANE;
1817
1818                 /* send one packet at a time */
1819                 count = td->max_packet_size;
1820                 if (td->remainder < count) {
1821                         /* we have a short packet */
1822                         td->short_pkt = 1;
1823                         count = td->remainder;
1824                 }
1825         } else {
1826                 /* Wait for ACK/NAK/STALL from device */
1827                 td->state = DWC_CHAN_ST_WAIT_ANE;
1828
1829                 /* send one packet at a time */
1830                 count = td->max_packet_size;
1831                 if (td->remainder < count) {
1832                         /* we have a short packet */
1833                         td->short_pkt = 1;
1834                         count = td->remainder;
1835                 }
1836         }
1837
1838         /* check for High-Speed multi-packets */
1839         if ((td->hcsplt == 0) && (td->max_packet_count > 1)) {
1840                 if (td->npkt == 0) {
1841                         if (td->remainder >= (3 * td->max_packet_size))
1842                                 td->npkt = 3;
1843                         else if (td->remainder >= (2 * td->max_packet_size))
1844                                 td->npkt = 2;
1845                         else
1846                                 td->npkt = 1;
1847
1848                         if (td->npkt > td->max_packet_count)
1849                                 td->npkt = td->max_packet_count;
1850
1851                         td->tt_xactpos = 1;     /* overload */
1852                 }
1853                 if (td->tt_xactpos == td->npkt) {
1854                         if (td->npkt == 1) {
1855                                 DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1856                                     (count << HCTSIZ_XFERSIZE_SHIFT) |
1857                                     (1 << HCTSIZ_PKTCNT_SHIFT) |
1858                                     (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT));
1859                         } else if (td->npkt == 2) {
1860                                 DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1861                                     (count << HCTSIZ_XFERSIZE_SHIFT) |
1862                                     (1 << HCTSIZ_PKTCNT_SHIFT) |
1863                                     (HCTSIZ_PID_DATA1 << HCTSIZ_PID_SHIFT));
1864                         } else {
1865                                 DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1866                                     (count << HCTSIZ_XFERSIZE_SHIFT) |
1867                                     (1 << HCTSIZ_PKTCNT_SHIFT) |
1868                                     (HCTSIZ_PID_DATA2 << HCTSIZ_PID_SHIFT));
1869                         }
1870                         td->npkt = 0;
1871                 } else {
1872                         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1873                             (count << HCTSIZ_XFERSIZE_SHIFT) |
1874                             (1 << HCTSIZ_PKTCNT_SHIFT) |
1875                             (HCTSIZ_PID_MDATA << HCTSIZ_PID_SHIFT));
1876                 }
1877         } else {
1878                 /* TODO: HCTSIZ_DOPNG */
1879
1880                 DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1881                     (count << HCTSIZ_XFERSIZE_SHIFT) |
1882                     (1 << HCTSIZ_PKTCNT_SHIFT) |
1883                     (td->toggle ? (HCTSIZ_PID_DATA1 << HCTSIZ_PID_SHIFT) :
1884                     (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT)));
1885         }
1886
1887         DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(channel), td->hcsplt);
1888
1889         hcchar = td->hcchar;
1890         hcchar &= ~HCCHAR_EPDIR_IN;
1891
1892         /* send after next SOF event */
1893         if ((sc->sc_last_frame_num & 1) == 0)
1894                 hcchar |= HCCHAR_ODDFRM;
1895         else
1896                 hcchar &= ~HCCHAR_ODDFRM;
1897
1898         /* must enable before writing data to FIFO */
1899         DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(channel), hcchar);
1900
1901         if (count != 0) {
1902
1903                 /* clear topmost word before copy */
1904                 sc->sc_tx_bounce_buffer[(count - 1) / 4] = 0;
1905
1906                 /* copy out data */
1907                 usbd_copy_out(td->pc, td->offset,
1908                     sc->sc_tx_bounce_buffer, count);
1909
1910                 /* transfer data into FIFO */
1911                 bus_space_write_region_4(sc->sc_io_tag, sc->sc_io_hdl,
1912                     DOTG_DFIFO(channel),
1913                     sc->sc_tx_bounce_buffer, (count + 3) / 4);
1914         }
1915
1916         /* store number of bytes transmitted */
1917         td->tx_bytes = count;
1918         goto busy;
1919
1920 send_cpkt:
1921         /* free existing channel, if any */
1922         dwc_otg_host_channel_free(sc, td);
1923
1924         delta = td->tt_complete_slot - sc->sc_last_frame_num - 1;
1925         if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
1926                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
1927                 goto busy;
1928         }
1929         delta = sc->sc_last_frame_num - td->tt_start_slot;
1930         if (delta > DWC_OTG_TT_SLOT_MAX) {
1931                 /* we missed the service interval */
1932                 if (td->ep_type != UE_ISOCHRONOUS)
1933                         td->error_any = 1;
1934                 goto complete;
1935         }
1936
1937         /* allocate a new channel */
1938         if (dwc_otg_host_channel_alloc(sc, td, 0)) {
1939                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
1940                 goto busy;
1941         }
1942
1943         channel = td->channel;
1944
1945         td->hcsplt |= HCSPLT_COMPSPLT;
1946         td->state = DWC_CHAN_ST_WAIT_C_ANE;
1947
1948         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1949             (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT));
1950
1951         DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(channel), td->hcsplt);
1952
1953         hcchar = td->hcchar;
1954         hcchar &= ~HCCHAR_EPDIR_IN;
1955
1956         /* receive complete split ASAP */
1957         if ((sc->sc_last_frame_num & 1) != 0)
1958                 hcchar |= HCCHAR_ODDFRM;
1959         else
1960                 hcchar &= ~HCCHAR_ODDFRM;
1961
1962         /* must enable channel before data can be received */
1963         DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(channel), hcchar);
1964
1965         /* wait until next slot before trying complete split */
1966         td->tt_complete_slot = sc->sc_last_frame_num + 1;
1967 busy:
1968         return (1);     /* busy */
1969
1970 complete:
1971         dwc_otg_host_channel_free(sc, td);
1972         return (0);     /* complete */
1973 }
1974
1975 static uint8_t
1976 dwc_otg_data_tx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1977 {
1978         uint32_t max_buffer;
1979         uint32_t count;
1980         uint32_t fifo_left;
1981         uint32_t mpkt;
1982         uint32_t temp;
1983         uint8_t to;
1984
1985         to = 3;                         /* don't loop forever! */
1986
1987         max_buffer = sc->sc_hw_ep_profile[td->ep_no].max_buffer;
1988
1989 repeat:
1990         /* check for for endpoint 0 data */
1991
1992         temp = sc->sc_last_rx_status;
1993
1994         if ((td->ep_no == 0) && (temp != 0) &&
1995             (GRXSTSRD_CHNUM_GET(temp) == 0)) {
1996
1997                 if ((temp & GRXSTSRD_PKTSTS_MASK) !=
1998                     GRXSTSRD_STP_DATA &&
1999                     (temp & GRXSTSRD_PKTSTS_MASK) !=
2000                     GRXSTSRD_STP_COMPLETE) {
2001
2002                         /* dump data - wrong direction */
2003                         dwc_otg_common_rx_ack(sc);
2004                 } else {
2005                         /*
2006                          * The current transfer was cancelled
2007                          * by the USB Host:
2008                          */
2009                         td->error_any = 1;
2010                         return (0);             /* complete */
2011                 }
2012         }
2013
2014         /* fill in more TX data, if possible */
2015         if (td->tx_bytes != 0) {
2016
2017                 uint16_t cpkt;
2018
2019                 /* check if packets have been transferred */
2020                 temp = DWC_OTG_READ_4(sc, DOTG_DIEPTSIZ(td->ep_no));
2021
2022                 /* get current packet number */
2023                 cpkt = DXEPTSIZ_GET_NPKT(temp);
2024
2025                 if (cpkt >= td->npkt) {
2026                         fifo_left = 0;
2027                 } else {
2028                         if (max_buffer != 0) {
2029                                 fifo_left = (td->npkt - cpkt) *
2030                                     td->max_packet_size;
2031
2032                                 if (fifo_left > max_buffer)
2033                                         fifo_left = max_buffer;
2034                         } else {
2035                                 fifo_left = td->max_packet_size;
2036                         }
2037                 }
2038
2039                 count = td->tx_bytes;
2040                 if (count > fifo_left)
2041                         count = fifo_left;
2042
2043                 if (count != 0) {
2044
2045                         /* clear topmost word before copy */
2046                         sc->sc_tx_bounce_buffer[(count - 1) / 4] = 0;
2047
2048                         /* copy out data */
2049                         usbd_copy_out(td->pc, td->offset,
2050                             sc->sc_tx_bounce_buffer, count);
2051
2052                         /* transfer data into FIFO */
2053                         bus_space_write_region_4(sc->sc_io_tag, sc->sc_io_hdl,
2054                             DOTG_DFIFO(td->ep_no),
2055                             sc->sc_tx_bounce_buffer, (count + 3) / 4);
2056
2057                         td->tx_bytes -= count;
2058                         td->remainder -= count;
2059                         td->offset += count;
2060                         td->npkt = cpkt;
2061                 }
2062                 if (td->tx_bytes != 0)
2063                         goto not_complete;
2064
2065                 /* check remainder */
2066                 if (td->remainder == 0) {
2067                         if (td->short_pkt)
2068                                 return (0);     /* complete */
2069
2070                         /* else we need to transmit a short packet */
2071                 }
2072         }
2073
2074         if (!to--)
2075                 goto not_complete;
2076
2077         /* check if not all packets have been transferred */
2078         temp = DWC_OTG_READ_4(sc, DOTG_DIEPTSIZ(td->ep_no));
2079
2080         if (DXEPTSIZ_GET_NPKT(temp) != 0) {
2081
2082                 DPRINTFN(5, "busy ep=%d npkt=%d DIEPTSIZ=0x%08x "
2083                     "DIEPCTL=0x%08x\n", td->ep_no,
2084                     DXEPTSIZ_GET_NPKT(temp),
2085                     temp, DWC_OTG_READ_4(sc, DOTG_DIEPCTL(td->ep_no)));
2086
2087                 goto not_complete;
2088         }
2089
2090         DPRINTFN(5, "rem=%u ep=%d\n", td->remainder, td->ep_no);
2091
2092         /* try to optimise by sending more data */
2093         if ((max_buffer != 0) && ((td->max_packet_size & 3) == 0)) {
2094
2095                 /* send multiple packets at the same time */
2096                 mpkt = max_buffer / td->max_packet_size;
2097
2098                 if (mpkt > 0x3FE)
2099                         mpkt = 0x3FE;
2100
2101                 count = td->remainder;
2102                 if (count > 0x7FFFFF)
2103                         count = 0x7FFFFF - (0x7FFFFF % td->max_packet_size);
2104
2105                 td->npkt = count / td->max_packet_size;
2106
2107                 /*
2108                  * NOTE: We could use 0x3FE instead of "mpkt" in the
2109                  * check below to get more throughput, but then we
2110                  * have a dependency towards non-generic chip features
2111                  * to disable the TX-FIFO-EMPTY interrupts on a per
2112                  * endpoint basis. Increase the maximum buffer size of
2113                  * the IN endpoint to increase the performance.
2114                  */
2115                 if (td->npkt > mpkt) {
2116                         td->npkt = mpkt;
2117                         count = td->max_packet_size * mpkt;
2118                 } else if ((count == 0) || (count % td->max_packet_size)) {
2119                         /* we are transmitting a short packet */
2120                         td->npkt++;
2121                         td->short_pkt = 1;
2122                 }
2123         } else {
2124                 /* send one packet at a time */
2125                 mpkt = 1;
2126                 count = td->max_packet_size;
2127                 if (td->remainder < count) {
2128                         /* we have a short packet */
2129                         td->short_pkt = 1;
2130                         count = td->remainder;
2131                 }
2132                 td->npkt = 1;
2133         }
2134         DWC_OTG_WRITE_4(sc, DOTG_DIEPTSIZ(td->ep_no),
2135             DXEPTSIZ_SET_MULTI(1) |
2136             DXEPTSIZ_SET_NPKT(td->npkt) |
2137             DXEPTSIZ_SET_NBYTES(count));
2138
2139         /* make room for buffering */
2140         td->npkt += mpkt;
2141
2142         temp = sc->sc_in_ctl[td->ep_no];
2143
2144         /* check for isochronous mode */
2145         if ((temp & DIEPCTL_EPTYPE_MASK) ==
2146             (DIEPCTL_EPTYPE_ISOC << DIEPCTL_EPTYPE_SHIFT)) {
2147                 /* toggle odd or even frame bit */
2148                 if (temp & DIEPCTL_SETD1PID) {
2149                         temp &= ~DIEPCTL_SETD1PID;
2150                         temp |= DIEPCTL_SETD0PID;
2151                 } else {
2152                         temp &= ~DIEPCTL_SETD0PID;
2153                         temp |= DIEPCTL_SETD1PID;
2154                 }
2155                 sc->sc_in_ctl[td->ep_no] = temp;
2156         }
2157
2158         /* must enable before writing data to FIFO */
2159         DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(td->ep_no), temp |
2160             DIEPCTL_EPENA | DIEPCTL_CNAK);
2161
2162         td->tx_bytes = count;
2163
2164         /* check remainder */
2165         if (td->tx_bytes == 0 &&
2166             td->remainder == 0) {
2167                 if (td->short_pkt)
2168                         return (0);     /* complete */
2169
2170                 /* else we need to transmit a short packet */
2171         }
2172         goto repeat;
2173
2174 not_complete:
2175         return (1);                     /* not complete */
2176 }
2177
2178 static uint8_t
2179 dwc_otg_data_tx_sync(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
2180 {
2181         uint32_t temp;
2182
2183         /*
2184          * If all packets are transferred we are complete:
2185          */
2186         temp = DWC_OTG_READ_4(sc, DOTG_DIEPTSIZ(td->ep_no));
2187
2188         /* check that all packets have been transferred */
2189         if (DXEPTSIZ_GET_NPKT(temp) != 0) {
2190                 DPRINTFN(5, "busy ep=%d\n", td->ep_no);
2191                 goto not_complete;
2192         }
2193         return (0);
2194
2195 not_complete:
2196
2197         /* we only want to know if there is a SETUP packet or free IN packet */
2198
2199         temp = sc->sc_last_rx_status;
2200
2201         if ((td->ep_no == 0) && (temp != 0) &&
2202             (GRXSTSRD_CHNUM_GET(temp) == 0)) {
2203
2204                 if ((temp & GRXSTSRD_PKTSTS_MASK) ==
2205                     GRXSTSRD_STP_DATA ||
2206                     (temp & GRXSTSRD_PKTSTS_MASK) ==
2207                     GRXSTSRD_STP_COMPLETE) {
2208                         DPRINTFN(5, "faking complete\n");
2209                         /*
2210                          * Race condition: We are complete!
2211                          */
2212                         return (0);
2213                 } else {
2214                         /* dump data - wrong direction */
2215                         dwc_otg_common_rx_ack(sc);
2216                 }
2217         }
2218         return (1);                     /* not complete */
2219 }
2220
2221 static void
2222 dwc_otg_xfer_do_fifo(struct dwc_otg_softc *sc, struct usb_xfer *xfer)
2223 {
2224         struct dwc_otg_td *td;
2225         uint8_t toggle;
2226         uint8_t tmr_val;
2227         uint8_t tmr_res;
2228
2229         DPRINTFN(9, "\n");
2230
2231         td = xfer->td_transfer_cache;
2232         if (td == NULL)
2233                 return;
2234
2235         while (1) {
2236                 if ((td->func) (sc, td)) {
2237                         /* operation in progress */
2238                         break;
2239                 }
2240                 if (((void *)td) == xfer->td_transfer_last) {
2241                         goto done;
2242                 }
2243                 if (td->error_any) {
2244                         goto done;
2245                 } else if (td->remainder > 0) {
2246                         /*
2247                          * We had a short transfer. If there is no alternate
2248                          * next, stop processing !
2249                          */
2250                         if (!td->alt_next)
2251                                 goto done;
2252                 }
2253
2254                 /*
2255                  * Fetch the next transfer descriptor and transfer
2256                  * some flags to the next transfer descriptor
2257                  */
2258                 tmr_res = td->tmr_res;
2259                 tmr_val = td->tmr_val;
2260                 toggle = td->toggle;
2261                 td = td->obj_next;
2262                 xfer->td_transfer_cache = td;
2263                 td->toggle = toggle;    /* transfer toggle */
2264                 td->tmr_res = tmr_res;
2265                 td->tmr_val = tmr_val;
2266         }
2267         return;
2268
2269 done:
2270         xfer->td_transfer_cache = NULL;
2271         sc->sc_xfer_complete = 1;
2272 }
2273
2274 static uint8_t
2275 dwc_otg_xfer_do_complete_locked(struct dwc_otg_softc *sc, struct usb_xfer *xfer)
2276 {
2277         struct dwc_otg_td *td;
2278
2279         DPRINTFN(9, "\n");
2280
2281         td = xfer->td_transfer_cache;
2282         if (td == NULL) {
2283                 /* compute all actual lengths */
2284                 dwc_otg_standard_done(xfer);
2285                 return (1);
2286         }
2287         return (0);
2288 }
2289
2290 static void
2291 dwc_otg_timer(void *_sc)
2292 {
2293         struct dwc_otg_softc *sc = _sc;
2294         struct usb_xfer *xfer;
2295         struct dwc_otg_td *td;
2296
2297         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
2298
2299         DPRINTF("\n");
2300
2301         USB_BUS_SPIN_LOCK(&sc->sc_bus);
2302
2303         /* increment timer value */
2304         sc->sc_tmr_val++;
2305
2306         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
2307                 td = xfer->td_transfer_cache;
2308                 if (td != NULL) {
2309                         /* reset NAK counter */
2310                         td->did_nak = 0;
2311                 }
2312         }
2313
2314         /* enable SOF interrupt, which will poll jobs */
2315         dwc_otg_enable_sof_irq(sc);
2316
2317         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
2318
2319         if (sc->sc_timer_active) {
2320                 /* restart timer */
2321                 usb_callout_reset(&sc->sc_timer,
2322                     hz / (1000 / DWC_OTG_HOST_TIMER_RATE),
2323                     &dwc_otg_timer, sc);
2324         }
2325 }
2326
2327 static void
2328 dwc_otg_timer_start(struct dwc_otg_softc *sc)
2329 {
2330         if (sc->sc_timer_active != 0)
2331                 return;
2332
2333         sc->sc_timer_active = 1;
2334
2335         /* restart timer */
2336         usb_callout_reset(&sc->sc_timer,
2337             hz / (1000 / DWC_OTG_HOST_TIMER_RATE),
2338             &dwc_otg_timer, sc);
2339 }
2340
2341 static void
2342 dwc_otg_timer_stop(struct dwc_otg_softc *sc)
2343 {
2344         if (sc->sc_timer_active == 0)
2345                 return;
2346
2347         sc->sc_timer_active = 0;
2348
2349         /* stop timer */
2350         usb_callout_stop(&sc->sc_timer);
2351 }
2352
2353 static void
2354 dwc_otg_host_channel_disable(struct dwc_otg_softc *sc, uint8_t x)
2355 {
2356         uint32_t hcchar;
2357
2358         hcchar = DWC_OTG_READ_4(sc, DOTG_HCCHAR(x));
2359
2360         /* disable host channel, if any */
2361         if (hcchar & (HCCHAR_CHENA | HCCHAR_CHDIS)) {
2362                 /* disable channel */
2363                 DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(x),
2364                     HCCHAR_CHENA | HCCHAR_CHDIS);
2365                 /* wait for chip to get its brains in order */
2366                 sc->sc_chan_state[x].wait_sof = 2;
2367         }
2368
2369         /* release TX FIFO usage, if any */
2370         sc->sc_tx_cur_p_level -= sc->sc_chan_state[x].tx_p_size;
2371         sc->sc_tx_cur_np_level -= sc->sc_chan_state[x].tx_np_size;
2372
2373         /* don't release TX FIFO usage twice */
2374         sc->sc_chan_state[x].tx_p_size = 0;
2375         sc->sc_chan_state[x].tx_np_size = 0;
2376 }
2377
2378 static uint16_t
2379 dwc_otg_compute_isoc_rx_tt_slot(struct dwc_otg_tt_info *pinfo)
2380 {
2381         if (pinfo->slot_index < DWC_OTG_TT_SLOT_MAX)
2382                 pinfo->slot_index++;
2383         return (pinfo->slot_index);
2384 }
2385
2386 static uint8_t
2387 dwc_otg_update_host_transfer_schedule_locked(struct dwc_otg_softc *sc)
2388 {
2389         TAILQ_HEAD(, usb_xfer) head;
2390         struct usb_xfer *xfer;
2391         struct usb_xfer *xfer_next;
2392         struct dwc_otg_td *td;
2393         uint16_t temp;
2394         uint16_t slot;
2395         uint8_t x;
2396
2397         temp = DWC_OTG_READ_4(sc, DOTG_HFNUM) & DWC_OTG_FRAME_MASK;
2398
2399         if (sc->sc_last_frame_num == temp)
2400                 return (0);
2401
2402         sc->sc_last_frame_num = temp;
2403
2404         TAILQ_INIT(&head);
2405
2406         for (x = 0; x != sc->sc_host_ch_max; x++) {
2407                 if (sc->sc_chan_state[x].wait_sof == 0)
2408                         continue;
2409
2410                 sc->sc_needsof = 1;
2411                 if (--(sc->sc_chan_state[x].wait_sof) == 0)
2412                         dwc_otg_host_channel_disable(sc, x);
2413         }
2414
2415         if ((temp & 7) == 0) {
2416
2417                 /* reset the schedule */
2418                 memset(sc->sc_tt_info, 0, sizeof(sc->sc_tt_info));
2419
2420                 TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2421                         td = xfer->td_transfer_cache;
2422                         if (td == NULL || td->ep_type != UE_ISOCHRONOUS)
2423                                 continue;
2424
2425                         /* check for IN direction */
2426                         if ((td->hcchar & HCCHAR_EPDIR_IN) != 0)
2427                                 continue;
2428
2429                         /* execute more frames */
2430                         td->tmr_val = 0;
2431
2432                         sc->sc_needsof = 1;
2433
2434                         if (td->hcsplt == 0 || td->tt_scheduled != 0)
2435                                 continue;
2436
2437                         /* compute slot */
2438                         slot = dwc_otg_compute_isoc_rx_tt_slot(
2439                             sc->sc_tt_info + td->tt_index);
2440                         if (slot > 3) {
2441                                 /*
2442                                  * Not enough time to get complete
2443                                  * split executed.
2444                                  */
2445                                 continue;
2446                         }
2447                         /* Delayed start */
2448                         td->tt_start_slot = temp + slot;
2449                         td->tt_scheduled = 1;
2450                         TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2451                         TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2452                 }
2453
2454                 TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2455                         td = xfer->td_transfer_cache;
2456                         if (td == NULL || td->ep_type != UE_ISOCHRONOUS)
2457                                 continue;
2458
2459                         /* check for OUT direction */
2460                         if ((td->hcchar & HCCHAR_EPDIR_IN) == 0)
2461                                 continue;
2462
2463                         /* execute more frames */
2464                         td->tmr_val = 0;
2465
2466                         sc->sc_needsof = 1;
2467
2468                         if (td->hcsplt == 0 || td->tt_scheduled != 0)
2469                                 continue;
2470
2471                         /* Start ASAP */
2472                         td->tt_start_slot = temp;
2473                         td->tt_scheduled = 1;
2474                         TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2475                         TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2476                 }
2477
2478                 TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2479                         td = xfer->td_transfer_cache;
2480                         if (td == NULL || td->ep_type != UE_INTERRUPT)
2481                                 continue;
2482
2483                         if (td->tt_scheduled != 0) {
2484                                 sc->sc_needsof = 1;
2485                                 continue;
2486                         }
2487
2488                         if (dwc_otg_host_rate_check_interrupt(sc, td))
2489                                 continue;
2490
2491                         if (td->hcsplt == 0) {
2492                                 sc->sc_needsof = 1;
2493                                 td->tt_scheduled = 1;
2494                                 continue;
2495                         }
2496
2497                         /* start ASAP */
2498                         td->tt_start_slot = temp;
2499                         sc->sc_needsof = 1;
2500                         td->tt_scheduled = 1;
2501                         TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2502                         TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2503                 }
2504
2505                 TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2506                         td = xfer->td_transfer_cache;
2507                         if (td == NULL ||
2508                             td->ep_type != UE_CONTROL ||
2509                             td->did_nak >= DWC_OTG_NAK_MAX) {
2510                                 continue;
2511                         }
2512
2513                         sc->sc_needsof = 1;
2514
2515                         if (td->hcsplt == 0 || td->tt_scheduled != 0)
2516                                 continue;
2517
2518                         /* start ASAP */
2519                         td->tt_start_slot = temp;
2520                         td->tt_scheduled = 1;
2521                         TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2522                         TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2523                 }
2524         }
2525         if ((temp & 7) < 6) {
2526                 TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2527                         td = xfer->td_transfer_cache;
2528                         if (td == NULL ||
2529                             td->ep_type != UE_BULK ||
2530                             td->did_nak >= DWC_OTG_NAK_MAX) {
2531                                 continue;
2532                         }
2533
2534                         sc->sc_needsof = 1;
2535
2536                         if (td->hcsplt == 0 || td->tt_scheduled != 0)
2537                                 continue;
2538
2539                         /* start ASAP */
2540                         td->tt_start_slot = temp;
2541                         td->tt_scheduled = 1;
2542                         TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2543                         TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2544                 }
2545         }
2546
2547         /* Put TT transfers in execution order at the end */
2548         TAILQ_CONCAT(&sc->sc_bus.intr_q.head, &head, wait_entry);
2549
2550         /* move all TT transfers in front, keeping the current order */
2551         TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2552                 td = xfer->td_transfer_cache;
2553                 if (td == NULL || td->hcsplt == 0)
2554                         continue;
2555                 TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2556                 TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2557         }
2558         TAILQ_CONCAT(&head, &sc->sc_bus.intr_q.head, wait_entry);
2559         TAILQ_CONCAT(&sc->sc_bus.intr_q.head, &head, wait_entry);
2560
2561         /* put non-TT BULK transfers last */
2562         TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2563                 td = xfer->td_transfer_cache;
2564                 if (td == NULL || td->hcsplt != 0 || td->ep_type != UE_BULK)
2565                         continue;
2566                 TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2567                 TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2568         }
2569         TAILQ_CONCAT(&sc->sc_bus.intr_q.head, &head, wait_entry);
2570
2571         if ((temp & 7) == 0) {
2572
2573                 DPRINTFN(12, "SOF interrupt #%d, needsof=%d\n",
2574                     (int)temp, (int)sc->sc_needsof);
2575
2576                 /* update SOF IRQ mask */
2577                 if (sc->sc_irq_mask & GINTMSK_SOFMSK) {
2578                         if (sc->sc_needsof == 0) {
2579                                 sc->sc_irq_mask &= ~GINTMSK_SOFMSK;
2580                                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2581                         }
2582                 } else {
2583                         if (sc->sc_needsof != 0) {
2584                                 sc->sc_irq_mask |= GINTMSK_SOFMSK;
2585                                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2586                         }
2587                 }
2588
2589                 /* clear need SOF flag */
2590                 sc->sc_needsof = 0;
2591         }
2592         return (1);
2593 }
2594
2595 static void
2596 dwc_otg_interrupt_poll_locked(struct dwc_otg_softc *sc)
2597 {
2598         struct usb_xfer *xfer;
2599         uint32_t temp;
2600         uint8_t got_rx_status;
2601         uint8_t x;
2602
2603 repeat:
2604         /* get all channel interrupts */
2605         for (x = 0; x != sc->sc_host_ch_max; x++) {
2606                 temp = DWC_OTG_READ_4(sc, DOTG_HCINT(x));
2607                 if (temp != 0) {
2608                         DWC_OTG_WRITE_4(sc, DOTG_HCINT(x), temp);
2609                         temp &= ~HCINT_SOFTWARE_ONLY;
2610                         sc->sc_chan_state[x].hcint |= temp;
2611                 }
2612         }
2613
2614         if (sc->sc_last_rx_status == 0) {
2615
2616                 temp = DWC_OTG_READ_4(sc, DOTG_GINTSTS);
2617                 if (temp & GINTSTS_RXFLVL) {
2618                         /* pop current status */
2619                         sc->sc_last_rx_status =
2620                             DWC_OTG_READ_4(sc, DOTG_GRXSTSPD);
2621                 }
2622
2623                 if (sc->sc_last_rx_status != 0) {
2624
2625                         uint8_t ep_no;
2626
2627                         temp = sc->sc_last_rx_status &
2628                             GRXSTSRD_PKTSTS_MASK;
2629
2630                         /* non-data messages we simply skip */
2631                         if (temp != GRXSTSRD_STP_DATA &&
2632                             temp != GRXSTSRD_STP_COMPLETE &&
2633                             temp != GRXSTSRD_OUT_DATA) {
2634                                 dwc_otg_common_rx_ack(sc);
2635                                 goto repeat;
2636                         }
2637
2638                         temp = GRXSTSRD_BCNT_GET(
2639                             sc->sc_last_rx_status);
2640                         ep_no = GRXSTSRD_CHNUM_GET(
2641                             sc->sc_last_rx_status);
2642
2643                         /* receive data, if any */
2644                         if (temp != 0) {
2645                                 DPRINTF("Reading %d bytes from ep %d\n", temp, ep_no);
2646                                 bus_space_read_region_4(sc->sc_io_tag, sc->sc_io_hdl,
2647                                     DOTG_DFIFO(ep_no),
2648                                     sc->sc_rx_bounce_buffer, (temp + 3) / 4);
2649                         }
2650
2651                         /* check if we should dump the data */
2652                         if (!(sc->sc_active_rx_ep & (1U << ep_no))) {
2653                                 dwc_otg_common_rx_ack(sc);
2654                                 goto repeat;
2655                         }
2656
2657                         got_rx_status = 1;
2658
2659                         DPRINTFN(5, "RX status = 0x%08x: ch=%d pid=%d bytes=%d sts=%d\n",
2660                             sc->sc_last_rx_status, ep_no,
2661                             (sc->sc_last_rx_status >> 15) & 3,
2662                             GRXSTSRD_BCNT_GET(sc->sc_last_rx_status),
2663                             (sc->sc_last_rx_status >> 17) & 15);
2664                 } else {
2665                         got_rx_status = 0;
2666                 }
2667         } else {
2668                 uint8_t ep_no;
2669
2670                 ep_no = GRXSTSRD_CHNUM_GET(
2671                     sc->sc_last_rx_status);
2672
2673                 /* check if we should dump the data */
2674                 if (!(sc->sc_active_rx_ep & (1U << ep_no))) {
2675                         dwc_otg_common_rx_ack(sc);
2676                         goto repeat;
2677                 }
2678
2679                 got_rx_status = 1;
2680         }
2681
2682         /* execute FIFOs */
2683         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry)
2684                 dwc_otg_xfer_do_fifo(sc, xfer);
2685
2686         if (got_rx_status) {
2687                 /* check if data was consumed */
2688                 if (sc->sc_last_rx_status == 0)
2689                         goto repeat;
2690
2691                 /* disable RX FIFO level interrupt */
2692                 sc->sc_irq_mask &= ~GINTMSK_RXFLVLMSK;
2693                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2694         }
2695
2696         if (sc->sc_flags.status_device_mode == 0 && sc->sc_xfer_complete == 0) {
2697                 /* update host transfer schedule, so that new transfers can be issued */
2698                 if (dwc_otg_update_host_transfer_schedule_locked(sc))
2699                         goto repeat;
2700         }
2701 }
2702
2703 static void
2704 dwc_otg_interrupt_complete_locked(struct dwc_otg_softc *sc)
2705 {
2706         struct usb_xfer *xfer;
2707 repeat:
2708         /* scan for completion events */
2709         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
2710                 if (dwc_otg_xfer_do_complete_locked(sc, xfer))
2711                         goto repeat;
2712         }
2713 }
2714
2715 static void
2716 dwc_otg_vbus_interrupt(struct dwc_otg_softc *sc, uint8_t is_on)
2717 {
2718         DPRINTFN(5, "vbus = %u\n", is_on);
2719
2720         /*
2721          * If the USB host mode is forced, then assume VBUS is always
2722          * present else rely on the input to this function:
2723          */
2724         if ((is_on != 0) || (sc->sc_mode == DWC_MODE_HOST)) {
2725
2726                 if (!sc->sc_flags.status_vbus) {
2727                         sc->sc_flags.status_vbus = 1;
2728
2729                         /* complete root HUB interrupt endpoint */
2730
2731                         dwc_otg_root_intr(sc);
2732                 }
2733         } else {
2734                 if (sc->sc_flags.status_vbus) {
2735                         sc->sc_flags.status_vbus = 0;
2736                         sc->sc_flags.status_bus_reset = 0;
2737                         sc->sc_flags.status_suspend = 0;
2738                         sc->sc_flags.change_suspend = 0;
2739                         sc->sc_flags.change_connect = 1;
2740
2741                         /* complete root HUB interrupt endpoint */
2742
2743                         dwc_otg_root_intr(sc);
2744                 }
2745         }
2746 }
2747
2748 int
2749 dwc_otg_filter_interrupt(void *arg)
2750 {
2751         struct dwc_otg_softc *sc = arg;
2752         int retval = FILTER_HANDLED;
2753         uint32_t status;
2754
2755         USB_BUS_SPIN_LOCK(&sc->sc_bus);
2756
2757         /* read and clear interrupt status */
2758         status = DWC_OTG_READ_4(sc, DOTG_GINTSTS);
2759
2760         /* clear interrupts we are handling here */
2761         DWC_OTG_WRITE_4(sc, DOTG_GINTSTS, status & ~DWC_OTG_MSK_GINT_THREAD_IRQ);
2762
2763         /* check for USB state change interrupts */
2764         if ((status & DWC_OTG_MSK_GINT_THREAD_IRQ) != 0)
2765                 retval = FILTER_SCHEDULE_THREAD;
2766
2767         /* clear all IN endpoint interrupts */
2768         if (status & GINTSTS_IEPINT) {
2769                 uint32_t temp;
2770                 uint8_t x;
2771
2772                 for (x = 0; x != sc->sc_dev_in_ep_max; x++) {
2773                         temp = DWC_OTG_READ_4(sc, DOTG_DIEPINT(x));
2774                         if (temp & DIEPMSK_XFERCOMPLMSK) {
2775                                 DWC_OTG_WRITE_4(sc, DOTG_DIEPINT(x),
2776                                     DIEPMSK_XFERCOMPLMSK);
2777                         }
2778                 }
2779         }
2780
2781         /* poll FIFOs, if any */
2782         dwc_otg_interrupt_poll_locked(sc);
2783
2784         if (sc->sc_xfer_complete != 0)
2785                 retval = FILTER_SCHEDULE_THREAD;
2786
2787         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
2788
2789         return (retval);
2790 }
2791
2792 void
2793 dwc_otg_interrupt(void *arg)
2794 {
2795         struct dwc_otg_softc *sc = arg;
2796         uint32_t status;
2797
2798         USB_BUS_LOCK(&sc->sc_bus);
2799         USB_BUS_SPIN_LOCK(&sc->sc_bus);
2800
2801         /* read and clear interrupt status */
2802         status = DWC_OTG_READ_4(sc, DOTG_GINTSTS);
2803
2804         /* clear interrupts we are handling here */
2805         DWC_OTG_WRITE_4(sc, DOTG_GINTSTS, status & DWC_OTG_MSK_GINT_THREAD_IRQ);
2806
2807         DPRINTFN(14, "GINTSTS=0x%08x HAINT=0x%08x HFNUM=0x%08x\n",
2808             status, DWC_OTG_READ_4(sc, DOTG_HAINT),
2809             DWC_OTG_READ_4(sc, DOTG_HFNUM));
2810
2811         if (status & GINTSTS_USBRST) {
2812
2813                 /* set correct state */
2814                 sc->sc_flags.status_device_mode = 1;
2815                 sc->sc_flags.status_bus_reset = 0;
2816                 sc->sc_flags.status_suspend = 0;
2817                 sc->sc_flags.change_suspend = 0;
2818                 sc->sc_flags.change_connect = 1;
2819
2820                 /* Disable SOF interrupt */
2821                 sc->sc_irq_mask &= ~GINTMSK_SOFMSK;
2822                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2823
2824                 /* complete root HUB interrupt endpoint */
2825                 dwc_otg_root_intr(sc);
2826         }
2827
2828         /* check for any bus state change interrupts */
2829         if (status & GINTSTS_ENUMDONE) {
2830
2831                 uint32_t temp;
2832
2833                 DPRINTFN(5, "end of reset\n");
2834
2835                 /* set correct state */
2836                 sc->sc_flags.status_device_mode = 1;
2837                 sc->sc_flags.status_bus_reset = 1;
2838                 sc->sc_flags.status_suspend = 0;
2839                 sc->sc_flags.change_suspend = 0;
2840                 sc->sc_flags.change_connect = 1;
2841                 sc->sc_flags.status_low_speed = 0;
2842                 sc->sc_flags.port_enabled = 1;
2843
2844                 /* reset FIFOs */
2845                 (void) dwc_otg_init_fifo(sc, DWC_MODE_DEVICE);
2846
2847                 /* reset function address */
2848                 dwc_otg_set_address(sc, 0);
2849
2850                 /* figure out enumeration speed */
2851                 temp = DWC_OTG_READ_4(sc, DOTG_DSTS);
2852                 if (DSTS_ENUMSPD_GET(temp) == DSTS_ENUMSPD_HI)
2853                         sc->sc_flags.status_high_speed = 1;
2854                 else
2855                         sc->sc_flags.status_high_speed = 0;
2856
2857                 /*
2858                  * Disable resume and SOF interrupt, and enable
2859                  * suspend and RX frame interrupt:
2860                  */
2861                 sc->sc_irq_mask &= ~(GINTMSK_WKUPINTMSK | GINTMSK_SOFMSK);
2862                 sc->sc_irq_mask |= GINTMSK_USBSUSPMSK;
2863                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2864
2865                 /* complete root HUB interrupt endpoint */
2866                 dwc_otg_root_intr(sc);
2867         }
2868
2869         if (status & GINTSTS_PRTINT) {
2870                 uint32_t hprt;
2871
2872                 hprt = DWC_OTG_READ_4(sc, DOTG_HPRT);
2873
2874                 /* clear change bits */
2875                 DWC_OTG_WRITE_4(sc, DOTG_HPRT, (hprt & (
2876                     HPRT_PRTPWR | HPRT_PRTENCHNG |
2877                     HPRT_PRTCONNDET | HPRT_PRTOVRCURRCHNG)) |
2878                     sc->sc_hprt_val);
2879
2880                 DPRINTFN(12, "GINTSTS=0x%08x, HPRT=0x%08x\n", status, hprt);
2881
2882                 sc->sc_flags.status_device_mode = 0;
2883
2884                 if (hprt & HPRT_PRTCONNSTS)
2885                         sc->sc_flags.status_bus_reset = 1;
2886                 else
2887                         sc->sc_flags.status_bus_reset = 0;
2888
2889                 if (hprt & HPRT_PRTENCHNG)
2890                         sc->sc_flags.change_enabled = 1;
2891
2892                 if (hprt & HPRT_PRTENA)
2893                         sc->sc_flags.port_enabled = 1;
2894                 else
2895                         sc->sc_flags.port_enabled = 0;
2896
2897                 if (hprt & HPRT_PRTOVRCURRCHNG)
2898                         sc->sc_flags.change_over_current = 1;
2899
2900                 if (hprt & HPRT_PRTOVRCURRACT)
2901                         sc->sc_flags.port_over_current = 1;
2902                 else
2903                         sc->sc_flags.port_over_current = 0;
2904
2905                 if (hprt & HPRT_PRTPWR)
2906                         sc->sc_flags.port_powered = 1;
2907                 else
2908                         sc->sc_flags.port_powered = 0;
2909
2910                 if (((hprt & HPRT_PRTSPD_MASK)
2911                     >> HPRT_PRTSPD_SHIFT) == HPRT_PRTSPD_LOW)
2912                         sc->sc_flags.status_low_speed = 1;
2913                 else
2914                         sc->sc_flags.status_low_speed = 0;
2915
2916                 if (((hprt & HPRT_PRTSPD_MASK)
2917                     >> HPRT_PRTSPD_SHIFT) == HPRT_PRTSPD_HIGH)
2918                         sc->sc_flags.status_high_speed = 1;
2919                 else
2920                         sc->sc_flags.status_high_speed = 0;
2921
2922                 if (hprt & HPRT_PRTCONNDET)
2923                         sc->sc_flags.change_connect = 1;
2924
2925                 if (hprt & HPRT_PRTSUSP)
2926                         dwc_otg_suspend_irq(sc);
2927                 else
2928                         dwc_otg_resume_irq(sc);
2929
2930                 /* complete root HUB interrupt endpoint */
2931                 dwc_otg_root_intr(sc);
2932
2933                 /* update host frame interval */
2934                 dwc_otg_update_host_frame_interval(sc);
2935         }
2936
2937         /*
2938          * If resume and suspend is set at the same time we interpret
2939          * that like RESUME. Resume is set when there is at least 3
2940          * milliseconds of inactivity on the USB BUS.
2941          */
2942         if (status & GINTSTS_WKUPINT) {
2943
2944                 DPRINTFN(5, "resume interrupt\n");
2945
2946                 dwc_otg_resume_irq(sc);
2947
2948         } else if (status & GINTSTS_USBSUSP) {
2949
2950                 DPRINTFN(5, "suspend interrupt\n");
2951
2952                 dwc_otg_suspend_irq(sc);
2953         }
2954         /* check VBUS */
2955         if (status & (GINTSTS_USBSUSP |
2956             GINTSTS_USBRST |
2957             GINTMSK_OTGINTMSK |
2958             GINTSTS_SESSREQINT)) {
2959                 uint32_t temp;
2960
2961                 temp = DWC_OTG_READ_4(sc, DOTG_GOTGCTL);
2962
2963                 DPRINTFN(5, "GOTGCTL=0x%08x\n", temp);
2964
2965                 dwc_otg_vbus_interrupt(sc,
2966                     (temp & (GOTGCTL_ASESVLD | GOTGCTL_BSESVLD)) ? 1 : 0);
2967         }
2968
2969         if (sc->sc_xfer_complete != 0) {
2970                 sc->sc_xfer_complete = 0;
2971
2972                 /* complete FIFOs, if any */
2973                 dwc_otg_interrupt_complete_locked(sc);
2974
2975                 if (sc->sc_flags.status_device_mode == 0) {
2976                         /* update host transfer schedule, so that new transfers can be issued */
2977                         if (dwc_otg_update_host_transfer_schedule_locked(sc))
2978                                 dwc_otg_interrupt_poll_locked(sc);
2979                 }
2980         }
2981         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
2982         USB_BUS_UNLOCK(&sc->sc_bus);
2983 }
2984
2985 static void
2986 dwc_otg_setup_standard_chain_sub(struct dwc_otg_std_temp *temp)
2987 {
2988         struct dwc_otg_td *td;
2989
2990         /* get current Transfer Descriptor */
2991         td = temp->td_next;
2992         temp->td = td;
2993
2994         /* prepare for next TD */
2995         temp->td_next = td->obj_next;
2996
2997         /* fill out the Transfer Descriptor */
2998         td->func = temp->func;
2999         td->pc = temp->pc;
3000         td->offset = temp->offset;
3001         td->remainder = temp->len;
3002         td->tx_bytes = 0;
3003         td->error_any = 0;
3004         td->error_stall = 0;
3005         td->npkt = 0;
3006         td->did_stall = temp->did_stall;
3007         td->short_pkt = temp->short_pkt;
3008         td->alt_next = temp->setup_alt_next;
3009         td->set_toggle = 0;
3010         td->got_short = 0;
3011         td->did_nak = 0;
3012         td->channel = DWC_OTG_MAX_CHANNELS;
3013         td->state = 0;
3014         td->errcnt = 0;
3015         td->tt_scheduled = 0;
3016         td->tt_xactpos = HCSPLT_XACTPOS_BEGIN;
3017 }
3018
3019 static void
3020 dwc_otg_setup_standard_chain(struct usb_xfer *xfer)
3021 {
3022         struct dwc_otg_std_temp temp;
3023         struct dwc_otg_td *td;
3024         uint32_t x;
3025         uint8_t need_sync;
3026         uint8_t is_host;
3027
3028         DPRINTFN(9, "addr=%d endpt=%d sumlen=%d speed=%d\n",
3029             xfer->address, UE_GET_ADDR(xfer->endpointno),
3030             xfer->sumlen, usbd_get_speed(xfer->xroot->udev));
3031
3032         temp.max_frame_size = xfer->max_frame_size;
3033
3034         td = xfer->td_start[0];
3035         xfer->td_transfer_first = td;
3036         xfer->td_transfer_cache = td;
3037
3038         /* setup temp */
3039
3040         temp.pc = NULL;
3041         temp.td = NULL;
3042         temp.td_next = xfer->td_start[0];
3043         temp.offset = 0;
3044         temp.setup_alt_next = xfer->flags_int.short_frames_ok ||
3045             xfer->flags_int.isochronous_xfr;
3046         temp.did_stall = !xfer->flags_int.control_stall;
3047
3048         is_host = (xfer->xroot->udev->flags.usb_mode == USB_MODE_HOST);
3049
3050         /* check if we should prepend a setup message */
3051
3052         if (xfer->flags_int.control_xfr) {
3053                 if (xfer->flags_int.control_hdr) {
3054
3055                         if (is_host)
3056                                 temp.func = &dwc_otg_host_setup_tx;
3057                         else
3058                                 temp.func = &dwc_otg_setup_rx;
3059
3060                         temp.len = xfer->frlengths[0];
3061                         temp.pc = xfer->frbuffers + 0;
3062                         temp.short_pkt = temp.len ? 1 : 0;
3063
3064                         /* check for last frame */
3065                         if (xfer->nframes == 1) {
3066                                 /* no STATUS stage yet, SETUP is last */
3067                                 if (xfer->flags_int.control_act)
3068                                         temp.setup_alt_next = 0;
3069                         }
3070
3071                         dwc_otg_setup_standard_chain_sub(&temp);
3072                 }
3073                 x = 1;
3074         } else {
3075                 x = 0;
3076         }
3077
3078         if (x != xfer->nframes) {
3079                 if (xfer->endpointno & UE_DIR_IN) {
3080                         if (is_host) {
3081                                 temp.func = &dwc_otg_host_data_rx;
3082                                 need_sync = 0;
3083                         } else {
3084                                 temp.func = &dwc_otg_data_tx;
3085                                 need_sync = 1;
3086                         }
3087                 } else {
3088                         if (is_host) {
3089                                 temp.func = &dwc_otg_host_data_tx;
3090                                 need_sync = 0;
3091                         } else {
3092                                 temp.func = &dwc_otg_data_rx;
3093                                 need_sync = 0;
3094                         }
3095                 }
3096
3097                 /* setup "pc" pointer */
3098                 temp.pc = xfer->frbuffers + x;
3099         } else {
3100                 need_sync = 0;
3101         }
3102         while (x != xfer->nframes) {
3103
3104                 /* DATA0 / DATA1 message */
3105
3106                 temp.len = xfer->frlengths[x];
3107
3108                 x++;
3109
3110                 if (x == xfer->nframes) {
3111                         if (xfer->flags_int.control_xfr) {
3112                                 if (xfer->flags_int.control_act) {
3113                                         temp.setup_alt_next = 0;
3114                                 }
3115                         } else {
3116                                 temp.setup_alt_next = 0;
3117                         }
3118                 }
3119                 if (temp.len == 0) {
3120
3121                         /* make sure that we send an USB packet */
3122
3123                         temp.short_pkt = 0;
3124
3125                 } else {
3126
3127                         /* regular data transfer */
3128
3129                         temp.short_pkt = (xfer->flags.force_short_xfer ? 0 : 1);
3130                 }
3131
3132                 dwc_otg_setup_standard_chain_sub(&temp);
3133
3134                 if (xfer->flags_int.isochronous_xfr) {
3135                         temp.offset += temp.len;
3136                 } else {
3137                         /* get next Page Cache pointer */
3138                         temp.pc = xfer->frbuffers + x;
3139                 }
3140         }
3141
3142         if (xfer->flags_int.control_xfr) {
3143
3144                 /* always setup a valid "pc" pointer for status and sync */
3145                 temp.pc = xfer->frbuffers + 0;
3146                 temp.len = 0;
3147                 temp.short_pkt = 0;
3148                 temp.setup_alt_next = 0;
3149
3150                 /* check if we need to sync */
3151                 if (need_sync) {
3152                         /* we need a SYNC point after TX */
3153                         temp.func = &dwc_otg_data_tx_sync;
3154                         dwc_otg_setup_standard_chain_sub(&temp);
3155                 }
3156
3157                 /* check if we should append a status stage */
3158                 if (!xfer->flags_int.control_act) {
3159
3160                         /*
3161                          * Send a DATA1 message and invert the current
3162                          * endpoint direction.
3163                          */
3164                         if (xfer->endpointno & UE_DIR_IN) {
3165                                 if (is_host) {
3166                                         temp.func = &dwc_otg_host_data_tx;
3167                                         need_sync = 0;
3168                                 } else {
3169                                         temp.func = &dwc_otg_data_rx;
3170                                         need_sync = 0;
3171                                 }
3172                         } else {
3173                                 if (is_host) {
3174                                         temp.func = &dwc_otg_host_data_rx;
3175                                         need_sync = 0;
3176                                 } else {
3177                                         temp.func = &dwc_otg_data_tx;
3178                                         need_sync = 1;
3179                                 }
3180                         }
3181
3182                         dwc_otg_setup_standard_chain_sub(&temp);
3183
3184                         /* data toggle should be DATA1 */
3185                         td = temp.td;
3186                         td->set_toggle = 1;
3187
3188                         if (need_sync) {
3189                                 /* we need a SYNC point after TX */
3190                                 temp.func = &dwc_otg_data_tx_sync;
3191                                 dwc_otg_setup_standard_chain_sub(&temp);
3192                         }
3193                 }
3194         } else {
3195                 /* check if we need to sync */
3196                 if (need_sync) {
3197
3198                         temp.pc = xfer->frbuffers + 0;
3199                         temp.len = 0;
3200                         temp.short_pkt = 0;
3201                         temp.setup_alt_next = 0;
3202
3203                         /* we need a SYNC point after TX */
3204                         temp.func = &dwc_otg_data_tx_sync;
3205                         dwc_otg_setup_standard_chain_sub(&temp);
3206                 }
3207         }
3208
3209         /* must have at least one frame! */
3210         td = temp.td;
3211         xfer->td_transfer_last = td;
3212
3213         if (is_host) {
3214
3215                 struct dwc_otg_softc *sc;
3216                 uint32_t hcchar;
3217                 uint32_t hcsplt;
3218
3219                 sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
3220
3221                 /* get first again */
3222                 td = xfer->td_transfer_first;
3223                 td->toggle = (xfer->endpoint->toggle_next ? 1 : 0);
3224
3225                 hcchar =
3226                         (xfer->address << HCCHAR_DEVADDR_SHIFT) |
3227                         ((xfer->endpointno & UE_ADDR) << HCCHAR_EPNUM_SHIFT) |
3228                         (xfer->max_packet_size << HCCHAR_MPS_SHIFT) |
3229                         HCCHAR_CHENA;
3230
3231                 /*
3232                  * We are not always able to meet the timing
3233                  * requirements of the USB interrupt endpoint's
3234                  * complete split token, when doing transfers going
3235                  * via a transaction translator. Use the CONTROL
3236                  * transfer type instead of the INTERRUPT transfer
3237                  * type in general, as a means to workaround
3238                  * that. This trick should work for both FULL and LOW
3239                  * speed USB traffic going through a TT. For non-TT
3240                  * traffic it works aswell. The reason for using
3241                  * CONTROL type instead of BULK is that some TTs might
3242                  * reject LOW speed BULK traffic.
3243                  */
3244                 if (td->ep_type == UE_INTERRUPT)
3245                         hcchar |= (UE_CONTROL << HCCHAR_EPTYPE_SHIFT);
3246                 else
3247                         hcchar |= (td->ep_type << HCCHAR_EPTYPE_SHIFT);
3248
3249                 if (usbd_get_speed(xfer->xroot->udev) == USB_SPEED_LOW)
3250                         hcchar |= HCCHAR_LSPDDEV;
3251                 if (UE_GET_DIR(xfer->endpointno) == UE_DIR_IN)
3252                         hcchar |= HCCHAR_EPDIR_IN;
3253
3254                 switch (xfer->xroot->udev->speed) {
3255                 case USB_SPEED_FULL:
3256                 case USB_SPEED_LOW:
3257                         /* check if root HUB port is running High Speed */
3258                         if (xfer->xroot->udev->parent_hs_hub != NULL) {
3259                                 hcsplt = HCSPLT_SPLTENA |
3260                                     (xfer->xroot->udev->hs_port_no <<
3261                                     HCSPLT_PRTADDR_SHIFT) |
3262                                     (xfer->xroot->udev->hs_hub_addr <<
3263                                     HCSPLT_HUBADDR_SHIFT);
3264                         } else {
3265                                 hcsplt = 0;
3266                         }
3267                         if (td->ep_type == UE_INTERRUPT) {
3268                                 uint32_t ival;
3269                                 ival = xfer->interval / DWC_OTG_HOST_TIMER_RATE;
3270                                 if (ival == 0)
3271                                         ival = 1;
3272                                 else if (ival > 127)
3273                                         ival = 127;
3274                                 td->tmr_val = sc->sc_tmr_val + ival;
3275                                 td->tmr_res = ival;
3276                         } else if (td->ep_type == UE_ISOCHRONOUS) {
3277                                 td->tmr_val = 0;
3278                                 td->tmr_res = 1;
3279                         } else {
3280                                 td->tmr_val = 0;
3281                                 td->tmr_res = 0;
3282                         }
3283                         break;
3284                 case USB_SPEED_HIGH:
3285                         hcsplt = 0;
3286                         if (td->ep_type == UE_INTERRUPT) {
3287                                 uint32_t ival;
3288 #if 0
3289                                 hcchar |= ((xfer->max_packet_count & 3)
3290                                     << HCCHAR_MC_SHIFT);
3291 #endif
3292                                 ival = xfer->interval / DWC_OTG_HOST_TIMER_RATE;
3293                                 if (ival == 0)
3294                                         ival = 1;
3295                                 else if (ival > 127)
3296                                         ival = 127;
3297                                 td->tmr_val = sc->sc_tmr_val + ival;
3298                                 td->tmr_res = ival;
3299                         } else if (td->ep_type == UE_ISOCHRONOUS) {
3300                                 hcchar |= ((xfer->max_packet_count & 3)
3301                                     << HCCHAR_MC_SHIFT);
3302                                 td->tmr_val = 0;
3303                                 td->tmr_res = 1 << usbd_xfer_get_fps_shift(xfer);
3304                         } else {
3305                                 td->tmr_val = 0;
3306                                 td->tmr_res = 0;
3307                         }
3308                         break;
3309                 default:
3310                         hcsplt = 0;
3311                         td->tmr_val = 0;
3312                         td->tmr_res = 0;
3313                         break;
3314                 }
3315
3316                 /* store configuration in all TD's */
3317                 while (1) {
3318                         td->hcchar = hcchar;
3319                         td->hcsplt = hcsplt;
3320
3321                         if (((void *)td) == xfer->td_transfer_last)
3322                                 break;
3323
3324                         td = td->obj_next;
3325                 }
3326         }
3327 }
3328
3329 static void
3330 dwc_otg_timeout(void *arg)
3331 {
3332         struct usb_xfer *xfer = arg;
3333
3334         DPRINTF("xfer=%p\n", xfer);
3335
3336         USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
3337
3338         /* transfer is transferred */
3339         dwc_otg_device_done(xfer, USB_ERR_TIMEOUT);
3340 }
3341
3342 static void
3343 dwc_otg_start_standard_chain(struct usb_xfer *xfer)
3344 {
3345         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
3346         struct usb_xfer_root *xroot;
3347         struct dwc_otg_td *td;
3348
3349         DPRINTFN(9, "\n");
3350
3351         /*
3352          * Poll one time in device mode, which will turn on the
3353          * endpoint interrupts. Else wait for SOF interrupt in host
3354          * mode.
3355          */
3356         USB_BUS_SPIN_LOCK(&sc->sc_bus);
3357
3358         if (sc->sc_flags.status_device_mode != 0) {
3359                 dwc_otg_xfer_do_fifo(sc, xfer);
3360                 if (dwc_otg_xfer_do_complete_locked(sc, xfer))
3361                         goto done;
3362         }
3363
3364         /* put transfer on interrupt queue */
3365         usbd_transfer_enqueue(&xfer->xroot->bus->intr_q, xfer);
3366
3367         /* start timeout, if any */
3368         if (xfer->timeout != 0) {
3369                 usbd_transfer_timeout_ms(xfer,
3370                     &dwc_otg_timeout, xfer->timeout);
3371         }
3372
3373         if (sc->sc_flags.status_device_mode != 0)
3374                 goto done;
3375
3376         /* enable SOF interrupt, if any */
3377         dwc_otg_enable_sof_irq(sc);
3378
3379         td = xfer->td_transfer_cache;
3380         if (td->ep_type != UE_BULK)
3381                 goto done;
3382
3383         xroot = xfer->xroot;
3384
3385         /*
3386          * Optimise the ping-pong effect by waking up other BULK
3387          * transfers belonging to the same device group:
3388          */
3389         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
3390                 td = xfer->td_transfer_cache;
3391                 if (td == NULL || td->ep_type != UE_BULK || xfer->xroot != xroot)
3392                         continue;
3393                 /* reset NAK counter */
3394                 td->did_nak = 0;
3395         }
3396 done:
3397         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3398 }
3399
3400 static void
3401 dwc_otg_root_intr(struct dwc_otg_softc *sc)
3402 {
3403         DPRINTFN(9, "\n");
3404
3405         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
3406
3407         /* set port bit */
3408         sc->sc_hub_idata[0] = 0x02;     /* we only have one port */
3409
3410         uhub_root_intr(&sc->sc_bus, sc->sc_hub_idata,
3411             sizeof(sc->sc_hub_idata));
3412 }
3413
3414 static usb_error_t
3415 dwc_otg_standard_done_sub(struct usb_xfer *xfer)
3416 {
3417         struct dwc_otg_td *td;
3418         uint32_t len;
3419         usb_error_t error;
3420
3421         DPRINTFN(9, "\n");
3422
3423         td = xfer->td_transfer_cache;
3424
3425         do {
3426                 len = td->remainder;
3427
3428                 /* store last data toggle */
3429                 xfer->endpoint->toggle_next = td->toggle;
3430
3431                 if (xfer->aframes != xfer->nframes) {
3432                         /*
3433                          * Verify the length and subtract
3434                          * the remainder from "frlengths[]":
3435                          */
3436                         if (len > xfer->frlengths[xfer->aframes]) {
3437                                 td->error_any = 1;
3438                         } else {
3439                                 xfer->frlengths[xfer->aframes] -= len;
3440                         }
3441                 }
3442                 /* Check for transfer error */
3443                 if (td->error_any) {
3444                         /* the transfer is finished */
3445                         error = (td->error_stall ?
3446                             USB_ERR_STALLED : USB_ERR_IOERROR);
3447                         td = NULL;
3448                         break;
3449                 }
3450                 /* Check for short transfer */
3451                 if (len > 0) {
3452                         if (xfer->flags_int.short_frames_ok ||
3453                             xfer->flags_int.isochronous_xfr) {
3454                                 /* follow alt next */
3455                                 if (td->alt_next) {
3456                                         td = td->obj_next;
3457                                 } else {
3458                                         td = NULL;
3459                                 }
3460                         } else {
3461                                 /* the transfer is finished */
3462                                 td = NULL;
3463                         }
3464                         error = 0;
3465                         break;
3466                 }
3467                 td = td->obj_next;
3468
3469                 /* this USB frame is complete */
3470                 error = 0;
3471                 break;
3472
3473         } while (0);
3474
3475         /* update transfer cache */
3476
3477         xfer->td_transfer_cache = td;
3478
3479         return (error);
3480 }
3481
3482 static void
3483 dwc_otg_standard_done(struct usb_xfer *xfer)
3484 {
3485         usb_error_t err = 0;
3486
3487         DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
3488             xfer, xfer->endpoint);
3489
3490         /* reset scanner */
3491
3492         xfer->td_transfer_cache = xfer->td_transfer_first;
3493
3494         if (xfer->flags_int.control_xfr) {
3495
3496                 if (xfer->flags_int.control_hdr) {
3497
3498                         err = dwc_otg_standard_done_sub(xfer);
3499                 }
3500                 xfer->aframes = 1;
3501
3502                 if (xfer->td_transfer_cache == NULL) {
3503                         goto done;
3504                 }
3505         }
3506         while (xfer->aframes != xfer->nframes) {
3507
3508                 err = dwc_otg_standard_done_sub(xfer);
3509                 xfer->aframes++;
3510
3511                 if (xfer->td_transfer_cache == NULL) {
3512                         goto done;
3513                 }
3514         }
3515
3516         if (xfer->flags_int.control_xfr &&
3517             !xfer->flags_int.control_act) {
3518
3519                 err = dwc_otg_standard_done_sub(xfer);
3520         }
3521 done:
3522         dwc_otg_device_done(xfer, err);
3523 }
3524
3525 /*------------------------------------------------------------------------*
3526  *      dwc_otg_device_done
3527  *
3528  * NOTE: this function can be called more than one time on the
3529  * same USB transfer!
3530  *------------------------------------------------------------------------*/
3531 static void
3532 dwc_otg_device_done(struct usb_xfer *xfer, usb_error_t error)
3533 {
3534         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
3535
3536         DPRINTFN(9, "xfer=%p, endpoint=%p, error=%d\n",
3537             xfer, xfer->endpoint, error);
3538
3539         USB_BUS_SPIN_LOCK(&sc->sc_bus);
3540
3541         if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) {
3542                 /* Interrupts are cleared by the interrupt handler */
3543         } else {
3544                 struct dwc_otg_td *td;
3545
3546                 td = xfer->td_transfer_cache;
3547                 if (td != NULL)
3548                         dwc_otg_host_channel_free(sc, td);
3549         }
3550         /* dequeue transfer and start next transfer */
3551         usbd_transfer_done(xfer, error);
3552
3553         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3554 }
3555
3556 static void
3557 dwc_otg_xfer_stall(struct usb_xfer *xfer)
3558 {
3559         dwc_otg_device_done(xfer, USB_ERR_STALLED);
3560 }
3561
3562 static void
3563 dwc_otg_set_stall(struct usb_device *udev,
3564     struct usb_endpoint *ep, uint8_t *did_stall)
3565 {
3566         struct dwc_otg_softc *sc;
3567         uint32_t temp;
3568         uint32_t reg;
3569         uint8_t ep_no;
3570
3571         USB_BUS_LOCK_ASSERT(udev->bus, MA_OWNED);
3572
3573         /* check mode */
3574         if (udev->flags.usb_mode != USB_MODE_DEVICE) {
3575                 /* not supported */
3576                 return;
3577         }
3578
3579         sc = DWC_OTG_BUS2SC(udev->bus);
3580
3581         USB_BUS_SPIN_LOCK(&sc->sc_bus);
3582
3583         /* get endpoint address */
3584         ep_no = ep->edesc->bEndpointAddress;
3585
3586         DPRINTFN(5, "endpoint=0x%x\n", ep_no);
3587
3588         if (ep_no & UE_DIR_IN) {
3589                 reg = DOTG_DIEPCTL(ep_no & UE_ADDR);
3590                 temp = sc->sc_in_ctl[ep_no & UE_ADDR];
3591         } else {
3592                 reg = DOTG_DOEPCTL(ep_no & UE_ADDR);
3593                 temp = sc->sc_out_ctl[ep_no & UE_ADDR];
3594         }
3595
3596         /* disable and stall endpoint */
3597         DWC_OTG_WRITE_4(sc, reg, temp | DOEPCTL_EPDIS);
3598         DWC_OTG_WRITE_4(sc, reg, temp | DOEPCTL_STALL);
3599
3600         /* clear active OUT ep */
3601         if (!(ep_no & UE_DIR_IN)) {
3602
3603                 sc->sc_active_rx_ep &= ~(1U << (ep_no & UE_ADDR));
3604
3605                 if (sc->sc_last_rx_status != 0 &&
3606                     (ep_no & UE_ADDR) == GRXSTSRD_CHNUM_GET(
3607                     sc->sc_last_rx_status)) {
3608                         /* dump data */
3609                         dwc_otg_common_rx_ack(sc);
3610                         /* poll interrupt */
3611                         dwc_otg_interrupt_poll_locked(sc);
3612                         dwc_otg_interrupt_complete_locked(sc);
3613                 }
3614         }
3615         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3616 }
3617
3618 static void
3619 dwc_otg_clear_stall_sub_locked(struct dwc_otg_softc *sc, uint32_t mps,
3620     uint8_t ep_no, uint8_t ep_type, uint8_t ep_dir)
3621 {
3622         uint32_t reg;
3623         uint32_t temp;
3624
3625         if (ep_type == UE_CONTROL) {
3626                 /* clearing stall is not needed */
3627                 return;
3628         }
3629
3630         if (ep_dir) {
3631                 reg = DOTG_DIEPCTL(ep_no);
3632         } else {
3633                 reg = DOTG_DOEPCTL(ep_no);
3634                 sc->sc_active_rx_ep |= (1U << ep_no);
3635         }
3636
3637         /* round up and mask away the multiplier count */
3638         mps = (mps + 3) & 0x7FC;
3639
3640         if (ep_type == UE_BULK) {
3641                 temp = DIEPCTL_EPTYPE_SET(
3642                     DIEPCTL_EPTYPE_BULK) |
3643                     DIEPCTL_USBACTEP;
3644         } else if (ep_type == UE_INTERRUPT) {
3645                 temp = DIEPCTL_EPTYPE_SET(
3646                     DIEPCTL_EPTYPE_INTERRUPT) |
3647                     DIEPCTL_USBACTEP;
3648         } else {
3649                 temp = DIEPCTL_EPTYPE_SET(
3650                     DIEPCTL_EPTYPE_ISOC) |
3651                     DIEPCTL_USBACTEP;
3652         }
3653
3654         temp |= DIEPCTL_MPS_SET(mps);
3655         temp |= DIEPCTL_TXFNUM_SET(ep_no);
3656
3657         if (ep_dir)
3658                 sc->sc_in_ctl[ep_no] = temp;
3659         else
3660                 sc->sc_out_ctl[ep_no] = temp;
3661
3662         DWC_OTG_WRITE_4(sc, reg, temp | DOEPCTL_EPDIS);
3663         DWC_OTG_WRITE_4(sc, reg, temp | DOEPCTL_SETD0PID);
3664         DWC_OTG_WRITE_4(sc, reg, temp | DIEPCTL_SNAK);
3665
3666         /* we only reset the transmit FIFO */
3667         if (ep_dir) {
3668                 dwc_otg_tx_fifo_reset(sc,
3669                     GRSTCTL_TXFIFO(ep_no) |
3670                     GRSTCTL_TXFFLSH);
3671
3672                 DWC_OTG_WRITE_4(sc,
3673                     DOTG_DIEPTSIZ(ep_no), 0);
3674         }
3675
3676         /* poll interrupt */
3677         dwc_otg_interrupt_poll_locked(sc);
3678         dwc_otg_interrupt_complete_locked(sc);
3679 }
3680
3681 static void
3682 dwc_otg_clear_stall(struct usb_device *udev, struct usb_endpoint *ep)
3683 {
3684         struct dwc_otg_softc *sc;
3685         struct usb_endpoint_descriptor *ed;
3686
3687         DPRINTFN(5, "endpoint=%p\n", ep);
3688
3689         USB_BUS_LOCK_ASSERT(udev->bus, MA_OWNED);
3690
3691         /* check mode */
3692         if (udev->flags.usb_mode != USB_MODE_DEVICE) {
3693                 /* not supported */
3694                 return;
3695         }
3696         /* get softc */
3697         sc = DWC_OTG_BUS2SC(udev->bus);
3698
3699         USB_BUS_SPIN_LOCK(&sc->sc_bus);
3700
3701         /* get endpoint descriptor */
3702         ed = ep->edesc;
3703
3704         /* reset endpoint */
3705         dwc_otg_clear_stall_sub_locked(sc,
3706             UGETW(ed->wMaxPacketSize),
3707             (ed->bEndpointAddress & UE_ADDR),
3708             (ed->bmAttributes & UE_XFERTYPE),
3709             (ed->bEndpointAddress & (UE_DIR_IN | UE_DIR_OUT)));
3710
3711         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3712 }
3713
3714 static void
3715 dwc_otg_device_state_change(struct usb_device *udev)
3716 {
3717         struct dwc_otg_softc *sc;
3718         uint8_t x;
3719
3720         /* check mode */
3721         if (udev->flags.usb_mode != USB_MODE_DEVICE) {
3722                 /* not supported */
3723                 return;
3724         }
3725
3726         /* get softc */
3727         sc = DWC_OTG_BUS2SC(udev->bus);
3728
3729         /* deactivate all other endpoint but the control endpoint */
3730         if (udev->state == USB_STATE_CONFIGURED ||
3731             udev->state == USB_STATE_ADDRESSED) {
3732
3733                 USB_BUS_LOCK(&sc->sc_bus);
3734
3735                 for (x = 1; x != sc->sc_dev_ep_max; x++) {
3736
3737                         if (x < sc->sc_dev_in_ep_max) {
3738                                 DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(x),
3739                                     DIEPCTL_EPDIS);
3740                                 DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(x), 0);
3741                         }
3742
3743                         DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(x),
3744                             DOEPCTL_EPDIS);
3745                         DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(x), 0);
3746                 }
3747                 USB_BUS_UNLOCK(&sc->sc_bus);
3748         }
3749 }
3750
3751 int
3752 dwc_otg_init(struct dwc_otg_softc *sc)
3753 {
3754         uint32_t temp;
3755
3756         DPRINTF("start\n");
3757
3758         /* set up the bus structure */
3759         sc->sc_bus.usbrev = USB_REV_2_0;
3760         sc->sc_bus.methods = &dwc_otg_bus_methods;
3761
3762         usb_callout_init_mtx(&sc->sc_timer,
3763             &sc->sc_bus.bus_mtx, 0);
3764
3765         USB_BUS_LOCK(&sc->sc_bus);
3766
3767         /* turn on clocks */
3768         dwc_otg_clocks_on(sc);
3769
3770         temp = DWC_OTG_READ_4(sc, DOTG_GSNPSID);
3771         DPRINTF("Version = 0x%08x\n", temp);
3772
3773         /* disconnect */
3774         DWC_OTG_WRITE_4(sc, DOTG_DCTL,
3775             DCTL_SFTDISCON);
3776
3777         /* wait for host to detect disconnect */
3778         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 32);
3779
3780         DWC_OTG_WRITE_4(sc, DOTG_GRSTCTL,
3781             GRSTCTL_CSFTRST);
3782
3783         /* wait a little bit for block to reset */
3784         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 128);
3785
3786         switch (sc->sc_mode) {
3787         case DWC_MODE_DEVICE:
3788                 temp = GUSBCFG_FORCEDEVMODE;
3789                 break;
3790         case DWC_MODE_HOST:
3791                 temp = GUSBCFG_FORCEHOSTMODE;
3792                 break;
3793         default:
3794                 temp = 0;
3795                 break;
3796         }
3797
3798         /* select HSIC, ULPI or internal PHY mode */
3799         switch (dwc_otg_phy_type) {
3800         case DWC_OTG_PHY_HSIC:
3801                 DWC_OTG_WRITE_4(sc, DOTG_GUSBCFG,
3802                     GUSBCFG_PHYIF |
3803                     GUSBCFG_TRD_TIM_SET(5) | temp);
3804                 DWC_OTG_WRITE_4(sc, DOTG_GOTGCTL,
3805                     0x000000EC);
3806
3807                 temp = DWC_OTG_READ_4(sc, DOTG_GLPMCFG);
3808                 DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3809                     temp & ~GLPMCFG_HSIC_CONN);
3810                 DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3811                     temp | GLPMCFG_HSIC_CONN);
3812                 break;
3813         case DWC_OTG_PHY_ULPI:
3814                 DWC_OTG_WRITE_4(sc, DOTG_GUSBCFG,
3815                     GUSBCFG_ULPI_UTMI_SEL |
3816                     GUSBCFG_TRD_TIM_SET(5) | temp);
3817                 DWC_OTG_WRITE_4(sc, DOTG_GOTGCTL, 0);
3818
3819                 temp = DWC_OTG_READ_4(sc, DOTG_GLPMCFG);
3820                 DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3821                     temp & ~GLPMCFG_HSIC_CONN);
3822                 break;
3823         case DWC_OTG_PHY_INTERNAL:
3824                 DWC_OTG_WRITE_4(sc, DOTG_GUSBCFG,
3825                     GUSBCFG_PHYSEL |
3826                     GUSBCFG_TRD_TIM_SET(5) | temp);
3827                 DWC_OTG_WRITE_4(sc, DOTG_GOTGCTL, 0);
3828
3829                 temp = DWC_OTG_READ_4(sc, DOTG_GLPMCFG);
3830                 DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3831                     temp & ~GLPMCFG_HSIC_CONN);
3832
3833                 temp = DWC_OTG_READ_4(sc, DOTG_GGPIO);
3834                 temp &= ~(DOTG_GGPIO_NOVBUSSENS | DOTG_GGPIO_I2CPADEN);
3835                 temp |= (DOTG_GGPIO_VBUSASEN | DOTG_GGPIO_VBUSBSEN |
3836                     DOTG_GGPIO_PWRDWN);
3837                 DWC_OTG_WRITE_4(sc, DOTG_GGPIO, temp);
3838                 break;
3839         default:
3840                 break;
3841         }
3842
3843         /* clear global nak */
3844         DWC_OTG_WRITE_4(sc, DOTG_DCTL,
3845             DCTL_CGOUTNAK |
3846             DCTL_CGNPINNAK);
3847
3848         /* disable USB port */
3849         DWC_OTG_WRITE_4(sc, DOTG_PCGCCTL, 0xFFFFFFFF);
3850
3851         /* wait 10ms */
3852         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 100);
3853
3854         /* enable USB port */
3855         DWC_OTG_WRITE_4(sc, DOTG_PCGCCTL, 0);
3856
3857         /* wait 10ms */
3858         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 100);
3859
3860         temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG3);
3861
3862         sc->sc_fifo_size = 4 * GHWCFG3_DFIFODEPTH_GET(temp);
3863
3864         temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG2);
3865
3866         sc->sc_dev_ep_max = GHWCFG2_NUMDEVEPS_GET(temp);
3867
3868         if (sc->sc_dev_ep_max > DWC_OTG_MAX_ENDPOINTS)
3869                 sc->sc_dev_ep_max = DWC_OTG_MAX_ENDPOINTS;
3870
3871         sc->sc_host_ch_max = GHWCFG2_NUMHSTCHNL_GET(temp);
3872
3873         if (sc->sc_host_ch_max > DWC_OTG_MAX_CHANNELS)
3874                 sc->sc_host_ch_max = DWC_OTG_MAX_CHANNELS;
3875
3876         temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG4);
3877
3878         sc->sc_dev_in_ep_max = GHWCFG4_NUM_IN_EP_GET(temp);
3879
3880         DPRINTF("Total FIFO size = %d bytes, Device EPs = %d/%d Host CHs = %d\n",
3881             sc->sc_fifo_size, sc->sc_dev_ep_max, sc->sc_dev_in_ep_max,
3882             sc->sc_host_ch_max);
3883
3884         /* setup FIFO */
3885         if (dwc_otg_init_fifo(sc, sc->sc_mode)) {
3886                 USB_BUS_UNLOCK(&sc->sc_bus);
3887                 return (EINVAL);
3888         }
3889
3890         /* enable interrupts */
3891         sc->sc_irq_mask = DWC_OTG_MSK_GINT_ENABLED;
3892         DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
3893
3894         if (sc->sc_mode == DWC_MODE_OTG || sc->sc_mode == DWC_MODE_DEVICE) {
3895
3896                 /* enable all endpoint interrupts */
3897                 temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG2);
3898                 if (temp & GHWCFG2_MPI) {
3899                         uint8_t x;
3900
3901                         DPRINTF("Disable Multi Process Interrupts\n");
3902
3903                         for (x = 0; x != sc->sc_dev_in_ep_max; x++) {
3904                                 DWC_OTG_WRITE_4(sc, DOTG_DIEPEACHINTMSK(x), 0);
3905                                 DWC_OTG_WRITE_4(sc, DOTG_DOEPEACHINTMSK(x), 0);
3906                         }
3907                         DWC_OTG_WRITE_4(sc, DOTG_DEACHINTMSK, 0);
3908                 }
3909                 DWC_OTG_WRITE_4(sc, DOTG_DIEPMSK,
3910                     DIEPMSK_XFERCOMPLMSK);
3911                 DWC_OTG_WRITE_4(sc, DOTG_DOEPMSK, 0);
3912                 DWC_OTG_WRITE_4(sc, DOTG_DAINTMSK, 0xFFFF);
3913         }
3914
3915         if (sc->sc_mode == DWC_MODE_OTG || sc->sc_mode == DWC_MODE_HOST) {
3916                 /* setup clocks */
3917                 temp = DWC_OTG_READ_4(sc, DOTG_HCFG);
3918                 temp &= ~(HCFG_FSLSSUPP | HCFG_FSLSPCLKSEL_MASK);
3919                 temp |= (1 << HCFG_FSLSPCLKSEL_SHIFT);
3920                 DWC_OTG_WRITE_4(sc, DOTG_HCFG, temp);
3921         }
3922
3923         /* only enable global IRQ */
3924         DWC_OTG_WRITE_4(sc, DOTG_GAHBCFG,
3925             GAHBCFG_GLBLINTRMSK);
3926
3927         /* turn off clocks */
3928         dwc_otg_clocks_off(sc);
3929
3930         /* read initial VBUS state */
3931
3932         temp = DWC_OTG_READ_4(sc, DOTG_GOTGCTL);
3933
3934         DPRINTFN(5, "GOTCTL=0x%08x\n", temp);
3935
3936         dwc_otg_vbus_interrupt(sc,
3937             (temp & (GOTGCTL_ASESVLD | GOTGCTL_BSESVLD)) ? 1 : 0);
3938
3939         USB_BUS_UNLOCK(&sc->sc_bus);
3940
3941         /* catch any lost interrupts */
3942
3943         dwc_otg_do_poll(&sc->sc_bus);
3944
3945         return (0);                     /* success */
3946 }
3947
3948 void
3949 dwc_otg_uninit(struct dwc_otg_softc *sc)
3950 {
3951         USB_BUS_LOCK(&sc->sc_bus);
3952
3953         /* stop host timer */
3954         dwc_otg_timer_stop(sc);
3955
3956         /* set disconnect */
3957         DWC_OTG_WRITE_4(sc, DOTG_DCTL,
3958             DCTL_SFTDISCON);
3959
3960         /* turn off global IRQ */
3961         DWC_OTG_WRITE_4(sc, DOTG_GAHBCFG, 0);
3962
3963         sc->sc_flags.port_enabled = 0;
3964         sc->sc_flags.port_powered = 0;
3965         sc->sc_flags.status_vbus = 0;
3966         sc->sc_flags.status_bus_reset = 0;
3967         sc->sc_flags.status_suspend = 0;
3968         sc->sc_flags.change_suspend = 0;
3969         sc->sc_flags.change_connect = 1;
3970
3971         dwc_otg_pull_down(sc);
3972         dwc_otg_clocks_off(sc);
3973
3974         USB_BUS_UNLOCK(&sc->sc_bus);
3975
3976         usb_callout_drain(&sc->sc_timer);
3977 }
3978
3979 static void
3980 dwc_otg_suspend(struct dwc_otg_softc *sc)
3981 {
3982         return;
3983 }
3984
3985 static void
3986 dwc_otg_resume(struct dwc_otg_softc *sc)
3987 {
3988         return;
3989 }
3990
3991 static void
3992 dwc_otg_do_poll(struct usb_bus *bus)
3993 {
3994         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(bus);
3995
3996         USB_BUS_LOCK(&sc->sc_bus);
3997         USB_BUS_SPIN_LOCK(&sc->sc_bus);
3998         dwc_otg_interrupt_poll_locked(sc);
3999         dwc_otg_interrupt_complete_locked(sc);
4000         if (sc->sc_flags.status_device_mode == 0) {
4001                 /* update host transfer schedule, so that new transfers can be issued */
4002                 if (dwc_otg_update_host_transfer_schedule_locked(sc))
4003                         dwc_otg_interrupt_poll_locked(sc);
4004         }
4005         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
4006         USB_BUS_UNLOCK(&sc->sc_bus);
4007 }
4008
4009 /*------------------------------------------------------------------------*
4010  * DWC OTG bulk support
4011  * DWC OTG control support
4012  * DWC OTG interrupt support
4013  *------------------------------------------------------------------------*/
4014 static void
4015 dwc_otg_device_non_isoc_open(struct usb_xfer *xfer)
4016 {
4017 }
4018
4019 static void
4020 dwc_otg_device_non_isoc_close(struct usb_xfer *xfer)
4021 {
4022         dwc_otg_device_done(xfer, USB_ERR_CANCELLED);
4023 }
4024
4025 static void
4026 dwc_otg_device_non_isoc_enter(struct usb_xfer *xfer)
4027 {
4028 }
4029
4030 static void
4031 dwc_otg_device_non_isoc_start(struct usb_xfer *xfer)
4032 {
4033         /* setup TDs */
4034         dwc_otg_setup_standard_chain(xfer);
4035         dwc_otg_start_standard_chain(xfer);
4036 }
4037
4038 static const struct usb_pipe_methods dwc_otg_device_non_isoc_methods =
4039 {
4040         .open = dwc_otg_device_non_isoc_open,
4041         .close = dwc_otg_device_non_isoc_close,
4042         .enter = dwc_otg_device_non_isoc_enter,
4043         .start = dwc_otg_device_non_isoc_start,
4044 };
4045
4046 /*------------------------------------------------------------------------*
4047  * DWC OTG full speed isochronous support
4048  *------------------------------------------------------------------------*/
4049 static void
4050 dwc_otg_device_isoc_open(struct usb_xfer *xfer)
4051 {
4052 }
4053
4054 static void
4055 dwc_otg_device_isoc_close(struct usb_xfer *xfer)
4056 {
4057         dwc_otg_device_done(xfer, USB_ERR_CANCELLED);
4058 }
4059
4060 static void
4061 dwc_otg_device_isoc_enter(struct usb_xfer *xfer)
4062 {
4063 }
4064
4065 static void
4066 dwc_otg_device_isoc_start(struct usb_xfer *xfer)
4067 {
4068         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
4069         uint32_t temp;
4070         uint32_t msframes;
4071         uint32_t framenum;
4072         uint8_t shift = usbd_xfer_get_fps_shift(xfer);
4073
4074         DPRINTFN(6, "xfer=%p next=%d nframes=%d\n",
4075             xfer, xfer->endpoint->isoc_next, xfer->nframes);
4076
4077         if (xfer->xroot->udev->flags.usb_mode == USB_MODE_HOST) {
4078                 temp = DWC_OTG_READ_4(sc, DOTG_HFNUM);
4079
4080                 /* get the current frame index */
4081                 framenum = (temp & HFNUM_FRNUM_MASK);
4082         } else {
4083                 temp = DWC_OTG_READ_4(sc, DOTG_DSTS);
4084
4085                 /* get the current frame index */
4086                 framenum = DSTS_SOFFN_GET(temp);
4087         }
4088
4089         if (xfer->xroot->udev->parent_hs_hub != NULL)
4090                 framenum /= 8;
4091
4092         framenum &= DWC_OTG_FRAME_MASK;
4093
4094         /*
4095          * Compute number of milliseconds worth of data traffic for
4096          * this USB transfer:
4097          */
4098         if (xfer->xroot->udev->speed == USB_SPEED_HIGH)
4099                 msframes = ((xfer->nframes << shift) + 7) / 8;
4100         else
4101                 msframes = xfer->nframes;
4102
4103         /*
4104          * check if the frame index is within the window where the frames
4105          * will be inserted
4106          */
4107         temp = (framenum - xfer->endpoint->isoc_next) & DWC_OTG_FRAME_MASK;
4108
4109         if ((xfer->endpoint->is_synced == 0) || (temp < msframes)) {
4110                 /*
4111                  * If there is data underflow or the pipe queue is
4112                  * empty we schedule the transfer a few frames ahead
4113                  * of the current frame position. Else two isochronous
4114                  * transfers might overlap.
4115                  */
4116                 xfer->endpoint->isoc_next = (framenum + 3) & DWC_OTG_FRAME_MASK;
4117                 xfer->endpoint->is_synced = 1;
4118                 DPRINTFN(3, "start next=%d\n", xfer->endpoint->isoc_next);
4119         }
4120         /*
4121          * compute how many milliseconds the insertion is ahead of the
4122          * current frame position:
4123          */
4124         temp = (xfer->endpoint->isoc_next - framenum) & DWC_OTG_FRAME_MASK;
4125
4126         /*
4127          * pre-compute when the isochronous transfer will be finished:
4128          */
4129         xfer->isoc_time_complete =
4130                 usb_isoc_time_expand(&sc->sc_bus, framenum) + temp + msframes;
4131
4132         /* setup TDs */
4133         dwc_otg_setup_standard_chain(xfer);
4134
4135         /* compute frame number for next insertion */
4136         xfer->endpoint->isoc_next += msframes;
4137
4138         /* start TD chain */
4139         dwc_otg_start_standard_chain(xfer);
4140 }
4141
4142 static const struct usb_pipe_methods dwc_otg_device_isoc_methods =
4143 {
4144         .open = dwc_otg_device_isoc_open,
4145         .close = dwc_otg_device_isoc_close,
4146         .enter = dwc_otg_device_isoc_enter,
4147         .start = dwc_otg_device_isoc_start,
4148 };
4149
4150 /*------------------------------------------------------------------------*
4151  * DWC OTG root control support
4152  *------------------------------------------------------------------------*
4153  * Simulate a hardware HUB by handling all the necessary requests.
4154  *------------------------------------------------------------------------*/
4155
4156 static const struct usb_device_descriptor dwc_otg_devd = {
4157         .bLength = sizeof(struct usb_device_descriptor),
4158         .bDescriptorType = UDESC_DEVICE,
4159         .bcdUSB = {0x00, 0x02},
4160         .bDeviceClass = UDCLASS_HUB,
4161         .bDeviceSubClass = UDSUBCLASS_HUB,
4162         .bDeviceProtocol = UDPROTO_HSHUBSTT,
4163         .bMaxPacketSize = 64,
4164         .bcdDevice = {0x00, 0x01},
4165         .iManufacturer = 1,
4166         .iProduct = 2,
4167         .bNumConfigurations = 1,
4168 };
4169
4170 static const struct dwc_otg_config_desc dwc_otg_confd = {
4171         .confd = {
4172                 .bLength = sizeof(struct usb_config_descriptor),
4173                 .bDescriptorType = UDESC_CONFIG,
4174                 .wTotalLength[0] = sizeof(dwc_otg_confd),
4175                 .bNumInterface = 1,
4176                 .bConfigurationValue = 1,
4177                 .iConfiguration = 0,
4178                 .bmAttributes = UC_SELF_POWERED,
4179                 .bMaxPower = 0,
4180         },
4181         .ifcd = {
4182                 .bLength = sizeof(struct usb_interface_descriptor),
4183                 .bDescriptorType = UDESC_INTERFACE,
4184                 .bNumEndpoints = 1,
4185                 .bInterfaceClass = UICLASS_HUB,
4186                 .bInterfaceSubClass = UISUBCLASS_HUB,
4187                 .bInterfaceProtocol = 0,
4188         },
4189         .endpd = {
4190                 .bLength = sizeof(struct usb_endpoint_descriptor),
4191                 .bDescriptorType = UDESC_ENDPOINT,
4192                 .bEndpointAddress = (UE_DIR_IN | DWC_OTG_INTR_ENDPT),
4193                 .bmAttributes = UE_INTERRUPT,
4194                 .wMaxPacketSize[0] = 8,
4195                 .bInterval = 255,
4196         },
4197 };
4198
4199 #define HSETW(ptr, val) ptr = { (uint8_t)(val), (uint8_t)((val) >> 8) }
4200
4201 static const struct usb_hub_descriptor_min dwc_otg_hubd = {
4202         .bDescLength = sizeof(dwc_otg_hubd),
4203         .bDescriptorType = UDESC_HUB,
4204         .bNbrPorts = 1,
4205         HSETW(.wHubCharacteristics, (UHD_PWR_NO_SWITCH | UHD_OC_INDIVIDUAL)),
4206         .bPwrOn2PwrGood = 50,
4207         .bHubContrCurrent = 0,
4208         .DeviceRemovable = {0},         /* port is removable */
4209 };
4210
4211 #define STRING_VENDOR \
4212   "D\0W\0C\0O\0T\0G"
4213
4214 #define STRING_PRODUCT \
4215   "O\0T\0G\0 \0R\0o\0o\0t\0 \0H\0U\0B"
4216
4217 USB_MAKE_STRING_DESC(STRING_VENDOR, dwc_otg_vendor);
4218 USB_MAKE_STRING_DESC(STRING_PRODUCT, dwc_otg_product);
4219
4220 static usb_error_t
4221 dwc_otg_roothub_exec(struct usb_device *udev,
4222     struct usb_device_request *req, const void **pptr, uint16_t *plength)
4223 {
4224         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(udev->bus);
4225         const void *ptr;
4226         uint16_t len;
4227         uint16_t value;
4228         uint16_t index;
4229         usb_error_t err;
4230
4231         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
4232
4233         /* buffer reset */
4234         ptr = (const void *)&sc->sc_hub_temp;
4235         len = 0;
4236         err = 0;
4237
4238         value = UGETW(req->wValue);
4239         index = UGETW(req->wIndex);
4240
4241         /* demultiplex the control request */
4242
4243         switch (req->bmRequestType) {
4244         case UT_READ_DEVICE:
4245                 switch (req->bRequest) {
4246                 case UR_GET_DESCRIPTOR:
4247                         goto tr_handle_get_descriptor;
4248                 case UR_GET_CONFIG:
4249                         goto tr_handle_get_config;
4250                 case UR_GET_STATUS:
4251                         goto tr_handle_get_status;
4252                 default:
4253                         goto tr_stalled;
4254                 }
4255                 break;
4256
4257         case UT_WRITE_DEVICE:
4258                 switch (req->bRequest) {
4259                 case UR_SET_ADDRESS:
4260                         goto tr_handle_set_address;
4261                 case UR_SET_CONFIG:
4262                         goto tr_handle_set_config;
4263                 case UR_CLEAR_FEATURE:
4264                         goto tr_valid;  /* nop */
4265                 case UR_SET_DESCRIPTOR:
4266                         goto tr_valid;  /* nop */
4267                 case UR_SET_FEATURE:
4268                 default:
4269                         goto tr_stalled;
4270                 }
4271                 break;
4272
4273         case UT_WRITE_ENDPOINT:
4274                 switch (req->bRequest) {
4275                 case UR_CLEAR_FEATURE:
4276                         switch (UGETW(req->wValue)) {
4277                         case UF_ENDPOINT_HALT:
4278                                 goto tr_handle_clear_halt;
4279                         case UF_DEVICE_REMOTE_WAKEUP:
4280                                 goto tr_handle_clear_wakeup;
4281                         default:
4282                                 goto tr_stalled;
4283                         }
4284                         break;
4285                 case UR_SET_FEATURE:
4286                         switch (UGETW(req->wValue)) {
4287                         case UF_ENDPOINT_HALT:
4288                                 goto tr_handle_set_halt;
4289                         case UF_DEVICE_REMOTE_WAKEUP:
4290                                 goto tr_handle_set_wakeup;
4291                         default:
4292                                 goto tr_stalled;
4293                         }
4294                         break;
4295                 case UR_SYNCH_FRAME:
4296                         goto tr_valid;  /* nop */
4297                 default:
4298                         goto tr_stalled;
4299                 }
4300                 break;
4301
4302         case UT_READ_ENDPOINT:
4303                 switch (req->bRequest) {
4304                 case UR_GET_STATUS:
4305                         goto tr_handle_get_ep_status;
4306                 default:
4307                         goto tr_stalled;
4308                 }
4309                 break;
4310
4311         case UT_WRITE_INTERFACE:
4312                 switch (req->bRequest) {
4313                 case UR_SET_INTERFACE:
4314                         goto tr_handle_set_interface;
4315                 case UR_CLEAR_FEATURE:
4316                         goto tr_valid;  /* nop */
4317                 case UR_SET_FEATURE:
4318                 default:
4319                         goto tr_stalled;
4320                 }
4321                 break;
4322
4323         case UT_READ_INTERFACE:
4324                 switch (req->bRequest) {
4325                 case UR_GET_INTERFACE:
4326                         goto tr_handle_get_interface;
4327                 case UR_GET_STATUS:
4328                         goto tr_handle_get_iface_status;
4329                 default:
4330                         goto tr_stalled;
4331                 }
4332                 break;
4333
4334         case UT_WRITE_CLASS_INTERFACE:
4335         case UT_WRITE_VENDOR_INTERFACE:
4336                 /* XXX forward */
4337                 break;
4338
4339         case UT_READ_CLASS_INTERFACE:
4340         case UT_READ_VENDOR_INTERFACE:
4341                 /* XXX forward */
4342                 break;
4343
4344         case UT_WRITE_CLASS_DEVICE:
4345                 switch (req->bRequest) {
4346                 case UR_CLEAR_FEATURE:
4347                         goto tr_valid;
4348                 case UR_SET_DESCRIPTOR:
4349                 case UR_SET_FEATURE:
4350                         break;
4351                 default:
4352                         goto tr_stalled;
4353                 }
4354                 break;
4355
4356         case UT_WRITE_CLASS_OTHER:
4357                 switch (req->bRequest) {
4358                 case UR_CLEAR_FEATURE:
4359                         goto tr_handle_clear_port_feature;
4360                 case UR_SET_FEATURE:
4361                         goto tr_handle_set_port_feature;
4362                 case UR_CLEAR_TT_BUFFER:
4363                 case UR_RESET_TT:
4364                 case UR_STOP_TT:
4365                         goto tr_valid;
4366
4367                 default:
4368                         goto tr_stalled;
4369                 }
4370                 break;
4371
4372         case UT_READ_CLASS_OTHER:
4373                 switch (req->bRequest) {
4374                 case UR_GET_TT_STATE:
4375                         goto tr_handle_get_tt_state;
4376                 case UR_GET_STATUS:
4377                         goto tr_handle_get_port_status;
4378                 default:
4379                         goto tr_stalled;
4380                 }
4381                 break;
4382
4383         case UT_READ_CLASS_DEVICE:
4384                 switch (req->bRequest) {
4385                 case UR_GET_DESCRIPTOR:
4386                         goto tr_handle_get_class_descriptor;
4387                 case UR_GET_STATUS:
4388                         goto tr_handle_get_class_status;
4389
4390                 default:
4391                         goto tr_stalled;
4392                 }
4393                 break;
4394         default:
4395                 goto tr_stalled;
4396         }
4397         goto tr_valid;
4398
4399 tr_handle_get_descriptor:
4400         switch (value >> 8) {
4401         case UDESC_DEVICE:
4402                 if (value & 0xff) {
4403                         goto tr_stalled;
4404                 }
4405                 len = sizeof(dwc_otg_devd);
4406                 ptr = (const void *)&dwc_otg_devd;
4407                 goto tr_valid;
4408         case UDESC_CONFIG:
4409                 if (value & 0xff) {
4410                         goto tr_stalled;
4411                 }
4412                 len = sizeof(dwc_otg_confd);
4413                 ptr = (const void *)&dwc_otg_confd;
4414                 goto tr_valid;
4415         case UDESC_STRING:
4416                 switch (value & 0xff) {
4417                 case 0:         /* Language table */
4418                         len = sizeof(usb_string_lang_en);
4419                         ptr = (const void *)&usb_string_lang_en;
4420                         goto tr_valid;
4421
4422                 case 1:         /* Vendor */
4423                         len = sizeof(dwc_otg_vendor);
4424                         ptr = (const void *)&dwc_otg_vendor;
4425                         goto tr_valid;
4426
4427                 case 2:         /* Product */
4428                         len = sizeof(dwc_otg_product);
4429                         ptr = (const void *)&dwc_otg_product;
4430                         goto tr_valid;
4431                 default:
4432                         break;
4433                 }
4434                 break;
4435         default:
4436                 goto tr_stalled;
4437         }
4438         goto tr_stalled;
4439
4440 tr_handle_get_config:
4441         len = 1;
4442         sc->sc_hub_temp.wValue[0] = sc->sc_conf;
4443         goto tr_valid;
4444
4445 tr_handle_get_status:
4446         len = 2;
4447         USETW(sc->sc_hub_temp.wValue, UDS_SELF_POWERED);
4448         goto tr_valid;
4449
4450 tr_handle_set_address:
4451         if (value & 0xFF00) {
4452                 goto tr_stalled;
4453         }
4454         sc->sc_rt_addr = value;
4455         goto tr_valid;
4456
4457 tr_handle_set_config:
4458         if (value >= 2) {
4459                 goto tr_stalled;
4460         }
4461         sc->sc_conf = value;
4462         goto tr_valid;
4463
4464 tr_handle_get_interface:
4465         len = 1;
4466         sc->sc_hub_temp.wValue[0] = 0;
4467         goto tr_valid;
4468
4469 tr_handle_get_tt_state:
4470 tr_handle_get_class_status:
4471 tr_handle_get_iface_status:
4472 tr_handle_get_ep_status:
4473         len = 2;
4474         USETW(sc->sc_hub_temp.wValue, 0);
4475         goto tr_valid;
4476
4477 tr_handle_set_halt:
4478 tr_handle_set_interface:
4479 tr_handle_set_wakeup:
4480 tr_handle_clear_wakeup:
4481 tr_handle_clear_halt:
4482         goto tr_valid;
4483
4484 tr_handle_clear_port_feature:
4485         if (index != 1)
4486                 goto tr_stalled;
4487
4488         DPRINTFN(9, "UR_CLEAR_PORT_FEATURE on port %d\n", index);
4489
4490         switch (value) {
4491         case UHF_PORT_SUSPEND:
4492                 dwc_otg_wakeup_peer(sc);
4493                 break;
4494
4495         case UHF_PORT_ENABLE:
4496                 if (sc->sc_flags.status_device_mode == 0) {
4497                         DWC_OTG_WRITE_4(sc, DOTG_HPRT,
4498                             sc->sc_hprt_val | HPRT_PRTENA);
4499                 }
4500                 sc->sc_flags.port_enabled = 0;
4501                 break;
4502
4503         case UHF_C_PORT_RESET:
4504                 sc->sc_flags.change_reset = 0;
4505                 break;
4506
4507         case UHF_C_PORT_ENABLE:
4508                 sc->sc_flags.change_enabled = 0;
4509                 break;
4510
4511         case UHF_C_PORT_OVER_CURRENT:
4512                 sc->sc_flags.change_over_current = 0;
4513                 break;
4514
4515         case UHF_PORT_TEST:
4516         case UHF_PORT_INDICATOR:
4517                 /* nops */
4518                 break;
4519
4520         case UHF_PORT_POWER:
4521                 sc->sc_flags.port_powered = 0;
4522                 if (sc->sc_mode == DWC_MODE_HOST || sc->sc_mode == DWC_MODE_OTG) {
4523                         sc->sc_hprt_val = 0;
4524                         DWC_OTG_WRITE_4(sc, DOTG_HPRT, HPRT_PRTENA);
4525                 }
4526                 dwc_otg_pull_down(sc);
4527                 dwc_otg_clocks_off(sc);
4528                 break;
4529
4530         case UHF_C_PORT_CONNECTION:
4531                 /* clear connect change flag */
4532                 sc->sc_flags.change_connect = 0;
4533                 break;
4534
4535         case UHF_C_PORT_SUSPEND:
4536                 sc->sc_flags.change_suspend = 0;
4537                 break;
4538
4539         default:
4540                 err = USB_ERR_IOERROR;
4541                 goto done;
4542         }
4543         goto tr_valid;
4544
4545 tr_handle_set_port_feature:
4546         if (index != 1) {
4547                 goto tr_stalled;
4548         }
4549         DPRINTFN(9, "UR_SET_PORT_FEATURE\n");
4550
4551         switch (value) {
4552         case UHF_PORT_ENABLE:
4553                 break;
4554
4555         case UHF_PORT_SUSPEND:
4556                 if (sc->sc_flags.status_device_mode == 0) {
4557                         /* set suspend BIT */
4558                         sc->sc_hprt_val |= HPRT_PRTSUSP;
4559                         DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
4560
4561                         /* generate HUB suspend event */
4562                         dwc_otg_suspend_irq(sc);
4563                 }
4564                 break;
4565
4566         case UHF_PORT_RESET:
4567                 if (sc->sc_flags.status_device_mode == 0) {
4568
4569                         DPRINTF("PORT RESET\n");
4570
4571                         /* enable PORT reset */
4572                         DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val | HPRT_PRTRST);
4573
4574                         /* Wait 62.5ms for reset to complete */
4575                         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 16);
4576
4577                         DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
4578
4579                         /* Wait 62.5ms for reset to complete */
4580                         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 16);
4581
4582                         /* reset FIFOs */
4583                         (void) dwc_otg_init_fifo(sc, DWC_MODE_HOST);
4584
4585                         sc->sc_flags.change_reset = 1;
4586                 } else {
4587                         err = USB_ERR_IOERROR;
4588                 }
4589                 break;
4590
4591         case UHF_PORT_TEST:
4592         case UHF_PORT_INDICATOR:
4593                 /* nops */
4594                 break;
4595         case UHF_PORT_POWER:
4596                 sc->sc_flags.port_powered = 1;
4597                 if (sc->sc_mode == DWC_MODE_HOST || sc->sc_mode == DWC_MODE_OTG) {
4598                         sc->sc_hprt_val |= HPRT_PRTPWR;
4599                         DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
4600                 }
4601                 if (sc->sc_mode == DWC_MODE_DEVICE || sc->sc_mode == DWC_MODE_OTG) {
4602                         /* pull up D+, if any */
4603                         dwc_otg_pull_up(sc);
4604                 }
4605                 break;
4606         default:
4607                 err = USB_ERR_IOERROR;
4608                 goto done;
4609         }
4610         goto tr_valid;
4611
4612 tr_handle_get_port_status:
4613
4614         DPRINTFN(9, "UR_GET_PORT_STATUS\n");
4615
4616         if (index != 1)
4617                 goto tr_stalled;
4618
4619         if (sc->sc_flags.status_vbus)
4620                 dwc_otg_clocks_on(sc);
4621         else
4622                 dwc_otg_clocks_off(sc);
4623
4624         /* Select Device Side Mode */
4625
4626         if (sc->sc_flags.status_device_mode) {
4627                 value = UPS_PORT_MODE_DEVICE;
4628                 dwc_otg_timer_stop(sc);
4629         } else {
4630                 value = 0;
4631                 dwc_otg_timer_start(sc);
4632         }
4633
4634         if (sc->sc_flags.status_high_speed)
4635                 value |= UPS_HIGH_SPEED;
4636         else if (sc->sc_flags.status_low_speed)
4637                 value |= UPS_LOW_SPEED;
4638
4639         if (sc->sc_flags.port_powered)
4640                 value |= UPS_PORT_POWER;
4641
4642         if (sc->sc_flags.port_enabled)
4643                 value |= UPS_PORT_ENABLED;
4644
4645         if (sc->sc_flags.port_over_current)
4646                 value |= UPS_OVERCURRENT_INDICATOR;
4647
4648         if (sc->sc_flags.status_vbus &&
4649             sc->sc_flags.status_bus_reset)
4650                 value |= UPS_CURRENT_CONNECT_STATUS;
4651
4652         if (sc->sc_flags.status_suspend)
4653                 value |= UPS_SUSPEND;
4654
4655         USETW(sc->sc_hub_temp.ps.wPortStatus, value);
4656
4657         value = 0;
4658
4659         if (sc->sc_flags.change_connect)
4660                 value |= UPS_C_CONNECT_STATUS;
4661         if (sc->sc_flags.change_suspend)
4662                 value |= UPS_C_SUSPEND;
4663         if (sc->sc_flags.change_reset)
4664                 value |= UPS_C_PORT_RESET;
4665         if (sc->sc_flags.change_over_current)
4666                 value |= UPS_C_OVERCURRENT_INDICATOR;
4667
4668         USETW(sc->sc_hub_temp.ps.wPortChange, value);
4669         len = sizeof(sc->sc_hub_temp.ps);
4670         goto tr_valid;
4671
4672 tr_handle_get_class_descriptor:
4673         if (value & 0xFF) {
4674                 goto tr_stalled;
4675         }
4676         ptr = (const void *)&dwc_otg_hubd;
4677         len = sizeof(dwc_otg_hubd);
4678         goto tr_valid;
4679
4680 tr_stalled:
4681         err = USB_ERR_STALLED;
4682 tr_valid:
4683 done:
4684         *plength = len;
4685         *pptr = ptr;
4686         return (err);
4687 }
4688
4689 static void
4690 dwc_otg_xfer_setup(struct usb_setup_params *parm)
4691 {
4692         struct usb_xfer *xfer;
4693         void *last_obj;
4694         uint32_t ntd;
4695         uint32_t n;
4696         uint8_t ep_no;
4697         uint8_t ep_type;
4698
4699         xfer = parm->curr_xfer;
4700
4701         /*
4702          * NOTE: This driver does not use any of the parameters that
4703          * are computed from the following values. Just set some
4704          * reasonable dummies:
4705          */
4706         parm->hc_max_packet_size = 0x500;
4707         parm->hc_max_packet_count = 3;
4708         parm->hc_max_frame_size = 3 * 0x500;
4709
4710         usbd_transfer_setup_sub(parm);
4711
4712         /*
4713          * compute maximum number of TDs
4714          */
4715         ep_type = (xfer->endpoint->edesc->bmAttributes & UE_XFERTYPE);
4716
4717         if (ep_type == UE_CONTROL) {
4718
4719                 ntd = xfer->nframes + 1 /* STATUS */ + 1 /* SYNC 1 */
4720                     + 1 /* SYNC 2 */ + 1 /* SYNC 3 */;
4721         } else {
4722
4723                 ntd = xfer->nframes + 1 /* SYNC */ ;
4724         }
4725
4726         /*
4727          * check if "usbd_transfer_setup_sub" set an error
4728          */
4729         if (parm->err)
4730                 return;
4731
4732         /*
4733          * allocate transfer descriptors
4734          */
4735         last_obj = NULL;
4736
4737         ep_no = xfer->endpointno & UE_ADDR;
4738
4739         /*
4740          * Check for a valid endpoint profile in USB device mode:
4741          */
4742         if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) {
4743                 const struct usb_hw_ep_profile *pf;
4744
4745                 dwc_otg_get_hw_ep_profile(parm->udev, &pf, ep_no);
4746
4747                 if (pf == NULL) {
4748                         /* should not happen */
4749                         parm->err = USB_ERR_INVAL;
4750                         return;
4751                 }
4752         }
4753
4754         /* align data */
4755         parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
4756
4757         for (n = 0; n != ntd; n++) {
4758
4759                 struct dwc_otg_td *td;
4760
4761                 if (parm->buf) {
4762
4763                         td = USB_ADD_BYTES(parm->buf, parm->size[0]);
4764
4765                         /* compute shared bandwidth resource index for TT */
4766                         if (parm->udev->parent_hs_hub != NULL && parm->udev->speed != USB_SPEED_HIGH) {
4767                                 if (parm->udev->parent_hs_hub->ddesc.bDeviceProtocol == UDPROTO_HSHUBMTT)
4768                                         td->tt_index = parm->udev->device_index;
4769                                 else
4770                                         td->tt_index = parm->udev->parent_hs_hub->device_index;
4771                         } else {
4772                                 td->tt_index = parm->udev->device_index;
4773                         }
4774
4775                         /* init TD */
4776                         td->max_packet_size = xfer->max_packet_size;
4777                         td->max_packet_count = xfer->max_packet_count;
4778                         td->ep_no = ep_no;
4779                         td->ep_type = ep_type;
4780                         td->obj_next = last_obj;
4781
4782                         last_obj = td;
4783                 }
4784                 parm->size[0] += sizeof(*td);
4785         }
4786
4787         xfer->td_start[0] = last_obj;
4788 }
4789
4790 static void
4791 dwc_otg_xfer_unsetup(struct usb_xfer *xfer)
4792 {
4793         return;
4794 }
4795
4796 static void
4797 dwc_otg_ep_init(struct usb_device *udev, struct usb_endpoint_descriptor *edesc,
4798     struct usb_endpoint *ep)
4799 {
4800         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(udev->bus);
4801
4802         DPRINTFN(2, "endpoint=%p, addr=%d, endpt=%d, mode=%d (%d,%d)\n",
4803             ep, udev->address,
4804             edesc->bEndpointAddress, udev->flags.usb_mode,
4805             sc->sc_rt_addr, udev->device_index);
4806
4807         if (udev->device_index != sc->sc_rt_addr) {
4808
4809                 if (udev->flags.usb_mode == USB_MODE_DEVICE) {
4810                         if (udev->speed != USB_SPEED_FULL &&
4811                             udev->speed != USB_SPEED_HIGH) {
4812                                 /* not supported */
4813                                 return;
4814                         }
4815                 } else {
4816                         if (udev->speed == USB_SPEED_HIGH) {
4817                                 if ((UGETW(edesc->wMaxPacketSize) >> 11) & 3) {
4818                                         /* high bandwidth endpoint - not tested */
4819                                         DPRINTF("High Bandwidth Endpoint - not tested\n");
4820                                         return;
4821                                 }
4822                         }
4823                 }
4824                 if ((edesc->bmAttributes & UE_XFERTYPE) == UE_ISOCHRONOUS)
4825                         ep->methods = &dwc_otg_device_isoc_methods;
4826                 else
4827                         ep->methods = &dwc_otg_device_non_isoc_methods;
4828         }
4829 }
4830
4831 static void
4832 dwc_otg_set_hw_power_sleep(struct usb_bus *bus, uint32_t state)
4833 {
4834         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(bus);
4835
4836         switch (state) {
4837         case USB_HW_POWER_SUSPEND:
4838                 dwc_otg_suspend(sc);
4839                 break;
4840         case USB_HW_POWER_SHUTDOWN:
4841                 dwc_otg_uninit(sc);
4842                 break;
4843         case USB_HW_POWER_RESUME:
4844                 dwc_otg_resume(sc);
4845                 break;
4846         default:
4847                 break;
4848         }
4849 }
4850
4851 static void
4852 dwc_otg_get_dma_delay(struct usb_device *udev, uint32_t *pus)
4853 {
4854         /* DMA delay - wait until any use of memory is finished */
4855         *pus = (2125);                  /* microseconds */
4856 }
4857
4858 static void
4859 dwc_otg_device_resume(struct usb_device *udev)
4860 {
4861         DPRINTF("\n");
4862
4863         /* poll all transfers again to restart resumed ones */
4864         dwc_otg_do_poll(udev->bus);
4865 }
4866
4867 static void
4868 dwc_otg_device_suspend(struct usb_device *udev)
4869 {
4870         DPRINTF("\n");
4871 }
4872
4873 static const struct usb_bus_methods dwc_otg_bus_methods =
4874 {
4875         .endpoint_init = &dwc_otg_ep_init,
4876         .xfer_setup = &dwc_otg_xfer_setup,
4877         .xfer_unsetup = &dwc_otg_xfer_unsetup,
4878         .get_hw_ep_profile = &dwc_otg_get_hw_ep_profile,
4879         .xfer_stall = &dwc_otg_xfer_stall,
4880         .set_stall = &dwc_otg_set_stall,
4881         .clear_stall = &dwc_otg_clear_stall,
4882         .roothub_exec = &dwc_otg_roothub_exec,
4883         .xfer_poll = &dwc_otg_do_poll,
4884         .device_state_change = &dwc_otg_device_state_change,
4885         .set_hw_power_sleep = &dwc_otg_set_hw_power_sleep,
4886         .get_dma_delay = &dwc_otg_get_dma_delay,
4887         .device_resume = &dwc_otg_device_resume,
4888         .device_suspend = &dwc_otg_device_suspend,
4889 };