sys/dev/usb/controller/dwc_otg.c

   1 /* $FreeBSD$ */
   2 /*-
   3  * Copyright (c) 2015 Daisuke Aoyama. All rights reserved.
   4  * Copyright (c) 2012-2015 Hans Petter Selasky. All rights reserved.
   5  * Copyright (c) 2010-2011 Aleksandr Rybalko. All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  26  * SUCH DAMAGE.
  27  */
  28
  29 /*
  30  * This file contains the driver for the DesignWare series USB 2.0 OTG
  31  * Controller.
  32  */
  33
  34 /*
  35  * LIMITATION: Drivers must be bound to all OUT endpoints in the
  36  * active configuration for this driver to work properly. Blocking any
  37  * OUT endpoint will block all OUT endpoints including the control
  38  * endpoint. Usually this is not a problem.
  39  */
  40
  41 /*
  42  * NOTE: Writing to non-existing registers appears to cause an
  43  * internal reset.
  44  */
  45
  46 #ifdef USB_GLOBAL_INCLUDE_FILE
  47 #include USB_GLOBAL_INCLUDE_FILE
  48 #else
  49 #include <sys/stdint.h>
  50 #include <sys/stddef.h>
  51 #include <sys/param.h>
  52 #include <sys/queue.h>
  53 #include <sys/types.h>
  54 #include <sys/systm.h>
  55 #include <sys/kernel.h>
  56 #include <sys/bus.h>
  57 #include <sys/module.h>
  58 #include <sys/lock.h>
  59 #include <sys/mutex.h>
  60 #include <sys/condvar.h>
  61 #include <sys/sysctl.h>
  62 #include <sys/sx.h>
  63 #include <sys/unistd.h>
  64 #include <sys/callout.h>
  65 #include <sys/malloc.h>
  66 #include <sys/priv.h>
  67
  68 #include <dev/usb/usb.h>
  69 #include <dev/usb/usbdi.h>
  70
  71 #define USB_DEBUG_VAR dwc_otg_debug
  72
  73 #include <dev/usb/usb_core.h>
  74 #include <dev/usb/usb_debug.h>
  75 #include <dev/usb/usb_busdma.h>
  76 #include <dev/usb/usb_process.h>
  77 #include <dev/usb/usb_transfer.h>
  78 #include <dev/usb/usb_device.h>
  79 #include <dev/usb/usb_hub.h>
  80 #include <dev/usb/usb_util.h>
  81
  82 #include <dev/usb/usb_controller.h>
  83 #include <dev/usb/usb_bus.h>
  84 #endif                  /* USB_GLOBAL_INCLUDE_FILE */
  85
  86 #include <dev/usb/controller/dwc_otg.h>
  87 #include <dev/usb/controller/dwc_otgreg.h>
  88
  89 #define DWC_OTG_BUS2SC(bus) \
  90    ((struct dwc_otg_softc *)(((uint8_t *)(bus)) - \
  91     ((uint8_t *)&(((struct dwc_otg_softc *)0)->sc_bus))))
  92
  93 #define DWC_OTG_PC2UDEV(pc) \
  94    (USB_DMATAG_TO_XROOT((pc)->tag_parent)->udev)
  95
  96 #define DWC_OTG_MSK_GINT_ENABLED        \
  97    (GINTMSK_ENUMDONEMSK |               \
  98    GINTMSK_USBRSTMSK |                  \
  99    GINTMSK_USBSUSPMSK |                 \
 100    GINTMSK_IEPINTMSK |                  \
 101    GINTMSK_SESSREQINTMSK |              \
 102    GINTMSK_RXFLVLMSK |                  \
 103    GINTMSK_HCHINTMSK |                  \
 104    GINTMSK_OTGINTMSK |                  \
 105    GINTMSK_PRTINTMSK)
 106
 107 #define DWC_OTG_MSK_GINT_THREAD_IRQ                             \
 108    (GINTSTS_USBRST | GINTSTS_ENUMDONE | GINTSTS_PRTINT |        \
 109    GINTSTS_WKUPINT | GINTSTS_USBSUSP | GINTMSK_OTGINTMSK |      \
 110    GINTSTS_SESSREQINT)
 111
 112 #define DWC_OTG_PHY_ULPI 0
 113 #define DWC_OTG_PHY_HSIC 1
 114 #define DWC_OTG_PHY_INTERNAL 2
 115
 116 #ifndef DWC_OTG_PHY_DEFAULT
 117 #define DWC_OTG_PHY_DEFAULT DWC_OTG_PHY_ULPI
 118 #endif
 119
 120 static int dwc_otg_phy_type = DWC_OTG_PHY_DEFAULT;
 121
 122 static SYSCTL_NODE(_hw_usb, OID_AUTO, dwc_otg, CTLFLAG_RW, 0, "USB DWC OTG");
 123 SYSCTL_INT(_hw_usb_dwc_otg, OID_AUTO, phy_type, CTLFLAG_RDTUN,
 124     &dwc_otg_phy_type, 0, "DWC OTG PHY TYPE - 0/1/2 - ULPI/HSIC/INTERNAL");
 125 TUNABLE_INT("hw.usb.dwc_otg.phy_type", &dwc_otg_phy_type);
 126
 127 #ifdef USB_DEBUG
 128 static int dwc_otg_debug;
 129
 130 SYSCTL_INT(_hw_usb_dwc_otg, OID_AUTO, debug, CTLFLAG_RW,
 131     &dwc_otg_debug, 0, "DWC OTG debug level");
 132 #endif
 133
 134 #define DWC_OTG_INTR_ENDPT 1
 135
 136 /* prototypes */
 137
 138 struct usb_bus_methods dwc_otg_bus_methods;
 139 struct usb_pipe_methods dwc_otg_device_non_isoc_methods;
 140 struct usb_pipe_methods dwc_otg_device_isoc_methods;
 141
 142 static dwc_otg_cmd_t dwc_otg_setup_rx;
 143 static dwc_otg_cmd_t dwc_otg_data_rx;
 144 static dwc_otg_cmd_t dwc_otg_data_tx;
 145 static dwc_otg_cmd_t dwc_otg_data_tx_sync;
 146
 147 static dwc_otg_cmd_t dwc_otg_host_setup_tx;
 148 static dwc_otg_cmd_t dwc_otg_host_data_tx;
 149 static dwc_otg_cmd_t dwc_otg_host_data_rx;
 150
 151 static void dwc_otg_device_done(struct usb_xfer *, usb_error_t);
 152 static void dwc_otg_do_poll(struct usb_bus *);
 153 static void dwc_otg_standard_done(struct usb_xfer *);
 154 static void dwc_otg_root_intr(struct dwc_otg_softc *);
 155 static void dwc_otg_interrupt_poll_locked(struct dwc_otg_softc *);
 156
 157 /*
 158  * Here is a configuration that the chip supports.
 159  */
 160 static const struct usb_hw_ep_profile dwc_otg_ep_profile[1] = {
 161
 162         [0] = {
 163                 .max_in_frame_size = 64,/* fixed */
 164                 .max_out_frame_size = 64,       /* fixed */
 165                 .is_simplex = 1,
 166                 .support_control = 1,
 167         }
 168 };
 169
 170 static void
 171 dwc_otg_get_hw_ep_profile(struct usb_device *udev,
 172     const struct usb_hw_ep_profile **ppf, uint8_t ep_addr)
 173 {
 174         struct dwc_otg_softc *sc;
 175
 176         sc = DWC_OTG_BUS2SC(udev->bus);
 177
 178         if (ep_addr < sc->sc_dev_ep_max)
 179                 *ppf = &sc->sc_hw_ep_profile[ep_addr].usb;
 180         else
 181                 *ppf = NULL;
 182 }
 183
 184 static void
 185 dwc_otg_tx_fifo_reset(struct dwc_otg_softc *sc, uint32_t value)
 186 {
 187         uint32_t temp;
 188
 189         /* reset FIFO */
 190         DWC_OTG_WRITE_4(sc, DOTG_GRSTCTL, value);
 191
 192         /* wait for reset to complete */
 193         for (temp = 0; temp != 16; temp++) {
 194                 value = DWC_OTG_READ_4(sc, DOTG_GRSTCTL);
 195                 if (!(value & (GRSTCTL_TXFFLSH | GRSTCTL_RXFFLSH)))
 196                         break;
 197         }
 198 }
 199
 200 static int
 201 dwc_otg_init_fifo(struct dwc_otg_softc *sc, uint8_t mode)
 202 {
 203         struct dwc_otg_profile *pf;
 204         uint32_t fifo_size;
 205         uint32_t fifo_regs;
 206         uint32_t tx_start;
 207         uint8_t x;
 208
 209         fifo_size = sc->sc_fifo_size;
 210
 211         /*
 212          * NOTE: Reserved fixed size area at end of RAM, which must
 213          * not be allocated to the FIFOs:
 214          */
 215         fifo_regs = 4 * 16;
 216
 217         if (fifo_size < fifo_regs) {
 218                 DPRINTF("Too little FIFO\n");
 219                 return (EINVAL);
 220         }
 221
 222         /* subtract FIFO regs from total once */
 223         fifo_size -= fifo_regs;
 224
 225         /* split equally for IN and OUT */
 226         fifo_size /= 2;
 227
 228         /* Align to 4 bytes boundary (refer to PGM) */
 229         fifo_size &= ~3;
 230
 231         /* set global receive FIFO size */
 232         DWC_OTG_WRITE_4(sc, DOTG_GRXFSIZ, fifo_size / 4);
 233
 234         tx_start = fifo_size;
 235
 236         if (fifo_size < 64) {
 237                 DPRINTFN(-1, "Not enough data space for EP0 FIFO.\n");
 238                 return (EINVAL);
 239         }
 240
 241         if (mode == DWC_MODE_HOST) {
 242
 243                 /* reset active endpoints */
 244                 sc->sc_active_rx_ep = 0;
 245
 246                 /* split equally for periodic and non-periodic */
 247                 fifo_size /= 2;
 248
 249                 DPRINTF("PTX/NPTX FIFO=%u\n", fifo_size);
 250
 251                 /* align to 4 bytes boundary */
 252                 fifo_size &= ~3;
 253
 254                 DWC_OTG_WRITE_4(sc, DOTG_GNPTXFSIZ,
 255                     ((fifo_size / 4) << 16) |
 256                     (tx_start / 4));
 257
 258                 tx_start += fifo_size;
 259
 260                 for (x = 0; x != sc->sc_host_ch_max; x++) {
 261                         /* enable all host interrupts */
 262                         DWC_OTG_WRITE_4(sc, DOTG_HCINTMSK(x),
 263                             HCINT_DEFAULT_MASK);
 264                 }
 265
 266                 DWC_OTG_WRITE_4(sc, DOTG_HPTXFSIZ,
 267                     ((fifo_size / 4) << 16) |
 268                     (tx_start / 4));
 269
 270                 /* reset host channel state */
 271                 memset(sc->sc_chan_state, 0, sizeof(sc->sc_chan_state));
 272
 273                 /* enable all host channel interrupts */
 274                 DWC_OTG_WRITE_4(sc, DOTG_HAINTMSK,
 275                     (1U << sc->sc_host_ch_max) - 1U);
 276         }
 277
 278         if (mode == DWC_MODE_DEVICE) {
 279
 280             DWC_OTG_WRITE_4(sc, DOTG_GNPTXFSIZ,
 281                 (0x10 << 16) | (tx_start / 4));
 282             fifo_size -= 0x40;
 283             tx_start += 0x40;
 284
 285             /* setup control endpoint profile */
 286             sc->sc_hw_ep_profile[0].usb = dwc_otg_ep_profile[0];
 287
 288             /* reset active endpoints */
 289             sc->sc_active_rx_ep = 1;
 290
 291             for (x = 1; x != sc->sc_dev_ep_max; x++) {
 292
 293                 pf = sc->sc_hw_ep_profile + x;
 294
 295                 pf->usb.max_out_frame_size = 1024 * 3;
 296                 pf->usb.is_simplex = 0; /* assume duplex */
 297                 pf->usb.support_bulk = 1;
 298                 pf->usb.support_interrupt = 1;
 299                 pf->usb.support_isochronous = 1;
 300                 pf->usb.support_out = 1;
 301
 302                 if (x < sc->sc_dev_in_ep_max) {
 303                         uint32_t limit;
 304
 305                         limit = (x == 1) ? MIN(DWC_OTG_TX_MAX_FIFO_SIZE,
 306                             DWC_OTG_MAX_TXN) : MIN(DWC_OTG_MAX_TXN / 2,
 307                             DWC_OTG_TX_MAX_FIFO_SIZE);
 308
 309                         /* see if there is enough FIFO space */
 310                         if (limit <= fifo_size) {
 311                                 pf->max_buffer = limit;
 312                                 pf->usb.support_in = 1;
 313                         } else {
 314                                 limit = MIN(DWC_OTG_TX_MAX_FIFO_SIZE, 0x40);
 315                                 if (limit <= fifo_size) {
 316                                         pf->usb.support_in = 1;
 317                                 } else {
 318                                         pf->usb.is_simplex = 1;
 319                                         limit = 0;
 320                                 }
 321                         }
 322                         /* set FIFO size */
 323                         DWC_OTG_WRITE_4(sc, DOTG_DIEPTXF(x),
 324                             ((limit / 4) << 16) | (tx_start / 4));
 325                         tx_start += limit;
 326                         fifo_size -= limit;
 327                         pf->usb.max_in_frame_size = limit;
 328                 } else {
 329                         pf->usb.is_simplex = 1;
 330                 }
 331
 332                 DPRINTF("FIFO%d = IN:%d / OUT:%d\n", x,
 333                     pf->usb.max_in_frame_size,
 334                     pf->usb.max_out_frame_size);
 335             }
 336         }
 337
 338         /* reset RX FIFO */
 339         dwc_otg_tx_fifo_reset(sc, GRSTCTL_RXFFLSH);
 340
 341         if (mode != DWC_MODE_OTG) {
 342                 /* reset all TX FIFOs */
 343                 dwc_otg_tx_fifo_reset(sc,
 344                     GRSTCTL_TXFIFO(0x10) |
 345                     GRSTCTL_TXFFLSH);
 346         } else {
 347                 /* reset active endpoints */
 348                 sc->sc_active_rx_ep = 0;
 349
 350                 /* reset host channel state */
 351                 memset(sc->sc_chan_state, 0, sizeof(sc->sc_chan_state));
 352         }
 353         return (0);
 354 }
 355
 356 static void
 357 dwc_otg_update_host_frame_interval(struct dwc_otg_softc *sc)
 358 {
 359
 360   /*
 361    * Disabled until further. Assuming that the register is already
 362    * programmed correctly by the boot loader.
 363    */
 364 #if 0
 365         uint32_t temp;
 366
 367         /* setup HOST frame interval register, based on existing value */
 368         temp = DWC_OTG_READ_4(sc, DOTG_HFIR) & HFIR_FRINT_MASK;
 369         if (temp >= 10000)
 370                 temp /= 1000;
 371         else
 372                 temp /= 125;
 373
 374         /* figure out nearest X-tal value */
 375         if (temp >= 54)
 376                 temp = 60;      /* MHz */
 377         else if (temp >= 39)
 378                 temp = 48;      /* MHz */
 379         else
 380                 temp = 30;      /* MHz */
 381
 382         if (sc->sc_flags.status_high_speed)
 383                 temp *= 125;
 384         else
 385                 temp *= 1000;
 386
 387         DPRINTF("HFIR=0x%08x\n", temp);
 388
 389         DWC_OTG_WRITE_4(sc, DOTG_HFIR, temp);
 390 #endif
 391 }
 392
 393 static void
 394 dwc_otg_clocks_on(struct dwc_otg_softc *sc)
 395 {
 396         if (sc->sc_flags.clocks_off &&
 397             sc->sc_flags.port_powered) {
 398
 399                 DPRINTFN(5, "\n");
 400
 401                 /* TODO - platform specific */
 402
 403                 sc->sc_flags.clocks_off = 0;
 404         }
 405 }
 406
 407 static void
 408 dwc_otg_clocks_off(struct dwc_otg_softc *sc)
 409 {
 410         if (!sc->sc_flags.clocks_off) {
 411
 412                 DPRINTFN(5, "\n");
 413
 414                 /* TODO - platform specific */
 415
 416                 sc->sc_flags.clocks_off = 1;
 417         }
 418 }
 419
 420 static void
 421 dwc_otg_pull_up(struct dwc_otg_softc *sc)
 422 {
 423         uint32_t temp;
 424
 425         /* pullup D+, if possible */
 426
 427         if (!sc->sc_flags.d_pulled_up &&
 428             sc->sc_flags.port_powered) {
 429                 sc->sc_flags.d_pulled_up = 1;
 430
 431                 temp = DWC_OTG_READ_4(sc, DOTG_DCTL);
 432                 temp &= ~DCTL_SFTDISCON;
 433                 DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
 434         }
 435 }
 436
 437 static void
 438 dwc_otg_pull_down(struct dwc_otg_softc *sc)
 439 {
 440         uint32_t temp;
 441
 442         /* pulldown D+, if possible */
 443
 444         if (sc->sc_flags.d_pulled_up) {
 445                 sc->sc_flags.d_pulled_up = 0;
 446
 447                 temp = DWC_OTG_READ_4(sc, DOTG_DCTL);
 448                 temp |= DCTL_SFTDISCON;
 449                 DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
 450         }
 451 }
 452
 453 static void
 454 dwc_otg_enable_sof_irq(struct dwc_otg_softc *sc)
 455 {
 456         /* In device mode we don't use the SOF interrupt */
 457         if (sc->sc_flags.status_device_mode != 0)
 458                 return;
 459         /* Ensure the SOF interrupt is not disabled */
 460         sc->sc_needsof = 1;
 461         /* Check if the SOF interrupt is already enabled */
 462         if ((sc->sc_irq_mask & GINTMSK_SOFMSK) != 0)
 463                 return;
 464         sc->sc_irq_mask |= GINTMSK_SOFMSK;
 465         DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 466 }
 467
 468 static void
 469 dwc_otg_resume_irq(struct dwc_otg_softc *sc)
 470 {
 471         if (sc->sc_flags.status_suspend) {
 472                 /* update status bits */
 473                 sc->sc_flags.status_suspend = 0;
 474                 sc->sc_flags.change_suspend = 1;
 475
 476                 if (sc->sc_flags.status_device_mode) {
 477                         /*
 478                          * Disable resume interrupt and enable suspend
 479                          * interrupt:
 480                          */
 481                         sc->sc_irq_mask &= ~GINTMSK_WKUPINTMSK;
 482                         sc->sc_irq_mask |= GINTMSK_USBSUSPMSK;
 483                         DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 484                 }
 485
 486                 /* complete root HUB interrupt endpoint */
 487                 dwc_otg_root_intr(sc);
 488         }
 489 }
 490
 491 static void
 492 dwc_otg_suspend_irq(struct dwc_otg_softc *sc)
 493 {
 494         if (!sc->sc_flags.status_suspend) {
 495                 /* update status bits */
 496                 sc->sc_flags.status_suspend = 1;
 497                 sc->sc_flags.change_suspend = 1;
 498
 499                 if (sc->sc_flags.status_device_mode) {
 500                         /*
 501                          * Disable suspend interrupt and enable resume
 502                          * interrupt:
 503                          */
 504                         sc->sc_irq_mask &= ~GINTMSK_USBSUSPMSK;
 505                         sc->sc_irq_mask |= GINTMSK_WKUPINTMSK;
 506                         DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 507                 }
 508
 509                 /* complete root HUB interrupt endpoint */
 510                 dwc_otg_root_intr(sc);
 511         }
 512 }
 513
 514 static void
 515 dwc_otg_wakeup_peer(struct dwc_otg_softc *sc)
 516 {
 517         if (!sc->sc_flags.status_suspend)
 518                 return;
 519
 520         DPRINTFN(5, "Remote wakeup\n");
 521
 522         if (sc->sc_flags.status_device_mode) {
 523                 uint32_t temp;
 524
 525                 /* enable remote wakeup signalling */
 526                 temp = DWC_OTG_READ_4(sc, DOTG_DCTL);
 527                 temp |= DCTL_RMTWKUPSIG;
 528                 DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
 529
 530                 /* Wait 8ms for remote wakeup to complete. */
 531                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 125);
 532
 533                 temp &= ~DCTL_RMTWKUPSIG;
 534                 DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
 535         } else {
 536                 /* enable USB port */
 537                 DWC_OTG_WRITE_4(sc, DOTG_PCGCCTL, 0);
 538
 539                 /* wait 10ms */
 540                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 100);
 541
 542                 /* resume port */
 543                 sc->sc_hprt_val |= HPRT_PRTRES;
 544                 DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
 545
 546                 /* Wait 100ms for resume signalling to complete. */
 547                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 10);
 548
 549                 /* clear suspend and resume */
 550                 sc->sc_hprt_val &= ~(HPRT_PRTSUSP | HPRT_PRTRES);
 551                 DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
 552
 553                 /* Wait 4ms */
 554                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 250);
 555         }
 556
 557         /* need to fake resume IRQ */
 558         dwc_otg_resume_irq(sc);
 559 }
 560
 561 static void
 562 dwc_otg_set_address(struct dwc_otg_softc *sc, uint8_t addr)
 563 {
 564         uint32_t temp;
 565
 566         DPRINTFN(5, "addr=%d\n", addr);
 567
 568         temp = DWC_OTG_READ_4(sc, DOTG_DCFG);
 569         temp &= ~DCFG_DEVADDR_SET(0x7F);
 570         temp |= DCFG_DEVADDR_SET(addr);
 571         DWC_OTG_WRITE_4(sc, DOTG_DCFG, temp);
 572 }
 573
 574 static void
 575 dwc_otg_common_rx_ack(struct dwc_otg_softc *sc)
 576 {
 577         DPRINTFN(5, "RX status clear\n");
 578
 579         /* enable RX FIFO level interrupt */
 580         sc->sc_irq_mask |= GINTMSK_RXFLVLMSK;
 581         DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 582
 583         /* clear cached status */
 584         sc->sc_last_rx_status = 0;
 585 }
 586
 587 static void
 588 dwc_otg_clear_hcint(struct dwc_otg_softc *sc, uint8_t x)
 589 {
 590         uint32_t hcint;
 591
 592         /* clear all pending interrupts */
 593         hcint = DWC_OTG_READ_4(sc, DOTG_HCINT(x));
 594         DWC_OTG_WRITE_4(sc, DOTG_HCINT(x), hcint);
 595
 596         /* clear buffered interrupts */
 597         sc->sc_chan_state[x].hcint = 0;
 598 }
 599
 600 static uint8_t
 601 dwc_otg_host_check_tx_fifo_empty(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
 602 {
 603         uint32_t temp;
 604
 605         temp = DWC_OTG_READ_4(sc, DOTG_GINTSTS);
 606
 607         if (td->ep_type == UE_ISOCHRONOUS) {
 608                 /*
 609                  * NOTE: USB INTERRUPT transactions are executed like
 610                  * USB CONTROL transactions! See the setup standard
 611                  * chain function for more information.
 612                  */
 613                 if (!(temp & GINTSTS_PTXFEMP)) {
 614                         DPRINTF("Periodic TX FIFO is not empty\n");
 615                         if (!(sc->sc_irq_mask & GINTMSK_PTXFEMPMSK)) {
 616                                 sc->sc_irq_mask |= GINTMSK_PTXFEMPMSK;
 617                                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 618                         }
 619                         return (1);     /* busy */
 620                 }
 621         } else {
 622                 if (!(temp & GINTSTS_NPTXFEMP)) {
 623                         DPRINTF("Non-periodic TX FIFO is not empty\n");
 624                         if (!(sc->sc_irq_mask & GINTMSK_NPTXFEMPMSK)) {
 625                                 sc->sc_irq_mask |= GINTMSK_NPTXFEMPMSK;
 626                                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 627                         }
 628                         return (1);     /* busy */
 629                 }
 630         }
 631         return (0);     /* ready for transmit */
 632 }
 633
 634 static uint8_t
 635 dwc_otg_host_channel_alloc(struct dwc_otg_softc *sc,
 636     struct dwc_otg_td *td, uint8_t is_out)
 637 {
 638         uint8_t x;
 639         uint8_t y;
 640         uint8_t z;
 641
 642         if (td->channel[0] < DWC_OTG_MAX_CHANNELS)
 643                 return (0);             /* already allocated */
 644
 645         /* check if device is suspended */
 646         if (DWC_OTG_PC2UDEV(td->pc)->flags.self_suspended != 0)
 647                 return (1);             /* busy - cannot transfer data */
 648
 649         /* compute needed TX FIFO size */
 650         if (is_out != 0) {
 651                 if (dwc_otg_host_check_tx_fifo_empty(sc, td) != 0)
 652                         return (1);     /* busy - cannot transfer data */
 653         }
 654         z = td->max_packet_count;
 655         for (x = y = 0; x != sc->sc_host_ch_max; x++) {
 656                 /* check if channel is allocated */
 657                 if (sc->sc_chan_state[x].allocated != 0)
 658                         continue;
 659                 /* check if channel is still enabled */
 660                 if (sc->sc_chan_state[x].wait_halted != 0)
 661                         continue;
 662                 /* store channel number */
 663                 td->channel[y++] = x;
 664                 /* check if we got all channels */
 665                 if (y == z)
 666                         break;
 667         }
 668         if (y != z) {
 669                 /* reset channel variable */
 670                 td->channel[0] = DWC_OTG_MAX_CHANNELS;
 671                 td->channel[1] = DWC_OTG_MAX_CHANNELS;
 672                 td->channel[2] = DWC_OTG_MAX_CHANNELS;
 673                 /* wait a bit */
 674                 dwc_otg_enable_sof_irq(sc);
 675                 return (1);     /* busy - not enough channels */
 676         }
 677
 678         for (y = 0; y != z; y++) {
 679                 x = td->channel[y];
 680
 681                 /* set allocated */
 682                 sc->sc_chan_state[x].allocated = 1;
 683
 684                 /* set wait halted */
 685                 sc->sc_chan_state[x].wait_halted = 1;
 686
 687                 /* clear interrupts */
 688                 dwc_otg_clear_hcint(sc, x);
 689
 690                 DPRINTF("CH=%d HCCHAR=0x%08x "
 691                     "HCSPLT=0x%08x\n", x, td->hcchar, td->hcsplt);
 692
 693                 /* set active channel */
 694                 sc->sc_active_rx_ep |= (1 << x);
 695         }
 696         return (0);     /* allocated */
 697 }
 698
 699 static void
 700 dwc_otg_host_channel_free_sub(struct dwc_otg_softc *sc, struct dwc_otg_td *td, uint8_t index)
 701 {
 702         uint32_t hcchar;
 703         uint8_t x;
 704
 705         if (td->channel[index] >= DWC_OTG_MAX_CHANNELS)
 706                 return;         /* already freed */
 707
 708         /* free channel */
 709         x = td->channel[index];
 710         td->channel[index] = DWC_OTG_MAX_CHANNELS;
 711
 712         DPRINTF("CH=%d\n", x);
 713
 714         /*
 715          * We need to let programmed host channels run till complete
 716          * else the host channel will stop functioning.
 717          */
 718         sc->sc_chan_state[x].allocated = 0;
 719
 720         /* ack any pending messages */
 721         if (sc->sc_last_rx_status != 0 &&
 722             GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) == x) {
 723                 dwc_otg_common_rx_ack(sc);
 724         }
 725
 726         /* clear active channel */
 727         sc->sc_active_rx_ep &= ~(1 << x);
 728
 729         /* check if already halted */
 730         if (sc->sc_chan_state[x].wait_halted == 0)
 731                 return;
 732
 733         /* disable host channel */
 734         hcchar = DWC_OTG_READ_4(sc, DOTG_HCCHAR(x));
 735         if (hcchar & HCCHAR_CHENA) {
 736                 DPRINTF("Halting channel %d\n", x);
 737                 DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(x),
 738                     hcchar | HCCHAR_CHDIS);
 739                 /* don't write HCCHAR until the channel is halted */
 740         } else {
 741                 sc->sc_chan_state[x].wait_halted = 0;
 742         }
 743 }
 744
 745 static void
 746 dwc_otg_host_channel_free(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
 747 {
 748         uint8_t x;
 749         for (x = 0; x != td->max_packet_count; x++)
 750                 dwc_otg_host_channel_free_sub(sc, td, x);
 751 }
 752
 753 static void
 754 dwc_otg_host_dump_rx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
 755 {
 756         uint8_t x;
 757         /* dump any pending messages */
 758         if (sc->sc_last_rx_status == 0)
 759                 return;
 760         for (x = 0; x != td->max_packet_count; x++) {
 761                 if (td->channel[x] >= DWC_OTG_MAX_CHANNELS ||
 762                     td->channel[x] != GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status))
 763                         continue;
 764                 dwc_otg_common_rx_ack(sc);
 765         }
 766 }
 767
 768 static uint8_t
 769 dwc_otg_host_setup_tx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
 770 {
 771         struct usb_device_request req __aligned(4);
 772         uint32_t hcint;
 773         uint32_t hcchar;
 774         uint8_t delta;
 775
 776         dwc_otg_host_dump_rx(sc, td);
 777
 778         if (td->channel[0] < DWC_OTG_MAX_CHANNELS) {
 779                 hcint = sc->sc_chan_state[td->channel[0]].hcint;
 780
 781                 DPRINTF("CH=%d ST=%d HCINT=0x%08x HCCHAR=0x%08x HCTSIZ=0x%08x\n",
 782                     td->channel[0], td->state, hcint,
 783                     DWC_OTG_READ_4(sc, DOTG_HCCHAR(td->channel[0])),
 784                     DWC_OTG_READ_4(sc, DOTG_HCTSIZ(td->channel[0])));
 785         } else {
 786                 hcint = 0;
 787                 goto check_state;
 788         }
 789
 790         if (hcint & (HCINT_RETRY |
 791             HCINT_ACK | HCINT_NYET)) {
 792                 /* give success bits priority over failure bits */
 793         } else if (hcint & HCINT_STALL) {
 794                 DPRINTF("CH=%d STALL\n", td->channel[0]);
 795                 td->error_stall = 1;
 796                 td->error_any = 1;
 797                 goto complete;
 798         } else if (hcint & HCINT_ERRORS) {
 799                 DPRINTF("CH=%d ERROR\n", td->channel[0]);
 800                 td->errcnt++;
 801                 if (td->hcsplt != 0 || td->errcnt >= 3) {
 802                         td->error_any = 1;
 803                         goto complete;
 804                 }
 805         }
 806
 807         if (hcint & (HCINT_ERRORS | HCINT_RETRY |
 808             HCINT_ACK | HCINT_NYET)) {
 809                 if (!(hcint & HCINT_ERRORS))
 810                         td->errcnt = 0;
 811         }
 812
 813 check_state:
 814         switch (td->state) {
 815         case DWC_CHAN_ST_START:
 816                 goto send_pkt;
 817
 818         case DWC_CHAN_ST_WAIT_ANE:
 819                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
 820                         td->did_nak = 1;
 821                         td->tt_scheduled = 0;
 822                         goto send_pkt;
 823                 } else if (hcint & (HCINT_ACK | HCINT_NYET)) {
 824                         td->offset += td->tx_bytes;
 825                         td->remainder -= td->tx_bytes;
 826                         td->toggle = 1;
 827                         td->tt_scheduled = 0;
 828                         goto complete;
 829                 }
 830                 break;
 831
 832         case DWC_CHAN_ST_WAIT_S_ANE:
 833                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
 834                         td->did_nak = 1;
 835                         td->tt_scheduled = 0;
 836                         goto send_pkt;
 837                 } else if (hcint & (HCINT_ACK | HCINT_NYET)) {
 838                         goto send_cpkt;
 839                 }
 840                 break;
 841
 842         case DWC_CHAN_ST_WAIT_C_ANE:
 843                 if (hcint & HCINT_NYET) {
 844                         goto send_cpkt;
 845                 } else if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
 846                         td->did_nak = 1;
 847                         td->tt_scheduled = 0;
 848                         goto send_pkt;
 849                 } else if (hcint & HCINT_ACK) {
 850                         td->offset += td->tx_bytes;
 851                         td->remainder -= td->tx_bytes;
 852                         td->toggle = 1;
 853                         goto complete;
 854                 }
 855                 break;
 856
 857         case DWC_CHAN_ST_WAIT_C_PKT:
 858                 goto send_cpkt;
 859
 860         default:
 861                 break;
 862         }
 863         goto busy;
 864
 865 send_pkt:
 866         /* free existing channel, if any */
 867         dwc_otg_host_channel_free(sc, td);
 868
 869         if (sizeof(req) != td->remainder) {
 870                 td->error_any = 1;
 871                 goto complete;
 872         }
 873
 874         if (td->hcsplt != 0) {
 875                 delta = td->tt_start_slot - sc->sc_last_frame_num - 1;
 876                 if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
 877                         td->state = DWC_CHAN_ST_START;
 878                         goto busy;
 879                 }
 880                 delta = sc->sc_last_frame_num - td->tt_start_slot;
 881                 if (delta > 5) {
 882                         /* missed it */
 883                         td->tt_scheduled = 0;
 884                         td->state = DWC_CHAN_ST_START;
 885                         goto busy;
 886                 }
 887         }
 888
 889         /* allocate a new channel */
 890         if (dwc_otg_host_channel_alloc(sc, td, 1)) {
 891                 td->state = DWC_CHAN_ST_START;
 892                 goto busy;
 893         }
 894
 895         if (td->hcsplt != 0) {
 896                 td->hcsplt &= ~HCSPLT_COMPSPLT;
 897                 td->state = DWC_CHAN_ST_WAIT_S_ANE;
 898         } else {
 899                 td->state = DWC_CHAN_ST_WAIT_ANE;
 900         }
 901
 902         usbd_copy_out(td->pc, 0, &req, sizeof(req));
 903
 904         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(td->channel[0]),
 905             (sizeof(req) << HCTSIZ_XFERSIZE_SHIFT) |
 906             (1 << HCTSIZ_PKTCNT_SHIFT) |
 907             (HCTSIZ_PID_SETUP << HCTSIZ_PID_SHIFT));
 908
 909         DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(td->channel[0]), td->hcsplt);
 910
 911         hcchar = td->hcchar;
 912         hcchar &= ~(HCCHAR_EPDIR_IN | HCCHAR_EPTYPE_MASK);
 913         hcchar |= UE_CONTROL << HCCHAR_EPTYPE_SHIFT;
 914
 915         /* must enable channel before writing data to FIFO */
 916         DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(td->channel[0]), hcchar);
 917
 918         /* transfer data into FIFO */
 919         bus_space_write_region_4(sc->sc_io_tag, sc->sc_io_hdl,
 920             DOTG_DFIFO(td->channel[0]), (uint32_t *)&req, sizeof(req) / 4);
 921
 922         /* wait until next slot before trying complete split */
 923         td->tt_complete_slot = sc->sc_last_frame_num + 1;
 924
 925         /* store number of bytes transmitted */
 926         td->tx_bytes = sizeof(req);
 927         goto busy;
 928
 929 send_cpkt:
 930         /* free existing channel, if any */
 931         dwc_otg_host_channel_free(sc, td);
 932
 933         delta = td->tt_complete_slot - sc->sc_last_frame_num - 1;
 934         if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
 935                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
 936                 goto busy;
 937         }
 938         delta = sc->sc_last_frame_num - td->tt_start_slot;
 939         if (delta > DWC_OTG_TT_SLOT_MAX) {
 940                 /* we missed the service interval */
 941                 if (td->ep_type != UE_ISOCHRONOUS)
 942                         td->error_any = 1;
 943                 goto complete;
 944         }
 945         /* allocate a new channel */
 946         if (dwc_otg_host_channel_alloc(sc, td, 0)) {
 947                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
 948                 goto busy;
 949         }
 950
 951         /* wait until next slot before trying complete split */
 952         td->tt_complete_slot = sc->sc_last_frame_num + 1;
 953
 954         td->hcsplt |= HCSPLT_COMPSPLT;
 955         td->state = DWC_CHAN_ST_WAIT_C_ANE;
 956
 957         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(td->channel[0]),
 958             (HCTSIZ_PID_SETUP << HCTSIZ_PID_SHIFT));
 959
 960         DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(td->channel[0]), td->hcsplt);
 961
 962         hcchar = td->hcchar;
 963         hcchar &= ~(HCCHAR_EPDIR_IN | HCCHAR_EPTYPE_MASK);
 964         hcchar |= UE_CONTROL << HCCHAR_EPTYPE_SHIFT;
 965
 966         /* must enable channel before writing data to FIFO */
 967         DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(td->channel[0]), hcchar);
 968
 969 busy:
 970         return (1);     /* busy */
 971
 972 complete:
 973         dwc_otg_host_channel_free(sc, td);
 974         return (0);     /* complete */
 975 }
 976
 977 static uint8_t
 978 dwc_otg_setup_rx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
 979 {
 980         struct usb_device_request req __aligned(4);
 981         uint32_t temp;
 982         uint16_t count;
 983
 984         /* check endpoint status */
 985
 986         if (sc->sc_last_rx_status == 0)
 987                 goto not_complete;
 988
 989         if (GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) != 0)
 990                 goto not_complete;
 991
 992         if ((sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) !=
 993             GRXSTSRD_STP_DATA) {
 994                 if ((sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) !=
 995                     GRXSTSRD_STP_COMPLETE || td->remainder != 0) {
 996                         /* release FIFO */
 997                         dwc_otg_common_rx_ack(sc);
 998                         goto not_complete;
 999                 }
1000                 /* release FIFO */
1001                 dwc_otg_common_rx_ack(sc);
1002                 return (0);     /* complete */
1003         }
1004
1005         if ((sc->sc_last_rx_status & GRXSTSRD_DPID_MASK) !=
1006             GRXSTSRD_DPID_DATA0) {
1007                 /* release FIFO */
1008                 dwc_otg_common_rx_ack(sc);
1009                 goto not_complete;
1010         }
1011
1012         DPRINTFN(5, "GRXSTSR=0x%08x\n", sc->sc_last_rx_status);
1013
1014         /* clear did stall */
1015         td->did_stall = 0;
1016
1017         /* get the packet byte count */
1018         count = GRXSTSRD_BCNT_GET(sc->sc_last_rx_status);
1019
1020         if (count != sizeof(req)) {
1021                 DPRINTFN(0, "Unsupported SETUP packet "
1022                     "length, %d bytes\n", count);
1023                 /* release FIFO */
1024                 dwc_otg_common_rx_ack(sc);
1025                 goto not_complete;
1026         }
1027
1028         /* copy in control request */
1029         memcpy(&req, sc->sc_rx_bounce_buffer, sizeof(req));
1030
1031         /* copy data into real buffer */
1032         usbd_copy_in(td->pc, 0, &req, sizeof(req));
1033
1034         td->offset = sizeof(req);
1035         td->remainder = 0;
1036
1037         /* sneak peek the set address */
1038         if ((req.bmRequestType == UT_WRITE_DEVICE) &&
1039             (req.bRequest == UR_SET_ADDRESS)) {
1040                 /* must write address before ZLP */
1041                 dwc_otg_set_address(sc, req.wValue[0] & 0x7F);
1042         }
1043
1044         /* don't send any data by default */
1045         DWC_OTG_WRITE_4(sc, DOTG_DIEPTSIZ(0), DIEPCTL_EPDIS);
1046         DWC_OTG_WRITE_4(sc, DOTG_DOEPTSIZ(0), DOEPCTL_EPDIS);
1047
1048         /* reset IN endpoint buffer */
1049         dwc_otg_tx_fifo_reset(sc,
1050             GRSTCTL_TXFIFO(0) |
1051             GRSTCTL_TXFFLSH);
1052
1053         /* acknowledge RX status */
1054         dwc_otg_common_rx_ack(sc);
1055         td->did_stall = 1;
1056
1057 not_complete:
1058         /* abort any ongoing transfer, before enabling again */
1059         if (!td->did_stall) {
1060                 td->did_stall = 1;
1061
1062                 DPRINTFN(5, "stalling IN and OUT direction\n");
1063
1064                 temp = sc->sc_out_ctl[0];
1065
1066                 /* set stall after enabling endpoint */
1067                 DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(0),
1068                     temp | DOEPCTL_STALL);
1069
1070                 temp = sc->sc_in_ctl[0];
1071
1072                 /* set stall assuming endpoint is enabled */
1073                 DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(0),
1074                     temp | DIEPCTL_STALL);
1075         }
1076         return (1);                     /* not complete */
1077 }
1078
1079 static uint8_t
1080 dwc_otg_host_rate_check_interrupt(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1081 {
1082         uint8_t delta;
1083
1084         delta = sc->sc_tmr_val - td->tmr_val;
1085         if (delta >= 128)
1086                 return (1);     /* busy */
1087
1088         td->tmr_val = sc->sc_tmr_val + td->tmr_res;
1089
1090         /* set toggle, if any */
1091         if (td->set_toggle) {
1092                 td->set_toggle = 0;
1093                 td->toggle = 1;
1094         }
1095         return (0);
1096 }
1097
1098 static uint8_t
1099 dwc_otg_host_rate_check(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1100 {
1101         uint8_t frame_num = (uint8_t)sc->sc_last_frame_num;
1102
1103         if (td->ep_type == UE_ISOCHRONOUS) {
1104                 /* non TT isochronous traffic */
1105                 if (frame_num & (td->tmr_res - 1))
1106                         goto busy;
1107                 if ((frame_num ^ td->tmr_val) & td->tmr_res)
1108                         goto busy;
1109                 td->tmr_val = td->tmr_res + sc->sc_last_frame_num;
1110                 td->toggle = 0;
1111                 return (0);
1112         } else if (td->ep_type == UE_INTERRUPT) {
1113                 if (!td->tt_scheduled)
1114                         goto busy;
1115                 td->tt_scheduled = 0;
1116                 return (0);
1117         } else if (td->did_nak != 0) {
1118                 /* check if we should pause sending queries for 125us */
1119                 if (td->tmr_res == frame_num) {
1120                         /* wait a bit */
1121                         dwc_otg_enable_sof_irq(sc);
1122                         goto busy;
1123                 }
1124         } else if (td->set_toggle) {
1125                 td->set_toggle = 0;
1126                 td->toggle = 1;
1127         }
1128         /* query for data one more time */
1129         td->tmr_res = frame_num;
1130         td->did_nak = 0;
1131         return (0);
1132 busy:
1133         return (1);
1134 }
1135
1136 static uint8_t
1137 dwc_otg_host_data_rx_sub(struct dwc_otg_softc *sc, struct dwc_otg_td *td,
1138     uint8_t channel)
1139 {
1140         uint32_t count;
1141
1142         /* check endpoint status */
1143         if (sc->sc_last_rx_status == 0)
1144                 goto busy;
1145
1146         if (channel >= DWC_OTG_MAX_CHANNELS)
1147                 goto busy;
1148
1149         if (GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) != channel)
1150                 goto busy;
1151
1152         switch (sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) {
1153         case GRXSTSRH_IN_DATA:
1154
1155                 DPRINTF("DATA ST=%d STATUS=0x%08x\n",
1156                     (int)td->state, (int)sc->sc_last_rx_status);
1157
1158                 if (sc->sc_chan_state[channel].hcint & HCINT_SOFTWARE_ONLY) {
1159                         /*
1160                          * When using SPLIT transactions on interrupt
1161                          * endpoints, sometimes data occurs twice.
1162                          */
1163                         DPRINTF("Data already received\n");
1164                         break;
1165                 }
1166
1167                 /* get the packet byte count */
1168                 count = GRXSTSRD_BCNT_GET(sc->sc_last_rx_status);
1169
1170                 /* check for ISOCHRONOUS endpoint */
1171                 if (td->ep_type == UE_ISOCHRONOUS) {
1172                         if ((sc->sc_last_rx_status & GRXSTSRD_DPID_MASK) !=
1173                             GRXSTSRD_DPID_DATA0) {
1174                                 /* more data to be received */
1175                                 td->tt_xactpos = HCSPLT_XACTPOS_MIDDLE;
1176                         } else {
1177                                 /* all data received */
1178                                 td->tt_xactpos = HCSPLT_XACTPOS_BEGIN;
1179                                 /* verify the packet byte count */
1180                                 if (count != td->remainder) {
1181                                         /* we have a short packet */
1182                                         td->short_pkt = 1;
1183                                         td->got_short = 1;
1184                                 }
1185                         }
1186                 } else {
1187                         /* verify the packet byte count */
1188                         if (count != td->max_packet_size) {
1189                                 if (count < td->max_packet_size) {
1190                                         /* we have a short packet */
1191                                         td->short_pkt = 1;
1192                                         td->got_short = 1;
1193                                 } else {
1194                                         /* invalid USB packet */
1195                                         td->error_any = 1;
1196
1197                                         /* release FIFO */
1198                                         dwc_otg_common_rx_ack(sc);
1199                                         goto complete;
1200                                 }
1201                         }
1202                         td->toggle ^= 1;
1203                         td->tt_scheduled = 0;
1204                 }
1205
1206                 /* verify the packet byte count */
1207                 if (count > td->remainder) {
1208                         /* invalid USB packet */
1209                         td->error_any = 1;
1210
1211                         /* release FIFO */
1212                         dwc_otg_common_rx_ack(sc);
1213                         goto complete;
1214                 }
1215
1216                 usbd_copy_in(td->pc, td->offset,
1217                     sc->sc_rx_bounce_buffer, count);
1218
1219                 td->remainder -= count;
1220                 td->offset += count;
1221                 sc->sc_chan_state[channel].hcint |= HCINT_SOFTWARE_ONLY;
1222                 break;
1223         default:
1224                 break;
1225         }
1226         /* release FIFO */
1227         dwc_otg_common_rx_ack(sc);
1228 busy:
1229         return (0);
1230 complete:
1231         return (1);
1232 }
1233
1234 static uint8_t
1235 dwc_otg_host_data_rx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1236 {
1237         uint32_t hcint = 0;
1238         uint32_t hcchar;
1239         uint8_t delta;
1240         uint8_t channel;
1241         uint8_t x;
1242
1243         for (x = 0; x != td->max_packet_count; x++) {
1244                 channel = td->channel[x];
1245                 if (channel >= DWC_OTG_MAX_CHANNELS)
1246                         continue;
1247                 hcint |= sc->sc_chan_state[channel].hcint;
1248
1249                 DPRINTF("CH=%d ST=%d HCINT=0x%08x HCCHAR=0x%08x HCTSIZ=0x%08x\n",
1250                     channel, td->state, hcint,
1251                     DWC_OTG_READ_4(sc, DOTG_HCCHAR(channel)),
1252                     DWC_OTG_READ_4(sc, DOTG_HCTSIZ(channel)));
1253
1254                 /* check interrupt bits */
1255                 if (hcint & (HCINT_RETRY |
1256                     HCINT_ACK | HCINT_NYET)) {
1257                         /* give success bits priority over failure bits */
1258                 } else if (hcint & HCINT_STALL) {
1259                         DPRINTF("CH=%d STALL\n", channel);
1260                         td->error_stall = 1;
1261                         td->error_any = 1;
1262                         goto complete;
1263                 } else if (hcint & HCINT_ERRORS) {
1264                         DPRINTF("CH=%d ERROR\n", channel);
1265                         td->errcnt++;
1266                         if (td->hcsplt != 0 || td->errcnt >= 3) {
1267                                 if (td->ep_type != UE_ISOCHRONOUS) {
1268                                         td->error_any = 1;
1269                                         goto complete;
1270                                 }
1271                         }
1272                 }
1273
1274                 /* check channels for data, if any */
1275                 if (dwc_otg_host_data_rx_sub(sc, td, channel))
1276                         goto complete;
1277
1278                 /* refresh interrupt status */
1279                 hcint |= sc->sc_chan_state[channel].hcint;
1280
1281                 if (hcint & (HCINT_ERRORS | HCINT_RETRY |
1282                     HCINT_ACK | HCINT_NYET)) {
1283                         if (!(hcint & HCINT_ERRORS))
1284                                 td->errcnt = 0;
1285                 }
1286         }
1287
1288         switch (td->state) {
1289         case DWC_CHAN_ST_START:
1290                 if (td->hcsplt != 0)
1291                         goto receive_spkt;
1292                 else
1293                         goto receive_pkt;
1294
1295         case DWC_CHAN_ST_WAIT_ANE:
1296                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1297                         if (td->ep_type == UE_INTERRUPT) {
1298                                 /*
1299                                  * The USB specification does not
1300                                  * mandate a particular data toggle
1301                                  * value for USB INTERRUPT
1302                                  * transfers. Switch the data toggle
1303                                  * value to receive the packet
1304                                  * correctly:
1305                                  */
1306                                 if (hcint & HCINT_DATATGLERR) {
1307                                         DPRINTF("Retrying packet due to "
1308                                             "data toggle error\n");
1309                                         td->toggle ^= 1;
1310                                         goto receive_pkt;
1311                                 }
1312                         } else if (td->ep_type == UE_ISOCHRONOUS) {
1313                                 goto complete;
1314                         }
1315                         td->did_nak = 1;
1316                         td->tt_scheduled = 0;
1317                         if (td->hcsplt != 0)
1318                                 goto receive_spkt;
1319                         else
1320                                 goto receive_pkt;
1321                 } else if (hcint & HCINT_NYET) {
1322                         if (td->hcsplt != 0) {
1323                                 /* try again */
1324                                 goto receive_pkt;
1325                         } else {
1326                                 /* not a valid token for IN endpoints */
1327                                 td->error_any = 1;
1328                                 goto complete;
1329                         }
1330                 } else if (hcint & HCINT_ACK) {
1331                         /* wait for data - ACK arrived first */
1332                         if (!(hcint & HCINT_SOFTWARE_ONLY))
1333                                 goto busy;
1334
1335                         if (td->ep_type == UE_ISOCHRONOUS) {
1336                                 /* check if we are complete */
1337                                 if (td->tt_xactpos == HCSPLT_XACTPOS_BEGIN) {
1338                                         goto complete;
1339                                 } else {
1340                                         /* get more packets */
1341                                         goto busy;
1342                                 }
1343                         } else {
1344                                 /* check if we are complete */
1345                                 if ((td->remainder == 0) || (td->got_short != 0)) {
1346                                         if (td->short_pkt)
1347                                                 goto complete;
1348
1349                                         /*
1350                                          * Else need to receive a zero length
1351                                          * packet.
1352                                          */
1353                                 }
1354                                 td->tt_scheduled = 0;
1355                                 td->did_nak = 0;
1356                                 if (td->hcsplt != 0)
1357                                         goto receive_spkt;
1358                                 else
1359                                         goto receive_pkt;
1360                         }
1361                 }
1362                 break;
1363
1364         case DWC_CHAN_ST_WAIT_S_ANE:
1365                 /*
1366                  * NOTE: The DWC OTG hardware provides a fake ACK in
1367                  * case of interrupt and isochronous transfers:
1368                  */
1369                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1370                         td->did_nak = 1;
1371                         td->tt_scheduled = 0;
1372                         goto receive_spkt;
1373                 } else if (hcint & HCINT_NYET) {
1374                         td->tt_scheduled = 0;
1375                         goto receive_spkt;
1376                 } else if (hcint & HCINT_ACK) {
1377                         td->did_nak = 0;
1378                         goto receive_pkt;
1379                 }
1380                 break;
1381
1382         case DWC_CHAN_ST_WAIT_C_PKT:
1383                 goto receive_pkt;
1384
1385         default:
1386                 break;
1387         }
1388         goto busy;
1389
1390 receive_pkt:
1391         /* free existing channel, if any */
1392         dwc_otg_host_channel_free(sc, td);
1393
1394         if (td->hcsplt != 0) {
1395                 delta = td->tt_complete_slot - sc->sc_last_frame_num - 1;
1396                 if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
1397                         td->state = DWC_CHAN_ST_WAIT_C_PKT;
1398                         goto busy;
1399                 }
1400                 delta = sc->sc_last_frame_num - td->tt_start_slot;
1401                 if (delta > DWC_OTG_TT_SLOT_MAX) {
1402                         if (td->ep_type != UE_ISOCHRONOUS) {
1403                                 /* we missed the service interval */
1404                                 td->error_any = 1;
1405                         }
1406                         goto complete;
1407                 }
1408                 /* complete split */
1409                 td->hcsplt |= HCSPLT_COMPSPLT;
1410         } else if (dwc_otg_host_rate_check(sc, td)) {
1411                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
1412                 goto busy;
1413         }
1414
1415         /* allocate a new channel */
1416         if (dwc_otg_host_channel_alloc(sc, td, 0)) {
1417                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
1418                 goto busy;
1419         }
1420
1421         /* set toggle, if any */
1422         if (td->set_toggle) {
1423                 td->set_toggle = 0;
1424                 td->toggle = 1;
1425         }
1426
1427         td->state = DWC_CHAN_ST_WAIT_ANE;
1428
1429         for (x = 0; x != td->max_packet_count; x++) {
1430                 channel = td->channel[x];
1431
1432                 /* receive one packet */
1433                 DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1434                     (td->max_packet_size << HCTSIZ_XFERSIZE_SHIFT) |
1435                     (1 << HCTSIZ_PKTCNT_SHIFT) |
1436                     (td->toggle ? (HCTSIZ_PID_DATA1 << HCTSIZ_PID_SHIFT) :
1437                     (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT)));
1438
1439                 DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(channel), td->hcsplt);
1440
1441                 hcchar = td->hcchar;
1442                 hcchar |= HCCHAR_EPDIR_IN;
1443
1444                 /* receive complete split ASAP */
1445                 if ((sc->sc_last_frame_num & 1) != 0 &&
1446                     td->ep_type == UE_ISOCHRONOUS)
1447                         hcchar |= HCCHAR_ODDFRM;
1448                 else
1449                         hcchar &= ~HCCHAR_ODDFRM;
1450
1451                 /* must enable channel before data can be received */
1452                 DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(channel), hcchar);
1453         }
1454         /* wait until next slot before trying complete split */
1455         td->tt_complete_slot = sc->sc_last_frame_num + 1;
1456
1457         goto busy;
1458
1459 receive_spkt:
1460         /* free existing channel(s), if any */
1461         dwc_otg_host_channel_free(sc, td);
1462
1463         delta = td->tt_start_slot - sc->sc_last_frame_num - 1;
1464         if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
1465                 td->state = DWC_CHAN_ST_START;
1466                 goto busy;
1467         }
1468         delta = sc->sc_last_frame_num - td->tt_start_slot;
1469         if (delta > 5) {
1470                 /* missed it */
1471                 td->tt_scheduled = 0;
1472                 td->state = DWC_CHAN_ST_START;
1473                 goto busy;
1474         }
1475
1476         /* allocate a new channel */
1477         if (dwc_otg_host_channel_alloc(sc, td, 0)) {
1478                 td->state = DWC_CHAN_ST_START;
1479                 goto busy;
1480         }
1481
1482         channel = td->channel[0];
1483
1484         td->hcsplt &= ~HCSPLT_COMPSPLT;
1485         td->state = DWC_CHAN_ST_WAIT_S_ANE;
1486
1487         /* receive one packet */
1488         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1489             (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT));
1490
1491         DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(channel), td->hcsplt);
1492
1493         /* send after next SOF event */
1494         if ((sc->sc_last_frame_num & 1) == 0 &&
1495             td->ep_type == UE_ISOCHRONOUS)
1496                 td->hcchar |= HCCHAR_ODDFRM;
1497         else
1498                 td->hcchar &= ~HCCHAR_ODDFRM;
1499
1500         hcchar = td->hcchar;
1501         hcchar |= HCCHAR_EPDIR_IN;
1502
1503         /* wait until next slot before trying complete split */
1504         td->tt_complete_slot = sc->sc_last_frame_num + 1;
1505
1506         /* must enable channel before data can be received */
1507         DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(channel), hcchar);
1508 busy:
1509         return (1);     /* busy */
1510
1511 complete:
1512         dwc_otg_host_channel_free(sc, td);
1513         return (0);     /* complete */
1514 }
1515
1516 static uint8_t
1517 dwc_otg_data_rx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1518 {
1519         uint32_t temp;
1520         uint16_t count;
1521         uint8_t got_short;
1522
1523         got_short = 0;
1524
1525         /* check endpoint status */
1526         if (sc->sc_last_rx_status == 0)
1527                 goto not_complete;
1528
1529         if (GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) != td->ep_no)
1530                 goto not_complete;
1531
1532         /* check for SETUP packet */
1533         if ((sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) ==
1534             GRXSTSRD_STP_DATA ||
1535             (sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) ==
1536             GRXSTSRD_STP_COMPLETE) {
1537                 if (td->remainder == 0) {
1538                         /*
1539                          * We are actually complete and have
1540                          * received the next SETUP
1541                          */
1542                         DPRINTFN(5, "faking complete\n");
1543                         return (0);     /* complete */
1544                 }
1545                 /*
1546                  * USB Host Aborted the transfer.
1547                  */
1548                 td->error_any = 1;
1549                 return (0);             /* complete */
1550         }
1551
1552         if ((sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) !=
1553             GRXSTSRD_OUT_DATA) {
1554                 /* release FIFO */
1555                 dwc_otg_common_rx_ack(sc);
1556                 goto not_complete;
1557         }
1558
1559         /* get the packet byte count */
1560         count = GRXSTSRD_BCNT_GET(sc->sc_last_rx_status);
1561
1562         /* verify the packet byte count */
1563         if (count != td->max_packet_size) {
1564                 if (count < td->max_packet_size) {
1565                         /* we have a short packet */
1566                         td->short_pkt = 1;
1567                         got_short = 1;
1568                 } else {
1569                         /* invalid USB packet */
1570                         td->error_any = 1;
1571
1572                         /* release FIFO */
1573                         dwc_otg_common_rx_ack(sc);
1574                         return (0);     /* we are complete */
1575                 }
1576         }
1577         /* verify the packet byte count */
1578         if (count > td->remainder) {
1579                 /* invalid USB packet */
1580                 td->error_any = 1;
1581
1582                 /* release FIFO */
1583                 dwc_otg_common_rx_ack(sc);
1584                 return (0);             /* we are complete */
1585         }
1586
1587         usbd_copy_in(td->pc, td->offset, sc->sc_rx_bounce_buffer, count);
1588         td->remainder -= count;
1589         td->offset += count;
1590
1591         /* release FIFO */
1592         dwc_otg_common_rx_ack(sc);
1593
1594         temp = sc->sc_out_ctl[td->ep_no];
1595
1596         /* check for isochronous mode */
1597         if ((temp & DIEPCTL_EPTYPE_MASK) ==
1598             (DIEPCTL_EPTYPE_ISOC << DIEPCTL_EPTYPE_SHIFT)) {
1599                 /* toggle odd or even frame bit */
1600                 if (temp & DIEPCTL_SETD1PID) {
1601                         temp &= ~DIEPCTL_SETD1PID;
1602                         temp |= DIEPCTL_SETD0PID;
1603                 } else {
1604                         temp &= ~DIEPCTL_SETD0PID;
1605                         temp |= DIEPCTL_SETD1PID;
1606                 }
1607                 sc->sc_out_ctl[td->ep_no] = temp;
1608         }
1609
1610         /* check if we are complete */
1611         if ((td->remainder == 0) || got_short) {
1612                 if (td->short_pkt) {
1613                         /* we are complete */
1614                         return (0);
1615                 }
1616                 /* else need to receive a zero length packet */
1617         }
1618
1619 not_complete:
1620
1621         /* enable SETUP and transfer complete interrupt */
1622         if (td->ep_no == 0) {
1623                 DWC_OTG_WRITE_4(sc, DOTG_DOEPTSIZ(0),
1624                     DXEPTSIZ_SET_MULTI(3) |
1625                     DXEPTSIZ_SET_NPKT(1) |
1626                     DXEPTSIZ_SET_NBYTES(td->max_packet_size));
1627         } else {
1628                 /* allow reception of multiple packets */
1629                 DWC_OTG_WRITE_4(sc, DOTG_DOEPTSIZ(td->ep_no),
1630                     DXEPTSIZ_SET_MULTI(1) |
1631                     DXEPTSIZ_SET_NPKT(4) |
1632                     DXEPTSIZ_SET_NBYTES(4 *
1633                         ((td->max_packet_size + 3) & ~3)));
1634         }
1635         temp = sc->sc_out_ctl[td->ep_no];
1636         DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(td->ep_no), temp |
1637             DOEPCTL_EPENA | DOEPCTL_CNAK);
1638
1639         return (1);                     /* not complete */
1640 }
1641
1642 static uint8_t
1643 dwc_otg_host_data_tx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1644 {
1645         uint32_t count;
1646         uint32_t hcint;
1647         uint32_t hcchar;
1648         uint8_t delta;
1649         uint8_t channel;
1650         uint8_t x;
1651
1652         dwc_otg_host_dump_rx(sc, td);
1653
1654         /* check that last channel is complete */
1655         channel = td->channel[td->npkt];
1656
1657         if (channel < DWC_OTG_MAX_CHANNELS) {
1658                 hcint = sc->sc_chan_state[channel].hcint;
1659
1660                 DPRINTF("CH=%d ST=%d HCINT=0x%08x HCCHAR=0x%08x HCTSIZ=0x%08x\n",
1661                     channel, td->state, hcint,
1662                     DWC_OTG_READ_4(sc, DOTG_HCCHAR(channel)),
1663                     DWC_OTG_READ_4(sc, DOTG_HCTSIZ(channel)));
1664
1665                 if (hcint & (HCINT_RETRY |
1666                     HCINT_ACK | HCINT_NYET)) {
1667                         /* give success bits priority over failure bits */
1668                 } else if (hcint & HCINT_STALL) {
1669                         DPRINTF("CH=%d STALL\n", channel);
1670                         td->error_stall = 1;
1671                         td->error_any = 1;
1672                         goto complete;
1673                 } else if (hcint & HCINT_ERRORS) {
1674                         DPRINTF("CH=%d ERROR\n", channel);
1675                         td->errcnt++;
1676                         if (td->hcsplt != 0 || td->errcnt >= 3) {
1677                                 td->error_any = 1;
1678                                 goto complete;
1679                         }
1680                 }
1681
1682                 if (hcint & (HCINT_ERRORS | HCINT_RETRY |
1683                     HCINT_ACK | HCINT_NYET)) {
1684
1685                         if (!(hcint & HCINT_ERRORS))
1686                                 td->errcnt = 0;
1687                 }
1688         } else {
1689                 hcint = 0;
1690         }
1691
1692         switch (td->state) {
1693         case DWC_CHAN_ST_START:
1694                 goto send_pkt;
1695
1696         case DWC_CHAN_ST_WAIT_ANE:
1697                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1698                         td->did_nak = 1;
1699                         td->tt_scheduled = 0;
1700                         goto send_pkt;
1701                 } else if (hcint & (HCINT_ACK | HCINT_NYET)) {
1702                         td->offset += td->tx_bytes;
1703                         td->remainder -= td->tx_bytes;
1704                         td->toggle ^= 1;
1705                         /* check if next response will be a NAK */
1706                         if (hcint & HCINT_NYET)
1707                                 td->did_nak = 1;
1708                         else
1709                                 td->did_nak = 0;
1710                         td->tt_scheduled = 0;
1711
1712                         /* check remainder */
1713                         if (td->remainder == 0) {
1714                                 if (td->short_pkt)
1715                                         goto complete;
1716
1717                                 /*
1718                                  * Else we need to transmit a short
1719                                  * packet:
1720                                  */
1721                         }
1722                         goto send_pkt;
1723                 }
1724                 break;
1725
1726         case DWC_CHAN_ST_WAIT_S_ANE:
1727                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1728                         td->did_nak = 1;
1729                         td->tt_scheduled = 0;
1730                         goto send_pkt;
1731                 } else if (hcint & (HCINT_ACK | HCINT_NYET)) {
1732                         td->did_nak = 0;
1733                         goto send_cpkt;
1734                 }
1735                 break;
1736
1737         case DWC_CHAN_ST_WAIT_C_ANE:
1738                 if (hcint & HCINT_NYET) {
1739                         goto send_cpkt;
1740                 } else if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1741                         td->did_nak = 1;
1742                         td->tt_scheduled = 0;
1743                         goto send_pkt;
1744                 } else if (hcint & HCINT_ACK) {
1745                         td->offset += td->tx_bytes;
1746                         td->remainder -= td->tx_bytes;
1747                         td->toggle ^= 1;
1748                         td->did_nak = 0;
1749                         td->tt_scheduled = 0;
1750
1751                         /* check remainder */
1752                         if (td->remainder == 0) {
1753                                 if (td->short_pkt)
1754                                         goto complete;
1755
1756                                 /* else we need to transmit a short packet */
1757                         }
1758                         goto send_pkt;
1759                 }
1760                 break;
1761
1762         case DWC_CHAN_ST_WAIT_C_PKT:
1763                 goto send_cpkt;
1764
1765         case DWC_CHAN_ST_TX_WAIT_ISOC:
1766                 /* Check if ISOCHRONOUS OUT traffic is complete */
1767                 if ((hcint & HCINT_HCH_DONE_MASK) == 0)
1768                         break;
1769
1770                 td->offset += td->tx_bytes;
1771                 td->remainder -= td->tx_bytes;
1772                 goto complete;
1773         default:
1774                 break;
1775         }
1776         goto busy;
1777
1778 send_pkt:
1779         /* free existing channel(s), if any */
1780         dwc_otg_host_channel_free(sc, td);
1781
1782         if (td->hcsplt != 0) {
1783                 delta = td->tt_start_slot - sc->sc_last_frame_num - 1;
1784                 if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
1785                         td->state = DWC_CHAN_ST_START;
1786                         goto busy;
1787                 }
1788                 delta = sc->sc_last_frame_num - td->tt_start_slot;
1789                 if (delta > 5) {
1790                         /* missed it */
1791                         td->tt_scheduled = 0;
1792                         td->state = DWC_CHAN_ST_START;
1793                         goto busy;
1794                 }
1795         } else if (dwc_otg_host_rate_check(sc, td)) {
1796                 td->state = DWC_CHAN_ST_START;
1797                 goto busy;
1798         }
1799
1800         /* allocate a new channel */
1801         if (dwc_otg_host_channel_alloc(sc, td, 1)) {
1802                 td->state = DWC_CHAN_ST_START;
1803                 goto busy;
1804         }
1805
1806         /* set toggle, if any */
1807         if (td->set_toggle) {
1808                 td->set_toggle = 0;
1809                 td->toggle = 1;
1810         }
1811
1812         if (td->ep_type == UE_ISOCHRONOUS) {
1813                 /* ISOCHRONOUS OUT transfers don't have any ACKs */
1814                 td->state = DWC_CHAN_ST_TX_WAIT_ISOC;
1815                 td->hcsplt &= ~HCSPLT_COMPSPLT;
1816                 if (td->hcsplt != 0) {
1817                         /* get maximum transfer length */
1818                         count = td->remainder;
1819                         if (count > HCSPLT_XACTLEN_BURST) {
1820                                 DPRINTF("TT overflow\n");
1821                                 td->error_any = 1;
1822                                 goto complete;
1823                         }
1824                         /* Update transaction position */
1825                         td->hcsplt &= ~HCSPLT_XACTPOS_MASK;
1826                         td->hcsplt |= (HCSPLT_XACTPOS_ALL << HCSPLT_XACTPOS_SHIFT);
1827                 }
1828         } else if (td->hcsplt != 0) {
1829                 td->hcsplt &= ~HCSPLT_COMPSPLT;
1830                 /* Wait for ACK/NAK/ERR from TT */
1831                 td->state = DWC_CHAN_ST_WAIT_S_ANE;
1832         } else {
1833                 /* Wait for ACK/NAK/STALL from device */
1834                 td->state = DWC_CHAN_ST_WAIT_ANE;
1835         }
1836
1837         td->tx_bytes = 0;
1838
1839         for (x = 0; x != td->max_packet_count; x++) {
1840                 uint32_t rem_bytes;
1841
1842                 channel = td->channel[x];
1843
1844                 /* send one packet at a time */
1845                 count = td->max_packet_size;
1846                 rem_bytes = td->remainder - td->tx_bytes;
1847                 if (rem_bytes < count) {
1848                         /* we have a short packet */
1849                         td->short_pkt = 1;
1850                         count = rem_bytes;
1851                 }
1852                 if (count == rem_bytes) {
1853                         /* last packet */
1854                         switch (x) {
1855                         case 0:
1856                                 DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1857                                     (count << HCTSIZ_XFERSIZE_SHIFT) |
1858                                     (1 << HCTSIZ_PKTCNT_SHIFT) |
1859                                     (td->toggle ? (HCTSIZ_PID_DATA1 << HCTSIZ_PID_SHIFT) :
1860                                     (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT)));
1861                                 break;
1862                         case 1:
1863                                 DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1864                                     (count << HCTSIZ_XFERSIZE_SHIFT) |
1865                                     (1 << HCTSIZ_PKTCNT_SHIFT) |
1866                                     (HCTSIZ_PID_DATA1 << HCTSIZ_PID_SHIFT));
1867                                 break;
1868                         default:
1869                                 DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1870                                     (count << HCTSIZ_XFERSIZE_SHIFT) |
1871                                     (1 << HCTSIZ_PKTCNT_SHIFT) |
1872                                     (HCTSIZ_PID_DATA2 << HCTSIZ_PID_SHIFT));
1873                                 break;
1874                         }
1875                 } else if (td->ep_type == UE_ISOCHRONOUS &&
1876                            td->max_packet_count > 1) {
1877                         /* ISOCHRONOUS multi packet */
1878                         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1879                             (count << HCTSIZ_XFERSIZE_SHIFT) |
1880                             (1 << HCTSIZ_PKTCNT_SHIFT) |
1881                             (HCTSIZ_PID_MDATA << HCTSIZ_PID_SHIFT));
1882                 } else {
1883                         /* TODO: HCTSIZ_DOPNG */
1884                         /* standard BULK/INTERRUPT/CONTROL packet */
1885                         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1886                             (count << HCTSIZ_XFERSIZE_SHIFT) |
1887                             (1 << HCTSIZ_PKTCNT_SHIFT) |
1888                             (td->toggle ? (HCTSIZ_PID_DATA1 << HCTSIZ_PID_SHIFT) :
1889                             (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT)));
1890                 }
1891
1892                 DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(channel), td->hcsplt);
1893
1894                 hcchar = td->hcchar;
1895                 hcchar &= ~HCCHAR_EPDIR_IN;
1896
1897                 /* send after next SOF event */
1898                 if ((sc->sc_last_frame_num & 1) == 0 &&
1899                     td->ep_type == UE_ISOCHRONOUS)
1900                         hcchar |= HCCHAR_ODDFRM;
1901                 else
1902                         hcchar &= ~HCCHAR_ODDFRM;
1903
1904                 /* must enable before writing data to FIFO */
1905                 DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(channel), hcchar);
1906
1907                 if (count != 0) {
1908                         /* clear topmost word before copy */
1909                         sc->sc_tx_bounce_buffer[(count - 1) / 4] = 0;
1910
1911                         /* copy out data */
1912                         usbd_copy_out(td->pc, td->offset + td->tx_bytes,
1913                             sc->sc_tx_bounce_buffer, count);
1914
1915                         /* transfer data into FIFO */
1916                         bus_space_write_region_4(sc->sc_io_tag, sc->sc_io_hdl,
1917                             DOTG_DFIFO(channel),
1918                             sc->sc_tx_bounce_buffer, (count + 3) / 4);
1919                 }
1920
1921                 /* store number of bytes transmitted */
1922                 td->tx_bytes += count;
1923
1924                 /* store last packet index */
1925                 td->npkt = x;
1926
1927                 /* check for last packet */
1928                 if (count == rem_bytes)
1929                         break;
1930         }
1931         goto busy;
1932
1933 send_cpkt:
1934         /* free existing channel, if any */
1935         dwc_otg_host_channel_free(sc, td);
1936
1937         delta = td->tt_complete_slot - sc->sc_last_frame_num - 1;
1938         if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
1939                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
1940                 goto busy;
1941         }
1942         delta = sc->sc_last_frame_num - td->tt_start_slot;
1943         if (delta > DWC_OTG_TT_SLOT_MAX) {
1944                 /* we missed the service interval */
1945                 if (td->ep_type != UE_ISOCHRONOUS)
1946                         td->error_any = 1;
1947                 goto complete;
1948         }
1949
1950         /* allocate a new channel */
1951         if (dwc_otg_host_channel_alloc(sc, td, 0)) {
1952                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
1953                 goto busy;
1954         }
1955
1956         channel = td->channel[0];
1957
1958         td->hcsplt |= HCSPLT_COMPSPLT;
1959         td->state = DWC_CHAN_ST_WAIT_C_ANE;
1960
1961         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1962             (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT));
1963
1964         DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(channel), td->hcsplt);
1965
1966         hcchar = td->hcchar;
1967         hcchar &= ~HCCHAR_EPDIR_IN;
1968
1969         /* receive complete split ASAP */
1970         if ((sc->sc_last_frame_num & 1) != 0 &&
1971             td->ep_type == UE_ISOCHRONOUS)
1972                 hcchar |= HCCHAR_ODDFRM;
1973         else
1974                 hcchar &= ~HCCHAR_ODDFRM;
1975
1976         /* must enable channel before data can be received */
1977         DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(channel), hcchar);
1978
1979         /* wait until next slot before trying complete split */
1980         td->tt_complete_slot = sc->sc_last_frame_num + 1;
1981 busy:
1982         return (1);     /* busy */
1983
1984 complete:
1985         dwc_otg_host_channel_free(sc, td);
1986         return (0);     /* complete */
1987 }
1988
1989 static uint8_t
1990 dwc_otg_data_tx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1991 {
1992         uint32_t max_buffer;
1993         uint32_t count;
1994         uint32_t fifo_left;
1995         uint32_t mpkt;
1996         uint32_t temp;
1997         uint8_t to;
1998
1999         to = 3;                         /* don't loop forever! */
2000
2001         max_buffer = sc->sc_hw_ep_profile[td->ep_no].max_buffer;
2002
2003 repeat:
2004         /* check for for endpoint 0 data */
2005
2006         temp = sc->sc_last_rx_status;
2007
2008         if ((td->ep_no == 0) && (temp != 0) &&
2009             (GRXSTSRD_CHNUM_GET(temp) == 0)) {
2010
2011                 if ((temp & GRXSTSRD_PKTSTS_MASK) !=
2012                     GRXSTSRD_STP_DATA &&
2013                     (temp & GRXSTSRD_PKTSTS_MASK) !=
2014                     GRXSTSRD_STP_COMPLETE) {
2015
2016                         /* dump data - wrong direction */
2017                         dwc_otg_common_rx_ack(sc);
2018                 } else {
2019                         /*
2020                          * The current transfer was cancelled
2021                          * by the USB Host:
2022                          */
2023                         td->error_any = 1;
2024                         return (0);             /* complete */
2025                 }
2026         }
2027
2028         /* fill in more TX data, if possible */
2029         if (td->tx_bytes != 0) {
2030
2031                 uint16_t cpkt;
2032
2033                 /* check if packets have been transferred */
2034                 temp = DWC_OTG_READ_4(sc, DOTG_DIEPTSIZ(td->ep_no));
2035
2036                 /* get current packet number */
2037                 cpkt = DXEPTSIZ_GET_NPKT(temp);
2038
2039                 if (cpkt >= td->npkt) {
2040                         fifo_left = 0;
2041                 } else {
2042                         if (max_buffer != 0) {
2043                                 fifo_left = (td->npkt - cpkt) *
2044                                     td->max_packet_size;
2045
2046                                 if (fifo_left > max_buffer)
2047                                         fifo_left = max_buffer;
2048                         } else {
2049                                 fifo_left = td->max_packet_size;
2050                         }
2051                 }
2052
2053                 count = td->tx_bytes;
2054                 if (count > fifo_left)
2055                         count = fifo_left;
2056
2057                 if (count != 0) {
2058
2059                         /* clear topmost word before copy */
2060                         sc->sc_tx_bounce_buffer[(count - 1) / 4] = 0;
2061
2062                         /* copy out data */
2063                         usbd_copy_out(td->pc, td->offset,
2064                             sc->sc_tx_bounce_buffer, count);
2065
2066                         /* transfer data into FIFO */
2067                         bus_space_write_region_4(sc->sc_io_tag, sc->sc_io_hdl,
2068                             DOTG_DFIFO(td->ep_no),
2069                             sc->sc_tx_bounce_buffer, (count + 3) / 4);
2070
2071                         td->tx_bytes -= count;
2072                         td->remainder -= count;
2073                         td->offset += count;
2074                         td->npkt = cpkt;
2075                 }
2076                 if (td->tx_bytes != 0)
2077                         goto not_complete;
2078
2079                 /* check remainder */
2080                 if (td->remainder == 0) {
2081                         if (td->short_pkt)
2082                                 return (0);     /* complete */
2083
2084                         /* else we need to transmit a short packet */
2085                 }
2086         }
2087
2088         if (!to--)
2089                 goto not_complete;
2090
2091         /* check if not all packets have been transferred */
2092         temp = DWC_OTG_READ_4(sc, DOTG_DIEPTSIZ(td->ep_no));
2093
2094         if (DXEPTSIZ_GET_NPKT(temp) != 0) {
2095
2096                 DPRINTFN(5, "busy ep=%d npkt=%d DIEPTSIZ=0x%08x "
2097                     "DIEPCTL=0x%08x\n", td->ep_no,
2098                     DXEPTSIZ_GET_NPKT(temp),
2099                     temp, DWC_OTG_READ_4(sc, DOTG_DIEPCTL(td->ep_no)));
2100
2101                 goto not_complete;
2102         }
2103
2104         DPRINTFN(5, "rem=%u ep=%d\n", td->remainder, td->ep_no);
2105
2106         /* try to optimise by sending more data */
2107         if ((max_buffer != 0) && ((td->max_packet_size & 3) == 0)) {
2108
2109                 /* send multiple packets at the same time */
2110                 mpkt = max_buffer / td->max_packet_size;
2111
2112                 if (mpkt > 0x3FE)
2113                         mpkt = 0x3FE;
2114
2115                 count = td->remainder;
2116                 if (count > 0x7FFFFF)
2117                         count = 0x7FFFFF - (0x7FFFFF % td->max_packet_size);
2118
2119                 td->npkt = count / td->max_packet_size;
2120
2121                 /*
2122                  * NOTE: We could use 0x3FE instead of "mpkt" in the
2123                  * check below to get more throughput, but then we
2124                  * have a dependency towards non-generic chip features
2125                  * to disable the TX-FIFO-EMPTY interrupts on a per
2126                  * endpoint basis. Increase the maximum buffer size of
2127                  * the IN endpoint to increase the performance.
2128                  */
2129                 if (td->npkt > mpkt) {
2130                         td->npkt = mpkt;
2131                         count = td->max_packet_size * mpkt;
2132                 } else if ((count == 0) || (count % td->max_packet_size)) {
2133                         /* we are transmitting a short packet */
2134                         td->npkt++;
2135                         td->short_pkt = 1;
2136                 }
2137         } else {
2138                 /* send one packet at a time */
2139                 mpkt = 1;
2140                 count = td->max_packet_size;
2141                 if (td->remainder < count) {
2142                         /* we have a short packet */
2143                         td->short_pkt = 1;
2144                         count = td->remainder;
2145                 }
2146                 td->npkt = 1;
2147         }
2148         DWC_OTG_WRITE_4(sc, DOTG_DIEPTSIZ(td->ep_no),
2149             DXEPTSIZ_SET_MULTI(1) |
2150             DXEPTSIZ_SET_NPKT(td->npkt) |
2151             DXEPTSIZ_SET_NBYTES(count));
2152
2153         /* make room for buffering */
2154         td->npkt += mpkt;
2155
2156         temp = sc->sc_in_ctl[td->ep_no];
2157
2158         /* check for isochronous mode */
2159         if ((temp & DIEPCTL_EPTYPE_MASK) ==
2160             (DIEPCTL_EPTYPE_ISOC << DIEPCTL_EPTYPE_SHIFT)) {
2161                 /* toggle odd or even frame bit */
2162                 if (temp & DIEPCTL_SETD1PID) {
2163                         temp &= ~DIEPCTL_SETD1PID;
2164                         temp |= DIEPCTL_SETD0PID;
2165                 } else {
2166                         temp &= ~DIEPCTL_SETD0PID;
2167                         temp |= DIEPCTL_SETD1PID;
2168                 }
2169                 sc->sc_in_ctl[td->ep_no] = temp;
2170         }
2171
2172         /* must enable before writing data to FIFO */
2173         DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(td->ep_no), temp |
2174             DIEPCTL_EPENA | DIEPCTL_CNAK);
2175
2176         td->tx_bytes = count;
2177
2178         /* check remainder */
2179         if (td->tx_bytes == 0 &&
2180             td->remainder == 0) {
2181                 if (td->short_pkt)
2182                         return (0);     /* complete */
2183
2184                 /* else we need to transmit a short packet */
2185         }
2186         goto repeat;
2187
2188 not_complete:
2189         return (1);                     /* not complete */
2190 }
2191
2192 static uint8_t
2193 dwc_otg_data_tx_sync(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
2194 {
2195         uint32_t temp;
2196
2197         /*
2198          * If all packets are transferred we are complete:
2199          */
2200         temp = DWC_OTG_READ_4(sc, DOTG_DIEPTSIZ(td->ep_no));
2201
2202         /* check that all packets have been transferred */
2203         if (DXEPTSIZ_GET_NPKT(temp) != 0) {
2204                 DPRINTFN(5, "busy ep=%d\n", td->ep_no);
2205                 goto not_complete;
2206         }
2207         return (0);
2208
2209 not_complete:
2210
2211         /* we only want to know if there is a SETUP packet or free IN packet */
2212
2213         temp = sc->sc_last_rx_status;
2214
2215         if ((td->ep_no == 0) && (temp != 0) &&
2216             (GRXSTSRD_CHNUM_GET(temp) == 0)) {
2217
2218                 if ((temp & GRXSTSRD_PKTSTS_MASK) ==
2219                     GRXSTSRD_STP_DATA ||
2220                     (temp & GRXSTSRD_PKTSTS_MASK) ==
2221                     GRXSTSRD_STP_COMPLETE) {
2222                         DPRINTFN(5, "faking complete\n");
2223                         /*
2224                          * Race condition: We are complete!
2225                          */
2226                         return (0);
2227                 } else {
2228                         /* dump data - wrong direction */
2229                         dwc_otg_common_rx_ack(sc);
2230                 }
2231         }
2232         return (1);                     /* not complete */
2233 }
2234
2235 static void
2236 dwc_otg_xfer_do_fifo(struct dwc_otg_softc *sc, struct usb_xfer *xfer)
2237 {
2238         struct dwc_otg_td *td;
2239         uint8_t toggle;
2240         uint8_t tmr_val;
2241         uint8_t tmr_res;
2242
2243         DPRINTFN(9, "\n");
2244
2245         td = xfer->td_transfer_cache;
2246         if (td == NULL)
2247                 return;
2248
2249         while (1) {
2250                 if ((td->func) (sc, td)) {
2251                         /* operation in progress */
2252                         break;
2253                 }
2254                 if (((void *)td) == xfer->td_transfer_last) {
2255                         goto done;
2256                 }
2257                 if (td->error_any) {
2258                         goto done;
2259                 } else if (td->remainder > 0) {
2260                         /*
2261                          * We had a short transfer. If there is no alternate
2262                          * next, stop processing !
2263                          */
2264                         if (!td->alt_next)
2265                                 goto done;
2266                 }
2267
2268                 /*
2269                  * Fetch the next transfer descriptor and transfer
2270                  * some flags to the next transfer descriptor
2271                  */
2272                 tmr_res = td->tmr_res;
2273                 tmr_val = td->tmr_val;
2274                 toggle = td->toggle;
2275                 td = td->obj_next;
2276                 xfer->td_transfer_cache = td;
2277                 td->toggle = toggle;    /* transfer toggle */
2278                 td->tmr_res = tmr_res;
2279                 td->tmr_val = tmr_val;
2280         }
2281         return;
2282
2283 done:
2284         xfer->td_transfer_cache = NULL;
2285         sc->sc_xfer_complete = 1;
2286 }
2287
2288 static uint8_t
2289 dwc_otg_xfer_do_complete_locked(struct dwc_otg_softc *sc, struct usb_xfer *xfer)
2290 {
2291         struct dwc_otg_td *td;
2292
2293         DPRINTFN(9, "\n");
2294
2295         td = xfer->td_transfer_cache;
2296         if (td == NULL) {
2297                 /* compute all actual lengths */
2298                 dwc_otg_standard_done(xfer);
2299                 return (1);
2300         }
2301         return (0);
2302 }
2303
2304 static void
2305 dwc_otg_timer(void *_sc)
2306 {
2307         struct dwc_otg_softc *sc = _sc;
2308
2309         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
2310
2311         DPRINTF("\n");
2312
2313         USB_BUS_SPIN_LOCK(&sc->sc_bus);
2314
2315         /* increment timer value */
2316         sc->sc_tmr_val++;
2317
2318         /* enable SOF interrupt, which will poll jobs */
2319         dwc_otg_enable_sof_irq(sc);
2320
2321         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
2322
2323         if (sc->sc_timer_active) {
2324                 /* restart timer */
2325                 usb_callout_reset(&sc->sc_timer,
2326                     hz / (1000 / DWC_OTG_HOST_TIMER_RATE),
2327                     &dwc_otg_timer, sc);
2328         }
2329 }
2330
2331 static void
2332 dwc_otg_timer_start(struct dwc_otg_softc *sc)
2333 {
2334         if (sc->sc_timer_active != 0)
2335                 return;
2336
2337         sc->sc_timer_active = 1;
2338
2339         /* restart timer */
2340         usb_callout_reset(&sc->sc_timer,
2341             hz / (1000 / DWC_OTG_HOST_TIMER_RATE),
2342             &dwc_otg_timer, sc);
2343 }
2344
2345 static void
2346 dwc_otg_timer_stop(struct dwc_otg_softc *sc)
2347 {
2348         if (sc->sc_timer_active == 0)
2349                 return;
2350
2351         sc->sc_timer_active = 0;
2352
2353         /* stop timer */
2354         usb_callout_stop(&sc->sc_timer);
2355 }
2356
2357 static uint16_t
2358 dwc_otg_compute_isoc_rx_tt_slot(struct dwc_otg_tt_info *pinfo)
2359 {
2360         if (pinfo->slot_index < DWC_OTG_TT_SLOT_MAX)
2361                 pinfo->slot_index++;
2362         return (pinfo->slot_index);
2363 }
2364
2365 static uint8_t
2366 dwc_otg_update_host_transfer_schedule_locked(struct dwc_otg_softc *sc)
2367 {
2368         TAILQ_HEAD(, usb_xfer) head;
2369         struct usb_xfer *xfer;
2370         struct usb_xfer *xfer_next;
2371         struct dwc_otg_td *td;
2372         uint16_t temp;
2373         uint16_t slot;
2374
2375         temp = DWC_OTG_READ_4(sc, DOTG_HFNUM) & DWC_OTG_FRAME_MASK;
2376
2377         if (sc->sc_last_frame_num == temp)
2378                 return (0);
2379
2380         sc->sc_last_frame_num = temp;
2381
2382         TAILQ_INIT(&head);
2383
2384         if ((temp & 7) == 0) {
2385
2386                 /* reset the schedule */
2387                 memset(sc->sc_tt_info, 0, sizeof(sc->sc_tt_info));
2388
2389                 TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2390                         td = xfer->td_transfer_cache;
2391                         if (td == NULL || td->ep_type != UE_ISOCHRONOUS)
2392                                 continue;
2393
2394                         /* check for IN direction */
2395                         if ((td->hcchar & HCCHAR_EPDIR_IN) != 0)
2396                                 continue;
2397
2398                         sc->sc_needsof = 1;
2399
2400                         if (td->hcsplt == 0 || td->tt_scheduled != 0)
2401                                 continue;
2402
2403                         /* compute slot */
2404                         slot = dwc_otg_compute_isoc_rx_tt_slot(
2405                             sc->sc_tt_info + td->tt_index);
2406                         if (slot > 3) {
2407                                 /*
2408                                  * Not enough time to get complete
2409                                  * split executed.
2410                                  */
2411                                 continue;
2412                         }
2413                         /* Delayed start */
2414                         td->tt_start_slot = temp + slot;
2415                         td->tt_scheduled = 1;
2416                         TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2417                         TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2418                 }
2419
2420                 TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2421                         td = xfer->td_transfer_cache;
2422                         if (td == NULL || td->ep_type != UE_ISOCHRONOUS)
2423                                 continue;
2424
2425                         /* check for OUT direction */
2426                         if ((td->hcchar & HCCHAR_EPDIR_IN) == 0)
2427                                 continue;
2428
2429                         sc->sc_needsof = 1;
2430
2431                         if (td->hcsplt == 0 || td->tt_scheduled != 0)
2432                                 continue;
2433
2434                         /* Start ASAP */
2435                         td->tt_start_slot = temp;
2436                         td->tt_scheduled = 1;
2437                         TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2438                         TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2439                 }
2440
2441                 TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2442                         td = xfer->td_transfer_cache;
2443                         if (td == NULL || td->ep_type != UE_INTERRUPT)
2444                                 continue;
2445
2446                         if (td->tt_scheduled != 0) {
2447                                 sc->sc_needsof = 1;
2448                                 continue;
2449                         }
2450
2451                         if (dwc_otg_host_rate_check_interrupt(sc, td))
2452                                 continue;
2453
2454                         if (td->hcsplt == 0) {
2455                                 sc->sc_needsof = 1;
2456                                 td->tt_scheduled = 1;
2457                                 continue;
2458                         }
2459
2460                         /* start ASAP */
2461                         td->tt_start_slot = temp;
2462                         sc->sc_needsof = 1;
2463                         td->tt_scheduled = 1;
2464                         TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2465                         TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2466                 }
2467
2468                 TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2469                         td = xfer->td_transfer_cache;
2470                         if (td == NULL ||
2471                             td->ep_type != UE_CONTROL) {
2472                                 continue;
2473                         }
2474
2475                         sc->sc_needsof = 1;
2476
2477                         if (td->hcsplt == 0 || td->tt_scheduled != 0)
2478                                 continue;
2479
2480                         /* start ASAP */
2481                         td->tt_start_slot = temp;
2482                         td->tt_scheduled = 1;
2483                         TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2484                         TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2485                 }
2486         }
2487         if ((temp & 7) < 6) {
2488                 TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2489                         td = xfer->td_transfer_cache;
2490                         if (td == NULL ||
2491                             td->ep_type != UE_BULK) {
2492                                 continue;
2493                         }
2494
2495                         sc->sc_needsof = 1;
2496
2497                         if (td->hcsplt == 0 || td->tt_scheduled != 0)
2498                                 continue;
2499
2500                         /* start ASAP */
2501                         td->tt_start_slot = temp;
2502                         td->tt_scheduled = 1;
2503                         TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2504                         TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2505                 }
2506         }
2507
2508         /* Put TT transfers in execution order at the end */
2509         TAILQ_CONCAT(&sc->sc_bus.intr_q.head, &head, wait_entry);
2510
2511         /* move all TT transfers in front, keeping the current order */
2512         TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2513                 td = xfer->td_transfer_cache;
2514                 if (td == NULL || td->hcsplt == 0)
2515                         continue;
2516                 TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2517                 TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2518         }
2519         TAILQ_CONCAT(&head, &sc->sc_bus.intr_q.head, wait_entry);
2520         TAILQ_CONCAT(&sc->sc_bus.intr_q.head, &head, wait_entry);
2521
2522         /* put non-TT non-ISOCHRONOUS transfers last */
2523         TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2524                 td = xfer->td_transfer_cache;
2525                 if (td == NULL || td->hcsplt != 0 || td->ep_type == UE_ISOCHRONOUS)
2526                         continue;
2527                 TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2528                 TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2529         }
2530         TAILQ_CONCAT(&sc->sc_bus.intr_q.head, &head, wait_entry);
2531
2532         if ((temp & 7) == 0) {
2533
2534                 DPRINTFN(12, "SOF interrupt #%d, needsof=%d\n",
2535                     (int)temp, (int)sc->sc_needsof);
2536
2537                 /* update SOF IRQ mask */
2538                 if (sc->sc_irq_mask & GINTMSK_SOFMSK) {
2539                         if (sc->sc_needsof == 0) {
2540                                 sc->sc_irq_mask &= ~GINTMSK_SOFMSK;
2541                                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2542                         }
2543                 } else {
2544                         if (sc->sc_needsof != 0) {
2545                                 sc->sc_irq_mask |= GINTMSK_SOFMSK;
2546                                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2547                         }
2548                 }
2549
2550                 /* clear need SOF flag */
2551                 sc->sc_needsof = 0;
2552         }
2553         return (1);
2554 }
2555
2556 static void
2557 dwc_otg_interrupt_poll_locked(struct dwc_otg_softc *sc)
2558 {
2559         struct usb_xfer *xfer;
2560         uint32_t count;
2561         uint32_t temp;
2562         uint8_t got_rx_status;
2563         uint8_t x;
2564
2565         if (sc->sc_flags.status_device_mode == 0) {
2566                 /*
2567                  * Update host transfer schedule, so that new
2568                  * transfers can be issued:
2569                  */
2570                 dwc_otg_update_host_transfer_schedule_locked(sc);
2571         }
2572         count = 0;
2573 repeat:
2574         if (++count == 16) {
2575                 /* give other interrupts a chance */
2576                 DPRINTF("Yield\n");
2577                 return;
2578         }
2579         /* get all host channel interrupts */
2580         for (x = 0; x != sc->sc_host_ch_max; x++) {
2581                 temp = DWC_OTG_READ_4(sc, DOTG_HCINT(x));
2582                 if (temp != 0) {
2583                         DWC_OTG_WRITE_4(sc, DOTG_HCINT(x), temp);
2584                         temp &= ~HCINT_SOFTWARE_ONLY;
2585                         sc->sc_chan_state[x].hcint |= temp;
2586                 }
2587         }
2588
2589         if (sc->sc_last_rx_status == 0) {
2590
2591                 temp = DWC_OTG_READ_4(sc, DOTG_GINTSTS);
2592                 if (temp & GINTSTS_RXFLVL) {
2593                         /* pop current status */
2594                         sc->sc_last_rx_status =
2595                             DWC_OTG_READ_4(sc, DOTG_GRXSTSPD);
2596                 }
2597
2598                 if (sc->sc_last_rx_status != 0) {
2599
2600                         uint8_t ep_no;
2601
2602                         temp = sc->sc_last_rx_status &
2603                             GRXSTSRD_PKTSTS_MASK;
2604
2605                         /* non-data messages we simply skip */
2606                         if (temp != GRXSTSRD_STP_DATA &&
2607                             temp != GRXSTSRD_STP_COMPLETE &&
2608                             temp != GRXSTSRD_OUT_DATA) {
2609                                 /* check for halted channel */
2610                                 if (temp == GRXSTSRH_HALTED) {
2611                                         ep_no = GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status);
2612                                         sc->sc_chan_state[ep_no].wait_halted = 0;
2613                                         DPRINTFN(5, "channel halt complete ch=%u\n", ep_no);
2614                                 }
2615                                 dwc_otg_common_rx_ack(sc);
2616                                 goto repeat;
2617                         }
2618
2619                         temp = GRXSTSRD_BCNT_GET(
2620                             sc->sc_last_rx_status);
2621                         ep_no = GRXSTSRD_CHNUM_GET(
2622                             sc->sc_last_rx_status);
2623
2624                         /* receive data, if any */
2625                         if (temp != 0) {
2626                                 DPRINTF("Reading %d bytes from ep %d\n", temp, ep_no);
2627                                 bus_space_read_region_4(sc->sc_io_tag, sc->sc_io_hdl,
2628                                     DOTG_DFIFO(ep_no),
2629                                     sc->sc_rx_bounce_buffer, (temp + 3) / 4);
2630                         }
2631
2632                         /* check if we should dump the data */
2633                         if (!(sc->sc_active_rx_ep & (1U << ep_no))) {
2634                                 dwc_otg_common_rx_ack(sc);
2635                                 goto repeat;
2636                         }
2637
2638                         got_rx_status = 1;
2639
2640                         DPRINTFN(5, "RX status = 0x%08x: ch=%d pid=%d bytes=%d sts=%d\n",
2641                             sc->sc_last_rx_status, ep_no,
2642                             (sc->sc_last_rx_status >> 15) & 3,
2643                             GRXSTSRD_BCNT_GET(sc->sc_last_rx_status),
2644                             (sc->sc_last_rx_status >> 17) & 15);
2645                 } else {
2646                         got_rx_status = 0;
2647                 }
2648         } else {
2649                 uint8_t ep_no;
2650
2651                 ep_no = GRXSTSRD_CHNUM_GET(
2652                     sc->sc_last_rx_status);
2653
2654                 /* check if we should dump the data */
2655                 if (!(sc->sc_active_rx_ep & (1U << ep_no))) {
2656                         dwc_otg_common_rx_ack(sc);
2657                         goto repeat;
2658                 }
2659
2660                 got_rx_status = 1;
2661         }
2662
2663         /* execute FIFOs */
2664         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry)
2665                 dwc_otg_xfer_do_fifo(sc, xfer);
2666
2667         if (got_rx_status) {
2668                 /* check if data was consumed */
2669                 if (sc->sc_last_rx_status == 0)
2670                         goto repeat;
2671
2672                 /* disable RX FIFO level interrupt */
2673                 sc->sc_irq_mask &= ~GINTMSK_RXFLVLMSK;
2674                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2675         }
2676 }
2677
2678 static void
2679 dwc_otg_interrupt_complete_locked(struct dwc_otg_softc *sc)
2680 {
2681         struct usb_xfer *xfer;
2682 repeat:
2683         /* scan for completion events */
2684         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
2685                 if (dwc_otg_xfer_do_complete_locked(sc, xfer))
2686                         goto repeat;
2687         }
2688 }
2689
2690 static void
2691 dwc_otg_vbus_interrupt(struct dwc_otg_softc *sc, uint8_t is_on)
2692 {
2693         DPRINTFN(5, "vbus = %u\n", is_on);
2694
2695         if (is_on) {
2696                 if (!sc->sc_flags.status_vbus) {
2697                         sc->sc_flags.status_vbus = 1;
2698
2699                         /* complete root HUB interrupt endpoint */
2700
2701                         dwc_otg_root_intr(sc);
2702                 }
2703         } else {
2704                 if (sc->sc_flags.status_vbus) {
2705                         sc->sc_flags.status_vbus = 0;
2706                         sc->sc_flags.status_bus_reset = 0;
2707                         sc->sc_flags.status_suspend = 0;
2708                         sc->sc_flags.change_suspend = 0;
2709                         sc->sc_flags.change_connect = 1;
2710
2711                         /* complete root HUB interrupt endpoint */
2712
2713                         dwc_otg_root_intr(sc);
2714                 }
2715         }
2716 }
2717
2718 int
2719 dwc_otg_filter_interrupt(void *arg)
2720 {
2721         struct dwc_otg_softc *sc = arg;
2722         int retval = FILTER_HANDLED;
2723         uint32_t status;
2724
2725         USB_BUS_SPIN_LOCK(&sc->sc_bus);
2726
2727         /* read and clear interrupt status */
2728         status = DWC_OTG_READ_4(sc, DOTG_GINTSTS);
2729
2730         /* clear interrupts we are handling here */
2731         DWC_OTG_WRITE_4(sc, DOTG_GINTSTS, status & ~DWC_OTG_MSK_GINT_THREAD_IRQ);
2732
2733         /* check for USB state change interrupts */
2734         if ((status & DWC_OTG_MSK_GINT_THREAD_IRQ) != 0)
2735                 retval = FILTER_SCHEDULE_THREAD;
2736
2737         /* clear FIFO empty interrupts */
2738         if (status & sc->sc_irq_mask &
2739             (GINTSTS_PTXFEMP | GINTSTS_NPTXFEMP)) {
2740                 sc->sc_irq_mask &= ~(GINTSTS_PTXFEMP | GINTSTS_NPTXFEMP);
2741                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2742         }
2743         /* clear all IN endpoint interrupts */
2744         if (status & GINTSTS_IEPINT) {
2745                 uint32_t temp;
2746                 uint8_t x;
2747
2748                 for (x = 0; x != sc->sc_dev_in_ep_max; x++) {
2749                         temp = DWC_OTG_READ_4(sc, DOTG_DIEPINT(x));
2750                         if (temp & DIEPMSK_XFERCOMPLMSK) {
2751                                 DWC_OTG_WRITE_4(sc, DOTG_DIEPINT(x),
2752                                     DIEPMSK_XFERCOMPLMSK);
2753                         }
2754                 }
2755         }
2756
2757         /* poll FIFOs, if any */
2758         dwc_otg_interrupt_poll_locked(sc);
2759
2760         if (sc->sc_xfer_complete != 0)
2761                 retval = FILTER_SCHEDULE_THREAD;
2762
2763         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
2764
2765         return (retval);
2766 }
2767
2768 void
2769 dwc_otg_interrupt(void *arg)
2770 {
2771         struct dwc_otg_softc *sc = arg;
2772         uint32_t status;
2773
2774         USB_BUS_LOCK(&sc->sc_bus);
2775         USB_BUS_SPIN_LOCK(&sc->sc_bus);
2776
2777         /* read and clear interrupt status */
2778         status = DWC_OTG_READ_4(sc, DOTG_GINTSTS);
2779
2780         /* clear interrupts we are handling here */
2781         DWC_OTG_WRITE_4(sc, DOTG_GINTSTS, status & DWC_OTG_MSK_GINT_THREAD_IRQ);
2782
2783         DPRINTFN(14, "GINTSTS=0x%08x HAINT=0x%08x HFNUM=0x%08x\n",
2784             status, DWC_OTG_READ_4(sc, DOTG_HAINT),
2785             DWC_OTG_READ_4(sc, DOTG_HFNUM));
2786
2787         if (status & GINTSTS_USBRST) {
2788
2789                 /* set correct state */
2790                 sc->sc_flags.status_device_mode = 1;
2791                 sc->sc_flags.status_bus_reset = 0;
2792                 sc->sc_flags.status_suspend = 0;
2793                 sc->sc_flags.change_suspend = 0;
2794                 sc->sc_flags.change_connect = 1;
2795
2796                 /* Disable SOF interrupt */
2797                 sc->sc_irq_mask &= ~GINTMSK_SOFMSK;
2798                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2799
2800                 /* complete root HUB interrupt endpoint */
2801                 dwc_otg_root_intr(sc);
2802         }
2803
2804         /* check for any bus state change interrupts */
2805         if (status & GINTSTS_ENUMDONE) {
2806
2807                 uint32_t temp;
2808
2809                 DPRINTFN(5, "end of reset\n");
2810
2811                 /* set correct state */
2812                 sc->sc_flags.status_device_mode = 1;
2813                 sc->sc_flags.status_bus_reset = 1;
2814                 sc->sc_flags.status_suspend = 0;
2815                 sc->sc_flags.change_suspend = 0;
2816                 sc->sc_flags.change_connect = 1;
2817                 sc->sc_flags.status_low_speed = 0;
2818                 sc->sc_flags.port_enabled = 1;
2819
2820                 /* reset FIFOs */
2821                 (void) dwc_otg_init_fifo(sc, DWC_MODE_DEVICE);
2822
2823                 /* reset function address */
2824                 dwc_otg_set_address(sc, 0);
2825
2826                 /* figure out enumeration speed */
2827                 temp = DWC_OTG_READ_4(sc, DOTG_DSTS);
2828                 if (DSTS_ENUMSPD_GET(temp) == DSTS_ENUMSPD_HI)
2829                         sc->sc_flags.status_high_speed = 1;
2830                 else
2831                         sc->sc_flags.status_high_speed = 0;
2832
2833                 /*
2834                  * Disable resume and SOF interrupt, and enable
2835                  * suspend and RX frame interrupt:
2836                  */
2837                 sc->sc_irq_mask &= ~(GINTMSK_WKUPINTMSK | GINTMSK_SOFMSK);
2838                 sc->sc_irq_mask |= GINTMSK_USBSUSPMSK;
2839                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2840
2841                 /* complete root HUB interrupt endpoint */
2842                 dwc_otg_root_intr(sc);
2843         }
2844
2845         if (status & GINTSTS_PRTINT) {
2846                 uint32_t hprt;
2847
2848                 hprt = DWC_OTG_READ_4(sc, DOTG_HPRT);
2849
2850                 /* clear change bits */
2851                 DWC_OTG_WRITE_4(sc, DOTG_HPRT, (hprt & (
2852                     HPRT_PRTPWR | HPRT_PRTENCHNG |
2853                     HPRT_PRTCONNDET | HPRT_PRTOVRCURRCHNG)) |
2854                     sc->sc_hprt_val);
2855
2856                 DPRINTFN(12, "GINTSTS=0x%08x, HPRT=0x%08x\n", status, hprt);
2857
2858                 sc->sc_flags.status_device_mode = 0;
2859
2860                 if (hprt & HPRT_PRTCONNSTS)
2861                         sc->sc_flags.status_bus_reset = 1;
2862                 else
2863                         sc->sc_flags.status_bus_reset = 0;
2864
2865                 if (hprt & HPRT_PRTENCHNG)
2866                         sc->sc_flags.change_enabled = 1;
2867
2868                 if (hprt & HPRT_PRTENA)
2869                         sc->sc_flags.port_enabled = 1;
2870                 else
2871                         sc->sc_flags.port_enabled = 0;
2872
2873                 if (hprt & HPRT_PRTOVRCURRCHNG)
2874                         sc->sc_flags.change_over_current = 1;
2875
2876                 if (hprt & HPRT_PRTOVRCURRACT)
2877                         sc->sc_flags.port_over_current = 1;
2878                 else
2879                         sc->sc_flags.port_over_current = 0;
2880
2881                 if (hprt & HPRT_PRTPWR)
2882                         sc->sc_flags.port_powered = 1;
2883                 else
2884                         sc->sc_flags.port_powered = 0;
2885
2886                 if (((hprt & HPRT_PRTSPD_MASK)
2887                     >> HPRT_PRTSPD_SHIFT) == HPRT_PRTSPD_LOW)
2888                         sc->sc_flags.status_low_speed = 1;
2889                 else
2890                         sc->sc_flags.status_low_speed = 0;
2891
2892                 if (((hprt & HPRT_PRTSPD_MASK)
2893                     >> HPRT_PRTSPD_SHIFT) == HPRT_PRTSPD_HIGH)
2894                         sc->sc_flags.status_high_speed = 1;
2895                 else
2896                         sc->sc_flags.status_high_speed = 0;
2897
2898                 if (hprt & HPRT_PRTCONNDET)
2899                         sc->sc_flags.change_connect = 1;
2900
2901                 if (hprt & HPRT_PRTSUSP)
2902                         dwc_otg_suspend_irq(sc);
2903                 else
2904                         dwc_otg_resume_irq(sc);
2905
2906                 /* complete root HUB interrupt endpoint */
2907                 dwc_otg_root_intr(sc);
2908
2909                 /* update host frame interval */
2910                 dwc_otg_update_host_frame_interval(sc);
2911         }
2912
2913         /*
2914          * If resume and suspend is set at the same time we interpret
2915          * that like RESUME. Resume is set when there is at least 3
2916          * milliseconds of inactivity on the USB BUS.
2917          */
2918         if (status & GINTSTS_WKUPINT) {
2919
2920                 DPRINTFN(5, "resume interrupt\n");
2921
2922                 dwc_otg_resume_irq(sc);
2923
2924         } else if (status & GINTSTS_USBSUSP) {
2925
2926                 DPRINTFN(5, "suspend interrupt\n");
2927
2928                 dwc_otg_suspend_irq(sc);
2929         }
2930         /* check VBUS */
2931         if (status & (GINTSTS_USBSUSP |
2932             GINTSTS_USBRST |
2933             GINTMSK_OTGINTMSK |
2934             GINTSTS_SESSREQINT)) {
2935                 uint32_t temp;
2936
2937                 temp = DWC_OTG_READ_4(sc, DOTG_GOTGCTL);
2938
2939                 DPRINTFN(5, "GOTGCTL=0x%08x\n", temp);
2940
2941                 dwc_otg_vbus_interrupt(sc,
2942                     (temp & (GOTGCTL_ASESVLD | GOTGCTL_BSESVLD)) ? 1 : 0);
2943         }
2944
2945         if (sc->sc_xfer_complete != 0) {
2946                 sc->sc_xfer_complete = 0;
2947
2948                 /* complete FIFOs, if any */
2949                 dwc_otg_interrupt_complete_locked(sc);
2950         }
2951         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
2952         USB_BUS_UNLOCK(&sc->sc_bus);
2953 }
2954
2955 static void
2956 dwc_otg_setup_standard_chain_sub(struct dwc_otg_std_temp *temp)
2957 {
2958         struct dwc_otg_td *td;
2959
2960         /* get current Transfer Descriptor */
2961         td = temp->td_next;
2962         temp->td = td;
2963
2964         /* prepare for next TD */
2965         temp->td_next = td->obj_next;
2966
2967         /* fill out the Transfer Descriptor */
2968         td->func = temp->func;
2969         td->pc = temp->pc;
2970         td->offset = temp->offset;
2971         td->remainder = temp->len;
2972         td->tx_bytes = 0;
2973         td->error_any = 0;
2974         td->error_stall = 0;
2975         td->npkt = 0;
2976         td->did_stall = temp->did_stall;
2977         td->short_pkt = temp->short_pkt;
2978         td->alt_next = temp->setup_alt_next;
2979         td->set_toggle = 0;
2980         td->got_short = 0;
2981         td->did_nak = 0;
2982         td->channel[0] = DWC_OTG_MAX_CHANNELS;
2983         td->channel[1] = DWC_OTG_MAX_CHANNELS;
2984         td->channel[2] = DWC_OTG_MAX_CHANNELS;
2985         td->state = 0;
2986         td->errcnt = 0;
2987         td->tt_scheduled = 0;
2988         td->tt_xactpos = HCSPLT_XACTPOS_BEGIN;
2989 }
2990
2991 static void
2992 dwc_otg_setup_standard_chain(struct usb_xfer *xfer)
2993 {
2994         struct dwc_otg_std_temp temp;
2995         struct dwc_otg_td *td;
2996         uint32_t x;
2997         uint8_t need_sync;
2998         uint8_t is_host;
2999
3000         DPRINTFN(9, "addr=%d endpt=%d sumlen=%d speed=%d\n",
3001             xfer->address, UE_GET_ADDR(xfer->endpointno),
3002             xfer->sumlen, usbd_get_speed(xfer->xroot->udev));
3003
3004         temp.max_frame_size = xfer->max_frame_size;
3005
3006         td = xfer->td_start[0];
3007         xfer->td_transfer_first = td;
3008         xfer->td_transfer_cache = td;
3009
3010         /* setup temp */
3011
3012         temp.pc = NULL;
3013         temp.td = NULL;
3014         temp.td_next = xfer->td_start[0];
3015         temp.offset = 0;
3016         temp.setup_alt_next = xfer->flags_int.short_frames_ok ||
3017             xfer->flags_int.isochronous_xfr;
3018         temp.did_stall = !xfer->flags_int.control_stall;
3019
3020         is_host = (xfer->xroot->udev->flags.usb_mode == USB_MODE_HOST);
3021
3022         /* check if we should prepend a setup message */
3023
3024         if (xfer->flags_int.control_xfr) {
3025                 if (xfer->flags_int.control_hdr) {
3026
3027                         if (is_host)
3028                                 temp.func = &dwc_otg_host_setup_tx;
3029                         else
3030                                 temp.func = &dwc_otg_setup_rx;
3031
3032                         temp.len = xfer->frlengths[0];
3033                         temp.pc = xfer->frbuffers + 0;
3034                         temp.short_pkt = temp.len ? 1 : 0;
3035
3036                         /* check for last frame */
3037                         if (xfer->nframes == 1) {
3038                                 /* no STATUS stage yet, SETUP is last */
3039                                 if (xfer->flags_int.control_act)
3040                                         temp.setup_alt_next = 0;
3041                         }
3042
3043                         dwc_otg_setup_standard_chain_sub(&temp);
3044                 }
3045                 x = 1;
3046         } else {
3047                 x = 0;
3048         }
3049
3050         if (x != xfer->nframes) {
3051                 if (xfer->endpointno & UE_DIR_IN) {
3052                         if (is_host) {
3053                                 temp.func = &dwc_otg_host_data_rx;
3054                                 need_sync = 0;
3055                         } else {
3056                                 temp.func = &dwc_otg_data_tx;
3057                                 need_sync = 1;
3058                         }
3059                 } else {
3060                         if (is_host) {
3061                                 temp.func = &dwc_otg_host_data_tx;
3062                                 need_sync = 0;
3063                         } else {
3064                                 temp.func = &dwc_otg_data_rx;
3065                                 need_sync = 0;
3066                         }
3067                 }
3068
3069                 /* setup "pc" pointer */
3070                 temp.pc = xfer->frbuffers + x;
3071         } else {
3072                 need_sync = 0;
3073         }
3074         while (x != xfer->nframes) {
3075
3076                 /* DATA0 / DATA1 message */
3077
3078                 temp.len = xfer->frlengths[x];
3079
3080                 x++;
3081
3082                 if (x == xfer->nframes) {
3083                         if (xfer->flags_int.control_xfr) {
3084                                 if (xfer->flags_int.control_act) {
3085                                         temp.setup_alt_next = 0;
3086                                 }
3087                         } else {
3088                                 temp.setup_alt_next = 0;
3089                         }
3090                 }
3091                 if (temp.len == 0) {
3092
3093                         /* make sure that we send an USB packet */
3094
3095                         temp.short_pkt = 0;
3096
3097                 } else {
3098
3099                         /* regular data transfer */
3100
3101                         temp.short_pkt = (xfer->flags.force_short_xfer ? 0 : 1);
3102                 }
3103
3104                 dwc_otg_setup_standard_chain_sub(&temp);
3105
3106                 if (xfer->flags_int.isochronous_xfr) {
3107                         temp.offset += temp.len;
3108                 } else {
3109                         /* get next Page Cache pointer */
3110                         temp.pc = xfer->frbuffers + x;
3111                 }
3112         }
3113
3114         if (xfer->flags_int.control_xfr) {
3115
3116                 /* always setup a valid "pc" pointer for status and sync */
3117                 temp.pc = xfer->frbuffers + 0;
3118                 temp.len = 0;
3119                 temp.short_pkt = 0;
3120                 temp.setup_alt_next = 0;
3121
3122                 /* check if we need to sync */
3123                 if (need_sync) {
3124                         /* we need a SYNC point after TX */
3125                         temp.func = &dwc_otg_data_tx_sync;
3126                         dwc_otg_setup_standard_chain_sub(&temp);
3127                 }
3128
3129                 /* check if we should append a status stage */
3130                 if (!xfer->flags_int.control_act) {
3131
3132                         /*
3133                          * Send a DATA1 message and invert the current
3134                          * endpoint direction.
3135                          */
3136                         if (xfer->endpointno & UE_DIR_IN) {
3137                                 if (is_host) {
3138                                         temp.func = &dwc_otg_host_data_tx;
3139                                         need_sync = 0;
3140                                 } else {
3141                                         temp.func = &dwc_otg_data_rx;
3142                                         need_sync = 0;
3143                                 }
3144                         } else {
3145                                 if (is_host) {
3146                                         temp.func = &dwc_otg_host_data_rx;
3147                                         need_sync = 0;
3148                                 } else {
3149                                         temp.func = &dwc_otg_data_tx;
3150                                         need_sync = 1;
3151                                 }
3152                         }
3153
3154                         dwc_otg_setup_standard_chain_sub(&temp);
3155
3156                         /* data toggle should be DATA1 */
3157                         td = temp.td;
3158                         td->set_toggle = 1;
3159
3160                         if (need_sync) {
3161                                 /* we need a SYNC point after TX */
3162                                 temp.func = &dwc_otg_data_tx_sync;
3163                                 dwc_otg_setup_standard_chain_sub(&temp);
3164                         }
3165                 }
3166         } else {
3167                 /* check if we need to sync */
3168                 if (need_sync) {
3169
3170                         temp.pc = xfer->frbuffers + 0;
3171                         temp.len = 0;
3172                         temp.short_pkt = 0;
3173                         temp.setup_alt_next = 0;
3174
3175                         /* we need a SYNC point after TX */
3176                         temp.func = &dwc_otg_data_tx_sync;
3177                         dwc_otg_setup_standard_chain_sub(&temp);
3178                 }
3179         }
3180
3181         /* must have at least one frame! */
3182         td = temp.td;
3183         xfer->td_transfer_last = td;
3184
3185         if (is_host) {
3186
3187                 struct dwc_otg_softc *sc;
3188                 uint32_t hcchar;
3189                 uint32_t hcsplt;
3190
3191                 sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
3192
3193                 /* get first again */
3194                 td = xfer->td_transfer_first;
3195                 td->toggle = (xfer->endpoint->toggle_next ? 1 : 0);
3196
3197                 hcchar =
3198                         (xfer->address << HCCHAR_DEVADDR_SHIFT) |
3199                         ((xfer->endpointno & UE_ADDR) << HCCHAR_EPNUM_SHIFT) |
3200                         (xfer->max_packet_size << HCCHAR_MPS_SHIFT) |
3201                         HCCHAR_CHENA;
3202
3203                 /*
3204                  * We are not always able to meet the timing
3205                  * requirements of the USB interrupt endpoint's
3206                  * complete split token, when doing transfers going
3207                  * via a transaction translator. Use the CONTROL
3208                  * transfer type instead of the INTERRUPT transfer
3209                  * type in general, as a means to workaround
3210                  * that. This trick should work for both FULL and LOW
3211                  * speed USB traffic going through a TT. For non-TT
3212                  * traffic it works aswell. The reason for using
3213                  * CONTROL type instead of BULK is that some TTs might
3214                  * reject LOW speed BULK traffic.
3215                  */
3216                 if (td->ep_type == UE_INTERRUPT)
3217                         hcchar |= (UE_CONTROL << HCCHAR_EPTYPE_SHIFT);
3218                 else
3219                         hcchar |= (td->ep_type << HCCHAR_EPTYPE_SHIFT);
3220
3221                 if (usbd_get_speed(xfer->xroot->udev) == USB_SPEED_LOW)
3222                         hcchar |= HCCHAR_LSPDDEV;
3223                 if (UE_GET_DIR(xfer->endpointno) == UE_DIR_IN)
3224                         hcchar |= HCCHAR_EPDIR_IN;
3225
3226                 switch (xfer->xroot->udev->speed) {
3227                 case USB_SPEED_FULL:
3228                 case USB_SPEED_LOW:
3229                         /* check if root HUB port is running High Speed */
3230                         if (xfer->xroot->udev->parent_hs_hub != NULL) {
3231                                 hcsplt = HCSPLT_SPLTENA |
3232                                     (xfer->xroot->udev->hs_port_no <<
3233                                     HCSPLT_PRTADDR_SHIFT) |
3234                                     (xfer->xroot->udev->hs_hub_addr <<
3235                                     HCSPLT_HUBADDR_SHIFT);
3236                         } else {
3237                                 hcsplt = 0;
3238                         }
3239                         if (td->ep_type == UE_INTERRUPT) {
3240                                 uint32_t ival;
3241                                 ival = xfer->interval / DWC_OTG_HOST_TIMER_RATE;
3242                                 if (ival == 0)
3243                                         ival = 1;
3244                                 else if (ival > 127)
3245                                         ival = 127;
3246                                 td->tmr_val = sc->sc_tmr_val + ival;
3247                                 td->tmr_res = ival;
3248                         } else if (td->ep_type == UE_ISOCHRONOUS) {
3249                                 td->tmr_res = 1;
3250                                 td->tmr_val = sc->sc_last_frame_num;
3251                                 if (td->hcchar & HCCHAR_EPDIR_IN)
3252                                         td->tmr_val++;
3253                         } else {
3254                                 td->tmr_val = 0;
3255                                 td->tmr_res = (uint8_t)sc->sc_last_frame_num;
3256                         }
3257                         break;
3258                 case USB_SPEED_HIGH:
3259                         hcsplt = 0;
3260                         if (td->ep_type == UE_INTERRUPT) {
3261                                 uint32_t ival;
3262                                 hcchar |= ((xfer->max_packet_count & 3)
3263                                     << HCCHAR_MC_SHIFT);
3264                                 ival = xfer->interval / DWC_OTG_HOST_TIMER_RATE;
3265                                 if (ival == 0)
3266                                         ival = 1;
3267                                 else if (ival > 127)
3268                                         ival = 127;
3269                                 td->tmr_val = sc->sc_tmr_val + ival;
3270                                 td->tmr_res = ival;
3271                         } else if (td->ep_type == UE_ISOCHRONOUS) {
3272                                 hcchar |= ((xfer->max_packet_count & 3)
3273                                     << HCCHAR_MC_SHIFT);
3274                                 td->tmr_res = 1 << usbd_xfer_get_fps_shift(xfer);
3275                                 td->tmr_val = sc->sc_last_frame_num;
3276                                 if (td->hcchar & HCCHAR_EPDIR_IN)
3277                                         td->tmr_val += td->tmr_res;
3278
3279                         } else {
3280                                 td->tmr_val = 0;
3281                                 td->tmr_res = (uint8_t)sc->sc_last_frame_num;
3282                         }
3283                         break;
3284                 default:
3285                         hcsplt = 0;
3286                         td->tmr_val = 0;
3287                         td->tmr_res = 0;
3288                         break;
3289                 }
3290
3291                 /* store configuration in all TD's */
3292                 while (1) {
3293                         td->hcchar = hcchar;
3294                         td->hcsplt = hcsplt;
3295
3296                         if (((void *)td) == xfer->td_transfer_last)
3297                                 break;
3298
3299                         td = td->obj_next;
3300                 }
3301         }
3302 }
3303
3304 static void
3305 dwc_otg_timeout(void *arg)
3306 {
3307         struct usb_xfer *xfer = arg;
3308
3309         DPRINTF("xfer=%p\n", xfer);
3310
3311         USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
3312
3313         /* transfer is transferred */
3314         dwc_otg_device_done(xfer, USB_ERR_TIMEOUT);
3315 }
3316
3317 static void
3318 dwc_otg_start_standard_chain(struct usb_xfer *xfer)
3319 {
3320         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
3321
3322         DPRINTFN(9, "\n");
3323
3324         /*
3325          * Poll one time in device mode, which will turn on the
3326          * endpoint interrupts. Else wait for SOF interrupt in host
3327          * mode.
3328          */
3329         USB_BUS_SPIN_LOCK(&sc->sc_bus);
3330
3331         if (sc->sc_flags.status_device_mode != 0) {
3332                 dwc_otg_xfer_do_fifo(sc, xfer);
3333                 if (dwc_otg_xfer_do_complete_locked(sc, xfer))
3334                         goto done;
3335         } else {
3336                 struct dwc_otg_td *td = xfer->td_transfer_cache;
3337                 if (td->ep_type == UE_ISOCHRONOUS &&
3338                     (td->hcchar & HCCHAR_EPDIR_IN) == 0) {
3339                         /*
3340                          * Need to start ISOCHRONOUS OUT transfer ASAP
3341                          * because execution is delayed by one 125us
3342                          * microframe:
3343                          */
3344                         dwc_otg_xfer_do_fifo(sc, xfer);
3345                         if (dwc_otg_xfer_do_complete_locked(sc, xfer))
3346                                 goto done;
3347                 }
3348         }
3349
3350         /* put transfer on interrupt queue */
3351         usbd_transfer_enqueue(&xfer->xroot->bus->intr_q, xfer);
3352
3353         /* start timeout, if any */
3354         if (xfer->timeout != 0) {
3355                 usbd_transfer_timeout_ms(xfer,
3356                     &dwc_otg_timeout, xfer->timeout);
3357         }
3358
3359         if (sc->sc_flags.status_device_mode != 0)
3360                 goto done;
3361
3362         /* enable SOF interrupt, if any */
3363         dwc_otg_enable_sof_irq(sc);
3364 done:
3365         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3366 }
3367
3368 static void
3369 dwc_otg_root_intr(struct dwc_otg_softc *sc)
3370 {
3371         DPRINTFN(9, "\n");
3372
3373         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
3374
3375         /* set port bit */
3376         sc->sc_hub_idata[0] = 0x02;     /* we only have one port */
3377
3378         uhub_root_intr(&sc->sc_bus, sc->sc_hub_idata,
3379             sizeof(sc->sc_hub_idata));
3380 }
3381
3382 static usb_error_t
3383 dwc_otg_standard_done_sub(struct usb_xfer *xfer)
3384 {
3385         struct dwc_otg_td *td;
3386         uint32_t len;
3387         usb_error_t error;
3388
3389         DPRINTFN(9, "\n");
3390
3391         td = xfer->td_transfer_cache;
3392
3393         do {
3394                 len = td->remainder;
3395
3396                 /* store last data toggle */
3397                 xfer->endpoint->toggle_next = td->toggle;
3398
3399                 if (xfer->aframes != xfer->nframes) {
3400                         /*
3401                          * Verify the length and subtract
3402                          * the remainder from "frlengths[]":
3403                          */
3404                         if (len > xfer->frlengths[xfer->aframes]) {
3405                                 td->error_any = 1;
3406                         } else {
3407                                 xfer->frlengths[xfer->aframes] -= len;
3408                         }
3409                 }
3410                 /* Check for transfer error */
3411                 if (td->error_any) {
3412                         /* the transfer is finished */
3413                         error = (td->error_stall ?
3414                             USB_ERR_STALLED : USB_ERR_IOERROR);
3415                         td = NULL;
3416                         break;
3417                 }
3418                 /* Check for short transfer */
3419                 if (len > 0) {
3420                         if (xfer->flags_int.short_frames_ok ||
3421                             xfer->flags_int.isochronous_xfr) {
3422                                 /* follow alt next */
3423                                 if (td->alt_next) {
3424                                         td = td->obj_next;
3425                                 } else {
3426                                         td = NULL;
3427                                 }
3428                         } else {
3429                                 /* the transfer is finished */
3430                                 td = NULL;
3431                         }
3432                         error = 0;
3433                         break;
3434                 }
3435                 td = td->obj_next;
3436
3437                 /* this USB frame is complete */
3438                 error = 0;
3439                 break;
3440
3441         } while (0);
3442
3443         /* update transfer cache */
3444
3445         xfer->td_transfer_cache = td;
3446
3447         return (error);
3448 }
3449
3450 static void
3451 dwc_otg_standard_done(struct usb_xfer *xfer)
3452 {
3453         usb_error_t err = 0;
3454
3455         DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
3456             xfer, xfer->endpoint);
3457
3458         /* reset scanner */
3459
3460         xfer->td_transfer_cache = xfer->td_transfer_first;
3461
3462         if (xfer->flags_int.control_xfr) {
3463
3464                 if (xfer->flags_int.control_hdr) {
3465
3466                         err = dwc_otg_standard_done_sub(xfer);
3467                 }
3468                 xfer->aframes = 1;
3469
3470                 if (xfer->td_transfer_cache == NULL) {
3471                         goto done;
3472                 }
3473         }
3474         while (xfer->aframes != xfer->nframes) {
3475
3476                 err = dwc_otg_standard_done_sub(xfer);
3477                 xfer->aframes++;
3478
3479                 if (xfer->td_transfer_cache == NULL) {
3480                         goto done;
3481                 }
3482         }
3483
3484         if (xfer->flags_int.control_xfr &&
3485             !xfer->flags_int.control_act) {
3486
3487                 err = dwc_otg_standard_done_sub(xfer);
3488         }
3489 done:
3490         dwc_otg_device_done(xfer, err);
3491 }
3492
3493 /*------------------------------------------------------------------------*
3494  *      dwc_otg_device_done
3495  *
3496  * NOTE: this function can be called more than one time on the
3497  * same USB transfer!
3498  *------------------------------------------------------------------------*/
3499 static void
3500 dwc_otg_device_done(struct usb_xfer *xfer, usb_error_t error)
3501 {
3502         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
3503
3504         DPRINTFN(9, "xfer=%p, endpoint=%p, error=%d\n",
3505             xfer, xfer->endpoint, error);
3506
3507         USB_BUS_SPIN_LOCK(&sc->sc_bus);
3508
3509         if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) {
3510                 /* Interrupts are cleared by the interrupt handler */
3511         } else {
3512                 struct dwc_otg_td *td;
3513
3514                 td = xfer->td_transfer_cache;
3515                 if (td != NULL)
3516                         dwc_otg_host_channel_free(sc, td);
3517         }
3518         /* dequeue transfer and start next transfer */
3519         usbd_transfer_done(xfer, error);
3520
3521         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3522 }
3523
3524 static void
3525 dwc_otg_xfer_stall(struct usb_xfer *xfer)
3526 {
3527         dwc_otg_device_done(xfer, USB_ERR_STALLED);
3528 }
3529
3530 static void
3531 dwc_otg_set_stall(struct usb_device *udev,
3532     struct usb_endpoint *ep, uint8_t *did_stall)
3533 {
3534         struct dwc_otg_softc *sc;
3535         uint32_t temp;
3536         uint32_t reg;
3537         uint8_t ep_no;
3538
3539         USB_BUS_LOCK_ASSERT(udev->bus, MA_OWNED);
3540
3541         /* check mode */
3542         if (udev->flags.usb_mode != USB_MODE_DEVICE) {
3543                 /* not supported */
3544                 return;
3545         }
3546
3547         sc = DWC_OTG_BUS2SC(udev->bus);
3548
3549         USB_BUS_SPIN_LOCK(&sc->sc_bus);
3550
3551         /* get endpoint address */
3552         ep_no = ep->edesc->bEndpointAddress;
3553
3554         DPRINTFN(5, "endpoint=0x%x\n", ep_no);
3555
3556         if (ep_no & UE_DIR_IN) {
3557                 reg = DOTG_DIEPCTL(ep_no & UE_ADDR);
3558                 temp = sc->sc_in_ctl[ep_no & UE_ADDR];
3559         } else {
3560                 reg = DOTG_DOEPCTL(ep_no & UE_ADDR);
3561                 temp = sc->sc_out_ctl[ep_no & UE_ADDR];
3562         }
3563
3564         /* disable and stall endpoint */
3565         DWC_OTG_WRITE_4(sc, reg, temp | DOEPCTL_EPDIS);
3566         DWC_OTG_WRITE_4(sc, reg, temp | DOEPCTL_STALL);
3567
3568         /* clear active OUT ep */
3569         if (!(ep_no & UE_DIR_IN)) {
3570
3571                 sc->sc_active_rx_ep &= ~(1U << (ep_no & UE_ADDR));
3572
3573                 if (sc->sc_last_rx_status != 0 &&
3574                     (ep_no & UE_ADDR) == GRXSTSRD_CHNUM_GET(
3575                     sc->sc_last_rx_status)) {
3576                         /* dump data */
3577                         dwc_otg_common_rx_ack(sc);
3578                         /* poll interrupt */
3579                         dwc_otg_interrupt_poll_locked(sc);
3580                         dwc_otg_interrupt_complete_locked(sc);
3581                 }
3582         }
3583         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3584 }
3585
3586 static void
3587 dwc_otg_clear_stall_sub_locked(struct dwc_otg_softc *sc, uint32_t mps,
3588     uint8_t ep_no, uint8_t ep_type, uint8_t ep_dir)
3589 {
3590         uint32_t reg;
3591         uint32_t temp;
3592
3593         if (ep_type == UE_CONTROL) {
3594                 /* clearing stall is not needed */
3595                 return;
3596         }
3597
3598         if (ep_dir) {
3599                 reg = DOTG_DIEPCTL(ep_no);
3600         } else {
3601                 reg = DOTG_DOEPCTL(ep_no);
3602                 sc->sc_active_rx_ep |= (1U << ep_no);
3603         }
3604
3605         /* round up and mask away the multiplier count */
3606         mps = (mps + 3) & 0x7FC;
3607
3608         if (ep_type == UE_BULK) {
3609                 temp = DIEPCTL_EPTYPE_SET(
3610                     DIEPCTL_EPTYPE_BULK) |
3611                     DIEPCTL_USBACTEP;
3612         } else if (ep_type == UE_INTERRUPT) {
3613                 temp = DIEPCTL_EPTYPE_SET(
3614                     DIEPCTL_EPTYPE_INTERRUPT) |
3615                     DIEPCTL_USBACTEP;
3616         } else {
3617                 temp = DIEPCTL_EPTYPE_SET(
3618                     DIEPCTL_EPTYPE_ISOC) |
3619                     DIEPCTL_USBACTEP;
3620         }
3621
3622         temp |= DIEPCTL_MPS_SET(mps);
3623         temp |= DIEPCTL_TXFNUM_SET(ep_no);
3624
3625         if (ep_dir)
3626                 sc->sc_in_ctl[ep_no] = temp;
3627         else
3628                 sc->sc_out_ctl[ep_no] = temp;
3629
3630         DWC_OTG_WRITE_4(sc, reg, temp | DOEPCTL_EPDIS);
3631         DWC_OTG_WRITE_4(sc, reg, temp | DOEPCTL_SETD0PID);
3632         DWC_OTG_WRITE_4(sc, reg, temp | DIEPCTL_SNAK);
3633
3634         /* we only reset the transmit FIFO */
3635         if (ep_dir) {
3636                 dwc_otg_tx_fifo_reset(sc,
3637                     GRSTCTL_TXFIFO(ep_no) |
3638                     GRSTCTL_TXFFLSH);
3639
3640                 DWC_OTG_WRITE_4(sc,
3641                     DOTG_DIEPTSIZ(ep_no), 0);
3642         }
3643
3644         /* poll interrupt */
3645         dwc_otg_interrupt_poll_locked(sc);
3646         dwc_otg_interrupt_complete_locked(sc);
3647 }
3648
3649 static void
3650 dwc_otg_clear_stall(struct usb_device *udev, struct usb_endpoint *ep)
3651 {
3652         struct dwc_otg_softc *sc;
3653         struct usb_endpoint_descriptor *ed;
3654
3655         DPRINTFN(5, "endpoint=%p\n", ep);
3656
3657         USB_BUS_LOCK_ASSERT(udev->bus, MA_OWNED);
3658
3659         /* check mode */
3660         if (udev->flags.usb_mode != USB_MODE_DEVICE) {
3661                 /* not supported */
3662                 return;
3663         }
3664         /* get softc */
3665         sc = DWC_OTG_BUS2SC(udev->bus);
3666
3667         USB_BUS_SPIN_LOCK(&sc->sc_bus);
3668
3669         /* get endpoint descriptor */
3670         ed = ep->edesc;
3671
3672         /* reset endpoint */
3673         dwc_otg_clear_stall_sub_locked(sc,
3674             UGETW(ed->wMaxPacketSize),
3675             (ed->bEndpointAddress & UE_ADDR),
3676             (ed->bmAttributes & UE_XFERTYPE),
3677             (ed->bEndpointAddress & (UE_DIR_IN | UE_DIR_OUT)));
3678
3679         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3680 }
3681
3682 static void
3683 dwc_otg_device_state_change(struct usb_device *udev)
3684 {
3685         struct dwc_otg_softc *sc;
3686         uint8_t x;
3687
3688         /* check mode */
3689         if (udev->flags.usb_mode != USB_MODE_DEVICE) {
3690                 /* not supported */
3691                 return;
3692         }
3693
3694         /* get softc */
3695         sc = DWC_OTG_BUS2SC(udev->bus);
3696
3697         /* deactivate all other endpoint but the control endpoint */
3698         if (udev->state == USB_STATE_CONFIGURED ||
3699             udev->state == USB_STATE_ADDRESSED) {
3700
3701                 USB_BUS_LOCK(&sc->sc_bus);
3702
3703                 for (x = 1; x != sc->sc_dev_ep_max; x++) {
3704
3705                         if (x < sc->sc_dev_in_ep_max) {
3706                                 DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(x),
3707                                     DIEPCTL_EPDIS);
3708                                 DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(x), 0);
3709                         }
3710
3711                         DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(x),
3712                             DOEPCTL_EPDIS);
3713                         DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(x), 0);
3714                 }
3715                 USB_BUS_UNLOCK(&sc->sc_bus);
3716         }
3717 }
3718
3719 int
3720 dwc_otg_init(struct dwc_otg_softc *sc)
3721 {
3722         uint32_t temp;
3723
3724         DPRINTF("start\n");
3725
3726         /* set up the bus structure */
3727         sc->sc_bus.usbrev = USB_REV_2_0;
3728         sc->sc_bus.methods = &dwc_otg_bus_methods;
3729
3730         usb_callout_init_mtx(&sc->sc_timer,
3731             &sc->sc_bus.bus_mtx, 0);
3732
3733         USB_BUS_LOCK(&sc->sc_bus);
3734
3735         /* turn on clocks */
3736         dwc_otg_clocks_on(sc);
3737
3738         temp = DWC_OTG_READ_4(sc, DOTG_GSNPSID);
3739         DPRINTF("Version = 0x%08x\n", temp);
3740
3741         /* disconnect */
3742         DWC_OTG_WRITE_4(sc, DOTG_DCTL,
3743             DCTL_SFTDISCON);
3744
3745         /* wait for host to detect disconnect */
3746         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 32);
3747
3748         DWC_OTG_WRITE_4(sc, DOTG_GRSTCTL,
3749             GRSTCTL_CSFTRST);
3750
3751         /* wait a little bit for block to reset */
3752         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 128);
3753
3754         switch (sc->sc_mode) {
3755         case DWC_MODE_DEVICE:
3756                 temp = GUSBCFG_FORCEDEVMODE;
3757                 break;
3758         case DWC_MODE_HOST:
3759                 temp = GUSBCFG_FORCEHOSTMODE;
3760                 break;
3761         default:
3762                 temp = 0;
3763                 break;
3764         }
3765
3766         /* select HSIC, ULPI or internal PHY mode */
3767         switch (dwc_otg_phy_type) {
3768         case DWC_OTG_PHY_HSIC:
3769                 DWC_OTG_WRITE_4(sc, DOTG_GUSBCFG,
3770                     GUSBCFG_PHYIF |
3771                     GUSBCFG_TRD_TIM_SET(5) | temp);
3772                 DWC_OTG_WRITE_4(sc, DOTG_GOTGCTL,
3773                     0x000000EC);
3774
3775                 temp = DWC_OTG_READ_4(sc, DOTG_GLPMCFG);
3776                 DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3777                     temp & ~GLPMCFG_HSIC_CONN);
3778                 DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3779                     temp | GLPMCFG_HSIC_CONN);
3780                 break;
3781         case DWC_OTG_PHY_ULPI:
3782                 DWC_OTG_WRITE_4(sc, DOTG_GUSBCFG,
3783                     GUSBCFG_ULPI_UTMI_SEL |
3784                     GUSBCFG_TRD_TIM_SET(5) | temp);
3785                 DWC_OTG_WRITE_4(sc, DOTG_GOTGCTL, 0);
3786
3787                 temp = DWC_OTG_READ_4(sc, DOTG_GLPMCFG);
3788                 DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3789                     temp & ~GLPMCFG_HSIC_CONN);
3790                 break;
3791         case DWC_OTG_PHY_INTERNAL:
3792                 DWC_OTG_WRITE_4(sc, DOTG_GUSBCFG,
3793                     GUSBCFG_PHYSEL |
3794                     GUSBCFG_TRD_TIM_SET(5) | temp);
3795                 DWC_OTG_WRITE_4(sc, DOTG_GOTGCTL, 0);
3796
3797                 temp = DWC_OTG_READ_4(sc, DOTG_GLPMCFG);
3798                 DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3799                     temp & ~GLPMCFG_HSIC_CONN);
3800
3801                 temp = DWC_OTG_READ_4(sc, DOTG_GGPIO);
3802                 temp &= ~(DOTG_GGPIO_NOVBUSSENS | DOTG_GGPIO_I2CPADEN);
3803                 temp |= (DOTG_GGPIO_VBUSASEN | DOTG_GGPIO_VBUSBSEN |
3804                     DOTG_GGPIO_PWRDWN);
3805                 DWC_OTG_WRITE_4(sc, DOTG_GGPIO, temp);
3806                 break;
3807         default:
3808                 break;
3809         }
3810
3811         /* clear global nak */
3812         DWC_OTG_WRITE_4(sc, DOTG_DCTL,
3813             DCTL_CGOUTNAK |
3814             DCTL_CGNPINNAK);
3815
3816         /* disable USB port */
3817         DWC_OTG_WRITE_4(sc, DOTG_PCGCCTL, 0xFFFFFFFF);
3818
3819         /* wait 10ms */
3820         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 100);
3821
3822         /* enable USB port */
3823         DWC_OTG_WRITE_4(sc, DOTG_PCGCCTL, 0);
3824
3825         /* wait 10ms */
3826         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 100);
3827
3828         temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG3);
3829
3830         sc->sc_fifo_size = 4 * GHWCFG3_DFIFODEPTH_GET(temp);
3831
3832         temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG2);
3833
3834         sc->sc_dev_ep_max = GHWCFG2_NUMDEVEPS_GET(temp);
3835
3836         if (sc->sc_dev_ep_max > DWC_OTG_MAX_ENDPOINTS)
3837                 sc->sc_dev_ep_max = DWC_OTG_MAX_ENDPOINTS;
3838
3839         sc->sc_host_ch_max = GHWCFG2_NUMHSTCHNL_GET(temp);
3840
3841         if (sc->sc_host_ch_max > DWC_OTG_MAX_CHANNELS)
3842                 sc->sc_host_ch_max = DWC_OTG_MAX_CHANNELS;
3843
3844         temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG4);
3845
3846         sc->sc_dev_in_ep_max = GHWCFG4_NUM_IN_EP_GET(temp);
3847
3848         DPRINTF("Total FIFO size = %d bytes, Device EPs = %d/%d Host CHs = %d\n",
3849             sc->sc_fifo_size, sc->sc_dev_ep_max, sc->sc_dev_in_ep_max,
3850             sc->sc_host_ch_max);
3851
3852         /* setup FIFO */
3853         if (dwc_otg_init_fifo(sc, DWC_MODE_OTG)) {
3854                 USB_BUS_UNLOCK(&sc->sc_bus);
3855                 return (EINVAL);
3856         }
3857
3858         /* enable interrupts */
3859         sc->sc_irq_mask = DWC_OTG_MSK_GINT_ENABLED;
3860         DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
3861
3862         if (sc->sc_mode == DWC_MODE_OTG || sc->sc_mode == DWC_MODE_DEVICE) {
3863
3864                 /* enable all endpoint interrupts */
3865                 temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG2);
3866                 if (temp & GHWCFG2_MPI) {
3867                         uint8_t x;
3868
3869                         DPRINTF("Disable Multi Process Interrupts\n");
3870
3871                         for (x = 0; x != sc->sc_dev_in_ep_max; x++) {
3872                                 DWC_OTG_WRITE_4(sc, DOTG_DIEPEACHINTMSK(x), 0);
3873                                 DWC_OTG_WRITE_4(sc, DOTG_DOEPEACHINTMSK(x), 0);
3874                         }
3875                         DWC_OTG_WRITE_4(sc, DOTG_DEACHINTMSK, 0);
3876                 }
3877                 DWC_OTG_WRITE_4(sc, DOTG_DIEPMSK,
3878                     DIEPMSK_XFERCOMPLMSK);
3879                 DWC_OTG_WRITE_4(sc, DOTG_DOEPMSK, 0);
3880                 DWC_OTG_WRITE_4(sc, DOTG_DAINTMSK, 0xFFFF);
3881         }
3882
3883         if (sc->sc_mode == DWC_MODE_OTG || sc->sc_mode == DWC_MODE_HOST) {
3884                 /* setup clocks */
3885                 temp = DWC_OTG_READ_4(sc, DOTG_HCFG);
3886                 temp &= ~(HCFG_FSLSSUPP | HCFG_FSLSPCLKSEL_MASK);
3887                 temp |= (1 << HCFG_FSLSPCLKSEL_SHIFT);
3888                 DWC_OTG_WRITE_4(sc, DOTG_HCFG, temp);
3889         }
3890
3891         /* only enable global IRQ */
3892         DWC_OTG_WRITE_4(sc, DOTG_GAHBCFG,
3893             GAHBCFG_GLBLINTRMSK);
3894
3895         /* turn off clocks */
3896         dwc_otg_clocks_off(sc);
3897
3898         /* read initial VBUS state */
3899
3900         temp = DWC_OTG_READ_4(sc, DOTG_GOTGCTL);
3901
3902         DPRINTFN(5, "GOTCTL=0x%08x\n", temp);
3903
3904         dwc_otg_vbus_interrupt(sc,
3905             (temp & (GOTGCTL_ASESVLD | GOTGCTL_BSESVLD)) ? 1 : 0);
3906
3907         USB_BUS_UNLOCK(&sc->sc_bus);
3908
3909         /* catch any lost interrupts */
3910
3911         dwc_otg_do_poll(&sc->sc_bus);
3912
3913         return (0);                     /* success */
3914 }
3915
3916 void
3917 dwc_otg_uninit(struct dwc_otg_softc *sc)
3918 {
3919         USB_BUS_LOCK(&sc->sc_bus);
3920
3921         /* stop host timer */
3922         dwc_otg_timer_stop(sc);
3923
3924         /* set disconnect */
3925         DWC_OTG_WRITE_4(sc, DOTG_DCTL,
3926             DCTL_SFTDISCON);
3927
3928         /* turn off global IRQ */
3929         DWC_OTG_WRITE_4(sc, DOTG_GAHBCFG, 0);
3930
3931         sc->sc_flags.port_enabled = 0;
3932         sc->sc_flags.port_powered = 0;
3933         sc->sc_flags.status_vbus = 0;
3934         sc->sc_flags.status_bus_reset = 0;
3935         sc->sc_flags.status_suspend = 0;
3936         sc->sc_flags.change_suspend = 0;
3937         sc->sc_flags.change_connect = 1;
3938
3939         dwc_otg_pull_down(sc);
3940         dwc_otg_clocks_off(sc);
3941
3942         USB_BUS_UNLOCK(&sc->sc_bus);
3943
3944         usb_callout_drain(&sc->sc_timer);
3945 }
3946
3947 static void
3948 dwc_otg_suspend(struct dwc_otg_softc *sc)
3949 {
3950         return;
3951 }
3952
3953 static void
3954 dwc_otg_resume(struct dwc_otg_softc *sc)
3955 {
3956         return;
3957 }
3958
3959 static void
3960 dwc_otg_do_poll(struct usb_bus *bus)
3961 {
3962         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(bus);
3963
3964         USB_BUS_LOCK(&sc->sc_bus);
3965         USB_BUS_SPIN_LOCK(&sc->sc_bus);
3966         dwc_otg_interrupt_poll_locked(sc);
3967         dwc_otg_interrupt_complete_locked(sc);
3968         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3969         USB_BUS_UNLOCK(&sc->sc_bus);
3970 }
3971
3972 /*------------------------------------------------------------------------*
3973  * DWC OTG bulk support
3974  * DWC OTG control support
3975  * DWC OTG interrupt support
3976  *------------------------------------------------------------------------*/
3977 static void
3978 dwc_otg_device_non_isoc_open(struct usb_xfer *xfer)
3979 {
3980 }
3981
3982 static void
3983 dwc_otg_device_non_isoc_close(struct usb_xfer *xfer)
3984 {
3985         dwc_otg_device_done(xfer, USB_ERR_CANCELLED);
3986 }
3987
3988 static void
3989 dwc_otg_device_non_isoc_enter(struct usb_xfer *xfer)
3990 {
3991 }
3992
3993 static void
3994 dwc_otg_device_non_isoc_start(struct usb_xfer *xfer)
3995 {
3996         /* setup TDs */
3997         dwc_otg_setup_standard_chain(xfer);
3998         dwc_otg_start_standard_chain(xfer);
3999 }
4000
4001 struct usb_pipe_methods dwc_otg_device_non_isoc_methods =
4002 {
4003         .open = dwc_otg_device_non_isoc_open,
4004         .close = dwc_otg_device_non_isoc_close,
4005         .enter = dwc_otg_device_non_isoc_enter,
4006         .start = dwc_otg_device_non_isoc_start,
4007 };
4008
4009 /*------------------------------------------------------------------------*
4010  * DWC OTG full speed isochronous support
4011  *------------------------------------------------------------------------*/
4012 static void
4013 dwc_otg_device_isoc_open(struct usb_xfer *xfer)
4014 {
4015 }
4016
4017 static void
4018 dwc_otg_device_isoc_close(struct usb_xfer *xfer)
4019 {
4020         dwc_otg_device_done(xfer, USB_ERR_CANCELLED);
4021 }
4022
4023 static void
4024 dwc_otg_device_isoc_enter(struct usb_xfer *xfer)
4025 {
4026 }
4027
4028 static void
4029 dwc_otg_device_isoc_start(struct usb_xfer *xfer)
4030 {
4031         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
4032         uint32_t temp;
4033         uint32_t msframes;
4034         uint32_t framenum;
4035         uint8_t shift = usbd_xfer_get_fps_shift(xfer);
4036
4037         DPRINTFN(6, "xfer=%p next=%d nframes=%d\n",
4038             xfer, xfer->endpoint->isoc_next, xfer->nframes);
4039
4040         if (xfer->xroot->udev->flags.usb_mode == USB_MODE_HOST) {
4041                 temp = DWC_OTG_READ_4(sc, DOTG_HFNUM);
4042
4043                 /* get the current frame index */
4044                 framenum = (temp & HFNUM_FRNUM_MASK);
4045         } else {
4046                 temp = DWC_OTG_READ_4(sc, DOTG_DSTS);
4047
4048                 /* get the current frame index */
4049                 framenum = DSTS_SOFFN_GET(temp);
4050         }
4051
4052         if (xfer->xroot->udev->parent_hs_hub != NULL)
4053                 framenum /= 8;
4054
4055         framenum &= DWC_OTG_FRAME_MASK;
4056
4057         /*
4058          * Compute number of milliseconds worth of data traffic for
4059          * this USB transfer:
4060          */
4061         if (xfer->xroot->udev->speed == USB_SPEED_HIGH)
4062                 msframes = ((xfer->nframes << shift) + 7) / 8;
4063         else
4064                 msframes = xfer->nframes;
4065
4066         /*
4067          * check if the frame index is within the window where the frames
4068          * will be inserted
4069          */
4070         temp = (framenum - xfer->endpoint->isoc_next) & DWC_OTG_FRAME_MASK;
4071
4072         if ((xfer->endpoint->is_synced == 0) || (temp < msframes)) {
4073                 /*
4074                  * If there is data underflow or the pipe queue is
4075                  * empty we schedule the transfer a few frames ahead
4076                  * of the current frame position. Else two isochronous
4077                  * transfers might overlap.
4078                  */
4079                 xfer->endpoint->isoc_next = (framenum + 3) & DWC_OTG_FRAME_MASK;
4080                 xfer->endpoint->is_synced = 1;
4081                 DPRINTFN(3, "start next=%d\n", xfer->endpoint->isoc_next);
4082         }
4083         /*
4084          * compute how many milliseconds the insertion is ahead of the
4085          * current frame position:
4086          */
4087         temp = (xfer->endpoint->isoc_next - framenum) & DWC_OTG_FRAME_MASK;
4088
4089         /*
4090          * pre-compute when the isochronous transfer will be finished:
4091          */
4092         xfer->isoc_time_complete =
4093                 usb_isoc_time_expand(&sc->sc_bus, framenum) + temp + msframes;
4094
4095         /* setup TDs */
4096         dwc_otg_setup_standard_chain(xfer);
4097
4098         /* compute frame number for next insertion */
4099         xfer->endpoint->isoc_next += msframes;
4100
4101         /* start TD chain */
4102         dwc_otg_start_standard_chain(xfer);
4103 }
4104
4105 struct usb_pipe_methods dwc_otg_device_isoc_methods =
4106 {
4107         .open = dwc_otg_device_isoc_open,
4108         .close = dwc_otg_device_isoc_close,
4109         .enter = dwc_otg_device_isoc_enter,
4110         .start = dwc_otg_device_isoc_start,
4111 };
4112
4113 /*------------------------------------------------------------------------*
4114  * DWC OTG root control support
4115  *------------------------------------------------------------------------*
4116  * Simulate a hardware HUB by handling all the necessary requests.
4117  *------------------------------------------------------------------------*/
4118
4119 static const struct usb_device_descriptor dwc_otg_devd = {
4120         .bLength = sizeof(struct usb_device_descriptor),
4121         .bDescriptorType = UDESC_DEVICE,
4122         .bcdUSB = {0x00, 0x02},
4123         .bDeviceClass = UDCLASS_HUB,
4124         .bDeviceSubClass = UDSUBCLASS_HUB,
4125         .bDeviceProtocol = UDPROTO_HSHUBSTT,
4126         .bMaxPacketSize = 64,
4127         .bcdDevice = {0x00, 0x01},
4128         .iManufacturer = 1,
4129         .iProduct = 2,
4130         .bNumConfigurations = 1,
4131 };
4132
4133 static const struct dwc_otg_config_desc dwc_otg_confd = {
4134         .confd = {
4135                 .bLength = sizeof(struct usb_config_descriptor),
4136                 .bDescriptorType = UDESC_CONFIG,
4137                 .wTotalLength[0] = sizeof(dwc_otg_confd),
4138                 .bNumInterface = 1,
4139                 .bConfigurationValue = 1,
4140                 .iConfiguration = 0,
4141                 .bmAttributes = UC_SELF_POWERED,
4142                 .bMaxPower = 0,
4143         },
4144         .ifcd = {
4145                 .bLength = sizeof(struct usb_interface_descriptor),
4146                 .bDescriptorType = UDESC_INTERFACE,
4147                 .bNumEndpoints = 1,
4148                 .bInterfaceClass = UICLASS_HUB,
4149                 .bInterfaceSubClass = UISUBCLASS_HUB,
4150                 .bInterfaceProtocol = 0,
4151         },
4152         .endpd = {
4153                 .bLength = sizeof(struct usb_endpoint_descriptor),
4154                 .bDescriptorType = UDESC_ENDPOINT,
4155                 .bEndpointAddress = (UE_DIR_IN | DWC_OTG_INTR_ENDPT),
4156                 .bmAttributes = UE_INTERRUPT,
4157                 .wMaxPacketSize[0] = 8,
4158                 .bInterval = 255,
4159         },
4160 };
4161
4162 #define HSETW(ptr, val) ptr = { (uint8_t)(val), (uint8_t)((val) >> 8) }
4163
4164 static const struct usb_hub_descriptor_min dwc_otg_hubd = {
4165         .bDescLength = sizeof(dwc_otg_hubd),
4166         .bDescriptorType = UDESC_HUB,
4167         .bNbrPorts = 1,
4168         HSETW(.wHubCharacteristics, (UHD_PWR_NO_SWITCH | UHD_OC_INDIVIDUAL)),
4169         .bPwrOn2PwrGood = 50,
4170         .bHubContrCurrent = 0,
4171         .DeviceRemovable = {0},         /* port is removable */
4172 };
4173
4174 #define STRING_VENDOR \
4175   "D\0W\0C\0O\0T\0G"
4176
4177 #define STRING_PRODUCT \
4178   "O\0T\0G\0 \0R\0o\0o\0t\0 \0H\0U\0B"
4179
4180 USB_MAKE_STRING_DESC(STRING_VENDOR, dwc_otg_vendor);
4181 USB_MAKE_STRING_DESC(STRING_PRODUCT, dwc_otg_product);
4182
4183 static usb_error_t
4184 dwc_otg_roothub_exec(struct usb_device *udev,
4185     struct usb_device_request *req, const void **pptr, uint16_t *plength)
4186 {
4187         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(udev->bus);
4188         const void *ptr;
4189         uint16_t len;
4190         uint16_t value;
4191         uint16_t index;
4192         usb_error_t err;
4193
4194         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
4195
4196         /* buffer reset */
4197         ptr = (const void *)&sc->sc_hub_temp;
4198         len = 0;
4199         err = 0;
4200
4201         value = UGETW(req->wValue);
4202         index = UGETW(req->wIndex);
4203
4204         /* demultiplex the control request */
4205
4206         switch (req->bmRequestType) {
4207         case UT_READ_DEVICE:
4208                 switch (req->bRequest) {
4209                 case UR_GET_DESCRIPTOR:
4210                         goto tr_handle_get_descriptor;
4211                 case UR_GET_CONFIG:
4212                         goto tr_handle_get_config;
4213                 case UR_GET_STATUS:
4214                         goto tr_handle_get_status;
4215                 default:
4216                         goto tr_stalled;
4217                 }
4218                 break;
4219
4220         case UT_WRITE_DEVICE:
4221                 switch (req->bRequest) {
4222                 case UR_SET_ADDRESS:
4223                         goto tr_handle_set_address;
4224                 case UR_SET_CONFIG:
4225                         goto tr_handle_set_config;
4226                 case UR_CLEAR_FEATURE:
4227                         goto tr_valid;  /* nop */
4228                 case UR_SET_DESCRIPTOR:
4229                         goto tr_valid;  /* nop */
4230                 case UR_SET_FEATURE:
4231                 default:
4232                         goto tr_stalled;
4233                 }
4234                 break;
4235
4236         case UT_WRITE_ENDPOINT:
4237                 switch (req->bRequest) {
4238                 case UR_CLEAR_FEATURE:
4239                         switch (UGETW(req->wValue)) {
4240                         case UF_ENDPOINT_HALT:
4241                                 goto tr_handle_clear_halt;
4242                         case UF_DEVICE_REMOTE_WAKEUP:
4243                                 goto tr_handle_clear_wakeup;
4244                         default:
4245                                 goto tr_stalled;
4246                         }
4247                         break;
4248                 case UR_SET_FEATURE:
4249                         switch (UGETW(req->wValue)) {
4250                         case UF_ENDPOINT_HALT:
4251                                 goto tr_handle_set_halt;
4252                         case UF_DEVICE_REMOTE_WAKEUP:
4253                                 goto tr_handle_set_wakeup;
4254                         default:
4255                                 goto tr_stalled;
4256                         }
4257                         break;
4258                 case UR_SYNCH_FRAME:
4259                         goto tr_valid;  /* nop */
4260                 default:
4261                         goto tr_stalled;
4262                 }
4263                 break;
4264
4265         case UT_READ_ENDPOINT:
4266                 switch (req->bRequest) {
4267                 case UR_GET_STATUS:
4268                         goto tr_handle_get_ep_status;
4269                 default:
4270                         goto tr_stalled;
4271                 }
4272                 break;
4273
4274         case UT_WRITE_INTERFACE:
4275                 switch (req->bRequest) {
4276                 case UR_SET_INTERFACE:
4277                         goto tr_handle_set_interface;
4278                 case UR_CLEAR_FEATURE:
4279                         goto tr_valid;  /* nop */
4280                 case UR_SET_FEATURE:
4281                 default:
4282                         goto tr_stalled;
4283                 }
4284                 break;
4285
4286         case UT_READ_INTERFACE:
4287                 switch (req->bRequest) {
4288                 case UR_GET_INTERFACE:
4289                         goto tr_handle_get_interface;
4290                 case UR_GET_STATUS:
4291                         goto tr_handle_get_iface_status;
4292                 default:
4293                         goto tr_stalled;
4294                 }
4295                 break;
4296
4297         case UT_WRITE_CLASS_INTERFACE:
4298         case UT_WRITE_VENDOR_INTERFACE:
4299                 /* XXX forward */
4300                 break;
4301
4302         case UT_READ_CLASS_INTERFACE:
4303         case UT_READ_VENDOR_INTERFACE:
4304                 /* XXX forward */
4305                 break;
4306
4307         case UT_WRITE_CLASS_DEVICE:
4308                 switch (req->bRequest) {
4309                 case UR_CLEAR_FEATURE:
4310                         goto tr_valid;
4311                 case UR_SET_DESCRIPTOR:
4312                 case UR_SET_FEATURE:
4313                         break;
4314                 default:
4315                         goto tr_stalled;
4316                 }
4317                 break;
4318
4319         case UT_WRITE_CLASS_OTHER:
4320                 switch (req->bRequest) {
4321                 case UR_CLEAR_FEATURE:
4322                         goto tr_handle_clear_port_feature;
4323                 case UR_SET_FEATURE:
4324                         goto tr_handle_set_port_feature;
4325                 case UR_CLEAR_TT_BUFFER:
4326                 case UR_RESET_TT:
4327                 case UR_STOP_TT:
4328                         goto tr_valid;
4329
4330                 default:
4331                         goto tr_stalled;
4332                 }
4333                 break;
4334
4335         case UT_READ_CLASS_OTHER:
4336                 switch (req->bRequest) {
4337                 case UR_GET_TT_STATE:
4338                         goto tr_handle_get_tt_state;
4339                 case UR_GET_STATUS:
4340                         goto tr_handle_get_port_status;
4341                 default:
4342                         goto tr_stalled;
4343                 }
4344                 break;
4345
4346         case UT_READ_CLASS_DEVICE:
4347                 switch (req->bRequest) {
4348                 case UR_GET_DESCRIPTOR:
4349                         goto tr_handle_get_class_descriptor;
4350                 case UR_GET_STATUS:
4351                         goto tr_handle_get_class_status;
4352
4353                 default:
4354                         goto tr_stalled;
4355                 }
4356                 break;
4357         default:
4358                 goto tr_stalled;
4359         }
4360         goto tr_valid;
4361
4362 tr_handle_get_descriptor:
4363         switch (value >> 8) {
4364         case UDESC_DEVICE:
4365                 if (value & 0xff) {
4366                         goto tr_stalled;
4367                 }
4368                 len = sizeof(dwc_otg_devd);
4369                 ptr = (const void *)&dwc_otg_devd;
4370                 goto tr_valid;
4371         case UDESC_CONFIG:
4372                 if (value & 0xff) {
4373                         goto tr_stalled;
4374                 }
4375                 len = sizeof(dwc_otg_confd);
4376                 ptr = (const void *)&dwc_otg_confd;
4377                 goto tr_valid;
4378         case UDESC_STRING:
4379                 switch (value & 0xff) {
4380                 case 0:         /* Language table */
4381                         len = sizeof(usb_string_lang_en);
4382                         ptr = (const void *)&usb_string_lang_en;
4383                         goto tr_valid;
4384
4385                 case 1:         /* Vendor */
4386                         len = sizeof(dwc_otg_vendor);
4387                         ptr = (const void *)&dwc_otg_vendor;
4388                         goto tr_valid;
4389
4390                 case 2:         /* Product */
4391                         len = sizeof(dwc_otg_product);
4392                         ptr = (const void *)&dwc_otg_product;
4393                         goto tr_valid;
4394                 default:
4395                         break;
4396                 }
4397                 break;
4398         default:
4399                 goto tr_stalled;
4400         }
4401         goto tr_stalled;
4402
4403 tr_handle_get_config:
4404         len = 1;
4405         sc->sc_hub_temp.wValue[0] = sc->sc_conf;
4406         goto tr_valid;
4407
4408 tr_handle_get_status:
4409         len = 2;
4410         USETW(sc->sc_hub_temp.wValue, UDS_SELF_POWERED);
4411         goto tr_valid;
4412
4413 tr_handle_set_address:
4414         if (value & 0xFF00) {
4415                 goto tr_stalled;
4416         }
4417         sc->sc_rt_addr = value;
4418         goto tr_valid;
4419
4420 tr_handle_set_config:
4421         if (value >= 2) {
4422                 goto tr_stalled;
4423         }
4424         sc->sc_conf = value;
4425         goto tr_valid;
4426
4427 tr_handle_get_interface:
4428         len = 1;
4429         sc->sc_hub_temp.wValue[0] = 0;
4430         goto tr_valid;
4431
4432 tr_handle_get_tt_state:
4433 tr_handle_get_class_status:
4434 tr_handle_get_iface_status:
4435 tr_handle_get_ep_status:
4436         len = 2;
4437         USETW(sc->sc_hub_temp.wValue, 0);
4438         goto tr_valid;
4439
4440 tr_handle_set_halt:
4441 tr_handle_set_interface:
4442 tr_handle_set_wakeup:
4443 tr_handle_clear_wakeup:
4444 tr_handle_clear_halt:
4445         goto tr_valid;
4446
4447 tr_handle_clear_port_feature:
4448         if (index != 1)
4449                 goto tr_stalled;
4450
4451         DPRINTFN(9, "UR_CLEAR_PORT_FEATURE on port %d\n", index);
4452
4453         switch (value) {
4454         case UHF_PORT_SUSPEND:
4455                 dwc_otg_wakeup_peer(sc);
4456                 break;
4457
4458         case UHF_PORT_ENABLE:
4459                 if (sc->sc_flags.status_device_mode == 0) {
4460                         DWC_OTG_WRITE_4(sc, DOTG_HPRT,
4461                             sc->sc_hprt_val | HPRT_PRTENA);
4462                 }
4463                 sc->sc_flags.port_enabled = 0;
4464                 break;
4465
4466         case UHF_C_PORT_RESET:
4467                 sc->sc_flags.change_reset = 0;
4468                 break;
4469
4470         case UHF_C_PORT_ENABLE:
4471                 sc->sc_flags.change_enabled = 0;
4472                 break;
4473
4474         case UHF_C_PORT_OVER_CURRENT:
4475                 sc->sc_flags.change_over_current = 0;
4476                 break;
4477
4478         case UHF_PORT_TEST:
4479         case UHF_PORT_INDICATOR:
4480                 /* nops */
4481                 break;
4482
4483         case UHF_PORT_POWER:
4484                 sc->sc_flags.port_powered = 0;
4485                 if (sc->sc_mode == DWC_MODE_HOST || sc->sc_mode == DWC_MODE_OTG) {
4486                         sc->sc_hprt_val = 0;
4487                         DWC_OTG_WRITE_4(sc, DOTG_HPRT, HPRT_PRTENA);
4488                 }
4489                 dwc_otg_pull_down(sc);
4490                 dwc_otg_clocks_off(sc);
4491                 break;
4492
4493         case UHF_C_PORT_CONNECTION:
4494                 /* clear connect change flag */
4495                 sc->sc_flags.change_connect = 0;
4496                 break;
4497
4498         case UHF_C_PORT_SUSPEND:
4499                 sc->sc_flags.change_suspend = 0;
4500                 break;
4501
4502         default:
4503                 err = USB_ERR_IOERROR;
4504                 goto done;
4505         }
4506         goto tr_valid;
4507
4508 tr_handle_set_port_feature:
4509         if (index != 1) {
4510                 goto tr_stalled;
4511         }
4512         DPRINTFN(9, "UR_SET_PORT_FEATURE\n");
4513
4514         switch (value) {
4515         case UHF_PORT_ENABLE:
4516                 break;
4517
4518         case UHF_PORT_SUSPEND:
4519                 if (sc->sc_flags.status_device_mode == 0) {
4520                         /* set suspend BIT */
4521                         sc->sc_hprt_val |= HPRT_PRTSUSP;
4522                         DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
4523
4524                         /* generate HUB suspend event */
4525                         dwc_otg_suspend_irq(sc);
4526                 }
4527                 break;
4528
4529         case UHF_PORT_RESET:
4530                 if (sc->sc_flags.status_device_mode == 0) {
4531
4532                         DPRINTF("PORT RESET\n");
4533
4534                         /* enable PORT reset */
4535                         DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val | HPRT_PRTRST);
4536
4537                         /* Wait 62.5ms for reset to complete */
4538                         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 16);
4539
4540                         DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
4541
4542                         /* Wait 62.5ms for reset to complete */
4543                         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 16);
4544
4545                         /* reset FIFOs */
4546                         (void) dwc_otg_init_fifo(sc, DWC_MODE_HOST);
4547
4548                         sc->sc_flags.change_reset = 1;
4549                 } else {
4550                         err = USB_ERR_IOERROR;
4551                 }
4552                 break;
4553
4554         case UHF_PORT_TEST:
4555         case UHF_PORT_INDICATOR:
4556                 /* nops */
4557                 break;
4558         case UHF_PORT_POWER:
4559                 sc->sc_flags.port_powered = 1;
4560                 if (sc->sc_mode == DWC_MODE_HOST || sc->sc_mode == DWC_MODE_OTG) {
4561                         sc->sc_hprt_val |= HPRT_PRTPWR;
4562                         DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
4563                 }
4564                 if (sc->sc_mode == DWC_MODE_DEVICE || sc->sc_mode == DWC_MODE_OTG) {
4565                         /* pull up D+, if any */
4566                         dwc_otg_pull_up(sc);
4567                 }
4568                 break;
4569         default:
4570                 err = USB_ERR_IOERROR;
4571                 goto done;
4572         }
4573         goto tr_valid;
4574
4575 tr_handle_get_port_status:
4576
4577         DPRINTFN(9, "UR_GET_PORT_STATUS\n");
4578
4579         if (index != 1)
4580                 goto tr_stalled;
4581
4582         if (sc->sc_flags.status_vbus)
4583                 dwc_otg_clocks_on(sc);
4584         else
4585                 dwc_otg_clocks_off(sc);
4586
4587         /* Select Device Side Mode */
4588
4589         if (sc->sc_flags.status_device_mode) {
4590                 value = UPS_PORT_MODE_DEVICE;
4591                 dwc_otg_timer_stop(sc);
4592         } else {
4593                 value = 0;
4594                 dwc_otg_timer_start(sc);
4595         }
4596
4597         if (sc->sc_flags.status_high_speed)
4598                 value |= UPS_HIGH_SPEED;
4599         else if (sc->sc_flags.status_low_speed)
4600                 value |= UPS_LOW_SPEED;
4601
4602         if (sc->sc_flags.port_powered)
4603                 value |= UPS_PORT_POWER;
4604
4605         if (sc->sc_flags.port_enabled)
4606                 value |= UPS_PORT_ENABLED;
4607
4608         if (sc->sc_flags.port_over_current)
4609                 value |= UPS_OVERCURRENT_INDICATOR;
4610
4611         if (sc->sc_flags.status_vbus &&
4612             sc->sc_flags.status_bus_reset)
4613                 value |= UPS_CURRENT_CONNECT_STATUS;
4614
4615         if (sc->sc_flags.status_suspend)
4616                 value |= UPS_SUSPEND;
4617
4618         USETW(sc->sc_hub_temp.ps.wPortStatus, value);
4619
4620         value = 0;
4621
4622         if (sc->sc_flags.change_connect)
4623                 value |= UPS_C_CONNECT_STATUS;
4624         if (sc->sc_flags.change_suspend)
4625                 value |= UPS_C_SUSPEND;
4626         if (sc->sc_flags.change_reset)
4627                 value |= UPS_C_PORT_RESET;
4628         if (sc->sc_flags.change_over_current)
4629                 value |= UPS_C_OVERCURRENT_INDICATOR;
4630
4631         USETW(sc->sc_hub_temp.ps.wPortChange, value);
4632         len = sizeof(sc->sc_hub_temp.ps);
4633         goto tr_valid;
4634
4635 tr_handle_get_class_descriptor:
4636         if (value & 0xFF) {
4637                 goto tr_stalled;
4638         }
4639         ptr = (const void *)&dwc_otg_hubd;
4640         len = sizeof(dwc_otg_hubd);
4641         goto tr_valid;
4642
4643 tr_stalled:
4644         err = USB_ERR_STALLED;
4645 tr_valid:
4646 done:
4647         *plength = len;
4648         *pptr = ptr;
4649         return (err);
4650 }
4651
4652 static void
4653 dwc_otg_xfer_setup(struct usb_setup_params *parm)
4654 {
4655         struct usb_xfer *xfer;
4656         void *last_obj;
4657         uint32_t ntd;
4658         uint32_t n;
4659         uint8_t ep_no;
4660         uint8_t ep_type;
4661
4662         xfer = parm->curr_xfer;
4663
4664         /*
4665          * NOTE: This driver does not use any of the parameters that
4666          * are computed from the following values. Just set some
4667          * reasonable dummies:
4668          */
4669         parm->hc_max_packet_size = 0x500;
4670         parm->hc_max_packet_count = 3;
4671         parm->hc_max_frame_size = 3 * 0x500;
4672
4673         usbd_transfer_setup_sub(parm);
4674
4675         /*
4676          * compute maximum number of TDs
4677          */
4678         ep_type = (xfer->endpoint->edesc->bmAttributes & UE_XFERTYPE);
4679
4680         if (ep_type == UE_CONTROL) {
4681
4682                 ntd = xfer->nframes + 1 /* STATUS */ + 1 /* SYNC 1 */
4683                     + 1 /* SYNC 2 */ + 1 /* SYNC 3 */;
4684         } else {
4685
4686                 ntd = xfer->nframes + 1 /* SYNC */ ;
4687         }
4688
4689         /*
4690          * check if "usbd_transfer_setup_sub" set an error
4691          */
4692         if (parm->err)
4693                 return;
4694
4695         /*
4696          * allocate transfer descriptors
4697          */
4698         last_obj = NULL;
4699
4700         ep_no = xfer->endpointno & UE_ADDR;
4701
4702         /*
4703          * Check for a valid endpoint profile in USB device mode:
4704          */
4705         if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) {
4706                 const struct usb_hw_ep_profile *pf;
4707
4708                 dwc_otg_get_hw_ep_profile(parm->udev, &pf, ep_no);
4709
4710                 if (pf == NULL) {
4711                         /* should not happen */
4712                         parm->err = USB_ERR_INVAL;
4713                         return;
4714                 }
4715         }
4716
4717         /* align data */
4718         parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
4719
4720         for (n = 0; n != ntd; n++) {
4721
4722                 struct dwc_otg_td *td;
4723
4724                 if (parm->buf) {
4725
4726                         td = USB_ADD_BYTES(parm->buf, parm->size[0]);
4727
4728                         /* compute shared bandwidth resource index for TT */
4729                         if (parm->udev->parent_hs_hub != NULL && parm->udev->speed != USB_SPEED_HIGH) {
4730                                 if (parm->udev->parent_hs_hub->ddesc.bDeviceProtocol == UDPROTO_HSHUBMTT)
4731                                         td->tt_index = parm->udev->device_index;
4732                                 else
4733                                         td->tt_index = parm->udev->parent_hs_hub->device_index;
4734                         } else {
4735                                 td->tt_index = parm->udev->device_index;
4736                         }
4737
4738                         /* init TD */
4739                         td->max_packet_size = xfer->max_packet_size;
4740                         td->max_packet_count = xfer->max_packet_count;
4741                         /* range check */
4742                         if (td->max_packet_count == 0 || td->max_packet_count > 3)
4743                                 td->max_packet_count = 1;
4744                         td->ep_no = ep_no;
4745                         td->ep_type = ep_type;
4746                         td->obj_next = last_obj;
4747
4748                         last_obj = td;
4749                 }
4750                 parm->size[0] += sizeof(*td);
4751         }
4752
4753         xfer->td_start[0] = last_obj;
4754 }
4755
4756 static void
4757 dwc_otg_xfer_unsetup(struct usb_xfer *xfer)
4758 {
4759         return;
4760 }
4761
4762 static void
4763 dwc_otg_ep_init(struct usb_device *udev, struct usb_endpoint_descriptor *edesc,
4764     struct usb_endpoint *ep)
4765 {
4766         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(udev->bus);
4767
4768         DPRINTFN(2, "endpoint=%p, addr=%d, endpt=%d, mode=%d (%d,%d)\n",
4769             ep, udev->address,
4770             edesc->bEndpointAddress, udev->flags.usb_mode,
4771             sc->sc_rt_addr, udev->device_index);
4772
4773         if (udev->device_index != sc->sc_rt_addr) {
4774
4775                 if (udev->flags.usb_mode == USB_MODE_DEVICE) {
4776                         if (udev->speed != USB_SPEED_FULL &&
4777                             udev->speed != USB_SPEED_HIGH) {
4778                                 /* not supported */
4779                                 return;
4780                         }
4781                 } else {
4782                         if (udev->speed == USB_SPEED_HIGH &&
4783                             (edesc->wMaxPacketSize[1] & 0x18) != 0 &&
4784                             (edesc->bmAttributes & UE_XFERTYPE) != UE_ISOCHRONOUS) {
4785                                 /* not supported */
4786                                 DPRINTFN(-1, "Non-isochronous high bandwidth "
4787                                     "endpoint not supported\n");
4788                                 return;
4789                         }
4790                 }
4791                 if ((edesc->bmAttributes & UE_XFERTYPE) == UE_ISOCHRONOUS)
4792                         ep->methods = &dwc_otg_device_isoc_methods;
4793                 else
4794                         ep->methods = &dwc_otg_device_non_isoc_methods;
4795         }
4796 }
4797
4798 static void
4799 dwc_otg_set_hw_power_sleep(struct usb_bus *bus, uint32_t state)
4800 {
4801         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(bus);
4802
4803         switch (state) {
4804         case USB_HW_POWER_SUSPEND:
4805                 dwc_otg_suspend(sc);
4806                 break;
4807         case USB_HW_POWER_SHUTDOWN:
4808                 dwc_otg_uninit(sc);
4809                 break;
4810         case USB_HW_POWER_RESUME:
4811                 dwc_otg_resume(sc);
4812                 break;
4813         default:
4814                 break;
4815         }
4816 }
4817
4818 static void
4819 dwc_otg_get_dma_delay(struct usb_device *udev, uint32_t *pus)
4820 {
4821         /* DMA delay - wait until any use of memory is finished */
4822         *pus = (2125);                  /* microseconds */
4823 }
4824
4825 static void
4826 dwc_otg_device_resume(struct usb_device *udev)
4827 {
4828         DPRINTF("\n");
4829
4830         /* poll all transfers again to restart resumed ones */
4831         dwc_otg_do_poll(udev->bus);
4832 }
4833
4834 static void
4835 dwc_otg_device_suspend(struct usb_device *udev)
4836 {
4837         DPRINTF("\n");
4838 }
4839
4840 struct usb_bus_methods dwc_otg_bus_methods =
4841 {
4842         .endpoint_init = &dwc_otg_ep_init,
4843         .xfer_setup = &dwc_otg_xfer_setup,
4844         .xfer_unsetup = &dwc_otg_xfer_unsetup,
4845         .get_hw_ep_profile = &dwc_otg_get_hw_ep_profile,
4846         .xfer_stall = &dwc_otg_xfer_stall,
4847         .set_stall = &dwc_otg_set_stall,
4848         .clear_stall = &dwc_otg_clear_stall,
4849         .roothub_exec = &dwc_otg_roothub_exec,
4850         .xfer_poll = &dwc_otg_do_poll,
4851         .device_state_change = &dwc_otg_device_state_change,
4852         .set_hw_power_sleep = &dwc_otg_set_hw_power_sleep,
4853         .get_dma_delay = &dwc_otg_get_dma_delay,
4854         .device_resume = &dwc_otg_device_resume,
4855         .device_suspend = &dwc_otg_device_suspend,
4856 };