sys/dev/usb/controller/dwc_otg.c

   1 /* $FreeBSD$ */
   2 /*-
   3  * Copyright (c) 2012 Hans Petter Selasky. All rights reserved.
   4  * Copyright (c) 2010-2011 Aleksandr Rybalko. All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  */
  27
  28 /*
  29  * This file contains the driver for the DesignWare series USB 2.0 OTG
  30  * Controller.
  31  */
  32
  33 /*
  34  * LIMITATION: Drivers must be bound to all OUT endpoints in the
  35  * active configuration for this driver to work properly. Blocking any
  36  * OUT endpoint will block all OUT endpoints including the control
  37  * endpoint. Usually this is not a problem.
  38  */
  39
  40 /*
  41  * NOTE: Writing to non-existing registers appears to cause an
  42  * internal reset.
  43  */
  44
  45 #ifdef USB_GLOBAL_INCLUDE_FILE
  46 #include USB_GLOBAL_INCLUDE_FILE
  47 #else
  48 #include <sys/stdint.h>
  49 #include <sys/stddef.h>
  50 #include <sys/param.h>
  51 #include <sys/queue.h>
  52 #include <sys/types.h>
  53 #include <sys/systm.h>
  54 #include <sys/kernel.h>
  55 #include <sys/bus.h>
  56 #include <sys/module.h>
  57 #include <sys/lock.h>
  58 #include <sys/mutex.h>
  59 #include <sys/condvar.h>
  60 #include <sys/sysctl.h>
  61 #include <sys/sx.h>
  62 #include <sys/unistd.h>
  63 #include <sys/callout.h>
  64 #include <sys/malloc.h>
  65 #include <sys/priv.h>
  66
  67 #include <dev/usb/usb.h>
  68 #include <dev/usb/usbdi.h>
  69
  70 #define USB_DEBUG_VAR dwc_otg_debug
  71
  72 #include <dev/usb/usb_core.h>
  73 #include <dev/usb/usb_debug.h>
  74 #include <dev/usb/usb_busdma.h>
  75 #include <dev/usb/usb_process.h>
  76 #include <dev/usb/usb_transfer.h>
  77 #include <dev/usb/usb_device.h>
  78 #include <dev/usb/usb_hub.h>
  79 #include <dev/usb/usb_util.h>
  80
  81 #include <dev/usb/usb_controller.h>
  82 #include <dev/usb/usb_bus.h>
  83 #endif                  /* USB_GLOBAL_INCLUDE_FILE */
  84
  85 #include <dev/usb/controller/dwc_otg.h>
  86 #include <dev/usb/controller/dwc_otgreg.h>
  87
  88 #define DWC_OTG_BUS2SC(bus) \
  89    ((struct dwc_otg_softc *)(((uint8_t *)(bus)) - \
  90     ((uint8_t *)&(((struct dwc_otg_softc *)0)->sc_bus))))
  91
  92 #define DWC_OTG_PC2UDEV(pc) \
  93    (USB_DMATAG_TO_XROOT((pc)->tag_parent)->udev)
  94
  95 #define DWC_OTG_MSK_GINT_ENABLED        \
  96    (GINTMSK_ENUMDONEMSK |               \
  97    GINTMSK_USBRSTMSK |                  \
  98    GINTMSK_USBSUSPMSK |                 \
  99    GINTMSK_IEPINTMSK |                  \
 100    GINTMSK_SESSREQINTMSK |              \
 101    GINTMSK_RXFLVLMSK |                  \
 102    GINTMSK_HCHINTMSK |                  \
 103    GINTMSK_OTGINTMSK |                  \
 104    GINTMSK_PRTINTMSK)
 105
 106 #define DWC_OTG_MSK_GINT_THREAD_IRQ                             \
 107    (GINTSTS_USBRST | GINTSTS_ENUMDONE | GINTSTS_PRTINT |        \
 108    GINTSTS_WKUPINT | GINTSTS_USBSUSP | GINTMSK_OTGINTMSK |      \
 109    GINTSTS_SESSREQINT)
 110
 111 #define DWC_OTG_PHY_ULPI 0
 112 #define DWC_OTG_PHY_HSIC 1
 113 #define DWC_OTG_PHY_INTERNAL 2
 114
 115 #ifndef DWC_OTG_PHY_DEFAULT
 116 #define DWC_OTG_PHY_DEFAULT DWC_OTG_PHY_ULPI
 117 #endif
 118
 119 static int dwc_otg_phy_type = DWC_OTG_PHY_DEFAULT;
 120
 121 static SYSCTL_NODE(_hw_usb, OID_AUTO, dwc_otg, CTLFLAG_RW, 0, "USB DWC OTG");
 122 SYSCTL_INT(_hw_usb_dwc_otg, OID_AUTO, phy_type, CTLFLAG_RDTUN,
 123     &dwc_otg_phy_type, 0, "DWC OTG PHY TYPE - 0/1/2 - ULPI/HSIC/INTERNAL");
 124 TUNABLE_INT("hw.usb.dwc_otg.phy_type", &dwc_otg_phy_type);
 125
 126 #ifdef USB_DEBUG
 127 static int dwc_otg_debug;
 128
 129 SYSCTL_INT(_hw_usb_dwc_otg, OID_AUTO, debug, CTLFLAG_RW,
 130     &dwc_otg_debug, 0, "DWC OTG debug level");
 131 #endif
 132
 133 #define DWC_OTG_INTR_ENDPT 1
 134
 135 /* prototypes */
 136
 137 struct usb_bus_methods dwc_otg_bus_methods;
 138 struct usb_pipe_methods dwc_otg_device_non_isoc_methods;
 139 struct usb_pipe_methods dwc_otg_device_isoc_methods;
 140
 141 static dwc_otg_cmd_t dwc_otg_setup_rx;
 142 static dwc_otg_cmd_t dwc_otg_data_rx;
 143 static dwc_otg_cmd_t dwc_otg_data_tx;
 144 static dwc_otg_cmd_t dwc_otg_data_tx_sync;
 145
 146 static dwc_otg_cmd_t dwc_otg_host_setup_tx;
 147 static dwc_otg_cmd_t dwc_otg_host_data_tx;
 148 static dwc_otg_cmd_t dwc_otg_host_data_rx;
 149
 150 static void dwc_otg_device_done(struct usb_xfer *, usb_error_t);
 151 static void dwc_otg_do_poll(struct usb_bus *);
 152 static void dwc_otg_standard_done(struct usb_xfer *);
 153 static void dwc_otg_root_intr(struct dwc_otg_softc *);
 154 static void dwc_otg_interrupt_poll_locked(struct dwc_otg_softc *);
 155 static void dwc_otg_host_channel_disable(struct dwc_otg_softc *, uint8_t);
 156
 157 /*
 158  * Here is a configuration that the chip supports.
 159  */
 160 static const struct usb_hw_ep_profile dwc_otg_ep_profile[1] = {
 161
 162         [0] = {
 163                 .max_in_frame_size = 64,/* fixed */
 164                 .max_out_frame_size = 64,       /* fixed */
 165                 .is_simplex = 1,
 166                 .support_control = 1,
 167         }
 168 };
 169
 170 static void
 171 dwc_otg_get_hw_ep_profile(struct usb_device *udev,
 172     const struct usb_hw_ep_profile **ppf, uint8_t ep_addr)
 173 {
 174         struct dwc_otg_softc *sc;
 175
 176         sc = DWC_OTG_BUS2SC(udev->bus);
 177
 178         if (ep_addr < sc->sc_dev_ep_max)
 179                 *ppf = &sc->sc_hw_ep_profile[ep_addr].usb;
 180         else
 181                 *ppf = NULL;
 182 }
 183
 184 static void
 185 dwc_otg_tx_fifo_reset(struct dwc_otg_softc *sc, uint32_t value)
 186 {
 187         uint32_t temp;
 188
 189         /* reset FIFO */
 190         DWC_OTG_WRITE_4(sc, DOTG_GRSTCTL, value);
 191
 192         /* wait for reset to complete */
 193         for (temp = 0; temp != 16; temp++) {
 194                 value = DWC_OTG_READ_4(sc, DOTG_GRSTCTL);
 195                 if (!(value & (GRSTCTL_TXFFLSH | GRSTCTL_RXFFLSH)))
 196                         break;
 197         }
 198 }
 199
 200 static int
 201 dwc_otg_init_fifo(struct dwc_otg_softc *sc, uint8_t mode)
 202 {
 203         struct dwc_otg_profile *pf;
 204         uint32_t fifo_size;
 205         uint32_t fifo_regs;
 206         uint32_t tx_start;
 207         uint8_t x;
 208
 209         fifo_size = sc->sc_fifo_size;
 210
 211         /*
 212          * NOTE: Reserved fixed size area at end of RAM, which must
 213          * not be allocated to the FIFOs:
 214          */
 215         fifo_regs = 4 * 16;
 216
 217         if (fifo_size < fifo_regs) {
 218                 DPRINTF("Too little FIFO\n");
 219                 return (EINVAL);
 220         }
 221
 222         /* subtract FIFO regs from total once */
 223         fifo_size -= fifo_regs;
 224
 225         /* split equally for IN and OUT */
 226         fifo_size /= 2;
 227
 228         /* align to 4 bytes boundary */
 229         fifo_size &= ~3;
 230
 231         /* set global receive FIFO size */
 232         DWC_OTG_WRITE_4(sc, DOTG_GRXFSIZ, fifo_size / 4);
 233
 234         tx_start = fifo_size;
 235
 236         if (fifo_size < 64) {
 237                 DPRINTFN(-1, "Not enough data space for EP0 FIFO.\n");
 238                 return (EINVAL);
 239         }
 240
 241         /* disable any leftover host channels */
 242         for (x = 0; x != sc->sc_host_ch_max; x++) {
 243                 if (sc->sc_chan_state[x].wait_sof == 0)
 244                         continue;
 245                 dwc_otg_host_channel_disable(sc, x);
 246         }
 247
 248         if (mode == DWC_MODE_HOST) {
 249
 250                 /* reset active endpoints */
 251                 sc->sc_active_rx_ep = 0;
 252
 253                 /* split equally for periodic and non-periodic */
 254                 fifo_size /= 2;
 255
 256                 /* align to 4 bytes boundary */
 257                 fifo_size &= ~3;
 258
 259                 DWC_OTG_WRITE_4(sc, DOTG_GNPTXFSIZ,
 260                     ((fifo_size / 4) << 16) |
 261                     (tx_start / 4));
 262
 263                 tx_start += fifo_size;
 264
 265                 for (x = 0; x != sc->sc_host_ch_max; x++) {
 266                         /* disable all host interrupts */
 267                         DWC_OTG_WRITE_4(sc, DOTG_HCINTMSK(x),
 268                             HCINT_DEFAULT_MASK);
 269                 }
 270
 271                 DWC_OTG_WRITE_4(sc, DOTG_HPTXFSIZ,
 272                     ((fifo_size / 4) << 16) |
 273                     (tx_start / 4));
 274
 275                 /* reset host channel state */
 276                 memset(sc->sc_chan_state, 0, sizeof(sc->sc_chan_state));
 277
 278                 /* reset FIFO TX levels */
 279                 sc->sc_tx_cur_p_level = 0;
 280                 sc->sc_tx_cur_np_level = 0;
 281
 282                 /* store maximum periodic and non-periodic FIFO TX size */
 283                 sc->sc_tx_max_size = fifo_size;
 284
 285                 /* enable all host channel interrupts */
 286                 DWC_OTG_WRITE_4(sc, DOTG_HAINTMSK,
 287                     (1U << sc->sc_host_ch_max) - 1U);
 288         }
 289
 290         if (mode == DWC_MODE_DEVICE) {
 291
 292             DWC_OTG_WRITE_4(sc, DOTG_GNPTXFSIZ,
 293                 (0x10 << 16) | (tx_start / 4));
 294             fifo_size -= 0x40;
 295             tx_start += 0x40;
 296
 297             /* setup control endpoint profile */
 298             sc->sc_hw_ep_profile[0].usb = dwc_otg_ep_profile[0];
 299
 300             /* reset active endpoints */
 301             sc->sc_active_rx_ep = 1;
 302
 303             for (x = 1; x != sc->sc_dev_ep_max; x++) {
 304
 305                 pf = sc->sc_hw_ep_profile + x;
 306
 307                 pf->usb.max_out_frame_size = 1024 * 3;
 308                 pf->usb.is_simplex = 0; /* assume duplex */
 309                 pf->usb.support_bulk = 1;
 310                 pf->usb.support_interrupt = 1;
 311                 pf->usb.support_isochronous = 1;
 312                 pf->usb.support_out = 1;
 313
 314                 if (x < sc->sc_dev_in_ep_max) {
 315                         uint32_t limit;
 316
 317                         limit = (x == 1) ? DWC_OTG_MAX_TXN :
 318                             (DWC_OTG_MAX_TXN / 2);
 319
 320                         if (fifo_size >= limit) {
 321                                 DWC_OTG_WRITE_4(sc, DOTG_DIEPTXF(x),
 322                                     ((limit / 4) << 16) |
 323                                     (tx_start / 4));
 324                                 tx_start += limit;
 325                                 fifo_size -= limit;
 326                                 pf->usb.max_in_frame_size = 0x200;
 327                                 pf->usb.support_in = 1;
 328                                 pf->max_buffer = limit;
 329
 330                         } else if (fifo_size >= 0x80) {
 331                                 DWC_OTG_WRITE_4(sc, DOTG_DIEPTXF(x),
 332                                     ((0x80 / 4) << 16) | (tx_start / 4));
 333                                 tx_start += 0x80;
 334                                 fifo_size -= 0x80;
 335                                 pf->usb.max_in_frame_size = 0x40;
 336                                 pf->usb.support_in = 1;
 337
 338                         } else {
 339                                 pf->usb.is_simplex = 1;
 340                                 DWC_OTG_WRITE_4(sc, DOTG_DIEPTXF(x),
 341                                     (0x0 << 16) | (tx_start / 4));
 342                         }
 343                 } else {
 344                         pf->usb.is_simplex = 1;
 345                 }
 346
 347                 DPRINTF("FIFO%d = IN:%d / OUT:%d\n", x,
 348                     pf->usb.max_in_frame_size,
 349                     pf->usb.max_out_frame_size);
 350             }
 351         }
 352
 353         /* reset RX FIFO */
 354         dwc_otg_tx_fifo_reset(sc, GRSTCTL_RXFFLSH);
 355
 356         if (mode != DWC_MODE_OTG) {
 357                 /* reset all TX FIFOs */
 358                 dwc_otg_tx_fifo_reset(sc,
 359                     GRSTCTL_TXFIFO(0x10) |
 360                     GRSTCTL_TXFFLSH);
 361         } else {
 362                 /* reset active endpoints */
 363                 sc->sc_active_rx_ep = 0;
 364
 365                 /* reset periodic and non-periodic FIFO TX size */
 366                 sc->sc_tx_max_size = fifo_size;
 367
 368                 /* reset host channel state */
 369                 memset(sc->sc_chan_state, 0, sizeof(sc->sc_chan_state));
 370
 371                 /* reset FIFO TX levels */
 372                 sc->sc_tx_cur_p_level = 0;
 373                 sc->sc_tx_cur_np_level = 0;
 374         }
 375         return (0);
 376 }
 377
 378 static void
 379 dwc_otg_update_host_frame_interval(struct dwc_otg_softc *sc)
 380 {
 381
 382   /*
 383    * Disabled until further. Assuming that the register is already
 384    * programmed correctly by the boot loader.
 385    */
 386 #if 0
 387         uint32_t temp;
 388
 389         /* setup HOST frame interval register, based on existing value */
 390         temp = DWC_OTG_READ_4(sc, DOTG_HFIR) & HFIR_FRINT_MASK;
 391         if (temp >= 10000)
 392                 temp /= 1000;
 393         else
 394                 temp /= 125;
 395
 396         /* figure out nearest X-tal value */
 397         if (temp >= 54)
 398                 temp = 60;      /* MHz */
 399         else if (temp >= 39)
 400                 temp = 48;      /* MHz */
 401         else
 402                 temp = 30;      /* MHz */
 403
 404         if (sc->sc_flags.status_high_speed)
 405                 temp *= 125;
 406         else
 407                 temp *= 1000;
 408
 409         DPRINTF("HFIR=0x%08x\n", temp);
 410
 411         DWC_OTG_WRITE_4(sc, DOTG_HFIR, temp);
 412 #endif
 413 }
 414
 415 static void
 416 dwc_otg_clocks_on(struct dwc_otg_softc *sc)
 417 {
 418         if (sc->sc_flags.clocks_off &&
 419             sc->sc_flags.port_powered) {
 420
 421                 DPRINTFN(5, "\n");
 422
 423                 /* TODO - platform specific */
 424
 425                 sc->sc_flags.clocks_off = 0;
 426         }
 427 }
 428
 429 static void
 430 dwc_otg_clocks_off(struct dwc_otg_softc *sc)
 431 {
 432         if (!sc->sc_flags.clocks_off) {
 433
 434                 DPRINTFN(5, "\n");
 435
 436                 /* TODO - platform specific */
 437
 438                 sc->sc_flags.clocks_off = 1;
 439         }
 440 }
 441
 442 static void
 443 dwc_otg_pull_up(struct dwc_otg_softc *sc)
 444 {
 445         uint32_t temp;
 446
 447         /* pullup D+, if possible */
 448
 449         if (!sc->sc_flags.d_pulled_up &&
 450             sc->sc_flags.port_powered) {
 451                 sc->sc_flags.d_pulled_up = 1;
 452
 453                 temp = DWC_OTG_READ_4(sc, DOTG_DCTL);
 454                 temp &= ~DCTL_SFTDISCON;
 455                 DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
 456         }
 457 }
 458
 459 static void
 460 dwc_otg_pull_down(struct dwc_otg_softc *sc)
 461 {
 462         uint32_t temp;
 463
 464         /* pulldown D+, if possible */
 465
 466         if (sc->sc_flags.d_pulled_up) {
 467                 sc->sc_flags.d_pulled_up = 0;
 468
 469                 temp = DWC_OTG_READ_4(sc, DOTG_DCTL);
 470                 temp |= DCTL_SFTDISCON;
 471                 DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
 472         }
 473 }
 474
 475 static void
 476 dwc_otg_enable_sof_irq(struct dwc_otg_softc *sc)
 477 {
 478         /* In device mode we don't use the SOF interrupt */
 479         if (sc->sc_flags.status_device_mode != 0 ||
 480             (sc->sc_irq_mask & GINTMSK_SOFMSK) != 0)
 481                 return;
 482         sc->sc_irq_mask |= GINTMSK_SOFMSK;
 483         DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 484 }
 485
 486 static void
 487 dwc_otg_resume_irq(struct dwc_otg_softc *sc)
 488 {
 489         if (sc->sc_flags.status_suspend) {
 490                 /* update status bits */
 491                 sc->sc_flags.status_suspend = 0;
 492                 sc->sc_flags.change_suspend = 1;
 493
 494                 if (sc->sc_flags.status_device_mode) {
 495                         /*
 496                          * Disable resume interrupt and enable suspend
 497                          * interrupt:
 498                          */
 499                         sc->sc_irq_mask &= ~GINTMSK_WKUPINTMSK;
 500                         sc->sc_irq_mask |= GINTMSK_USBSUSPMSK;
 501                         DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 502                 }
 503
 504                 /* complete root HUB interrupt endpoint */
 505                 dwc_otg_root_intr(sc);
 506         }
 507 }
 508
 509 static void
 510 dwc_otg_suspend_irq(struct dwc_otg_softc *sc)
 511 {
 512         if (!sc->sc_flags.status_suspend) {
 513                 /* update status bits */
 514                 sc->sc_flags.status_suspend = 1;
 515                 sc->sc_flags.change_suspend = 1;
 516
 517                 if (sc->sc_flags.status_device_mode) {
 518                         /*
 519                          * Disable suspend interrupt and enable resume
 520                          * interrupt:
 521                          */
 522                         sc->sc_irq_mask &= ~GINTMSK_USBSUSPMSK;
 523                         sc->sc_irq_mask |= GINTMSK_WKUPINTMSK;
 524                         DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 525                 }
 526
 527                 /* complete root HUB interrupt endpoint */
 528                 dwc_otg_root_intr(sc);
 529         }
 530 }
 531
 532 static void
 533 dwc_otg_wakeup_peer(struct dwc_otg_softc *sc)
 534 {
 535         if (!sc->sc_flags.status_suspend)
 536                 return;
 537
 538         DPRINTFN(5, "Remote wakeup\n");
 539
 540         if (sc->sc_flags.status_device_mode) {
 541                 uint32_t temp;
 542
 543                 /* enable remote wakeup signalling */
 544                 temp = DWC_OTG_READ_4(sc, DOTG_DCTL);
 545                 temp |= DCTL_RMTWKUPSIG;
 546                 DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
 547
 548                 /* Wait 8ms for remote wakeup to complete. */
 549                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 125);
 550
 551                 temp &= ~DCTL_RMTWKUPSIG;
 552                 DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
 553         } else {
 554                 /* enable USB port */
 555                 DWC_OTG_WRITE_4(sc, DOTG_PCGCCTL, 0);
 556
 557                 /* wait 10ms */
 558                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 100);
 559
 560                 /* resume port */
 561                 sc->sc_hprt_val |= HPRT_PRTRES;
 562                 DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
 563
 564                 /* Wait 100ms for resume signalling to complete. */
 565                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 10);
 566
 567                 /* clear suspend and resume */
 568                 sc->sc_hprt_val &= ~(HPRT_PRTSUSP | HPRT_PRTRES);
 569                 DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
 570
 571                 /* Wait 4ms */
 572                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 250);
 573         }
 574
 575         /* need to fake resume IRQ */
 576         dwc_otg_resume_irq(sc);
 577 }
 578
 579 static void
 580 dwc_otg_set_address(struct dwc_otg_softc *sc, uint8_t addr)
 581 {
 582         uint32_t temp;
 583
 584         DPRINTFN(5, "addr=%d\n", addr);
 585
 586         temp = DWC_OTG_READ_4(sc, DOTG_DCFG);
 587         temp &= ~DCFG_DEVADDR_SET(0x7F);
 588         temp |= DCFG_DEVADDR_SET(addr);
 589         DWC_OTG_WRITE_4(sc, DOTG_DCFG, temp);
 590 }
 591
 592 static void
 593 dwc_otg_common_rx_ack(struct dwc_otg_softc *sc)
 594 {
 595         DPRINTFN(5, "RX status clear\n");
 596
 597         /* enable RX FIFO level interrupt */
 598         sc->sc_irq_mask |= GINTMSK_RXFLVLMSK;
 599         DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 600
 601         /* clear cached status */
 602         sc->sc_last_rx_status = 0;
 603 }
 604
 605 static void
 606 dwc_otg_clear_hcint(struct dwc_otg_softc *sc, uint8_t x)
 607 {
 608         uint32_t hcint;
 609
 610         /* clear all pending interrupts */
 611         hcint = DWC_OTG_READ_4(sc, DOTG_HCINT(x));
 612         DWC_OTG_WRITE_4(sc, DOTG_HCINT(x), hcint);
 613
 614         /* clear buffered interrupts */
 615         sc->sc_chan_state[x].hcint = 0;
 616 }
 617
 618 static uint8_t
 619 dwc_otg_host_channel_alloc(struct dwc_otg_softc *sc, struct dwc_otg_td *td, uint8_t is_out)
 620 {
 621         uint32_t tx_p_size;
 622         uint32_t tx_np_size;
 623         uint8_t x;
 624
 625         if (td->channel < DWC_OTG_MAX_CHANNELS)
 626                 return (0);             /* already allocated */
 627
 628         /* check if device is suspended */
 629         if (DWC_OTG_PC2UDEV(td->pc)->flags.self_suspended != 0)
 630                 return (1);             /* busy - cannot transfer data */
 631
 632         /* compute needed TX FIFO size */
 633         if (is_out != 0) {
 634                 if (td->ep_type == UE_ISOCHRONOUS) {
 635                         tx_p_size = td->max_packet_size;
 636                         tx_np_size = 0;
 637                         if (td->hcsplt != 0 && tx_p_size > HCSPLT_XACTLEN_BURST)
 638                                 tx_p_size = HCSPLT_XACTLEN_BURST;
 639                         if ((sc->sc_tx_cur_p_level + tx_p_size) > sc->sc_tx_max_size) {
 640                                 DPRINTF("Too little FIFO space\n");
 641                                 return (1);     /* too little FIFO */
 642                         }
 643                 } else {
 644                         tx_p_size = 0;
 645                         tx_np_size = td->max_packet_size;
 646                         if (td->hcsplt != 0 && tx_np_size > HCSPLT_XACTLEN_BURST)
 647                                 tx_np_size = HCSPLT_XACTLEN_BURST;
 648                         if ((sc->sc_tx_cur_np_level + tx_np_size) > sc->sc_tx_max_size) {
 649                                 DPRINTF("Too little FIFO space\n");
 650                                 return (1);     /* too little FIFO */
 651                         }
 652                 }
 653         } else {
 654                 /* not a TX transaction */
 655                 tx_p_size = 0;
 656                 tx_np_size = 0;
 657         }
 658
 659         for (x = 0; x != sc->sc_host_ch_max; x++) {
 660                 if (sc->sc_chan_state[x].allocated != 0)
 661                         continue;
 662                 /* check if channel is still enabled */
 663                 if (sc->sc_chan_state[x].wait_sof != 0)
 664                         continue;
 665
 666                 sc->sc_chan_state[x].allocated = 1;
 667                 sc->sc_chan_state[x].tx_p_size = tx_p_size;
 668                 sc->sc_chan_state[x].tx_np_size = tx_np_size;
 669
 670                 /* keep track of used TX FIFO, if any */
 671                 sc->sc_tx_cur_p_level += tx_p_size;
 672                 sc->sc_tx_cur_np_level += tx_np_size;
 673
 674                 /* clear interrupts */
 675                 dwc_otg_clear_hcint(sc, x);
 676
 677                 DPRINTF("CH=%d HCCHAR=0x%08x "
 678                     "HCSPLT=0x%08x\n", x, td->hcchar, td->hcsplt);
 679
 680                 /* set active channel */
 681                 sc->sc_active_rx_ep |= (1 << x);
 682
 683                 /* set channel */
 684                 td->channel = x;
 685
 686                 return (0);     /* allocated */
 687         }
 688         /* wait a bit */
 689         dwc_otg_enable_sof_irq(sc);
 690         return (1);     /* busy */
 691 }
 692
 693 static void
 694 dwc_otg_host_channel_free(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
 695 {
 696         uint8_t x;
 697
 698         if (td->channel >= DWC_OTG_MAX_CHANNELS)
 699                 return;         /* already freed */
 700
 701         /* free channel */
 702         x = td->channel;
 703         td->channel = DWC_OTG_MAX_CHANNELS;
 704
 705         DPRINTF("CH=%d\n", x);
 706
 707         /*
 708          * We need to let programmed host channels run till complete
 709          * else the host channel will stop functioning. Assume that
 710          * after a fixed given amount of time the host channel is no
 711          * longer doing any USB traffic:
 712          */
 713         if (td->ep_type == UE_ISOCHRONOUS) {
 714                 /* double buffered */
 715                 sc->sc_chan_state[x].wait_sof = DWC_OTG_SLOT_IDLE_MAX;
 716         } else {
 717                 /* single buffered */
 718                 sc->sc_chan_state[x].wait_sof = DWC_OTG_SLOT_IDLE_MIN;
 719         }
 720
 721         sc->sc_chan_state[x].allocated = 0;
 722
 723         /* ack any pending messages */
 724         if (sc->sc_last_rx_status != 0 &&
 725             GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) == x) {
 726                 dwc_otg_common_rx_ack(sc);
 727         }
 728
 729         /* clear active channel */
 730         sc->sc_active_rx_ep &= ~(1 << x);
 731 }
 732
 733 static void
 734 dwc_otg_host_dump_rx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
 735 {
 736         /* dump any pending messages */
 737         if (sc->sc_last_rx_status != 0) {
 738                 if (td->channel < DWC_OTG_MAX_CHANNELS &&
 739                     td->channel == GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status)) {
 740                         dwc_otg_common_rx_ack(sc);
 741                 }
 742         }
 743 }
 744
 745 static uint8_t
 746 dwc_otg_host_setup_tx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
 747 {
 748         struct usb_device_request req __aligned(4);
 749         uint32_t hcint;
 750         uint32_t hcchar;
 751         uint8_t delta;
 752
 753         dwc_otg_host_dump_rx(sc, td);
 754
 755         if (td->channel < DWC_OTG_MAX_CHANNELS) {
 756                 hcint = sc->sc_chan_state[td->channel].hcint;
 757
 758                 DPRINTF("CH=%d ST=%d HCINT=0x%08x HCCHAR=0x%08x HCTSIZ=0x%08x\n",
 759                     td->channel, td->state, hcint,
 760                     DWC_OTG_READ_4(sc, DOTG_HCCHAR(td->channel)),
 761                     DWC_OTG_READ_4(sc, DOTG_HCTSIZ(td->channel)));
 762         } else {
 763                 hcint = 0;
 764                 goto check_state;
 765         }
 766
 767         if (hcint & (HCINT_RETRY |
 768             HCINT_ACK | HCINT_NYET)) {
 769                 /* give success bits priority over failure bits */
 770         } else if (hcint & HCINT_STALL) {
 771                 DPRINTF("CH=%d STALL\n", td->channel);
 772                 td->error_stall = 1;
 773                 td->error_any = 1;
 774                 goto complete;
 775         } else if (hcint & HCINT_ERRORS) {
 776                 DPRINTF("CH=%d ERROR\n", td->channel);
 777                 td->errcnt++;
 778                 if (td->hcsplt != 0 || td->errcnt >= 3) {
 779                         td->error_any = 1;
 780                         goto complete;
 781                 }
 782         }
 783
 784         if (hcint & (HCINT_ERRORS | HCINT_RETRY |
 785             HCINT_ACK | HCINT_NYET)) {
 786                 if (!(hcint & HCINT_ERRORS))
 787                         td->errcnt = 0;
 788         }
 789
 790 check_state:
 791         switch (td->state) {
 792         case DWC_CHAN_ST_START:
 793                 goto send_pkt;
 794
 795         case DWC_CHAN_ST_WAIT_ANE:
 796                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
 797                         td->did_nak++;
 798                         td->tt_scheduled = 0;
 799                         goto send_pkt;
 800                 } else if (hcint & (HCINT_ACK | HCINT_NYET)) {
 801                         td->offset += td->tx_bytes;
 802                         td->remainder -= td->tx_bytes;
 803                         td->toggle = 1;
 804                         td->tt_scheduled = 0;
 805                         goto complete;
 806                 }
 807                 break;
 808
 809         case DWC_CHAN_ST_WAIT_S_ANE:
 810                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
 811                         td->did_nak++;
 812                         td->tt_scheduled = 0;
 813                         goto send_pkt;
 814                 } else if (hcint & (HCINT_ACK | HCINT_NYET)) {
 815                         goto send_cpkt;
 816                 }
 817                 break;
 818
 819         case DWC_CHAN_ST_WAIT_C_ANE:
 820                 if (hcint & HCINT_NYET) {
 821                         goto send_cpkt;
 822                 } else if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
 823                         td->did_nak++;
 824                         td->tt_scheduled = 0;
 825                         goto send_pkt;
 826                 } else if (hcint & HCINT_ACK) {
 827                         td->offset += td->tx_bytes;
 828                         td->remainder -= td->tx_bytes;
 829                         td->toggle = 1;
 830                         goto complete;
 831                 }
 832                 break;
 833
 834         case DWC_CHAN_ST_WAIT_C_PKT:
 835                 goto send_cpkt;
 836
 837         default:
 838                 break;
 839         }
 840         goto busy;
 841
 842 send_pkt:
 843         /* free existing channel, if any */
 844         dwc_otg_host_channel_free(sc, td);
 845
 846         if (sizeof(req) != td->remainder) {
 847                 td->error_any = 1;
 848                 goto complete;
 849         }
 850
 851         if (td->hcsplt != 0) {
 852                 delta = td->tt_start_slot - sc->sc_last_frame_num - 1;
 853                 if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
 854                         td->state = DWC_CHAN_ST_START;
 855                         goto busy;
 856                 }
 857                 delta = sc->sc_last_frame_num - td->tt_start_slot;
 858                 if (delta > 5) {
 859                         /* missed it */
 860                         td->tt_scheduled = 0;
 861                         td->state = DWC_CHAN_ST_START;
 862                         goto busy;
 863                 }
 864         }
 865
 866         /* allocate a new channel */
 867         if (dwc_otg_host_channel_alloc(sc, td, 1)) {
 868                 td->state = DWC_CHAN_ST_START;
 869                 goto busy;
 870         }
 871
 872         if (td->hcsplt != 0) {
 873                 td->hcsplt &= ~HCSPLT_COMPSPLT;
 874                 td->state = DWC_CHAN_ST_WAIT_S_ANE;
 875         } else {
 876                 td->state = DWC_CHAN_ST_WAIT_ANE;
 877         }
 878
 879         usbd_copy_out(td->pc, 0, &req, sizeof(req));
 880
 881         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(td->channel),
 882             (sizeof(req) << HCTSIZ_XFERSIZE_SHIFT) |
 883             (1 << HCTSIZ_PKTCNT_SHIFT) |
 884             (HCTSIZ_PID_SETUP << HCTSIZ_PID_SHIFT));
 885
 886         DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(td->channel), td->hcsplt);
 887
 888         hcchar = td->hcchar;
 889         hcchar &= ~(HCCHAR_EPDIR_IN | HCCHAR_EPTYPE_MASK);
 890         hcchar |= UE_CONTROL << HCCHAR_EPTYPE_SHIFT;
 891
 892         /* must enable channel before writing data to FIFO */
 893         DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(td->channel), hcchar);
 894
 895         /* transfer data into FIFO */
 896         bus_space_write_region_4(sc->sc_io_tag, sc->sc_io_hdl,
 897             DOTG_DFIFO(td->channel), (uint32_t *)&req, sizeof(req) / 4);
 898
 899         /* wait until next slot before trying complete split */
 900         td->tt_complete_slot = sc->sc_last_frame_num + 1;
 901
 902         /* store number of bytes transmitted */
 903         td->tx_bytes = sizeof(req);
 904         goto busy;
 905
 906 send_cpkt:
 907         /* free existing channel, if any */
 908         dwc_otg_host_channel_free(sc, td);
 909
 910         delta = td->tt_complete_slot - sc->sc_last_frame_num - 1;
 911         if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
 912                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
 913                 goto busy;
 914         }
 915         delta = sc->sc_last_frame_num - td->tt_start_slot;
 916         if (delta > DWC_OTG_TT_SLOT_MAX) {
 917                 /* we missed the service interval */
 918                 if (td->ep_type != UE_ISOCHRONOUS)
 919                         td->error_any = 1;
 920                 goto complete;
 921         }
 922         /* allocate a new channel */
 923         if (dwc_otg_host_channel_alloc(sc, td, 0)) {
 924                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
 925                 goto busy;
 926         }
 927
 928         /* wait until next slot before trying complete split */
 929         td->tt_complete_slot = sc->sc_last_frame_num + 1;
 930
 931         td->hcsplt |= HCSPLT_COMPSPLT;
 932         td->state = DWC_CHAN_ST_WAIT_C_ANE;
 933
 934         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(td->channel),
 935             (HCTSIZ_PID_SETUP << HCTSIZ_PID_SHIFT));
 936
 937         DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(td->channel), td->hcsplt);
 938
 939         hcchar = td->hcchar;
 940         hcchar &= ~(HCCHAR_EPDIR_IN | HCCHAR_EPTYPE_MASK);
 941         hcchar |= UE_CONTROL << HCCHAR_EPTYPE_SHIFT;
 942
 943         /* must enable channel before writing data to FIFO */
 944         DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(td->channel), hcchar);
 945
 946 busy:
 947         return (1);     /* busy */
 948
 949 complete:
 950         dwc_otg_host_channel_free(sc, td);
 951         return (0);     /* complete */
 952 }
 953
 954 static uint8_t
 955 dwc_otg_setup_rx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
 956 {
 957         struct usb_device_request req __aligned(4);
 958         uint32_t temp;
 959         uint16_t count;
 960
 961         /* check endpoint status */
 962
 963         if (sc->sc_last_rx_status == 0)
 964                 goto not_complete;
 965
 966         if (GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) != 0)
 967                 goto not_complete;
 968
 969         if ((sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) !=
 970             GRXSTSRD_STP_DATA) {
 971                 if ((sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) !=
 972                     GRXSTSRD_STP_COMPLETE || td->remainder != 0) {
 973                         /* release FIFO */
 974                         dwc_otg_common_rx_ack(sc);
 975                         goto not_complete;
 976                 }
 977                 /* release FIFO */
 978                 dwc_otg_common_rx_ack(sc);
 979                 return (0);     /* complete */
 980         }
 981
 982         if ((sc->sc_last_rx_status & GRXSTSRD_DPID_MASK) !=
 983             GRXSTSRD_DPID_DATA0) {
 984                 /* release FIFO */
 985                 dwc_otg_common_rx_ack(sc);
 986                 goto not_complete;
 987         }
 988
 989         DPRINTFN(5, "GRXSTSR=0x%08x\n", sc->sc_last_rx_status);
 990
 991         /* clear did stall */
 992         td->did_stall = 0;
 993
 994         /* get the packet byte count */
 995         count = GRXSTSRD_BCNT_GET(sc->sc_last_rx_status);
 996
 997         if (count != sizeof(req)) {
 998                 DPRINTFN(0, "Unsupported SETUP packet "
 999                     "length, %d bytes\n", count);
1000                 /* release FIFO */
1001                 dwc_otg_common_rx_ack(sc);
1002                 goto not_complete;
1003         }
1004
1005         /* copy in control request */
1006         memcpy(&req, sc->sc_rx_bounce_buffer, sizeof(req));
1007
1008         /* copy data into real buffer */
1009         usbd_copy_in(td->pc, 0, &req, sizeof(req));
1010
1011         td->offset = sizeof(req);
1012         td->remainder = 0;
1013
1014         /* sneak peek the set address */
1015         if ((req.bmRequestType == UT_WRITE_DEVICE) &&
1016             (req.bRequest == UR_SET_ADDRESS)) {
1017                 /* must write address before ZLP */
1018                 dwc_otg_set_address(sc, req.wValue[0] & 0x7F);
1019         }
1020
1021         /* don't send any data by default */
1022         DWC_OTG_WRITE_4(sc, DOTG_DIEPTSIZ(0), DIEPCTL_EPDIS);
1023         DWC_OTG_WRITE_4(sc, DOTG_DOEPTSIZ(0), DOEPCTL_EPDIS);
1024
1025         /* reset IN endpoint buffer */
1026         dwc_otg_tx_fifo_reset(sc,
1027             GRSTCTL_TXFIFO(0) |
1028             GRSTCTL_TXFFLSH);
1029
1030         /* acknowledge RX status */
1031         dwc_otg_common_rx_ack(sc);
1032         td->did_stall = 1;
1033
1034 not_complete:
1035         /* abort any ongoing transfer, before enabling again */
1036         if (!td->did_stall) {
1037                 td->did_stall = 1;
1038
1039                 DPRINTFN(5, "stalling IN and OUT direction\n");
1040
1041                 temp = sc->sc_out_ctl[0];
1042
1043                 /* set stall after enabling endpoint */
1044                 DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(0),
1045                     temp | DOEPCTL_STALL);
1046
1047                 temp = sc->sc_in_ctl[0];
1048
1049                 /* set stall assuming endpoint is enabled */
1050                 DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(0),
1051                     temp | DIEPCTL_STALL);
1052         }
1053         return (1);                     /* not complete */
1054 }
1055
1056 static uint8_t
1057 dwc_otg_host_rate_check_interrupt(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1058 {
1059         uint8_t delta;
1060
1061         delta = sc->sc_tmr_val - td->tmr_val;
1062         if (delta >= 128)
1063                 return (1);     /* busy */
1064
1065         td->tmr_val = sc->sc_tmr_val + td->tmr_res;
1066
1067         /* set toggle, if any */
1068         if (td->set_toggle) {
1069                 td->set_toggle = 0;
1070                 td->toggle = 1;
1071         }
1072         return (0);
1073 }
1074
1075 static uint8_t
1076 dwc_otg_host_rate_check(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1077 {
1078         if (td->ep_type == UE_ISOCHRONOUS) {
1079                 /* non TT isochronous traffic */
1080                 if ((td->tmr_val != 0) ||
1081                     (sc->sc_last_frame_num & (td->tmr_res - 1))) {
1082                         goto busy;
1083                 }
1084                 td->tmr_val = 1;        /* executed */
1085                 td->toggle = 0;
1086
1087         } else if (td->ep_type == UE_INTERRUPT) {
1088                 if (!td->tt_scheduled)
1089                         goto busy;
1090                 td->tt_scheduled = 0;
1091         } else if (td->did_nak >= DWC_OTG_NAK_MAX) {
1092                 goto busy;
1093         } else if (td->set_toggle) {
1094                 td->set_toggle = 0;
1095                 td->toggle = 1;
1096         }
1097         return (0);
1098 busy:
1099         return (1);
1100 }
1101
1102 static uint8_t
1103 dwc_otg_host_data_rx_sub(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1104 {
1105         uint32_t count;
1106         uint8_t channel;
1107
1108         /* check endpoint status */
1109         if (sc->sc_last_rx_status == 0)
1110                 goto busy;
1111
1112         channel = td->channel;
1113         if (channel >= DWC_OTG_MAX_CHANNELS)
1114                 goto busy;
1115
1116         if (GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) != channel)
1117                 goto busy;
1118
1119         switch (sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) {
1120         case GRXSTSRH_IN_DATA:
1121
1122                 DPRINTF("DATA ST=%d STATUS=0x%08x\n",
1123                     (int)td->state, (int)sc->sc_last_rx_status);
1124
1125                 if (sc->sc_chan_state[channel].hcint & HCINT_SOFTWARE_ONLY) {
1126                         /*
1127                          * When using SPLIT transactions on interrupt
1128                          * endpoints, sometimes data occurs twice.
1129                          */
1130                         DPRINTF("Data already received\n");
1131                         break;
1132                 }
1133
1134                 /* get the packet byte count */
1135                 count = GRXSTSRD_BCNT_GET(sc->sc_last_rx_status);
1136
1137                 /* check for isochronous transfer or high-speed bandwidth endpoint */
1138                 if (td->ep_type == UE_ISOCHRONOUS || td->max_packet_count > 1) {
1139                         if ((sc->sc_last_rx_status & GRXSTSRD_DPID_MASK) != GRXSTSRD_DPID_DATA0) {
1140                                 td->tt_xactpos = HCSPLT_XACTPOS_MIDDLE;
1141                         } else {
1142                                 td->tt_xactpos = HCSPLT_XACTPOS_BEGIN;
1143
1144                                 /* verify the packet byte count */
1145                                 if (count < td->max_packet_size) {
1146                                         /* we have a short packet */
1147                                         td->short_pkt = 1;
1148                                         td->got_short = 1;
1149                                 }
1150                         }
1151                         td->toggle = 0;
1152                 } else {
1153                         /* verify the packet byte count */
1154                         if (count != td->max_packet_size) {
1155                                 if (count < td->max_packet_size) {
1156                                         /* we have a short packet */
1157                                         td->short_pkt = 1;
1158                                         td->got_short = 1;
1159                                 } else {
1160                                         /* invalid USB packet */
1161                                         td->error_any = 1;
1162
1163                                         /* release FIFO */
1164                                         dwc_otg_common_rx_ack(sc);
1165                                         goto complete;
1166                                 }
1167                         }
1168                         td->toggle ^= 1;
1169                         td->tt_scheduled = 0;
1170                 }
1171
1172                 /* verify the packet byte count */
1173                 if (count > td->remainder) {
1174                         /* invalid USB packet */
1175                         td->error_any = 1;
1176
1177                         /* release FIFO */
1178                         dwc_otg_common_rx_ack(sc);
1179                         goto complete;
1180                 }
1181
1182                 usbd_copy_in(td->pc, td->offset,
1183                     sc->sc_rx_bounce_buffer, count);
1184
1185                 td->remainder -= count;
1186                 td->offset += count;
1187                 sc->sc_chan_state[channel].hcint |= HCINT_SOFTWARE_ONLY;
1188                 break;
1189         default:
1190                 break;
1191         }
1192         /* release FIFO */
1193         dwc_otg_common_rx_ack(sc);
1194 busy:
1195         return (0);
1196 complete:
1197         return (1);
1198 }
1199
1200 static uint8_t
1201 dwc_otg_host_data_rx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1202 {
1203         uint32_t hcint;
1204         uint32_t hcchar;
1205         uint8_t delta;
1206         uint8_t channel;
1207
1208         channel = td->channel;
1209
1210         if (channel < DWC_OTG_MAX_CHANNELS) {
1211                 hcint = sc->sc_chan_state[channel].hcint;
1212
1213                 DPRINTF("CH=%d ST=%d HCINT=0x%08x HCCHAR=0x%08x HCTSIZ=0x%08x\n",
1214                     channel, td->state, hcint,
1215                     DWC_OTG_READ_4(sc, DOTG_HCCHAR(channel)),
1216                     DWC_OTG_READ_4(sc, DOTG_HCTSIZ(channel)));
1217
1218                 /* check interrupt bits */
1219                 if (hcint & (HCINT_RETRY |
1220                     HCINT_ACK | HCINT_NYET)) {
1221                         /* give success bits priority over failure bits */
1222                 } else if (hcint & HCINT_STALL) {
1223                         DPRINTF("CH=%d STALL\n", channel);
1224                         td->error_stall = 1;
1225                         td->error_any = 1;
1226                         goto complete;
1227                 } else if (hcint & HCINT_ERRORS) {
1228                         DPRINTF("CH=%d ERROR\n", channel);
1229                         td->errcnt++;
1230                         if (td->hcsplt != 0 || td->errcnt >= 3) {
1231                                 if (td->ep_type != UE_ISOCHRONOUS) {
1232                                         td->error_any = 1;
1233                                         goto complete;
1234                                 }
1235                         }
1236                 }
1237
1238                 /* check channels for data, if any */
1239                 if (dwc_otg_host_data_rx_sub(sc, td))
1240                         goto complete;
1241
1242                 /* refresh interrupt status */
1243                 hcint = sc->sc_chan_state[channel].hcint;
1244
1245                 if (hcint & (HCINT_ERRORS | HCINT_RETRY |
1246                     HCINT_ACK | HCINT_NYET)) {
1247                         if (!(hcint & HCINT_ERRORS))
1248                                 td->errcnt = 0;
1249                 }
1250         } else {
1251                 hcint = 0;
1252         }
1253
1254         switch (td->state) {
1255         case DWC_CHAN_ST_START:
1256                 if (td->hcsplt != 0)
1257                         goto receive_spkt;
1258                 else
1259                         goto receive_pkt;
1260
1261         case DWC_CHAN_ST_WAIT_ANE:
1262                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1263                         if (td->ep_type == UE_INTERRUPT) {
1264                                 /*
1265                                  * The USB specification does not
1266                                  * mandate a particular data toggle
1267                                  * value for USB INTERRUPT
1268                                  * transfers. Switch the data toggle
1269                                  * value to receive the packet
1270                                  * correctly:
1271                                  */
1272                                 if (hcint & HCINT_DATATGLERR) {
1273                                         DPRINTF("Retrying packet due to "
1274                                             "data toggle error\n");
1275                                         td->toggle ^= 1;
1276                                         goto receive_pkt;
1277                                 }
1278                         }
1279                         td->did_nak++;
1280                         td->tt_scheduled = 0;
1281                         if (td->hcsplt != 0)
1282                                 goto receive_spkt;
1283                         else
1284                                 goto receive_pkt;
1285                 } else if (hcint & HCINT_NYET) {
1286                         if (td->hcsplt != 0) {
1287                                 /* try again */
1288                                 goto receive_pkt;
1289                         } else {
1290                                 /* not a valid token for IN endpoints */
1291                                 td->error_any = 1;
1292                                 goto complete;
1293                         }
1294                 } else if (hcint & HCINT_ACK) {
1295                         /* wait for data - ACK arrived first */
1296                         if (!(hcint & HCINT_SOFTWARE_ONLY))
1297                                 goto busy;
1298
1299                         if (td->ep_type == UE_ISOCHRONOUS) {
1300                                 /* check if we are complete */
1301                                 if ((td->remainder == 0) ||
1302                                     (td->tt_xactpos == HCSPLT_XACTPOS_BEGIN)) {
1303                                         goto complete;
1304                                 }
1305                                 /* get another packet */
1306                                 goto receive_pkt;
1307                         } else {
1308                                 /* check if we are complete */
1309                                 if ((td->remainder == 0) || (td->got_short != 0)) {
1310                                         if (td->short_pkt)
1311                                                 goto complete;
1312
1313                                         /*
1314                                          * Else need to receive a zero length
1315                                          * packet.
1316                                          */
1317                                 }
1318                                 td->tt_scheduled = 0;
1319                                 td->did_nak = 0;
1320                                 if (td->hcsplt != 0)
1321                                         goto receive_spkt;
1322                                 else
1323                                         goto receive_pkt;
1324                         }
1325                 }
1326                 break;
1327
1328         case DWC_CHAN_ST_WAIT_S_ANE:
1329                 /*
1330                  * NOTE: The DWC OTG hardware provides a fake ACK in
1331                  * case of interrupt and isochronous transfers:
1332                  */
1333                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1334                         td->did_nak++;
1335                         td->tt_scheduled = 0;
1336                         goto receive_spkt;
1337                 } else if (hcint & HCINT_NYET) {
1338                         td->tt_scheduled = 0;
1339                         goto receive_spkt;
1340                 } else if (hcint & HCINT_ACK) {
1341                         td->did_nak = 0;
1342                         goto receive_pkt;
1343                 }
1344                 break;
1345
1346         case DWC_CHAN_ST_WAIT_C_PKT:
1347                 goto receive_pkt;
1348
1349         default:
1350                 break;
1351         }
1352         goto busy;
1353
1354 receive_pkt:
1355         /* free existing channel, if any */
1356         dwc_otg_host_channel_free(sc, td);
1357
1358         if (td->hcsplt != 0) {
1359                 delta = td->tt_complete_slot - sc->sc_last_frame_num - 1;
1360                 if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
1361                         td->state = DWC_CHAN_ST_WAIT_C_PKT;
1362                         goto busy;
1363                 }
1364                 delta = sc->sc_last_frame_num - td->tt_start_slot;
1365                 if (delta > DWC_OTG_TT_SLOT_MAX) {
1366                         if (td->ep_type != UE_ISOCHRONOUS) {
1367                                 /* we missed the service interval */
1368                                 td->error_any = 1;
1369                         }
1370                         goto complete;
1371                 }
1372                 /* complete split */
1373                 td->hcsplt |= HCSPLT_COMPSPLT;
1374         } else if (td->tt_xactpos == HCSPLT_XACTPOS_BEGIN &&
1375             dwc_otg_host_rate_check(sc, td)) {
1376                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
1377                 goto busy;
1378         }
1379
1380         /* allocate a new channel */
1381         if (dwc_otg_host_channel_alloc(sc, td, 0)) {
1382                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
1383                 goto busy;
1384         }
1385
1386         channel = td->channel;
1387
1388         /* set toggle, if any */
1389         if (td->set_toggle) {
1390                 td->set_toggle = 0;
1391                 td->toggle = 1;
1392         }
1393
1394         td->state = DWC_CHAN_ST_WAIT_ANE;
1395
1396         /* receive one packet */
1397         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1398             (td->max_packet_size << HCTSIZ_XFERSIZE_SHIFT) |
1399             (1 << HCTSIZ_PKTCNT_SHIFT) |
1400             (td->toggle ? (HCTSIZ_PID_DATA1 << HCTSIZ_PID_SHIFT) :
1401             (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT)));
1402
1403         DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(channel), td->hcsplt);
1404
1405         hcchar = td->hcchar;
1406         hcchar |= HCCHAR_EPDIR_IN;
1407
1408         /* receive complete split ASAP */
1409         if ((sc->sc_last_frame_num & 1) != 0)
1410                 hcchar |= HCCHAR_ODDFRM;
1411         else
1412                 hcchar &= ~HCCHAR_ODDFRM;
1413
1414         /* must enable channel before data can be received */
1415         DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(channel), hcchar);
1416
1417         /* wait until next slot before trying complete split */
1418         td->tt_complete_slot = sc->sc_last_frame_num + 1;
1419
1420         goto busy;
1421
1422 receive_spkt:
1423         /* free existing channel(s), if any */
1424         dwc_otg_host_channel_free(sc, td);
1425
1426         delta = td->tt_start_slot - sc->sc_last_frame_num - 1;
1427         if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
1428                 td->state = DWC_CHAN_ST_START;
1429                 goto busy;
1430         }
1431         delta = sc->sc_last_frame_num - td->tt_start_slot;
1432         if (delta > 5) {
1433                 /* missed it */
1434                 td->tt_scheduled = 0;
1435                 td->state = DWC_CHAN_ST_START;
1436                 goto busy;
1437         }
1438
1439         /* allocate a new channel */
1440         if (dwc_otg_host_channel_alloc(sc, td, 0)) {
1441                 td->state = DWC_CHAN_ST_START;
1442                 goto busy;
1443         }
1444
1445         channel = td->channel;
1446
1447         td->hcsplt &= ~HCSPLT_COMPSPLT;
1448         td->state = DWC_CHAN_ST_WAIT_S_ANE;
1449
1450         /* receive one packet */
1451         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1452             (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT));
1453
1454         DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(channel), td->hcsplt);
1455
1456         /* send after next SOF event */
1457         if ((sc->sc_last_frame_num & 1) == 0)
1458                 td->hcchar |= HCCHAR_ODDFRM;
1459         else
1460                 td->hcchar &= ~HCCHAR_ODDFRM;
1461
1462         hcchar = td->hcchar;
1463         hcchar |= HCCHAR_EPDIR_IN;
1464
1465         /* wait until next slot before trying complete split */
1466         td->tt_complete_slot = sc->sc_last_frame_num + 1;
1467
1468         /* must enable channel before data can be received */
1469         DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(channel), hcchar);
1470 busy:
1471         return (1);     /* busy */
1472
1473 complete:
1474         dwc_otg_host_channel_free(sc, td);
1475         return (0);     /* complete */
1476 }
1477
1478 static uint8_t
1479 dwc_otg_data_rx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1480 {
1481         uint32_t temp;
1482         uint16_t count;
1483         uint8_t got_short;
1484
1485         got_short = 0;
1486
1487         /* check endpoint status */
1488         if (sc->sc_last_rx_status == 0)
1489                 goto not_complete;
1490
1491         if (GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) != td->ep_no)
1492                 goto not_complete;
1493
1494         /* check for SETUP packet */
1495         if ((sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) ==
1496             GRXSTSRD_STP_DATA ||
1497             (sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) ==
1498             GRXSTSRD_STP_COMPLETE) {
1499                 if (td->remainder == 0) {
1500                         /*
1501                          * We are actually complete and have
1502                          * received the next SETUP
1503                          */
1504                         DPRINTFN(5, "faking complete\n");
1505                         return (0);     /* complete */
1506                 }
1507                 /*
1508                  * USB Host Aborted the transfer.
1509                  */
1510                 td->error_any = 1;
1511                 return (0);             /* complete */
1512         }
1513
1514         if ((sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) !=
1515             GRXSTSRD_OUT_DATA) {
1516                 /* release FIFO */
1517                 dwc_otg_common_rx_ack(sc);
1518                 goto not_complete;
1519         }
1520
1521         /* get the packet byte count */
1522         count = GRXSTSRD_BCNT_GET(sc->sc_last_rx_status);
1523
1524         /* verify the packet byte count */
1525         if (count != td->max_packet_size) {
1526                 if (count < td->max_packet_size) {
1527                         /* we have a short packet */
1528                         td->short_pkt = 1;
1529                         got_short = 1;
1530                 } else {
1531                         /* invalid USB packet */
1532                         td->error_any = 1;
1533
1534                         /* release FIFO */
1535                         dwc_otg_common_rx_ack(sc);
1536                         return (0);     /* we are complete */
1537                 }
1538         }
1539         /* verify the packet byte count */
1540         if (count > td->remainder) {
1541                 /* invalid USB packet */
1542                 td->error_any = 1;
1543
1544                 /* release FIFO */
1545                 dwc_otg_common_rx_ack(sc);
1546                 return (0);             /* we are complete */
1547         }
1548
1549         usbd_copy_in(td->pc, td->offset, sc->sc_rx_bounce_buffer, count);
1550         td->remainder -= count;
1551         td->offset += count;
1552
1553         /* release FIFO */
1554         dwc_otg_common_rx_ack(sc);
1555
1556         temp = sc->sc_out_ctl[td->ep_no];
1557
1558         /* check for isochronous mode */
1559         if ((temp & DIEPCTL_EPTYPE_MASK) ==
1560             (DIEPCTL_EPTYPE_ISOC << DIEPCTL_EPTYPE_SHIFT)) {
1561                 /* toggle odd or even frame bit */
1562                 if (temp & DIEPCTL_SETD1PID) {
1563                         temp &= ~DIEPCTL_SETD1PID;
1564                         temp |= DIEPCTL_SETD0PID;
1565                 } else {
1566                         temp &= ~DIEPCTL_SETD0PID;
1567                         temp |= DIEPCTL_SETD1PID;
1568                 }
1569                 sc->sc_out_ctl[td->ep_no] = temp;
1570         }
1571
1572         /* check if we are complete */
1573         if ((td->remainder == 0) || got_short) {
1574                 if (td->short_pkt) {
1575                         /* we are complete */
1576                         return (0);
1577                 }
1578                 /* else need to receive a zero length packet */
1579         }
1580
1581 not_complete:
1582
1583         /* enable SETUP and transfer complete interrupt */
1584         if (td->ep_no == 0) {
1585                 DWC_OTG_WRITE_4(sc, DOTG_DOEPTSIZ(0),
1586                     DXEPTSIZ_SET_MULTI(3) |
1587                     DXEPTSIZ_SET_NPKT(1) |
1588                     DXEPTSIZ_SET_NBYTES(td->max_packet_size));
1589         } else {
1590                 /* allow reception of multiple packets */
1591                 DWC_OTG_WRITE_4(sc, DOTG_DOEPTSIZ(td->ep_no),
1592                     DXEPTSIZ_SET_MULTI(1) |
1593                     DXEPTSIZ_SET_NPKT(4) |
1594                     DXEPTSIZ_SET_NBYTES(4 *
1595                         ((td->max_packet_size + 3) & ~3)));
1596         }
1597         temp = sc->sc_out_ctl[td->ep_no];
1598         DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(td->ep_no), temp |
1599             DOEPCTL_EPENA | DOEPCTL_CNAK);
1600
1601         return (1);                     /* not complete */
1602 }
1603
1604 static uint8_t
1605 dwc_otg_host_data_tx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1606 {
1607         uint32_t count;
1608         uint32_t hcint;
1609         uint32_t hcchar;
1610         uint8_t delta;
1611         uint8_t channel;
1612
1613         dwc_otg_host_dump_rx(sc, td);
1614
1615         channel = td->channel;
1616
1617         if (channel < DWC_OTG_MAX_CHANNELS) {
1618                 hcint = sc->sc_chan_state[channel].hcint;
1619
1620                 DPRINTF("CH=%d ST=%d HCINT=0x%08x HCCHAR=0x%08x HCTSIZ=0x%08x\n",
1621                     channel, td->state, hcint,
1622                     DWC_OTG_READ_4(sc, DOTG_HCCHAR(channel)),
1623                     DWC_OTG_READ_4(sc, DOTG_HCTSIZ(channel)));
1624
1625                 if (hcint & (HCINT_RETRY |
1626                     HCINT_ACK | HCINT_NYET)) {
1627                         /* give success bits priority over failure bits */
1628                 } else if (hcint & HCINT_STALL) {
1629                         DPRINTF("CH=%d STALL\n", channel);
1630                         td->error_stall = 1;
1631                         td->error_any = 1;
1632                         goto complete;
1633                 } else if (hcint & HCINT_ERRORS) {
1634                         DPRINTF("CH=%d ERROR\n", channel);
1635                         td->errcnt++;
1636                         if (td->hcsplt != 0 || td->errcnt >= 3) {
1637                                 td->error_any = 1;
1638                                 goto complete;
1639                         }
1640                 }
1641
1642                 if (hcint & (HCINT_ERRORS | HCINT_RETRY |
1643                     HCINT_ACK | HCINT_NYET)) {
1644
1645                         if (!(hcint & HCINT_ERRORS))
1646                                 td->errcnt = 0;
1647                 }
1648         } else {
1649                 hcint = 0;
1650         }
1651
1652         switch (td->state) {
1653         case DWC_CHAN_ST_START:
1654                 goto send_pkt;
1655
1656         case DWC_CHAN_ST_WAIT_ANE:
1657                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1658                         td->did_nak++;
1659                         td->tt_scheduled = 0;
1660                         goto send_pkt;
1661                 } else if (hcint & (HCINT_ACK | HCINT_NYET)) {
1662                         td->offset += td->tx_bytes;
1663                         td->remainder -= td->tx_bytes;
1664                         td->toggle ^= 1;
1665                         td->did_nak = 0;
1666                         td->tt_scheduled = 0;
1667
1668                         /* check remainder */
1669                         if (td->remainder == 0) {
1670                                 if (td->short_pkt)
1671                                         goto complete;
1672
1673                                 /*
1674                                  * Else we need to transmit a short
1675                                  * packet:
1676                                  */
1677                         }
1678                         goto send_pkt;
1679                 }
1680                 break;
1681
1682         case DWC_CHAN_ST_WAIT_S_ANE:
1683                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1684                         td->did_nak++;
1685                         td->tt_scheduled = 0;
1686                         goto send_pkt;
1687                 } else if (hcint & (HCINT_ACK | HCINT_NYET)) {
1688                         td->did_nak = 0;
1689                         goto send_cpkt;
1690                 }
1691                 break;
1692
1693         case DWC_CHAN_ST_WAIT_C_ANE:
1694                 if (hcint & HCINT_NYET) {
1695                         goto send_cpkt;
1696                 } else if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1697                         td->did_nak++;
1698                         td->tt_scheduled = 0;
1699                         goto send_pkt;
1700                 } else if (hcint & HCINT_ACK) {
1701                         td->offset += td->tx_bytes;
1702                         td->remainder -= td->tx_bytes;
1703                         td->toggle ^= 1;
1704                         td->did_nak = 0;
1705                         td->tt_scheduled = 0;
1706
1707                         /* check remainder */
1708                         if (td->remainder == 0) {
1709                                 if (td->short_pkt)
1710                                         goto complete;
1711
1712                                 /* else we need to transmit a short packet */
1713                         }
1714                         goto send_pkt;
1715                 }
1716                 break;
1717
1718         case DWC_CHAN_ST_WAIT_C_PKT:
1719                 goto send_cpkt;
1720
1721         case DWC_CHAN_ST_TX_WAIT_ISOC:
1722
1723                 /* Check if isochronous OUT traffic is complete */
1724                 if ((hcint & HCINT_HCH_DONE_MASK) == 0)
1725                         break;
1726
1727                 td->offset += td->tx_bytes;
1728                 td->remainder -= td->tx_bytes;
1729
1730                 if (td->hcsplt != 0 || td->remainder == 0)
1731                         goto complete;
1732
1733                 /* check for next packet */
1734                 if (td->max_packet_count > 1)
1735                         td->tt_xactpos++;
1736
1737                 /* free existing channel, if any */
1738                 dwc_otg_host_channel_free(sc, td);
1739
1740                 td->state = DWC_CHAN_ST_TX_PKT_ISOC;
1741
1742                 /* FALLTHROUGH */
1743
1744         case DWC_CHAN_ST_TX_PKT_ISOC:
1745                 if (dwc_otg_host_channel_alloc(sc, td, 1))
1746                         break;
1747                 channel = td->channel;
1748                 goto send_isoc_pkt;
1749         default:
1750                 break;
1751         }
1752         goto busy;
1753
1754 send_pkt:
1755         /* free existing channel(s), if any */
1756         dwc_otg_host_channel_free(sc, td);
1757
1758         if (td->hcsplt != 0) {
1759                 delta = td->tt_start_slot - sc->sc_last_frame_num - 1;
1760                 if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
1761                         td->state = DWC_CHAN_ST_START;
1762                         goto busy;
1763                 }
1764                 delta = sc->sc_last_frame_num - td->tt_start_slot;
1765                 if (delta > 5) {
1766                         /* missed it */
1767                         td->tt_scheduled = 0;
1768                         td->state = DWC_CHAN_ST_START;
1769                         goto busy;
1770                 }
1771         } else if (dwc_otg_host_rate_check(sc, td)) {
1772                 td->state = DWC_CHAN_ST_START;
1773                 goto busy;
1774         }
1775
1776         /* allocate a new channel */
1777         if (dwc_otg_host_channel_alloc(sc, td, 1)) {
1778                 td->state = DWC_CHAN_ST_START;
1779                 goto busy;
1780         }
1781
1782         channel = td->channel;
1783
1784         /* set toggle, if any */
1785         if (td->set_toggle) {
1786                 td->set_toggle = 0;
1787                 td->toggle = 1;
1788         }
1789
1790         if (td->ep_type == UE_ISOCHRONOUS) {
1791 send_isoc_pkt:
1792                 /* Isochronous OUT transfers don't have any ACKs */
1793                 td->state = DWC_CHAN_ST_TX_WAIT_ISOC;
1794                 td->hcsplt &= ~HCSPLT_COMPSPLT;
1795                 if (td->hcsplt != 0) {
1796                         /* get maximum transfer length */
1797                         count = td->remainder;
1798                         if (count > HCSPLT_XACTLEN_BURST) {
1799                                 DPRINTF("TT overflow\n");
1800                                 td->error_any = 1;
1801                                 goto complete;
1802                         }
1803                         /* Update transaction position */
1804                         td->hcsplt &= ~HCSPLT_XACTPOS_MASK;
1805                         td->hcsplt |= (HCSPLT_XACTPOS_ALL << HCSPLT_XACTPOS_SHIFT);
1806                 } else {
1807                         /* send one packet at a time */
1808                         count = td->max_packet_size;
1809                         if (td->remainder < count) {
1810                                 /* we have a short packet */
1811                                 td->short_pkt = 1;
1812                                 count = td->remainder;
1813                         }
1814                 }
1815         } else if (td->hcsplt != 0) {
1816
1817                 td->hcsplt &= ~HCSPLT_COMPSPLT;
1818
1819                 /* Wait for ACK/NAK/ERR from TT */
1820                 td->state = DWC_CHAN_ST_WAIT_S_ANE;
1821
1822                 /* send one packet at a time */
1823                 count = td->max_packet_size;
1824                 if (td->remainder < count) {
1825                         /* we have a short packet */
1826                         td->short_pkt = 1;
1827                         count = td->remainder;
1828                 }
1829         } else {
1830                 /* Wait for ACK/NAK/STALL from device */
1831                 td->state = DWC_CHAN_ST_WAIT_ANE;
1832
1833                 /* send one packet at a time */
1834                 count = td->max_packet_size;
1835                 if (td->remainder < count) {
1836                         /* we have a short packet */
1837                         td->short_pkt = 1;
1838                         count = td->remainder;
1839                 }
1840         }
1841
1842         /* check for High-Speed multi-packets */
1843         if ((td->hcsplt == 0) && (td->max_packet_count > 1)) {
1844                 if (td->npkt == 0) {
1845                         if (td->remainder >= (3 * td->max_packet_size))
1846                                 td->npkt = 3;
1847                         else if (td->remainder >= (2 * td->max_packet_size))
1848                                 td->npkt = 2;
1849                         else
1850                                 td->npkt = 1;
1851
1852                         if (td->npkt > td->max_packet_count)
1853                                 td->npkt = td->max_packet_count;
1854
1855                         td->tt_xactpos = 1;     /* overload */
1856                 }
1857                 if (td->tt_xactpos == td->npkt) {
1858                         if (td->npkt == 1) {
1859                                 DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1860                                     (count << HCTSIZ_XFERSIZE_SHIFT) |
1861                                     (1 << HCTSIZ_PKTCNT_SHIFT) |
1862                                     (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT));
1863                         } else if (td->npkt == 2) {
1864                                 DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1865                                     (count << HCTSIZ_XFERSIZE_SHIFT) |
1866                                     (1 << HCTSIZ_PKTCNT_SHIFT) |
1867                                     (HCTSIZ_PID_DATA1 << HCTSIZ_PID_SHIFT));
1868                         } else {
1869                                 DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1870                                     (count << HCTSIZ_XFERSIZE_SHIFT) |
1871                                     (1 << HCTSIZ_PKTCNT_SHIFT) |
1872                                     (HCTSIZ_PID_DATA2 << HCTSIZ_PID_SHIFT));
1873                         }
1874                         td->npkt = 0;
1875                 } else {
1876                         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1877                             (count << HCTSIZ_XFERSIZE_SHIFT) |
1878                             (1 << HCTSIZ_PKTCNT_SHIFT) |
1879                             (HCTSIZ_PID_MDATA << HCTSIZ_PID_SHIFT));
1880                 }
1881         } else {
1882                 /* TODO: HCTSIZ_DOPNG */
1883
1884                 DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1885                     (count << HCTSIZ_XFERSIZE_SHIFT) |
1886                     (1 << HCTSIZ_PKTCNT_SHIFT) |
1887                     (td->toggle ? (HCTSIZ_PID_DATA1 << HCTSIZ_PID_SHIFT) :
1888                     (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT)));
1889         }
1890
1891         DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(channel), td->hcsplt);
1892
1893         hcchar = td->hcchar;
1894         hcchar &= ~HCCHAR_EPDIR_IN;
1895
1896         /* send after next SOF event */
1897         if ((sc->sc_last_frame_num & 1) == 0)
1898                 hcchar |= HCCHAR_ODDFRM;
1899         else
1900                 hcchar &= ~HCCHAR_ODDFRM;
1901
1902         /* must enable before writing data to FIFO */
1903         DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(channel), hcchar);
1904
1905         if (count != 0) {
1906
1907                 /* clear topmost word before copy */
1908                 sc->sc_tx_bounce_buffer[(count - 1) / 4] = 0;
1909
1910                 /* copy out data */
1911                 usbd_copy_out(td->pc, td->offset,
1912                     sc->sc_tx_bounce_buffer, count);
1913
1914                 /* transfer data into FIFO */
1915                 bus_space_write_region_4(sc->sc_io_tag, sc->sc_io_hdl,
1916                     DOTG_DFIFO(channel),
1917                     sc->sc_tx_bounce_buffer, (count + 3) / 4);
1918         }
1919
1920         /* store number of bytes transmitted */
1921         td->tx_bytes = count;
1922         goto busy;
1923
1924 send_cpkt:
1925         /* free existing channel, if any */
1926         dwc_otg_host_channel_free(sc, td);
1927
1928         delta = td->tt_complete_slot - sc->sc_last_frame_num - 1;
1929         if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
1930                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
1931                 goto busy;
1932         }
1933         delta = sc->sc_last_frame_num - td->tt_start_slot;
1934         if (delta > DWC_OTG_TT_SLOT_MAX) {
1935                 /* we missed the service interval */
1936                 if (td->ep_type != UE_ISOCHRONOUS)
1937                         td->error_any = 1;
1938                 goto complete;
1939         }
1940
1941         /* allocate a new channel */
1942         if (dwc_otg_host_channel_alloc(sc, td, 0)) {
1943                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
1944                 goto busy;
1945         }
1946
1947         channel = td->channel;
1948
1949         td->hcsplt |= HCSPLT_COMPSPLT;
1950         td->state = DWC_CHAN_ST_WAIT_C_ANE;
1951
1952         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1953             (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT));
1954
1955         DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(channel), td->hcsplt);
1956
1957         hcchar = td->hcchar;
1958         hcchar &= ~HCCHAR_EPDIR_IN;
1959
1960         /* receive complete split ASAP */
1961         if ((sc->sc_last_frame_num & 1) != 0)
1962                 hcchar |= HCCHAR_ODDFRM;
1963         else
1964                 hcchar &= ~HCCHAR_ODDFRM;
1965
1966         /* must enable channel before data can be received */
1967         DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(channel), hcchar);
1968
1969         /* wait until next slot before trying complete split */
1970         td->tt_complete_slot = sc->sc_last_frame_num + 1;
1971 busy:
1972         return (1);     /* busy */
1973
1974 complete:
1975         dwc_otg_host_channel_free(sc, td);
1976         return (0);     /* complete */
1977 }
1978
1979 static uint8_t
1980 dwc_otg_data_tx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1981 {
1982         uint32_t max_buffer;
1983         uint32_t count;
1984         uint32_t fifo_left;
1985         uint32_t mpkt;
1986         uint32_t temp;
1987         uint8_t to;
1988
1989         to = 3;                         /* don't loop forever! */
1990
1991         max_buffer = sc->sc_hw_ep_profile[td->ep_no].max_buffer;
1992
1993 repeat:
1994         /* check for for endpoint 0 data */
1995
1996         temp = sc->sc_last_rx_status;
1997
1998         if ((td->ep_no == 0) && (temp != 0) &&
1999             (GRXSTSRD_CHNUM_GET(temp) == 0)) {
2000
2001                 if ((temp & GRXSTSRD_PKTSTS_MASK) !=
2002                     GRXSTSRD_STP_DATA &&
2003                     (temp & GRXSTSRD_PKTSTS_MASK) !=
2004                     GRXSTSRD_STP_COMPLETE) {
2005
2006                         /* dump data - wrong direction */
2007                         dwc_otg_common_rx_ack(sc);
2008                 } else {
2009                         /*
2010                          * The current transfer was cancelled
2011                          * by the USB Host:
2012                          */
2013                         td->error_any = 1;
2014                         return (0);             /* complete */
2015                 }
2016         }
2017
2018         /* fill in more TX data, if possible */
2019         if (td->tx_bytes != 0) {
2020
2021                 uint16_t cpkt;
2022
2023                 /* check if packets have been transferred */
2024                 temp = DWC_OTG_READ_4(sc, DOTG_DIEPTSIZ(td->ep_no));
2025
2026                 /* get current packet number */
2027                 cpkt = DXEPTSIZ_GET_NPKT(temp);
2028
2029                 if (cpkt >= td->npkt) {
2030                         fifo_left = 0;
2031                 } else {
2032                         if (max_buffer != 0) {
2033                                 fifo_left = (td->npkt - cpkt) *
2034                                     td->max_packet_size;
2035
2036                                 if (fifo_left > max_buffer)
2037                                         fifo_left = max_buffer;
2038                         } else {
2039                                 fifo_left = td->max_packet_size;
2040                         }
2041                 }
2042
2043                 count = td->tx_bytes;
2044                 if (count > fifo_left)
2045                         count = fifo_left;
2046
2047                 if (count != 0) {
2048
2049                         /* clear topmost word before copy */
2050                         sc->sc_tx_bounce_buffer[(count - 1) / 4] = 0;
2051
2052                         /* copy out data */
2053                         usbd_copy_out(td->pc, td->offset,
2054                             sc->sc_tx_bounce_buffer, count);
2055
2056                         /* transfer data into FIFO */
2057                         bus_space_write_region_4(sc->sc_io_tag, sc->sc_io_hdl,
2058                             DOTG_DFIFO(td->ep_no),
2059                             sc->sc_tx_bounce_buffer, (count + 3) / 4);
2060
2061                         td->tx_bytes -= count;
2062                         td->remainder -= count;
2063                         td->offset += count;
2064                         td->npkt = cpkt;
2065                 }
2066                 if (td->tx_bytes != 0)
2067                         goto not_complete;
2068
2069                 /* check remainder */
2070                 if (td->remainder == 0) {
2071                         if (td->short_pkt)
2072                                 return (0);     /* complete */
2073
2074                         /* else we need to transmit a short packet */
2075                 }
2076         }
2077
2078         if (!to--)
2079                 goto not_complete;
2080
2081         /* check if not all packets have been transferred */
2082         temp = DWC_OTG_READ_4(sc, DOTG_DIEPTSIZ(td->ep_no));
2083
2084         if (DXEPTSIZ_GET_NPKT(temp) != 0) {
2085
2086                 DPRINTFN(5, "busy ep=%d npkt=%d DIEPTSIZ=0x%08x "
2087                     "DIEPCTL=0x%08x\n", td->ep_no,
2088                     DXEPTSIZ_GET_NPKT(temp),
2089                     temp, DWC_OTG_READ_4(sc, DOTG_DIEPCTL(td->ep_no)));
2090
2091                 goto not_complete;
2092         }
2093
2094         DPRINTFN(5, "rem=%u ep=%d\n", td->remainder, td->ep_no);
2095
2096         /* try to optimise by sending more data */
2097         if ((max_buffer != 0) && ((td->max_packet_size & 3) == 0)) {
2098
2099                 /* send multiple packets at the same time */
2100                 mpkt = max_buffer / td->max_packet_size;
2101
2102                 if (mpkt > 0x3FE)
2103                         mpkt = 0x3FE;
2104
2105                 count = td->remainder;
2106                 if (count > 0x7FFFFF)
2107                         count = 0x7FFFFF - (0x7FFFFF % td->max_packet_size);
2108
2109                 td->npkt = count / td->max_packet_size;
2110
2111                 /*
2112                  * NOTE: We could use 0x3FE instead of "mpkt" in the
2113                  * check below to get more throughput, but then we
2114                  * have a dependency towards non-generic chip features
2115                  * to disable the TX-FIFO-EMPTY interrupts on a per
2116                  * endpoint basis. Increase the maximum buffer size of
2117                  * the IN endpoint to increase the performance.
2118                  */
2119                 if (td->npkt > mpkt) {
2120                         td->npkt = mpkt;
2121                         count = td->max_packet_size * mpkt;
2122                 } else if ((count == 0) || (count % td->max_packet_size)) {
2123                         /* we are transmitting a short packet */
2124                         td->npkt++;
2125                         td->short_pkt = 1;
2126                 }
2127         } else {
2128                 /* send one packet at a time */
2129                 mpkt = 1;
2130                 count = td->max_packet_size;
2131                 if (td->remainder < count) {
2132                         /* we have a short packet */
2133                         td->short_pkt = 1;
2134                         count = td->remainder;
2135                 }
2136                 td->npkt = 1;
2137         }
2138         DWC_OTG_WRITE_4(sc, DOTG_DIEPTSIZ(td->ep_no),
2139             DXEPTSIZ_SET_MULTI(1) |
2140             DXEPTSIZ_SET_NPKT(td->npkt) |
2141             DXEPTSIZ_SET_NBYTES(count));
2142
2143         /* make room for buffering */
2144         td->npkt += mpkt;
2145
2146         temp = sc->sc_in_ctl[td->ep_no];
2147
2148         /* check for isochronous mode */
2149         if ((temp & DIEPCTL_EPTYPE_MASK) ==
2150             (DIEPCTL_EPTYPE_ISOC << DIEPCTL_EPTYPE_SHIFT)) {
2151                 /* toggle odd or even frame bit */
2152                 if (temp & DIEPCTL_SETD1PID) {
2153                         temp &= ~DIEPCTL_SETD1PID;
2154                         temp |= DIEPCTL_SETD0PID;
2155                 } else {
2156                         temp &= ~DIEPCTL_SETD0PID;
2157                         temp |= DIEPCTL_SETD1PID;
2158                 }
2159                 sc->sc_in_ctl[td->ep_no] = temp;
2160         }
2161
2162         /* must enable before writing data to FIFO */
2163         DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(td->ep_no), temp |
2164             DIEPCTL_EPENA | DIEPCTL_CNAK);
2165
2166         td->tx_bytes = count;
2167
2168         /* check remainder */
2169         if (td->tx_bytes == 0 &&
2170             td->remainder == 0) {
2171                 if (td->short_pkt)
2172                         return (0);     /* complete */
2173
2174                 /* else we need to transmit a short packet */
2175         }
2176         goto repeat;
2177
2178 not_complete:
2179         return (1);                     /* not complete */
2180 }
2181
2182 static uint8_t
2183 dwc_otg_data_tx_sync(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
2184 {
2185         uint32_t temp;
2186
2187         /*
2188          * If all packets are transferred we are complete:
2189          */
2190         temp = DWC_OTG_READ_4(sc, DOTG_DIEPTSIZ(td->ep_no));
2191
2192         /* check that all packets have been transferred */
2193         if (DXEPTSIZ_GET_NPKT(temp) != 0) {
2194                 DPRINTFN(5, "busy ep=%d\n", td->ep_no);
2195                 goto not_complete;
2196         }
2197         return (0);
2198
2199 not_complete:
2200
2201         /* we only want to know if there is a SETUP packet or free IN packet */
2202
2203         temp = sc->sc_last_rx_status;
2204
2205         if ((td->ep_no == 0) && (temp != 0) &&
2206             (GRXSTSRD_CHNUM_GET(temp) == 0)) {
2207
2208                 if ((temp & GRXSTSRD_PKTSTS_MASK) ==
2209                     GRXSTSRD_STP_DATA ||
2210                     (temp & GRXSTSRD_PKTSTS_MASK) ==
2211                     GRXSTSRD_STP_COMPLETE) {
2212                         DPRINTFN(5, "faking complete\n");
2213                         /*
2214                          * Race condition: We are complete!
2215                          */
2216                         return (0);
2217                 } else {
2218                         /* dump data - wrong direction */
2219                         dwc_otg_common_rx_ack(sc);
2220                 }
2221         }
2222         return (1);                     /* not complete */
2223 }
2224
2225 static void
2226 dwc_otg_xfer_do_fifo(struct dwc_otg_softc *sc, struct usb_xfer *xfer)
2227 {
2228         struct dwc_otg_td *td;
2229         uint8_t toggle;
2230         uint8_t tmr_val;
2231         uint8_t tmr_res;
2232
2233         DPRINTFN(9, "\n");
2234
2235         td = xfer->td_transfer_cache;
2236         if (td == NULL)
2237                 return;
2238
2239         while (1) {
2240                 if ((td->func) (sc, td)) {
2241                         /* operation in progress */
2242                         break;
2243                 }
2244                 if (((void *)td) == xfer->td_transfer_last) {
2245                         goto done;
2246                 }
2247                 if (td->error_any) {
2248                         goto done;
2249                 } else if (td->remainder > 0) {
2250                         /*
2251                          * We had a short transfer. If there is no alternate
2252                          * next, stop processing !
2253                          */
2254                         if (!td->alt_next)
2255                                 goto done;
2256                 }
2257
2258                 /*
2259                  * Fetch the next transfer descriptor and transfer
2260                  * some flags to the next transfer descriptor
2261                  */
2262                 tmr_res = td->tmr_res;
2263                 tmr_val = td->tmr_val;
2264                 toggle = td->toggle;
2265                 td = td->obj_next;
2266                 xfer->td_transfer_cache = td;
2267                 td->toggle = toggle;    /* transfer toggle */
2268                 td->tmr_res = tmr_res;
2269                 td->tmr_val = tmr_val;
2270         }
2271         return;
2272
2273 done:
2274         xfer->td_transfer_cache = NULL;
2275         sc->sc_xfer_complete = 1;
2276 }
2277
2278 static uint8_t
2279 dwc_otg_xfer_do_complete_locked(struct dwc_otg_softc *sc, struct usb_xfer *xfer)
2280 {
2281         struct dwc_otg_td *td;
2282
2283         DPRINTFN(9, "\n");
2284
2285         td = xfer->td_transfer_cache;
2286         if (td == NULL) {
2287                 /* compute all actual lengths */
2288                 dwc_otg_standard_done(xfer);
2289                 return (1);
2290         }
2291         return (0);
2292 }
2293
2294 static void
2295 dwc_otg_timer(void *_sc)
2296 {
2297         struct dwc_otg_softc *sc = _sc;
2298         struct usb_xfer *xfer;
2299         struct dwc_otg_td *td;
2300
2301         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
2302
2303         DPRINTF("\n");
2304
2305         USB_BUS_SPIN_LOCK(&sc->sc_bus);
2306
2307         /* increment timer value */
2308         sc->sc_tmr_val++;
2309
2310         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
2311                 td = xfer->td_transfer_cache;
2312                 if (td != NULL) {
2313                         /* reset NAK counter */
2314                         td->did_nak = 0;
2315                 }
2316         }
2317
2318         /* enable SOF interrupt, which will poll jobs */
2319         dwc_otg_enable_sof_irq(sc);
2320
2321         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
2322
2323         if (sc->sc_timer_active) {
2324                 /* restart timer */
2325                 usb_callout_reset(&sc->sc_timer,
2326                     hz / (1000 / DWC_OTG_HOST_TIMER_RATE),
2327                     &dwc_otg_timer, sc);
2328         }
2329 }
2330
2331 static void
2332 dwc_otg_timer_start(struct dwc_otg_softc *sc)
2333 {
2334         if (sc->sc_timer_active != 0)
2335                 return;
2336
2337         sc->sc_timer_active = 1;
2338
2339         /* restart timer */
2340         usb_callout_reset(&sc->sc_timer,
2341             hz / (1000 / DWC_OTG_HOST_TIMER_RATE),
2342             &dwc_otg_timer, sc);
2343 }
2344
2345 static void
2346 dwc_otg_timer_stop(struct dwc_otg_softc *sc)
2347 {
2348         if (sc->sc_timer_active == 0)
2349                 return;
2350
2351         sc->sc_timer_active = 0;
2352
2353         /* stop timer */
2354         usb_callout_stop(&sc->sc_timer);
2355 }
2356
2357 static void
2358 dwc_otg_host_channel_disable(struct dwc_otg_softc *sc, uint8_t x)
2359 {
2360         uint32_t hcchar;
2361
2362         hcchar = DWC_OTG_READ_4(sc, DOTG_HCCHAR(x));
2363
2364         /* disable host channel, if any */
2365         if (hcchar & (HCCHAR_CHENA | HCCHAR_CHDIS)) {
2366                 /* disable channel */
2367                 DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(x),
2368                     HCCHAR_CHENA | HCCHAR_CHDIS);
2369                 /* wait for chip to get its brains in order */
2370                 sc->sc_chan_state[x].wait_sof = 2;
2371         }
2372
2373         /* release TX FIFO usage, if any */
2374         sc->sc_tx_cur_p_level -= sc->sc_chan_state[x].tx_p_size;
2375         sc->sc_tx_cur_np_level -= sc->sc_chan_state[x].tx_np_size;
2376
2377         /* don't release TX FIFO usage twice */
2378         sc->sc_chan_state[x].tx_p_size = 0;
2379         sc->sc_chan_state[x].tx_np_size = 0;
2380 }
2381
2382 static uint16_t
2383 dwc_otg_compute_isoc_rx_tt_slot(struct dwc_otg_tt_info *pinfo)
2384 {
2385         if (pinfo->slot_index < DWC_OTG_TT_SLOT_MAX)
2386                 pinfo->slot_index++;
2387         return (pinfo->slot_index);
2388 }
2389
2390 static uint8_t
2391 dwc_otg_update_host_transfer_schedule_locked(struct dwc_otg_softc *sc)
2392 {
2393         TAILQ_HEAD(, usb_xfer) head;
2394         struct usb_xfer *xfer;
2395         struct usb_xfer *xfer_next;
2396         struct dwc_otg_td *td;
2397         uint16_t temp;
2398         uint16_t slot;
2399         uint8_t x;
2400
2401         temp = DWC_OTG_READ_4(sc, DOTG_HFNUM) & DWC_OTG_FRAME_MASK;
2402
2403         if (sc->sc_last_frame_num == temp)
2404                 return (0);
2405
2406         sc->sc_last_frame_num = temp;
2407
2408         TAILQ_INIT(&head);
2409
2410         for (x = 0; x != sc->sc_host_ch_max; x++) {
2411                 if (sc->sc_chan_state[x].wait_sof == 0)
2412                         continue;
2413
2414                 sc->sc_needsof = 1;
2415                 if (--(sc->sc_chan_state[x].wait_sof) == 0)
2416                         dwc_otg_host_channel_disable(sc, x);
2417         }
2418
2419         if ((temp & 7) == 0) {
2420
2421                 /* reset the schedule */
2422                 memset(sc->sc_tt_info, 0, sizeof(sc->sc_tt_info));
2423
2424                 TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2425                         td = xfer->td_transfer_cache;
2426                         if (td == NULL || td->ep_type != UE_ISOCHRONOUS)
2427                                 continue;
2428
2429                         /* check for IN direction */
2430                         if ((td->hcchar & HCCHAR_EPDIR_IN) != 0)
2431                                 continue;
2432
2433                         /* execute more frames */
2434                         td->tmr_val = 0;
2435
2436                         sc->sc_needsof = 1;
2437
2438                         if (td->hcsplt == 0 || td->tt_scheduled != 0)
2439                                 continue;
2440
2441                         /* compute slot */
2442                         slot = dwc_otg_compute_isoc_rx_tt_slot(
2443                             sc->sc_tt_info + td->tt_index);
2444                         if (slot > 3) {
2445                                 /*
2446                                  * Not enough time to get complete
2447                                  * split executed.
2448                                  */
2449                                 continue;
2450                         }
2451                         /* Delayed start */
2452                         td->tt_start_slot = temp + slot;
2453                         td->tt_scheduled = 1;
2454                         TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2455                         TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2456                 }
2457
2458                 TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2459                         td = xfer->td_transfer_cache;
2460                         if (td == NULL || td->ep_type != UE_ISOCHRONOUS)
2461                                 continue;
2462
2463                         /* check for OUT direction */
2464                         if ((td->hcchar & HCCHAR_EPDIR_IN) == 0)
2465                                 continue;
2466
2467                         /* execute more frames */
2468                         td->tmr_val = 0;
2469
2470                         sc->sc_needsof = 1;
2471
2472                         if (td->hcsplt == 0 || td->tt_scheduled != 0)
2473                                 continue;
2474
2475                         /* Start ASAP */
2476                         td->tt_start_slot = temp;
2477                         td->tt_scheduled = 1;
2478                         TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2479                         TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2480                 }
2481
2482                 TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2483                         td = xfer->td_transfer_cache;
2484                         if (td == NULL || td->ep_type != UE_INTERRUPT)
2485                                 continue;
2486
2487                         if (td->tt_scheduled != 0) {
2488                                 sc->sc_needsof = 1;
2489                                 continue;
2490                         }
2491
2492                         if (dwc_otg_host_rate_check_interrupt(sc, td))
2493                                 continue;
2494
2495                         if (td->hcsplt == 0) {
2496                                 sc->sc_needsof = 1;
2497                                 td->tt_scheduled = 1;
2498                                 continue;
2499                         }
2500
2501                         /* start ASAP */
2502                         td->tt_start_slot = temp;
2503                         sc->sc_needsof = 1;
2504                         td->tt_scheduled = 1;
2505                         TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2506                         TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2507                 }
2508
2509                 TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2510                         td = xfer->td_transfer_cache;
2511                         if (td == NULL ||
2512                             td->ep_type != UE_CONTROL ||
2513                             td->did_nak >= DWC_OTG_NAK_MAX) {
2514                                 continue;
2515                         }
2516
2517                         sc->sc_needsof = 1;
2518
2519                         if (td->hcsplt == 0 || td->tt_scheduled != 0)
2520                                 continue;
2521
2522                         /* start ASAP */
2523                         td->tt_start_slot = temp;
2524                         td->tt_scheduled = 1;
2525                         TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2526                         TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2527                 }
2528         }
2529         if ((temp & 7) < 6) {
2530                 TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2531                         td = xfer->td_transfer_cache;
2532                         if (td == NULL ||
2533                             td->ep_type != UE_BULK ||
2534                             td->did_nak >= DWC_OTG_NAK_MAX) {
2535                                 continue;
2536                         }
2537
2538                         sc->sc_needsof = 1;
2539
2540                         if (td->hcsplt == 0 || td->tt_scheduled != 0)
2541                                 continue;
2542
2543                         /* start ASAP */
2544                         td->tt_start_slot = temp;
2545                         td->tt_scheduled = 1;
2546                         TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2547                         TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2548                 }
2549         }
2550
2551         /* Put TT transfers in execution order at the end */
2552         TAILQ_CONCAT(&sc->sc_bus.intr_q.head, &head, wait_entry);
2553
2554         /* move all TT transfers in front, keeping the current order */
2555         TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2556                 td = xfer->td_transfer_cache;
2557                 if (td == NULL || td->hcsplt == 0)
2558                         continue;
2559                 TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2560                 TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2561         }
2562         TAILQ_CONCAT(&head, &sc->sc_bus.intr_q.head, wait_entry);
2563         TAILQ_CONCAT(&sc->sc_bus.intr_q.head, &head, wait_entry);
2564
2565         /* put non-TT BULK transfers last */
2566         TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2567                 td = xfer->td_transfer_cache;
2568                 if (td == NULL || td->hcsplt != 0 || td->ep_type != UE_BULK)
2569                         continue;
2570                 TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2571                 TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2572         }
2573         TAILQ_CONCAT(&sc->sc_bus.intr_q.head, &head, wait_entry);
2574
2575         if ((temp & 7) == 0) {
2576
2577                 DPRINTFN(12, "SOF interrupt #%d, needsof=%d\n",
2578                     (int)temp, (int)sc->sc_needsof);
2579
2580                 /* update SOF IRQ mask */
2581                 if (sc->sc_irq_mask & GINTMSK_SOFMSK) {
2582                         if (sc->sc_needsof == 0) {
2583                                 sc->sc_irq_mask &= ~GINTMSK_SOFMSK;
2584                                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2585                         }
2586                 } else {
2587                         if (sc->sc_needsof != 0) {
2588                                 sc->sc_irq_mask |= GINTMSK_SOFMSK;
2589                                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2590                         }
2591                 }
2592
2593                 /* clear need SOF flag */
2594                 sc->sc_needsof = 0;
2595         }
2596         return (1);
2597 }
2598
2599 static void
2600 dwc_otg_interrupt_poll_locked(struct dwc_otg_softc *sc)
2601 {
2602         struct usb_xfer *xfer;
2603         uint32_t temp;
2604         uint8_t got_rx_status;
2605         uint8_t x;
2606
2607 repeat:
2608         /* get all channel interrupts */
2609         for (x = 0; x != sc->sc_host_ch_max; x++) {
2610                 temp = DWC_OTG_READ_4(sc, DOTG_HCINT(x));
2611                 if (temp != 0) {
2612                         DWC_OTG_WRITE_4(sc, DOTG_HCINT(x), temp);
2613                         temp &= ~HCINT_SOFTWARE_ONLY;
2614                         sc->sc_chan_state[x].hcint |= temp;
2615                 }
2616         }
2617
2618         if (sc->sc_last_rx_status == 0) {
2619
2620                 temp = DWC_OTG_READ_4(sc, DOTG_GINTSTS);
2621                 if (temp & GINTSTS_RXFLVL) {
2622                         /* pop current status */
2623                         sc->sc_last_rx_status =
2624                             DWC_OTG_READ_4(sc, DOTG_GRXSTSPD);
2625                 }
2626
2627                 if (sc->sc_last_rx_status != 0) {
2628
2629                         uint8_t ep_no;
2630
2631                         temp = sc->sc_last_rx_status &
2632                             GRXSTSRD_PKTSTS_MASK;
2633
2634                         /* non-data messages we simply skip */
2635                         if (temp != GRXSTSRD_STP_DATA &&
2636                             temp != GRXSTSRD_STP_COMPLETE &&
2637                             temp != GRXSTSRD_OUT_DATA) {
2638                                 dwc_otg_common_rx_ack(sc);
2639                                 goto repeat;
2640                         }
2641
2642                         temp = GRXSTSRD_BCNT_GET(
2643                             sc->sc_last_rx_status);
2644                         ep_no = GRXSTSRD_CHNUM_GET(
2645                             sc->sc_last_rx_status);
2646
2647                         /* receive data, if any */
2648                         if (temp != 0) {
2649                                 DPRINTF("Reading %d bytes from ep %d\n", temp, ep_no);
2650                                 bus_space_read_region_4(sc->sc_io_tag, sc->sc_io_hdl,
2651                                     DOTG_DFIFO(ep_no),
2652                                     sc->sc_rx_bounce_buffer, (temp + 3) / 4);
2653                         }
2654
2655                         /* check if we should dump the data */
2656                         if (!(sc->sc_active_rx_ep & (1U << ep_no))) {
2657                                 dwc_otg_common_rx_ack(sc);
2658                                 goto repeat;
2659                         }
2660
2661                         got_rx_status = 1;
2662
2663                         DPRINTFN(5, "RX status = 0x%08x: ch=%d pid=%d bytes=%d sts=%d\n",
2664                             sc->sc_last_rx_status, ep_no,
2665                             (sc->sc_last_rx_status >> 15) & 3,
2666                             GRXSTSRD_BCNT_GET(sc->sc_last_rx_status),
2667                             (sc->sc_last_rx_status >> 17) & 15);
2668                 } else {
2669                         got_rx_status = 0;
2670                 }
2671         } else {
2672                 uint8_t ep_no;
2673
2674                 ep_no = GRXSTSRD_CHNUM_GET(
2675                     sc->sc_last_rx_status);
2676
2677                 /* check if we should dump the data */
2678                 if (!(sc->sc_active_rx_ep & (1U << ep_no))) {
2679                         dwc_otg_common_rx_ack(sc);
2680                         goto repeat;
2681                 }
2682
2683                 got_rx_status = 1;
2684         }
2685
2686         /* execute FIFOs */
2687         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry)
2688                 dwc_otg_xfer_do_fifo(sc, xfer);
2689
2690         if (got_rx_status) {
2691                 /* check if data was consumed */
2692                 if (sc->sc_last_rx_status == 0)
2693                         goto repeat;
2694
2695                 /* disable RX FIFO level interrupt */
2696                 sc->sc_irq_mask &= ~GINTMSK_RXFLVLMSK;
2697                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2698         }
2699
2700         if (sc->sc_flags.status_device_mode == 0 && sc->sc_xfer_complete == 0) {
2701                 /* update host transfer schedule, so that new transfers can be issued */
2702                 if (dwc_otg_update_host_transfer_schedule_locked(sc))
2703                         goto repeat;
2704         }
2705 }
2706
2707 static void
2708 dwc_otg_interrupt_complete_locked(struct dwc_otg_softc *sc)
2709 {
2710         struct usb_xfer *xfer;
2711 repeat:
2712         /* scan for completion events */
2713         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
2714                 if (dwc_otg_xfer_do_complete_locked(sc, xfer))
2715                         goto repeat;
2716         }
2717 }
2718
2719 static void
2720 dwc_otg_vbus_interrupt(struct dwc_otg_softc *sc, uint8_t is_on)
2721 {
2722         DPRINTFN(5, "vbus = %u\n", is_on);
2723
2724         if (is_on) {
2725                 if (!sc->sc_flags.status_vbus) {
2726                         sc->sc_flags.status_vbus = 1;
2727
2728                         /* complete root HUB interrupt endpoint */
2729
2730                         dwc_otg_root_intr(sc);
2731                 }
2732         } else {
2733                 if (sc->sc_flags.status_vbus) {
2734                         sc->sc_flags.status_vbus = 0;
2735                         sc->sc_flags.status_bus_reset = 0;
2736                         sc->sc_flags.status_suspend = 0;
2737                         sc->sc_flags.change_suspend = 0;
2738                         sc->sc_flags.change_connect = 1;
2739
2740                         /* complete root HUB interrupt endpoint */
2741
2742                         dwc_otg_root_intr(sc);
2743                 }
2744         }
2745 }
2746
2747 int
2748 dwc_otg_filter_interrupt(void *arg)
2749 {
2750         struct dwc_otg_softc *sc = arg;
2751         int retval = FILTER_HANDLED;
2752         uint32_t status;
2753
2754         USB_BUS_SPIN_LOCK(&sc->sc_bus);
2755
2756         /* read and clear interrupt status */
2757         status = DWC_OTG_READ_4(sc, DOTG_GINTSTS);
2758
2759         /* clear interrupts we are handling here */
2760         DWC_OTG_WRITE_4(sc, DOTG_GINTSTS, status & ~DWC_OTG_MSK_GINT_THREAD_IRQ);
2761
2762         /* check for USB state change interrupts */
2763         if ((status & DWC_OTG_MSK_GINT_THREAD_IRQ) != 0)
2764                 retval = FILTER_SCHEDULE_THREAD;
2765
2766         /* clear all IN endpoint interrupts */
2767         if (status & GINTSTS_IEPINT) {
2768                 uint32_t temp;
2769                 uint8_t x;
2770
2771                 for (x = 0; x != sc->sc_dev_in_ep_max; x++) {
2772                         temp = DWC_OTG_READ_4(sc, DOTG_DIEPINT(x));
2773                         if (temp & DIEPMSK_XFERCOMPLMSK) {
2774                                 DWC_OTG_WRITE_4(sc, DOTG_DIEPINT(x),
2775                                     DIEPMSK_XFERCOMPLMSK);
2776                         }
2777                 }
2778         }
2779
2780         /* poll FIFOs, if any */
2781         dwc_otg_interrupt_poll_locked(sc);
2782
2783         if (sc->sc_xfer_complete != 0)
2784                 retval = FILTER_SCHEDULE_THREAD;
2785
2786         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
2787
2788         return (retval);
2789 }
2790
2791 void
2792 dwc_otg_interrupt(void *arg)
2793 {
2794         struct dwc_otg_softc *sc = arg;
2795         uint32_t status;
2796
2797         USB_BUS_LOCK(&sc->sc_bus);
2798         USB_BUS_SPIN_LOCK(&sc->sc_bus);
2799
2800         /* read and clear interrupt status */
2801         status = DWC_OTG_READ_4(sc, DOTG_GINTSTS);
2802
2803         /* clear interrupts we are handling here */
2804         DWC_OTG_WRITE_4(sc, DOTG_GINTSTS, status & DWC_OTG_MSK_GINT_THREAD_IRQ);
2805
2806         DPRINTFN(14, "GINTSTS=0x%08x HAINT=0x%08x HFNUM=0x%08x\n",
2807             status, DWC_OTG_READ_4(sc, DOTG_HAINT),
2808             DWC_OTG_READ_4(sc, DOTG_HFNUM));
2809
2810         if (status & GINTSTS_USBRST) {
2811
2812                 /* set correct state */
2813                 sc->sc_flags.status_device_mode = 1;
2814                 sc->sc_flags.status_bus_reset = 0;
2815                 sc->sc_flags.status_suspend = 0;
2816                 sc->sc_flags.change_suspend = 0;
2817                 sc->sc_flags.change_connect = 1;
2818
2819                 /* Disable SOF interrupt */
2820                 sc->sc_irq_mask &= ~GINTMSK_SOFMSK;
2821                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2822
2823                 /* complete root HUB interrupt endpoint */
2824                 dwc_otg_root_intr(sc);
2825         }
2826
2827         /* check for any bus state change interrupts */
2828         if (status & GINTSTS_ENUMDONE) {
2829
2830                 uint32_t temp;
2831
2832                 DPRINTFN(5, "end of reset\n");
2833
2834                 /* set correct state */
2835                 sc->sc_flags.status_device_mode = 1;
2836                 sc->sc_flags.status_bus_reset = 1;
2837                 sc->sc_flags.status_suspend = 0;
2838                 sc->sc_flags.change_suspend = 0;
2839                 sc->sc_flags.change_connect = 1;
2840                 sc->sc_flags.status_low_speed = 0;
2841                 sc->sc_flags.port_enabled = 1;
2842
2843                 /* reset FIFOs */
2844                 (void) dwc_otg_init_fifo(sc, DWC_MODE_DEVICE);
2845
2846                 /* reset function address */
2847                 dwc_otg_set_address(sc, 0);
2848
2849                 /* figure out enumeration speed */
2850                 temp = DWC_OTG_READ_4(sc, DOTG_DSTS);
2851                 if (DSTS_ENUMSPD_GET(temp) == DSTS_ENUMSPD_HI)
2852                         sc->sc_flags.status_high_speed = 1;
2853                 else
2854                         sc->sc_flags.status_high_speed = 0;
2855
2856                 /*
2857                  * Disable resume and SOF interrupt, and enable
2858                  * suspend and RX frame interrupt:
2859                  */
2860                 sc->sc_irq_mask &= ~(GINTMSK_WKUPINTMSK | GINTMSK_SOFMSK);
2861                 sc->sc_irq_mask |= GINTMSK_USBSUSPMSK;
2862                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2863
2864                 /* complete root HUB interrupt endpoint */
2865                 dwc_otg_root_intr(sc);
2866         }
2867
2868         if (status & GINTSTS_PRTINT) {
2869                 uint32_t hprt;
2870
2871                 hprt = DWC_OTG_READ_4(sc, DOTG_HPRT);
2872
2873                 /* clear change bits */
2874                 DWC_OTG_WRITE_4(sc, DOTG_HPRT, (hprt & (
2875                     HPRT_PRTPWR | HPRT_PRTENCHNG |
2876                     HPRT_PRTCONNDET | HPRT_PRTOVRCURRCHNG)) |
2877                     sc->sc_hprt_val);
2878
2879                 DPRINTFN(12, "GINTSTS=0x%08x, HPRT=0x%08x\n", status, hprt);
2880
2881                 sc->sc_flags.status_device_mode = 0;
2882
2883                 if (hprt & HPRT_PRTCONNSTS)
2884                         sc->sc_flags.status_bus_reset = 1;
2885                 else
2886                         sc->sc_flags.status_bus_reset = 0;
2887
2888                 if (hprt & HPRT_PRTENCHNG)
2889                         sc->sc_flags.change_enabled = 1;
2890
2891                 if (hprt & HPRT_PRTENA)
2892                         sc->sc_flags.port_enabled = 1;
2893                 else
2894                         sc->sc_flags.port_enabled = 0;
2895
2896                 if (hprt & HPRT_PRTOVRCURRCHNG)
2897                         sc->sc_flags.change_over_current = 1;
2898
2899                 if (hprt & HPRT_PRTOVRCURRACT)
2900                         sc->sc_flags.port_over_current = 1;
2901                 else
2902                         sc->sc_flags.port_over_current = 0;
2903
2904                 if (hprt & HPRT_PRTPWR)
2905                         sc->sc_flags.port_powered = 1;
2906                 else
2907                         sc->sc_flags.port_powered = 0;
2908
2909                 if (((hprt & HPRT_PRTSPD_MASK)
2910                     >> HPRT_PRTSPD_SHIFT) == HPRT_PRTSPD_LOW)
2911                         sc->sc_flags.status_low_speed = 1;
2912                 else
2913                         sc->sc_flags.status_low_speed = 0;
2914
2915                 if (((hprt & HPRT_PRTSPD_MASK)
2916                     >> HPRT_PRTSPD_SHIFT) == HPRT_PRTSPD_HIGH)
2917                         sc->sc_flags.status_high_speed = 1;
2918                 else
2919                         sc->sc_flags.status_high_speed = 0;
2920
2921                 if (hprt & HPRT_PRTCONNDET)
2922                         sc->sc_flags.change_connect = 1;
2923
2924                 if (hprt & HPRT_PRTSUSP)
2925                         dwc_otg_suspend_irq(sc);
2926                 else
2927                         dwc_otg_resume_irq(sc);
2928
2929                 /* complete root HUB interrupt endpoint */
2930                 dwc_otg_root_intr(sc);
2931
2932                 /* update host frame interval */
2933                 dwc_otg_update_host_frame_interval(sc);
2934         }
2935
2936         /*
2937          * If resume and suspend is set at the same time we interpret
2938          * that like RESUME. Resume is set when there is at least 3
2939          * milliseconds of inactivity on the USB BUS.
2940          */
2941         if (status & GINTSTS_WKUPINT) {
2942
2943                 DPRINTFN(5, "resume interrupt\n");
2944
2945                 dwc_otg_resume_irq(sc);
2946
2947         } else if (status & GINTSTS_USBSUSP) {
2948
2949                 DPRINTFN(5, "suspend interrupt\n");
2950
2951                 dwc_otg_suspend_irq(sc);
2952         }
2953         /* check VBUS */
2954         if (status & (GINTSTS_USBSUSP |
2955             GINTSTS_USBRST |
2956             GINTMSK_OTGINTMSK |
2957             GINTSTS_SESSREQINT)) {
2958                 uint32_t temp;
2959
2960                 temp = DWC_OTG_READ_4(sc, DOTG_GOTGCTL);
2961
2962                 DPRINTFN(5, "GOTGCTL=0x%08x\n", temp);
2963
2964                 dwc_otg_vbus_interrupt(sc,
2965                     (temp & (GOTGCTL_ASESVLD | GOTGCTL_BSESVLD)) ? 1 : 0);
2966         }
2967
2968         if (sc->sc_xfer_complete != 0) {
2969                 sc->sc_xfer_complete = 0;
2970
2971                 /* complete FIFOs, if any */
2972                 dwc_otg_interrupt_complete_locked(sc);
2973
2974                 if (sc->sc_flags.status_device_mode == 0) {
2975                         /* update host transfer schedule, so that new transfers can be issued */
2976                         if (dwc_otg_update_host_transfer_schedule_locked(sc))
2977                                 dwc_otg_interrupt_poll_locked(sc);
2978                 }
2979         }
2980         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
2981         USB_BUS_UNLOCK(&sc->sc_bus);
2982 }
2983
2984 static void
2985 dwc_otg_setup_standard_chain_sub(struct dwc_otg_std_temp *temp)
2986 {
2987         struct dwc_otg_td *td;
2988
2989         /* get current Transfer Descriptor */
2990         td = temp->td_next;
2991         temp->td = td;
2992
2993         /* prepare for next TD */
2994         temp->td_next = td->obj_next;
2995
2996         /* fill out the Transfer Descriptor */
2997         td->func = temp->func;
2998         td->pc = temp->pc;
2999         td->offset = temp->offset;
3000         td->remainder = temp->len;
3001         td->tx_bytes = 0;
3002         td->error_any = 0;
3003         td->error_stall = 0;
3004         td->npkt = 0;
3005         td->did_stall = temp->did_stall;
3006         td->short_pkt = temp->short_pkt;
3007         td->alt_next = temp->setup_alt_next;
3008         td->set_toggle = 0;
3009         td->got_short = 0;
3010         td->did_nak = 0;
3011         td->channel = DWC_OTG_MAX_CHANNELS;
3012         td->state = 0;
3013         td->errcnt = 0;
3014         td->tt_scheduled = 0;
3015         td->tt_xactpos = HCSPLT_XACTPOS_BEGIN;
3016 }
3017
3018 static void
3019 dwc_otg_setup_standard_chain(struct usb_xfer *xfer)
3020 {
3021         struct dwc_otg_std_temp temp;
3022         struct dwc_otg_td *td;
3023         uint32_t x;
3024         uint8_t need_sync;
3025         uint8_t is_host;
3026
3027         DPRINTFN(9, "addr=%d endpt=%d sumlen=%d speed=%d\n",
3028             xfer->address, UE_GET_ADDR(xfer->endpointno),
3029             xfer->sumlen, usbd_get_speed(xfer->xroot->udev));
3030
3031         temp.max_frame_size = xfer->max_frame_size;
3032
3033         td = xfer->td_start[0];
3034         xfer->td_transfer_first = td;
3035         xfer->td_transfer_cache = td;
3036
3037         /* setup temp */
3038
3039         temp.pc = NULL;
3040         temp.td = NULL;
3041         temp.td_next = xfer->td_start[0];
3042         temp.offset = 0;
3043         temp.setup_alt_next = xfer->flags_int.short_frames_ok ||
3044             xfer->flags_int.isochronous_xfr;
3045         temp.did_stall = !xfer->flags_int.control_stall;
3046
3047         is_host = (xfer->xroot->udev->flags.usb_mode == USB_MODE_HOST);
3048
3049         /* check if we should prepend a setup message */
3050
3051         if (xfer->flags_int.control_xfr) {
3052                 if (xfer->flags_int.control_hdr) {
3053
3054                         if (is_host)
3055                                 temp.func = &dwc_otg_host_setup_tx;
3056                         else
3057                                 temp.func = &dwc_otg_setup_rx;
3058
3059                         temp.len = xfer->frlengths[0];
3060                         temp.pc = xfer->frbuffers + 0;
3061                         temp.short_pkt = temp.len ? 1 : 0;
3062
3063                         /* check for last frame */
3064                         if (xfer->nframes == 1) {
3065                                 /* no STATUS stage yet, SETUP is last */
3066                                 if (xfer->flags_int.control_act)
3067                                         temp.setup_alt_next = 0;
3068                         }
3069
3070                         dwc_otg_setup_standard_chain_sub(&temp);
3071                 }
3072                 x = 1;
3073         } else {
3074                 x = 0;
3075         }
3076
3077         if (x != xfer->nframes) {
3078                 if (xfer->endpointno & UE_DIR_IN) {
3079                         if (is_host) {
3080                                 temp.func = &dwc_otg_host_data_rx;
3081                                 need_sync = 0;
3082                         } else {
3083                                 temp.func = &dwc_otg_data_tx;
3084                                 need_sync = 1;
3085                         }
3086                 } else {
3087                         if (is_host) {
3088                                 temp.func = &dwc_otg_host_data_tx;
3089                                 need_sync = 0;
3090                         } else {
3091                                 temp.func = &dwc_otg_data_rx;
3092                                 need_sync = 0;
3093                         }
3094                 }
3095
3096                 /* setup "pc" pointer */
3097                 temp.pc = xfer->frbuffers + x;
3098         } else {
3099                 need_sync = 0;
3100         }
3101         while (x != xfer->nframes) {
3102
3103                 /* DATA0 / DATA1 message */
3104
3105                 temp.len = xfer->frlengths[x];
3106
3107                 x++;
3108
3109                 if (x == xfer->nframes) {
3110                         if (xfer->flags_int.control_xfr) {
3111                                 if (xfer->flags_int.control_act) {
3112                                         temp.setup_alt_next = 0;
3113                                 }
3114                         } else {
3115                                 temp.setup_alt_next = 0;
3116                         }
3117                 }
3118                 if (temp.len == 0) {
3119
3120                         /* make sure that we send an USB packet */
3121
3122                         temp.short_pkt = 0;
3123
3124                 } else {
3125
3126                         /* regular data transfer */
3127
3128                         temp.short_pkt = (xfer->flags.force_short_xfer ? 0 : 1);
3129                 }
3130
3131                 dwc_otg_setup_standard_chain_sub(&temp);
3132
3133                 if (xfer->flags_int.isochronous_xfr) {
3134                         temp.offset += temp.len;
3135                 } else {
3136                         /* get next Page Cache pointer */
3137                         temp.pc = xfer->frbuffers + x;
3138                 }
3139         }
3140
3141         if (xfer->flags_int.control_xfr) {
3142
3143                 /* always setup a valid "pc" pointer for status and sync */
3144                 temp.pc = xfer->frbuffers + 0;
3145                 temp.len = 0;
3146                 temp.short_pkt = 0;
3147                 temp.setup_alt_next = 0;
3148
3149                 /* check if we need to sync */
3150                 if (need_sync) {
3151                         /* we need a SYNC point after TX */
3152                         temp.func = &dwc_otg_data_tx_sync;
3153                         dwc_otg_setup_standard_chain_sub(&temp);
3154                 }
3155
3156                 /* check if we should append a status stage */
3157                 if (!xfer->flags_int.control_act) {
3158
3159                         /*
3160                          * Send a DATA1 message and invert the current
3161                          * endpoint direction.
3162                          */
3163                         if (xfer->endpointno & UE_DIR_IN) {
3164                                 if (is_host) {
3165                                         temp.func = &dwc_otg_host_data_tx;
3166                                         need_sync = 0;
3167                                 } else {
3168                                         temp.func = &dwc_otg_data_rx;
3169                                         need_sync = 0;
3170                                 }
3171                         } else {
3172                                 if (is_host) {
3173                                         temp.func = &dwc_otg_host_data_rx;
3174                                         need_sync = 0;
3175                                 } else {
3176                                         temp.func = &dwc_otg_data_tx;
3177                                         need_sync = 1;
3178                                 }
3179                         }
3180
3181                         dwc_otg_setup_standard_chain_sub(&temp);
3182
3183                         /* data toggle should be DATA1 */
3184                         td = temp.td;
3185                         td->set_toggle = 1;
3186
3187                         if (need_sync) {
3188                                 /* we need a SYNC point after TX */
3189                                 temp.func = &dwc_otg_data_tx_sync;
3190                                 dwc_otg_setup_standard_chain_sub(&temp);
3191                         }
3192                 }
3193         } else {
3194                 /* check if we need to sync */
3195                 if (need_sync) {
3196
3197                         temp.pc = xfer->frbuffers + 0;
3198                         temp.len = 0;
3199                         temp.short_pkt = 0;
3200                         temp.setup_alt_next = 0;
3201
3202                         /* we need a SYNC point after TX */
3203                         temp.func = &dwc_otg_data_tx_sync;
3204                         dwc_otg_setup_standard_chain_sub(&temp);
3205                 }
3206         }
3207
3208         /* must have at least one frame! */
3209         td = temp.td;
3210         xfer->td_transfer_last = td;
3211
3212         if (is_host) {
3213
3214                 struct dwc_otg_softc *sc;
3215                 uint32_t hcchar;
3216                 uint32_t hcsplt;
3217
3218                 sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
3219
3220                 /* get first again */
3221                 td = xfer->td_transfer_first;
3222                 td->toggle = (xfer->endpoint->toggle_next ? 1 : 0);
3223
3224                 hcchar =
3225                         (xfer->address << HCCHAR_DEVADDR_SHIFT) |
3226                         ((xfer->endpointno & UE_ADDR) << HCCHAR_EPNUM_SHIFT) |
3227                         (xfer->max_packet_size << HCCHAR_MPS_SHIFT) |
3228                         HCCHAR_CHENA;
3229
3230                 /*
3231                  * We are not always able to meet the timing
3232                  * requirements of the USB interrupt endpoint's
3233                  * complete split token, when doing transfers going
3234                  * via a transaction translator. Use the CONTROL
3235                  * transfer type instead of the INTERRUPT transfer
3236                  * type in general, as a means to workaround
3237                  * that. This trick should work for both FULL and LOW
3238                  * speed USB traffic going through a TT. For non-TT
3239                  * traffic it works aswell. The reason for using
3240                  * CONTROL type instead of BULK is that some TTs might
3241                  * reject LOW speed BULK traffic.
3242                  */
3243                 if (td->ep_type == UE_INTERRUPT)
3244                         hcchar |= (UE_CONTROL << HCCHAR_EPTYPE_SHIFT);
3245                 else
3246                         hcchar |= (td->ep_type << HCCHAR_EPTYPE_SHIFT);
3247
3248                 if (usbd_get_speed(xfer->xroot->udev) == USB_SPEED_LOW)
3249                         hcchar |= HCCHAR_LSPDDEV;
3250                 if (UE_GET_DIR(xfer->endpointno) == UE_DIR_IN)
3251                         hcchar |= HCCHAR_EPDIR_IN;
3252
3253                 switch (xfer->xroot->udev->speed) {
3254                 case USB_SPEED_FULL:
3255                 case USB_SPEED_LOW:
3256                         /* check if root HUB port is running High Speed */
3257                         if (xfer->xroot->udev->parent_hs_hub != NULL) {
3258                                 hcsplt = HCSPLT_SPLTENA |
3259                                     (xfer->xroot->udev->hs_port_no <<
3260                                     HCSPLT_PRTADDR_SHIFT) |
3261                                     (xfer->xroot->udev->hs_hub_addr <<
3262                                     HCSPLT_HUBADDR_SHIFT);
3263                         } else {
3264                                 hcsplt = 0;
3265                         }
3266                         if (td->ep_type == UE_INTERRUPT) {
3267                                 uint32_t ival;
3268                                 ival = xfer->interval / DWC_OTG_HOST_TIMER_RATE;
3269                                 if (ival == 0)
3270                                         ival = 1;
3271                                 else if (ival > 127)
3272                                         ival = 127;
3273                                 td->tmr_val = sc->sc_tmr_val + ival;
3274                                 td->tmr_res = ival;
3275                         } else if (td->ep_type == UE_ISOCHRONOUS) {
3276                                 td->tmr_val = 0;
3277                                 td->tmr_res = 1;
3278                         } else {
3279                                 td->tmr_val = 0;
3280                                 td->tmr_res = 0;
3281                         }
3282                         break;
3283                 case USB_SPEED_HIGH:
3284                         hcsplt = 0;
3285                         if (td->ep_type == UE_INTERRUPT) {
3286                                 uint32_t ival;
3287 #if 0
3288                                 hcchar |= ((xfer->max_packet_count & 3)
3289                                     << HCCHAR_MC_SHIFT);
3290 #endif
3291                                 ival = xfer->interval / DWC_OTG_HOST_TIMER_RATE;
3292                                 if (ival == 0)
3293                                         ival = 1;
3294                                 else if (ival > 127)
3295                                         ival = 127;
3296                                 td->tmr_val = sc->sc_tmr_val + ival;
3297                                 td->tmr_res = ival;
3298                         } else if (td->ep_type == UE_ISOCHRONOUS) {
3299                                 hcchar |= ((xfer->max_packet_count & 3)
3300                                     << HCCHAR_MC_SHIFT);
3301                                 td->tmr_val = 0;
3302                                 td->tmr_res = 1 << usbd_xfer_get_fps_shift(xfer);
3303                         } else {
3304                                 td->tmr_val = 0;
3305                                 td->tmr_res = 0;
3306                         }
3307                         break;
3308                 default:
3309                         hcsplt = 0;
3310                         td->tmr_val = 0;
3311                         td->tmr_res = 0;
3312                         break;
3313                 }
3314
3315                 /* store configuration in all TD's */
3316                 while (1) {
3317                         td->hcchar = hcchar;
3318                         td->hcsplt = hcsplt;
3319
3320                         if (((void *)td) == xfer->td_transfer_last)
3321                                 break;
3322
3323                         td = td->obj_next;
3324                 }
3325         }
3326 }
3327
3328 static void
3329 dwc_otg_timeout(void *arg)
3330 {
3331         struct usb_xfer *xfer = arg;
3332
3333         DPRINTF("xfer=%p\n", xfer);
3334
3335         USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
3336
3337         /* transfer is transferred */
3338         dwc_otg_device_done(xfer, USB_ERR_TIMEOUT);
3339 }
3340
3341 static void
3342 dwc_otg_start_standard_chain(struct usb_xfer *xfer)
3343 {
3344         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
3345         struct usb_xfer_root *xroot;
3346         struct dwc_otg_td *td;
3347
3348         DPRINTFN(9, "\n");
3349
3350         /*
3351          * Poll one time in device mode, which will turn on the
3352          * endpoint interrupts. Else wait for SOF interrupt in host
3353          * mode.
3354          */
3355         USB_BUS_SPIN_LOCK(&sc->sc_bus);
3356
3357         if (sc->sc_flags.status_device_mode != 0) {
3358                 dwc_otg_xfer_do_fifo(sc, xfer);
3359                 if (dwc_otg_xfer_do_complete_locked(sc, xfer))
3360                         goto done;
3361         }
3362
3363         /* put transfer on interrupt queue */
3364         usbd_transfer_enqueue(&xfer->xroot->bus->intr_q, xfer);
3365
3366         /* start timeout, if any */
3367         if (xfer->timeout != 0) {
3368                 usbd_transfer_timeout_ms(xfer,
3369                     &dwc_otg_timeout, xfer->timeout);
3370         }
3371
3372         if (sc->sc_flags.status_device_mode != 0)
3373                 goto done;
3374
3375         /* enable SOF interrupt, if any */
3376         dwc_otg_enable_sof_irq(sc);
3377
3378         td = xfer->td_transfer_cache;
3379         if (td->ep_type != UE_BULK)
3380                 goto done;
3381
3382         xroot = xfer->xroot;
3383
3384         /*
3385          * Optimise the ping-pong effect by waking up other BULK
3386          * transfers belonging to the same device group:
3387          */
3388         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
3389                 td = xfer->td_transfer_cache;
3390                 if (td == NULL || td->ep_type != UE_BULK || xfer->xroot != xroot)
3391                         continue;
3392                 /* reset NAK counter */
3393                 td->did_nak = 0;
3394         }
3395 done:
3396         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3397 }
3398
3399 static void
3400 dwc_otg_root_intr(struct dwc_otg_softc *sc)
3401 {
3402         DPRINTFN(9, "\n");
3403
3404         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
3405
3406         /* set port bit */
3407         sc->sc_hub_idata[0] = 0x02;     /* we only have one port */
3408
3409         uhub_root_intr(&sc->sc_bus, sc->sc_hub_idata,
3410             sizeof(sc->sc_hub_idata));
3411 }
3412
3413 static usb_error_t
3414 dwc_otg_standard_done_sub(struct usb_xfer *xfer)
3415 {
3416         struct dwc_otg_td *td;
3417         uint32_t len;
3418         usb_error_t error;
3419
3420         DPRINTFN(9, "\n");
3421
3422         td = xfer->td_transfer_cache;
3423
3424         do {
3425                 len = td->remainder;
3426
3427                 /* store last data toggle */
3428                 xfer->endpoint->toggle_next = td->toggle;
3429
3430                 if (xfer->aframes != xfer->nframes) {
3431                         /*
3432                          * Verify the length and subtract
3433                          * the remainder from "frlengths[]":
3434                          */
3435                         if (len > xfer->frlengths[xfer->aframes]) {
3436                                 td->error_any = 1;
3437                         } else {
3438                                 xfer->frlengths[xfer->aframes] -= len;
3439                         }
3440                 }
3441                 /* Check for transfer error */
3442                 if (td->error_any) {
3443                         /* the transfer is finished */
3444                         error = (td->error_stall ?
3445                             USB_ERR_STALLED : USB_ERR_IOERROR);
3446                         td = NULL;
3447                         break;
3448                 }
3449                 /* Check for short transfer */
3450                 if (len > 0) {
3451                         if (xfer->flags_int.short_frames_ok ||
3452                             xfer->flags_int.isochronous_xfr) {
3453                                 /* follow alt next */
3454                                 if (td->alt_next) {
3455                                         td = td->obj_next;
3456                                 } else {
3457                                         td = NULL;
3458                                 }
3459                         } else {
3460                                 /* the transfer is finished */
3461                                 td = NULL;
3462                         }
3463                         error = 0;
3464                         break;
3465                 }
3466                 td = td->obj_next;
3467
3468                 /* this USB frame is complete */
3469                 error = 0;
3470                 break;
3471
3472         } while (0);
3473
3474         /* update transfer cache */
3475
3476         xfer->td_transfer_cache = td;
3477
3478         return (error);
3479 }
3480
3481 static void
3482 dwc_otg_standard_done(struct usb_xfer *xfer)
3483 {
3484         usb_error_t err = 0;
3485
3486         DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
3487             xfer, xfer->endpoint);
3488
3489         /* reset scanner */
3490
3491         xfer->td_transfer_cache = xfer->td_transfer_first;
3492
3493         if (xfer->flags_int.control_xfr) {
3494
3495                 if (xfer->flags_int.control_hdr) {
3496
3497                         err = dwc_otg_standard_done_sub(xfer);
3498                 }
3499                 xfer->aframes = 1;
3500
3501                 if (xfer->td_transfer_cache == NULL) {
3502                         goto done;
3503                 }
3504         }
3505         while (xfer->aframes != xfer->nframes) {
3506
3507                 err = dwc_otg_standard_done_sub(xfer);
3508                 xfer->aframes++;
3509
3510                 if (xfer->td_transfer_cache == NULL) {
3511                         goto done;
3512                 }
3513         }
3514
3515         if (xfer->flags_int.control_xfr &&
3516             !xfer->flags_int.control_act) {
3517
3518                 err = dwc_otg_standard_done_sub(xfer);
3519         }
3520 done:
3521         dwc_otg_device_done(xfer, err);
3522 }
3523
3524 /*------------------------------------------------------------------------*
3525  *      dwc_otg_device_done
3526  *
3527  * NOTE: this function can be called more than one time on the
3528  * same USB transfer!
3529  *------------------------------------------------------------------------*/
3530 static void
3531 dwc_otg_device_done(struct usb_xfer *xfer, usb_error_t error)
3532 {
3533         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
3534
3535         DPRINTFN(9, "xfer=%p, endpoint=%p, error=%d\n",
3536             xfer, xfer->endpoint, error);
3537
3538         USB_BUS_SPIN_LOCK(&sc->sc_bus);
3539
3540         if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) {
3541                 /* Interrupts are cleared by the interrupt handler */
3542         } else {
3543                 struct dwc_otg_td *td;
3544
3545                 td = xfer->td_transfer_cache;
3546                 if (td != NULL)
3547                         dwc_otg_host_channel_free(sc, td);
3548         }
3549         /* dequeue transfer and start next transfer */
3550         usbd_transfer_done(xfer, error);
3551
3552         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3553 }
3554
3555 static void
3556 dwc_otg_xfer_stall(struct usb_xfer *xfer)
3557 {
3558         dwc_otg_device_done(xfer, USB_ERR_STALLED);
3559 }
3560
3561 static void
3562 dwc_otg_set_stall(struct usb_device *udev,
3563     struct usb_endpoint *ep, uint8_t *did_stall)
3564 {
3565         struct dwc_otg_softc *sc;
3566         uint32_t temp;
3567         uint32_t reg;
3568         uint8_t ep_no;
3569
3570         USB_BUS_LOCK_ASSERT(udev->bus, MA_OWNED);
3571
3572         /* check mode */
3573         if (udev->flags.usb_mode != USB_MODE_DEVICE) {
3574                 /* not supported */
3575                 return;
3576         }
3577
3578         sc = DWC_OTG_BUS2SC(udev->bus);
3579
3580         USB_BUS_SPIN_LOCK(&sc->sc_bus);
3581
3582         /* get endpoint address */
3583         ep_no = ep->edesc->bEndpointAddress;
3584
3585         DPRINTFN(5, "endpoint=0x%x\n", ep_no);
3586
3587         if (ep_no & UE_DIR_IN) {
3588                 reg = DOTG_DIEPCTL(ep_no & UE_ADDR);
3589                 temp = sc->sc_in_ctl[ep_no & UE_ADDR];
3590         } else {
3591                 reg = DOTG_DOEPCTL(ep_no & UE_ADDR);
3592                 temp = sc->sc_out_ctl[ep_no & UE_ADDR];
3593         }
3594
3595         /* disable and stall endpoint */
3596         DWC_OTG_WRITE_4(sc, reg, temp | DOEPCTL_EPDIS);
3597         DWC_OTG_WRITE_4(sc, reg, temp | DOEPCTL_STALL);
3598
3599         /* clear active OUT ep */
3600         if (!(ep_no & UE_DIR_IN)) {
3601
3602                 sc->sc_active_rx_ep &= ~(1U << (ep_no & UE_ADDR));
3603
3604                 if (sc->sc_last_rx_status != 0 &&
3605                     (ep_no & UE_ADDR) == GRXSTSRD_CHNUM_GET(
3606                     sc->sc_last_rx_status)) {
3607                         /* dump data */
3608                         dwc_otg_common_rx_ack(sc);
3609                         /* poll interrupt */
3610                         dwc_otg_interrupt_poll_locked(sc);
3611                         dwc_otg_interrupt_complete_locked(sc);
3612                 }
3613         }
3614         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3615 }
3616
3617 static void
3618 dwc_otg_clear_stall_sub_locked(struct dwc_otg_softc *sc, uint32_t mps,
3619     uint8_t ep_no, uint8_t ep_type, uint8_t ep_dir)
3620 {
3621         uint32_t reg;
3622         uint32_t temp;
3623
3624         if (ep_type == UE_CONTROL) {
3625                 /* clearing stall is not needed */
3626                 return;
3627         }
3628
3629         if (ep_dir) {
3630                 reg = DOTG_DIEPCTL(ep_no);
3631         } else {
3632                 reg = DOTG_DOEPCTL(ep_no);
3633                 sc->sc_active_rx_ep |= (1U << ep_no);
3634         }
3635
3636         /* round up and mask away the multiplier count */
3637         mps = (mps + 3) & 0x7FC;
3638
3639         if (ep_type == UE_BULK) {
3640                 temp = DIEPCTL_EPTYPE_SET(
3641                     DIEPCTL_EPTYPE_BULK) |
3642                     DIEPCTL_USBACTEP;
3643         } else if (ep_type == UE_INTERRUPT) {
3644                 temp = DIEPCTL_EPTYPE_SET(
3645                     DIEPCTL_EPTYPE_INTERRUPT) |
3646                     DIEPCTL_USBACTEP;
3647         } else {
3648                 temp = DIEPCTL_EPTYPE_SET(
3649                     DIEPCTL_EPTYPE_ISOC) |
3650                     DIEPCTL_USBACTEP;
3651         }
3652
3653         temp |= DIEPCTL_MPS_SET(mps);
3654         temp |= DIEPCTL_TXFNUM_SET(ep_no);
3655
3656         if (ep_dir)
3657                 sc->sc_in_ctl[ep_no] = temp;
3658         else
3659                 sc->sc_out_ctl[ep_no] = temp;
3660
3661         DWC_OTG_WRITE_4(sc, reg, temp | DOEPCTL_EPDIS);
3662         DWC_OTG_WRITE_4(sc, reg, temp | DOEPCTL_SETD0PID);
3663         DWC_OTG_WRITE_4(sc, reg, temp | DIEPCTL_SNAK);
3664
3665         /* we only reset the transmit FIFO */
3666         if (ep_dir) {
3667                 dwc_otg_tx_fifo_reset(sc,
3668                     GRSTCTL_TXFIFO(ep_no) |
3669                     GRSTCTL_TXFFLSH);
3670
3671                 DWC_OTG_WRITE_4(sc,
3672                     DOTG_DIEPTSIZ(ep_no), 0);
3673         }
3674
3675         /* poll interrupt */
3676         dwc_otg_interrupt_poll_locked(sc);
3677         dwc_otg_interrupt_complete_locked(sc);
3678 }
3679
3680 static void
3681 dwc_otg_clear_stall(struct usb_device *udev, struct usb_endpoint *ep)
3682 {
3683         struct dwc_otg_softc *sc;
3684         struct usb_endpoint_descriptor *ed;
3685
3686         DPRINTFN(5, "endpoint=%p\n", ep);
3687
3688         USB_BUS_LOCK_ASSERT(udev->bus, MA_OWNED);
3689
3690         /* check mode */
3691         if (udev->flags.usb_mode != USB_MODE_DEVICE) {
3692                 /* not supported */
3693                 return;
3694         }
3695         /* get softc */
3696         sc = DWC_OTG_BUS2SC(udev->bus);
3697
3698         USB_BUS_SPIN_LOCK(&sc->sc_bus);
3699
3700         /* get endpoint descriptor */
3701         ed = ep->edesc;
3702
3703         /* reset endpoint */
3704         dwc_otg_clear_stall_sub_locked(sc,
3705             UGETW(ed->wMaxPacketSize),
3706             (ed->bEndpointAddress & UE_ADDR),
3707             (ed->bmAttributes & UE_XFERTYPE),
3708             (ed->bEndpointAddress & (UE_DIR_IN | UE_DIR_OUT)));
3709
3710         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3711 }
3712
3713 static void
3714 dwc_otg_device_state_change(struct usb_device *udev)
3715 {
3716         struct dwc_otg_softc *sc;
3717         uint8_t x;
3718
3719         /* check mode */
3720         if (udev->flags.usb_mode != USB_MODE_DEVICE) {
3721                 /* not supported */
3722                 return;
3723         }
3724
3725         /* get softc */
3726         sc = DWC_OTG_BUS2SC(udev->bus);
3727
3728         /* deactivate all other endpoint but the control endpoint */
3729         if (udev->state == USB_STATE_CONFIGURED ||
3730             udev->state == USB_STATE_ADDRESSED) {
3731
3732                 USB_BUS_LOCK(&sc->sc_bus);
3733
3734                 for (x = 1; x != sc->sc_dev_ep_max; x++) {
3735
3736                         if (x < sc->sc_dev_in_ep_max) {
3737                                 DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(x),
3738                                     DIEPCTL_EPDIS);
3739                                 DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(x), 0);
3740                         }
3741
3742                         DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(x),
3743                             DOEPCTL_EPDIS);
3744                         DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(x), 0);
3745                 }
3746                 USB_BUS_UNLOCK(&sc->sc_bus);
3747         }
3748 }
3749
3750 int
3751 dwc_otg_init(struct dwc_otg_softc *sc)
3752 {
3753         uint32_t temp;
3754
3755         DPRINTF("start\n");
3756
3757         /* set up the bus structure */
3758         sc->sc_bus.usbrev = USB_REV_2_0;
3759         sc->sc_bus.methods = &dwc_otg_bus_methods;
3760
3761         usb_callout_init_mtx(&sc->sc_timer,
3762             &sc->sc_bus.bus_mtx, 0);
3763
3764         USB_BUS_LOCK(&sc->sc_bus);
3765
3766         /* turn on clocks */
3767         dwc_otg_clocks_on(sc);
3768
3769         temp = DWC_OTG_READ_4(sc, DOTG_GSNPSID);
3770         DPRINTF("Version = 0x%08x\n", temp);
3771
3772         /* disconnect */
3773         DWC_OTG_WRITE_4(sc, DOTG_DCTL,
3774             DCTL_SFTDISCON);
3775
3776         /* wait for host to detect disconnect */
3777         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 32);
3778
3779         DWC_OTG_WRITE_4(sc, DOTG_GRSTCTL,
3780             GRSTCTL_CSFTRST);
3781
3782         /* wait a little bit for block to reset */
3783         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 128);
3784
3785         switch (sc->sc_mode) {
3786         case DWC_MODE_DEVICE:
3787                 temp = GUSBCFG_FORCEDEVMODE;
3788                 break;
3789         case DWC_MODE_HOST:
3790                 temp = GUSBCFG_FORCEHOSTMODE;
3791                 break;
3792         default:
3793                 temp = 0;
3794                 break;
3795         }
3796
3797         /* select HSIC, ULPI or internal PHY mode */
3798         switch (dwc_otg_phy_type) {
3799         case DWC_OTG_PHY_HSIC:
3800                 DWC_OTG_WRITE_4(sc, DOTG_GUSBCFG,
3801                     GUSBCFG_PHYIF |
3802                     GUSBCFG_TRD_TIM_SET(5) | temp);
3803                 DWC_OTG_WRITE_4(sc, DOTG_GOTGCTL,
3804                     0x000000EC);
3805
3806                 temp = DWC_OTG_READ_4(sc, DOTG_GLPMCFG);
3807                 DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3808                     temp & ~GLPMCFG_HSIC_CONN);
3809                 DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3810                     temp | GLPMCFG_HSIC_CONN);
3811                 break;
3812         case DWC_OTG_PHY_ULPI:
3813                 DWC_OTG_WRITE_4(sc, DOTG_GUSBCFG,
3814                     GUSBCFG_ULPI_UTMI_SEL |
3815                     GUSBCFG_TRD_TIM_SET(5) | temp);
3816                 DWC_OTG_WRITE_4(sc, DOTG_GOTGCTL, 0);
3817
3818                 temp = DWC_OTG_READ_4(sc, DOTG_GLPMCFG);
3819                 DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3820                     temp & ~GLPMCFG_HSIC_CONN);
3821                 break;
3822         case DWC_OTG_PHY_INTERNAL:
3823                 DWC_OTG_WRITE_4(sc, DOTG_GUSBCFG,
3824                     GUSBCFG_PHYSEL |
3825                     GUSBCFG_TRD_TIM_SET(5) | temp);
3826                 DWC_OTG_WRITE_4(sc, DOTG_GOTGCTL, 0);
3827
3828                 temp = DWC_OTG_READ_4(sc, DOTG_GLPMCFG);
3829                 DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3830                     temp & ~GLPMCFG_HSIC_CONN);
3831
3832                 temp = DWC_OTG_READ_4(sc, DOTG_GGPIO);
3833                 temp &= ~(DOTG_GGPIO_NOVBUSSENS | DOTG_GGPIO_I2CPADEN);
3834                 temp |= (DOTG_GGPIO_VBUSASEN | DOTG_GGPIO_VBUSBSEN |
3835                     DOTG_GGPIO_PWRDWN);
3836                 DWC_OTG_WRITE_4(sc, DOTG_GGPIO, temp);
3837                 break;
3838         default:
3839                 break;
3840         }
3841
3842         /* clear global nak */
3843         DWC_OTG_WRITE_4(sc, DOTG_DCTL,
3844             DCTL_CGOUTNAK |
3845             DCTL_CGNPINNAK);
3846
3847         /* disable USB port */
3848         DWC_OTG_WRITE_4(sc, DOTG_PCGCCTL, 0xFFFFFFFF);
3849
3850         /* wait 10ms */
3851         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 100);
3852
3853         /* enable USB port */
3854         DWC_OTG_WRITE_4(sc, DOTG_PCGCCTL, 0);
3855
3856         /* wait 10ms */
3857         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 100);
3858
3859         temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG3);
3860
3861         sc->sc_fifo_size = 4 * GHWCFG3_DFIFODEPTH_GET(temp);
3862
3863         temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG2);
3864
3865         sc->sc_dev_ep_max = GHWCFG2_NUMDEVEPS_GET(temp);
3866
3867         if (sc->sc_dev_ep_max > DWC_OTG_MAX_ENDPOINTS)
3868                 sc->sc_dev_ep_max = DWC_OTG_MAX_ENDPOINTS;
3869
3870         sc->sc_host_ch_max = GHWCFG2_NUMHSTCHNL_GET(temp);
3871
3872         if (sc->sc_host_ch_max > DWC_OTG_MAX_CHANNELS)
3873                 sc->sc_host_ch_max = DWC_OTG_MAX_CHANNELS;
3874
3875         temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG4);
3876
3877         sc->sc_dev_in_ep_max = GHWCFG4_NUM_IN_EP_GET(temp);
3878
3879         DPRINTF("Total FIFO size = %d bytes, Device EPs = %d/%d Host CHs = %d\n",
3880             sc->sc_fifo_size, sc->sc_dev_ep_max, sc->sc_dev_in_ep_max,
3881             sc->sc_host_ch_max);
3882
3883         /* setup FIFO */
3884         if (dwc_otg_init_fifo(sc, DWC_MODE_OTG)) {
3885                 USB_BUS_UNLOCK(&sc->sc_bus);
3886                 return (EINVAL);
3887         }
3888
3889         /* enable interrupts */
3890         sc->sc_irq_mask = DWC_OTG_MSK_GINT_ENABLED;
3891         DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
3892
3893         if (sc->sc_mode == DWC_MODE_OTG || sc->sc_mode == DWC_MODE_DEVICE) {
3894
3895                 /* enable all endpoint interrupts */
3896                 temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG2);
3897                 if (temp & GHWCFG2_MPI) {
3898                         uint8_t x;
3899
3900                         DPRINTF("Disable Multi Process Interrupts\n");
3901
3902                         for (x = 0; x != sc->sc_dev_in_ep_max; x++) {
3903                                 DWC_OTG_WRITE_4(sc, DOTG_DIEPEACHINTMSK(x), 0);
3904                                 DWC_OTG_WRITE_4(sc, DOTG_DOEPEACHINTMSK(x), 0);
3905                         }
3906                         DWC_OTG_WRITE_4(sc, DOTG_DEACHINTMSK, 0);
3907                 }
3908                 DWC_OTG_WRITE_4(sc, DOTG_DIEPMSK,
3909                     DIEPMSK_XFERCOMPLMSK);
3910                 DWC_OTG_WRITE_4(sc, DOTG_DOEPMSK, 0);
3911                 DWC_OTG_WRITE_4(sc, DOTG_DAINTMSK, 0xFFFF);
3912         }
3913
3914         if (sc->sc_mode == DWC_MODE_OTG || sc->sc_mode == DWC_MODE_HOST) {
3915                 /* setup clocks */
3916                 temp = DWC_OTG_READ_4(sc, DOTG_HCFG);
3917                 temp &= ~(HCFG_FSLSSUPP | HCFG_FSLSPCLKSEL_MASK);
3918                 temp |= (1 << HCFG_FSLSPCLKSEL_SHIFT);
3919                 DWC_OTG_WRITE_4(sc, DOTG_HCFG, temp);
3920         }
3921
3922         /* only enable global IRQ */
3923         DWC_OTG_WRITE_4(sc, DOTG_GAHBCFG,
3924             GAHBCFG_GLBLINTRMSK);
3925
3926         /* turn off clocks */
3927         dwc_otg_clocks_off(sc);
3928
3929         /* read initial VBUS state */
3930
3931         temp = DWC_OTG_READ_4(sc, DOTG_GOTGCTL);
3932
3933         DPRINTFN(5, "GOTCTL=0x%08x\n", temp);
3934
3935         dwc_otg_vbus_interrupt(sc,
3936             (temp & (GOTGCTL_ASESVLD | GOTGCTL_BSESVLD)) ? 1 : 0);
3937
3938         USB_BUS_UNLOCK(&sc->sc_bus);
3939
3940         /* catch any lost interrupts */
3941
3942         dwc_otg_do_poll(&sc->sc_bus);
3943
3944         return (0);                     /* success */
3945 }
3946
3947 void
3948 dwc_otg_uninit(struct dwc_otg_softc *sc)
3949 {
3950         USB_BUS_LOCK(&sc->sc_bus);
3951
3952         /* stop host timer */
3953         dwc_otg_timer_stop(sc);
3954
3955         /* set disconnect */
3956         DWC_OTG_WRITE_4(sc, DOTG_DCTL,
3957             DCTL_SFTDISCON);
3958
3959         /* turn off global IRQ */
3960         DWC_OTG_WRITE_4(sc, DOTG_GAHBCFG, 0);
3961
3962         sc->sc_flags.port_enabled = 0;
3963         sc->sc_flags.port_powered = 0;
3964         sc->sc_flags.status_vbus = 0;
3965         sc->sc_flags.status_bus_reset = 0;
3966         sc->sc_flags.status_suspend = 0;
3967         sc->sc_flags.change_suspend = 0;
3968         sc->sc_flags.change_connect = 1;
3969
3970         dwc_otg_pull_down(sc);
3971         dwc_otg_clocks_off(sc);
3972
3973         USB_BUS_UNLOCK(&sc->sc_bus);
3974
3975         usb_callout_drain(&sc->sc_timer);
3976 }
3977
3978 static void
3979 dwc_otg_suspend(struct dwc_otg_softc *sc)
3980 {
3981         return;
3982 }
3983
3984 static void
3985 dwc_otg_resume(struct dwc_otg_softc *sc)
3986 {
3987         return;
3988 }
3989
3990 static void
3991 dwc_otg_do_poll(struct usb_bus *bus)
3992 {
3993         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(bus);
3994
3995         USB_BUS_LOCK(&sc->sc_bus);
3996         USB_BUS_SPIN_LOCK(&sc->sc_bus);
3997         dwc_otg_interrupt_poll_locked(sc);
3998         dwc_otg_interrupt_complete_locked(sc);
3999         if (sc->sc_flags.status_device_mode == 0) {
4000                 /* update host transfer schedule, so that new transfers can be issued */
4001                 if (dwc_otg_update_host_transfer_schedule_locked(sc))
4002                         dwc_otg_interrupt_poll_locked(sc);
4003         }
4004         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
4005         USB_BUS_UNLOCK(&sc->sc_bus);
4006 }
4007
4008 /*------------------------------------------------------------------------*
4009  * DWC OTG bulk support
4010  * DWC OTG control support
4011  * DWC OTG interrupt support
4012  *------------------------------------------------------------------------*/
4013 static void
4014 dwc_otg_device_non_isoc_open(struct usb_xfer *xfer)
4015 {
4016 }
4017
4018 static void
4019 dwc_otg_device_non_isoc_close(struct usb_xfer *xfer)
4020 {
4021         dwc_otg_device_done(xfer, USB_ERR_CANCELLED);
4022 }
4023
4024 static void
4025 dwc_otg_device_non_isoc_enter(struct usb_xfer *xfer)
4026 {
4027 }
4028
4029 static void
4030 dwc_otg_device_non_isoc_start(struct usb_xfer *xfer)
4031 {
4032         /* setup TDs */
4033         dwc_otg_setup_standard_chain(xfer);
4034         dwc_otg_start_standard_chain(xfer);
4035 }
4036
4037 struct usb_pipe_methods dwc_otg_device_non_isoc_methods =
4038 {
4039         .open = dwc_otg_device_non_isoc_open,
4040         .close = dwc_otg_device_non_isoc_close,
4041         .enter = dwc_otg_device_non_isoc_enter,
4042         .start = dwc_otg_device_non_isoc_start,
4043 };
4044
4045 /*------------------------------------------------------------------------*
4046  * DWC OTG full speed isochronous support
4047  *------------------------------------------------------------------------*/
4048 static void
4049 dwc_otg_device_isoc_open(struct usb_xfer *xfer)
4050 {
4051 }
4052
4053 static void
4054 dwc_otg_device_isoc_close(struct usb_xfer *xfer)
4055 {
4056         dwc_otg_device_done(xfer, USB_ERR_CANCELLED);
4057 }
4058
4059 static void
4060 dwc_otg_device_isoc_enter(struct usb_xfer *xfer)
4061 {
4062 }
4063
4064 static void
4065 dwc_otg_device_isoc_start(struct usb_xfer *xfer)
4066 {
4067         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
4068         uint32_t temp;
4069         uint32_t msframes;
4070         uint32_t framenum;
4071         uint8_t shift = usbd_xfer_get_fps_shift(xfer);
4072
4073         DPRINTFN(6, "xfer=%p next=%d nframes=%d\n",
4074             xfer, xfer->endpoint->isoc_next, xfer->nframes);
4075
4076         if (xfer->xroot->udev->flags.usb_mode == USB_MODE_HOST) {
4077                 temp = DWC_OTG_READ_4(sc, DOTG_HFNUM);
4078
4079                 /* get the current frame index */
4080                 framenum = (temp & HFNUM_FRNUM_MASK);
4081         } else {
4082                 temp = DWC_OTG_READ_4(sc, DOTG_DSTS);
4083
4084                 /* get the current frame index */
4085                 framenum = DSTS_SOFFN_GET(temp);
4086         }
4087
4088         if (xfer->xroot->udev->parent_hs_hub != NULL)
4089                 framenum /= 8;
4090
4091         framenum &= DWC_OTG_FRAME_MASK;
4092
4093         /*
4094          * Compute number of milliseconds worth of data traffic for
4095          * this USB transfer:
4096          */
4097         if (xfer->xroot->udev->speed == USB_SPEED_HIGH)
4098                 msframes = ((xfer->nframes << shift) + 7) / 8;
4099         else
4100                 msframes = xfer->nframes;
4101
4102         /*
4103          * check if the frame index is within the window where the frames
4104          * will be inserted
4105          */
4106         temp = (framenum - xfer->endpoint->isoc_next) & DWC_OTG_FRAME_MASK;
4107
4108         if ((xfer->endpoint->is_synced == 0) || (temp < msframes)) {
4109                 /*
4110                  * If there is data underflow or the pipe queue is
4111                  * empty we schedule the transfer a few frames ahead
4112                  * of the current frame position. Else two isochronous
4113                  * transfers might overlap.
4114                  */
4115                 xfer->endpoint->isoc_next = (framenum + 3) & DWC_OTG_FRAME_MASK;
4116                 xfer->endpoint->is_synced = 1;
4117                 DPRINTFN(3, "start next=%d\n", xfer->endpoint->isoc_next);
4118         }
4119         /*
4120          * compute how many milliseconds the insertion is ahead of the
4121          * current frame position:
4122          */
4123         temp = (xfer->endpoint->isoc_next - framenum) & DWC_OTG_FRAME_MASK;
4124
4125         /*
4126          * pre-compute when the isochronous transfer will be finished:
4127          */
4128         xfer->isoc_time_complete =
4129                 usb_isoc_time_expand(&sc->sc_bus, framenum) + temp + msframes;
4130
4131         /* setup TDs */
4132         dwc_otg_setup_standard_chain(xfer);
4133
4134         /* compute frame number for next insertion */
4135         xfer->endpoint->isoc_next += msframes;
4136
4137         /* start TD chain */
4138         dwc_otg_start_standard_chain(xfer);
4139 }
4140
4141 struct usb_pipe_methods dwc_otg_device_isoc_methods =
4142 {
4143         .open = dwc_otg_device_isoc_open,
4144         .close = dwc_otg_device_isoc_close,
4145         .enter = dwc_otg_device_isoc_enter,
4146         .start = dwc_otg_device_isoc_start,
4147 };
4148
4149 /*------------------------------------------------------------------------*
4150  * DWC OTG root control support
4151  *------------------------------------------------------------------------*
4152  * Simulate a hardware HUB by handling all the necessary requests.
4153  *------------------------------------------------------------------------*/
4154
4155 static const struct usb_device_descriptor dwc_otg_devd = {
4156         .bLength = sizeof(struct usb_device_descriptor),
4157         .bDescriptorType = UDESC_DEVICE,
4158         .bcdUSB = {0x00, 0x02},
4159         .bDeviceClass = UDCLASS_HUB,
4160         .bDeviceSubClass = UDSUBCLASS_HUB,
4161         .bDeviceProtocol = UDPROTO_HSHUBSTT,
4162         .bMaxPacketSize = 64,
4163         .bcdDevice = {0x00, 0x01},
4164         .iManufacturer = 1,
4165         .iProduct = 2,
4166         .bNumConfigurations = 1,
4167 };
4168
4169 static const struct dwc_otg_config_desc dwc_otg_confd = {
4170         .confd = {
4171                 .bLength = sizeof(struct usb_config_descriptor),
4172                 .bDescriptorType = UDESC_CONFIG,
4173                 .wTotalLength[0] = sizeof(dwc_otg_confd),
4174                 .bNumInterface = 1,
4175                 .bConfigurationValue = 1,
4176                 .iConfiguration = 0,
4177                 .bmAttributes = UC_SELF_POWERED,
4178                 .bMaxPower = 0,
4179         },
4180         .ifcd = {
4181                 .bLength = sizeof(struct usb_interface_descriptor),
4182                 .bDescriptorType = UDESC_INTERFACE,
4183                 .bNumEndpoints = 1,
4184                 .bInterfaceClass = UICLASS_HUB,
4185                 .bInterfaceSubClass = UISUBCLASS_HUB,
4186                 .bInterfaceProtocol = 0,
4187         },
4188         .endpd = {
4189                 .bLength = sizeof(struct usb_endpoint_descriptor),
4190                 .bDescriptorType = UDESC_ENDPOINT,
4191                 .bEndpointAddress = (UE_DIR_IN | DWC_OTG_INTR_ENDPT),
4192                 .bmAttributes = UE_INTERRUPT,
4193                 .wMaxPacketSize[0] = 8,
4194                 .bInterval = 255,
4195         },
4196 };
4197
4198 #define HSETW(ptr, val) ptr = { (uint8_t)(val), (uint8_t)((val) >> 8) }
4199
4200 static const struct usb_hub_descriptor_min dwc_otg_hubd = {
4201         .bDescLength = sizeof(dwc_otg_hubd),
4202         .bDescriptorType = UDESC_HUB,
4203         .bNbrPorts = 1,
4204         HSETW(.wHubCharacteristics, (UHD_PWR_NO_SWITCH | UHD_OC_INDIVIDUAL)),
4205         .bPwrOn2PwrGood = 50,
4206         .bHubContrCurrent = 0,
4207         .DeviceRemovable = {0},         /* port is removable */
4208 };
4209
4210 #define STRING_VENDOR \
4211   "D\0W\0C\0O\0T\0G"
4212
4213 #define STRING_PRODUCT \
4214   "O\0T\0G\0 \0R\0o\0o\0t\0 \0H\0U\0B"
4215
4216 USB_MAKE_STRING_DESC(STRING_VENDOR, dwc_otg_vendor);
4217 USB_MAKE_STRING_DESC(STRING_PRODUCT, dwc_otg_product);
4218
4219 static usb_error_t
4220 dwc_otg_roothub_exec(struct usb_device *udev,
4221     struct usb_device_request *req, const void **pptr, uint16_t *plength)
4222 {
4223         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(udev->bus);
4224         const void *ptr;
4225         uint16_t len;
4226         uint16_t value;
4227         uint16_t index;
4228         usb_error_t err;
4229
4230         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
4231
4232         /* buffer reset */
4233         ptr = (const void *)&sc->sc_hub_temp;
4234         len = 0;
4235         err = 0;
4236
4237         value = UGETW(req->wValue);
4238         index = UGETW(req->wIndex);
4239
4240         /* demultiplex the control request */
4241
4242         switch (req->bmRequestType) {
4243         case UT_READ_DEVICE:
4244                 switch (req->bRequest) {
4245                 case UR_GET_DESCRIPTOR:
4246                         goto tr_handle_get_descriptor;
4247                 case UR_GET_CONFIG:
4248                         goto tr_handle_get_config;
4249                 case UR_GET_STATUS:
4250                         goto tr_handle_get_status;
4251                 default:
4252                         goto tr_stalled;
4253                 }
4254                 break;
4255
4256         case UT_WRITE_DEVICE:
4257                 switch (req->bRequest) {
4258                 case UR_SET_ADDRESS:
4259                         goto tr_handle_set_address;
4260                 case UR_SET_CONFIG:
4261                         goto tr_handle_set_config;
4262                 case UR_CLEAR_FEATURE:
4263                         goto tr_valid;  /* nop */
4264                 case UR_SET_DESCRIPTOR:
4265                         goto tr_valid;  /* nop */
4266                 case UR_SET_FEATURE:
4267                 default:
4268                         goto tr_stalled;
4269                 }
4270                 break;
4271
4272         case UT_WRITE_ENDPOINT:
4273                 switch (req->bRequest) {
4274                 case UR_CLEAR_FEATURE:
4275                         switch (UGETW(req->wValue)) {
4276                         case UF_ENDPOINT_HALT:
4277                                 goto tr_handle_clear_halt;
4278                         case UF_DEVICE_REMOTE_WAKEUP:
4279                                 goto tr_handle_clear_wakeup;
4280                         default:
4281                                 goto tr_stalled;
4282                         }
4283                         break;
4284                 case UR_SET_FEATURE:
4285                         switch (UGETW(req->wValue)) {
4286                         case UF_ENDPOINT_HALT:
4287                                 goto tr_handle_set_halt;
4288                         case UF_DEVICE_REMOTE_WAKEUP:
4289                                 goto tr_handle_set_wakeup;
4290                         default:
4291                                 goto tr_stalled;
4292                         }
4293                         break;
4294                 case UR_SYNCH_FRAME:
4295                         goto tr_valid;  /* nop */
4296                 default:
4297                         goto tr_stalled;
4298                 }
4299                 break;
4300
4301         case UT_READ_ENDPOINT:
4302                 switch (req->bRequest) {
4303                 case UR_GET_STATUS:
4304                         goto tr_handle_get_ep_status;
4305                 default:
4306                         goto tr_stalled;
4307                 }
4308                 break;
4309
4310         case UT_WRITE_INTERFACE:
4311                 switch (req->bRequest) {
4312                 case UR_SET_INTERFACE:
4313                         goto tr_handle_set_interface;
4314                 case UR_CLEAR_FEATURE:
4315                         goto tr_valid;  /* nop */
4316                 case UR_SET_FEATURE:
4317                 default:
4318                         goto tr_stalled;
4319                 }
4320                 break;
4321
4322         case UT_READ_INTERFACE:
4323                 switch (req->bRequest) {
4324                 case UR_GET_INTERFACE:
4325                         goto tr_handle_get_interface;
4326                 case UR_GET_STATUS:
4327                         goto tr_handle_get_iface_status;
4328                 default:
4329                         goto tr_stalled;
4330                 }
4331                 break;
4332
4333         case UT_WRITE_CLASS_INTERFACE:
4334         case UT_WRITE_VENDOR_INTERFACE:
4335                 /* XXX forward */
4336                 break;
4337
4338         case UT_READ_CLASS_INTERFACE:
4339         case UT_READ_VENDOR_INTERFACE:
4340                 /* XXX forward */
4341                 break;
4342
4343         case UT_WRITE_CLASS_DEVICE:
4344                 switch (req->bRequest) {
4345                 case UR_CLEAR_FEATURE:
4346                         goto tr_valid;
4347                 case UR_SET_DESCRIPTOR:
4348                 case UR_SET_FEATURE:
4349                         break;
4350                 default:
4351                         goto tr_stalled;
4352                 }
4353                 break;
4354
4355         case UT_WRITE_CLASS_OTHER:
4356                 switch (req->bRequest) {
4357                 case UR_CLEAR_FEATURE:
4358                         goto tr_handle_clear_port_feature;
4359                 case UR_SET_FEATURE:
4360                         goto tr_handle_set_port_feature;
4361                 case UR_CLEAR_TT_BUFFER:
4362                 case UR_RESET_TT:
4363                 case UR_STOP_TT:
4364                         goto tr_valid;
4365
4366                 default:
4367                         goto tr_stalled;
4368                 }
4369                 break;
4370
4371         case UT_READ_CLASS_OTHER:
4372                 switch (req->bRequest) {
4373                 case UR_GET_TT_STATE:
4374                         goto tr_handle_get_tt_state;
4375                 case UR_GET_STATUS:
4376                         goto tr_handle_get_port_status;
4377                 default:
4378                         goto tr_stalled;
4379                 }
4380                 break;
4381
4382         case UT_READ_CLASS_DEVICE:
4383                 switch (req->bRequest) {
4384                 case UR_GET_DESCRIPTOR:
4385                         goto tr_handle_get_class_descriptor;
4386                 case UR_GET_STATUS:
4387                         goto tr_handle_get_class_status;
4388
4389                 default:
4390                         goto tr_stalled;
4391                 }
4392                 break;
4393         default:
4394                 goto tr_stalled;
4395         }
4396         goto tr_valid;
4397
4398 tr_handle_get_descriptor:
4399         switch (value >> 8) {
4400         case UDESC_DEVICE:
4401                 if (value & 0xff) {
4402                         goto tr_stalled;
4403                 }
4404                 len = sizeof(dwc_otg_devd);
4405                 ptr = (const void *)&dwc_otg_devd;
4406                 goto tr_valid;
4407         case UDESC_CONFIG:
4408                 if (value & 0xff) {
4409                         goto tr_stalled;
4410                 }
4411                 len = sizeof(dwc_otg_confd);
4412                 ptr = (const void *)&dwc_otg_confd;
4413                 goto tr_valid;
4414         case UDESC_STRING:
4415                 switch (value & 0xff) {
4416                 case 0:         /* Language table */
4417                         len = sizeof(usb_string_lang_en);
4418                         ptr = (const void *)&usb_string_lang_en;
4419                         goto tr_valid;
4420
4421                 case 1:         /* Vendor */
4422                         len = sizeof(dwc_otg_vendor);
4423                         ptr = (const void *)&dwc_otg_vendor;
4424                         goto tr_valid;
4425
4426                 case 2:         /* Product */
4427                         len = sizeof(dwc_otg_product);
4428                         ptr = (const void *)&dwc_otg_product;
4429                         goto tr_valid;
4430                 default:
4431                         break;
4432                 }
4433                 break;
4434         default:
4435                 goto tr_stalled;
4436         }
4437         goto tr_stalled;
4438
4439 tr_handle_get_config:
4440         len = 1;
4441         sc->sc_hub_temp.wValue[0] = sc->sc_conf;
4442         goto tr_valid;
4443
4444 tr_handle_get_status:
4445         len = 2;
4446         USETW(sc->sc_hub_temp.wValue, UDS_SELF_POWERED);
4447         goto tr_valid;
4448
4449 tr_handle_set_address:
4450         if (value & 0xFF00) {
4451                 goto tr_stalled;
4452         }
4453         sc->sc_rt_addr = value;
4454         goto tr_valid;
4455
4456 tr_handle_set_config:
4457         if (value >= 2) {
4458                 goto tr_stalled;
4459         }
4460         sc->sc_conf = value;
4461         goto tr_valid;
4462
4463 tr_handle_get_interface:
4464         len = 1;
4465         sc->sc_hub_temp.wValue[0] = 0;
4466         goto tr_valid;
4467
4468 tr_handle_get_tt_state:
4469 tr_handle_get_class_status:
4470 tr_handle_get_iface_status:
4471 tr_handle_get_ep_status:
4472         len = 2;
4473         USETW(sc->sc_hub_temp.wValue, 0);
4474         goto tr_valid;
4475
4476 tr_handle_set_halt:
4477 tr_handle_set_interface:
4478 tr_handle_set_wakeup:
4479 tr_handle_clear_wakeup:
4480 tr_handle_clear_halt:
4481         goto tr_valid;
4482
4483 tr_handle_clear_port_feature:
4484         if (index != 1)
4485                 goto tr_stalled;
4486
4487         DPRINTFN(9, "UR_CLEAR_PORT_FEATURE on port %d\n", index);
4488
4489         switch (value) {
4490         case UHF_PORT_SUSPEND:
4491                 dwc_otg_wakeup_peer(sc);
4492                 break;
4493
4494         case UHF_PORT_ENABLE:
4495                 if (sc->sc_flags.status_device_mode == 0) {
4496                         DWC_OTG_WRITE_4(sc, DOTG_HPRT,
4497                             sc->sc_hprt_val | HPRT_PRTENA);
4498                 }
4499                 sc->sc_flags.port_enabled = 0;
4500                 break;
4501
4502         case UHF_C_PORT_RESET:
4503                 sc->sc_flags.change_reset = 0;
4504                 break;
4505
4506         case UHF_C_PORT_ENABLE:
4507                 sc->sc_flags.change_enabled = 0;
4508                 break;
4509
4510         case UHF_C_PORT_OVER_CURRENT:
4511                 sc->sc_flags.change_over_current = 0;
4512                 break;
4513
4514         case UHF_PORT_TEST:
4515         case UHF_PORT_INDICATOR:
4516                 /* nops */
4517                 break;
4518
4519         case UHF_PORT_POWER:
4520                 sc->sc_flags.port_powered = 0;
4521                 if (sc->sc_mode == DWC_MODE_HOST || sc->sc_mode == DWC_MODE_OTG) {
4522                         sc->sc_hprt_val = 0;
4523                         DWC_OTG_WRITE_4(sc, DOTG_HPRT, HPRT_PRTENA);
4524                 }
4525                 dwc_otg_pull_down(sc);
4526                 dwc_otg_clocks_off(sc);
4527                 break;
4528
4529         case UHF_C_PORT_CONNECTION:
4530                 /* clear connect change flag */
4531                 sc->sc_flags.change_connect = 0;
4532                 break;
4533
4534         case UHF_C_PORT_SUSPEND:
4535                 sc->sc_flags.change_suspend = 0;
4536                 break;
4537
4538         default:
4539                 err = USB_ERR_IOERROR;
4540                 goto done;
4541         }
4542         goto tr_valid;
4543
4544 tr_handle_set_port_feature:
4545         if (index != 1) {
4546                 goto tr_stalled;
4547         }
4548         DPRINTFN(9, "UR_SET_PORT_FEATURE\n");
4549
4550         switch (value) {
4551         case UHF_PORT_ENABLE:
4552                 break;
4553
4554         case UHF_PORT_SUSPEND:
4555                 if (sc->sc_flags.status_device_mode == 0) {
4556                         /* set suspend BIT */
4557                         sc->sc_hprt_val |= HPRT_PRTSUSP;
4558                         DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
4559
4560                         /* generate HUB suspend event */
4561                         dwc_otg_suspend_irq(sc);
4562                 }
4563                 break;
4564
4565         case UHF_PORT_RESET:
4566                 if (sc->sc_flags.status_device_mode == 0) {
4567
4568                         DPRINTF("PORT RESET\n");
4569
4570                         /* enable PORT reset */
4571                         DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val | HPRT_PRTRST);
4572
4573                         /* Wait 62.5ms for reset to complete */
4574                         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 16);
4575
4576                         DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
4577
4578                         /* Wait 62.5ms for reset to complete */
4579                         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 16);
4580
4581                         /* reset FIFOs */
4582                         (void) dwc_otg_init_fifo(sc, DWC_MODE_HOST);
4583
4584                         sc->sc_flags.change_reset = 1;
4585                 } else {
4586                         err = USB_ERR_IOERROR;
4587                 }
4588                 break;
4589
4590         case UHF_PORT_TEST:
4591         case UHF_PORT_INDICATOR:
4592                 /* nops */
4593                 break;
4594         case UHF_PORT_POWER:
4595                 sc->sc_flags.port_powered = 1;
4596                 if (sc->sc_mode == DWC_MODE_HOST || sc->sc_mode == DWC_MODE_OTG) {
4597                         sc->sc_hprt_val |= HPRT_PRTPWR;
4598                         DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
4599                 }
4600                 if (sc->sc_mode == DWC_MODE_DEVICE || sc->sc_mode == DWC_MODE_OTG) {
4601                         /* pull up D+, if any */
4602                         dwc_otg_pull_up(sc);
4603                 }
4604                 break;
4605         default:
4606                 err = USB_ERR_IOERROR;
4607                 goto done;
4608         }
4609         goto tr_valid;
4610
4611 tr_handle_get_port_status:
4612
4613         DPRINTFN(9, "UR_GET_PORT_STATUS\n");
4614
4615         if (index != 1)
4616                 goto tr_stalled;
4617
4618         if (sc->sc_flags.status_vbus)
4619                 dwc_otg_clocks_on(sc);
4620         else
4621                 dwc_otg_clocks_off(sc);
4622
4623         /* Select Device Side Mode */
4624
4625         if (sc->sc_flags.status_device_mode) {
4626                 value = UPS_PORT_MODE_DEVICE;
4627                 dwc_otg_timer_stop(sc);
4628         } else {
4629                 value = 0;
4630                 dwc_otg_timer_start(sc);
4631         }
4632
4633         if (sc->sc_flags.status_high_speed)
4634                 value |= UPS_HIGH_SPEED;
4635         else if (sc->sc_flags.status_low_speed)
4636                 value |= UPS_LOW_SPEED;
4637
4638         if (sc->sc_flags.port_powered)
4639                 value |= UPS_PORT_POWER;
4640
4641         if (sc->sc_flags.port_enabled)
4642                 value |= UPS_PORT_ENABLED;
4643
4644         if (sc->sc_flags.port_over_current)
4645                 value |= UPS_OVERCURRENT_INDICATOR;
4646
4647         if (sc->sc_flags.status_vbus &&
4648             sc->sc_flags.status_bus_reset)
4649                 value |= UPS_CURRENT_CONNECT_STATUS;
4650
4651         if (sc->sc_flags.status_suspend)
4652                 value |= UPS_SUSPEND;
4653
4654         USETW(sc->sc_hub_temp.ps.wPortStatus, value);
4655
4656         value = 0;
4657
4658         if (sc->sc_flags.change_connect)
4659                 value |= UPS_C_CONNECT_STATUS;
4660         if (sc->sc_flags.change_suspend)
4661                 value |= UPS_C_SUSPEND;
4662         if (sc->sc_flags.change_reset)
4663                 value |= UPS_C_PORT_RESET;
4664         if (sc->sc_flags.change_over_current)
4665                 value |= UPS_C_OVERCURRENT_INDICATOR;
4666
4667         USETW(sc->sc_hub_temp.ps.wPortChange, value);
4668         len = sizeof(sc->sc_hub_temp.ps);
4669         goto tr_valid;
4670
4671 tr_handle_get_class_descriptor:
4672         if (value & 0xFF) {
4673                 goto tr_stalled;
4674         }
4675         ptr = (const void *)&dwc_otg_hubd;
4676         len = sizeof(dwc_otg_hubd);
4677         goto tr_valid;
4678
4679 tr_stalled:
4680         err = USB_ERR_STALLED;
4681 tr_valid:
4682 done:
4683         *plength = len;
4684         *pptr = ptr;
4685         return (err);
4686 }
4687
4688 static void
4689 dwc_otg_xfer_setup(struct usb_setup_params *parm)
4690 {
4691         struct usb_xfer *xfer;
4692         void *last_obj;
4693         uint32_t ntd;
4694         uint32_t n;
4695         uint8_t ep_no;
4696         uint8_t ep_type;
4697
4698         xfer = parm->curr_xfer;
4699
4700         /*
4701          * NOTE: This driver does not use any of the parameters that
4702          * are computed from the following values. Just set some
4703          * reasonable dummies:
4704          */
4705         parm->hc_max_packet_size = 0x500;
4706         parm->hc_max_packet_count = 3;
4707         parm->hc_max_frame_size = 3 * 0x500;
4708
4709         usbd_transfer_setup_sub(parm);
4710
4711         /*
4712          * compute maximum number of TDs
4713          */
4714         ep_type = (xfer->endpoint->edesc->bmAttributes & UE_XFERTYPE);
4715
4716         if (ep_type == UE_CONTROL) {
4717
4718                 ntd = xfer->nframes + 1 /* STATUS */ + 1 /* SYNC 1 */
4719                     + 1 /* SYNC 2 */ + 1 /* SYNC 3 */;
4720         } else {
4721
4722                 ntd = xfer->nframes + 1 /* SYNC */ ;
4723         }
4724
4725         /*
4726          * check if "usbd_transfer_setup_sub" set an error
4727          */
4728         if (parm->err)
4729                 return;
4730
4731         /*
4732          * allocate transfer descriptors
4733          */
4734         last_obj = NULL;
4735
4736         ep_no = xfer->endpointno & UE_ADDR;
4737
4738         /*
4739          * Check for a valid endpoint profile in USB device mode:
4740          */
4741         if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) {
4742                 const struct usb_hw_ep_profile *pf;
4743
4744                 dwc_otg_get_hw_ep_profile(parm->udev, &pf, ep_no);
4745
4746                 if (pf == NULL) {
4747                         /* should not happen */
4748                         parm->err = USB_ERR_INVAL;
4749                         return;
4750                 }
4751         }
4752
4753         /* align data */
4754         parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
4755
4756         for (n = 0; n != ntd; n++) {
4757
4758                 struct dwc_otg_td *td;
4759
4760                 if (parm->buf) {
4761
4762                         td = USB_ADD_BYTES(parm->buf, parm->size[0]);
4763
4764                         /* compute shared bandwidth resource index for TT */
4765                         if (parm->udev->parent_hs_hub != NULL && parm->udev->speed != USB_SPEED_HIGH) {
4766                                 if (parm->udev->parent_hs_hub->ddesc.bDeviceProtocol == UDPROTO_HSHUBMTT)
4767                                         td->tt_index = parm->udev->device_index;
4768                                 else
4769                                         td->tt_index = parm->udev->parent_hs_hub->device_index;
4770                         } else {
4771                                 td->tt_index = parm->udev->device_index;
4772                         }
4773
4774                         /* init TD */
4775                         td->max_packet_size = xfer->max_packet_size;
4776                         td->max_packet_count = xfer->max_packet_count;
4777                         td->ep_no = ep_no;
4778                         td->ep_type = ep_type;
4779                         td->obj_next = last_obj;
4780
4781                         last_obj = td;
4782                 }
4783                 parm->size[0] += sizeof(*td);
4784         }
4785
4786         xfer->td_start[0] = last_obj;
4787 }
4788
4789 static void
4790 dwc_otg_xfer_unsetup(struct usb_xfer *xfer)
4791 {
4792         return;
4793 }
4794
4795 static void
4796 dwc_otg_ep_init(struct usb_device *udev, struct usb_endpoint_descriptor *edesc,
4797     struct usb_endpoint *ep)
4798 {
4799         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(udev->bus);
4800
4801         DPRINTFN(2, "endpoint=%p, addr=%d, endpt=%d, mode=%d (%d,%d)\n",
4802             ep, udev->address,
4803             edesc->bEndpointAddress, udev->flags.usb_mode,
4804             sc->sc_rt_addr, udev->device_index);
4805
4806         if (udev->device_index != sc->sc_rt_addr) {
4807
4808                 if (udev->flags.usb_mode == USB_MODE_DEVICE) {
4809                         if (udev->speed != USB_SPEED_FULL &&
4810                             udev->speed != USB_SPEED_HIGH) {
4811                                 /* not supported */
4812                                 return;
4813                         }
4814                 } else {
4815                         if (udev->speed == USB_SPEED_HIGH) {
4816                                 if ((UGETW(edesc->wMaxPacketSize) >> 11) & 3) {
4817                                         /* high bandwidth endpoint - not tested */
4818                                         DPRINTF("High Bandwidth Endpoint - not tested\n");
4819                                         return;
4820                                 }
4821                         }
4822                 }
4823                 if ((edesc->bmAttributes & UE_XFERTYPE) == UE_ISOCHRONOUS)
4824                         ep->methods = &dwc_otg_device_isoc_methods;
4825                 else
4826                         ep->methods = &dwc_otg_device_non_isoc_methods;
4827         }
4828 }
4829
4830 static void
4831 dwc_otg_set_hw_power_sleep(struct usb_bus *bus, uint32_t state)
4832 {
4833         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(bus);
4834
4835         switch (state) {
4836         case USB_HW_POWER_SUSPEND:
4837                 dwc_otg_suspend(sc);
4838                 break;
4839         case USB_HW_POWER_SHUTDOWN:
4840                 dwc_otg_uninit(sc);
4841                 break;
4842         case USB_HW_POWER_RESUME:
4843                 dwc_otg_resume(sc);
4844                 break;
4845         default:
4846                 break;
4847         }
4848 }
4849
4850 static void
4851 dwc_otg_get_dma_delay(struct usb_device *udev, uint32_t *pus)
4852 {
4853         /* DMA delay - wait until any use of memory is finished */
4854         *pus = (2125);                  /* microseconds */
4855 }
4856
4857 static void
4858 dwc_otg_device_resume(struct usb_device *udev)
4859 {
4860         DPRINTF("\n");
4861
4862         /* poll all transfers again to restart resumed ones */
4863         dwc_otg_do_poll(udev->bus);
4864 }
4865
4866 static void
4867 dwc_otg_device_suspend(struct usb_device *udev)
4868 {
4869         DPRINTF("\n");
4870 }
4871
4872 struct usb_bus_methods dwc_otg_bus_methods =
4873 {
4874         .endpoint_init = &dwc_otg_ep_init,
4875         .xfer_setup = &dwc_otg_xfer_setup,
4876         .xfer_unsetup = &dwc_otg_xfer_unsetup,
4877         .get_hw_ep_profile = &dwc_otg_get_hw_ep_profile,
4878         .xfer_stall = &dwc_otg_xfer_stall,
4879         .set_stall = &dwc_otg_set_stall,
4880         .clear_stall = &dwc_otg_clear_stall,
4881         .roothub_exec = &dwc_otg_roothub_exec,
4882         .xfer_poll = &dwc_otg_do_poll,
4883         .device_state_change = &dwc_otg_device_state_change,
4884         .set_hw_power_sleep = &dwc_otg_set_hw_power_sleep,
4885         .get_dma_delay = &dwc_otg_get_dma_delay,
4886         .device_resume = &dwc_otg_device_resume,
4887         .device_suspend = &dwc_otg_device_suspend,
4888 };