sys/dev/usb/controller/dwc_otg.c

   1 /* $FreeBSD$ */
   2 /*-
   3  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   4  *
   5  * Copyright (c) 2015 Daisuke Aoyama. All rights reserved.
   6  * Copyright (c) 2012-2015 Hans Petter Selasky. All rights reserved.
   7  * Copyright (c) 2010-2011 Aleksandr Rybalko. All rights reserved.
   8  *
   9  * Redistribution and use in source and binary forms, with or without
  10  * modification, are permitted provided that the following conditions
  11  * are met:
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions and the following disclaimer.
  14  * 2. Redistributions in binary form must reproduce the above copyright
  15  *    notice, this list of conditions and the following disclaimer in the
  16  *    documentation and/or other materials provided with the distribution.
  17  *
  18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  21  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  28  * SUCH DAMAGE.
  29  */
  30
  31 /*
  32  * This file contains the driver for the DesignWare series USB 2.0 OTG
  33  * Controller.
  34  */
  35
  36 /*
  37  * LIMITATION: Drivers must be bound to all OUT endpoints in the
  38  * active configuration for this driver to work properly. Blocking any
  39  * OUT endpoint will block all OUT endpoints including the control
  40  * endpoint. Usually this is not a problem.
  41  */
  42
  43 /*
  44  * NOTE: Writing to non-existing registers appears to cause an
  45  * internal reset.
  46  */
  47
  48 #ifdef USB_GLOBAL_INCLUDE_FILE
  49 #include USB_GLOBAL_INCLUDE_FILE
  50 #else
  51 #include <sys/stdint.h>
  52 #include <sys/stddef.h>
  53 #include <sys/param.h>
  54 #include <sys/queue.h>
  55 #include <sys/types.h>
  56 #include <sys/systm.h>
  57 #include <sys/kernel.h>
  58 #include <sys/bus.h>
  59 #include <sys/module.h>
  60 #include <sys/lock.h>
  61 #include <sys/mutex.h>
  62 #include <sys/condvar.h>
  63 #include <sys/sysctl.h>
  64 #include <sys/sx.h>
  65 #include <sys/unistd.h>
  66 #include <sys/callout.h>
  67 #include <sys/malloc.h>
  68 #include <sys/priv.h>
  69 #include <sys/rman.h>
  70
  71 #include <dev/usb/usb.h>
  72 #include <dev/usb/usbdi.h>
  73
  74 #define USB_DEBUG_VAR dwc_otg_debug
  75
  76 #include <dev/usb/usb_core.h>
  77 #include <dev/usb/usb_debug.h>
  78 #include <dev/usb/usb_busdma.h>
  79 #include <dev/usb/usb_process.h>
  80 #include <dev/usb/usb_transfer.h>
  81 #include <dev/usb/usb_device.h>
  82 #include <dev/usb/usb_hub.h>
  83 #include <dev/usb/usb_util.h>
  84
  85 #include <dev/usb/usb_controller.h>
  86 #include <dev/usb/usb_bus.h>
  87 #endif                  /* USB_GLOBAL_INCLUDE_FILE */
  88
  89 #include <dev/usb/controller/dwc_otg.h>
  90 #include <dev/usb/controller/dwc_otgreg.h>
  91
  92 #define DWC_OTG_BUS2SC(bus) \
  93    ((struct dwc_otg_softc *)(((uint8_t *)(bus)) - \
  94     ((uint8_t *)&(((struct dwc_otg_softc *)0)->sc_bus))))
  95
  96 #define DWC_OTG_PC2UDEV(pc) \
  97    (USB_DMATAG_TO_XROOT((pc)->tag_parent)->udev)
  98
  99 #define DWC_OTG_MSK_GINT_THREAD_IRQ                             \
 100    (GINTSTS_USBRST | GINTSTS_ENUMDONE | GINTSTS_PRTINT |        \
 101    GINTSTS_WKUPINT | GINTSTS_USBSUSP | GINTMSK_OTGINTMSK |      \
 102    GINTSTS_SESSREQINT)
 103
 104 #ifndef DWC_OTG_PHY_DEFAULT
 105 #define DWC_OTG_PHY_DEFAULT DWC_OTG_PHY_ULPI
 106 #endif
 107
 108 static int dwc_otg_phy_type = DWC_OTG_PHY_DEFAULT;
 109
 110 static SYSCTL_NODE(_hw_usb, OID_AUTO, dwc_otg, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
 111     "USB DWC OTG");
 112 SYSCTL_INT(_hw_usb_dwc_otg, OID_AUTO, phy_type, CTLFLAG_RDTUN,
 113     &dwc_otg_phy_type, 0, "DWC OTG PHY TYPE - 0/1/2/3 - ULPI/HSIC/INTERNAL/UTMI+");
 114
 115 #ifdef USB_DEBUG
 116 static int dwc_otg_debug = 0;
 117
 118 SYSCTL_INT(_hw_usb_dwc_otg, OID_AUTO, debug, CTLFLAG_RWTUN,
 119     &dwc_otg_debug, 0, "DWC OTG debug level");
 120 #endif
 121
 122 #define DWC_OTG_INTR_ENDPT 1
 123
 124 /* prototypes */
 125
 126 static const struct usb_bus_methods dwc_otg_bus_methods;
 127 static const struct usb_pipe_methods dwc_otg_device_non_isoc_methods;
 128 static const struct usb_pipe_methods dwc_otg_device_isoc_methods;
 129
 130 static dwc_otg_cmd_t dwc_otg_setup_rx;
 131 static dwc_otg_cmd_t dwc_otg_data_rx;
 132 static dwc_otg_cmd_t dwc_otg_data_tx;
 133 static dwc_otg_cmd_t dwc_otg_data_tx_sync;
 134
 135 static dwc_otg_cmd_t dwc_otg_host_setup_tx;
 136 static dwc_otg_cmd_t dwc_otg_host_data_tx;
 137 static dwc_otg_cmd_t dwc_otg_host_data_rx;
 138
 139 static void dwc_otg_device_done(struct usb_xfer *, usb_error_t);
 140 static void dwc_otg_do_poll(struct usb_bus *);
 141 static void dwc_otg_standard_done(struct usb_xfer *);
 142 static void dwc_otg_root_intr(struct dwc_otg_softc *);
 143 static void dwc_otg_interrupt_poll_locked(struct dwc_otg_softc *);
 144
 145 /*
 146  * Here is a configuration that the chip supports.
 147  */
 148 static const struct usb_hw_ep_profile dwc_otg_ep_profile[1] = {
 149
 150         [0] = {
 151                 .max_in_frame_size = 64,/* fixed */
 152                 .max_out_frame_size = 64,       /* fixed */
 153                 .is_simplex = 1,
 154                 .support_control = 1,
 155         }
 156 };
 157
 158 static void
 159 dwc_otg_get_hw_ep_profile(struct usb_device *udev,
 160     const struct usb_hw_ep_profile **ppf, uint8_t ep_addr)
 161 {
 162         struct dwc_otg_softc *sc;
 163
 164         sc = DWC_OTG_BUS2SC(udev->bus);
 165
 166         if (ep_addr < sc->sc_dev_ep_max)
 167                 *ppf = &sc->sc_hw_ep_profile[ep_addr].usb;
 168         else
 169                 *ppf = NULL;
 170 }
 171
 172 static void
 173 dwc_otg_write_fifo(struct dwc_otg_softc *sc, struct usb_page_cache *pc,
 174     uint32_t offset, uint32_t fifo, uint32_t count)
 175 {
 176         uint32_t temp;
 177
 178         /* round down length to nearest 4-bytes */
 179         temp = count & ~3;
 180
 181         /* check if we can write the data directly */
 182         if (temp != 0 && usb_pc_buffer_is_aligned(pc, offset, temp, 3)) {
 183                 struct usb_page_search buf_res;
 184
 185                 /* pre-subtract length */
 186                 count -= temp;
 187
 188                 /* iterate buffer list */
 189                 do {
 190                         /* get current buffer pointer */
 191                         usbd_get_page(pc, offset, &buf_res);
 192
 193                         if (buf_res.length > temp)
 194                                 buf_res.length = temp;
 195
 196                         /* transfer data into FIFO */
 197                         bus_space_write_region_4(sc->sc_io_tag, sc->sc_io_hdl,
 198                             fifo, buf_res.buffer, buf_res.length / 4);
 199
 200                         offset += buf_res.length;
 201                         fifo += buf_res.length;
 202                         temp -= buf_res.length;
 203                 } while (temp != 0);
 204         }
 205
 206         /* check for remainder */
 207         if (count != 0) {
 208                 /* clear topmost word before copy */
 209                 sc->sc_bounce_buffer[(count - 1) / 4] = 0;
 210
 211                 /* copy out data */
 212                 usbd_copy_out(pc, offset,
 213                     sc->sc_bounce_buffer, count);
 214
 215                 /* transfer data into FIFO */
 216                 bus_space_write_region_4(sc->sc_io_tag,
 217                     sc->sc_io_hdl, fifo, sc->sc_bounce_buffer,
 218                     (count + 3) / 4);
 219         }
 220 }
 221
 222 static void
 223 dwc_otg_read_fifo(struct dwc_otg_softc *sc, struct usb_page_cache *pc,
 224     uint32_t offset, uint32_t count)
 225 {
 226         uint32_t temp;
 227
 228         /* round down length to nearest 4-bytes */
 229         temp = count & ~3;
 230
 231         /* check if we can read the data directly */
 232         if (temp != 0 && usb_pc_buffer_is_aligned(pc, offset, temp, 3)) {
 233                 struct usb_page_search buf_res;
 234
 235                 /* pre-subtract length */
 236                 count -= temp;
 237
 238                 /* iterate buffer list */
 239                 do {
 240                         /* get current buffer pointer */
 241                         usbd_get_page(pc, offset, &buf_res);
 242
 243                         if (buf_res.length > temp)
 244                                 buf_res.length = temp;
 245
 246                         /* transfer data from FIFO */
 247                         bus_space_read_region_4(sc->sc_io_tag, sc->sc_io_hdl,
 248                             sc->sc_current_rx_fifo, buf_res.buffer, buf_res.length / 4);
 249
 250                         offset += buf_res.length;
 251                         sc->sc_current_rx_fifo += buf_res.length;
 252                         sc->sc_current_rx_bytes -= buf_res.length;
 253                         temp -= buf_res.length;
 254                 } while (temp != 0);
 255         }
 256
 257         /* check for remainder */
 258         if (count != 0) {
 259                 /* read data into bounce buffer */
 260                 bus_space_read_region_4(sc->sc_io_tag, sc->sc_io_hdl,
 261                         sc->sc_current_rx_fifo,
 262                         sc->sc_bounce_buffer, (count + 3) / 4);
 263
 264                 /* store data into proper buffer */
 265                 usbd_copy_in(pc, offset, sc->sc_bounce_buffer, count);
 266
 267                 /* round length up to nearest 4 bytes */
 268                 count = (count + 3) & ~3;
 269
 270                 /* update counters */
 271                 sc->sc_current_rx_bytes -= count;
 272                 sc->sc_current_rx_fifo += count;
 273         }
 274 }
 275
 276 static void
 277 dwc_otg_tx_fifo_reset(struct dwc_otg_softc *sc, uint32_t value)
 278 {
 279         uint32_t temp;
 280
 281         /* reset FIFO */
 282         DWC_OTG_WRITE_4(sc, DOTG_GRSTCTL, value);
 283
 284         /* wait for reset to complete */
 285         for (temp = 0; temp != 16; temp++) {
 286                 value = DWC_OTG_READ_4(sc, DOTG_GRSTCTL);
 287                 if (!(value & (GRSTCTL_TXFFLSH | GRSTCTL_RXFFLSH)))
 288                         break;
 289         }
 290 }
 291
 292 static int
 293 dwc_otg_init_fifo(struct dwc_otg_softc *sc, uint8_t mode)
 294 {
 295         struct dwc_otg_profile *pf;
 296         uint32_t fifo_size;
 297         uint32_t fifo_regs;
 298         uint32_t tx_start;
 299         uint8_t x;
 300
 301         fifo_size = sc->sc_fifo_size;
 302
 303         /*
 304          * NOTE: Reserved fixed size area at end of RAM, which must
 305          * not be allocated to the FIFOs:
 306          */
 307         fifo_regs = 4 * 16;
 308
 309         if (fifo_size < fifo_regs) {
 310                 DPRINTF("Too little FIFO\n");
 311                 return (EINVAL);
 312         }
 313
 314         /* subtract FIFO regs from total once */
 315         fifo_size -= fifo_regs;
 316
 317         /* split equally for IN and OUT */
 318         fifo_size /= 2;
 319
 320         /* Align to 4 bytes boundary (refer to PGM) */
 321         fifo_size &= ~3;
 322
 323         /* set global receive FIFO size */
 324         DWC_OTG_WRITE_4(sc, DOTG_GRXFSIZ, fifo_size / 4);
 325
 326         tx_start = fifo_size;
 327
 328         if (fifo_size < 64) {
 329                 DPRINTFN(-1, "Not enough data space for EP0 FIFO.\n");
 330                 return (EINVAL);
 331         }
 332
 333         if (mode == DWC_MODE_HOST) {
 334
 335                 /* reset active endpoints */
 336                 sc->sc_active_rx_ep = 0;
 337
 338                 /* split equally for periodic and non-periodic */
 339                 fifo_size /= 2;
 340
 341                 DPRINTF("PTX/NPTX FIFO=%u\n", fifo_size);
 342
 343                 /* align to 4 bytes boundary */
 344                 fifo_size &= ~3;
 345
 346                 DWC_OTG_WRITE_4(sc, DOTG_GNPTXFSIZ,
 347                     ((fifo_size / 4) << 16) |
 348                     (tx_start / 4));
 349
 350                 tx_start += fifo_size;
 351
 352                 for (x = 0; x != sc->sc_host_ch_max; x++) {
 353                         /* enable all host interrupts */
 354                         DWC_OTG_WRITE_4(sc, DOTG_HCINTMSK(x),
 355                             HCINT_DEFAULT_MASK);
 356                 }
 357
 358                 DWC_OTG_WRITE_4(sc, DOTG_HPTXFSIZ,
 359                     ((fifo_size / 4) << 16) |
 360                     (tx_start / 4));
 361
 362                 /* reset host channel state */
 363                 memset(sc->sc_chan_state, 0, sizeof(sc->sc_chan_state));
 364
 365                 /* enable all host channel interrupts */
 366                 DWC_OTG_WRITE_4(sc, DOTG_HAINTMSK,
 367                     (1U << sc->sc_host_ch_max) - 1U);
 368
 369                 /* enable proper host channel interrupts */
 370                 sc->sc_irq_mask |= GINTMSK_HCHINTMSK;
 371                 sc->sc_irq_mask &= ~GINTMSK_IEPINTMSK;
 372                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 373         }
 374
 375         if (mode == DWC_MODE_DEVICE) {
 376
 377             DWC_OTG_WRITE_4(sc, DOTG_GNPTXFSIZ,
 378                 (0x10 << 16) | (tx_start / 4));
 379             fifo_size -= 0x40;
 380             tx_start += 0x40;
 381
 382             /* setup control endpoint profile */
 383             sc->sc_hw_ep_profile[0].usb = dwc_otg_ep_profile[0];
 384
 385             /* reset active endpoints */
 386             sc->sc_active_rx_ep = 1;
 387
 388             for (x = 1; x != sc->sc_dev_ep_max; x++) {
 389
 390                 pf = sc->sc_hw_ep_profile + x;
 391
 392                 pf->usb.max_out_frame_size = 1024 * 3;
 393                 pf->usb.is_simplex = 0; /* assume duplex */
 394                 pf->usb.support_bulk = 1;
 395                 pf->usb.support_interrupt = 1;
 396                 pf->usb.support_isochronous = 1;
 397                 pf->usb.support_out = 1;
 398
 399                 if (x < sc->sc_dev_in_ep_max) {
 400                         uint32_t limit;
 401
 402                         limit = (x == 1) ? MIN(DWC_OTG_TX_MAX_FIFO_SIZE,
 403                             DWC_OTG_MAX_TXN) : MIN(DWC_OTG_MAX_TXN / 2,
 404                             DWC_OTG_TX_MAX_FIFO_SIZE);
 405
 406                         /* see if there is enough FIFO space */
 407                         if (limit <= fifo_size) {
 408                                 pf->max_buffer = limit;
 409                                 pf->usb.support_in = 1;
 410                         } else {
 411                                 limit = MIN(DWC_OTG_TX_MAX_FIFO_SIZE, 0x40);
 412                                 if (limit <= fifo_size) {
 413                                         pf->usb.support_in = 1;
 414                                 } else {
 415                                         pf->usb.is_simplex = 1;
 416                                         limit = 0;
 417                                 }
 418                         }
 419                         /* set FIFO size */
 420                         DWC_OTG_WRITE_4(sc, DOTG_DIEPTXF(x),
 421                             ((limit / 4) << 16) | (tx_start / 4));
 422                         tx_start += limit;
 423                         fifo_size -= limit;
 424                         pf->usb.max_in_frame_size = limit;
 425                 } else {
 426                         pf->usb.is_simplex = 1;
 427                 }
 428
 429                 DPRINTF("FIFO%d = IN:%d / OUT:%d\n", x,
 430                     pf->usb.max_in_frame_size,
 431                     pf->usb.max_out_frame_size);
 432             }
 433
 434             /* enable proper device channel interrupts */
 435             sc->sc_irq_mask &= ~GINTMSK_HCHINTMSK;
 436             sc->sc_irq_mask |= GINTMSK_IEPINTMSK;
 437             DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 438         }
 439
 440         /* reset RX FIFO */
 441         dwc_otg_tx_fifo_reset(sc, GRSTCTL_RXFFLSH);
 442
 443         if (mode != DWC_MODE_OTG) {
 444                 /* reset all TX FIFOs */
 445                 dwc_otg_tx_fifo_reset(sc,
 446                     GRSTCTL_TXFIFO(0x10) |
 447                     GRSTCTL_TXFFLSH);
 448         } else {
 449                 /* reset active endpoints */
 450                 sc->sc_active_rx_ep = 0;
 451
 452                 /* reset host channel state */
 453                 memset(sc->sc_chan_state, 0, sizeof(sc->sc_chan_state));
 454         }
 455         return (0);
 456 }
 457
 458 static uint8_t
 459 dwc_otg_uses_split(struct usb_device *udev)
 460 {
 461         /*
 462          * When a LOW or FULL speed device is connected directly to
 463          * the USB port we don't use split transactions:
 464          */
 465         return (udev->speed != USB_SPEED_HIGH &&
 466             udev->parent_hs_hub != NULL &&
 467             udev->parent_hs_hub->parent_hub != NULL);
 468 }
 469
 470 static void
 471 dwc_otg_update_host_frame_interval(struct dwc_otg_softc *sc)
 472 {
 473
 474   /*
 475    * Disabled until further. Assuming that the register is already
 476    * programmed correctly by the boot loader.
 477    */
 478 #if 0
 479         uint32_t temp;
 480
 481         /* setup HOST frame interval register, based on existing value */
 482         temp = DWC_OTG_READ_4(sc, DOTG_HFIR) & HFIR_FRINT_MASK;
 483         if (temp >= 10000)
 484                 temp /= 1000;
 485         else
 486                 temp /= 125;
 487
 488         /* figure out nearest X-tal value */
 489         if (temp >= 54)
 490                 temp = 60;      /* MHz */
 491         else if (temp >= 39)
 492                 temp = 48;      /* MHz */
 493         else
 494                 temp = 30;      /* MHz */
 495
 496         if (sc->sc_flags.status_high_speed)
 497                 temp *= 125;
 498         else
 499                 temp *= 1000;
 500
 501         DPRINTF("HFIR=0x%08x\n", temp);
 502
 503         DWC_OTG_WRITE_4(sc, DOTG_HFIR, temp);
 504 #endif
 505 }
 506
 507 static void
 508 dwc_otg_clocks_on(struct dwc_otg_softc *sc)
 509 {
 510         if (sc->sc_flags.clocks_off &&
 511             sc->sc_flags.port_powered) {
 512
 513                 DPRINTFN(5, "\n");
 514
 515                 /* TODO - platform specific */
 516
 517                 sc->sc_flags.clocks_off = 0;
 518         }
 519 }
 520
 521 static void
 522 dwc_otg_clocks_off(struct dwc_otg_softc *sc)
 523 {
 524         if (!sc->sc_flags.clocks_off) {
 525
 526                 DPRINTFN(5, "\n");
 527
 528                 /* TODO - platform specific */
 529
 530                 sc->sc_flags.clocks_off = 1;
 531         }
 532 }
 533
 534 static void
 535 dwc_otg_pull_up(struct dwc_otg_softc *sc)
 536 {
 537         uint32_t temp;
 538
 539         /* pullup D+, if possible */
 540
 541         if (!sc->sc_flags.d_pulled_up &&
 542             sc->sc_flags.port_powered) {
 543                 sc->sc_flags.d_pulled_up = 1;
 544
 545                 temp = DWC_OTG_READ_4(sc, DOTG_DCTL);
 546                 temp &= ~DCTL_SFTDISCON;
 547                 DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
 548         }
 549 }
 550
 551 static void
 552 dwc_otg_pull_down(struct dwc_otg_softc *sc)
 553 {
 554         uint32_t temp;
 555
 556         /* pulldown D+, if possible */
 557
 558         if (sc->sc_flags.d_pulled_up) {
 559                 sc->sc_flags.d_pulled_up = 0;
 560
 561                 temp = DWC_OTG_READ_4(sc, DOTG_DCTL);
 562                 temp |= DCTL_SFTDISCON;
 563                 DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
 564         }
 565 }
 566
 567 static void
 568 dwc_otg_enable_sof_irq(struct dwc_otg_softc *sc)
 569 {
 570         /* In device mode we don't use the SOF interrupt */
 571         if (sc->sc_flags.status_device_mode != 0)
 572                 return;
 573         /* Ensure the SOF interrupt is not disabled */
 574         sc->sc_needsof = 1;
 575         /* Check if the SOF interrupt is already enabled */
 576         if ((sc->sc_irq_mask & GINTMSK_SOFMSK) != 0)
 577                 return;
 578         sc->sc_irq_mask |= GINTMSK_SOFMSK;
 579         DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 580 }
 581
 582 static void
 583 dwc_otg_resume_irq(struct dwc_otg_softc *sc)
 584 {
 585         if (sc->sc_flags.status_suspend) {
 586                 /* update status bits */
 587                 sc->sc_flags.status_suspend = 0;
 588                 sc->sc_flags.change_suspend = 1;
 589
 590                 if (sc->sc_flags.status_device_mode) {
 591                         /*
 592                          * Disable resume interrupt and enable suspend
 593                          * interrupt:
 594                          */
 595                         sc->sc_irq_mask &= ~GINTMSK_WKUPINTMSK;
 596                         sc->sc_irq_mask |= GINTMSK_USBSUSPMSK;
 597                         DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 598                 }
 599
 600                 /* complete root HUB interrupt endpoint */
 601                 dwc_otg_root_intr(sc);
 602         }
 603 }
 604
 605 static void
 606 dwc_otg_suspend_irq(struct dwc_otg_softc *sc)
 607 {
 608         if (!sc->sc_flags.status_suspend) {
 609                 /* update status bits */
 610                 sc->sc_flags.status_suspend = 1;
 611                 sc->sc_flags.change_suspend = 1;
 612
 613                 if (sc->sc_flags.status_device_mode) {
 614                         /*
 615                          * Disable suspend interrupt and enable resume
 616                          * interrupt:
 617                          */
 618                         sc->sc_irq_mask &= ~GINTMSK_USBSUSPMSK;
 619                         sc->sc_irq_mask |= GINTMSK_WKUPINTMSK;
 620                         DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 621                 }
 622
 623                 /* complete root HUB interrupt endpoint */
 624                 dwc_otg_root_intr(sc);
 625         }
 626 }
 627
 628 static void
 629 dwc_otg_wakeup_peer(struct dwc_otg_softc *sc)
 630 {
 631         if (!sc->sc_flags.status_suspend)
 632                 return;
 633
 634         DPRINTFN(5, "Remote wakeup\n");
 635
 636         if (sc->sc_flags.status_device_mode) {
 637                 uint32_t temp;
 638
 639                 /* enable remote wakeup signalling */
 640                 temp = DWC_OTG_READ_4(sc, DOTG_DCTL);
 641                 temp |= DCTL_RMTWKUPSIG;
 642                 DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
 643
 644                 /* Wait 8ms for remote wakeup to complete. */
 645                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 125);
 646
 647                 temp &= ~DCTL_RMTWKUPSIG;
 648                 DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
 649         } else {
 650                 /* enable USB port */
 651                 DWC_OTG_WRITE_4(sc, DOTG_PCGCCTL, 0);
 652
 653                 /* wait 10ms */
 654                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 100);
 655
 656                 /* resume port */
 657                 sc->sc_hprt_val |= HPRT_PRTRES;
 658                 DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
 659
 660                 /* Wait 100ms for resume signalling to complete. */
 661                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 10);
 662
 663                 /* clear suspend and resume */
 664                 sc->sc_hprt_val &= ~(HPRT_PRTSUSP | HPRT_PRTRES);
 665                 DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
 666
 667                 /* Wait 4ms */
 668                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 250);
 669         }
 670
 671         /* need to fake resume IRQ */
 672         dwc_otg_resume_irq(sc);
 673 }
 674
 675 static void
 676 dwc_otg_set_address(struct dwc_otg_softc *sc, uint8_t addr)
 677 {
 678         uint32_t temp;
 679
 680         DPRINTFN(5, "addr=%d\n", addr);
 681
 682         temp = DWC_OTG_READ_4(sc, DOTG_DCFG);
 683         temp &= ~DCFG_DEVADDR_SET(0x7F);
 684         temp |= DCFG_DEVADDR_SET(addr);
 685         DWC_OTG_WRITE_4(sc, DOTG_DCFG, temp);
 686 }
 687
 688 static void
 689 dwc_otg_common_rx_ack(struct dwc_otg_softc *sc)
 690 {
 691         DPRINTFN(5, "RX status clear\n");
 692
 693         /* enable RX FIFO level interrupt */
 694         sc->sc_irq_mask |= GINTMSK_RXFLVLMSK;
 695         DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 696
 697         if (sc->sc_current_rx_bytes != 0) {
 698                 /* need to dump remaining data */
 699                 bus_space_read_region_4(sc->sc_io_tag, sc->sc_io_hdl,
 700                     sc->sc_current_rx_fifo, sc->sc_bounce_buffer,
 701                     sc->sc_current_rx_bytes / 4);
 702                 /* clear number of active bytes to receive */
 703                 sc->sc_current_rx_bytes = 0;
 704         }
 705         /* clear cached status */
 706         sc->sc_last_rx_status = 0;
 707 }
 708
 709 static void
 710 dwc_otg_clear_hcint(struct dwc_otg_softc *sc, uint8_t x)
 711 {
 712         uint32_t hcint;
 713
 714         /* clear all pending interrupts */
 715         hcint = DWC_OTG_READ_4(sc, DOTG_HCINT(x));
 716         DWC_OTG_WRITE_4(sc, DOTG_HCINT(x), hcint);
 717
 718         /* clear buffered interrupts */
 719         sc->sc_chan_state[x].hcint = 0;
 720 }
 721
 722 static uint8_t
 723 dwc_otg_host_check_tx_fifo_empty(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
 724 {
 725         uint32_t temp;
 726
 727         temp = DWC_OTG_READ_4(sc, DOTG_GINTSTS);
 728
 729         if (td->ep_type == UE_ISOCHRONOUS) {
 730                 /*
 731                  * NOTE: USB INTERRUPT transactions are executed like
 732                  * USB CONTROL transactions! See the setup standard
 733                  * chain function for more information.
 734                  */
 735                 if (!(temp & GINTSTS_PTXFEMP)) {
 736                         DPRINTF("Periodic TX FIFO is not empty\n");
 737                         if (!(sc->sc_irq_mask & GINTMSK_PTXFEMPMSK)) {
 738                                 sc->sc_irq_mask |= GINTMSK_PTXFEMPMSK;
 739                                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 740                         }
 741                         return (1);     /* busy */
 742                 }
 743         } else {
 744                 if (!(temp & GINTSTS_NPTXFEMP)) {
 745                         DPRINTF("Non-periodic TX FIFO is not empty\n");
 746                         if (!(sc->sc_irq_mask & GINTMSK_NPTXFEMPMSK)) {
 747                                 sc->sc_irq_mask |= GINTMSK_NPTXFEMPMSK;
 748                                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 749                         }
 750                         return (1);     /* busy */
 751                 }
 752         }
 753         return (0);     /* ready for transmit */
 754 }
 755
 756 static uint8_t
 757 dwc_otg_host_channel_alloc(struct dwc_otg_softc *sc,
 758     struct dwc_otg_td *td, uint8_t is_out)
 759 {
 760         uint8_t x;
 761         uint8_t y;
 762         uint8_t z;
 763
 764         if (td->channel[0] < DWC_OTG_MAX_CHANNELS)
 765                 return (0);             /* already allocated */
 766
 767         /* check if device is suspended */
 768         if (DWC_OTG_PC2UDEV(td->pc)->flags.self_suspended != 0)
 769                 return (1);             /* busy - cannot transfer data */
 770
 771         /* compute needed TX FIFO size */
 772         if (is_out != 0) {
 773                 if (dwc_otg_host_check_tx_fifo_empty(sc, td) != 0)
 774                         return (1);     /* busy - cannot transfer data */
 775         }
 776         z = td->max_packet_count;
 777         for (x = y = 0; x != sc->sc_host_ch_max; x++) {
 778                 /* check if channel is allocated */
 779                 if (sc->sc_chan_state[x].allocated != 0)
 780                         continue;
 781                 /* check if channel is still enabled */
 782                 if (sc->sc_chan_state[x].wait_halted != 0)
 783                         continue;
 784                 /* store channel number */
 785                 td->channel[y++] = x;
 786                 /* check if we got all channels */
 787                 if (y == z)
 788                         break;
 789         }
 790         if (y != z) {
 791                 /* reset channel variable */
 792                 td->channel[0] = DWC_OTG_MAX_CHANNELS;
 793                 td->channel[1] = DWC_OTG_MAX_CHANNELS;
 794                 td->channel[2] = DWC_OTG_MAX_CHANNELS;
 795                 /* wait a bit */
 796                 dwc_otg_enable_sof_irq(sc);
 797                 return (1);     /* busy - not enough channels */
 798         }
 799
 800         for (y = 0; y != z; y++) {
 801                 x = td->channel[y];
 802
 803                 /* set allocated */
 804                 sc->sc_chan_state[x].allocated = 1;
 805
 806                 /* set wait halted */
 807                 sc->sc_chan_state[x].wait_halted = 1;
 808
 809                 /* clear interrupts */
 810                 dwc_otg_clear_hcint(sc, x);
 811
 812                 DPRINTF("CH=%d HCCHAR=0x%08x "
 813                     "HCSPLT=0x%08x\n", x, td->hcchar, td->hcsplt);
 814
 815                 /* set active channel */
 816                 sc->sc_active_rx_ep |= (1 << x);
 817         }
 818         return (0);     /* allocated */
 819 }
 820
 821 static void
 822 dwc_otg_host_channel_free_sub(struct dwc_otg_softc *sc, struct dwc_otg_td *td, uint8_t index)
 823 {
 824         uint32_t hcchar;
 825         uint8_t x;
 826
 827         if (td->channel[index] >= DWC_OTG_MAX_CHANNELS)
 828                 return;         /* already freed */
 829
 830         /* free channel */
 831         x = td->channel[index];
 832         td->channel[index] = DWC_OTG_MAX_CHANNELS;
 833
 834         DPRINTF("CH=%d\n", x);
 835
 836         /*
 837          * We need to let programmed host channels run till complete
 838          * else the host channel will stop functioning.
 839          */
 840         sc->sc_chan_state[x].allocated = 0;
 841
 842         /* ack any pending messages */
 843         if (sc->sc_last_rx_status != 0 &&
 844             GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) == x) {
 845                 dwc_otg_common_rx_ack(sc);
 846         }
 847
 848         /* clear active channel */
 849         sc->sc_active_rx_ep &= ~(1 << x);
 850
 851         /* check if already halted */
 852         if (sc->sc_chan_state[x].wait_halted == 0)
 853                 return;
 854
 855         /* disable host channel */
 856         hcchar = DWC_OTG_READ_4(sc, DOTG_HCCHAR(x));
 857         if (hcchar & HCCHAR_CHENA) {
 858                 DPRINTF("Halting channel %d\n", x);
 859                 DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(x),
 860                     hcchar | HCCHAR_CHDIS);
 861                 /* don't write HCCHAR until the channel is halted */
 862         } else {
 863                 sc->sc_chan_state[x].wait_halted = 0;
 864         }
 865 }
 866
 867 static void
 868 dwc_otg_host_channel_free(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
 869 {
 870         uint8_t x;
 871         for (x = 0; x != td->max_packet_count; x++)
 872                 dwc_otg_host_channel_free_sub(sc, td, x);
 873 }
 874
 875 static void
 876 dwc_otg_host_dump_rx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
 877 {
 878         uint8_t x;
 879         /* dump any pending messages */
 880         if (sc->sc_last_rx_status == 0)
 881                 return;
 882         for (x = 0; x != td->max_packet_count; x++) {
 883                 if (td->channel[x] >= DWC_OTG_MAX_CHANNELS ||
 884                     td->channel[x] != GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status))
 885                         continue;
 886                 dwc_otg_common_rx_ack(sc);
 887                 break;
 888         }
 889 }
 890
 891 static uint8_t
 892 dwc_otg_host_setup_tx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
 893 {
 894         struct usb_device_request req __aligned(4);
 895         uint32_t hcint;
 896         uint32_t hcchar;
 897         uint8_t delta;
 898
 899         dwc_otg_host_dump_rx(sc, td);
 900
 901         if (td->channel[0] < DWC_OTG_MAX_CHANNELS) {
 902                 hcint = sc->sc_chan_state[td->channel[0]].hcint;
 903
 904                 DPRINTF("CH=%d ST=%d HCINT=0x%08x HCCHAR=0x%08x HCTSIZ=0x%08x\n",
 905                     td->channel[0], td->state, hcint,
 906                     DWC_OTG_READ_4(sc, DOTG_HCCHAR(td->channel[0])),
 907                     DWC_OTG_READ_4(sc, DOTG_HCTSIZ(td->channel[0])));
 908         } else {
 909                 hcint = 0;
 910                 goto check_state;
 911         }
 912
 913         if (hcint & (HCINT_RETRY |
 914             HCINT_ACK | HCINT_NYET)) {
 915                 /* give success bits priority over failure bits */
 916         } else if (hcint & HCINT_STALL) {
 917                 DPRINTF("CH=%d STALL\n", td->channel[0]);
 918                 td->error_stall = 1;
 919                 td->error_any = 1;
 920                 goto complete;
 921         } else if (hcint & HCINT_ERRORS) {
 922                 DPRINTF("CH=%d ERROR\n", td->channel[0]);
 923                 td->errcnt++;
 924                 if (td->hcsplt != 0 || td->errcnt >= 3) {
 925                         td->error_any = 1;
 926                         goto complete;
 927                 }
 928         }
 929
 930         if (hcint & (HCINT_ERRORS | HCINT_RETRY |
 931             HCINT_ACK | HCINT_NYET)) {
 932                 if (!(hcint & HCINT_ERRORS))
 933                         td->errcnt = 0;
 934         }
 935
 936 check_state:
 937         switch (td->state) {
 938         case DWC_CHAN_ST_START:
 939                 goto send_pkt;
 940
 941         case DWC_CHAN_ST_WAIT_ANE:
 942                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
 943                         td->did_nak = 1;
 944                         td->tt_scheduled = 0;
 945                         goto send_pkt;
 946                 } else if (hcint & (HCINT_ACK | HCINT_NYET)) {
 947                         td->offset += td->tx_bytes;
 948                         td->remainder -= td->tx_bytes;
 949                         td->toggle = 1;
 950                         td->tt_scheduled = 0;
 951                         goto complete;
 952                 }
 953                 break;
 954
 955         case DWC_CHAN_ST_WAIT_S_ANE:
 956                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
 957                         td->did_nak = 1;
 958                         td->tt_scheduled = 0;
 959                         goto send_pkt;
 960                 } else if (hcint & (HCINT_ACK | HCINT_NYET)) {
 961                         goto send_cpkt;
 962                 }
 963                 break;
 964
 965         case DWC_CHAN_ST_WAIT_C_ANE:
 966                 if (hcint & HCINT_NYET) {
 967                         goto send_cpkt;
 968                 } else if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
 969                         td->did_nak = 1;
 970                         td->tt_scheduled = 0;
 971                         goto send_pkt;
 972                 } else if (hcint & HCINT_ACK) {
 973                         td->offset += td->tx_bytes;
 974                         td->remainder -= td->tx_bytes;
 975                         td->toggle = 1;
 976                         goto complete;
 977                 }
 978                 break;
 979
 980         case DWC_CHAN_ST_WAIT_C_PKT:
 981                 goto send_cpkt;
 982
 983         default:
 984                 break;
 985         }
 986         goto busy;
 987
 988 send_pkt:
 989         /* free existing channel, if any */
 990         dwc_otg_host_channel_free(sc, td);
 991
 992         if (sizeof(req) != td->remainder) {
 993                 td->error_any = 1;
 994                 goto complete;
 995         }
 996
 997         if (td->hcsplt != 0) {
 998                 delta = td->tt_start_slot - sc->sc_last_frame_num - 1;
 999                 if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
1000                         td->state = DWC_CHAN_ST_START;
1001                         goto busy;
1002                 }
1003                 delta = sc->sc_last_frame_num - td->tt_start_slot;
1004                 if (delta > 5) {
1005                         /* missed it */
1006                         td->tt_scheduled = 0;
1007                         td->state = DWC_CHAN_ST_START;
1008                         goto busy;
1009                 }
1010         }
1011
1012         /* allocate a new channel */
1013         if (dwc_otg_host_channel_alloc(sc, td, 1)) {
1014                 td->state = DWC_CHAN_ST_START;
1015                 goto busy;
1016         }
1017
1018         if (td->hcsplt != 0) {
1019                 td->hcsplt &= ~HCSPLT_COMPSPLT;
1020                 td->state = DWC_CHAN_ST_WAIT_S_ANE;
1021         } else {
1022                 td->state = DWC_CHAN_ST_WAIT_ANE;
1023         }
1024
1025         /* copy out control request */
1026         usbd_copy_out(td->pc, 0, &req, sizeof(req));
1027
1028         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(td->channel[0]),
1029             (sizeof(req) << HCTSIZ_XFERSIZE_SHIFT) |
1030             (1 << HCTSIZ_PKTCNT_SHIFT) |
1031             (HCTSIZ_PID_SETUP << HCTSIZ_PID_SHIFT));
1032
1033         DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(td->channel[0]), td->hcsplt);
1034
1035         hcchar = td->hcchar;
1036         hcchar &= ~(HCCHAR_EPDIR_IN | HCCHAR_EPTYPE_MASK);
1037         hcchar |= UE_CONTROL << HCCHAR_EPTYPE_SHIFT;
1038
1039         /* must enable channel before writing data to FIFO */
1040         DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(td->channel[0]), hcchar);
1041
1042         /* transfer data into FIFO */
1043         bus_space_write_region_4(sc->sc_io_tag, sc->sc_io_hdl,
1044             DOTG_DFIFO(td->channel[0]), (uint32_t *)&req, sizeof(req) / 4);
1045
1046         /* wait until next slot before trying complete split */
1047         td->tt_complete_slot = sc->sc_last_frame_num + 1;
1048
1049         /* store number of bytes transmitted */
1050         td->tx_bytes = sizeof(req);
1051         goto busy;
1052
1053 send_cpkt:
1054         /* free existing channel, if any */
1055         dwc_otg_host_channel_free(sc, td);
1056
1057         delta = td->tt_complete_slot - sc->sc_last_frame_num - 1;
1058         if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
1059                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
1060                 goto busy;
1061         }
1062         delta = sc->sc_last_frame_num - td->tt_start_slot;
1063         if (delta > DWC_OTG_TT_SLOT_MAX) {
1064                 /* we missed the service interval */
1065                 if (td->ep_type != UE_ISOCHRONOUS)
1066                         td->error_any = 1;
1067                 goto complete;
1068         }
1069         /* allocate a new channel */
1070         if (dwc_otg_host_channel_alloc(sc, td, 0)) {
1071                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
1072                 goto busy;
1073         }
1074
1075         /* wait until next slot before trying complete split */
1076         td->tt_complete_slot = sc->sc_last_frame_num + 1;
1077
1078         td->hcsplt |= HCSPLT_COMPSPLT;
1079         td->state = DWC_CHAN_ST_WAIT_C_ANE;
1080
1081         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(td->channel[0]),
1082             (HCTSIZ_PID_SETUP << HCTSIZ_PID_SHIFT));
1083
1084         DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(td->channel[0]), td->hcsplt);
1085
1086         hcchar = td->hcchar;
1087         hcchar &= ~(HCCHAR_EPDIR_IN | HCCHAR_EPTYPE_MASK);
1088         hcchar |= UE_CONTROL << HCCHAR_EPTYPE_SHIFT;
1089
1090         /* must enable channel before writing data to FIFO */
1091         DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(td->channel[0]), hcchar);
1092
1093 busy:
1094         return (1);     /* busy */
1095
1096 complete:
1097         dwc_otg_host_channel_free(sc, td);
1098         return (0);     /* complete */
1099 }
1100
1101 static uint8_t
1102 dwc_otg_setup_rx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1103 {
1104         struct usb_device_request req __aligned(4);
1105         uint32_t temp;
1106         uint16_t count;
1107
1108         /* check endpoint status */
1109
1110         if (sc->sc_last_rx_status == 0)
1111                 goto not_complete;
1112
1113         if (GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) != 0)
1114                 goto not_complete;
1115
1116         if ((sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) !=
1117             GRXSTSRD_STP_DATA) {
1118                 if ((sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) !=
1119                     GRXSTSRD_STP_COMPLETE || td->remainder != 0) {
1120                         /* release FIFO */
1121                         dwc_otg_common_rx_ack(sc);
1122                         goto not_complete;
1123                 }
1124                 /* release FIFO */
1125                 dwc_otg_common_rx_ack(sc);
1126                 return (0);     /* complete */
1127         }
1128
1129         if ((sc->sc_last_rx_status & GRXSTSRD_DPID_MASK) !=
1130             GRXSTSRD_DPID_DATA0) {
1131                 /* release FIFO */
1132                 dwc_otg_common_rx_ack(sc);
1133                 goto not_complete;
1134         }
1135
1136         DPRINTFN(5, "GRXSTSR=0x%08x\n", sc->sc_last_rx_status);
1137
1138         /* clear did stall */
1139         td->did_stall = 0;
1140
1141         /* get the packet byte count */
1142         count = GRXSTSRD_BCNT_GET(sc->sc_last_rx_status);
1143
1144         if (count != sizeof(req)) {
1145                 DPRINTFN(0, "Unsupported SETUP packet "
1146                     "length, %d bytes\n", count);
1147                 /* release FIFO */
1148                 dwc_otg_common_rx_ack(sc);
1149                 goto not_complete;
1150         }
1151
1152         /* read FIFO */
1153         dwc_otg_read_fifo(sc, td->pc, 0, sizeof(req));
1154
1155         /* copy out control request */
1156         usbd_copy_out(td->pc, 0, &req, sizeof(req));
1157
1158         td->offset = sizeof(req);
1159         td->remainder = 0;
1160
1161         /* sneak peek the set address */
1162         if ((req.bmRequestType == UT_WRITE_DEVICE) &&
1163             (req.bRequest == UR_SET_ADDRESS)) {
1164                 /* must write address before ZLP */
1165                 dwc_otg_set_address(sc, req.wValue[0] & 0x7F);
1166         }
1167
1168         /* don't send any data by default */
1169         DWC_OTG_WRITE_4(sc, DOTG_DIEPTSIZ(0), DIEPCTL_EPDIS);
1170         DWC_OTG_WRITE_4(sc, DOTG_DOEPTSIZ(0), DOEPCTL_EPDIS);
1171
1172         /* reset IN endpoint buffer */
1173         dwc_otg_tx_fifo_reset(sc,
1174             GRSTCTL_TXFIFO(0) |
1175             GRSTCTL_TXFFLSH);
1176
1177         /* acknowledge RX status */
1178         dwc_otg_common_rx_ack(sc);
1179         td->did_stall = 1;
1180
1181 not_complete:
1182         /* abort any ongoing transfer, before enabling again */
1183         if (!td->did_stall) {
1184                 td->did_stall = 1;
1185
1186                 DPRINTFN(5, "stalling IN and OUT direction\n");
1187
1188                 temp = sc->sc_out_ctl[0];
1189
1190                 /* set stall after enabling endpoint */
1191                 DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(0),
1192                     temp | DOEPCTL_STALL);
1193
1194                 temp = sc->sc_in_ctl[0];
1195
1196                 /* set stall assuming endpoint is enabled */
1197                 DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(0),
1198                     temp | DIEPCTL_STALL);
1199         }
1200         return (1);                     /* not complete */
1201 }
1202
1203 static uint8_t
1204 dwc_otg_host_rate_check_interrupt(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1205 {
1206         uint8_t delta;
1207
1208         delta = sc->sc_tmr_val - td->tmr_val;
1209         if (delta >= 128)
1210                 return (1);     /* busy */
1211
1212         td->tmr_val = sc->sc_tmr_val + td->tmr_res;
1213
1214         /* set toggle, if any */
1215         if (td->set_toggle) {
1216                 td->set_toggle = 0;
1217                 td->toggle = 1;
1218         }
1219         return (0);
1220 }
1221
1222 static uint8_t
1223 dwc_otg_host_rate_check(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1224 {
1225         uint8_t frame_num = (uint8_t)sc->sc_last_frame_num;
1226
1227         if (td->ep_type == UE_ISOCHRONOUS) {
1228                 /* non TT isochronous traffic */
1229                 if (frame_num & (td->tmr_res - 1))
1230                         goto busy;
1231                 if ((frame_num ^ td->tmr_val) & td->tmr_res)
1232                         goto busy;
1233                 td->tmr_val = td->tmr_res + sc->sc_last_frame_num;
1234                 td->toggle = 0;
1235                 return (0);
1236         } else if (td->ep_type == UE_INTERRUPT) {
1237                 if (!td->tt_scheduled)
1238                         goto busy;
1239                 td->tt_scheduled = 0;
1240                 return (0);
1241         } else if (td->did_nak != 0) {
1242                 /* check if we should pause sending queries for 125us */
1243                 if (td->tmr_res == frame_num) {
1244                         /* wait a bit */
1245                         dwc_otg_enable_sof_irq(sc);
1246                         goto busy;
1247                 }
1248         } else if (td->set_toggle) {
1249                 td->set_toggle = 0;
1250                 td->toggle = 1;
1251         }
1252         /* query for data one more time */
1253         td->tmr_res = frame_num;
1254         td->did_nak = 0;
1255         return (0);
1256 busy:
1257         return (1);
1258 }
1259
1260 static uint8_t
1261 dwc_otg_host_data_rx_sub(struct dwc_otg_softc *sc, struct dwc_otg_td *td,
1262     uint8_t channel)
1263 {
1264         uint32_t count;
1265
1266         /* check endpoint status */
1267         if (sc->sc_last_rx_status == 0)
1268                 goto busy;
1269
1270         if (channel >= DWC_OTG_MAX_CHANNELS)
1271                 goto busy;
1272
1273         if (GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) != channel)
1274                 goto busy;
1275
1276         switch (sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) {
1277         case GRXSTSRH_IN_DATA:
1278
1279                 DPRINTF("DATA ST=%d STATUS=0x%08x\n",
1280                     (int)td->state, (int)sc->sc_last_rx_status);
1281
1282                 if (sc->sc_chan_state[channel].hcint & HCINT_SOFTWARE_ONLY) {
1283                         /*
1284                          * When using SPLIT transactions on interrupt
1285                          * endpoints, sometimes data occurs twice.
1286                          */
1287                         DPRINTF("Data already received\n");
1288                         break;
1289                 }
1290
1291                 /* get the packet byte count */
1292                 count = GRXSTSRD_BCNT_GET(sc->sc_last_rx_status);
1293
1294                 /* check for ISOCHRONOUS endpoint */
1295                 if (td->ep_type == UE_ISOCHRONOUS) {
1296                         if ((sc->sc_last_rx_status & GRXSTSRD_DPID_MASK) !=
1297                             GRXSTSRD_DPID_DATA0) {
1298                                 /* more data to be received */
1299                                 td->tt_xactpos = HCSPLT_XACTPOS_MIDDLE;
1300                         } else {
1301                                 /* all data received */
1302                                 td->tt_xactpos = HCSPLT_XACTPOS_BEGIN;
1303                                 /* verify the packet byte count */
1304                                 if (count != td->remainder) {
1305                                         /* we have a short packet */
1306                                         td->short_pkt = 1;
1307                                         td->got_short = 1;
1308                                 }
1309                         }
1310                 } else {
1311                         /* verify the packet byte count */
1312                         if (count != td->max_packet_size) {
1313                                 if (count < td->max_packet_size) {
1314                                         /* we have a short packet */
1315                                         td->short_pkt = 1;
1316                                         td->got_short = 1;
1317                                 } else {
1318                                         /* invalid USB packet */
1319                                         td->error_any = 1;
1320
1321                                         /* release FIFO */
1322                                         dwc_otg_common_rx_ack(sc);
1323                                         goto complete;
1324                                 }
1325                         }
1326                         td->toggle ^= 1;
1327                         td->tt_scheduled = 0;
1328                 }
1329
1330                 /* verify the packet byte count */
1331                 if (count > td->remainder) {
1332                         /* invalid USB packet */
1333                         td->error_any = 1;
1334
1335                         /* release FIFO */
1336                         dwc_otg_common_rx_ack(sc);
1337                         goto complete;
1338                 }
1339
1340                 /* read data from FIFO */
1341                 dwc_otg_read_fifo(sc, td->pc, td->offset, count);
1342
1343                 td->remainder -= count;
1344                 td->offset += count;
1345                 sc->sc_chan_state[channel].hcint |= HCINT_SOFTWARE_ONLY;
1346                 break;
1347         default:
1348                 break;
1349         }
1350         /* release FIFO */
1351         dwc_otg_common_rx_ack(sc);
1352 busy:
1353         return (0);
1354 complete:
1355         return (1);
1356 }
1357
1358 static uint8_t
1359 dwc_otg_host_data_rx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1360 {
1361         uint32_t hcint = 0;
1362         uint32_t hcchar;
1363         uint8_t delta;
1364         uint8_t channel;
1365         uint8_t x;
1366
1367         for (x = 0; x != td->max_packet_count; x++) {
1368                 channel = td->channel[x];
1369                 if (channel >= DWC_OTG_MAX_CHANNELS)
1370                         continue;
1371                 hcint |= sc->sc_chan_state[channel].hcint;
1372
1373                 DPRINTF("CH=%d ST=%d HCINT=0x%08x HCCHAR=0x%08x HCTSIZ=0x%08x\n",
1374                     channel, td->state, hcint,
1375                     DWC_OTG_READ_4(sc, DOTG_HCCHAR(channel)),
1376                     DWC_OTG_READ_4(sc, DOTG_HCTSIZ(channel)));
1377
1378                 /* check interrupt bits */
1379                 if (hcint & (HCINT_RETRY |
1380                     HCINT_ACK | HCINT_NYET)) {
1381                         /* give success bits priority over failure bits */
1382                 } else if (hcint & HCINT_STALL) {
1383                         DPRINTF("CH=%d STALL\n", channel);
1384                         td->error_stall = 1;
1385                         td->error_any = 1;
1386                         goto complete;
1387                 } else if (hcint & HCINT_ERRORS) {
1388                         DPRINTF("CH=%d ERROR\n", channel);
1389                         td->errcnt++;
1390                         if (td->hcsplt != 0 || td->errcnt >= 3) {
1391                                 if (td->ep_type != UE_ISOCHRONOUS) {
1392                                         td->error_any = 1;
1393                                         goto complete;
1394                                 }
1395                         }
1396                 }
1397
1398                 /* check channels for data, if any */
1399                 if (dwc_otg_host_data_rx_sub(sc, td, channel))
1400                         goto complete;
1401
1402                 /* refresh interrupt status */
1403                 hcint |= sc->sc_chan_state[channel].hcint;
1404
1405                 if (hcint & (HCINT_ERRORS | HCINT_RETRY |
1406                     HCINT_ACK | HCINT_NYET)) {
1407                         if (!(hcint & HCINT_ERRORS))
1408                                 td->errcnt = 0;
1409                 }
1410         }
1411
1412         switch (td->state) {
1413         case DWC_CHAN_ST_START:
1414                 if (td->hcsplt != 0)
1415                         goto receive_spkt;
1416                 else
1417                         goto receive_pkt;
1418
1419         case DWC_CHAN_ST_WAIT_ANE:
1420                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1421                         if (td->ep_type == UE_INTERRUPT) {
1422                                 /*
1423                                  * The USB specification does not
1424                                  * mandate a particular data toggle
1425                                  * value for USB INTERRUPT
1426                                  * transfers. Switch the data toggle
1427                                  * value to receive the packet
1428                                  * correctly:
1429                                  */
1430                                 if (hcint & HCINT_DATATGLERR) {
1431                                         DPRINTF("Retrying packet due to "
1432                                             "data toggle error\n");
1433                                         td->toggle ^= 1;
1434                                         goto receive_pkt;
1435                                 }
1436                         } else if (td->ep_type == UE_ISOCHRONOUS) {
1437                                 if (td->hcsplt != 0) {
1438                                         /*
1439                                          * Sometimes the complete
1440                                          * split packet may be queued
1441                                          * too early and the
1442                                          * transaction translator will
1443                                          * return a NAK. Ignore
1444                                          * this message and retry the
1445                                          * complete split instead.
1446                                          */
1447                                         DPRINTF("Retrying complete split\n");
1448                                         goto receive_pkt;
1449                                 }
1450                                 goto complete;
1451                         }
1452                         td->did_nak = 1;
1453                         td->tt_scheduled = 0;
1454                         if (td->hcsplt != 0)
1455                                 goto receive_spkt;
1456                         else
1457                                 goto receive_pkt;
1458                 } else if (hcint & HCINT_NYET) {
1459                         if (td->hcsplt != 0) {
1460                                 /* try again */
1461                                 goto receive_pkt;
1462                         } else {
1463                                 /* not a valid token for IN endpoints */
1464                                 td->error_any = 1;
1465                                 goto complete;
1466                         }
1467                 } else if (hcint & HCINT_ACK) {
1468                         /* wait for data - ACK arrived first */
1469                         if (!(hcint & HCINT_SOFTWARE_ONLY))
1470                                 goto busy;
1471
1472                         if (td->ep_type == UE_ISOCHRONOUS) {
1473                                 /* check if we are complete */
1474                                 if (td->tt_xactpos == HCSPLT_XACTPOS_BEGIN) {
1475                                         goto complete;
1476                                 } else if (td->hcsplt != 0) {
1477                                         goto receive_pkt;
1478                                 } else {
1479                                         /* get more packets */
1480                                         goto busy;
1481                                 }
1482                         } else {
1483                                 /* check if we are complete */
1484                                 if ((td->remainder == 0) || (td->got_short != 0)) {
1485                                         if (td->short_pkt)
1486                                                 goto complete;
1487
1488                                         /*
1489                                          * Else need to receive a zero length
1490                                          * packet.
1491                                          */
1492                                 }
1493                                 td->tt_scheduled = 0;
1494                                 td->did_nak = 0;
1495                                 if (td->hcsplt != 0)
1496                                         goto receive_spkt;
1497                                 else
1498                                         goto receive_pkt;
1499                         }
1500                 }
1501                 break;
1502
1503         case DWC_CHAN_ST_WAIT_S_ANE:
1504                 /*
1505                  * NOTE: The DWC OTG hardware provides a fake ACK in
1506                  * case of interrupt and isochronous transfers:
1507                  */
1508                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1509                         td->did_nak = 1;
1510                         td->tt_scheduled = 0;
1511                         goto receive_spkt;
1512                 } else if (hcint & HCINT_NYET) {
1513                         td->tt_scheduled = 0;
1514                         goto receive_spkt;
1515                 } else if (hcint & HCINT_ACK) {
1516                         td->did_nak = 0;
1517                         goto receive_pkt;
1518                 }
1519                 break;
1520
1521         case DWC_CHAN_ST_WAIT_C_PKT:
1522                 goto receive_pkt;
1523
1524         default:
1525                 break;
1526         }
1527         goto busy;
1528
1529 receive_pkt:
1530         /* free existing channel, if any */
1531         dwc_otg_host_channel_free(sc, td);
1532
1533         if (td->hcsplt != 0) {
1534                 delta = td->tt_complete_slot - sc->sc_last_frame_num - 1;
1535                 if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
1536                         if (td->ep_type != UE_ISOCHRONOUS) {
1537                                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
1538                                 goto busy;
1539                         }
1540                 }
1541                 delta = sc->sc_last_frame_num - td->tt_start_slot;
1542                 if (delta > DWC_OTG_TT_SLOT_MAX) {
1543                         if (td->ep_type != UE_ISOCHRONOUS) {
1544                                 /* we missed the service interval */
1545                                 td->error_any = 1;
1546                         }
1547                         goto complete;
1548                 }
1549                 /* complete split */
1550                 td->hcsplt |= HCSPLT_COMPSPLT;
1551         } else if (dwc_otg_host_rate_check(sc, td)) {
1552                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
1553                 goto busy;
1554         }
1555
1556         /* allocate a new channel */
1557         if (dwc_otg_host_channel_alloc(sc, td, 0)) {
1558                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
1559                 goto busy;
1560         }
1561
1562         /* set toggle, if any */
1563         if (td->set_toggle) {
1564                 td->set_toggle = 0;
1565                 td->toggle = 1;
1566         }
1567
1568         td->state = DWC_CHAN_ST_WAIT_ANE;
1569
1570         for (x = 0; x != td->max_packet_count; x++) {
1571                 channel = td->channel[x];
1572
1573                 /* receive one packet */
1574                 DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1575                     (td->max_packet_size << HCTSIZ_XFERSIZE_SHIFT) |
1576                     (1 << HCTSIZ_PKTCNT_SHIFT) |
1577                     (td->toggle ? (HCTSIZ_PID_DATA1 << HCTSIZ_PID_SHIFT) :
1578                     (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT)));
1579
1580                 DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(channel), td->hcsplt);
1581
1582                 hcchar = td->hcchar;
1583                 hcchar |= HCCHAR_EPDIR_IN;
1584
1585                 if (td->ep_type == UE_ISOCHRONOUS) {
1586                         if (td->hcsplt != 0) {
1587                                 /* continously buffer */
1588                                 if (sc->sc_last_frame_num & 1)
1589                                         hcchar &= ~HCCHAR_ODDFRM;
1590                                 else
1591                                         hcchar |= HCCHAR_ODDFRM;
1592                         } else {
1593                                 /* multi buffer, if any */
1594                                 if (sc->sc_last_frame_num & 1)
1595                                         hcchar |= HCCHAR_ODDFRM;
1596                                 else
1597                                         hcchar &= ~HCCHAR_ODDFRM;
1598                         }
1599                 } else {
1600                         hcchar &= ~HCCHAR_ODDFRM;
1601                 }
1602
1603                 /* must enable channel before data can be received */
1604                 DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(channel), hcchar);
1605         }
1606         /* wait until next slot before trying complete split */
1607         td->tt_complete_slot = sc->sc_last_frame_num + 1;
1608
1609         goto busy;
1610
1611 receive_spkt:
1612         /* free existing channel(s), if any */
1613         dwc_otg_host_channel_free(sc, td);
1614
1615         delta = td->tt_start_slot - sc->sc_last_frame_num - 1;
1616         if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
1617                 td->state = DWC_CHAN_ST_START;
1618                 goto busy;
1619         }
1620         delta = sc->sc_last_frame_num - td->tt_start_slot;
1621         if (delta > 5) {
1622                 /* missed it */
1623                 td->tt_scheduled = 0;
1624                 td->state = DWC_CHAN_ST_START;
1625                 goto busy;
1626         }
1627
1628         /* allocate a new channel */
1629         if (dwc_otg_host_channel_alloc(sc, td, 0)) {
1630                 td->state = DWC_CHAN_ST_START;
1631                 goto busy;
1632         }
1633
1634         channel = td->channel[0];
1635
1636         td->hcsplt &= ~HCSPLT_COMPSPLT;
1637         td->state = DWC_CHAN_ST_WAIT_S_ANE;
1638
1639         /* receive one packet */
1640         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
1641             (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT));
1642
1643         DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(channel), td->hcsplt);
1644
1645         /* send after next SOF event */
1646         if ((sc->sc_last_frame_num & 1) == 0 &&
1647             td->ep_type == UE_ISOCHRONOUS)
1648                 td->hcchar |= HCCHAR_ODDFRM;
1649         else
1650                 td->hcchar &= ~HCCHAR_ODDFRM;
1651
1652         hcchar = td->hcchar;
1653         hcchar |= HCCHAR_EPDIR_IN;
1654
1655         /* wait until next slot before trying complete split */
1656         td->tt_complete_slot = sc->sc_last_frame_num + 1;
1657
1658         /* must enable channel before data can be received */
1659         DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(channel), hcchar);
1660 busy:
1661         return (1);     /* busy */
1662
1663 complete:
1664         dwc_otg_host_channel_free(sc, td);
1665         return (0);     /* complete */
1666 }
1667
1668 static uint8_t
1669 dwc_otg_data_rx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1670 {
1671         uint32_t temp;
1672         uint16_t count;
1673         uint8_t got_short;
1674
1675         got_short = 0;
1676
1677         /* check endpoint status */
1678         if (sc->sc_last_rx_status == 0)
1679                 goto not_complete;
1680
1681         if (GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) != td->ep_no)
1682                 goto not_complete;
1683
1684         /* check for SETUP packet */
1685         if ((sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) ==
1686             GRXSTSRD_STP_DATA ||
1687             (sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) ==
1688             GRXSTSRD_STP_COMPLETE) {
1689                 if (td->remainder == 0) {
1690                         /*
1691                          * We are actually complete and have
1692                          * received the next SETUP
1693                          */
1694                         DPRINTFN(5, "faking complete\n");
1695                         return (0);     /* complete */
1696                 }
1697                 /*
1698                  * USB Host Aborted the transfer.
1699                  */
1700                 td->error_any = 1;
1701                 return (0);             /* complete */
1702         }
1703
1704         if ((sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) !=
1705             GRXSTSRD_OUT_DATA) {
1706                 /* release FIFO */
1707                 dwc_otg_common_rx_ack(sc);
1708                 goto not_complete;
1709         }
1710
1711         /* get the packet byte count */
1712         count = GRXSTSRD_BCNT_GET(sc->sc_last_rx_status);
1713
1714         /* verify the packet byte count */
1715         if (count != td->max_packet_size) {
1716                 if (count < td->max_packet_size) {
1717                         /* we have a short packet */
1718                         td->short_pkt = 1;
1719                         got_short = 1;
1720                 } else {
1721                         /* invalid USB packet */
1722                         td->error_any = 1;
1723
1724                         /* release FIFO */
1725                         dwc_otg_common_rx_ack(sc);
1726                         return (0);     /* we are complete */
1727                 }
1728         }
1729         /* verify the packet byte count */
1730         if (count > td->remainder) {
1731                 /* invalid USB packet */
1732                 td->error_any = 1;
1733
1734                 /* release FIFO */
1735                 dwc_otg_common_rx_ack(sc);
1736                 return (0);             /* we are complete */
1737         }
1738
1739         /* read data from FIFO */
1740         dwc_otg_read_fifo(sc, td->pc, td->offset, count);
1741
1742         td->remainder -= count;
1743         td->offset += count;
1744
1745         /* release FIFO */
1746         dwc_otg_common_rx_ack(sc);
1747
1748         temp = sc->sc_out_ctl[td->ep_no];
1749
1750         /* check for isochronous mode */
1751         if ((temp & DIEPCTL_EPTYPE_MASK) ==
1752             (DIEPCTL_EPTYPE_ISOC << DIEPCTL_EPTYPE_SHIFT)) {
1753                 /* toggle odd or even frame bit */
1754                 if (temp & DIEPCTL_SETD1PID) {
1755                         temp &= ~DIEPCTL_SETD1PID;
1756                         temp |= DIEPCTL_SETD0PID;
1757                 } else {
1758                         temp &= ~DIEPCTL_SETD0PID;
1759                         temp |= DIEPCTL_SETD1PID;
1760                 }
1761                 sc->sc_out_ctl[td->ep_no] = temp;
1762         }
1763
1764         /* check if we are complete */
1765         if ((td->remainder == 0) || got_short) {
1766                 if (td->short_pkt) {
1767                         /* we are complete */
1768                         return (0);
1769                 }
1770                 /* else need to receive a zero length packet */
1771         }
1772
1773 not_complete:
1774
1775         /* enable SETUP and transfer complete interrupt */
1776         if (td->ep_no == 0) {
1777                 DWC_OTG_WRITE_4(sc, DOTG_DOEPTSIZ(0),
1778                     DXEPTSIZ_SET_MULTI(3) |
1779                     DXEPTSIZ_SET_NPKT(1) |
1780                     DXEPTSIZ_SET_NBYTES(td->max_packet_size));
1781         } else {
1782                 /* allow reception of multiple packets */
1783                 DWC_OTG_WRITE_4(sc, DOTG_DOEPTSIZ(td->ep_no),
1784                     DXEPTSIZ_SET_MULTI(1) |
1785                     DXEPTSIZ_SET_NPKT(4) |
1786                     DXEPTSIZ_SET_NBYTES(4 *
1787                         ((td->max_packet_size + 3) & ~3)));
1788         }
1789         temp = sc->sc_out_ctl[td->ep_no];
1790         DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(td->ep_no), temp |
1791             DOEPCTL_EPENA | DOEPCTL_CNAK);
1792
1793         return (1);                     /* not complete */
1794 }
1795
1796 static uint8_t
1797 dwc_otg_host_data_tx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
1798 {
1799         uint32_t count;
1800         uint32_t hcint;
1801         uint32_t hcchar;
1802         uint8_t delta;
1803         uint8_t channel;
1804         uint8_t x;
1805
1806         dwc_otg_host_dump_rx(sc, td);
1807
1808         /* check that last channel is complete */
1809         channel = td->channel[td->npkt];
1810
1811         if (channel < DWC_OTG_MAX_CHANNELS) {
1812                 hcint = sc->sc_chan_state[channel].hcint;
1813
1814                 DPRINTF("CH=%d ST=%d HCINT=0x%08x HCCHAR=0x%08x HCTSIZ=0x%08x\n",
1815                     channel, td->state, hcint,
1816                     DWC_OTG_READ_4(sc, DOTG_HCCHAR(channel)),
1817                     DWC_OTG_READ_4(sc, DOTG_HCTSIZ(channel)));
1818
1819                 if (hcint & (HCINT_RETRY |
1820                     HCINT_ACK | HCINT_NYET)) {
1821                         /* give success bits priority over failure bits */
1822                 } else if (hcint & HCINT_STALL) {
1823                         DPRINTF("CH=%d STALL\n", channel);
1824                         td->error_stall = 1;
1825                         td->error_any = 1;
1826                         goto complete;
1827                 } else if (hcint & HCINT_ERRORS) {
1828                         DPRINTF("CH=%d ERROR\n", channel);
1829                         td->errcnt++;
1830                         if (td->hcsplt != 0 || td->errcnt >= 3) {
1831                                 td->error_any = 1;
1832                                 goto complete;
1833                         }
1834                 }
1835
1836                 if (hcint & (HCINT_ERRORS | HCINT_RETRY |
1837                     HCINT_ACK | HCINT_NYET)) {
1838
1839                         if (!(hcint & HCINT_ERRORS))
1840                                 td->errcnt = 0;
1841                 }
1842         } else {
1843                 hcint = 0;
1844         }
1845
1846         switch (td->state) {
1847         case DWC_CHAN_ST_START:
1848                 goto send_pkt;
1849
1850         case DWC_CHAN_ST_WAIT_ANE:
1851                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1852                         td->did_nak = 1;
1853                         td->tt_scheduled = 0;
1854                         goto send_pkt;
1855                 } else if (hcint & (HCINT_ACK | HCINT_NYET)) {
1856                         td->offset += td->tx_bytes;
1857                         td->remainder -= td->tx_bytes;
1858                         td->toggle ^= 1;
1859                         /* check if next response will be a NAK */
1860                         if (hcint & HCINT_NYET)
1861                                 td->did_nak = 1;
1862                         else
1863                                 td->did_nak = 0;
1864                         td->tt_scheduled = 0;
1865
1866                         /* check remainder */
1867                         if (td->remainder == 0) {
1868                                 if (td->short_pkt)
1869                                         goto complete;
1870
1871                                 /*
1872                                  * Else we need to transmit a short
1873                                  * packet:
1874                                  */
1875                         }
1876                         goto send_pkt;
1877                 }
1878                 break;
1879
1880         case DWC_CHAN_ST_WAIT_S_ANE:
1881                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1882                         td->did_nak = 1;
1883                         td->tt_scheduled = 0;
1884                         goto send_pkt;
1885                 } else if (hcint & (HCINT_ACK | HCINT_NYET)) {
1886                         td->did_nak = 0;
1887                         goto send_cpkt;
1888                 }
1889                 break;
1890
1891         case DWC_CHAN_ST_WAIT_C_ANE:
1892                 if (hcint & HCINT_NYET) {
1893                         goto send_cpkt;
1894                 } else if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1895                         td->did_nak = 1;
1896                         td->tt_scheduled = 0;
1897                         goto send_pkt;
1898                 } else if (hcint & HCINT_ACK) {
1899                         td->offset += td->tx_bytes;
1900                         td->remainder -= td->tx_bytes;
1901                         td->toggle ^= 1;
1902                         td->did_nak = 0;
1903                         td->tt_scheduled = 0;
1904
1905                         /* check remainder */
1906                         if (td->remainder == 0) {
1907                                 if (td->short_pkt)
1908                                         goto complete;
1909
1910                                 /* else we need to transmit a short packet */
1911                         }
1912                         goto send_pkt;
1913                 }
1914                 break;
1915
1916         case DWC_CHAN_ST_WAIT_C_PKT:
1917                 goto send_cpkt;
1918
1919         case DWC_CHAN_ST_TX_WAIT_ISOC:
1920                 /* Check if ISOCHRONOUS OUT traffic is complete */
1921                 if ((hcint & HCINT_HCH_DONE_MASK) == 0)
1922                         break;
1923
1924                 td->offset += td->tx_bytes;
1925                 td->remainder -= td->tx_bytes;
1926                 goto complete;
1927         default:
1928                 break;
1929         }
1930         goto busy;
1931
1932 send_pkt:
1933         /* free existing channel(s), if any */
1934         dwc_otg_host_channel_free(sc, td);
1935
1936         if (td->hcsplt != 0) {
1937                 delta = td->tt_start_slot - sc->sc_last_frame_num - 1;
1938                 if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
1939                         td->state = DWC_CHAN_ST_START;
1940                         goto busy;
1941                 }
1942                 delta = sc->sc_last_frame_num - td->tt_start_slot;
1943                 if (delta > 5) {
1944                         /* missed it */
1945                         td->tt_scheduled = 0;
1946                         td->state = DWC_CHAN_ST_START;
1947                         goto busy;
1948                 }
1949         } else if (dwc_otg_host_rate_check(sc, td)) {
1950                 td->state = DWC_CHAN_ST_START;
1951                 goto busy;
1952         }
1953
1954         /* allocate a new channel */
1955         if (dwc_otg_host_channel_alloc(sc, td, 1)) {
1956                 td->state = DWC_CHAN_ST_START;
1957                 goto busy;
1958         }
1959
1960         /* set toggle, if any */
1961         if (td->set_toggle) {
1962                 td->set_toggle = 0;
1963                 td->toggle = 1;
1964         }
1965
1966         if (td->ep_type == UE_ISOCHRONOUS) {
1967                 /* ISOCHRONOUS OUT transfers don't have any ACKs */
1968                 td->state = DWC_CHAN_ST_TX_WAIT_ISOC;
1969                 td->hcsplt &= ~HCSPLT_COMPSPLT;
1970                 if (td->hcsplt != 0) {
1971                         /* get maximum transfer length */
1972                         count = td->remainder;
1973                         if (count > HCSPLT_XACTLEN_BURST) {
1974                                 DPRINTF("TT overflow\n");
1975                                 td->error_any = 1;
1976                                 goto complete;
1977                         }
1978                         /* Update transaction position */
1979                         td->hcsplt &= ~HCSPLT_XACTPOS_MASK;
1980                         td->hcsplt |= (HCSPLT_XACTPOS_ALL << HCSPLT_XACTPOS_SHIFT);
1981                 }
1982         } else if (td->hcsplt != 0) {
1983                 td->hcsplt &= ~HCSPLT_COMPSPLT;
1984                 /* Wait for ACK/NAK/ERR from TT */
1985                 td->state = DWC_CHAN_ST_WAIT_S_ANE;
1986         } else {
1987                 /* Wait for ACK/NAK/STALL from device */
1988                 td->state = DWC_CHAN_ST_WAIT_ANE;
1989         }
1990
1991         td->tx_bytes = 0;
1992
1993         for (x = 0; x != td->max_packet_count; x++) {
1994                 uint32_t rem_bytes;
1995
1996                 channel = td->channel[x];
1997
1998                 /* send one packet at a time */
1999                 count = td->max_packet_size;
2000                 rem_bytes = td->remainder - td->tx_bytes;
2001                 if (rem_bytes < count) {
2002                         /* we have a short packet */
2003                         td->short_pkt = 1;
2004                         count = rem_bytes;
2005                 }
2006                 if (count == rem_bytes) {
2007                         /* last packet */
2008                         switch (x) {
2009                         case 0:
2010                                 DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
2011                                     (count << HCTSIZ_XFERSIZE_SHIFT) |
2012                                     (1 << HCTSIZ_PKTCNT_SHIFT) |
2013                                     (td->toggle ? (HCTSIZ_PID_DATA1 << HCTSIZ_PID_SHIFT) :
2014                                     (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT)));
2015                                 break;
2016                         case 1:
2017                                 DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
2018                                     (count << HCTSIZ_XFERSIZE_SHIFT) |
2019                                     (1 << HCTSIZ_PKTCNT_SHIFT) |
2020                                     (HCTSIZ_PID_DATA1 << HCTSIZ_PID_SHIFT));
2021                                 break;
2022                         default:
2023                                 DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
2024                                     (count << HCTSIZ_XFERSIZE_SHIFT) |
2025                                     (1 << HCTSIZ_PKTCNT_SHIFT) |
2026                                     (HCTSIZ_PID_DATA2 << HCTSIZ_PID_SHIFT));
2027                                 break;
2028                         }
2029                 } else if (td->ep_type == UE_ISOCHRONOUS &&
2030                            td->max_packet_count > 1) {
2031                         /* ISOCHRONOUS multi packet */
2032                         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
2033                             (count << HCTSIZ_XFERSIZE_SHIFT) |
2034                             (1 << HCTSIZ_PKTCNT_SHIFT) |
2035                             (HCTSIZ_PID_MDATA << HCTSIZ_PID_SHIFT));
2036                 } else {
2037                         /* TODO: HCTSIZ_DOPNG */
2038                         /* standard BULK/INTERRUPT/CONTROL packet */
2039                         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
2040                             (count << HCTSIZ_XFERSIZE_SHIFT) |
2041                             (1 << HCTSIZ_PKTCNT_SHIFT) |
2042                             (td->toggle ? (HCTSIZ_PID_DATA1 << HCTSIZ_PID_SHIFT) :
2043                             (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT)));
2044                 }
2045
2046                 DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(channel), td->hcsplt);
2047
2048                 hcchar = td->hcchar;
2049                 hcchar &= ~HCCHAR_EPDIR_IN;
2050
2051                 /* send after next SOF event */
2052                 if ((sc->sc_last_frame_num & 1) == 0 &&
2053                     td->ep_type == UE_ISOCHRONOUS)
2054                         hcchar |= HCCHAR_ODDFRM;
2055                 else
2056                         hcchar &= ~HCCHAR_ODDFRM;
2057
2058                 /* must enable before writing data to FIFO */
2059                 DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(channel), hcchar);
2060
2061                 if (count != 0) {
2062                         /* write data into FIFO */
2063                         dwc_otg_write_fifo(sc, td->pc, td->offset +
2064                             td->tx_bytes, DOTG_DFIFO(channel), count);
2065                 }
2066
2067                 /* store number of bytes transmitted */
2068                 td->tx_bytes += count;
2069
2070                 /* store last packet index */
2071                 td->npkt = x;
2072
2073                 /* check for last packet */
2074                 if (count == rem_bytes)
2075                         break;
2076         }
2077         goto busy;
2078
2079 send_cpkt:
2080         /* free existing channel, if any */
2081         dwc_otg_host_channel_free(sc, td);
2082
2083         delta = td->tt_complete_slot - sc->sc_last_frame_num - 1;
2084         if (td->tt_scheduled == 0 || delta < DWC_OTG_TT_SLOT_MAX) {
2085                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
2086                 goto busy;
2087         }
2088         delta = sc->sc_last_frame_num - td->tt_start_slot;
2089         if (delta > DWC_OTG_TT_SLOT_MAX) {
2090                 /* we missed the service interval */
2091                 if (td->ep_type != UE_ISOCHRONOUS)
2092                         td->error_any = 1;
2093                 goto complete;
2094         }
2095
2096         /* allocate a new channel */
2097         if (dwc_otg_host_channel_alloc(sc, td, 0)) {
2098                 td->state = DWC_CHAN_ST_WAIT_C_PKT;
2099                 goto busy;
2100         }
2101
2102         channel = td->channel[0];
2103
2104         td->hcsplt |= HCSPLT_COMPSPLT;
2105         td->state = DWC_CHAN_ST_WAIT_C_ANE;
2106
2107         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(channel),
2108             (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT));
2109
2110         DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(channel), td->hcsplt);
2111
2112         hcchar = td->hcchar;
2113         hcchar &= ~HCCHAR_EPDIR_IN;
2114
2115         /* receive complete split ASAP */
2116         if ((sc->sc_last_frame_num & 1) != 0 &&
2117             td->ep_type == UE_ISOCHRONOUS)
2118                 hcchar |= HCCHAR_ODDFRM;
2119         else
2120                 hcchar &= ~HCCHAR_ODDFRM;
2121
2122         /* must enable channel before data can be received */
2123         DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(channel), hcchar);
2124
2125         /* wait until next slot before trying complete split */
2126         td->tt_complete_slot = sc->sc_last_frame_num + 1;
2127 busy:
2128         return (1);     /* busy */
2129
2130 complete:
2131         dwc_otg_host_channel_free(sc, td);
2132         return (0);     /* complete */
2133 }
2134
2135 static uint8_t
2136 dwc_otg_data_tx(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
2137 {
2138         uint32_t max_buffer;
2139         uint32_t count;
2140         uint32_t fifo_left;
2141         uint32_t mpkt;
2142         uint32_t temp;
2143         uint8_t to;
2144
2145         to = 3;                         /* don't loop forever! */
2146
2147         max_buffer = sc->sc_hw_ep_profile[td->ep_no].max_buffer;
2148
2149 repeat:
2150         /* check for for endpoint 0 data */
2151
2152         temp = sc->sc_last_rx_status;
2153
2154         if ((td->ep_no == 0) && (temp != 0) &&
2155             (GRXSTSRD_CHNUM_GET(temp) == 0)) {
2156
2157                 if ((temp & GRXSTSRD_PKTSTS_MASK) !=
2158                     GRXSTSRD_STP_DATA &&
2159                     (temp & GRXSTSRD_PKTSTS_MASK) !=
2160                     GRXSTSRD_STP_COMPLETE) {
2161
2162                         /* dump data - wrong direction */
2163                         dwc_otg_common_rx_ack(sc);
2164                 } else {
2165                         /*
2166                          * The current transfer was cancelled
2167                          * by the USB Host:
2168                          */
2169                         td->error_any = 1;
2170                         return (0);             /* complete */
2171                 }
2172         }
2173
2174         /* fill in more TX data, if possible */
2175         if (td->tx_bytes != 0) {
2176
2177                 uint16_t cpkt;
2178
2179                 /* check if packets have been transferred */
2180                 temp = DWC_OTG_READ_4(sc, DOTG_DIEPTSIZ(td->ep_no));
2181
2182                 /* get current packet number */
2183                 cpkt = DXEPTSIZ_GET_NPKT(temp);
2184
2185                 if (cpkt >= td->npkt) {
2186                         fifo_left = 0;
2187                 } else {
2188                         if (max_buffer != 0) {
2189                                 fifo_left = (td->npkt - cpkt) *
2190                                     td->max_packet_size;
2191
2192                                 if (fifo_left > max_buffer)
2193                                         fifo_left = max_buffer;
2194                         } else {
2195                                 fifo_left = td->max_packet_size;
2196                         }
2197                 }
2198
2199                 count = td->tx_bytes;
2200                 if (count > fifo_left)
2201                         count = fifo_left;
2202
2203                 if (count != 0) {
2204                         /* write data into FIFO */
2205                         dwc_otg_write_fifo(sc, td->pc, td->offset,
2206                             DOTG_DFIFO(td->ep_no), count);
2207
2208                         td->tx_bytes -= count;
2209                         td->remainder -= count;
2210                         td->offset += count;
2211                         td->npkt = cpkt;
2212                 }
2213                 if (td->tx_bytes != 0)
2214                         goto not_complete;
2215
2216                 /* check remainder */
2217                 if (td->remainder == 0) {
2218                         if (td->short_pkt)
2219                                 return (0);     /* complete */
2220
2221                         /* else we need to transmit a short packet */
2222                 }
2223         }
2224
2225         if (!to--)
2226                 goto not_complete;
2227
2228         /* check if not all packets have been transferred */
2229         temp = DWC_OTG_READ_4(sc, DOTG_DIEPTSIZ(td->ep_no));
2230
2231         if (DXEPTSIZ_GET_NPKT(temp) != 0) {
2232
2233                 DPRINTFN(5, "busy ep=%d npkt=%d DIEPTSIZ=0x%08x "
2234                     "DIEPCTL=0x%08x\n", td->ep_no,
2235                     DXEPTSIZ_GET_NPKT(temp),
2236                     temp, DWC_OTG_READ_4(sc, DOTG_DIEPCTL(td->ep_no)));
2237
2238                 goto not_complete;
2239         }
2240
2241         DPRINTFN(5, "rem=%u ep=%d\n", td->remainder, td->ep_no);
2242
2243         /* try to optimise by sending more data */
2244         if ((max_buffer != 0) && ((td->max_packet_size & 3) == 0)) {
2245
2246                 /* send multiple packets at the same time */
2247                 mpkt = max_buffer / td->max_packet_size;
2248
2249                 if (mpkt > 0x3FE)
2250                         mpkt = 0x3FE;
2251
2252                 count = td->remainder;
2253                 if (count > 0x7FFFFF)
2254                         count = 0x7FFFFF - (0x7FFFFF % td->max_packet_size);
2255
2256                 td->npkt = count / td->max_packet_size;
2257
2258                 /*
2259                  * NOTE: We could use 0x3FE instead of "mpkt" in the
2260                  * check below to get more throughput, but then we
2261                  * have a dependency towards non-generic chip features
2262                  * to disable the TX-FIFO-EMPTY interrupts on a per
2263                  * endpoint basis. Increase the maximum buffer size of
2264                  * the IN endpoint to increase the performance.
2265                  */
2266                 if (td->npkt > mpkt) {
2267                         td->npkt = mpkt;
2268                         count = td->max_packet_size * mpkt;
2269                 } else if ((count == 0) || (count % td->max_packet_size)) {
2270                         /* we are transmitting a short packet */
2271                         td->npkt++;
2272                         td->short_pkt = 1;
2273                 }
2274         } else {
2275                 /* send one packet at a time */
2276                 mpkt = 1;
2277                 count = td->max_packet_size;
2278                 if (td->remainder < count) {
2279                         /* we have a short packet */
2280                         td->short_pkt = 1;
2281                         count = td->remainder;
2282                 }
2283                 td->npkt = 1;
2284         }
2285         DWC_OTG_WRITE_4(sc, DOTG_DIEPTSIZ(td->ep_no),
2286             DXEPTSIZ_SET_MULTI(1) |
2287             DXEPTSIZ_SET_NPKT(td->npkt) |
2288             DXEPTSIZ_SET_NBYTES(count));
2289
2290         /* make room for buffering */
2291         td->npkt += mpkt;
2292
2293         temp = sc->sc_in_ctl[td->ep_no];
2294
2295         /* check for isochronous mode */
2296         if ((temp & DIEPCTL_EPTYPE_MASK) ==
2297             (DIEPCTL_EPTYPE_ISOC << DIEPCTL_EPTYPE_SHIFT)) {
2298                 /* toggle odd or even frame bit */
2299                 if (temp & DIEPCTL_SETD1PID) {
2300                         temp &= ~DIEPCTL_SETD1PID;
2301                         temp |= DIEPCTL_SETD0PID;
2302                 } else {
2303                         temp &= ~DIEPCTL_SETD0PID;
2304                         temp |= DIEPCTL_SETD1PID;
2305                 }
2306                 sc->sc_in_ctl[td->ep_no] = temp;
2307         }
2308
2309         /* must enable before writing data to FIFO */
2310         DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(td->ep_no), temp |
2311             DIEPCTL_EPENA | DIEPCTL_CNAK);
2312
2313         td->tx_bytes = count;
2314
2315         /* check remainder */
2316         if (td->tx_bytes == 0 &&
2317             td->remainder == 0) {
2318                 if (td->short_pkt)
2319                         return (0);     /* complete */
2320
2321                 /* else we need to transmit a short packet */
2322         }
2323         goto repeat;
2324
2325 not_complete:
2326         return (1);                     /* not complete */
2327 }
2328
2329 static uint8_t
2330 dwc_otg_data_tx_sync(struct dwc_otg_softc *sc, struct dwc_otg_td *td)
2331 {
2332         uint32_t temp;
2333
2334         /*
2335          * If all packets are transferred we are complete:
2336          */
2337         temp = DWC_OTG_READ_4(sc, DOTG_DIEPTSIZ(td->ep_no));
2338
2339         /* check that all packets have been transferred */
2340         if (DXEPTSIZ_GET_NPKT(temp) != 0) {
2341                 DPRINTFN(5, "busy ep=%d\n", td->ep_no);
2342                 goto not_complete;
2343         }
2344         return (0);
2345
2346 not_complete:
2347
2348         /* we only want to know if there is a SETUP packet or free IN packet */
2349
2350         temp = sc->sc_last_rx_status;
2351
2352         if ((td->ep_no == 0) && (temp != 0) &&
2353             (GRXSTSRD_CHNUM_GET(temp) == 0)) {
2354
2355                 if ((temp & GRXSTSRD_PKTSTS_MASK) ==
2356                     GRXSTSRD_STP_DATA ||
2357                     (temp & GRXSTSRD_PKTSTS_MASK) ==
2358                     GRXSTSRD_STP_COMPLETE) {
2359                         DPRINTFN(5, "faking complete\n");
2360                         /*
2361                          * Race condition: We are complete!
2362                          */
2363                         return (0);
2364                 } else {
2365                         /* dump data - wrong direction */
2366                         dwc_otg_common_rx_ack(sc);
2367                 }
2368         }
2369         return (1);                     /* not complete */
2370 }
2371
2372 static void
2373 dwc_otg_xfer_do_fifo(struct dwc_otg_softc *sc, struct usb_xfer *xfer)
2374 {
2375         struct dwc_otg_td *td;
2376         uint8_t toggle;
2377         uint8_t tmr_val;
2378         uint8_t tmr_res;
2379
2380         DPRINTFN(9, "\n");
2381
2382         td = xfer->td_transfer_cache;
2383         if (td == NULL)
2384                 return;
2385
2386         while (1) {
2387                 if ((td->func) (sc, td)) {
2388                         /* operation in progress */
2389                         break;
2390                 }
2391                 if (((void *)td) == xfer->td_transfer_last) {
2392                         goto done;
2393                 }
2394                 if (td->error_any) {
2395                         goto done;
2396                 } else if (td->remainder > 0) {
2397                         /*
2398                          * We had a short transfer. If there is no alternate
2399                          * next, stop processing !
2400                          */
2401                         if (!td->alt_next)
2402                                 goto done;
2403                 }
2404
2405                 /*
2406                  * Fetch the next transfer descriptor and transfer
2407                  * some flags to the next transfer descriptor
2408                  */
2409                 tmr_res = td->tmr_res;
2410                 tmr_val = td->tmr_val;
2411                 toggle = td->toggle;
2412                 td = td->obj_next;
2413                 xfer->td_transfer_cache = td;
2414                 td->toggle = toggle;    /* transfer toggle */
2415                 td->tmr_res = tmr_res;
2416                 td->tmr_val = tmr_val;
2417         }
2418         return;
2419
2420 done:
2421         xfer->td_transfer_cache = NULL;
2422         sc->sc_xfer_complete = 1;
2423 }
2424
2425 static uint8_t
2426 dwc_otg_xfer_do_complete_locked(struct dwc_otg_softc *sc, struct usb_xfer *xfer)
2427 {
2428         struct dwc_otg_td *td;
2429
2430         DPRINTFN(9, "\n");
2431
2432         td = xfer->td_transfer_cache;
2433         if (td == NULL) {
2434                 /* compute all actual lengths */
2435                 dwc_otg_standard_done(xfer);
2436                 return (1);
2437         }
2438         return (0);
2439 }
2440
2441 static void
2442 dwc_otg_timer(void *_sc)
2443 {
2444         struct dwc_otg_softc *sc = _sc;
2445
2446         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
2447
2448         DPRINTF("\n");
2449
2450         USB_BUS_SPIN_LOCK(&sc->sc_bus);
2451
2452         /* increment timer value */
2453         sc->sc_tmr_val++;
2454
2455         /* enable SOF interrupt, which will poll jobs */
2456         dwc_otg_enable_sof_irq(sc);
2457
2458         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
2459
2460         if (sc->sc_timer_active) {
2461                 /* restart timer */
2462                 usb_callout_reset(&sc->sc_timer,
2463                     hz / (1000 / DWC_OTG_HOST_TIMER_RATE),
2464                     &dwc_otg_timer, sc);
2465         }
2466 }
2467
2468 static void
2469 dwc_otg_timer_start(struct dwc_otg_softc *sc)
2470 {
2471         if (sc->sc_timer_active != 0)
2472                 return;
2473
2474         sc->sc_timer_active = 1;
2475
2476         /* restart timer */
2477         usb_callout_reset(&sc->sc_timer,
2478             hz / (1000 / DWC_OTG_HOST_TIMER_RATE),
2479             &dwc_otg_timer, sc);
2480 }
2481
2482 static void
2483 dwc_otg_timer_stop(struct dwc_otg_softc *sc)
2484 {
2485         if (sc->sc_timer_active == 0)
2486                 return;
2487
2488         sc->sc_timer_active = 0;
2489
2490         /* stop timer */
2491         usb_callout_stop(&sc->sc_timer);
2492 }
2493
2494 static uint16_t
2495 dwc_otg_compute_isoc_rx_tt_slot(struct dwc_otg_tt_info *pinfo)
2496 {
2497         if (pinfo->slot_index < DWC_OTG_TT_SLOT_MAX)
2498                 pinfo->slot_index++;
2499         return (pinfo->slot_index);
2500 }
2501
2502 static uint8_t
2503 dwc_otg_update_host_transfer_schedule_locked(struct dwc_otg_softc *sc)
2504 {
2505         TAILQ_HEAD(, usb_xfer) head;
2506         struct usb_xfer *xfer;
2507         struct usb_xfer *xfer_next;
2508         struct dwc_otg_td *td;
2509         uint16_t temp;
2510         uint16_t slot;
2511
2512         temp = DWC_OTG_READ_4(sc, DOTG_HFNUM) & DWC_OTG_FRAME_MASK;
2513
2514         if (sc->sc_last_frame_num == temp)
2515                 return (0);
2516
2517         sc->sc_last_frame_num = temp;
2518
2519         TAILQ_INIT(&head);
2520
2521         if ((temp & 7) == 0) {
2522
2523                 /* reset the schedule */
2524                 memset(sc->sc_tt_info, 0, sizeof(sc->sc_tt_info));
2525
2526                 TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2527                         td = xfer->td_transfer_cache;
2528                         if (td == NULL || td->ep_type != UE_ISOCHRONOUS)
2529                                 continue;
2530
2531                         /* check for IN direction */
2532                         if ((td->hcchar & HCCHAR_EPDIR_IN) != 0)
2533                                 continue;
2534
2535                         sc->sc_needsof = 1;
2536
2537                         if (td->hcsplt == 0 || td->tt_scheduled != 0)
2538                                 continue;
2539
2540                         /* compute slot */
2541                         slot = dwc_otg_compute_isoc_rx_tt_slot(
2542                             sc->sc_tt_info + td->tt_index);
2543                         if (slot > 3) {
2544                                 /*
2545                                  * Not enough time to get complete
2546                                  * split executed.
2547                                  */
2548                                 continue;
2549                         }
2550                         /* Delayed start */
2551                         td->tt_start_slot = temp + slot;
2552                         td->tt_scheduled = 1;
2553                         TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2554                         TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2555                 }
2556
2557                 TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2558                         td = xfer->td_transfer_cache;
2559                         if (td == NULL || td->ep_type != UE_ISOCHRONOUS)
2560                                 continue;
2561
2562                         /* check for OUT direction */
2563                         if ((td->hcchar & HCCHAR_EPDIR_IN) == 0)
2564                                 continue;
2565
2566                         sc->sc_needsof = 1;
2567
2568                         if (td->hcsplt == 0 || td->tt_scheduled != 0)
2569                                 continue;
2570
2571                         /* Start ASAP */
2572                         td->tt_start_slot = temp;
2573                         td->tt_scheduled = 1;
2574                         TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2575                         TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2576                 }
2577
2578                 TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2579                         td = xfer->td_transfer_cache;
2580                         if (td == NULL || td->ep_type != UE_INTERRUPT)
2581                                 continue;
2582
2583                         if (td->tt_scheduled != 0) {
2584                                 sc->sc_needsof = 1;
2585                                 continue;
2586                         }
2587
2588                         if (dwc_otg_host_rate_check_interrupt(sc, td))
2589                                 continue;
2590
2591                         if (td->hcsplt == 0) {
2592                                 sc->sc_needsof = 1;
2593                                 td->tt_scheduled = 1;
2594                                 continue;
2595                         }
2596
2597                         /* start ASAP */
2598                         td->tt_start_slot = temp;
2599                         sc->sc_needsof = 1;
2600                         td->tt_scheduled = 1;
2601                         TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2602                         TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2603                 }
2604
2605                 TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2606                         td = xfer->td_transfer_cache;
2607                         if (td == NULL ||
2608                             td->ep_type != UE_CONTROL) {
2609                                 continue;
2610                         }
2611
2612                         sc->sc_needsof = 1;
2613
2614                         if (td->hcsplt == 0 || td->tt_scheduled != 0)
2615                                 continue;
2616
2617                         /* start ASAP */
2618                         td->tt_start_slot = temp;
2619                         td->tt_scheduled = 1;
2620                         TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2621                         TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2622                 }
2623         }
2624         if ((temp & 7) < 6) {
2625                 TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2626                         td = xfer->td_transfer_cache;
2627                         if (td == NULL ||
2628                             td->ep_type != UE_BULK) {
2629                                 continue;
2630                         }
2631
2632                         sc->sc_needsof = 1;
2633
2634                         if (td->hcsplt == 0 || td->tt_scheduled != 0)
2635                                 continue;
2636
2637                         /* start ASAP */
2638                         td->tt_start_slot = temp;
2639                         td->tt_scheduled = 1;
2640                         TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2641                         TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2642                 }
2643         }
2644
2645         /* Put TT transfers in execution order at the end */
2646         TAILQ_CONCAT(&sc->sc_bus.intr_q.head, &head, wait_entry);
2647
2648         /* move all TT transfers in front, keeping the current order */
2649         TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2650                 td = xfer->td_transfer_cache;
2651                 if (td == NULL || td->hcsplt == 0)
2652                         continue;
2653                 TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2654                 TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2655         }
2656         TAILQ_CONCAT(&head, &sc->sc_bus.intr_q.head, wait_entry);
2657         TAILQ_CONCAT(&sc->sc_bus.intr_q.head, &head, wait_entry);
2658
2659         /* put non-TT non-ISOCHRONOUS transfers last */
2660         TAILQ_FOREACH_SAFE(xfer, &sc->sc_bus.intr_q.head, wait_entry, xfer_next) {
2661                 td = xfer->td_transfer_cache;
2662                 if (td == NULL || td->hcsplt != 0 || td->ep_type == UE_ISOCHRONOUS)
2663                         continue;
2664                 TAILQ_REMOVE(&sc->sc_bus.intr_q.head, xfer, wait_entry);
2665                 TAILQ_INSERT_TAIL(&head, xfer, wait_entry);
2666         }
2667         TAILQ_CONCAT(&sc->sc_bus.intr_q.head, &head, wait_entry);
2668
2669         if ((temp & 7) == 0) {
2670
2671                 DPRINTFN(12, "SOF interrupt #%d, needsof=%d\n",
2672                     (int)temp, (int)sc->sc_needsof);
2673
2674                 /* update SOF IRQ mask */
2675                 if (sc->sc_irq_mask & GINTMSK_SOFMSK) {
2676                         if (sc->sc_needsof == 0) {
2677                                 sc->sc_irq_mask &= ~GINTMSK_SOFMSK;
2678                                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2679                         }
2680                 } else {
2681                         if (sc->sc_needsof != 0) {
2682                                 sc->sc_irq_mask |= GINTMSK_SOFMSK;
2683                                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2684                         }
2685                 }
2686
2687                 /* clear need SOF flag */
2688                 sc->sc_needsof = 0;
2689         }
2690         return (1);
2691 }
2692
2693 static void
2694 dwc_otg_interrupt_poll_locked(struct dwc_otg_softc *sc)
2695 {
2696         struct usb_xfer *xfer;
2697         uint32_t count;
2698         uint32_t temp;
2699         uint32_t haint;
2700         uint8_t got_rx_status;
2701         uint8_t x;
2702
2703         if (sc->sc_flags.status_device_mode == 0) {
2704                 /*
2705                  * Update host transfer schedule, so that new
2706                  * transfers can be issued:
2707                  */
2708                 dwc_otg_update_host_transfer_schedule_locked(sc);
2709         }
2710         count = 0;
2711 repeat:
2712         if (++count == 16) {
2713                 /* give other interrupts a chance */
2714                 DPRINTF("Yield\n");
2715                 return;
2716         }
2717
2718         /* get all host channel interrupts */
2719         haint = DWC_OTG_READ_4(sc, DOTG_HAINT);
2720         while (1) {
2721                 x = ffs(haint) - 1;
2722                 if (x >= sc->sc_host_ch_max)
2723                         break;
2724                 temp = DWC_OTG_READ_4(sc, DOTG_HCINT(x));
2725                 DWC_OTG_WRITE_4(sc, DOTG_HCINT(x), temp);
2726                 temp &= ~HCINT_SOFTWARE_ONLY;
2727                 sc->sc_chan_state[x].hcint |= temp;
2728                 haint &= ~(1U << x);
2729         }
2730
2731         if (sc->sc_last_rx_status == 0) {
2732
2733                 temp = DWC_OTG_READ_4(sc, DOTG_GINTSTS);
2734                 if (temp & GINTSTS_RXFLVL) {
2735                         /* pop current status */
2736                         sc->sc_last_rx_status =
2737                             DWC_OTG_READ_4(sc, DOTG_GRXSTSPD);
2738                 }
2739
2740                 if (sc->sc_last_rx_status != 0) {
2741
2742                         uint8_t ep_no;
2743
2744                         temp = sc->sc_last_rx_status &
2745                             GRXSTSRD_PKTSTS_MASK;
2746
2747                         /* non-data messages we simply skip */
2748                         if (temp != GRXSTSRD_STP_DATA &&
2749                             temp != GRXSTSRD_STP_COMPLETE &&
2750                             temp != GRXSTSRD_OUT_DATA) {
2751                                 /* check for halted channel */
2752                                 if (temp == GRXSTSRH_HALTED) {
2753                                         ep_no = GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status);
2754                                         sc->sc_chan_state[ep_no].wait_halted = 0;
2755                                         DPRINTFN(5, "channel halt complete ch=%u\n", ep_no);
2756                                 }
2757                                 /* store bytes and FIFO offset */
2758                                 sc->sc_current_rx_bytes = 0;
2759                                 sc->sc_current_rx_fifo = 0;
2760
2761                                 /* acknowledge status */
2762                                 dwc_otg_common_rx_ack(sc);
2763                                 goto repeat;
2764                         }
2765
2766                         temp = GRXSTSRD_BCNT_GET(
2767                             sc->sc_last_rx_status);
2768                         ep_no = GRXSTSRD_CHNUM_GET(
2769                             sc->sc_last_rx_status);
2770
2771                         /* store bytes and FIFO offset */
2772                         sc->sc_current_rx_bytes = (temp + 3) & ~3;
2773                         sc->sc_current_rx_fifo = DOTG_DFIFO(ep_no);
2774
2775                         DPRINTF("Reading %d bytes from ep %d\n", temp, ep_no);
2776
2777                         /* check if we should dump the data */
2778                         if (!(sc->sc_active_rx_ep & (1U << ep_no))) {
2779                                 dwc_otg_common_rx_ack(sc);
2780                                 goto repeat;
2781                         }
2782
2783                         got_rx_status = 1;
2784
2785                         DPRINTFN(5, "RX status = 0x%08x: ch=%d pid=%d bytes=%d sts=%d\n",
2786                             sc->sc_last_rx_status, ep_no,
2787                             (sc->sc_last_rx_status >> 15) & 3,
2788                             GRXSTSRD_BCNT_GET(sc->sc_last_rx_status),
2789                             (sc->sc_last_rx_status >> 17) & 15);
2790                 } else {
2791                         got_rx_status = 0;
2792                 }
2793         } else {
2794                 uint8_t ep_no;
2795
2796                 ep_no = GRXSTSRD_CHNUM_GET(
2797                     sc->sc_last_rx_status);
2798
2799                 /* check if we should dump the data */
2800                 if (!(sc->sc_active_rx_ep & (1U << ep_no))) {
2801                         dwc_otg_common_rx_ack(sc);
2802                         goto repeat;
2803                 }
2804
2805                 got_rx_status = 1;
2806         }
2807
2808         /* execute FIFOs */
2809         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry)
2810                 dwc_otg_xfer_do_fifo(sc, xfer);
2811
2812         if (got_rx_status) {
2813                 /* check if data was consumed */
2814                 if (sc->sc_last_rx_status == 0)
2815                         goto repeat;
2816
2817                 /* disable RX FIFO level interrupt */
2818                 sc->sc_irq_mask &= ~GINTMSK_RXFLVLMSK;
2819                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2820         }
2821 }
2822
2823 static void
2824 dwc_otg_interrupt_complete_locked(struct dwc_otg_softc *sc)
2825 {
2826         struct usb_xfer *xfer;
2827 repeat:
2828         /* scan for completion events */
2829         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
2830                 if (dwc_otg_xfer_do_complete_locked(sc, xfer))
2831                         goto repeat;
2832         }
2833 }
2834
2835 static void
2836 dwc_otg_vbus_interrupt(struct dwc_otg_softc *sc, uint8_t is_on)
2837 {
2838         DPRINTFN(5, "vbus = %u\n", is_on);
2839
2840         /*
2841          * If the USB host mode is forced, then assume VBUS is always
2842          * present else rely on the input to this function:
2843          */
2844         if ((is_on != 0) || (sc->sc_mode == DWC_MODE_HOST)) {
2845
2846                 if (!sc->sc_flags.status_vbus) {
2847                         sc->sc_flags.status_vbus = 1;
2848
2849                         /* complete root HUB interrupt endpoint */
2850
2851                         dwc_otg_root_intr(sc);
2852                 }
2853         } else {
2854                 if (sc->sc_flags.status_vbus) {
2855                         sc->sc_flags.status_vbus = 0;
2856                         sc->sc_flags.status_bus_reset = 0;
2857                         sc->sc_flags.status_suspend = 0;
2858                         sc->sc_flags.change_suspend = 0;
2859                         sc->sc_flags.change_connect = 1;
2860
2861                         /* complete root HUB interrupt endpoint */
2862
2863                         dwc_otg_root_intr(sc);
2864                 }
2865         }
2866 }
2867
2868 int
2869 dwc_otg_filter_interrupt(void *arg)
2870 {
2871         struct dwc_otg_softc *sc = arg;
2872         int retval = FILTER_HANDLED;
2873         uint32_t status;
2874
2875         USB_BUS_SPIN_LOCK(&sc->sc_bus);
2876
2877         /* read and clear interrupt status */
2878         status = DWC_OTG_READ_4(sc, DOTG_GINTSTS);
2879
2880         /* clear interrupts we are handling here */
2881         DWC_OTG_WRITE_4(sc, DOTG_GINTSTS, status & ~DWC_OTG_MSK_GINT_THREAD_IRQ);
2882
2883         /* check for USB state change interrupts */
2884         if ((status & DWC_OTG_MSK_GINT_THREAD_IRQ) != 0)
2885                 retval = FILTER_SCHEDULE_THREAD;
2886
2887         /* clear FIFO empty interrupts */
2888         if (status & sc->sc_irq_mask &
2889             (GINTSTS_PTXFEMP | GINTSTS_NPTXFEMP)) {
2890                 sc->sc_irq_mask &= ~(GINTSTS_PTXFEMP | GINTSTS_NPTXFEMP);
2891                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2892         }
2893         /* clear all IN endpoint interrupts */
2894         if (status & GINTSTS_IEPINT) {
2895                 uint32_t temp;
2896                 uint8_t x;
2897
2898                 for (x = 0; x != sc->sc_dev_in_ep_max; x++) {
2899                         temp = DWC_OTG_READ_4(sc, DOTG_DIEPINT(x));
2900                         /*
2901                          * NOTE: Need to clear all interrupt bits,
2902                          * because some appears to be unmaskable and
2903                          * can cause an interrupt loop:
2904                          */
2905                         if (temp != 0)
2906                                 DWC_OTG_WRITE_4(sc, DOTG_DIEPINT(x), temp);
2907                 }
2908         }
2909
2910         /* poll FIFOs, if any */
2911         dwc_otg_interrupt_poll_locked(sc);
2912
2913         if (sc->sc_xfer_complete != 0)
2914                 retval = FILTER_SCHEDULE_THREAD;
2915
2916         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
2917
2918         return (retval);
2919 }
2920
2921 void
2922 dwc_otg_interrupt(void *arg)
2923 {
2924         struct dwc_otg_softc *sc = arg;
2925         uint32_t status;
2926
2927         USB_BUS_LOCK(&sc->sc_bus);
2928         USB_BUS_SPIN_LOCK(&sc->sc_bus);
2929
2930         /* read and clear interrupt status */
2931         status = DWC_OTG_READ_4(sc, DOTG_GINTSTS);
2932
2933         /* clear interrupts we are handling here */
2934         DWC_OTG_WRITE_4(sc, DOTG_GINTSTS, status & DWC_OTG_MSK_GINT_THREAD_IRQ);
2935
2936         DPRINTFN(14, "GINTSTS=0x%08x HAINT=0x%08x HFNUM=0x%08x\n",
2937             status, DWC_OTG_READ_4(sc, DOTG_HAINT),
2938             DWC_OTG_READ_4(sc, DOTG_HFNUM));
2939
2940         if (status & GINTSTS_USBRST) {
2941
2942                 /* set correct state */
2943                 sc->sc_flags.status_device_mode = 1;
2944                 sc->sc_flags.status_bus_reset = 0;
2945                 sc->sc_flags.status_suspend = 0;
2946                 sc->sc_flags.change_suspend = 0;
2947                 sc->sc_flags.change_connect = 1;
2948
2949                 /* Disable SOF interrupt */
2950                 sc->sc_irq_mask &= ~GINTMSK_SOFMSK;
2951                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2952
2953                 /* complete root HUB interrupt endpoint */
2954                 dwc_otg_root_intr(sc);
2955         }
2956
2957         /* check for any bus state change interrupts */
2958         if (status & GINTSTS_ENUMDONE) {
2959
2960                 uint32_t temp;
2961
2962                 DPRINTFN(5, "end of reset\n");
2963
2964                 /* set correct state */
2965                 sc->sc_flags.status_device_mode = 1;
2966                 sc->sc_flags.status_bus_reset = 1;
2967                 sc->sc_flags.status_suspend = 0;
2968                 sc->sc_flags.change_suspend = 0;
2969                 sc->sc_flags.change_connect = 1;
2970                 sc->sc_flags.status_low_speed = 0;
2971                 sc->sc_flags.port_enabled = 1;
2972
2973                 /* reset FIFOs */
2974                 (void) dwc_otg_init_fifo(sc, DWC_MODE_DEVICE);
2975
2976                 /* reset function address */
2977                 dwc_otg_set_address(sc, 0);
2978
2979                 /* figure out enumeration speed */
2980                 temp = DWC_OTG_READ_4(sc, DOTG_DSTS);
2981                 if (DSTS_ENUMSPD_GET(temp) == DSTS_ENUMSPD_HI)
2982                         sc->sc_flags.status_high_speed = 1;
2983                 else
2984                         sc->sc_flags.status_high_speed = 0;
2985
2986                 /*
2987                  * Disable resume and SOF interrupt, and enable
2988                  * suspend and RX frame interrupt:
2989                  */
2990                 sc->sc_irq_mask &= ~(GINTMSK_WKUPINTMSK | GINTMSK_SOFMSK);
2991                 sc->sc_irq_mask |= GINTMSK_USBSUSPMSK;
2992                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2993
2994                 /* complete root HUB interrupt endpoint */
2995                 dwc_otg_root_intr(sc);
2996         }
2997
2998         if (status & GINTSTS_PRTINT) {
2999                 uint32_t hprt;
3000
3001                 hprt = DWC_OTG_READ_4(sc, DOTG_HPRT);
3002
3003                 /* clear change bits */
3004                 DWC_OTG_WRITE_4(sc, DOTG_HPRT, (hprt & (
3005                     HPRT_PRTPWR | HPRT_PRTENCHNG |
3006                     HPRT_PRTCONNDET | HPRT_PRTOVRCURRCHNG)) |
3007                     sc->sc_hprt_val);
3008
3009                 DPRINTFN(12, "GINTSTS=0x%08x, HPRT=0x%08x\n", status, hprt);
3010
3011                 sc->sc_flags.status_device_mode = 0;
3012
3013                 if (hprt & HPRT_PRTCONNSTS)
3014                         sc->sc_flags.status_bus_reset = 1;
3015                 else
3016                         sc->sc_flags.status_bus_reset = 0;
3017
3018                 if ((hprt & HPRT_PRTENCHNG) &&
3019                     (hprt & HPRT_PRTENA) == 0)
3020                         sc->sc_flags.change_enabled = 1;
3021
3022                 if (hprt & HPRT_PRTENA)
3023                         sc->sc_flags.port_enabled = 1;
3024                 else
3025                         sc->sc_flags.port_enabled = 0;
3026
3027                 if (hprt & HPRT_PRTOVRCURRCHNG)
3028                         sc->sc_flags.change_over_current = 1;
3029
3030                 if (hprt & HPRT_PRTOVRCURRACT)
3031                         sc->sc_flags.port_over_current = 1;
3032                 else
3033                         sc->sc_flags.port_over_current = 0;
3034
3035                 if (hprt & HPRT_PRTPWR)
3036                         sc->sc_flags.port_powered = 1;
3037                 else
3038                         sc->sc_flags.port_powered = 0;
3039
3040                 if (((hprt & HPRT_PRTSPD_MASK)
3041                     >> HPRT_PRTSPD_SHIFT) == HPRT_PRTSPD_LOW)
3042                         sc->sc_flags.status_low_speed = 1;
3043                 else
3044                         sc->sc_flags.status_low_speed = 0;
3045
3046                 if (((hprt & HPRT_PRTSPD_MASK)
3047                     >> HPRT_PRTSPD_SHIFT) == HPRT_PRTSPD_HIGH)
3048                         sc->sc_flags.status_high_speed = 1;
3049                 else
3050                         sc->sc_flags.status_high_speed = 0;
3051
3052                 if (hprt & HPRT_PRTCONNDET)
3053                         sc->sc_flags.change_connect = 1;
3054
3055                 if (hprt & HPRT_PRTSUSP)
3056                         dwc_otg_suspend_irq(sc);
3057                 else
3058                         dwc_otg_resume_irq(sc);
3059
3060                 /* complete root HUB interrupt endpoint */
3061                 dwc_otg_root_intr(sc);
3062
3063                 /* update host frame interval */
3064                 dwc_otg_update_host_frame_interval(sc);
3065         }
3066
3067         /*
3068          * If resume and suspend is set at the same time we interpret
3069          * that like RESUME. Resume is set when there is at least 3
3070          * milliseconds of inactivity on the USB BUS.
3071          */
3072         if (status & GINTSTS_WKUPINT) {
3073
3074                 DPRINTFN(5, "resume interrupt\n");
3075
3076                 dwc_otg_resume_irq(sc);
3077
3078         } else if (status & GINTSTS_USBSUSP) {
3079
3080                 DPRINTFN(5, "suspend interrupt\n");
3081
3082                 dwc_otg_suspend_irq(sc);
3083         }
3084         /* check VBUS */
3085         if (status & (GINTSTS_USBSUSP |
3086             GINTSTS_USBRST |
3087             GINTMSK_OTGINTMSK |
3088             GINTSTS_SESSREQINT)) {
3089                 uint32_t temp;
3090
3091                 temp = DWC_OTG_READ_4(sc, DOTG_GOTGCTL);
3092
3093                 DPRINTFN(5, "GOTGCTL=0x%08x\n", temp);
3094
3095                 dwc_otg_vbus_interrupt(sc,
3096                     (temp & (GOTGCTL_ASESVLD | GOTGCTL_BSESVLD)) ? 1 : 0);
3097         }
3098
3099         if (sc->sc_xfer_complete != 0) {
3100                 sc->sc_xfer_complete = 0;
3101
3102                 /* complete FIFOs, if any */
3103                 dwc_otg_interrupt_complete_locked(sc);
3104         }
3105         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3106         USB_BUS_UNLOCK(&sc->sc_bus);
3107 }
3108
3109 static void
3110 dwc_otg_setup_standard_chain_sub(struct dwc_otg_std_temp *temp)
3111 {
3112         struct dwc_otg_td *td;
3113
3114         /* get current Transfer Descriptor */
3115         td = temp->td_next;
3116         temp->td = td;
3117
3118         /* prepare for next TD */
3119         temp->td_next = td->obj_next;
3120
3121         /* fill out the Transfer Descriptor */
3122         td->func = temp->func;
3123         td->pc = temp->pc;
3124         td->offset = temp->offset;
3125         td->remainder = temp->len;
3126         td->tx_bytes = 0;
3127         td->error_any = 0;
3128         td->error_stall = 0;
3129         td->npkt = 0;
3130         td->did_stall = temp->did_stall;
3131         td->short_pkt = temp->short_pkt;
3132         td->alt_next = temp->setup_alt_next;
3133         td->set_toggle = 0;
3134         td->got_short = 0;
3135         td->did_nak = 0;
3136         td->channel[0] = DWC_OTG_MAX_CHANNELS;
3137         td->channel[1] = DWC_OTG_MAX_CHANNELS;
3138         td->channel[2] = DWC_OTG_MAX_CHANNELS;
3139         td->state = 0;
3140         td->errcnt = 0;
3141         td->tt_scheduled = 0;
3142         td->tt_xactpos = HCSPLT_XACTPOS_BEGIN;
3143 }
3144
3145 static void
3146 dwc_otg_setup_standard_chain(struct usb_xfer *xfer)
3147 {
3148         struct dwc_otg_std_temp temp;
3149         struct dwc_otg_td *td;
3150         uint32_t x;
3151         uint8_t need_sync;
3152         uint8_t is_host;
3153
3154         DPRINTFN(9, "addr=%d endpt=%d sumlen=%d speed=%d\n",
3155             xfer->address, UE_GET_ADDR(xfer->endpointno),
3156             xfer->sumlen, usbd_get_speed(xfer->xroot->udev));
3157
3158         temp.max_frame_size = xfer->max_frame_size;
3159
3160         td = xfer->td_start[0];
3161         xfer->td_transfer_first = td;
3162         xfer->td_transfer_cache = td;
3163
3164         /* setup temp */
3165
3166         temp.pc = NULL;
3167         temp.td = NULL;
3168         temp.td_next = xfer->td_start[0];
3169         temp.offset = 0;
3170         temp.setup_alt_next = xfer->flags_int.short_frames_ok ||
3171             xfer->flags_int.isochronous_xfr;
3172         temp.did_stall = !xfer->flags_int.control_stall;
3173
3174         is_host = (xfer->xroot->udev->flags.usb_mode == USB_MODE_HOST);
3175
3176         /* check if we should prepend a setup message */
3177
3178         if (xfer->flags_int.control_xfr) {
3179                 if (xfer->flags_int.control_hdr) {
3180
3181                         if (is_host)
3182                                 temp.func = &dwc_otg_host_setup_tx;
3183                         else
3184                                 temp.func = &dwc_otg_setup_rx;
3185
3186                         temp.len = xfer->frlengths[0];
3187                         temp.pc = xfer->frbuffers + 0;
3188                         temp.short_pkt = temp.len ? 1 : 0;
3189
3190                         /* check for last frame */
3191                         if (xfer->nframes == 1) {
3192                                 /* no STATUS stage yet, SETUP is last */
3193                                 if (xfer->flags_int.control_act)
3194                                         temp.setup_alt_next = 0;
3195                         }
3196
3197                         dwc_otg_setup_standard_chain_sub(&temp);
3198                 }
3199                 x = 1;
3200         } else {
3201                 x = 0;
3202         }
3203
3204         if (x != xfer->nframes) {
3205                 if (xfer->endpointno & UE_DIR_IN) {
3206                         if (is_host) {
3207                                 temp.func = &dwc_otg_host_data_rx;
3208                                 need_sync = 0;
3209                         } else {
3210                                 temp.func = &dwc_otg_data_tx;
3211                                 need_sync = 1;
3212                         }
3213                 } else {
3214                         if (is_host) {
3215                                 temp.func = &dwc_otg_host_data_tx;
3216                                 need_sync = 0;
3217                         } else {
3218                                 temp.func = &dwc_otg_data_rx;
3219                                 need_sync = 0;
3220                         }
3221                 }
3222
3223                 /* setup "pc" pointer */
3224                 temp.pc = xfer->frbuffers + x;
3225         } else {
3226                 need_sync = 0;
3227         }
3228         while (x != xfer->nframes) {
3229
3230                 /* DATA0 / DATA1 message */
3231
3232                 temp.len = xfer->frlengths[x];
3233
3234                 x++;
3235
3236                 if (x == xfer->nframes) {
3237                         if (xfer->flags_int.control_xfr) {
3238                                 if (xfer->flags_int.control_act) {
3239                                         temp.setup_alt_next = 0;
3240                                 }
3241                         } else {
3242                                 temp.setup_alt_next = 0;
3243                         }
3244                 }
3245                 if (temp.len == 0) {
3246
3247                         /* make sure that we send an USB packet */
3248
3249                         temp.short_pkt = 0;
3250
3251                 } else {
3252
3253                         /* regular data transfer */
3254
3255                         temp.short_pkt = (xfer->flags.force_short_xfer ? 0 : 1);
3256                 }
3257
3258                 dwc_otg_setup_standard_chain_sub(&temp);
3259
3260                 if (xfer->flags_int.isochronous_xfr) {
3261                         temp.offset += temp.len;
3262                 } else {
3263                         /* get next Page Cache pointer */
3264                         temp.pc = xfer->frbuffers + x;
3265                 }
3266         }
3267
3268         if (xfer->flags_int.control_xfr) {
3269
3270                 /* always setup a valid "pc" pointer for status and sync */
3271                 temp.pc = xfer->frbuffers + 0;
3272                 temp.len = 0;
3273                 temp.short_pkt = 0;
3274                 temp.setup_alt_next = 0;
3275
3276                 /* check if we need to sync */
3277                 if (need_sync) {
3278                         /* we need a SYNC point after TX */
3279                         temp.func = &dwc_otg_data_tx_sync;
3280                         dwc_otg_setup_standard_chain_sub(&temp);
3281                 }
3282
3283                 /* check if we should append a status stage */
3284                 if (!xfer->flags_int.control_act) {
3285
3286                         /*
3287                          * Send a DATA1 message and invert the current
3288                          * endpoint direction.
3289                          */
3290                         if (xfer->endpointno & UE_DIR_IN) {
3291                                 if (is_host) {
3292                                         temp.func = &dwc_otg_host_data_tx;
3293                                         need_sync = 0;
3294                                 } else {
3295                                         temp.func = &dwc_otg_data_rx;
3296                                         need_sync = 0;
3297                                 }
3298                         } else {
3299                                 if (is_host) {
3300                                         temp.func = &dwc_otg_host_data_rx;
3301                                         need_sync = 0;
3302                                 } else {
3303                                         temp.func = &dwc_otg_data_tx;
3304                                         need_sync = 1;
3305                                 }
3306                         }
3307
3308                         dwc_otg_setup_standard_chain_sub(&temp);
3309
3310                         /* data toggle should be DATA1 */
3311                         td = temp.td;
3312                         td->set_toggle = 1;
3313
3314                         if (need_sync) {
3315                                 /* we need a SYNC point after TX */
3316                                 temp.func = &dwc_otg_data_tx_sync;
3317                                 dwc_otg_setup_standard_chain_sub(&temp);
3318                         }
3319                 }
3320         } else {
3321                 /* check if we need to sync */
3322                 if (need_sync) {
3323
3324                         temp.pc = xfer->frbuffers + 0;
3325                         temp.len = 0;
3326                         temp.short_pkt = 0;
3327                         temp.setup_alt_next = 0;
3328
3329                         /* we need a SYNC point after TX */
3330                         temp.func = &dwc_otg_data_tx_sync;
3331                         dwc_otg_setup_standard_chain_sub(&temp);
3332                 }
3333         }
3334
3335         /* must have at least one frame! */
3336         td = temp.td;
3337         xfer->td_transfer_last = td;
3338
3339         if (is_host) {
3340
3341                 struct dwc_otg_softc *sc;
3342                 uint32_t hcchar;
3343                 uint32_t hcsplt;
3344
3345                 sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
3346
3347                 /* get first again */
3348                 td = xfer->td_transfer_first;
3349                 td->toggle = (xfer->endpoint->toggle_next ? 1 : 0);
3350
3351                 hcchar =
3352                         (xfer->address << HCCHAR_DEVADDR_SHIFT) |
3353                         ((xfer->endpointno & UE_ADDR) << HCCHAR_EPNUM_SHIFT) |
3354                         (xfer->max_packet_size << HCCHAR_MPS_SHIFT) |
3355                         HCCHAR_CHENA;
3356
3357                 /*
3358                  * We are not always able to meet the timing
3359                  * requirements of the USB interrupt endpoint's
3360                  * complete split token, when doing transfers going
3361                  * via a transaction translator. Use the CONTROL
3362                  * transfer type instead of the INTERRUPT transfer
3363                  * type in general, as a means to workaround
3364                  * that. This trick should work for both FULL and LOW
3365                  * speed USB traffic going through a TT. For non-TT
3366                  * traffic it works as well. The reason for using
3367                  * CONTROL type instead of BULK is that some TTs might
3368                  * reject LOW speed BULK traffic.
3369                  */
3370                 if (td->ep_type == UE_INTERRUPT)
3371                         hcchar |= (UE_CONTROL << HCCHAR_EPTYPE_SHIFT);
3372                 else
3373                         hcchar |= (td->ep_type << HCCHAR_EPTYPE_SHIFT);
3374
3375                 if (UE_GET_DIR(xfer->endpointno) == UE_DIR_IN)
3376                         hcchar |= HCCHAR_EPDIR_IN;
3377
3378                 switch (xfer->xroot->udev->speed) {
3379                 case USB_SPEED_LOW:
3380                         hcchar |= HCCHAR_LSPDDEV;
3381                         /* FALLTHROUGH */
3382                 case USB_SPEED_FULL:
3383                         /* check if root HUB port is running High Speed */
3384                         if (dwc_otg_uses_split(xfer->xroot->udev)) {
3385                                 hcsplt = HCSPLT_SPLTENA |
3386                                     (xfer->xroot->udev->hs_port_no <<
3387                                     HCSPLT_PRTADDR_SHIFT) |
3388                                     (xfer->xroot->udev->hs_hub_addr <<
3389                                     HCSPLT_HUBADDR_SHIFT);
3390                         } else {
3391                                 hcsplt = 0;
3392                         }
3393                         if (td->ep_type == UE_INTERRUPT) {
3394                                 uint32_t ival;
3395                                 ival = xfer->interval / DWC_OTG_HOST_TIMER_RATE;
3396                                 if (ival == 0)
3397                                         ival = 1;
3398                                 else if (ival > 127)
3399                                         ival = 127;
3400                                 td->tmr_val = sc->sc_tmr_val + ival;
3401                                 td->tmr_res = ival;
3402                         } else if (td->ep_type == UE_ISOCHRONOUS) {
3403                                 td->tmr_res = 1;
3404                                 td->tmr_val = sc->sc_last_frame_num;
3405                                 if (td->hcchar & HCCHAR_EPDIR_IN)
3406                                         td->tmr_val++;
3407                         } else {
3408                                 td->tmr_val = 0;
3409                                 td->tmr_res = (uint8_t)sc->sc_last_frame_num;
3410                         }
3411                         break;
3412                 case USB_SPEED_HIGH:
3413                         hcsplt = 0;
3414                         if (td->ep_type == UE_INTERRUPT) {
3415                                 uint32_t ival;
3416                                 hcchar |= ((xfer->max_packet_count & 3)
3417                                     << HCCHAR_MC_SHIFT);
3418                                 ival = xfer->interval / DWC_OTG_HOST_TIMER_RATE;
3419                                 if (ival == 0)
3420                                         ival = 1;
3421                                 else if (ival > 127)
3422                                         ival = 127;
3423                                 td->tmr_val = sc->sc_tmr_val + ival;
3424                                 td->tmr_res = ival;
3425                         } else if (td->ep_type == UE_ISOCHRONOUS) {
3426                                 hcchar |= ((xfer->max_packet_count & 3)
3427                                     << HCCHAR_MC_SHIFT);
3428                                 td->tmr_res = 1 << usbd_xfer_get_fps_shift(xfer);
3429                                 td->tmr_val = sc->sc_last_frame_num;
3430                                 if (td->hcchar & HCCHAR_EPDIR_IN)
3431                                         td->tmr_val += td->tmr_res;
3432
3433                         } else {
3434                                 td->tmr_val = 0;
3435                                 td->tmr_res = (uint8_t)sc->sc_last_frame_num;
3436                         }
3437                         break;
3438                 default:
3439                         hcsplt = 0;
3440                         td->tmr_val = 0;
3441                         td->tmr_res = 0;
3442                         break;
3443                 }
3444
3445                 /* store configuration in all TD's */
3446                 while (1) {
3447                         td->hcchar = hcchar;
3448                         td->hcsplt = hcsplt;
3449
3450                         if (((void *)td) == xfer->td_transfer_last)
3451                                 break;
3452
3453                         td = td->obj_next;
3454                 }
3455         }
3456 }
3457
3458 static void
3459 dwc_otg_timeout(void *arg)
3460 {
3461         struct usb_xfer *xfer = arg;
3462
3463         DPRINTF("xfer=%p\n", xfer);
3464
3465         USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
3466
3467         /* transfer is transferred */
3468         dwc_otg_device_done(xfer, USB_ERR_TIMEOUT);
3469 }
3470
3471 static void
3472 dwc_otg_start_standard_chain(struct usb_xfer *xfer)
3473 {
3474         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
3475
3476         DPRINTFN(9, "\n");
3477
3478         /*
3479          * Poll one time in device mode, which will turn on the
3480          * endpoint interrupts. Else wait for SOF interrupt in host
3481          * mode.
3482          */
3483         USB_BUS_SPIN_LOCK(&sc->sc_bus);
3484
3485         if (sc->sc_flags.status_device_mode != 0) {
3486                 dwc_otg_xfer_do_fifo(sc, xfer);
3487                 if (dwc_otg_xfer_do_complete_locked(sc, xfer))
3488                         goto done;
3489         } else {
3490                 struct dwc_otg_td *td = xfer->td_transfer_cache;
3491                 if (td->ep_type == UE_ISOCHRONOUS &&
3492                     (td->hcchar & HCCHAR_EPDIR_IN) == 0) {
3493                         /*
3494                          * Need to start ISOCHRONOUS OUT transfer ASAP
3495                          * because execution is delayed by one 125us
3496                          * microframe:
3497                          */
3498                         dwc_otg_xfer_do_fifo(sc, xfer);
3499                         if (dwc_otg_xfer_do_complete_locked(sc, xfer))
3500                                 goto done;
3501                 }
3502         }
3503
3504         /* put transfer on interrupt queue */
3505         usbd_transfer_enqueue(&xfer->xroot->bus->intr_q, xfer);
3506
3507         /* start timeout, if any */
3508         if (xfer->timeout != 0) {
3509                 usbd_transfer_timeout_ms(xfer,
3510                     &dwc_otg_timeout, xfer->timeout);
3511         }
3512
3513         if (sc->sc_flags.status_device_mode != 0)
3514                 goto done;
3515
3516         /* enable SOF interrupt, if any */
3517         dwc_otg_enable_sof_irq(sc);
3518 done:
3519         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3520 }
3521
3522 static void
3523 dwc_otg_root_intr(struct dwc_otg_softc *sc)
3524 {
3525         DPRINTFN(9, "\n");
3526
3527         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
3528
3529         /* set port bit */
3530         sc->sc_hub_idata[0] = 0x02;     /* we only have one port */
3531
3532         uhub_root_intr(&sc->sc_bus, sc->sc_hub_idata,
3533             sizeof(sc->sc_hub_idata));
3534 }
3535
3536 static usb_error_t
3537 dwc_otg_standard_done_sub(struct usb_xfer *xfer)
3538 {
3539         struct dwc_otg_td *td;
3540         uint32_t len;
3541         usb_error_t error;
3542
3543         DPRINTFN(9, "\n");
3544
3545         td = xfer->td_transfer_cache;
3546
3547         do {
3548                 len = td->remainder;
3549
3550                 /* store last data toggle */
3551                 xfer->endpoint->toggle_next = td->toggle;
3552
3553                 if (xfer->aframes != xfer->nframes) {
3554                         /*
3555                          * Verify the length and subtract
3556                          * the remainder from "frlengths[]":
3557                          */
3558                         if (len > xfer->frlengths[xfer->aframes]) {
3559                                 td->error_any = 1;
3560                         } else {
3561                                 xfer->frlengths[xfer->aframes] -= len;
3562                         }
3563                 }
3564                 /* Check for transfer error */
3565                 if (td->error_any) {
3566                         /* the transfer is finished */
3567                         error = (td->error_stall ?
3568                             USB_ERR_STALLED : USB_ERR_IOERROR);
3569                         td = NULL;
3570                         break;
3571                 }
3572                 /* Check for short transfer */
3573                 if (len > 0) {
3574                         if (xfer->flags_int.short_frames_ok ||
3575                             xfer->flags_int.isochronous_xfr) {
3576                                 /* follow alt next */
3577                                 if (td->alt_next) {
3578                                         td = td->obj_next;
3579                                 } else {
3580                                         td = NULL;
3581                                 }
3582                         } else {
3583                                 /* the transfer is finished */
3584                                 td = NULL;
3585                         }
3586                         error = 0;
3587                         break;
3588                 }
3589                 td = td->obj_next;
3590
3591                 /* this USB frame is complete */
3592                 error = 0;
3593                 break;
3594
3595         } while (0);
3596
3597         /* update transfer cache */
3598
3599         xfer->td_transfer_cache = td;
3600
3601         return (error);
3602 }
3603
3604 static void
3605 dwc_otg_standard_done(struct usb_xfer *xfer)
3606 {
3607         usb_error_t err = 0;
3608
3609         DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
3610             xfer, xfer->endpoint);
3611
3612         /* reset scanner */
3613
3614         xfer->td_transfer_cache = xfer->td_transfer_first;
3615
3616         if (xfer->flags_int.control_xfr) {
3617
3618                 if (xfer->flags_int.control_hdr) {
3619
3620                         err = dwc_otg_standard_done_sub(xfer);
3621                 }
3622                 xfer->aframes = 1;
3623
3624                 if (xfer->td_transfer_cache == NULL) {
3625                         goto done;
3626                 }
3627         }
3628         while (xfer->aframes != xfer->nframes) {
3629
3630                 err = dwc_otg_standard_done_sub(xfer);
3631                 xfer->aframes++;
3632
3633                 if (xfer->td_transfer_cache == NULL) {
3634                         goto done;
3635                 }
3636         }
3637
3638         if (xfer->flags_int.control_xfr &&
3639             !xfer->flags_int.control_act) {
3640
3641                 err = dwc_otg_standard_done_sub(xfer);
3642         }
3643 done:
3644         dwc_otg_device_done(xfer, err);
3645 }
3646
3647 /*------------------------------------------------------------------------*
3648  *      dwc_otg_device_done
3649  *
3650  * NOTE: this function can be called more than one time on the
3651  * same USB transfer!
3652  *------------------------------------------------------------------------*/
3653 static void
3654 dwc_otg_device_done(struct usb_xfer *xfer, usb_error_t error)
3655 {
3656         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
3657
3658         DPRINTFN(9, "xfer=%p, endpoint=%p, error=%d\n",
3659             xfer, xfer->endpoint, error);
3660
3661         USB_BUS_SPIN_LOCK(&sc->sc_bus);
3662
3663         if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) {
3664                 /* Interrupts are cleared by the interrupt handler */
3665         } else {
3666                 struct dwc_otg_td *td;
3667
3668                 td = xfer->td_transfer_cache;
3669                 if (td != NULL)
3670                         dwc_otg_host_channel_free(sc, td);
3671         }
3672         /* dequeue transfer and start next transfer */
3673         usbd_transfer_done(xfer, error);
3674
3675         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3676 }
3677
3678 static void
3679 dwc_otg_xfer_stall(struct usb_xfer *xfer)
3680 {
3681         dwc_otg_device_done(xfer, USB_ERR_STALLED);
3682 }
3683
3684 static void
3685 dwc_otg_set_stall(struct usb_device *udev,
3686     struct usb_endpoint *ep, uint8_t *did_stall)
3687 {
3688         struct dwc_otg_softc *sc;
3689         uint32_t temp;
3690         uint32_t reg;
3691         uint8_t ep_no;
3692
3693         USB_BUS_LOCK_ASSERT(udev->bus, MA_OWNED);
3694
3695         /* check mode */
3696         if (udev->flags.usb_mode != USB_MODE_DEVICE) {
3697                 /* not supported */
3698                 return;
3699         }
3700
3701         sc = DWC_OTG_BUS2SC(udev->bus);
3702
3703         USB_BUS_SPIN_LOCK(&sc->sc_bus);
3704
3705         /* get endpoint address */
3706         ep_no = ep->edesc->bEndpointAddress;
3707
3708         DPRINTFN(5, "endpoint=0x%x\n", ep_no);
3709
3710         if (ep_no & UE_DIR_IN) {
3711                 reg = DOTG_DIEPCTL(ep_no & UE_ADDR);
3712                 temp = sc->sc_in_ctl[ep_no & UE_ADDR];
3713         } else {
3714                 reg = DOTG_DOEPCTL(ep_no & UE_ADDR);
3715                 temp = sc->sc_out_ctl[ep_no & UE_ADDR];
3716         }
3717
3718         /* disable and stall endpoint */
3719         DWC_OTG_WRITE_4(sc, reg, temp | DOEPCTL_EPDIS);
3720         DWC_OTG_WRITE_4(sc, reg, temp | DOEPCTL_STALL);
3721
3722         /* clear active OUT ep */
3723         if (!(ep_no & UE_DIR_IN)) {
3724
3725                 sc->sc_active_rx_ep &= ~(1U << (ep_no & UE_ADDR));
3726
3727                 if (sc->sc_last_rx_status != 0 &&
3728                     (ep_no & UE_ADDR) == GRXSTSRD_CHNUM_GET(
3729                     sc->sc_last_rx_status)) {
3730                         /* dump data */
3731                         dwc_otg_common_rx_ack(sc);
3732                         /* poll interrupt */
3733                         dwc_otg_interrupt_poll_locked(sc);
3734                         dwc_otg_interrupt_complete_locked(sc);
3735                 }
3736         }
3737         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3738 }
3739
3740 static void
3741 dwc_otg_clear_stall_sub_locked(struct dwc_otg_softc *sc, uint32_t mps,
3742     uint8_t ep_no, uint8_t ep_type, uint8_t ep_dir)
3743 {
3744         uint32_t reg;
3745         uint32_t temp;
3746
3747         if (ep_type == UE_CONTROL) {
3748                 /* clearing stall is not needed */
3749                 return;
3750         }
3751
3752         if (ep_dir) {
3753                 reg = DOTG_DIEPCTL(ep_no);
3754         } else {
3755                 reg = DOTG_DOEPCTL(ep_no);
3756                 sc->sc_active_rx_ep |= (1U << ep_no);
3757         }
3758
3759         /* round up and mask away the multiplier count */
3760         mps = (mps + 3) & 0x7FC;
3761
3762         if (ep_type == UE_BULK) {
3763                 temp = DIEPCTL_EPTYPE_SET(
3764                     DIEPCTL_EPTYPE_BULK) |
3765                     DIEPCTL_USBACTEP;
3766         } else if (ep_type == UE_INTERRUPT) {
3767                 temp = DIEPCTL_EPTYPE_SET(
3768                     DIEPCTL_EPTYPE_INTERRUPT) |
3769                     DIEPCTL_USBACTEP;
3770         } else {
3771                 temp = DIEPCTL_EPTYPE_SET(
3772                     DIEPCTL_EPTYPE_ISOC) |
3773                     DIEPCTL_USBACTEP;
3774         }
3775
3776         temp |= DIEPCTL_MPS_SET(mps);
3777         temp |= DIEPCTL_TXFNUM_SET(ep_no);
3778
3779         if (ep_dir)
3780                 sc->sc_in_ctl[ep_no] = temp;
3781         else
3782                 sc->sc_out_ctl[ep_no] = temp;
3783
3784         DWC_OTG_WRITE_4(sc, reg, temp | DOEPCTL_EPDIS);
3785         DWC_OTG_WRITE_4(sc, reg, temp | DOEPCTL_SETD0PID);
3786         DWC_OTG_WRITE_4(sc, reg, temp | DIEPCTL_SNAK);
3787
3788         /* we only reset the transmit FIFO */
3789         if (ep_dir) {
3790                 dwc_otg_tx_fifo_reset(sc,
3791                     GRSTCTL_TXFIFO(ep_no) |
3792                     GRSTCTL_TXFFLSH);
3793
3794                 DWC_OTG_WRITE_4(sc,
3795                     DOTG_DIEPTSIZ(ep_no), 0);
3796         }
3797
3798         /* poll interrupt */
3799         dwc_otg_interrupt_poll_locked(sc);
3800         dwc_otg_interrupt_complete_locked(sc);
3801 }
3802
3803 static void
3804 dwc_otg_clear_stall(struct usb_device *udev, struct usb_endpoint *ep)
3805 {
3806         struct dwc_otg_softc *sc;
3807         struct usb_endpoint_descriptor *ed;
3808
3809         DPRINTFN(5, "endpoint=%p\n", ep);
3810
3811         USB_BUS_LOCK_ASSERT(udev->bus, MA_OWNED);
3812
3813         /* check mode */
3814         if (udev->flags.usb_mode != USB_MODE_DEVICE) {
3815                 /* not supported */
3816                 return;
3817         }
3818         /* get softc */
3819         sc = DWC_OTG_BUS2SC(udev->bus);
3820
3821         USB_BUS_SPIN_LOCK(&sc->sc_bus);
3822
3823         /* get endpoint descriptor */
3824         ed = ep->edesc;
3825
3826         /* reset endpoint */
3827         dwc_otg_clear_stall_sub_locked(sc,
3828             UGETW(ed->wMaxPacketSize),
3829             (ed->bEndpointAddress & UE_ADDR),
3830             (ed->bmAttributes & UE_XFERTYPE),
3831             (ed->bEndpointAddress & (UE_DIR_IN | UE_DIR_OUT)));
3832
3833         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
3834 }
3835
3836 static void
3837 dwc_otg_device_state_change(struct usb_device *udev)
3838 {
3839         struct dwc_otg_softc *sc;
3840         uint8_t x;
3841
3842         /* check mode */
3843         if (udev->flags.usb_mode != USB_MODE_DEVICE) {
3844                 /* not supported */
3845                 return;
3846         }
3847
3848         /* get softc */
3849         sc = DWC_OTG_BUS2SC(udev->bus);
3850
3851         /* deactivate all other endpoint but the control endpoint */
3852         if (udev->state == USB_STATE_CONFIGURED ||
3853             udev->state == USB_STATE_ADDRESSED) {
3854
3855                 USB_BUS_LOCK(&sc->sc_bus);
3856
3857                 for (x = 1; x != sc->sc_dev_ep_max; x++) {
3858
3859                         if (x < sc->sc_dev_in_ep_max) {
3860                                 DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(x),
3861                                     DIEPCTL_EPDIS);
3862                                 DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(x), 0);
3863                         }
3864
3865                         DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(x),
3866                             DOEPCTL_EPDIS);
3867                         DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(x), 0);
3868                 }
3869                 USB_BUS_UNLOCK(&sc->sc_bus);
3870         }
3871 }
3872
3873 int
3874 dwc_otg_init(struct dwc_otg_softc *sc)
3875 {
3876         uint32_t temp;
3877         int err;
3878
3879         DPRINTF("start\n");
3880
3881         sc->sc_io_tag = rman_get_bustag(sc->sc_io_res);
3882         sc->sc_io_hdl = rman_get_bushandle(sc->sc_io_res);
3883         sc->sc_io_size = rman_get_size(sc->sc_io_res);
3884
3885         /* set up the bus structure */
3886         sc->sc_bus.devices = sc->sc_devices;
3887         sc->sc_bus.devices_max = DWC_OTG_MAX_DEVICES;
3888         sc->sc_bus.dma_bits = 32;
3889         sc->sc_bus.usbrev = USB_REV_2_0;
3890         sc->sc_bus.methods = &dwc_otg_bus_methods;
3891
3892         /* get all DMA memory */
3893         if (usb_bus_mem_alloc_all(&sc->sc_bus,
3894             USB_GET_DMA_TAG(sc->sc_bus.parent), NULL)) {
3895                 return (ENOMEM);
3896         }
3897
3898         sc->sc_bus.bdev = device_add_child(sc->sc_bus.parent, "usbus", -1);
3899         if (sc->sc_bus.bdev == NULL)
3900                 return (ENXIO);
3901
3902         device_set_ivars(sc->sc_bus.bdev, &sc->sc_bus);
3903
3904         err = bus_setup_intr(sc->sc_bus.parent, sc->sc_irq_res,
3905             INTR_TYPE_TTY | INTR_MPSAFE, &dwc_otg_filter_interrupt,
3906             &dwc_otg_interrupt, sc, &sc->sc_intr_hdl);
3907         if (err) {
3908                 sc->sc_intr_hdl = NULL;
3909                 return (ENXIO);
3910         }
3911
3912         usb_callout_init_mtx(&sc->sc_timer,
3913             &sc->sc_bus.bus_mtx, 0);
3914
3915         USB_BUS_LOCK(&sc->sc_bus);
3916
3917         /* turn on clocks */
3918         dwc_otg_clocks_on(sc);
3919
3920         temp = DWC_OTG_READ_4(sc, DOTG_GSNPSID);
3921         DPRINTF("Version = 0x%08x\n", temp);
3922
3923         /* disconnect */
3924         DWC_OTG_WRITE_4(sc, DOTG_DCTL,
3925             DCTL_SFTDISCON);
3926
3927         /* wait for host to detect disconnect */
3928         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 32);
3929
3930         DWC_OTG_WRITE_4(sc, DOTG_GRSTCTL,
3931             GRSTCTL_CSFTRST);
3932
3933         /* wait a little bit for block to reset */
3934         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 128);
3935
3936         switch (sc->sc_mode) {
3937         case DWC_MODE_DEVICE:
3938                 temp = GUSBCFG_FORCEDEVMODE;
3939                 break;
3940         case DWC_MODE_HOST:
3941                 temp = GUSBCFG_FORCEHOSTMODE;
3942                 break;
3943         default:
3944                 temp = 0;
3945                 break;
3946         }
3947
3948         if (sc->sc_phy_type == 0)
3949                 sc->sc_phy_type = dwc_otg_phy_type + 1;
3950         if (sc->sc_phy_bits == 0)
3951                 sc->sc_phy_bits = 16;
3952
3953         /* select HSIC, ULPI, UTMI+ or internal PHY mode */
3954         switch (sc->sc_phy_type) {
3955         case DWC_OTG_PHY_HSIC:
3956                 DWC_OTG_WRITE_4(sc, DOTG_GUSBCFG,
3957                     GUSBCFG_PHYIF |
3958                     GUSBCFG_TRD_TIM_SET(5) | temp);
3959                 DWC_OTG_WRITE_4(sc, DOTG_GOTGCTL,
3960                     0x000000EC);
3961
3962                 temp = DWC_OTG_READ_4(sc, DOTG_GLPMCFG);
3963                 DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3964                     temp & ~GLPMCFG_HSIC_CONN);
3965                 DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3966                     temp | GLPMCFG_HSIC_CONN);
3967                 break;
3968         case DWC_OTG_PHY_ULPI:
3969                 DWC_OTG_WRITE_4(sc, DOTG_GUSBCFG,
3970                     GUSBCFG_ULPI_UTMI_SEL |
3971                     GUSBCFG_TRD_TIM_SET(5) | temp);
3972                 DWC_OTG_WRITE_4(sc, DOTG_GOTGCTL, 0);
3973
3974                 temp = DWC_OTG_READ_4(sc, DOTG_GLPMCFG);
3975                 DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3976                     temp & ~GLPMCFG_HSIC_CONN);
3977                 break;
3978         case DWC_OTG_PHY_UTMI:
3979                 DWC_OTG_WRITE_4(sc, DOTG_GUSBCFG,
3980                     (sc->sc_phy_bits == 16 ? GUSBCFG_PHYIF : 0) |
3981                     GUSBCFG_TRD_TIM_SET(5) | temp);
3982                 DWC_OTG_WRITE_4(sc, DOTG_GOTGCTL, 0);
3983
3984                 temp = DWC_OTG_READ_4(sc, DOTG_GLPMCFG);
3985                 DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3986                     temp & ~GLPMCFG_HSIC_CONN);
3987                 break;
3988         case DWC_OTG_PHY_INTERNAL:
3989                 DWC_OTG_WRITE_4(sc, DOTG_GUSBCFG,
3990                     GUSBCFG_PHYSEL |
3991                     GUSBCFG_TRD_TIM_SET(5) | temp);
3992                 DWC_OTG_WRITE_4(sc, DOTG_GOTGCTL, 0);
3993
3994                 temp = DWC_OTG_READ_4(sc, DOTG_GLPMCFG);
3995                 DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3996                     temp & ~GLPMCFG_HSIC_CONN);
3997
3998                 temp = DWC_OTG_READ_4(sc, DOTG_GGPIO);
3999                 temp &= ~(DOTG_GGPIO_NOVBUSSENS | DOTG_GGPIO_I2CPADEN);
4000                 temp |= (DOTG_GGPIO_VBUSASEN | DOTG_GGPIO_VBUSBSEN |
4001                     DOTG_GGPIO_PWRDWN);
4002                 DWC_OTG_WRITE_4(sc, DOTG_GGPIO, temp);
4003                 break;
4004         default:
4005                 break;
4006         }
4007
4008         /* clear global nak */
4009         DWC_OTG_WRITE_4(sc, DOTG_DCTL,
4010             DCTL_CGOUTNAK |
4011             DCTL_CGNPINNAK);
4012
4013         /* disable USB port */
4014         DWC_OTG_WRITE_4(sc, DOTG_PCGCCTL, 0xFFFFFFFF);
4015
4016         /* wait 10ms */
4017         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 100);
4018
4019         /* enable USB port */
4020         DWC_OTG_WRITE_4(sc, DOTG_PCGCCTL, 0);
4021
4022         /* wait 10ms */
4023         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 100);
4024
4025         temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG3);
4026
4027         sc->sc_fifo_size = 4 * GHWCFG3_DFIFODEPTH_GET(temp);
4028
4029         temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG2);
4030
4031         sc->sc_dev_ep_max = GHWCFG2_NUMDEVEPS_GET(temp);
4032
4033         if (sc->sc_dev_ep_max > DWC_OTG_MAX_ENDPOINTS)
4034                 sc->sc_dev_ep_max = DWC_OTG_MAX_ENDPOINTS;
4035
4036         sc->sc_host_ch_max = GHWCFG2_NUMHSTCHNL_GET(temp);
4037
4038         if (sc->sc_host_ch_max > DWC_OTG_MAX_CHANNELS)
4039                 sc->sc_host_ch_max = DWC_OTG_MAX_CHANNELS;
4040
4041         temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG4);
4042
4043         sc->sc_dev_in_ep_max = GHWCFG4_NUM_IN_EP_GET(temp);
4044
4045         DPRINTF("Total FIFO size = %d bytes, Device EPs = %d/%d Host CHs = %d\n",
4046             sc->sc_fifo_size, sc->sc_dev_ep_max, sc->sc_dev_in_ep_max,
4047             sc->sc_host_ch_max);
4048
4049         /* setup FIFO */
4050         if (dwc_otg_init_fifo(sc, sc->sc_mode)) {
4051                 USB_BUS_UNLOCK(&sc->sc_bus);
4052                 return (EINVAL);
4053         }
4054
4055         /* enable interrupts */
4056         sc->sc_irq_mask |= DWC_OTG_MSK_GINT_THREAD_IRQ;
4057         DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
4058
4059         if (sc->sc_mode == DWC_MODE_OTG || sc->sc_mode == DWC_MODE_DEVICE) {
4060
4061                 /* enable all endpoint interrupts */
4062                 temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG2);
4063                 if (temp & GHWCFG2_MPI) {
4064                         uint8_t x;
4065
4066                         DPRINTF("Disable Multi Process Interrupts\n");
4067
4068                         for (x = 0; x != sc->sc_dev_in_ep_max; x++) {
4069                                 DWC_OTG_WRITE_4(sc, DOTG_DIEPEACHINTMSK(x), 0);
4070                                 DWC_OTG_WRITE_4(sc, DOTG_DOEPEACHINTMSK(x), 0);
4071                         }
4072                         DWC_OTG_WRITE_4(sc, DOTG_DEACHINTMSK, 0);
4073                 }
4074                 DWC_OTG_WRITE_4(sc, DOTG_DIEPMSK,
4075                     DIEPMSK_XFERCOMPLMSK);
4076                 DWC_OTG_WRITE_4(sc, DOTG_DOEPMSK, 0);
4077                 DWC_OTG_WRITE_4(sc, DOTG_DAINTMSK, 0xFFFF);
4078         }
4079
4080         if (sc->sc_mode == DWC_MODE_OTG || sc->sc_mode == DWC_MODE_HOST) {
4081                 /* setup clocks */
4082                 temp = DWC_OTG_READ_4(sc, DOTG_HCFG);
4083                 temp &= ~(HCFG_FSLSSUPP | HCFG_FSLSPCLKSEL_MASK);
4084                 temp |= (1 << HCFG_FSLSPCLKSEL_SHIFT);
4085                 DWC_OTG_WRITE_4(sc, DOTG_HCFG, temp);
4086         }
4087
4088         /* only enable global IRQ */
4089         DWC_OTG_WRITE_4(sc, DOTG_GAHBCFG,
4090             GAHBCFG_GLBLINTRMSK);
4091
4092         /* turn off clocks */
4093         dwc_otg_clocks_off(sc);
4094
4095         /* read initial VBUS state */
4096
4097         temp = DWC_OTG_READ_4(sc, DOTG_GOTGCTL);
4098
4099         DPRINTFN(5, "GOTCTL=0x%08x\n", temp);
4100
4101         dwc_otg_vbus_interrupt(sc,
4102             (temp & (GOTGCTL_ASESVLD | GOTGCTL_BSESVLD)) ? 1 : 0);
4103
4104         USB_BUS_UNLOCK(&sc->sc_bus);
4105
4106         /* catch any lost interrupts */
4107
4108         dwc_otg_do_poll(&sc->sc_bus);
4109
4110         return (0);                     /* success */
4111 }
4112
4113 void
4114 dwc_otg_uninit(struct dwc_otg_softc *sc)
4115 {
4116         USB_BUS_LOCK(&sc->sc_bus);
4117
4118         /* stop host timer */
4119         dwc_otg_timer_stop(sc);
4120
4121         /* set disconnect */
4122         DWC_OTG_WRITE_4(sc, DOTG_DCTL,
4123             DCTL_SFTDISCON);
4124
4125         /* turn off global IRQ */
4126         DWC_OTG_WRITE_4(sc, DOTG_GAHBCFG, 0);
4127
4128         sc->sc_flags.port_enabled = 0;
4129         sc->sc_flags.port_powered = 0;
4130         sc->sc_flags.status_vbus = 0;
4131         sc->sc_flags.status_bus_reset = 0;
4132         sc->sc_flags.status_suspend = 0;
4133         sc->sc_flags.change_suspend = 0;
4134         sc->sc_flags.change_connect = 1;
4135
4136         dwc_otg_pull_down(sc);
4137         dwc_otg_clocks_off(sc);
4138
4139         USB_BUS_UNLOCK(&sc->sc_bus);
4140
4141         usb_callout_drain(&sc->sc_timer);
4142 }
4143
4144 static void
4145 dwc_otg_suspend(struct dwc_otg_softc *sc)
4146 {
4147         return;
4148 }
4149
4150 static void
4151 dwc_otg_resume(struct dwc_otg_softc *sc)
4152 {
4153         return;
4154 }
4155
4156 static void
4157 dwc_otg_do_poll(struct usb_bus *bus)
4158 {
4159         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(bus);
4160
4161         USB_BUS_LOCK(&sc->sc_bus);
4162         USB_BUS_SPIN_LOCK(&sc->sc_bus);
4163         dwc_otg_interrupt_poll_locked(sc);
4164         dwc_otg_interrupt_complete_locked(sc);
4165         USB_BUS_SPIN_UNLOCK(&sc->sc_bus);
4166         USB_BUS_UNLOCK(&sc->sc_bus);
4167 }
4168
4169 /*------------------------------------------------------------------------*
4170  * DWC OTG bulk support
4171  * DWC OTG control support
4172  * DWC OTG interrupt support
4173  *------------------------------------------------------------------------*/
4174 static void
4175 dwc_otg_device_non_isoc_open(struct usb_xfer *xfer)
4176 {
4177 }
4178
4179 static void
4180 dwc_otg_device_non_isoc_close(struct usb_xfer *xfer)
4181 {
4182         dwc_otg_device_done(xfer, USB_ERR_CANCELLED);
4183 }
4184
4185 static void
4186 dwc_otg_device_non_isoc_enter(struct usb_xfer *xfer)
4187 {
4188 }
4189
4190 static void
4191 dwc_otg_device_non_isoc_start(struct usb_xfer *xfer)
4192 {
4193         /* setup TDs */
4194         dwc_otg_setup_standard_chain(xfer);
4195         dwc_otg_start_standard_chain(xfer);
4196 }
4197
4198 static const struct usb_pipe_methods dwc_otg_device_non_isoc_methods =
4199 {
4200         .open = dwc_otg_device_non_isoc_open,
4201         .close = dwc_otg_device_non_isoc_close,
4202         .enter = dwc_otg_device_non_isoc_enter,
4203         .start = dwc_otg_device_non_isoc_start,
4204 };
4205
4206 /*------------------------------------------------------------------------*
4207  * DWC OTG full speed isochronous support
4208  *------------------------------------------------------------------------*/
4209 static void
4210 dwc_otg_device_isoc_open(struct usb_xfer *xfer)
4211 {
4212 }
4213
4214 static void
4215 dwc_otg_device_isoc_close(struct usb_xfer *xfer)
4216 {
4217         dwc_otg_device_done(xfer, USB_ERR_CANCELLED);
4218 }
4219
4220 static void
4221 dwc_otg_device_isoc_enter(struct usb_xfer *xfer)
4222 {
4223 }
4224
4225 static void
4226 dwc_otg_device_isoc_start(struct usb_xfer *xfer)
4227 {
4228         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
4229         uint32_t temp;
4230         uint32_t msframes;
4231         uint32_t framenum;
4232         uint8_t shift = usbd_xfer_get_fps_shift(xfer);
4233
4234         DPRINTFN(6, "xfer=%p next=%d nframes=%d\n",
4235             xfer, xfer->endpoint->isoc_next, xfer->nframes);
4236
4237         if (xfer->xroot->udev->flags.usb_mode == USB_MODE_HOST) {
4238                 temp = DWC_OTG_READ_4(sc, DOTG_HFNUM);
4239
4240                 /* get the current frame index */
4241                 framenum = (temp & HFNUM_FRNUM_MASK);
4242         } else {
4243                 temp = DWC_OTG_READ_4(sc, DOTG_DSTS);
4244
4245                 /* get the current frame index */
4246                 framenum = DSTS_SOFFN_GET(temp);
4247         }
4248
4249         /*
4250          * Check if port is doing 8000 or 1000 frames per second:
4251          */
4252         if (sc->sc_flags.status_high_speed)
4253                 framenum /= 8;
4254
4255         framenum &= DWC_OTG_FRAME_MASK;
4256
4257         /*
4258          * Compute number of milliseconds worth of data traffic for
4259          * this USB transfer:
4260          */
4261         if (xfer->xroot->udev->speed == USB_SPEED_HIGH)
4262                 msframes = ((xfer->nframes << shift) + 7) / 8;
4263         else
4264                 msframes = xfer->nframes;
4265
4266         /*
4267          * check if the frame index is within the window where the frames
4268          * will be inserted
4269          */
4270         temp = (framenum - xfer->endpoint->isoc_next) & DWC_OTG_FRAME_MASK;
4271
4272         if ((xfer->endpoint->is_synced == 0) || (temp < msframes)) {
4273                 /*
4274                  * If there is data underflow or the pipe queue is
4275                  * empty we schedule the transfer a few frames ahead
4276                  * of the current frame position. Else two isochronous
4277                  * transfers might overlap.
4278                  */
4279                 xfer->endpoint->isoc_next = (framenum + 3) & DWC_OTG_FRAME_MASK;
4280                 xfer->endpoint->is_synced = 1;
4281                 DPRINTFN(3, "start next=%d\n", xfer->endpoint->isoc_next);
4282         }
4283         /*
4284          * compute how many milliseconds the insertion is ahead of the
4285          * current frame position:
4286          */
4287         temp = (xfer->endpoint->isoc_next - framenum) & DWC_OTG_FRAME_MASK;
4288
4289         /*
4290          * pre-compute when the isochronous transfer will be finished:
4291          */
4292         xfer->isoc_time_complete =
4293                 usb_isoc_time_expand(&sc->sc_bus, framenum) + temp + msframes;
4294
4295         /* setup TDs */
4296         dwc_otg_setup_standard_chain(xfer);
4297
4298         /* compute frame number for next insertion */
4299         xfer->endpoint->isoc_next += msframes;
4300
4301         /* start TD chain */
4302         dwc_otg_start_standard_chain(xfer);
4303 }
4304
4305 static const struct usb_pipe_methods dwc_otg_device_isoc_methods =
4306 {
4307         .open = dwc_otg_device_isoc_open,
4308         .close = dwc_otg_device_isoc_close,
4309         .enter = dwc_otg_device_isoc_enter,
4310         .start = dwc_otg_device_isoc_start,
4311 };
4312
4313 /*------------------------------------------------------------------------*
4314  * DWC OTG root control support
4315  *------------------------------------------------------------------------*
4316  * Simulate a hardware HUB by handling all the necessary requests.
4317  *------------------------------------------------------------------------*/
4318
4319 static const struct usb_device_descriptor dwc_otg_devd = {
4320         .bLength = sizeof(struct usb_device_descriptor),
4321         .bDescriptorType = UDESC_DEVICE,
4322         .bcdUSB = {0x00, 0x02},
4323         .bDeviceClass = UDCLASS_HUB,
4324         .bDeviceSubClass = UDSUBCLASS_HUB,
4325         .bDeviceProtocol = UDPROTO_HSHUBSTT,
4326         .bMaxPacketSize = 64,
4327         .bcdDevice = {0x00, 0x01},
4328         .iManufacturer = 1,
4329         .iProduct = 2,
4330         .bNumConfigurations = 1,
4331 };
4332
4333 static const struct dwc_otg_config_desc dwc_otg_confd = {
4334         .confd = {
4335                 .bLength = sizeof(struct usb_config_descriptor),
4336                 .bDescriptorType = UDESC_CONFIG,
4337                 .wTotalLength[0] = sizeof(dwc_otg_confd),
4338                 .bNumInterface = 1,
4339                 .bConfigurationValue = 1,
4340                 .iConfiguration = 0,
4341                 .bmAttributes = UC_SELF_POWERED,
4342                 .bMaxPower = 0,
4343         },
4344         .ifcd = {
4345                 .bLength = sizeof(struct usb_interface_descriptor),
4346                 .bDescriptorType = UDESC_INTERFACE,
4347                 .bNumEndpoints = 1,
4348                 .bInterfaceClass = UICLASS_HUB,
4349                 .bInterfaceSubClass = UISUBCLASS_HUB,
4350                 .bInterfaceProtocol = 0,
4351         },
4352         .endpd = {
4353                 .bLength = sizeof(struct usb_endpoint_descriptor),
4354                 .bDescriptorType = UDESC_ENDPOINT,
4355                 .bEndpointAddress = (UE_DIR_IN | DWC_OTG_INTR_ENDPT),
4356                 .bmAttributes = UE_INTERRUPT,
4357                 .wMaxPacketSize[0] = 8,
4358                 .bInterval = 255,
4359         },
4360 };
4361
4362 #define HSETW(ptr, val) ptr = { (uint8_t)(val), (uint8_t)((val) >> 8) }
4363
4364 static const struct usb_hub_descriptor_min dwc_otg_hubd = {
4365         .bDescLength = sizeof(dwc_otg_hubd),
4366         .bDescriptorType = UDESC_HUB,
4367         .bNbrPorts = 1,
4368         HSETW(.wHubCharacteristics, (UHD_PWR_NO_SWITCH | UHD_OC_INDIVIDUAL)),
4369         .bPwrOn2PwrGood = 50,
4370         .bHubContrCurrent = 0,
4371         .DeviceRemovable = {0},         /* port is removable */
4372 };
4373
4374 #define STRING_VENDOR \
4375   "D\0W\0C\0O\0T\0G"
4376
4377 #define STRING_PRODUCT \
4378   "O\0T\0G\0 \0R\0o\0o\0t\0 \0H\0U\0B"
4379
4380 USB_MAKE_STRING_DESC(STRING_VENDOR, dwc_otg_vendor);
4381 USB_MAKE_STRING_DESC(STRING_PRODUCT, dwc_otg_product);
4382
4383 static usb_error_t
4384 dwc_otg_roothub_exec(struct usb_device *udev,
4385     struct usb_device_request *req, const void **pptr, uint16_t *plength)
4386 {
4387         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(udev->bus);
4388         const void *ptr;
4389         uint16_t len;
4390         uint16_t value;
4391         uint16_t index;
4392         usb_error_t err;
4393
4394         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
4395
4396         /* buffer reset */
4397         ptr = (const void *)&sc->sc_hub_temp;
4398         len = 0;
4399         err = 0;
4400
4401         value = UGETW(req->wValue);
4402         index = UGETW(req->wIndex);
4403
4404         /* demultiplex the control request */
4405
4406         switch (req->bmRequestType) {
4407         case UT_READ_DEVICE:
4408                 switch (req->bRequest) {
4409                 case UR_GET_DESCRIPTOR:
4410                         goto tr_handle_get_descriptor;
4411                 case UR_GET_CONFIG:
4412                         goto tr_handle_get_config;
4413                 case UR_GET_STATUS:
4414                         goto tr_handle_get_status;
4415                 default:
4416                         goto tr_stalled;
4417                 }
4418                 break;
4419
4420         case UT_WRITE_DEVICE:
4421                 switch (req->bRequest) {
4422                 case UR_SET_ADDRESS:
4423                         goto tr_handle_set_address;
4424                 case UR_SET_CONFIG:
4425                         goto tr_handle_set_config;
4426                 case UR_CLEAR_FEATURE:
4427                         goto tr_valid;  /* nop */
4428                 case UR_SET_DESCRIPTOR:
4429                         goto tr_valid;  /* nop */
4430                 case UR_SET_FEATURE:
4431                 default:
4432                         goto tr_stalled;
4433                 }
4434                 break;
4435
4436         case UT_WRITE_ENDPOINT:
4437                 switch (req->bRequest) {
4438                 case UR_CLEAR_FEATURE:
4439                         switch (UGETW(req->wValue)) {
4440                         case UF_ENDPOINT_HALT:
4441                                 goto tr_handle_clear_halt;
4442                         case UF_DEVICE_REMOTE_WAKEUP:
4443                                 goto tr_handle_clear_wakeup;
4444                         default:
4445                                 goto tr_stalled;
4446                         }
4447                         break;
4448                 case UR_SET_FEATURE:
4449                         switch (UGETW(req->wValue)) {
4450                         case UF_ENDPOINT_HALT:
4451                                 goto tr_handle_set_halt;
4452                         case UF_DEVICE_REMOTE_WAKEUP:
4453                                 goto tr_handle_set_wakeup;
4454                         default:
4455                                 goto tr_stalled;
4456                         }
4457                         break;
4458                 case UR_SYNCH_FRAME:
4459                         goto tr_valid;  /* nop */
4460                 default:
4461                         goto tr_stalled;
4462                 }
4463                 break;
4464
4465         case UT_READ_ENDPOINT:
4466                 switch (req->bRequest) {
4467                 case UR_GET_STATUS:
4468                         goto tr_handle_get_ep_status;
4469                 default:
4470                         goto tr_stalled;
4471                 }
4472                 break;
4473
4474         case UT_WRITE_INTERFACE:
4475                 switch (req->bRequest) {
4476                 case UR_SET_INTERFACE:
4477                         goto tr_handle_set_interface;
4478                 case UR_CLEAR_FEATURE:
4479                         goto tr_valid;  /* nop */
4480                 case UR_SET_FEATURE:
4481                 default:
4482                         goto tr_stalled;
4483                 }
4484                 break;
4485
4486         case UT_READ_INTERFACE:
4487                 switch (req->bRequest) {
4488                 case UR_GET_INTERFACE:
4489                         goto tr_handle_get_interface;
4490                 case UR_GET_STATUS:
4491                         goto tr_handle_get_iface_status;
4492                 default:
4493                         goto tr_stalled;
4494                 }
4495                 break;
4496
4497         case UT_WRITE_CLASS_INTERFACE:
4498         case UT_WRITE_VENDOR_INTERFACE:
4499                 /* XXX forward */
4500                 break;
4501
4502         case UT_READ_CLASS_INTERFACE:
4503         case UT_READ_VENDOR_INTERFACE:
4504                 /* XXX forward */
4505                 break;
4506
4507         case UT_WRITE_CLASS_DEVICE:
4508                 switch (req->bRequest) {
4509                 case UR_CLEAR_FEATURE:
4510                         goto tr_valid;
4511                 case UR_SET_DESCRIPTOR:
4512                 case UR_SET_FEATURE:
4513                         break;
4514                 default:
4515                         goto tr_stalled;
4516                 }
4517                 break;
4518
4519         case UT_WRITE_CLASS_OTHER:
4520                 switch (req->bRequest) {
4521                 case UR_CLEAR_FEATURE:
4522                         goto tr_handle_clear_port_feature;
4523                 case UR_SET_FEATURE:
4524                         goto tr_handle_set_port_feature;
4525                 case UR_CLEAR_TT_BUFFER:
4526                 case UR_RESET_TT:
4527                 case UR_STOP_TT:
4528                         goto tr_valid;
4529
4530                 default:
4531                         goto tr_stalled;
4532                 }
4533                 break;
4534
4535         case UT_READ_CLASS_OTHER:
4536                 switch (req->bRequest) {
4537                 case UR_GET_TT_STATE:
4538                         goto tr_handle_get_tt_state;
4539                 case UR_GET_STATUS:
4540                         goto tr_handle_get_port_status;
4541                 default:
4542                         goto tr_stalled;
4543                 }
4544                 break;
4545
4546         case UT_READ_CLASS_DEVICE:
4547                 switch (req->bRequest) {
4548                 case UR_GET_DESCRIPTOR:
4549                         goto tr_handle_get_class_descriptor;
4550                 case UR_GET_STATUS:
4551                         goto tr_handle_get_class_status;
4552
4553                 default:
4554                         goto tr_stalled;
4555                 }
4556                 break;
4557         default:
4558                 goto tr_stalled;
4559         }
4560         goto tr_valid;
4561
4562 tr_handle_get_descriptor:
4563         switch (value >> 8) {
4564         case UDESC_DEVICE:
4565                 if (value & 0xff) {
4566                         goto tr_stalled;
4567                 }
4568                 len = sizeof(dwc_otg_devd);
4569                 ptr = (const void *)&dwc_otg_devd;
4570                 goto tr_valid;
4571         case UDESC_CONFIG:
4572                 if (value & 0xff) {
4573                         goto tr_stalled;
4574                 }
4575                 len = sizeof(dwc_otg_confd);
4576                 ptr = (const void *)&dwc_otg_confd;
4577                 goto tr_valid;
4578         case UDESC_STRING:
4579                 switch (value & 0xff) {
4580                 case 0:         /* Language table */
4581                         len = sizeof(usb_string_lang_en);
4582                         ptr = (const void *)&usb_string_lang_en;
4583                         goto tr_valid;
4584
4585                 case 1:         /* Vendor */
4586                         len = sizeof(dwc_otg_vendor);
4587                         ptr = (const void *)&dwc_otg_vendor;
4588                         goto tr_valid;
4589
4590                 case 2:         /* Product */
4591                         len = sizeof(dwc_otg_product);
4592                         ptr = (const void *)&dwc_otg_product;
4593                         goto tr_valid;
4594                 default:
4595                         break;
4596                 }
4597                 break;
4598         default:
4599                 goto tr_stalled;
4600         }
4601         goto tr_stalled;
4602
4603 tr_handle_get_config:
4604         len = 1;
4605         sc->sc_hub_temp.wValue[0] = sc->sc_conf;
4606         goto tr_valid;
4607
4608 tr_handle_get_status:
4609         len = 2;
4610         USETW(sc->sc_hub_temp.wValue, UDS_SELF_POWERED);
4611         goto tr_valid;
4612
4613 tr_handle_set_address:
4614         if (value & 0xFF00) {
4615                 goto tr_stalled;
4616         }
4617         sc->sc_rt_addr = value;
4618         goto tr_valid;
4619
4620 tr_handle_set_config:
4621         if (value >= 2) {
4622                 goto tr_stalled;
4623         }
4624         sc->sc_conf = value;
4625         goto tr_valid;
4626
4627 tr_handle_get_interface:
4628         len = 1;
4629         sc->sc_hub_temp.wValue[0] = 0;
4630         goto tr_valid;
4631
4632 tr_handle_get_tt_state:
4633 tr_handle_get_class_status:
4634 tr_handle_get_iface_status:
4635 tr_handle_get_ep_status:
4636         len = 2;
4637         USETW(sc->sc_hub_temp.wValue, 0);
4638         goto tr_valid;
4639
4640 tr_handle_set_halt:
4641 tr_handle_set_interface:
4642 tr_handle_set_wakeup:
4643 tr_handle_clear_wakeup:
4644 tr_handle_clear_halt:
4645         goto tr_valid;
4646
4647 tr_handle_clear_port_feature:
4648         if (index != 1)
4649                 goto tr_stalled;
4650
4651         DPRINTFN(9, "UR_CLEAR_PORT_FEATURE on port %d\n", index);
4652
4653         switch (value) {
4654         case UHF_PORT_SUSPEND:
4655                 dwc_otg_wakeup_peer(sc);
4656                 break;
4657
4658         case UHF_PORT_ENABLE:
4659                 if (sc->sc_flags.status_device_mode == 0) {
4660                         DWC_OTG_WRITE_4(sc, DOTG_HPRT,
4661                             sc->sc_hprt_val | HPRT_PRTENA);
4662                 }
4663                 sc->sc_flags.port_enabled = 0;
4664                 break;
4665
4666         case UHF_C_PORT_RESET:
4667                 sc->sc_flags.change_reset = 0;
4668                 break;
4669
4670         case UHF_C_PORT_ENABLE:
4671                 sc->sc_flags.change_enabled = 0;
4672                 break;
4673
4674         case UHF_C_PORT_OVER_CURRENT:
4675                 sc->sc_flags.change_over_current = 0;
4676                 break;
4677
4678         case UHF_PORT_TEST:
4679         case UHF_PORT_INDICATOR:
4680                 /* nops */
4681                 break;
4682
4683         case UHF_PORT_POWER:
4684                 sc->sc_flags.port_powered = 0;
4685                 if (sc->sc_mode == DWC_MODE_HOST || sc->sc_mode == DWC_MODE_OTG) {
4686                         sc->sc_hprt_val = 0;
4687                         DWC_OTG_WRITE_4(sc, DOTG_HPRT, HPRT_PRTENA);
4688                 }
4689                 dwc_otg_pull_down(sc);
4690                 dwc_otg_clocks_off(sc);
4691                 break;
4692
4693         case UHF_C_PORT_CONNECTION:
4694                 /* clear connect change flag */
4695                 sc->sc_flags.change_connect = 0;
4696                 break;
4697
4698         case UHF_C_PORT_SUSPEND:
4699                 sc->sc_flags.change_suspend = 0;
4700                 break;
4701
4702         default:
4703                 err = USB_ERR_IOERROR;
4704                 goto done;
4705         }
4706         goto tr_valid;
4707
4708 tr_handle_set_port_feature:
4709         if (index != 1) {
4710                 goto tr_stalled;
4711         }
4712         DPRINTFN(9, "UR_SET_PORT_FEATURE\n");
4713
4714         switch (value) {
4715         case UHF_PORT_ENABLE:
4716                 break;
4717
4718         case UHF_PORT_SUSPEND:
4719                 if (sc->sc_flags.status_device_mode == 0) {
4720                         /* set suspend BIT */
4721                         sc->sc_hprt_val |= HPRT_PRTSUSP;
4722                         DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
4723
4724                         /* generate HUB suspend event */
4725                         dwc_otg_suspend_irq(sc);
4726                 }
4727                 break;
4728
4729         case UHF_PORT_RESET:
4730                 if (sc->sc_flags.status_device_mode == 0) {
4731
4732                         DPRINTF("PORT RESET\n");
4733
4734                         /* enable PORT reset */
4735                         DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val | HPRT_PRTRST);
4736
4737                         /* Wait 62.5ms for reset to complete */
4738                         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 16);
4739
4740                         DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
4741
4742                         /* Wait 62.5ms for reset to complete */
4743                         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 16);
4744
4745                         /* reset FIFOs */
4746                         (void) dwc_otg_init_fifo(sc, DWC_MODE_HOST);
4747
4748                         sc->sc_flags.change_reset = 1;
4749                 } else {
4750                         err = USB_ERR_IOERROR;
4751                 }
4752                 break;
4753
4754         case UHF_PORT_TEST:
4755         case UHF_PORT_INDICATOR:
4756                 /* nops */
4757                 break;
4758         case UHF_PORT_POWER:
4759                 sc->sc_flags.port_powered = 1;
4760                 if (sc->sc_mode == DWC_MODE_HOST || sc->sc_mode == DWC_MODE_OTG) {
4761                         sc->sc_hprt_val |= HPRT_PRTPWR;
4762                         DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
4763                 }
4764                 if (sc->sc_mode == DWC_MODE_DEVICE || sc->sc_mode == DWC_MODE_OTG) {
4765                         /* pull up D+, if any */
4766                         dwc_otg_pull_up(sc);
4767                 }
4768                 break;
4769         default:
4770                 err = USB_ERR_IOERROR;
4771                 goto done;
4772         }
4773         goto tr_valid;
4774
4775 tr_handle_get_port_status:
4776
4777         DPRINTFN(9, "UR_GET_PORT_STATUS\n");
4778
4779         if (index != 1)
4780                 goto tr_stalled;
4781
4782         if (sc->sc_flags.status_vbus)
4783                 dwc_otg_clocks_on(sc);
4784         else
4785                 dwc_otg_clocks_off(sc);
4786
4787         /* Select Device Side Mode */
4788
4789         if (sc->sc_flags.status_device_mode) {
4790                 value = UPS_PORT_MODE_DEVICE;
4791                 dwc_otg_timer_stop(sc);
4792         } else {
4793                 value = 0;
4794                 dwc_otg_timer_start(sc);
4795         }
4796
4797         if (sc->sc_flags.status_high_speed)
4798                 value |= UPS_HIGH_SPEED;
4799         else if (sc->sc_flags.status_low_speed)
4800                 value |= UPS_LOW_SPEED;
4801
4802         if (sc->sc_flags.port_powered)
4803                 value |= UPS_PORT_POWER;
4804
4805         if (sc->sc_flags.port_enabled)
4806                 value |= UPS_PORT_ENABLED;
4807
4808         if (sc->sc_flags.port_over_current)
4809                 value |= UPS_OVERCURRENT_INDICATOR;
4810
4811         if (sc->sc_flags.status_vbus &&
4812             sc->sc_flags.status_bus_reset)
4813                 value |= UPS_CURRENT_CONNECT_STATUS;
4814
4815         if (sc->sc_flags.status_suspend)
4816                 value |= UPS_SUSPEND;
4817
4818         USETW(sc->sc_hub_temp.ps.wPortStatus, value);
4819
4820         value = 0;
4821
4822         if (sc->sc_flags.change_enabled)
4823                 value |= UPS_C_PORT_ENABLED;
4824         if (sc->sc_flags.change_connect)
4825                 value |= UPS_C_CONNECT_STATUS;
4826         if (sc->sc_flags.change_suspend)
4827                 value |= UPS_C_SUSPEND;
4828         if (sc->sc_flags.change_reset)
4829                 value |= UPS_C_PORT_RESET;
4830         if (sc->sc_flags.change_over_current)
4831                 value |= UPS_C_OVERCURRENT_INDICATOR;
4832
4833         USETW(sc->sc_hub_temp.ps.wPortChange, value);
4834         len = sizeof(sc->sc_hub_temp.ps);
4835         goto tr_valid;
4836
4837 tr_handle_get_class_descriptor:
4838         if (value & 0xFF) {
4839                 goto tr_stalled;
4840         }
4841         ptr = (const void *)&dwc_otg_hubd;
4842         len = sizeof(dwc_otg_hubd);
4843         goto tr_valid;
4844
4845 tr_stalled:
4846         err = USB_ERR_STALLED;
4847 tr_valid:
4848 done:
4849         *plength = len;
4850         *pptr = ptr;
4851         return (err);
4852 }
4853
4854 static void
4855 dwc_otg_xfer_setup(struct usb_setup_params *parm)
4856 {
4857         struct usb_xfer *xfer;
4858         void *last_obj;
4859         uint32_t ntd;
4860         uint32_t n;
4861         uint8_t ep_no;
4862         uint8_t ep_type;
4863
4864         xfer = parm->curr_xfer;
4865
4866         /*
4867          * NOTE: This driver does not use any of the parameters that
4868          * are computed from the following values. Just set some
4869          * reasonable dummies:
4870          */
4871         parm->hc_max_packet_size = 0x500;
4872         parm->hc_max_packet_count = 3;
4873         parm->hc_max_frame_size = 3 * 0x500;
4874
4875         usbd_transfer_setup_sub(parm);
4876
4877         /*
4878          * compute maximum number of TDs
4879          */
4880         ep_type = (xfer->endpoint->edesc->bmAttributes & UE_XFERTYPE);
4881
4882         if (ep_type == UE_CONTROL) {
4883
4884                 ntd = xfer->nframes + 1 /* STATUS */ + 1 /* SYNC 1 */
4885                     + 1 /* SYNC 2 */ + 1 /* SYNC 3 */;
4886         } else {
4887
4888                 ntd = xfer->nframes + 1 /* SYNC */ ;
4889         }
4890
4891         /*
4892          * check if "usbd_transfer_setup_sub" set an error
4893          */
4894         if (parm->err)
4895                 return;
4896
4897         /*
4898          * allocate transfer descriptors
4899          */
4900         last_obj = NULL;
4901
4902         ep_no = xfer->endpointno & UE_ADDR;
4903
4904         /*
4905          * Check for a valid endpoint profile in USB device mode:
4906          */
4907         if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) {
4908                 const struct usb_hw_ep_profile *pf;
4909
4910                 dwc_otg_get_hw_ep_profile(parm->udev, &pf, ep_no);
4911
4912                 if (pf == NULL) {
4913                         /* should not happen */
4914                         parm->err = USB_ERR_INVAL;
4915                         return;
4916                 }
4917         }
4918
4919         /* align data */
4920         parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
4921
4922         for (n = 0; n != ntd; n++) {
4923
4924                 struct dwc_otg_td *td;
4925
4926                 if (parm->buf) {
4927
4928                         td = USB_ADD_BYTES(parm->buf, parm->size[0]);
4929
4930                         /* compute shared bandwidth resource index for TT */
4931                         if (dwc_otg_uses_split(parm->udev)) {
4932                                 if (parm->udev->parent_hs_hub->ddesc.bDeviceProtocol == UDPROTO_HSHUBMTT)
4933                                         td->tt_index = parm->udev->device_index;
4934                                 else
4935                                         td->tt_index = parm->udev->parent_hs_hub->device_index;
4936                         } else {
4937                                 td->tt_index = parm->udev->device_index;
4938                         }
4939
4940                         /* init TD */
4941                         td->max_packet_size = xfer->max_packet_size;
4942                         td->max_packet_count = xfer->max_packet_count;
4943                         /* range check */
4944                         if (td->max_packet_count == 0 || td->max_packet_count > 3)
4945                                 td->max_packet_count = 1;
4946                         td->ep_no = ep_no;
4947                         td->ep_type = ep_type;
4948                         td->obj_next = last_obj;
4949
4950                         last_obj = td;
4951                 }
4952                 parm->size[0] += sizeof(*td);
4953         }
4954
4955         xfer->td_start[0] = last_obj;
4956 }
4957
4958 static void
4959 dwc_otg_xfer_unsetup(struct usb_xfer *xfer)
4960 {
4961         return;
4962 }
4963
4964 static void
4965 dwc_otg_ep_init(struct usb_device *udev, struct usb_endpoint_descriptor *edesc,
4966     struct usb_endpoint *ep)
4967 {
4968         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(udev->bus);
4969
4970         DPRINTFN(2, "endpoint=%p, addr=%d, endpt=%d, mode=%d (%d,%d)\n",
4971             ep, udev->address,
4972             edesc->bEndpointAddress, udev->flags.usb_mode,
4973             sc->sc_rt_addr, udev->device_index);
4974
4975         if (udev->device_index != sc->sc_rt_addr) {
4976
4977                 if (udev->flags.usb_mode == USB_MODE_DEVICE) {
4978                         if (udev->speed != USB_SPEED_FULL &&
4979                             udev->speed != USB_SPEED_HIGH) {
4980                                 /* not supported */
4981                                 return;
4982                         }
4983                 } else {
4984                         if (udev->speed == USB_SPEED_HIGH &&
4985                             (edesc->wMaxPacketSize[1] & 0x18) != 0 &&
4986                             (edesc->bmAttributes & UE_XFERTYPE) != UE_ISOCHRONOUS) {
4987                                 /* not supported */
4988                                 DPRINTFN(-1, "Non-isochronous high bandwidth "
4989                                     "endpoint not supported\n");
4990                                 return;
4991                         }
4992                 }
4993                 if ((edesc->bmAttributes & UE_XFERTYPE) == UE_ISOCHRONOUS)
4994                         ep->methods = &dwc_otg_device_isoc_methods;
4995                 else
4996                         ep->methods = &dwc_otg_device_non_isoc_methods;
4997         }
4998 }
4999
5000 static void
5001 dwc_otg_set_hw_power_sleep(struct usb_bus *bus, uint32_t state)
5002 {
5003         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(bus);
5004
5005         switch (state) {
5006         case USB_HW_POWER_SUSPEND:
5007                 dwc_otg_suspend(sc);
5008                 break;
5009         case USB_HW_POWER_SHUTDOWN:
5010                 dwc_otg_uninit(sc);
5011                 break;
5012         case USB_HW_POWER_RESUME:
5013                 dwc_otg_resume(sc);
5014                 break;
5015         default:
5016                 break;
5017         }
5018 }
5019
5020 static void
5021 dwc_otg_get_dma_delay(struct usb_device *udev, uint32_t *pus)
5022 {
5023         /* DMA delay - wait until any use of memory is finished */
5024         *pus = (2125);                  /* microseconds */
5025 }
5026
5027 static void
5028 dwc_otg_device_resume(struct usb_device *udev)
5029 {
5030         DPRINTF("\n");
5031
5032         /* poll all transfers again to restart resumed ones */
5033         dwc_otg_do_poll(udev->bus);
5034 }
5035
5036 static void
5037 dwc_otg_device_suspend(struct usb_device *udev)
5038 {
5039         DPRINTF("\n");
5040 }
5041
5042 static const struct usb_bus_methods dwc_otg_bus_methods =
5043 {
5044         .endpoint_init = &dwc_otg_ep_init,
5045         .xfer_setup = &dwc_otg_xfer_setup,
5046         .xfer_unsetup = &dwc_otg_xfer_unsetup,
5047         .get_hw_ep_profile = &dwc_otg_get_hw_ep_profile,
5048         .xfer_stall = &dwc_otg_xfer_stall,
5049         .set_stall = &dwc_otg_set_stall,
5050         .clear_stall = &dwc_otg_clear_stall,
5051         .roothub_exec = &dwc_otg_roothub_exec,
5052         .xfer_poll = &dwc_otg_do_poll,
5053         .device_state_change = &dwc_otg_device_state_change,
5054         .set_hw_power_sleep = &dwc_otg_set_hw_power_sleep,
5055         .get_dma_delay = &dwc_otg_get_dma_delay,
5056         .device_resume = &dwc_otg_device_resume,
5057         .device_suspend = &dwc_otg_device_suspend,
5058 };