sys/dev/usb/controller/dwc_otg.c

   1 /* $FreeBSD$ */
   2 /*-
   3  * Copyright (c) 2012 Hans Petter Selasky. All rights reserved.
   4  * Copyright (c) 2010-2011 Aleksandr Rybalko. All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  */
  27
  28 /*
  29  * This file contains the driver for the DesignWare series USB 2.0 OTG
  30  * Controller.
  31  */
  32
  33 /*
  34  * LIMITATION: Drivers must be bound to all OUT endpoints in the
  35  * active configuration for this driver to work properly. Blocking any
  36  * OUT endpoint will block all OUT endpoints including the control
  37  * endpoint. Usually this is not a problem.
  38  */
  39
  40 /*
  41  * NOTE: Writing to non-existing registers appears to cause an
  42  * internal reset.
  43  */
  44
  45 #ifdef USB_GLOBAL_INCLUDE_FILE
  46 #include USB_GLOBAL_INCLUDE_FILE
  47 #else
  48 #include <sys/stdint.h>
  49 #include <sys/stddef.h>
  50 #include <sys/param.h>
  51 #include <sys/queue.h>
  52 #include <sys/types.h>
  53 #include <sys/systm.h>
  54 #include <sys/kernel.h>
  55 #include <sys/bus.h>
  56 #include <sys/module.h>
  57 #include <sys/lock.h>
  58 #include <sys/mutex.h>
  59 #include <sys/condvar.h>
  60 #include <sys/sysctl.h>
  61 #include <sys/sx.h>
  62 #include <sys/unistd.h>
  63 #include <sys/callout.h>
  64 #include <sys/malloc.h>
  65 #include <sys/priv.h>
  66
  67 #include <dev/usb/usb.h>
  68 #include <dev/usb/usbdi.h>
  69
  70 #define USB_DEBUG_VAR dwc_otg_debug
  71
  72 #include <dev/usb/usb_core.h>
  73 #include <dev/usb/usb_debug.h>
  74 #include <dev/usb/usb_busdma.h>
  75 #include <dev/usb/usb_process.h>
  76 #include <dev/usb/usb_transfer.h>
  77 #include <dev/usb/usb_device.h>
  78 #include <dev/usb/usb_hub.h>
  79 #include <dev/usb/usb_util.h>
  80
  81 #include <dev/usb/usb_controller.h>
  82 #include <dev/usb/usb_bus.h>
  83 #endif                  /* USB_GLOBAL_INCLUDE_FILE */
  84
  85 #include <dev/usb/controller/dwc_otg.h>
  86 #include <dev/usb/controller/dwc_otgreg.h>
  87
  88 #define DWC_OTG_BUS2SC(bus) \
  89    ((struct dwc_otg_softc *)(((uint8_t *)(bus)) - \
  90     ((uint8_t *)&(((struct dwc_otg_softc *)0)->sc_bus))))
  91
  92 #define DWC_OTG_PC2SC(pc) \
  93    DWC_OTG_BUS2SC(USB_DMATAG_TO_XROOT((pc)->tag_parent)->bus)
  94
  95 #define DWC_OTG_MSK_GINT_ENABLED        \
  96    (GINTSTS_ENUMDONE |                  \
  97    GINTSTS_USBRST |                     \
  98    GINTSTS_USBSUSP |                    \
  99    GINTSTS_IEPINT |                     \
 100    GINTSTS_RXFLVL |                     \
 101    GINTSTS_SESSREQINT |                 \
 102    GINTMSK_OTGINTMSK |                  \
 103    GINTMSK_HCHINTMSK |                  \
 104    GINTSTS_PRTINT)
 105
 106 static int dwc_otg_use_hsic;
 107
 108 static SYSCTL_NODE(_hw_usb, OID_AUTO, dwc_otg, CTLFLAG_RW, 0, "USB DWC OTG");
 109
 110 SYSCTL_INT(_hw_usb_dwc_otg, OID_AUTO, use_hsic, CTLFLAG_RD | CTLFLAG_TUN,
 111     &dwc_otg_use_hsic, 0, "DWC OTG uses HSIC interface");
 112 TUNABLE_INT("hw.usb.dwc_otg.use_hsic", &dwc_otg_use_hsic);
 113
 114 #ifdef USB_DEBUG
 115 static int dwc_otg_debug;
 116
 117 SYSCTL_INT(_hw_usb_dwc_otg, OID_AUTO, debug, CTLFLAG_RW,
 118     &dwc_otg_debug, 0, "DWC OTG debug level");
 119 #endif
 120
 121 #define DWC_OTG_INTR_ENDPT 1
 122
 123 /* prototypes */
 124
 125 struct usb_bus_methods dwc_otg_bus_methods;
 126 struct usb_pipe_methods dwc_otg_device_non_isoc_methods;
 127 struct usb_pipe_methods dwc_otg_device_isoc_methods;
 128
 129 static dwc_otg_cmd_t dwc_otg_setup_rx;
 130 static dwc_otg_cmd_t dwc_otg_data_rx;
 131 static dwc_otg_cmd_t dwc_otg_data_tx;
 132 static dwc_otg_cmd_t dwc_otg_data_tx_sync;
 133
 134 static dwc_otg_cmd_t dwc_otg_host_setup_tx;
 135 static dwc_otg_cmd_t dwc_otg_host_data_tx;
 136 static dwc_otg_cmd_t dwc_otg_host_data_rx;
 137
 138 static void dwc_otg_device_done(struct usb_xfer *, usb_error_t);
 139 static void dwc_otg_do_poll(struct usb_bus *);
 140 static void dwc_otg_standard_done(struct usb_xfer *);
 141 static void dwc_otg_root_intr(struct dwc_otg_softc *sc);
 142 static void dwc_otg_interrupt_poll(struct dwc_otg_softc *sc);
 143
 144 /*
 145  * Here is a configuration that the chip supports.
 146  */
 147 static const struct usb_hw_ep_profile dwc_otg_ep_profile[1] = {
 148
 149         [0] = {
 150                 .max_in_frame_size = 64,/* fixed */
 151                 .max_out_frame_size = 64,       /* fixed */
 152                 .is_simplex = 1,
 153                 .support_control = 1,
 154         }
 155 };
 156
 157 static void
 158 dwc_otg_get_hw_ep_profile(struct usb_device *udev,
 159     const struct usb_hw_ep_profile **ppf, uint8_t ep_addr)
 160 {
 161         struct dwc_otg_softc *sc;
 162
 163         sc = DWC_OTG_BUS2SC(udev->bus);
 164
 165         if (ep_addr < sc->sc_dev_ep_max)
 166                 *ppf = &sc->sc_hw_ep_profile[ep_addr].usb;
 167         else
 168                 *ppf = NULL;
 169 }
 170
 171 static int
 172 dwc_otg_init_fifo(struct dwc_otg_softc *sc, uint8_t mode)
 173 {
 174         struct dwc_otg_profile *pf;
 175         uint32_t fifo_size;
 176         uint32_t fifo_regs;
 177         uint32_t tx_start;
 178         uint8_t x;
 179
 180         fifo_size = sc->sc_fifo_size;
 181
 182         fifo_regs = 4 * (sc->sc_dev_ep_max + sc->sc_dev_in_ep_max);
 183
 184         if (fifo_size >= fifo_regs)
 185                 fifo_size -= fifo_regs;
 186         else
 187                 fifo_size = 0;
 188
 189         /* split equally for IN and OUT */
 190         fifo_size /= 2;
 191
 192         DWC_OTG_WRITE_4(sc, DOTG_GRXFSIZ, fifo_size / 4);
 193
 194         /* align to 4-bytes */
 195         fifo_size &= ~3;
 196
 197         tx_start = fifo_size;
 198
 199         if (fifo_size < 0x40) {
 200                 DPRINTFN(-1, "Not enough data space for EP0 FIFO.\n");
 201                 USB_BUS_UNLOCK(&sc->sc_bus);
 202                 return (EINVAL);
 203         }
 204
 205         if (mode == DWC_MODE_HOST) {
 206
 207                 /* reset active endpoints */
 208                 sc->sc_active_rx_ep = 0;
 209
 210                 fifo_size /= 2;
 211
 212                 DWC_OTG_WRITE_4(sc, DOTG_GNPTXFSIZ,
 213                     ((fifo_size / 4) << 16) |
 214                     (tx_start / 4));
 215
 216                 tx_start += fifo_size;
 217
 218                 DWC_OTG_WRITE_4(sc, DOTG_HPTXFSIZ,
 219                     ((fifo_size / 4) << 16) |
 220                     (tx_start / 4));
 221
 222                 for (x = 0; x != sc->sc_host_ch_max; x++) {
 223                         /* enable interrupts */
 224                         DWC_OTG_WRITE_4(sc, DOTG_HCINTMSK(x),
 225                             HCINT_STALL | HCINT_BBLERR |
 226                             HCINT_XACTERR |
 227                             HCINT_NAK | HCINT_ACK | HCINT_NYET |
 228                             HCINT_CHHLTD | HCINT_FRMOVRUN |
 229                             HCINT_DATATGLERR);
 230                 }
 231
 232                 /* enable host channel interrupts */
 233                 DWC_OTG_WRITE_4(sc, DOTG_HAINTMSK,
 234                     (1U << sc->sc_host_ch_max) - 1U);
 235         }
 236
 237         if (mode == DWC_MODE_DEVICE) {
 238
 239             DWC_OTG_WRITE_4(sc, DOTG_GNPTXFSIZ,
 240                 (0x10 << 16) | (tx_start / 4));
 241             fifo_size -= 0x40;
 242             tx_start += 0x40;
 243
 244             /* setup control endpoint profile */
 245             sc->sc_hw_ep_profile[0].usb = dwc_otg_ep_profile[0];
 246
 247             /* reset active endpoints */
 248             sc->sc_active_rx_ep = 1;
 249
 250             for (x = 1; x != sc->sc_dev_ep_max; x++) {
 251
 252                 pf = sc->sc_hw_ep_profile + x;
 253
 254                 pf->usb.max_out_frame_size = 1024 * 3;
 255                 pf->usb.is_simplex = 0; /* assume duplex */
 256                 pf->usb.support_bulk = 1;
 257                 pf->usb.support_interrupt = 1;
 258                 pf->usb.support_isochronous = 1;
 259                 pf->usb.support_out = 1;
 260
 261                 if (x < sc->sc_dev_in_ep_max) {
 262                         uint32_t limit;
 263
 264                         limit = (x == 1) ? DWC_OTG_MAX_TXN :
 265                             (DWC_OTG_MAX_TXN / 2);
 266
 267                         if (fifo_size >= limit) {
 268                                 DWC_OTG_WRITE_4(sc, DOTG_DIEPTXF(x),
 269                                     ((limit / 4) << 16) |
 270                                     (tx_start / 4));
 271                                 tx_start += limit;
 272                                 fifo_size -= limit;
 273                                 pf->usb.max_in_frame_size = 0x200;
 274                                 pf->usb.support_in = 1;
 275                                 pf->max_buffer = limit;
 276
 277                         } else if (fifo_size >= 0x80) {
 278                                 DWC_OTG_WRITE_4(sc, DOTG_DIEPTXF(x),
 279                                     ((0x80 / 4) << 16) | (tx_start / 4));
 280                                 tx_start += 0x80;
 281                                 fifo_size -= 0x80;
 282                                 pf->usb.max_in_frame_size = 0x40;
 283                                 pf->usb.support_in = 1;
 284
 285                         } else {
 286                                 pf->usb.is_simplex = 1;
 287                                 DWC_OTG_WRITE_4(sc, DOTG_DIEPTXF(x),
 288                                     (0x0 << 16) | (tx_start / 4));
 289                         }
 290                 } else {
 291                         pf->usb.is_simplex = 1;
 292                 }
 293
 294                 DPRINTF("FIFO%d = IN:%d / OUT:%d\n", x,
 295                     pf->usb.max_in_frame_size,
 296                     pf->usb.max_out_frame_size);
 297             }
 298         }
 299
 300         /* reset RX FIFO */
 301         DWC_OTG_WRITE_4(sc, DOTG_GRSTCTL,
 302             GRSTCTL_RXFFLSH);
 303
 304         if (mode != DWC_MODE_OTG) {
 305                 /* reset all TX FIFOs */
 306                 DWC_OTG_WRITE_4(sc, DOTG_GRSTCTL,
 307                     GRSTCTL_TXFIFO(0x10) |
 308                     GRSTCTL_TXFFLSH);
 309         } else {
 310                 /* reset active endpoints */
 311                 sc->sc_active_rx_ep = 0;
 312         }
 313         return (0);
 314 }
 315
 316 static void
 317 dwc_otg_clocks_on(struct dwc_otg_softc *sc)
 318 {
 319         if (sc->sc_flags.clocks_off &&
 320             sc->sc_flags.port_powered) {
 321
 322                 DPRINTFN(5, "\n");
 323
 324                 /* TODO - platform specific */
 325
 326                 sc->sc_flags.clocks_off = 0;
 327         }
 328 }
 329
 330 static void
 331 dwc_otg_clocks_off(struct dwc_otg_softc *sc)
 332 {
 333         if (!sc->sc_flags.clocks_off) {
 334
 335                 DPRINTFN(5, "\n");
 336
 337                 /* TODO - platform specific */
 338
 339                 sc->sc_flags.clocks_off = 1;
 340         }
 341 }
 342
 343 static void
 344 dwc_otg_pull_up(struct dwc_otg_softc *sc)
 345 {
 346         uint32_t temp;
 347
 348         /* pullup D+, if possible */
 349
 350         if (!sc->sc_flags.d_pulled_up &&
 351             sc->sc_flags.port_powered) {
 352                 sc->sc_flags.d_pulled_up = 1;
 353
 354                 temp = DWC_OTG_READ_4(sc, DOTG_DCTL);
 355                 temp &= ~DCTL_SFTDISCON;
 356                 DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
 357         }
 358 }
 359
 360 static void
 361 dwc_otg_pull_down(struct dwc_otg_softc *sc)
 362 {
 363         uint32_t temp;
 364
 365         /* pulldown D+, if possible */
 366
 367         if (sc->sc_flags.d_pulled_up) {
 368                 sc->sc_flags.d_pulled_up = 0;
 369
 370                 temp = DWC_OTG_READ_4(sc, DOTG_DCTL);
 371                 temp |= DCTL_SFTDISCON;
 372                 DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
 373         }
 374 }
 375
 376 static void
 377 dwc_otg_enable_sof_irq(struct dwc_otg_softc *sc)
 378 {
 379         if (sc->sc_irq_mask & GINTSTS_SOF)
 380                 return;
 381         sc->sc_irq_mask |= GINTSTS_SOF;
 382         DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 383 }
 384
 385 static void
 386 dwc_otg_resume_irq(struct dwc_otg_softc *sc)
 387 {
 388         if (sc->sc_flags.status_suspend) {
 389                 /* update status bits */
 390                 sc->sc_flags.status_suspend = 0;
 391                 sc->sc_flags.change_suspend = 1;
 392
 393                 if (sc->sc_flags.status_device_mode) {
 394                         /*
 395                          * Disable resume interrupt and enable suspend
 396                          * interrupt:
 397                          */
 398                         sc->sc_irq_mask &= ~GINTSTS_WKUPINT;
 399                         sc->sc_irq_mask |= GINTSTS_USBSUSP;
 400                         DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 401                 }
 402
 403                 /* complete root HUB interrupt endpoint */
 404                 dwc_otg_root_intr(sc);
 405         }
 406 }
 407
 408 static void
 409 dwc_otg_suspend_irq(struct dwc_otg_softc *sc)
 410 {
 411         if (!sc->sc_flags.status_suspend) {
 412                 /* update status bits */
 413                 sc->sc_flags.status_suspend = 1;
 414                 sc->sc_flags.change_suspend = 1;
 415
 416                 if (sc->sc_flags.status_device_mode) {
 417                         /*
 418                          * Disable suspend interrupt and enable resume
 419                          * interrupt:
 420                          */
 421                         sc->sc_irq_mask &= ~GINTSTS_USBSUSP;
 422                         sc->sc_irq_mask |= GINTSTS_WKUPINT;
 423                         DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 424                 }
 425
 426                 /* complete root HUB interrupt endpoint */
 427                 dwc_otg_root_intr(sc);
 428         }
 429 }
 430
 431 static void
 432 dwc_otg_wakeup_peer(struct dwc_otg_softc *sc)
 433 {
 434         if (!sc->sc_flags.status_suspend)
 435                 return;
 436
 437         DPRINTFN(5, "Remote wakeup\n");
 438
 439         if (sc->sc_flags.status_device_mode) {
 440                 uint32_t temp;
 441
 442                 /* enable remote wakeup signalling */
 443                 temp = DWC_OTG_READ_4(sc, DOTG_DCTL);
 444                 temp |= DCTL_RMTWKUPSIG;
 445                 DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
 446
 447                 /* Wait 8ms for remote wakeup to complete. */
 448                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 125);
 449
 450                 temp &= ~DCTL_RMTWKUPSIG;
 451                 DWC_OTG_WRITE_4(sc, DOTG_DCTL, temp);
 452         } else {
 453                 /* enable USB port */
 454                 DWC_OTG_WRITE_4(sc, DOTG_PCGCCTL, 0);
 455
 456                 /* wait 10ms */
 457                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 100);
 458
 459                 /* resume port */
 460                 sc->sc_hprt_val |= HPRT_PRTRES;
 461                 DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
 462
 463                 /* Wait 100ms for resume signalling to complete. */
 464                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 10);
 465
 466                 /* clear suspend and resume */
 467                 sc->sc_hprt_val &= ~(HPRT_PRTSUSP | HPRT_PRTRES);
 468                 DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
 469
 470                 /* Wait 4ms */
 471                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 250);
 472         }
 473
 474         /* need to fake resume IRQ */
 475         dwc_otg_resume_irq(sc);
 476 }
 477
 478 static void
 479 dwc_otg_set_address(struct dwc_otg_softc *sc, uint8_t addr)
 480 {
 481         uint32_t temp;
 482
 483         DPRINTFN(5, "addr=%d\n", addr);
 484
 485         temp = DWC_OTG_READ_4(sc, DOTG_DCFG);
 486         temp &= ~DCFG_DEVADDR_SET(0x7F);
 487         temp |= DCFG_DEVADDR_SET(addr);
 488         DWC_OTG_WRITE_4(sc, DOTG_DCFG, temp);
 489 }
 490
 491 static void
 492 dwc_otg_common_rx_ack(struct dwc_otg_softc *sc)
 493 {
 494         DPRINTFN(5, "RX status clear\n");
 495
 496         /* enable RX FIFO level interrupt */
 497         sc->sc_irq_mask |= GINTSTS_RXFLVL;
 498         DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
 499
 500         /* clear cached status */
 501         sc->sc_last_rx_status = 0;
 502 }
 503
 504 static void
 505 dwc_otg_clear_hcint(struct dwc_otg_softc *sc, uint8_t x)
 506 {
 507         uint32_t hcint;
 508
 509         hcint = DWC_OTG_READ_4(sc, DOTG_HCINT(x));
 510         DWC_OTG_WRITE_4(sc, DOTG_HCINT(x), hcint);
 511
 512         /* clear buffered interrupts */
 513         sc->sc_chan_state[x].hcint = 0;
 514 }
 515
 516 static uint8_t
 517 dwc_otg_host_channel_wait(struct dwc_otg_td *td)
 518 {
 519         struct dwc_otg_softc *sc;
 520         uint8_t x;
 521
 522         x = td->channel;
 523
 524         DPRINTF("CH=%d\n", x);
 525
 526         /* get pointer to softc */
 527         sc = DWC_OTG_PC2SC(td->pc);
 528
 529         if (sc->sc_chan_state[x].wait_sof == 0) {
 530                 dwc_otg_clear_hcint(sc, x);
 531                 return (1);     /* done */
 532         }
 533
 534         if (x == 0)
 535                 return (0);     /* wait */
 536
 537         /* find new disabled channel */
 538         for (x = 1; x != sc->sc_host_ch_max; x++) {
 539
 540                 if (sc->sc_chan_state[x].allocated)
 541                         continue;
 542                 if (sc->sc_chan_state[x].wait_sof != 0)
 543                         continue;
 544
 545                 sc->sc_chan_state[td->channel].allocated = 0;
 546                 sc->sc_chan_state[x].allocated = 1;
 547
 548                 if (sc->sc_chan_state[td->channel].suspended) {
 549                         sc->sc_chan_state[td->channel].suspended = 0;
 550                         sc->sc_chan_state[x].suspended = 1;
 551                 }
 552
 553                 /* clear interrupts */
 554                 dwc_otg_clear_hcint(sc, x);
 555
 556                 DPRINTF("CH=%d HCCHAR=0x%08x "
 557                     "HCSPLT=0x%08x\n", x, td->hcchar, td->hcsplt);
 558
 559                 /* ack any pending messages */
 560                 if (sc->sc_last_rx_status != 0 &&
 561                     GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) == td->channel) {
 562                         /* get rid of message */
 563                         dwc_otg_common_rx_ack(sc);
 564                 }
 565
 566                 /* move active channel */
 567                 sc->sc_active_rx_ep &= ~(1 << td->channel);
 568                 sc->sc_active_rx_ep |= (1 << x);
 569
 570                 /* set channel */
 571                 td->channel = x;
 572
 573                 return (1);     /* new channel allocated */
 574         }
 575         return (0);     /* wait */
 576 }
 577
 578 static uint8_t
 579 dwc_otg_host_channel_alloc(struct dwc_otg_td *td)
 580 {
 581         struct dwc_otg_softc *sc;
 582         uint8_t x;
 583         uint8_t max_channel;
 584
 585         if (td->channel < DWC_OTG_MAX_CHANNELS)
 586                 return (0);             /* already allocated */
 587
 588         /* get pointer to softc */
 589         sc = DWC_OTG_PC2SC(td->pc);
 590
 591         if ((td->hcchar & HCCHAR_EPNUM_MASK) == 0) {
 592                 max_channel = 1;
 593                 x = 0;
 594         } else {
 595                 max_channel = sc->sc_host_ch_max;
 596                 x = 1;
 597         }
 598
 599         for (; x != max_channel; x++) {
 600
 601                 if (sc->sc_chan_state[x].allocated)
 602                         continue;
 603                 if (sc->sc_chan_state[x].wait_sof != 0)
 604                         continue;
 605
 606                 sc->sc_chan_state[x].allocated = 1;
 607
 608                 /* clear interrupts */
 609                 dwc_otg_clear_hcint(sc, x);
 610
 611                 DPRINTF("CH=%d HCCHAR=0x%08x "
 612                     "HCSPLT=0x%08x\n", x, td->hcchar, td->hcsplt);
 613
 614                 /* set active channel */
 615                 sc->sc_active_rx_ep |= (1 << x);
 616
 617                 /* set channel */
 618                 td->channel = x;
 619
 620                 return (0);     /* allocated */
 621         }
 622         return (1);     /* busy */
 623 }
 624
 625 static void
 626 dwc_otg_host_channel_disable(struct dwc_otg_softc *sc, uint8_t x)
 627 {
 628         uint32_t hcchar;
 629         if (sc->sc_chan_state[x].wait_sof != 0)
 630                 return;
 631         hcchar = DWC_OTG_READ_4(sc, DOTG_HCCHAR(x));
 632         if (hcchar & (HCCHAR_CHENA | HCCHAR_CHDIS)) {
 633                 /* disable channel */
 634                 DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(x),
 635                     HCCHAR_CHENA | HCCHAR_CHDIS);
 636                 /* don't re-use channel until next SOF is transmitted */
 637                 sc->sc_chan_state[x].wait_sof = 2;
 638                 /* enable SOF interrupt */
 639                 dwc_otg_enable_sof_irq(sc);
 640         }
 641 }
 642
 643 static void
 644 dwc_otg_host_channel_free(struct dwc_otg_td *td)
 645 {
 646         struct dwc_otg_softc *sc;
 647         uint8_t x;
 648
 649         if (td->channel >= DWC_OTG_MAX_CHANNELS)
 650                 return;         /* already freed */
 651
 652         /* free channel */
 653         x = td->channel;
 654         td->channel = DWC_OTG_MAX_CHANNELS;
 655
 656         DPRINTF("CH=%d\n", x);
 657
 658         /* get pointer to softc */
 659         sc = DWC_OTG_PC2SC(td->pc);
 660
 661         dwc_otg_host_channel_disable(sc, x);
 662
 663         sc->sc_chan_state[x].allocated = 0;
 664         sc->sc_chan_state[x].suspended = 0;
 665
 666         /* ack any pending messages */
 667         if (sc->sc_last_rx_status != 0 &&
 668             GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) == x) {
 669                 dwc_otg_common_rx_ack(sc);
 670         }
 671
 672         /* clear active channel */
 673         sc->sc_active_rx_ep &= ~(1 << x);
 674 }
 675
 676 static uint8_t
 677 dwc_otg_host_setup_tx(struct dwc_otg_td *td)
 678 {
 679         struct usb_device_request req __aligned(4);
 680         struct dwc_otg_softc *sc;
 681         uint32_t hcint;
 682         uint32_t hcchar;
 683
 684         if (dwc_otg_host_channel_alloc(td))
 685                 return (1);             /* busy */
 686
 687         /* get pointer to softc */
 688         sc = DWC_OTG_PC2SC(td->pc);
 689
 690         hcint = sc->sc_chan_state[td->channel].hcint;
 691
 692         DPRINTF("CH=%d ST=%d HCINT=0x%08x HCCHAR=0x%08x HCTSIZ=0x%08x\n",
 693             td->channel, td->state, hcint,
 694             DWC_OTG_READ_4(sc, DOTG_HCCHAR(td->channel)),
 695             DWC_OTG_READ_4(sc, DOTG_HCTSIZ(td->channel)));
 696
 697         if (hcint & (HCINT_RETRY |
 698             HCINT_ACK | HCINT_NYET)) {
 699                 /* give success bits priority over failure bits */
 700         } else if (hcint & HCINT_STALL) {
 701                 DPRINTF("CH=%d STALL\n", td->channel);
 702                 td->error_stall = 1;
 703                 td->error_any = 1;
 704                 return (0);             /* complete */
 705         } else if (hcint & HCINT_ERRORS) {
 706                 DPRINTF("CH=%d ERROR\n", td->channel);
 707                 td->errcnt++;
 708                 if (td->hcsplt != 0 || td->errcnt >= 3) {
 709                         td->error_any = 1;
 710                         return (0);             /* complete */
 711                 }
 712         }
 713
 714         /* channel must be disabled before we can complete the transfer */
 715
 716         if (hcint & (HCINT_ERRORS | HCINT_RETRY |
 717             HCINT_ACK | HCINT_NYET)) {
 718
 719                 dwc_otg_host_channel_disable(sc, td->channel);
 720
 721                 if (!(hcint & HCINT_ERRORS))
 722                         td->errcnt = 0;
 723         }
 724
 725         switch (td->state) {
 726         case DWC_CHAN_ST_START:
 727                 goto send_pkt;
 728
 729         case DWC_CHAN_ST_WAIT_ANE:
 730                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
 731                         if (!dwc_otg_host_channel_wait(td))
 732                                 break;
 733                         td->did_nak = 1;
 734                         goto send_pkt;
 735                 }
 736                 if (hcint & (HCINT_ACK | HCINT_NYET)) {
 737                         if (!dwc_otg_host_channel_wait(td))
 738                                 break;
 739                         td->offset += td->tx_bytes;
 740                         td->remainder -= td->tx_bytes;
 741                         td->toggle = 1;
 742                         return (0);     /* complete */
 743                 }
 744                 break;
 745         case DWC_CHAN_ST_WAIT_S_ANE:
 746                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
 747                         if (!dwc_otg_host_channel_wait(td))
 748                                 break;
 749                         td->did_nak = 1;
 750                         goto send_pkt;
 751                 }
 752                 if (hcint & (HCINT_ACK | HCINT_NYET)) {
 753                         if (!dwc_otg_host_channel_wait(td))
 754                                 break;
 755                         goto send_cpkt;
 756                 }
 757                 break;
 758         case DWC_CHAN_ST_WAIT_C_ANE:
 759                 if (hcint & HCINT_NYET) {
 760                         if (!dwc_otg_host_channel_wait(td))
 761                                 break;
 762                         goto send_cpkt;
 763                 }
 764                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
 765                         if (!dwc_otg_host_channel_wait(td))
 766                                 break;
 767                         td->did_nak = 1;
 768                         goto send_pkt;
 769                 }
 770                 if (hcint & HCINT_ACK) {
 771                         if (!dwc_otg_host_channel_wait(td))
 772                                 break;
 773                         td->offset += td->tx_bytes;
 774                         td->remainder -= td->tx_bytes;
 775                         td->toggle = 1;
 776                         return (0);     /* complete */
 777                 }
 778                 break;
 779         case DWC_CHAN_ST_TX_PKT_SYNC:
 780                 goto send_pkt_sync;
 781         default:
 782                 break;
 783         }
 784         return (1);             /* busy */
 785
 786 send_pkt:
 787         if (sizeof(req) != td->remainder) {
 788                 td->error_any = 1;
 789                 return (0);             /* complete */
 790         }
 791
 792 send_pkt_sync:
 793         if (td->hcsplt != 0) {
 794                 uint32_t count;
 795
 796                 count = DWC_OTG_READ_4(sc, DOTG_HFNUM) & 7;
 797                 /* check for not first microframe */
 798                 if (count != 0) {
 799                         /* enable SOF interrupt */
 800                         dwc_otg_enable_sof_irq(sc);
 801                         /* set state */
 802                         td->state = DWC_CHAN_ST_TX_PKT_SYNC;
 803                         dwc_otg_host_channel_free(td);
 804                         return (1);     /* busy */
 805                 }
 806
 807                 td->hcsplt &= ~HCSPLT_COMPSPLT;
 808                 td->state = DWC_CHAN_ST_WAIT_S_ANE;
 809         } else {
 810                 td->state = DWC_CHAN_ST_WAIT_ANE;
 811         }
 812
 813         usbd_copy_out(td->pc, 0, &req, sizeof(req));
 814
 815         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(td->channel),
 816             (sizeof(req) << HCTSIZ_XFERSIZE_SHIFT) |
 817             (1 << HCTSIZ_PKTCNT_SHIFT) |
 818             (HCTSIZ_PID_SETUP << HCTSIZ_PID_SHIFT));
 819
 820         DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(td->channel), td->hcsplt);
 821
 822         hcchar = td->hcchar;
 823         hcchar &= ~HCCHAR_EPDIR_IN;
 824
 825         /* must enable channel before writing data to FIFO */
 826         DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(td->channel), hcchar);
 827
 828         /* transfer data into FIFO */
 829         bus_space_write_region_4(sc->sc_io_tag, sc->sc_io_hdl,
 830             DOTG_DFIFO(td->channel), (uint32_t *)&req, sizeof(req) / 4);
 831
 832         /* store number of bytes transmitted */
 833         td->tx_bytes = sizeof(req);
 834
 835         return (1);     /* busy */
 836
 837 send_cpkt:
 838         td->hcsplt |= HCSPLT_COMPSPLT;
 839         td->state = DWC_CHAN_ST_WAIT_C_ANE;
 840
 841         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(td->channel),
 842             (HCTSIZ_PID_SETUP << HCTSIZ_PID_SHIFT));
 843
 844         DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(td->channel), td->hcsplt);
 845
 846         hcchar = td->hcchar;
 847         hcchar &= ~HCCHAR_EPDIR_IN;
 848
 849         /* must enable channel before writing data to FIFO */
 850         DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(td->channel), hcchar);
 851
 852         return (1);     /* busy */
 853 }
 854
 855 static uint8_t
 856 dwc_otg_setup_rx(struct dwc_otg_td *td)
 857 {
 858         struct dwc_otg_softc *sc;
 859         struct usb_device_request req __aligned(4);
 860         uint32_t temp;
 861         uint16_t count;
 862
 863         /* get pointer to softc */
 864         sc = DWC_OTG_PC2SC(td->pc);
 865
 866         /* check endpoint status */
 867
 868         if (sc->sc_last_rx_status == 0)
 869                 goto not_complete;
 870
 871         if (GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) != 0)
 872                 goto not_complete;
 873
 874         if ((sc->sc_last_rx_status & GRXSTSRD_DPID_MASK) !=
 875             GRXSTSRD_DPID_DATA0) {
 876                 /* release FIFO */
 877                 dwc_otg_common_rx_ack(sc);
 878                 goto not_complete;
 879         }
 880
 881         if ((sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) !=
 882             GRXSTSRD_STP_DATA) {
 883                 /* release FIFO */
 884                 dwc_otg_common_rx_ack(sc);
 885                 goto not_complete;
 886         }
 887
 888         DPRINTFN(5, "GRXSTSR=0x%08x\n", sc->sc_last_rx_status);
 889
 890         /* clear did stall */
 891         td->did_stall = 0;
 892
 893         /* get the packet byte count */
 894         count = GRXSTSRD_BCNT_GET(sc->sc_last_rx_status);
 895
 896         /* verify data length */
 897         if (count != td->remainder) {
 898                 DPRINTFN(0, "Invalid SETUP packet "
 899                     "length, %d bytes\n", count);
 900                 /* release FIFO */
 901                 dwc_otg_common_rx_ack(sc);
 902                 goto not_complete;
 903         }
 904         if (count != sizeof(req)) {
 905                 DPRINTFN(0, "Unsupported SETUP packet "
 906                     "length, %d bytes\n", count);
 907                 /* release FIFO */
 908                 dwc_otg_common_rx_ack(sc);
 909                 goto not_complete;
 910         }
 911
 912         /* copy in control request */
 913         memcpy(&req, sc->sc_rx_bounce_buffer, sizeof(req));
 914
 915         /* copy data into real buffer */
 916         usbd_copy_in(td->pc, 0, &req, sizeof(req));
 917
 918         td->offset = sizeof(req);
 919         td->remainder = 0;
 920
 921         /* sneak peek the set address */
 922         if ((req.bmRequestType == UT_WRITE_DEVICE) &&
 923             (req.bRequest == UR_SET_ADDRESS)) {
 924                 /* must write address before ZLP */
 925                 dwc_otg_set_address(sc, req.wValue[0] & 0x7F);
 926         }
 927
 928         /* don't send any data by default */
 929         DWC_OTG_WRITE_4(sc, DOTG_DIEPTSIZ(0),
 930             DXEPTSIZ_SET_NPKT(0) |
 931             DXEPTSIZ_SET_NBYTES(0));
 932
 933         temp = sc->sc_in_ctl[0];
 934
 935         /* enable IN endpoint */
 936         DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(0),
 937             temp | DIEPCTL_EPENA);
 938         DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(0),
 939             temp | DIEPCTL_SNAK);
 940
 941         /* reset IN endpoint buffer */
 942         DWC_OTG_WRITE_4(sc, DOTG_GRSTCTL,
 943             GRSTCTL_TXFIFO(0) |
 944             GRSTCTL_TXFFLSH);
 945
 946         /* acknowledge RX status */
 947         dwc_otg_common_rx_ack(sc);
 948         return (0);                     /* complete */
 949
 950 not_complete:
 951         /* abort any ongoing transfer, before enabling again */
 952
 953         temp = sc->sc_out_ctl[0];
 954
 955         temp |= DOEPCTL_EPENA |
 956             DOEPCTL_SNAK;
 957
 958         /* enable OUT endpoint */
 959         DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(0), temp);
 960
 961         if (!td->did_stall) {
 962                 td->did_stall = 1;
 963
 964                 DPRINTFN(5, "stalling IN and OUT direction\n");
 965
 966                 /* set stall after enabling endpoint */
 967                 DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(0),
 968                     temp | DOEPCTL_STALL);
 969
 970                 temp = sc->sc_in_ctl[0];
 971
 972                 /* set stall assuming endpoint is enabled */
 973                 DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(0),
 974                     temp | DIEPCTL_STALL);
 975         }
 976
 977         /* setup number of buffers to receive */
 978         DWC_OTG_WRITE_4(sc, DOTG_DOEPTSIZ(0),
 979             DXEPTSIZ_SET_MULTI(3) |
 980             DXEPTSIZ_SET_NPKT(1) |
 981             DXEPTSIZ_SET_NBYTES(sizeof(req)));
 982
 983         return (1);                     /* not complete */
 984 }
 985
 986 static uint8_t
 987 dwc_otg_host_rate_check(struct dwc_otg_td *td)
 988 {
 989         struct dwc_otg_softc *sc;
 990         uint8_t ep_type;
 991
 992         /* get pointer to softc */
 993         sc = DWC_OTG_PC2SC(td->pc);
 994
 995         ep_type = ((td->hcchar &
 996             HCCHAR_EPTYPE_MASK) >> HCCHAR_EPTYPE_SHIFT);
 997
 998         if (sc->sc_chan_state[td->channel].suspended)
 999                 goto busy;
1000
1001         if (ep_type == UE_ISOCHRONOUS) {
1002                 if (td->tmr_val & 1)
1003                         td->hcchar |= HCCHAR_ODDFRM;
1004                 else
1005                         td->hcchar &= ~HCCHAR_ODDFRM;
1006                 td->tmr_val += td->tmr_res;
1007         } else if (ep_type == UE_INTERRUPT) {
1008                 uint8_t delta;
1009
1010                 delta = sc->sc_tmr_val - td->tmr_val;
1011                 if (delta >= 128)
1012                         goto busy;
1013                 td->tmr_val = sc->sc_tmr_val + td->tmr_res;
1014         } else if (td->did_nak != 0) {
1015                 goto busy;
1016         }
1017
1018         if (ep_type == UE_ISOCHRONOUS) {
1019                 td->toggle = 0;
1020         } else if (td->set_toggle) {
1021                 td->set_toggle = 0;
1022                 td->toggle = 1;
1023         }
1024         return (0);
1025 busy:
1026         return (1);
1027 }
1028
1029 static uint8_t
1030 dwc_otg_host_data_rx(struct dwc_otg_td *td)
1031 {
1032         struct dwc_otg_softc *sc;
1033         uint32_t hcint;
1034         uint32_t hcchar;
1035         uint32_t count;
1036         uint8_t ep_type;
1037
1038         if (dwc_otg_host_channel_alloc(td))
1039                 return (1);             /* busy */
1040
1041         /* get pointer to softc */
1042         sc = DWC_OTG_PC2SC(td->pc);
1043
1044         ep_type = ((td->hcchar &
1045             HCCHAR_EPTYPE_MASK) >> HCCHAR_EPTYPE_SHIFT);
1046
1047         hcint = sc->sc_chan_state[td->channel].hcint;
1048
1049         DPRINTF("CH=%d ST=%d HCINT=0x%08x HCCHAR=0x%08x HCTSIZ=0x%08x\n",
1050             td->channel, td->state, hcint,
1051             DWC_OTG_READ_4(sc, DOTG_HCCHAR(td->channel)),
1052             DWC_OTG_READ_4(sc, DOTG_HCTSIZ(td->channel)));
1053
1054         /* check interrupt bits */
1055
1056         if (hcint & (HCINT_RETRY |
1057             HCINT_ACK | HCINT_NYET)) {
1058                 /* give success bits priority over failure bits */
1059         } else if (hcint & HCINT_STALL) {
1060                 DPRINTF("CH=%d STALL\n", td->channel);
1061                 td->error_stall = 1;
1062                 td->error_any = 1;
1063                 return (0);             /* complete */
1064         } else if (hcint & HCINT_ERRORS) {
1065                 DPRINTF("CH=%d ERROR\n", td->channel);
1066                 td->errcnt++;
1067                 if (td->hcsplt != 0 || td->errcnt >= 3) {
1068                         td->error_any = 1;
1069                         return (0);             /* complete */
1070                 }
1071         }
1072
1073         /* channel must be disabled before we can complete the transfer */
1074
1075         if (hcint & (HCINT_ERRORS | HCINT_RETRY |
1076             HCINT_ACK | HCINT_NYET)) {
1077
1078                 dwc_otg_host_channel_disable(sc, td->channel);
1079
1080                 if (!(hcint & HCINT_ERRORS))
1081                         td->errcnt = 0;
1082         }
1083
1084         /* check endpoint status */
1085         if (sc->sc_last_rx_status == 0)
1086                 goto check_state;
1087
1088         if (GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) != td->channel)
1089                 goto check_state;
1090
1091         switch (sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) {
1092         case GRXSTSRH_IN_DATA:
1093
1094                 DPRINTF("DATA ST=%d STATUS=0x%08x\n",
1095                     (int)td->state, (int)sc->sc_last_rx_status);
1096
1097                 if (hcint & HCINT_SOFTWARE_ONLY) {
1098                         /*
1099                          * When using SPLIT transactions on interrupt
1100                          * endpoints, sometimes data occurs twice.
1101                          */
1102                         DPRINTF("Data already received\n");
1103                         break;
1104                 }
1105
1106                 td->toggle ^= 1;
1107
1108                 /* get the packet byte count */
1109                 count = GRXSTSRD_BCNT_GET(sc->sc_last_rx_status);
1110
1111                 /* verify the packet byte count */
1112                 if (count != td->max_packet_size) {
1113                         if (count < td->max_packet_size) {
1114                                 /* we have a short packet */
1115                                 td->short_pkt = 1;
1116                                 td->got_short = 1;
1117                         } else {
1118                                 /* invalid USB packet */
1119                                 td->error_any = 1;
1120
1121                                 /* release FIFO */
1122                                 dwc_otg_common_rx_ack(sc);
1123                                 return (0);     /* we are complete */
1124                         }
1125                 }
1126
1127                 /* verify the packet byte count */
1128                 if (count > td->remainder) {
1129                         /* invalid USB packet */
1130                         td->error_any = 1;
1131
1132                         /* release FIFO */
1133                         dwc_otg_common_rx_ack(sc);
1134                         return (0);             /* we are complete */
1135                 }
1136
1137                 usbd_copy_in(td->pc, td->offset,
1138                     sc->sc_rx_bounce_buffer, count);
1139
1140                 td->remainder -= count;
1141                 td->offset += count;
1142                 hcint |= HCINT_SOFTWARE_ONLY | HCINT_ACK;
1143                 sc->sc_chan_state[td->channel].hcint = hcint;
1144                 break;
1145
1146         default:
1147                 DPRINTF("OTHER\n");
1148                 break;
1149         }
1150         /* release FIFO */
1151         dwc_otg_common_rx_ack(sc);
1152
1153 check_state:
1154         switch (td->state) {
1155         case DWC_CHAN_ST_START:
1156                 if (td->hcsplt != 0)
1157                         goto receive_spkt;
1158                 else
1159                         goto receive_pkt;
1160
1161         case DWC_CHAN_ST_WAIT_ANE:
1162                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1163                         if (!dwc_otg_host_channel_wait(td))
1164                                 break;
1165
1166                         td->did_nak = 1;
1167                         if (td->hcsplt != 0)
1168                                 goto receive_spkt;
1169                         else
1170                                 goto receive_pkt;
1171                 }
1172                 if (!(hcint & HCINT_SOFTWARE_ONLY)) {
1173                         if (hcint & HCINT_NYET) {
1174                                 if (td->hcsplt != 0) {
1175                                         if (!dwc_otg_host_channel_wait(td))
1176                                                 break;
1177                                         goto receive_pkt;
1178                                 }
1179                         }
1180                         break;
1181                 }
1182                 if (hcint & (HCINT_ACK | HCINT_NYET)) {
1183                         if (!dwc_otg_host_channel_wait(td))
1184                                 break;
1185
1186                         /* check if we are complete */
1187                         if ((td->remainder == 0) || (td->got_short != 0)) {
1188                                 if (td->short_pkt)
1189                                         return (0);     /* complete */
1190
1191                                 /*
1192                                  * Else need to receive a zero length
1193                                  * packet.
1194                                  */
1195                         }
1196                         if (td->hcsplt != 0)
1197                                 goto receive_spkt;
1198                         else
1199                                 goto receive_pkt;
1200                 }
1201                 break;
1202
1203         case DWC_CHAN_ST_WAIT_S_ANE:
1204                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1205                         if (!dwc_otg_host_channel_wait(td))
1206                                 break;
1207
1208                         td->did_nak = 1;
1209                         goto receive_spkt;
1210                 }
1211                 if (hcint & (HCINT_ACK | HCINT_NYET)) {
1212                         if (!dwc_otg_host_channel_wait(td))
1213                                 break;
1214                         goto receive_pkt;
1215                 }
1216                 break;
1217
1218         case DWC_CHAN_ST_RX_PKT:
1219                 goto receive_pkt;
1220
1221         case DWC_CHAN_ST_RX_SPKT:
1222                 goto receive_spkt;
1223
1224         case DWC_CHAN_ST_RX_SPKT_SYNC:
1225                 goto receive_spkt_sync;
1226
1227         default:
1228                 break;
1229         }
1230         goto busy;
1231
1232 receive_pkt:
1233         if (td->hcsplt != 0) {
1234                 count = DWC_OTG_READ_4(sc, DOTG_HFNUM) & 7;
1235
1236                 /* check for even microframes */
1237                 if (count == td->curr_frame) {
1238                         td->state = DWC_CHAN_ST_RX_PKT;
1239                         dwc_otg_host_channel_free(td);
1240                         /* enable SOF interrupt */
1241                         dwc_otg_enable_sof_irq(sc);
1242                         goto busy;
1243                 } else if (count == 0) {
1244                         /* check for start split timeout */
1245                         goto receive_spkt;
1246                 }
1247
1248                 td->curr_frame = count;
1249                 td->hcsplt |= HCSPLT_COMPSPLT;
1250         } else if (dwc_otg_host_rate_check(td)) {
1251                 td->state = DWC_CHAN_ST_RX_PKT;
1252                 dwc_otg_host_channel_free(td);
1253                 goto busy;
1254         }
1255
1256         td->state = DWC_CHAN_ST_WAIT_ANE;
1257
1258         /* receive one packet */
1259         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(td->channel),
1260             (td->max_packet_size << HCTSIZ_XFERSIZE_SHIFT) |
1261             (1 << HCTSIZ_PKTCNT_SHIFT) |
1262             (td->toggle ? (HCTSIZ_PID_DATA1 << HCTSIZ_PID_SHIFT) :
1263             (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT)));
1264
1265         DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(td->channel), td->hcsplt);
1266
1267         hcchar = td->hcchar;
1268         hcchar |= HCCHAR_EPDIR_IN;
1269
1270         /* must enable channel before data can be received */
1271         DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(td->channel), hcchar);
1272
1273         goto busy;
1274
1275 receive_spkt:
1276         if (dwc_otg_host_rate_check(td)) {
1277                 td->state = DWC_CHAN_ST_RX_SPKT;
1278                 dwc_otg_host_channel_free(td);
1279                 goto busy;
1280         }
1281
1282 receive_spkt_sync:
1283         if (ep_type == UE_INTERRUPT ||
1284             ep_type == UE_ISOCHRONOUS) {
1285                 count = DWC_OTG_READ_4(sc, DOTG_HFNUM) & 7;
1286                 td->curr_frame = count;
1287
1288                 /* check for non-zero microframe */
1289                 if (count != 0) {
1290                         /* enable SOF interrupt */
1291                         dwc_otg_enable_sof_irq(sc);
1292                         /* set state */
1293                         td->state = DWC_CHAN_ST_RX_SPKT_SYNC;
1294                         dwc_otg_host_channel_free(td);
1295                         goto busy;
1296                 }
1297         } else {
1298                 count = DWC_OTG_READ_4(sc, DOTG_HFNUM) & 7;
1299                 td->curr_frame = count;
1300
1301                 /* check for two last frames */
1302                 if (count >= 6) {
1303                         /* enable SOF interrupt */
1304                         dwc_otg_enable_sof_irq(sc);
1305                         /* set state */
1306                         td->state = DWC_CHAN_ST_RX_SPKT_SYNC;
1307                         dwc_otg_host_channel_free(td);
1308                         goto busy;
1309                 }
1310         }
1311
1312         td->hcsplt &= ~HCSPLT_COMPSPLT;
1313         td->state = DWC_CHAN_ST_WAIT_S_ANE;
1314
1315         /* receive one packet */
1316         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(td->channel),
1317             (td->toggle ? (HCTSIZ_PID_DATA1 << HCTSIZ_PID_SHIFT) :
1318             (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT)));
1319
1320         DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(td->channel), td->hcsplt);
1321
1322         hcchar = td->hcchar;
1323         hcchar |= HCCHAR_EPDIR_IN;
1324
1325         /* must enable channel before data can be received */
1326         DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(td->channel), hcchar);
1327
1328 busy:
1329         return (1);     /* busy */
1330 }
1331
1332 static uint8_t
1333 dwc_otg_data_rx(struct dwc_otg_td *td)
1334 {
1335         struct dwc_otg_softc *sc;
1336         uint32_t temp;
1337         uint16_t count;
1338         uint8_t got_short;
1339
1340         got_short = 0;
1341
1342         /* get pointer to softc */
1343         sc = DWC_OTG_PC2SC(td->pc);
1344
1345         /* check endpoint status */
1346         if (sc->sc_last_rx_status == 0)
1347                 goto not_complete;
1348
1349         if (GRXSTSRD_CHNUM_GET(sc->sc_last_rx_status) != td->ep_no)
1350                 goto not_complete;
1351
1352         /* check for SETUP packet */
1353         if ((sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) ==
1354             GRXSTSRD_STP_DATA) {
1355                 if (td->remainder == 0) {
1356                         /*
1357                          * We are actually complete and have
1358                          * received the next SETUP
1359                          */
1360                         DPRINTFN(5, "faking complete\n");
1361                         return (0);     /* complete */
1362                 }
1363                 /*
1364                  * USB Host Aborted the transfer.
1365                  */
1366                 td->error_any = 1;
1367                 return (0);             /* complete */
1368         }
1369
1370         if ((sc->sc_last_rx_status & GRXSTSRD_PKTSTS_MASK) !=
1371             GRXSTSRD_OUT_DATA) {
1372                 /* release FIFO */
1373                 dwc_otg_common_rx_ack(sc);
1374                 goto not_complete;
1375         }
1376
1377         /* get the packet byte count */
1378         count = GRXSTSRD_BCNT_GET(sc->sc_last_rx_status);
1379
1380         /* verify the packet byte count */
1381         if (count != td->max_packet_size) {
1382                 if (count < td->max_packet_size) {
1383                         /* we have a short packet */
1384                         td->short_pkt = 1;
1385                         got_short = 1;
1386                 } else {
1387                         /* invalid USB packet */
1388                         td->error_any = 1;
1389
1390                         /* release FIFO */
1391                         dwc_otg_common_rx_ack(sc);
1392                         return (0);     /* we are complete */
1393                 }
1394         }
1395         /* verify the packet byte count */
1396         if (count > td->remainder) {
1397                 /* invalid USB packet */
1398                 td->error_any = 1;
1399
1400                 /* release FIFO */
1401                 dwc_otg_common_rx_ack(sc);
1402                 return (0);             /* we are complete */
1403         }
1404
1405         usbd_copy_in(td->pc, td->offset, sc->sc_rx_bounce_buffer, count);
1406         td->remainder -= count;
1407         td->offset += count;
1408
1409         /* release FIFO */
1410         dwc_otg_common_rx_ack(sc);
1411
1412         /* check if we are complete */
1413         if ((td->remainder == 0) || got_short) {
1414                 if (td->short_pkt) {
1415                         /* we are complete */
1416                         return (0);
1417                 }
1418                 /* else need to receive a zero length packet */
1419         }
1420
1421 not_complete:
1422
1423         temp = sc->sc_out_ctl[td->ep_no];
1424
1425         temp |= DOEPCTL_EPENA | DOEPCTL_CNAK;
1426
1427         DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(td->ep_no), temp);
1428
1429         /* enable SETUP and transfer complete interrupt */
1430         if (td->ep_no == 0) {
1431                 DWC_OTG_WRITE_4(sc, DOTG_DOEPTSIZ(0),
1432                     DXEPTSIZ_SET_NPKT(1) |
1433                     DXEPTSIZ_SET_NBYTES(td->max_packet_size));
1434         } else {
1435                 /* allow reception of multiple packets */
1436                 DWC_OTG_WRITE_4(sc, DOTG_DOEPTSIZ(td->ep_no),
1437                     DXEPTSIZ_SET_MULTI(1) |
1438                     DXEPTSIZ_SET_NPKT(4) |
1439                     DXEPTSIZ_SET_NBYTES(4 *
1440                     ((td->max_packet_size + 3) & ~3)));
1441         }
1442         return (1);                     /* not complete */
1443 }
1444
1445 static uint8_t
1446 dwc_otg_host_data_tx(struct dwc_otg_td *td)
1447 {
1448         struct dwc_otg_softc *sc;
1449         uint32_t count;
1450         uint32_t hcint;
1451         uint32_t hcchar;
1452         uint8_t ep_type;
1453
1454         if (dwc_otg_host_channel_alloc(td))
1455                 return (1);             /* busy */
1456
1457         /* get pointer to softc */
1458         sc = DWC_OTG_PC2SC(td->pc);
1459
1460         ep_type = ((td->hcchar &
1461             HCCHAR_EPTYPE_MASK) >> HCCHAR_EPTYPE_SHIFT);
1462
1463         hcint = sc->sc_chan_state[td->channel].hcint;
1464
1465         DPRINTF("CH=%d ST=%d HCINT=0x%08x HCCHAR=0x%08x HCTSIZ=0x%08x\n",
1466             td->channel, td->state, hcint,
1467             DWC_OTG_READ_4(sc, DOTG_HCCHAR(td->channel)),
1468             DWC_OTG_READ_4(sc, DOTG_HCTSIZ(td->channel)));
1469
1470         if (hcint & (HCINT_RETRY |
1471             HCINT_ACK | HCINT_NYET)) {
1472                 /* give success bits priority over failure bits */
1473         } else if (hcint & HCINT_STALL) {
1474                 DPRINTF("CH=%d STALL\n", td->channel);
1475                 td->error_stall = 1;
1476                 td->error_any = 1;
1477                 return (0);             /* complete */
1478         } else if (hcint & HCINT_ERRORS) {
1479                 DPRINTF("CH=%d ERROR\n", td->channel);
1480                 td->errcnt++;
1481                 if (td->hcsplt != 0 || td->errcnt >= 3) {
1482                         td->error_any = 1;
1483                         return (0);             /* complete */
1484                 }
1485         }
1486
1487         /* channel must be disabled before we can complete the transfer */
1488
1489         if (hcint & (HCINT_ERRORS | HCINT_RETRY |
1490             HCINT_ACK | HCINT_NYET)) {
1491
1492                 dwc_otg_host_channel_disable(sc, td->channel);
1493
1494                 if (!(hcint & HCINT_ERRORS))
1495                         td->errcnt = 0;
1496         }
1497
1498         switch (td->state) {
1499         case DWC_CHAN_ST_START:
1500                 goto send_pkt;
1501
1502         case DWC_CHAN_ST_WAIT_ANE:
1503                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1504                         if (!dwc_otg_host_channel_wait(td))
1505                                 break;
1506                         td->did_nak = 1;
1507                         goto send_pkt;
1508                 }
1509                 if (hcint & (HCINT_ACK | HCINT_NYET)) {
1510                         if (!dwc_otg_host_channel_wait(td))
1511                                 break;
1512
1513                         td->offset += td->tx_bytes;
1514                         td->remainder -= td->tx_bytes;
1515                         td->toggle ^= 1;
1516
1517                         /* check remainder */
1518                         if (td->remainder == 0) {
1519                                 if (td->short_pkt)
1520                                         return (0);     /* complete */
1521
1522                                 /*
1523                                  * Else we need to transmit a short
1524                                  * packet:
1525                                  */
1526                         }
1527                         goto send_pkt;
1528                 }
1529                 break;
1530         case DWC_CHAN_ST_WAIT_S_ANE:
1531                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1532                         if (!dwc_otg_host_channel_wait(td))
1533                                 break;
1534                         td->did_nak = 1;
1535                         goto send_pkt;
1536                 }
1537                 if (hcint & (HCINT_ACK | HCINT_NYET)) {
1538                         if (!dwc_otg_host_channel_wait(td))
1539                                 break;
1540                         goto send_cpkt;
1541                 }
1542                 break;
1543         case DWC_CHAN_ST_WAIT_C_ANE:
1544                 if (hcint & HCINT_NYET) {
1545                         if (!dwc_otg_host_channel_wait(td))
1546                                 break;
1547                         goto send_cpkt;
1548                 }
1549                 if (hcint & (HCINT_RETRY | HCINT_ERRORS)) {
1550                         if (!dwc_otg_host_channel_wait(td))
1551                                 break;
1552                         td->did_nak = 1;
1553                         goto send_pkt;
1554                 }
1555                 if (hcint & HCINT_ACK) {
1556                         if (!dwc_otg_host_channel_wait(td))
1557                                 break;
1558                         td->offset += td->tx_bytes;
1559                         td->remainder -= td->tx_bytes;
1560                         td->toggle ^= 1;
1561
1562                         /* check remainder */
1563                         if (td->remainder == 0) {
1564                                 if (td->short_pkt)
1565                                         return (0);     /* complete */
1566
1567                                 /* else we need to transmit a short packet */
1568                         }
1569                         goto send_pkt;
1570                 }
1571                 break;
1572
1573         case DWC_CHAN_ST_TX_PKT:
1574                 goto send_pkt;
1575
1576         case DWC_CHAN_ST_TX_PKT_SYNC:
1577                 goto send_pkt_sync;
1578
1579         case DWC_CHAN_ST_TX_CPKT:
1580                 goto send_cpkt;
1581
1582         default:
1583                 break;
1584         }
1585         goto busy;
1586
1587 send_pkt:
1588         if (dwc_otg_host_rate_check(td)) {
1589                 td->state = DWC_CHAN_ST_TX_PKT;
1590                 dwc_otg_host_channel_free(td);
1591                 goto busy;
1592         }
1593
1594 send_pkt_sync:
1595         if (td->hcsplt != 0) {
1596                 count = DWC_OTG_READ_4(sc, DOTG_HFNUM) & 7;
1597                 /* check for first or last microframe */
1598                 if (count == 7 || count == 0) {
1599                         /* enable SOF interrupt */
1600                         dwc_otg_enable_sof_irq(sc);
1601                         /* set state */
1602                         td->state = DWC_CHAN_ST_TX_PKT_SYNC;
1603                         dwc_otg_host_channel_free(td);
1604                         goto busy;
1605                 }
1606
1607                 td->hcsplt &= ~HCSPLT_COMPSPLT;
1608                 td->state = DWC_CHAN_ST_WAIT_S_ANE;
1609         } else {
1610                 td->state = DWC_CHAN_ST_WAIT_ANE;
1611         }
1612
1613         /* send one packet at a time */
1614         count = td->max_packet_size;
1615         if (td->remainder < count) {
1616                 /* we have a short packet */
1617                 td->short_pkt = 1;
1618                 count = td->remainder;
1619         }
1620
1621         /* TODO: HCTSIZ_DOPNG */
1622
1623         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(td->channel),
1624             (count << HCTSIZ_XFERSIZE_SHIFT) |
1625             (1 << HCTSIZ_PKTCNT_SHIFT) |
1626             (td->toggle ? (HCTSIZ_PID_DATA1 << HCTSIZ_PID_SHIFT) :
1627             (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT)));
1628
1629         DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(td->channel), td->hcsplt);
1630
1631         hcchar = td->hcchar;
1632         hcchar &= ~HCCHAR_EPDIR_IN;
1633
1634         /* must enable before writing data to FIFO */
1635         DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(td->channel), hcchar);
1636
1637         if (count != 0) {
1638
1639                 /* clear topmost word before copy */
1640                 sc->sc_tx_bounce_buffer[(count - 1) / 4] = 0;
1641
1642                 /* copy out data */
1643                 usbd_copy_out(td->pc, td->offset,
1644                     sc->sc_tx_bounce_buffer, count);
1645
1646                 /* transfer data into FIFO */
1647                 bus_space_write_region_4(sc->sc_io_tag, sc->sc_io_hdl,
1648                     DOTG_DFIFO(td->channel),
1649                     sc->sc_tx_bounce_buffer, (count + 3) / 4);
1650         }
1651
1652         /* store number of bytes transmitted */
1653         td->tx_bytes = count;
1654
1655         goto busy;
1656
1657 send_cpkt:
1658         count = DWC_OTG_READ_4(sc, DOTG_HFNUM) & 7;
1659         /* check for first microframe */
1660         if (count == 0) {
1661                 /* send packet again */
1662                 goto send_pkt;
1663         }
1664
1665         td->hcsplt |= HCSPLT_COMPSPLT;
1666         td->state = DWC_CHAN_ST_WAIT_C_ANE;
1667
1668         DWC_OTG_WRITE_4(sc, DOTG_HCTSIZ(td->channel),
1669             (td->toggle ? (HCTSIZ_PID_DATA1 << HCTSIZ_PID_SHIFT) :
1670             (HCTSIZ_PID_DATA0 << HCTSIZ_PID_SHIFT)));
1671
1672         DWC_OTG_WRITE_4(sc, DOTG_HCSPLT(td->channel), td->hcsplt);
1673
1674         hcchar = td->hcchar;
1675         hcchar &= ~HCCHAR_EPDIR_IN;
1676
1677         /* must enable channel before writing data to FIFO */
1678         DWC_OTG_WRITE_4(sc, DOTG_HCCHAR(td->channel), hcchar);
1679
1680 busy:
1681         return (1);     /* busy */
1682 }
1683
1684 static uint8_t
1685 dwc_otg_data_tx(struct dwc_otg_td *td)
1686 {
1687         struct dwc_otg_softc *sc;
1688         uint32_t max_buffer;
1689         uint32_t count;
1690         uint32_t fifo_left;
1691         uint32_t mpkt;
1692         uint32_t temp;
1693         uint8_t to;
1694
1695         to = 3;                         /* don't loop forever! */
1696
1697         /* get pointer to softc */
1698         sc = DWC_OTG_PC2SC(td->pc);
1699
1700         max_buffer = sc->sc_hw_ep_profile[td->ep_no].max_buffer;
1701
1702 repeat:
1703         /* check for for endpoint 0 data */
1704
1705         temp = sc->sc_last_rx_status;
1706
1707         if ((td->ep_no == 0) && (temp != 0) &&
1708             (GRXSTSRD_CHNUM_GET(temp) == 0)) {
1709
1710                 if ((temp & GRXSTSRD_PKTSTS_MASK) !=
1711                     GRXSTSRD_STP_DATA) {
1712
1713                         /* dump data - wrong direction */
1714                         dwc_otg_common_rx_ack(sc);
1715                 } else {
1716                         /*
1717                          * The current transfer was cancelled
1718                          * by the USB Host:
1719                          */
1720                         td->error_any = 1;
1721                         return (0);             /* complete */
1722                 }
1723         }
1724
1725         /* fill in more TX data, if possible */
1726         if (td->tx_bytes != 0) {
1727
1728                 uint16_t cpkt;
1729
1730                 /* check if packets have been transferred */
1731                 temp = DWC_OTG_READ_4(sc, DOTG_DIEPTSIZ(td->ep_no));
1732
1733                 /* get current packet number */
1734                 cpkt = DXEPTSIZ_GET_NPKT(temp);
1735
1736                 if (cpkt >= td->npkt) {
1737                         fifo_left = 0;
1738                 } else {
1739                         if (max_buffer != 0) {
1740                                 fifo_left = (td->npkt - cpkt) *
1741                                     td->max_packet_size;
1742
1743                                 if (fifo_left > max_buffer)
1744                                         fifo_left = max_buffer;
1745                         } else {
1746                                 fifo_left = td->max_packet_size;
1747                         }
1748                 }
1749
1750                 count = td->tx_bytes;
1751                 if (count > fifo_left)
1752                         count = fifo_left;
1753
1754                 if (count != 0) {
1755
1756                         /* clear topmost word before copy */
1757                         sc->sc_tx_bounce_buffer[(count - 1) / 4] = 0;
1758
1759                         /* copy out data */
1760                         usbd_copy_out(td->pc, td->offset,
1761                             sc->sc_tx_bounce_buffer, count);
1762
1763                         /* transfer data into FIFO */
1764                         bus_space_write_region_4(sc->sc_io_tag, sc->sc_io_hdl,
1765                             DOTG_DFIFO(td->ep_no),
1766                             sc->sc_tx_bounce_buffer, (count + 3) / 4);
1767
1768                         td->tx_bytes -= count;
1769                         td->remainder -= count;
1770                         td->offset += count;
1771                         td->npkt = cpkt;
1772                 }
1773                 if (td->tx_bytes != 0)
1774                         goto not_complete;
1775
1776                 /* check remainder */
1777                 if (td->remainder == 0) {
1778                         if (td->short_pkt)
1779                                 return (0);     /* complete */
1780
1781                         /* else we need to transmit a short packet */
1782                 }
1783         }
1784
1785         if (!to--)
1786                 goto not_complete;
1787
1788         /* check if not all packets have been transferred */
1789         temp = DWC_OTG_READ_4(sc, DOTG_DIEPTSIZ(td->ep_no));
1790
1791         if (DXEPTSIZ_GET_NPKT(temp) != 0) {
1792
1793                 DPRINTFN(5, "busy ep=%d npkt=%d DIEPTSIZ=0x%08x "
1794                     "DIEPCTL=0x%08x\n", td->ep_no,
1795                     DXEPTSIZ_GET_NPKT(temp),
1796                     temp, DWC_OTG_READ_4(sc, DOTG_DIEPCTL(td->ep_no)));
1797
1798                 goto not_complete;
1799         }
1800
1801         DPRINTFN(5, "rem=%u ep=%d\n", td->remainder, td->ep_no);
1802
1803         /* try to optimise by sending more data */
1804         if ((max_buffer != 0) && ((td->max_packet_size & 3) == 0)) {
1805
1806                 /* send multiple packets at the same time */
1807                 mpkt = max_buffer / td->max_packet_size;
1808
1809                 if (mpkt > 0x3FE)
1810                         mpkt = 0x3FE;
1811
1812                 count = td->remainder;
1813                 if (count > 0x7FFFFF)
1814                         count = 0x7FFFFF - (0x7FFFFF % td->max_packet_size);
1815
1816                 td->npkt = count / td->max_packet_size;
1817
1818                 /*
1819                  * NOTE: We could use 0x3FE instead of "mpkt" in the
1820                  * check below to get more throughput, but then we
1821                  * have a dependency towards non-generic chip features
1822                  * to disable the TX-FIFO-EMPTY interrupts on a per
1823                  * endpoint basis. Increase the maximum buffer size of
1824                  * the IN endpoint to increase the performance.
1825                  */
1826                 if (td->npkt > mpkt) {
1827                         td->npkt = mpkt;
1828                         count = td->max_packet_size * mpkt;
1829                 } else if ((count == 0) || (count % td->max_packet_size)) {
1830                         /* we are transmitting a short packet */
1831                         td->npkt++;
1832                         td->short_pkt = 1;
1833                 }
1834         } else {
1835                 /* send one packet at a time */
1836                 mpkt = 1;
1837                 count = td->max_packet_size;
1838                 if (td->remainder < count) {
1839                         /* we have a short packet */
1840                         td->short_pkt = 1;
1841                         count = td->remainder;
1842                 }
1843                 td->npkt = 1;
1844         }
1845         DWC_OTG_WRITE_4(sc, DOTG_DIEPTSIZ(td->ep_no),
1846             DXEPTSIZ_SET_MULTI(1) |
1847             DXEPTSIZ_SET_NPKT(td->npkt) |
1848             DXEPTSIZ_SET_NBYTES(count));
1849
1850         /* make room for buffering */
1851         td->npkt += mpkt;
1852
1853         temp = sc->sc_in_ctl[td->ep_no];
1854
1855         /* must enable before writing data to FIFO */
1856         DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(td->ep_no), temp |
1857             DIEPCTL_EPENA |
1858             DIEPCTL_CNAK);
1859
1860         td->tx_bytes = count;
1861
1862         /* check remainder */
1863         if (td->tx_bytes == 0 &&
1864             td->remainder == 0) {
1865                 if (td->short_pkt)
1866                         return (0);     /* complete */
1867
1868                 /* else we need to transmit a short packet */
1869         }
1870         goto repeat;
1871
1872 not_complete:
1873         return (1);                     /* not complete */
1874 }
1875
1876 static uint8_t
1877 dwc_otg_data_tx_sync(struct dwc_otg_td *td)
1878 {
1879         struct dwc_otg_softc *sc;
1880         uint32_t temp;
1881
1882         /* get pointer to softc */
1883         sc = DWC_OTG_PC2SC(td->pc);
1884
1885         /*
1886          * If all packets are transferred we are complete:
1887          */
1888         temp = DWC_OTG_READ_4(sc, DOTG_DIEPTSIZ(td->ep_no));
1889
1890         /* check that all packets have been transferred */
1891         if (DXEPTSIZ_GET_NPKT(temp) != 0) {
1892                 DPRINTFN(5, "busy ep=%d\n", td->ep_no);
1893                 goto not_complete;
1894         }
1895         return (0);
1896
1897 not_complete:
1898
1899         /* we only want to know if there is a SETUP packet or free IN packet */
1900
1901         temp = sc->sc_last_rx_status;
1902
1903         if ((td->ep_no == 0) && (temp != 0) &&
1904             (GRXSTSRD_CHNUM_GET(temp) == 0)) {
1905
1906                 if ((temp & GRXSTSRD_PKTSTS_MASK) ==
1907                     GRXSTSRD_STP_DATA) {
1908                         DPRINTFN(5, "faking complete\n");
1909                         /*
1910                          * Race condition: We are complete!
1911                          */
1912                         return (0);
1913                 } else {
1914                         /* dump data - wrong direction */
1915                         dwc_otg_common_rx_ack(sc);
1916                 }
1917         }
1918         return (1);                     /* not complete */
1919 }
1920
1921 static uint8_t
1922 dwc_otg_xfer_do_fifo(struct usb_xfer *xfer)
1923 {
1924         struct dwc_otg_td *td;
1925         uint8_t toggle;
1926         uint8_t channel;
1927         uint8_t tmr_val;
1928         uint8_t tmr_res;
1929
1930         DPRINTFN(9, "\n");
1931
1932         td = xfer->td_transfer_cache;
1933
1934         /*
1935          * If we are suspended in host mode and no channel is
1936          * allocated, simply return:
1937          */
1938         if (xfer->xroot->udev->flags.self_suspended != 0 &&
1939             xfer->xroot->udev->flags.usb_mode == USB_MODE_HOST &&
1940             td->channel >= DWC_OTG_MAX_CHANNELS) {
1941                 return (1);     /* not complete */
1942         }
1943         while (1) {
1944                 if ((td->func) (td)) {
1945                         /* operation in progress */
1946                         break;
1947                 }
1948                 if (((void *)td) == xfer->td_transfer_last) {
1949                         goto done;
1950                 }
1951                 if (td->error_any) {
1952                         goto done;
1953                 } else if (td->remainder > 0) {
1954                         /*
1955                          * We had a short transfer. If there is no alternate
1956                          * next, stop processing !
1957                          */
1958                         if (!td->alt_next)
1959                                 goto done;
1960                 }
1961
1962                 /*
1963                  * Fetch the next transfer descriptor and transfer
1964                  * some flags to the next transfer descriptor
1965                  */
1966                 tmr_res = td->tmr_res;
1967                 tmr_val = td->tmr_val;
1968                 toggle = td->toggle;
1969                 channel = td->channel;
1970                 td = td->obj_next;
1971                 xfer->td_transfer_cache = td;
1972                 td->toggle = toggle;    /* transfer toggle */
1973                 td->channel = channel;  /* transfer channel */
1974                 td->tmr_res = tmr_res;
1975                 td->tmr_val = tmr_val;
1976         }
1977         return (1);                     /* not complete */
1978
1979 done:
1980         /* compute all actual lengths */
1981
1982         dwc_otg_standard_done(xfer);
1983         return (0);                     /* complete */
1984 }
1985
1986 static void
1987 dwc_otg_timer(void *_sc)
1988 {
1989         struct dwc_otg_softc *sc = _sc;
1990         struct usb_xfer *xfer;
1991         struct dwc_otg_td *td;
1992
1993         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
1994
1995         DPRINTF("\n");
1996
1997         /* increment timer value */
1998         sc->sc_tmr_val++;
1999
2000         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
2001                 td = xfer->td_transfer_cache;
2002                 if (td != NULL)
2003                         td->did_nak = 0;
2004         }
2005
2006         /* poll jobs */
2007         dwc_otg_interrupt_poll(sc);
2008
2009         if (sc->sc_timer_active) {
2010                 /* restart timer */
2011                 usb_callout_reset(&sc->sc_timer,
2012                     hz / (1000 / DWC_OTG_HOST_TIMER_RATE),
2013                     &dwc_otg_timer, sc);
2014         }
2015 }
2016
2017 static void
2018 dwc_otg_timer_start(struct dwc_otg_softc *sc)
2019 {
2020         if (sc->sc_timer_active != 0)
2021                 return;
2022
2023         sc->sc_timer_active = 1;
2024
2025         /* restart timer */
2026         usb_callout_reset(&sc->sc_timer,
2027             hz / (1000 / DWC_OTG_HOST_TIMER_RATE),
2028             &dwc_otg_timer, sc);
2029 }
2030
2031 static void
2032 dwc_otg_timer_stop(struct dwc_otg_softc *sc)
2033 {
2034         if (sc->sc_timer_active == 0)
2035                 return;
2036
2037         sc->sc_timer_active = 0;
2038
2039         /* stop timer */
2040         usb_callout_stop(&sc->sc_timer);
2041 }
2042
2043 static void
2044 dwc_otg_interrupt_poll(struct dwc_otg_softc *sc)
2045 {
2046         struct usb_xfer *xfer;
2047         uint32_t temp;
2048         uint8_t got_rx_status;
2049         uint8_t x;
2050
2051 repeat:
2052         /* get all channel interrupts */
2053         for (x = 0; x != sc->sc_host_ch_max; x++) {
2054                 temp = DWC_OTG_READ_4(sc, DOTG_HCINT(x));
2055                 if (temp != 0) {
2056                         DWC_OTG_WRITE_4(sc, DOTG_HCINT(x), temp);
2057                         temp &= ~HCINT_SOFTWARE_ONLY;
2058                         sc->sc_chan_state[x].hcint |= temp;
2059                 }
2060         }
2061
2062         if (sc->sc_last_rx_status == 0) {
2063
2064                 temp = DWC_OTG_READ_4(sc, DOTG_GINTSTS);
2065                 if (temp & GINTSTS_RXFLVL) {
2066                         /* pop current status */
2067                         sc->sc_last_rx_status =
2068                             DWC_OTG_READ_4(sc, DOTG_GRXSTSPD);
2069                 }
2070
2071                 if (sc->sc_last_rx_status != 0) {
2072
2073                         uint8_t ep_no;
2074
2075                         temp = sc->sc_last_rx_status &
2076                             GRXSTSRD_PKTSTS_MASK;
2077
2078                         /* non-data messages we simply skip */
2079                         if (temp != GRXSTSRD_STP_DATA &&
2080                             temp != GRXSTSRD_OUT_DATA) {
2081                                 dwc_otg_common_rx_ack(sc);
2082                                 goto repeat;
2083                         }
2084
2085                         temp = GRXSTSRD_BCNT_GET(
2086                             sc->sc_last_rx_status);
2087                         ep_no = GRXSTSRD_CHNUM_GET(
2088                             sc->sc_last_rx_status);
2089
2090                         /* receive data, if any */
2091                         if (temp != 0) {
2092                                 DPRINTF("Reading %d bytes from ep %d\n", temp, ep_no);
2093                                 bus_space_read_region_4(sc->sc_io_tag, sc->sc_io_hdl,
2094                                     DOTG_DFIFO(ep_no),
2095                                     sc->sc_rx_bounce_buffer, (temp + 3) / 4);
2096                         }
2097
2098                         /* check if we should dump the data */
2099                         if (!(sc->sc_active_rx_ep & (1U << ep_no))) {
2100                                 dwc_otg_common_rx_ack(sc);
2101                                 goto repeat;
2102                         }
2103
2104                         got_rx_status = 1;
2105
2106                         DPRINTFN(5, "RX status = 0x%08x: ch=%d pid=%d bytes=%d sts=%d\n",
2107                             sc->sc_last_rx_status, ep_no,
2108                             (sc->sc_last_rx_status >> 15) & 3,
2109                             GRXSTSRD_BCNT_GET(sc->sc_last_rx_status),
2110                             (sc->sc_last_rx_status >> 17) & 15);
2111                 } else {
2112                         got_rx_status = 0;
2113                 }
2114         } else {
2115                 uint8_t ep_no;
2116
2117                 ep_no = GRXSTSRD_CHNUM_GET(
2118                     sc->sc_last_rx_status);
2119
2120                 /* check if we should dump the data */
2121                 if (!(sc->sc_active_rx_ep & (1U << ep_no))) {
2122                         dwc_otg_common_rx_ack(sc);
2123                         goto repeat;
2124                 }
2125
2126                 got_rx_status = 1;
2127         }
2128
2129         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
2130                 if (!dwc_otg_xfer_do_fifo(xfer)) {
2131                         /* queue has been modified */
2132                         goto repeat;
2133                 }
2134         }
2135
2136         if (got_rx_status) {
2137                 /* check if data was consumed */
2138                 if (sc->sc_last_rx_status == 0)
2139                         goto repeat;
2140
2141                 /* disable RX FIFO level interrupt */
2142                 sc->sc_irq_mask &= ~GINTSTS_RXFLVL;
2143                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2144         }
2145 }
2146
2147 static void
2148 dwc_otg_vbus_interrupt(struct dwc_otg_softc *sc, uint8_t is_on)
2149 {
2150         DPRINTFN(5, "vbus = %u\n", is_on);
2151
2152         if (is_on) {
2153                 if (!sc->sc_flags.status_vbus) {
2154                         sc->sc_flags.status_vbus = 1;
2155
2156                         /* complete root HUB interrupt endpoint */
2157
2158                         dwc_otg_root_intr(sc);
2159                 }
2160         } else {
2161                 if (sc->sc_flags.status_vbus) {
2162                         sc->sc_flags.status_vbus = 0;
2163                         sc->sc_flags.status_bus_reset = 0;
2164                         sc->sc_flags.status_suspend = 0;
2165                         sc->sc_flags.change_suspend = 0;
2166                         sc->sc_flags.change_connect = 1;
2167
2168                         /* complete root HUB interrupt endpoint */
2169
2170                         dwc_otg_root_intr(sc);
2171                 }
2172         }
2173 }
2174
2175 void
2176 dwc_otg_interrupt(struct dwc_otg_softc *sc)
2177 {
2178         uint32_t status;
2179
2180         USB_BUS_LOCK(&sc->sc_bus);
2181
2182         /* read and clear interrupt status */
2183         status = DWC_OTG_READ_4(sc, DOTG_GINTSTS);
2184         DWC_OTG_WRITE_4(sc, DOTG_GINTSTS, status);
2185
2186         DPRINTFN(14, "GINTSTS=0x%08x HAINT=0x%08x HFNUM=0x%08x\n",
2187             status, DWC_OTG_READ_4(sc, DOTG_HAINT),
2188             DWC_OTG_READ_4(sc, DOTG_HFNUM));
2189
2190         if (status & GINTSTS_USBRST) {
2191
2192                 /* set correct state */
2193                 sc->sc_flags.status_device_mode = 1;
2194                 sc->sc_flags.status_bus_reset = 0;
2195                 sc->sc_flags.status_suspend = 0;
2196                 sc->sc_flags.change_suspend = 0;
2197                 sc->sc_flags.change_connect = 1;
2198
2199                 /* complete root HUB interrupt endpoint */
2200                 dwc_otg_root_intr(sc);
2201         }
2202
2203         /* check for any bus state change interrupts */
2204         if (status & GINTSTS_ENUMDONE) {
2205
2206                 uint32_t temp;
2207
2208                 DPRINTFN(5, "end of reset\n");
2209
2210                 /* set correct state */
2211                 sc->sc_flags.status_device_mode = 1;
2212                 sc->sc_flags.status_bus_reset = 1;
2213                 sc->sc_flags.status_suspend = 0;
2214                 sc->sc_flags.change_suspend = 0;
2215                 sc->sc_flags.change_connect = 1;
2216                 sc->sc_flags.status_low_speed = 0;
2217                 sc->sc_flags.port_enabled = 1;
2218
2219                 /* reset FIFOs */
2220                 dwc_otg_init_fifo(sc, DWC_MODE_DEVICE);
2221
2222                 /* reset function address */
2223                 dwc_otg_set_address(sc, 0);
2224
2225                 /* figure out enumeration speed */
2226                 temp = DWC_OTG_READ_4(sc, DOTG_DSTS);
2227                 if (DSTS_ENUMSPD_GET(temp) == DSTS_ENUMSPD_HI)
2228                         sc->sc_flags.status_high_speed = 1;
2229                 else
2230                         sc->sc_flags.status_high_speed = 0;
2231
2232                 /* disable resume interrupt and enable suspend interrupt */
2233
2234                 sc->sc_irq_mask &= ~GINTSTS_WKUPINT;
2235                 sc->sc_irq_mask |= GINTSTS_USBSUSP;
2236                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2237
2238                 /* complete root HUB interrupt endpoint */
2239                 dwc_otg_root_intr(sc);
2240         }
2241
2242         if (status & GINTSTS_PRTINT) {
2243                 uint32_t hprt;
2244
2245                 hprt = DWC_OTG_READ_4(sc, DOTG_HPRT);
2246
2247                 /* clear change bits */
2248                 DWC_OTG_WRITE_4(sc, DOTG_HPRT, (hprt & (
2249                     HPRT_PRTPWR | HPRT_PRTENCHNG |
2250                     HPRT_PRTCONNDET | HPRT_PRTOVRCURRCHNG)) |
2251                     sc->sc_hprt_val);
2252
2253                 DPRINTFN(12, "GINTSTS=0x%08x, HPRT=0x%08x\n", status, hprt);
2254
2255                 sc->sc_flags.status_device_mode = 0;
2256
2257                 if (hprt & HPRT_PRTCONNSTS)
2258                         sc->sc_flags.status_bus_reset = 1;
2259                 else
2260                         sc->sc_flags.status_bus_reset = 0;
2261
2262                 if (hprt & HPRT_PRTENCHNG)
2263                         sc->sc_flags.change_enabled = 1;
2264
2265                 if (hprt & HPRT_PRTENA)
2266                         sc->sc_flags.port_enabled = 1;
2267                 else
2268                         sc->sc_flags.port_enabled = 0;
2269
2270                 if (hprt & HPRT_PRTOVRCURRCHNG)
2271                         sc->sc_flags.change_over_current = 1;
2272
2273                 if (hprt & HPRT_PRTOVRCURRACT)
2274                         sc->sc_flags.port_over_current = 1;
2275                 else
2276                         sc->sc_flags.port_over_current = 0;
2277
2278                 if (hprt & HPRT_PRTPWR)
2279                         sc->sc_flags.port_powered = 1;
2280                 else
2281                         sc->sc_flags.port_powered = 0;
2282
2283                 if (((hprt & HPRT_PRTSPD_MASK)
2284                     >> HPRT_PRTSPD_SHIFT) == HPRT_PRTSPD_LOW)
2285                         sc->sc_flags.status_low_speed = 1;
2286                 else
2287                         sc->sc_flags.status_low_speed = 0;
2288
2289                 if (((hprt & HPRT_PRTSPD_MASK)
2290                     >> HPRT_PRTSPD_SHIFT) == HPRT_PRTSPD_HIGH)
2291                         sc->sc_flags.status_high_speed = 1;
2292                 else
2293                         sc->sc_flags.status_high_speed = 0;
2294
2295                 if (hprt & HPRT_PRTCONNDET)
2296                         sc->sc_flags.change_connect = 1;
2297
2298                 if (hprt & HPRT_PRTSUSP)
2299                         dwc_otg_suspend_irq(sc);
2300                 else
2301                         dwc_otg_resume_irq(sc);
2302
2303                 /* complete root HUB interrupt endpoint */
2304                 dwc_otg_root_intr(sc);
2305         }
2306
2307         /*
2308          * If resume and suspend is set at the same time we interpret
2309          * that like RESUME. Resume is set when there is at least 3
2310          * milliseconds of inactivity on the USB BUS.
2311          */
2312         if (status & GINTSTS_WKUPINT) {
2313
2314                 DPRINTFN(5, "resume interrupt\n");
2315
2316                 dwc_otg_resume_irq(sc);
2317
2318         } else if (status & GINTSTS_USBSUSP) {
2319
2320                 DPRINTFN(5, "suspend interrupt\n");
2321
2322                 dwc_otg_suspend_irq(sc);
2323         }
2324         /* check VBUS */
2325         if (status & (GINTSTS_USBSUSP |
2326             GINTSTS_USBRST |
2327             GINTMSK_OTGINTMSK |
2328             GINTSTS_SESSREQINT)) {
2329                 uint32_t temp;
2330
2331                 temp = DWC_OTG_READ_4(sc, DOTG_GOTGCTL);
2332
2333                 DPRINTFN(5, "GOTGCTL=0x%08x\n", temp);
2334
2335                 dwc_otg_vbus_interrupt(sc,
2336                     (temp & (GOTGCTL_ASESVLD | GOTGCTL_BSESVLD)) ? 1 : 0);
2337         }
2338
2339         /* clear all IN endpoint interrupts */
2340         if (status & GINTSTS_IEPINT) {
2341                 uint32_t temp;
2342                 uint8_t x;
2343
2344                 for (x = 0; x != sc->sc_dev_in_ep_max; x++) {
2345                         temp = DWC_OTG_READ_4(sc, DOTG_DIEPINT(x));
2346                         if (temp & DIEPMSK_XFERCOMPLMSK) {
2347                                 DWC_OTG_WRITE_4(sc, DOTG_DIEPINT(x),
2348                                     DIEPMSK_XFERCOMPLMSK);
2349                         }
2350                 }
2351         }
2352
2353         /* check for SOF interrupt */
2354         if (status & GINTSTS_SOF) {
2355                 if (sc->sc_irq_mask & GINTMSK_SOFMSK) {
2356                         uint8_t x;
2357                         uint8_t y;
2358
2359                         DPRINTFN(12, "SOF interrupt\n");
2360
2361                         for (x = y = 0; x != sc->sc_host_ch_max; x++) {
2362                                 if (sc->sc_chan_state[x].wait_sof != 0) {
2363                                         if (--(sc->sc_chan_state[x].wait_sof) != 0)
2364                                                 y = 1;
2365                                 }
2366                         }
2367                         if (y == 0) {
2368                                 sc->sc_irq_mask &= ~GINTMSK_SOFMSK;
2369                                 DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
2370                         }
2371                 }
2372         }
2373
2374         /* poll FIFO(s) */
2375         dwc_otg_interrupt_poll(sc);
2376
2377         USB_BUS_UNLOCK(&sc->sc_bus);
2378 }
2379
2380 static void
2381 dwc_otg_setup_standard_chain_sub(struct dwc_otg_std_temp *temp)
2382 {
2383         struct dwc_otg_td *td;
2384
2385         /* get current Transfer Descriptor */
2386         td = temp->td_next;
2387         temp->td = td;
2388
2389         /* prepare for next TD */
2390         temp->td_next = td->obj_next;
2391
2392         /* fill out the Transfer Descriptor */
2393         td->func = temp->func;
2394         td->pc = temp->pc;
2395         td->offset = temp->offset;
2396         td->remainder = temp->len;
2397         td->tx_bytes = 0;
2398         td->error_any = 0;
2399         td->error_stall = 0;
2400         td->npkt = 0;
2401         td->did_stall = temp->did_stall;
2402         td->short_pkt = temp->short_pkt;
2403         td->alt_next = temp->setup_alt_next;
2404         td->set_toggle = 0;
2405         td->got_short = 0;
2406         td->did_nak = 0;
2407         td->channel = DWC_OTG_MAX_CHANNELS;
2408         td->state = 0;
2409         td->errcnt = 0;
2410 }
2411
2412 static void
2413 dwc_otg_setup_standard_chain(struct usb_xfer *xfer)
2414 {
2415         struct dwc_otg_std_temp temp;
2416         struct dwc_otg_td *td;
2417         uint32_t x;
2418         uint8_t need_sync;
2419         uint8_t is_host;
2420
2421         DPRINTFN(9, "addr=%d endpt=%d sumlen=%d speed=%d\n",
2422             xfer->address, UE_GET_ADDR(xfer->endpointno),
2423             xfer->sumlen, usbd_get_speed(xfer->xroot->udev));
2424
2425         temp.max_frame_size = xfer->max_frame_size;
2426
2427         td = xfer->td_start[0];
2428         xfer->td_transfer_first = td;
2429         xfer->td_transfer_cache = td;
2430
2431         /* setup temp */
2432
2433         temp.pc = NULL;
2434         temp.td = NULL;
2435         temp.td_next = xfer->td_start[0];
2436         temp.offset = 0;
2437         temp.setup_alt_next = xfer->flags_int.short_frames_ok;
2438         temp.did_stall = !xfer->flags_int.control_stall;
2439
2440         is_host = (xfer->xroot->udev->flags.usb_mode == USB_MODE_HOST);
2441
2442         /* check if we should prepend a setup message */
2443
2444         if (xfer->flags_int.control_xfr) {
2445                 if (xfer->flags_int.control_hdr) {
2446
2447                         if (is_host)
2448                                 temp.func = &dwc_otg_host_setup_tx;
2449                         else
2450                                 temp.func = &dwc_otg_setup_rx;
2451
2452                         temp.len = xfer->frlengths[0];
2453                         temp.pc = xfer->frbuffers + 0;
2454                         temp.short_pkt = temp.len ? 1 : 0;
2455
2456                         /* check for last frame */
2457                         if (xfer->nframes == 1) {
2458                                 /* no STATUS stage yet, SETUP is last */
2459                                 if (xfer->flags_int.control_act)
2460                                         temp.setup_alt_next = 0;
2461                         }
2462
2463                         dwc_otg_setup_standard_chain_sub(&temp);
2464                 }
2465                 x = 1;
2466         } else {
2467                 x = 0;
2468         }
2469
2470         if (x != xfer->nframes) {
2471                 if (xfer->endpointno & UE_DIR_IN) {
2472                         if (is_host) {
2473                                 temp.func = &dwc_otg_host_data_rx;
2474                                 need_sync = 0;
2475                         } else {
2476                                 temp.func = &dwc_otg_data_tx;
2477                                 need_sync = 1;
2478                         }
2479                 } else {
2480                         if (is_host) {
2481                                 temp.func = &dwc_otg_host_data_tx;
2482                                 need_sync = 0;
2483                         } else {
2484                                 temp.func = &dwc_otg_data_rx;
2485                                 need_sync = 0;
2486                         }
2487                 }
2488
2489                 /* setup "pc" pointer */
2490                 temp.pc = xfer->frbuffers + x;
2491         } else {
2492                 need_sync = 0;
2493         }
2494         while (x != xfer->nframes) {
2495
2496                 /* DATA0 / DATA1 message */
2497
2498                 temp.len = xfer->frlengths[x];
2499
2500                 x++;
2501
2502                 if (x == xfer->nframes) {
2503                         if (xfer->flags_int.control_xfr) {
2504                                 if (xfer->flags_int.control_act) {
2505                                         temp.setup_alt_next = 0;
2506                                 }
2507                         } else {
2508                                 temp.setup_alt_next = 0;
2509                         }
2510                 }
2511                 if (temp.len == 0) {
2512
2513                         /* make sure that we send an USB packet */
2514
2515                         temp.short_pkt = 0;
2516
2517                 } else {
2518
2519                         /* regular data transfer */
2520
2521                         temp.short_pkt = (xfer->flags.force_short_xfer ? 0 : 1);
2522                 }
2523
2524                 dwc_otg_setup_standard_chain_sub(&temp);
2525
2526                 if (xfer->flags_int.isochronous_xfr) {
2527                         temp.offset += temp.len;
2528                 } else {
2529                         /* get next Page Cache pointer */
2530                         temp.pc = xfer->frbuffers + x;
2531                 }
2532         }
2533
2534         if (xfer->flags_int.control_xfr) {
2535
2536                 /* always setup a valid "pc" pointer for status and sync */
2537                 temp.pc = xfer->frbuffers + 0;
2538                 temp.len = 0;
2539                 temp.short_pkt = 0;
2540                 temp.setup_alt_next = 0;
2541
2542                 /* check if we need to sync */
2543                 if (need_sync) {
2544                         /* we need a SYNC point after TX */
2545                         temp.func = &dwc_otg_data_tx_sync;
2546                         dwc_otg_setup_standard_chain_sub(&temp);
2547                 }
2548
2549                 /* check if we should append a status stage */
2550                 if (!xfer->flags_int.control_act) {
2551
2552                         /*
2553                          * Send a DATA1 message and invert the current
2554                          * endpoint direction.
2555                          */
2556                         if (xfer->endpointno & UE_DIR_IN) {
2557                                 if (is_host) {
2558                                         temp.func = &dwc_otg_host_data_tx;
2559                                         need_sync = 0;
2560                                 } else {
2561                                         temp.func = &dwc_otg_data_rx;
2562                                         need_sync = 0;
2563                                 }
2564                         } else {
2565                                 if (is_host) {
2566                                         temp.func = &dwc_otg_host_data_rx;
2567                                         need_sync = 0;
2568                                 } else {
2569                                         temp.func = &dwc_otg_data_tx;
2570                                         need_sync = 1;
2571                                 }
2572                         }
2573
2574                         dwc_otg_setup_standard_chain_sub(&temp);
2575
2576                         /* data toggle should be DATA1 */
2577                         td = temp.td;
2578                         td->set_toggle = 1;
2579
2580                         if (need_sync) {
2581                                 /* we need a SYNC point after TX */
2582                                 temp.func = &dwc_otg_data_tx_sync;
2583                                 dwc_otg_setup_standard_chain_sub(&temp);
2584                         }
2585                 }
2586         } else {
2587                 /* check if we need to sync */
2588                 if (need_sync) {
2589
2590                         temp.pc = xfer->frbuffers + 0;
2591                         temp.len = 0;
2592                         temp.short_pkt = 0;
2593                         temp.setup_alt_next = 0;
2594
2595                         /* we need a SYNC point after TX */
2596                         temp.func = &dwc_otg_data_tx_sync;
2597                         dwc_otg_setup_standard_chain_sub(&temp);
2598                 }
2599         }
2600
2601         /* must have at least one frame! */
2602         td = temp.td;
2603         xfer->td_transfer_last = td;
2604
2605         if (is_host) {
2606
2607                 struct dwc_otg_softc *sc;
2608                 uint32_t hcchar;
2609                 uint32_t hcsplt;
2610                 uint8_t xfer_type;
2611
2612                 sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
2613                 xfer_type = xfer->endpoint->edesc->bmAttributes & UE_XFERTYPE;
2614
2615                 /* get first again */
2616                 td = xfer->td_transfer_first;
2617                 td->toggle = (xfer->endpoint->toggle_next ? 1 : 0);
2618
2619                 hcchar =
2620                         (xfer->address << HCCHAR_DEVADDR_SHIFT) |
2621                         (xfer_type << HCCHAR_EPTYPE_SHIFT) |
2622                         ((xfer->endpointno & UE_ADDR) << HCCHAR_EPNUM_SHIFT) |
2623                         (xfer->max_packet_size << HCCHAR_MPS_SHIFT) |
2624                         HCCHAR_CHENA;
2625
2626                 if (usbd_get_speed(xfer->xroot->udev) == USB_SPEED_LOW)
2627                         hcchar |= HCCHAR_LSPDDEV;
2628                 if (UE_GET_DIR(xfer->endpointno) == UE_DIR_IN)
2629                         hcchar |= HCCHAR_EPDIR_IN;
2630
2631                 switch (xfer->xroot->udev->speed) {
2632                 case USB_SPEED_FULL:
2633                 case USB_SPEED_LOW:
2634                         /* check if root HUB port is running High Speed */
2635                         if (sc->sc_flags.status_high_speed != 0) {
2636                                 hcsplt = HCSPLT_SPLTENA |
2637                                     (xfer->xroot->udev->hs_port_no <<
2638                                     HCSPLT_PRTADDR_SHIFT) |
2639                                     (xfer->xroot->udev->hs_hub_addr <<
2640                                     HCSPLT_HUBADDR_SHIFT);
2641                                 if (xfer_type == UE_ISOCHRONOUS)  /* XXX */
2642                                         hcsplt |= (3 << HCSPLT_XACTPOS_SHIFT);
2643                         } else {
2644                                 hcsplt = 0;
2645                         }
2646                         if (xfer_type == UE_INTERRUPT) {
2647                                 uint32_t ival;
2648                                 ival = xfer->interval / DWC_OTG_HOST_TIMER_RATE;
2649                                 if (ival == 0)
2650                                         ival = 1;
2651                                 else if (ival > 127)
2652                                         ival = 127;
2653                                 td->tmr_val = sc->sc_tmr_val + ival;
2654                                 td->tmr_res = ival;
2655                         }
2656                         break;
2657                 case USB_SPEED_HIGH:
2658                         hcsplt = 0;
2659                         if (xfer_type == UE_ISOCHRONOUS ||
2660                             xfer_type == UE_INTERRUPT) {
2661                                 hcchar |= ((xfer->max_packet_count & 3)
2662                                     << HCCHAR_MC_SHIFT);
2663                         }
2664                         if (xfer_type == UE_INTERRUPT) {
2665                                 uint32_t ival;
2666                                 ival = xfer->interval / DWC_OTG_HOST_TIMER_RATE;
2667                                 if (ival == 0)
2668                                         ival = 1;
2669                                 else if (ival > 127)
2670                                         ival = 127;
2671                                 td->tmr_val = sc->sc_tmr_val + ival;
2672                                 td->tmr_res = ival;
2673                         }
2674                         break;
2675                 default:
2676                         hcsplt = 0;
2677                         break;
2678                 }
2679
2680                 if (xfer_type == UE_ISOCHRONOUS) {
2681                         td->tmr_val = xfer->endpoint->isoc_next & 0xFF;
2682                         td->tmr_res = 1 << usbd_xfer_get_fps_shift(xfer);
2683                 } else if (xfer_type != UE_INTERRUPT) {
2684                         td->tmr_val = 0;
2685                         td->tmr_res = 0;
2686                 }
2687
2688                 /* store configuration in all TD's */
2689                 while (1) {
2690                         td->hcchar = hcchar;
2691                         td->hcsplt = hcsplt;
2692
2693                         if (((void *)td) == xfer->td_transfer_last)
2694                                 break;
2695
2696                         td = td->obj_next;
2697                 }
2698         }
2699 }
2700
2701 static void
2702 dwc_otg_timeout(void *arg)
2703 {
2704         struct usb_xfer *xfer = arg;
2705
2706         DPRINTF("xfer=%p\n", xfer);
2707
2708         USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
2709
2710         /* transfer is transferred */
2711         dwc_otg_device_done(xfer, USB_ERR_TIMEOUT);
2712 }
2713
2714 static void
2715 dwc_otg_start_standard_chain(struct usb_xfer *xfer)
2716 {
2717         DPRINTFN(9, "\n");
2718
2719         /* poll one time - will turn on interrupts */
2720         if (dwc_otg_xfer_do_fifo(xfer)) {
2721
2722                 /* put transfer on interrupt queue */
2723                 usbd_transfer_enqueue(&xfer->xroot->bus->intr_q, xfer);
2724
2725                 /* start timeout, if any */
2726                 if (xfer->timeout != 0) {
2727                         usbd_transfer_timeout_ms(xfer,
2728                             &dwc_otg_timeout, xfer->timeout);
2729                 }
2730         }
2731 }
2732
2733 static void
2734 dwc_otg_root_intr(struct dwc_otg_softc *sc)
2735 {
2736         DPRINTFN(9, "\n");
2737
2738         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
2739
2740         /* set port bit */
2741         sc->sc_hub_idata[0] = 0x02;     /* we only have one port */
2742
2743         uhub_root_intr(&sc->sc_bus, sc->sc_hub_idata,
2744             sizeof(sc->sc_hub_idata));
2745 }
2746
2747 static usb_error_t
2748 dwc_otg_standard_done_sub(struct usb_xfer *xfer)
2749 {
2750         struct dwc_otg_td *td;
2751         uint32_t len;
2752         usb_error_t error;
2753
2754         DPRINTFN(9, "\n");
2755
2756         td = xfer->td_transfer_cache;
2757
2758         do {
2759                 len = td->remainder;
2760
2761                 /* store last data toggle */
2762                 xfer->endpoint->toggle_next = td->toggle;
2763
2764                 if (xfer->aframes != xfer->nframes) {
2765                         /*
2766                          * Verify the length and subtract
2767                          * the remainder from "frlengths[]":
2768                          */
2769                         if (len > xfer->frlengths[xfer->aframes]) {
2770                                 td->error_any = 1;
2771                         } else {
2772                                 xfer->frlengths[xfer->aframes] -= len;
2773                         }
2774                 }
2775                 /* Check for transfer error */
2776                 if (td->error_any) {
2777                         /* the transfer is finished */
2778                         error = (td->error_stall ?
2779                             USB_ERR_STALLED : USB_ERR_IOERROR);
2780                         td = NULL;
2781                         break;
2782                 }
2783                 /* Check for short transfer */
2784                 if (len > 0) {
2785                         if (xfer->flags_int.short_frames_ok) {
2786                                 /* follow alt next */
2787                                 if (td->alt_next) {
2788                                         td = td->obj_next;
2789                                 } else {
2790                                         td = NULL;
2791                                 }
2792                         } else {
2793                                 /* the transfer is finished */
2794                                 td = NULL;
2795                         }
2796                         error = 0;
2797                         break;
2798                 }
2799                 td = td->obj_next;
2800
2801                 /* this USB frame is complete */
2802                 error = 0;
2803                 break;
2804
2805         } while (0);
2806
2807         /* update transfer cache */
2808
2809         xfer->td_transfer_cache = td;
2810
2811         return (error);
2812 }
2813
2814 static void
2815 dwc_otg_standard_done(struct usb_xfer *xfer)
2816 {
2817         usb_error_t err = 0;
2818
2819         DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
2820             xfer, xfer->endpoint);
2821
2822         /* reset scanner */
2823
2824         xfer->td_transfer_cache = xfer->td_transfer_first;
2825
2826         if (xfer->flags_int.control_xfr) {
2827
2828                 if (xfer->flags_int.control_hdr) {
2829
2830                         err = dwc_otg_standard_done_sub(xfer);
2831                 }
2832                 xfer->aframes = 1;
2833
2834                 if (xfer->td_transfer_cache == NULL) {
2835                         goto done;
2836                 }
2837         }
2838         while (xfer->aframes != xfer->nframes) {
2839
2840                 err = dwc_otg_standard_done_sub(xfer);
2841                 xfer->aframes++;
2842
2843                 if (xfer->td_transfer_cache == NULL) {
2844                         goto done;
2845                 }
2846         }
2847
2848         if (xfer->flags_int.control_xfr &&
2849             !xfer->flags_int.control_act) {
2850
2851                 err = dwc_otg_standard_done_sub(xfer);
2852         }
2853 done:
2854         dwc_otg_device_done(xfer, err);
2855 }
2856
2857 /*------------------------------------------------------------------------*
2858  *      dwc_otg_device_done
2859  *
2860  * NOTE: this function can be called more than one time on the
2861  * same USB transfer!
2862  *------------------------------------------------------------------------*/
2863 static void
2864 dwc_otg_device_done(struct usb_xfer *xfer, usb_error_t error)
2865 {
2866         DPRINTFN(9, "xfer=%p, endpoint=%p, error=%d\n",
2867             xfer, xfer->endpoint, error);
2868
2869         if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) {
2870                 DPRINTFN(15, "disabled interrupts!\n");
2871         } else {
2872                 struct dwc_otg_td *td;
2873
2874                 td = xfer->td_transfer_first;
2875
2876                 if (td != NULL)
2877                         dwc_otg_host_channel_free(td);
2878         }
2879         /* dequeue transfer and start next transfer */
2880         usbd_transfer_done(xfer, error);
2881 }
2882
2883 static void
2884 dwc_otg_xfer_stall(struct usb_xfer *xfer)
2885 {
2886         dwc_otg_device_done(xfer, USB_ERR_STALLED);
2887 }
2888
2889 static void
2890 dwc_otg_set_stall(struct usb_device *udev,
2891     struct usb_endpoint *ep, uint8_t *did_stall)
2892 {
2893         struct dwc_otg_softc *sc;
2894         uint32_t temp;
2895         uint32_t reg;
2896         uint8_t ep_no;
2897
2898         USB_BUS_LOCK_ASSERT(udev->bus, MA_OWNED);
2899
2900         /* check mode */
2901         if (udev->flags.usb_mode != USB_MODE_DEVICE) {
2902                 /* not supported */
2903                 return;
2904         }
2905
2906         sc = DWC_OTG_BUS2SC(udev->bus);
2907
2908         /* get endpoint address */
2909         ep_no = ep->edesc->bEndpointAddress;
2910
2911         DPRINTFN(5, "endpoint=0x%x\n", ep_no);
2912
2913         if (ep_no & UE_DIR_IN) {
2914                 reg = DOTG_DIEPCTL(ep_no & UE_ADDR);
2915                 temp = sc->sc_in_ctl[ep_no & UE_ADDR];
2916         } else {
2917                 reg = DOTG_DOEPCTL(ep_no & UE_ADDR);
2918                 temp = sc->sc_out_ctl[ep_no & UE_ADDR];
2919         }
2920
2921         /* disable and stall endpoint */
2922         DWC_OTG_WRITE_4(sc, reg, temp | DOEPCTL_EPDIS);
2923         DWC_OTG_WRITE_4(sc, reg, temp | DOEPCTL_STALL);
2924
2925         /* clear active OUT ep */
2926         if (!(ep_no & UE_DIR_IN)) {
2927
2928                 sc->sc_active_rx_ep &= ~(1U << (ep_no & UE_ADDR));
2929
2930                 if (sc->sc_last_rx_status != 0 &&
2931                     (ep_no & UE_ADDR) == GRXSTSRD_CHNUM_GET(
2932                     sc->sc_last_rx_status)) {
2933                         /* dump data */
2934                         dwc_otg_common_rx_ack(sc);
2935                         /* poll interrupt */
2936                         dwc_otg_interrupt_poll(sc);
2937                 }
2938         }
2939 }
2940
2941 static void
2942 dwc_otg_clear_stall_sub(struct dwc_otg_softc *sc, uint32_t mps,
2943     uint8_t ep_no, uint8_t ep_type, uint8_t ep_dir)
2944 {
2945         uint32_t reg;
2946         uint32_t temp;
2947
2948         if (ep_type == UE_CONTROL) {
2949                 /* clearing stall is not needed */
2950                 return;
2951         }
2952
2953         if (ep_dir) {
2954                 reg = DOTG_DIEPCTL(ep_no);
2955         } else {
2956                 reg = DOTG_DOEPCTL(ep_no);
2957                 sc->sc_active_rx_ep |= (1U << ep_no);
2958         }
2959
2960         /* round up and mask away the multiplier count */
2961         mps = (mps + 3) & 0x7FC;
2962
2963         if (ep_type == UE_BULK) {
2964                 temp = DIEPCTL_EPTYPE_SET(
2965                     DIEPCTL_EPTYPE_BULK) |
2966                     DIEPCTL_USBACTEP;
2967         } else if (ep_type == UE_INTERRUPT) {
2968                 temp = DIEPCTL_EPTYPE_SET(
2969                     DIEPCTL_EPTYPE_INTERRUPT) |
2970                     DIEPCTL_USBACTEP;
2971         } else {
2972                 temp = DIEPCTL_EPTYPE_SET(
2973                     DIEPCTL_EPTYPE_ISOC) |
2974                     DIEPCTL_USBACTEP;
2975         }
2976
2977         temp |= DIEPCTL_MPS_SET(mps);
2978         temp |= DIEPCTL_TXFNUM_SET(ep_no);
2979
2980         if (ep_dir)
2981                 sc->sc_in_ctl[ep_no] = temp;
2982         else
2983                 sc->sc_out_ctl[ep_no] = temp;
2984
2985         DWC_OTG_WRITE_4(sc, reg, temp | DOEPCTL_EPDIS);
2986         DWC_OTG_WRITE_4(sc, reg, temp | DOEPCTL_SETD0PID);
2987         DWC_OTG_WRITE_4(sc, reg, temp | DIEPCTL_SNAK);
2988
2989         /* we only reset the transmit FIFO */
2990         if (ep_dir) {
2991                 DWC_OTG_WRITE_4(sc, DOTG_GRSTCTL,
2992                     GRSTCTL_TXFIFO(ep_no) |
2993                     GRSTCTL_TXFFLSH);
2994
2995                 DWC_OTG_WRITE_4(sc,
2996                     DOTG_DIEPTSIZ(ep_no), 0);
2997         }
2998
2999         /* poll interrupt */
3000         dwc_otg_interrupt_poll(sc);
3001 }
3002
3003 static void
3004 dwc_otg_clear_stall(struct usb_device *udev, struct usb_endpoint *ep)
3005 {
3006         struct dwc_otg_softc *sc;
3007         struct usb_endpoint_descriptor *ed;
3008
3009         DPRINTFN(5, "endpoint=%p\n", ep);
3010
3011         USB_BUS_LOCK_ASSERT(udev->bus, MA_OWNED);
3012
3013         /* check mode */
3014         if (udev->flags.usb_mode != USB_MODE_DEVICE) {
3015                 /* not supported */
3016                 return;
3017         }
3018         /* get softc */
3019         sc = DWC_OTG_BUS2SC(udev->bus);
3020
3021         /* get endpoint descriptor */
3022         ed = ep->edesc;
3023
3024         /* reset endpoint */
3025         dwc_otg_clear_stall_sub(sc,
3026             UGETW(ed->wMaxPacketSize),
3027             (ed->bEndpointAddress & UE_ADDR),
3028             (ed->bmAttributes & UE_XFERTYPE),
3029             (ed->bEndpointAddress & (UE_DIR_IN | UE_DIR_OUT)));
3030 }
3031
3032 static void
3033 dwc_otg_device_state_change(struct usb_device *udev)
3034 {
3035         struct dwc_otg_softc *sc;
3036         uint8_t x;
3037
3038         /* check mode */
3039         if (udev->flags.usb_mode != USB_MODE_DEVICE) {
3040                 /* not supported */
3041                 return;
3042         }
3043
3044         /* get softc */
3045         sc = DWC_OTG_BUS2SC(udev->bus);
3046
3047         /* deactivate all other endpoint but the control endpoint */
3048         if (udev->state == USB_STATE_CONFIGURED ||
3049             udev->state == USB_STATE_ADDRESSED) {
3050
3051                 USB_BUS_LOCK(&sc->sc_bus);
3052
3053                 for (x = 1; x != sc->sc_dev_ep_max; x++) {
3054
3055                         if (x < sc->sc_dev_in_ep_max) {
3056                                 DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(x),
3057                                     DIEPCTL_EPDIS);
3058                                 DWC_OTG_WRITE_4(sc, DOTG_DIEPCTL(x), 0);
3059                         }
3060
3061                         DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(x),
3062                             DOEPCTL_EPDIS);
3063                         DWC_OTG_WRITE_4(sc, DOTG_DOEPCTL(x), 0);
3064                 }
3065                 USB_BUS_UNLOCK(&sc->sc_bus);
3066         }
3067 }
3068
3069 int
3070 dwc_otg_init(struct dwc_otg_softc *sc)
3071 {
3072         uint32_t temp;
3073
3074         DPRINTF("start\n");
3075
3076         /* set up the bus structure */
3077         sc->sc_bus.usbrev = USB_REV_2_0;
3078         sc->sc_bus.methods = &dwc_otg_bus_methods;
3079
3080         usb_callout_init_mtx(&sc->sc_timer,
3081             &sc->sc_bus.bus_mtx, 0);
3082
3083         USB_BUS_LOCK(&sc->sc_bus);
3084
3085         /* turn on clocks */
3086         dwc_otg_clocks_on(sc);
3087
3088         temp = DWC_OTG_READ_4(sc, DOTG_GSNPSID);
3089         DPRINTF("Version = 0x%08x\n", temp);
3090
3091         /* disconnect */
3092         DWC_OTG_WRITE_4(sc, DOTG_DCTL,
3093             DCTL_SFTDISCON);
3094
3095         /* wait for host to detect disconnect */
3096         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 32);
3097
3098         DWC_OTG_WRITE_4(sc, DOTG_GRSTCTL,
3099             GRSTCTL_CSFTRST);
3100
3101         /* wait a little bit for block to reset */
3102         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 128);
3103
3104         switch (sc->sc_mode) {
3105         case DWC_MODE_DEVICE:
3106                 temp = GUSBCFG_FORCEDEVMODE;
3107                 break;
3108         case DWC_MODE_HOST:
3109                 temp = GUSBCFG_FORCEHOSTMODE;
3110                 break;
3111         default:
3112                 temp = 0;
3113                 break;
3114         }
3115
3116         /* select HSIC or non-HSIC mode */
3117         if (dwc_otg_use_hsic) {
3118                 DWC_OTG_WRITE_4(sc, DOTG_GUSBCFG,
3119                     GUSBCFG_PHYIF |
3120                     GUSBCFG_TRD_TIM_SET(5) | temp);
3121                 DWC_OTG_WRITE_4(sc, DOTG_GOTGCTL,
3122                     0x000000EC);
3123
3124                 temp = DWC_OTG_READ_4(sc, DOTG_GLPMCFG);
3125                 DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3126                     temp & ~GLPMCFG_HSIC_CONN);
3127                 DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3128                     temp | GLPMCFG_HSIC_CONN);
3129         } else {
3130                 DWC_OTG_WRITE_4(sc, DOTG_GUSBCFG,
3131                     GUSBCFG_ULPI_UTMI_SEL |
3132                     GUSBCFG_TRD_TIM_SET(5) | temp);
3133                 DWC_OTG_WRITE_4(sc, DOTG_GOTGCTL, 0);
3134
3135                 temp = DWC_OTG_READ_4(sc, DOTG_GLPMCFG);
3136                 DWC_OTG_WRITE_4(sc, DOTG_GLPMCFG,
3137                     temp & ~GLPMCFG_HSIC_CONN);
3138         }
3139
3140         /* clear global nak */
3141         DWC_OTG_WRITE_4(sc, DOTG_DCTL,
3142             DCTL_CGOUTNAK |
3143             DCTL_CGNPINNAK);
3144
3145         /* disable USB port */
3146         DWC_OTG_WRITE_4(sc, DOTG_PCGCCTL, 0xFFFFFFFF);
3147
3148         /* wait 10ms */
3149         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 100);
3150
3151         /* enable USB port */
3152         DWC_OTG_WRITE_4(sc, DOTG_PCGCCTL, 0);
3153
3154         /* wait 10ms */
3155         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 100);
3156
3157         /* pull up D+ */
3158         dwc_otg_pull_up(sc);
3159
3160         temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG3);
3161
3162         sc->sc_fifo_size = 4 * GHWCFG3_DFIFODEPTH_GET(temp);
3163
3164         temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG2);
3165
3166         sc->sc_dev_ep_max = GHWCFG2_NUMDEVEPS_GET(temp);
3167
3168         if (sc->sc_dev_ep_max > DWC_OTG_MAX_ENDPOINTS)
3169                 sc->sc_dev_ep_max = DWC_OTG_MAX_ENDPOINTS;
3170
3171         sc->sc_host_ch_max = GHWCFG2_NUMHSTCHNL_GET(temp);
3172
3173         if (sc->sc_host_ch_max > DWC_OTG_MAX_CHANNELS)
3174                 sc->sc_host_ch_max = DWC_OTG_MAX_CHANNELS;
3175
3176         temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG4);
3177
3178         sc->sc_dev_in_ep_max = GHWCFG4_NUM_IN_EP_GET(temp);
3179
3180         DPRINTF("Total FIFO size = %d bytes, Device EPs = %d/%d Host CHs = %d\n",
3181             sc->sc_fifo_size, sc->sc_dev_ep_max, sc->sc_dev_in_ep_max,
3182             sc->sc_host_ch_max);
3183
3184         /* setup FIFO */
3185         if (dwc_otg_init_fifo(sc, DWC_MODE_OTG))
3186                 return (EINVAL);
3187
3188         /* enable interrupts */
3189         sc->sc_irq_mask = DWC_OTG_MSK_GINT_ENABLED;
3190         DWC_OTG_WRITE_4(sc, DOTG_GINTMSK, sc->sc_irq_mask);
3191
3192         if (sc->sc_mode == DWC_MODE_OTG || sc->sc_mode == DWC_MODE_DEVICE) {
3193
3194                 /* enable all endpoint interrupts */
3195                 temp = DWC_OTG_READ_4(sc, DOTG_GHWCFG2);
3196                 if (temp & GHWCFG2_MPI) {
3197                         uint8_t x;
3198
3199                         DPRINTF("Multi Process Interrupts\n");
3200
3201                         for (x = 0; x != sc->sc_dev_in_ep_max; x++) {
3202                                 DWC_OTG_WRITE_4(sc, DOTG_DIEPEACHINTMSK(x),
3203                                     DIEPMSK_XFERCOMPLMSK);
3204                                 DWC_OTG_WRITE_4(sc, DOTG_DOEPEACHINTMSK(x), 0);
3205                         }
3206                         DWC_OTG_WRITE_4(sc, DOTG_DEACHINTMSK, 0xFFFF);
3207                 } else {
3208                         DWC_OTG_WRITE_4(sc, DOTG_DIEPMSK,
3209                             DIEPMSK_XFERCOMPLMSK);
3210                         DWC_OTG_WRITE_4(sc, DOTG_DOEPMSK, 0);
3211                         DWC_OTG_WRITE_4(sc, DOTG_DAINTMSK, 0xFFFF);
3212                 }
3213         }
3214
3215         if (sc->sc_mode == DWC_MODE_OTG || sc->sc_mode == DWC_MODE_HOST) {
3216                 /* setup clocks */
3217                 temp = DWC_OTG_READ_4(sc, DOTG_HCFG);
3218                 temp &= ~(HCFG_FSLSSUPP | HCFG_FSLSPCLKSEL_MASK);
3219                 temp |= (1 << HCFG_FSLSPCLKSEL_SHIFT);
3220                 DWC_OTG_WRITE_4(sc, DOTG_HCFG, temp);
3221         }
3222
3223         /* only enable global IRQ */
3224         DWC_OTG_WRITE_4(sc, DOTG_GAHBCFG,
3225             GAHBCFG_GLBLINTRMSK);
3226
3227         /* turn off clocks */
3228         dwc_otg_clocks_off(sc);
3229
3230         /* read initial VBUS state */
3231
3232         temp = DWC_OTG_READ_4(sc, DOTG_GOTGCTL);
3233
3234         DPRINTFN(5, "GOTCTL=0x%08x\n", temp);
3235
3236         dwc_otg_vbus_interrupt(sc,
3237             (temp & (GOTGCTL_ASESVLD | GOTGCTL_BSESVLD)) ? 1 : 0);
3238
3239         USB_BUS_UNLOCK(&sc->sc_bus);
3240
3241         /* catch any lost interrupts */
3242
3243         dwc_otg_do_poll(&sc->sc_bus);
3244
3245         return (0);                     /* success */
3246 }
3247
3248 void
3249 dwc_otg_uninit(struct dwc_otg_softc *sc)
3250 {
3251         USB_BUS_LOCK(&sc->sc_bus);
3252
3253         /* stop host timer */
3254         dwc_otg_timer_stop(sc);
3255
3256         /* set disconnect */
3257         DWC_OTG_WRITE_4(sc, DOTG_DCTL,
3258             DCTL_SFTDISCON);
3259
3260         /* turn off global IRQ */
3261         DWC_OTG_WRITE_4(sc, DOTG_GAHBCFG, 0);
3262
3263         sc->sc_flags.port_enabled = 0;
3264         sc->sc_flags.port_powered = 0;
3265         sc->sc_flags.status_vbus = 0;
3266         sc->sc_flags.status_bus_reset = 0;
3267         sc->sc_flags.status_suspend = 0;
3268         sc->sc_flags.change_suspend = 0;
3269         sc->sc_flags.change_connect = 1;
3270
3271         dwc_otg_pull_down(sc);
3272         dwc_otg_clocks_off(sc);
3273
3274         USB_BUS_UNLOCK(&sc->sc_bus);
3275
3276         usb_callout_drain(&sc->sc_timer);
3277 }
3278
3279 static void
3280 dwc_otg_suspend(struct dwc_otg_softc *sc)
3281 {
3282         return;
3283 }
3284
3285 static void
3286 dwc_otg_resume(struct dwc_otg_softc *sc)
3287 {
3288         return;
3289 }
3290
3291 static void
3292 dwc_otg_do_poll(struct usb_bus *bus)
3293 {
3294         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(bus);
3295
3296         USB_BUS_LOCK(&sc->sc_bus);
3297         dwc_otg_interrupt_poll(sc);
3298         USB_BUS_UNLOCK(&sc->sc_bus);
3299 }
3300
3301 /*------------------------------------------------------------------------*
3302  * DWC OTG bulk support
3303  * DWC OTG control support
3304  * DWC OTG interrupt support
3305  *------------------------------------------------------------------------*/
3306 static void
3307 dwc_otg_device_non_isoc_open(struct usb_xfer *xfer)
3308 {
3309 }
3310
3311 static void
3312 dwc_otg_device_non_isoc_close(struct usb_xfer *xfer)
3313 {
3314         dwc_otg_device_done(xfer, USB_ERR_CANCELLED);
3315 }
3316
3317 static void
3318 dwc_otg_device_non_isoc_enter(struct usb_xfer *xfer)
3319 {
3320 }
3321
3322 static void
3323 dwc_otg_device_non_isoc_start(struct usb_xfer *xfer)
3324 {
3325         /* setup TDs */
3326         dwc_otg_setup_standard_chain(xfer);
3327         dwc_otg_start_standard_chain(xfer);
3328 }
3329
3330 struct usb_pipe_methods dwc_otg_device_non_isoc_methods =
3331 {
3332         .open = dwc_otg_device_non_isoc_open,
3333         .close = dwc_otg_device_non_isoc_close,
3334         .enter = dwc_otg_device_non_isoc_enter,
3335         .start = dwc_otg_device_non_isoc_start,
3336 };
3337
3338 /*------------------------------------------------------------------------*
3339  * DWC OTG full speed isochronous support
3340  *------------------------------------------------------------------------*/
3341 static void
3342 dwc_otg_device_isoc_open(struct usb_xfer *xfer)
3343 {
3344 }
3345
3346 static void
3347 dwc_otg_device_isoc_close(struct usb_xfer *xfer)
3348 {
3349         dwc_otg_device_done(xfer, USB_ERR_CANCELLED);
3350 }
3351
3352 static void
3353 dwc_otg_device_isoc_enter(struct usb_xfer *xfer)
3354 {
3355         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(xfer->xroot->bus);
3356         uint32_t temp;
3357         uint32_t nframes;
3358         uint8_t shift = usbd_xfer_get_fps_shift(xfer);
3359
3360         DPRINTFN(6, "xfer=%p next=%d nframes=%d\n",
3361             xfer, xfer->endpoint->isoc_next, xfer->nframes);
3362
3363         if (xfer->xroot->udev->flags.usb_mode == USB_MODE_HOST) {
3364                 temp = DWC_OTG_READ_4(sc, DOTG_HFNUM);
3365
3366                 /* get the current frame index */
3367                 nframes = (temp & HFNUM_FRNUM_MASK);
3368         } else {
3369                 temp = DWC_OTG_READ_4(sc, DOTG_DSTS);
3370
3371                 /* get the current frame index */
3372                 nframes = DSTS_SOFFN_GET(temp);
3373         }
3374
3375         if (sc->sc_flags.status_high_speed)
3376                 nframes /= 8;
3377
3378         nframes &= DWC_OTG_FRAME_MASK;
3379
3380         /*
3381          * check if the frame index is within the window where the frames
3382          * will be inserted
3383          */
3384         temp = (nframes - xfer->endpoint->isoc_next) & DWC_OTG_FRAME_MASK;
3385
3386         if ((xfer->endpoint->is_synced == 0) ||
3387             (temp < (((xfer->nframes << shift) + 7) / 8))) {
3388                 /*
3389                  * If there is data underflow or the pipe queue is
3390                  * empty we schedule the transfer a few frames ahead
3391                  * of the current frame position. Else two isochronous
3392                  * transfers might overlap.
3393                  */
3394                 xfer->endpoint->isoc_next = (nframes + 3) & DWC_OTG_FRAME_MASK;
3395                 xfer->endpoint->is_synced = 1;
3396                 DPRINTFN(3, "start next=%d\n", xfer->endpoint->isoc_next);
3397         }
3398         /*
3399          * compute how many milliseconds the insertion is ahead of the
3400          * current frame position:
3401          */
3402         temp = (xfer->endpoint->isoc_next - nframes) & DWC_OTG_FRAME_MASK;
3403
3404         /*
3405          * pre-compute when the isochronous transfer will be finished:
3406          */
3407         xfer->isoc_time_complete =
3408             usb_isoc_time_expand(&sc->sc_bus, nframes) + temp +
3409             (((xfer->nframes << shift) + 7) / 8);
3410
3411         /* setup TDs */
3412         dwc_otg_setup_standard_chain(xfer);
3413
3414         /* compute frame number for next insertion */
3415         xfer->endpoint->isoc_next += (xfer->nframes << shift);
3416 }
3417
3418 static void
3419 dwc_otg_device_isoc_start(struct usb_xfer *xfer)
3420 {
3421         /* start TD chain */
3422         dwc_otg_start_standard_chain(xfer);
3423 }
3424
3425 struct usb_pipe_methods dwc_otg_device_isoc_methods =
3426 {
3427         .open = dwc_otg_device_isoc_open,
3428         .close = dwc_otg_device_isoc_close,
3429         .enter = dwc_otg_device_isoc_enter,
3430         .start = dwc_otg_device_isoc_start,
3431 };
3432
3433 /*------------------------------------------------------------------------*
3434  * DWC OTG root control support
3435  *------------------------------------------------------------------------*
3436  * Simulate a hardware HUB by handling all the necessary requests.
3437  *------------------------------------------------------------------------*/
3438
3439 static const struct usb_device_descriptor dwc_otg_devd = {
3440         .bLength = sizeof(struct usb_device_descriptor),
3441         .bDescriptorType = UDESC_DEVICE,
3442         .bcdUSB = {0x00, 0x02},
3443         .bDeviceClass = UDCLASS_HUB,
3444         .bDeviceSubClass = UDSUBCLASS_HUB,
3445         .bDeviceProtocol = UDPROTO_HSHUBSTT,
3446         .bMaxPacketSize = 64,
3447         .bcdDevice = {0x00, 0x01},
3448         .iManufacturer = 1,
3449         .iProduct = 2,
3450         .bNumConfigurations = 1,
3451 };
3452
3453 static const struct dwc_otg_config_desc dwc_otg_confd = {
3454         .confd = {
3455                 .bLength = sizeof(struct usb_config_descriptor),
3456                 .bDescriptorType = UDESC_CONFIG,
3457                 .wTotalLength[0] = sizeof(dwc_otg_confd),
3458                 .bNumInterface = 1,
3459                 .bConfigurationValue = 1,
3460                 .iConfiguration = 0,
3461                 .bmAttributes = UC_SELF_POWERED,
3462                 .bMaxPower = 0,
3463         },
3464         .ifcd = {
3465                 .bLength = sizeof(struct usb_interface_descriptor),
3466                 .bDescriptorType = UDESC_INTERFACE,
3467                 .bNumEndpoints = 1,
3468                 .bInterfaceClass = UICLASS_HUB,
3469                 .bInterfaceSubClass = UISUBCLASS_HUB,
3470                 .bInterfaceProtocol = 0,
3471         },
3472         .endpd = {
3473                 .bLength = sizeof(struct usb_endpoint_descriptor),
3474                 .bDescriptorType = UDESC_ENDPOINT,
3475                 .bEndpointAddress = (UE_DIR_IN | DWC_OTG_INTR_ENDPT),
3476                 .bmAttributes = UE_INTERRUPT,
3477                 .wMaxPacketSize[0] = 8,
3478                 .bInterval = 255,
3479         },
3480 };
3481
3482 #define HSETW(ptr, val) ptr = { (uint8_t)(val), (uint8_t)((val) >> 8) }
3483
3484 static const struct usb_hub_descriptor_min dwc_otg_hubd = {
3485         .bDescLength = sizeof(dwc_otg_hubd),
3486         .bDescriptorType = UDESC_HUB,
3487         .bNbrPorts = 1,
3488         HSETW(.wHubCharacteristics, (UHD_PWR_NO_SWITCH | UHD_OC_INDIVIDUAL)),
3489         .bPwrOn2PwrGood = 50,
3490         .bHubContrCurrent = 0,
3491         .DeviceRemovable = {0},         /* port is removable */
3492 };
3493
3494 #define STRING_VENDOR \
3495   "D\0W\0C\0O\0T\0G"
3496
3497 #define STRING_PRODUCT \
3498   "O\0T\0G\0 \0R\0o\0o\0t\0 \0H\0U\0B"
3499
3500 USB_MAKE_STRING_DESC(STRING_VENDOR, dwc_otg_vendor);
3501 USB_MAKE_STRING_DESC(STRING_PRODUCT, dwc_otg_product);
3502
3503 static usb_error_t
3504 dwc_otg_roothub_exec(struct usb_device *udev,
3505     struct usb_device_request *req, const void **pptr, uint16_t *plength)
3506 {
3507         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(udev->bus);
3508         const void *ptr;
3509         uint16_t len;
3510         uint16_t value;
3511         uint16_t index;
3512         usb_error_t err;
3513
3514         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
3515
3516         /* buffer reset */
3517         ptr = (const void *)&sc->sc_hub_temp;
3518         len = 0;
3519         err = 0;
3520
3521         value = UGETW(req->wValue);
3522         index = UGETW(req->wIndex);
3523
3524         /* demultiplex the control request */
3525
3526         switch (req->bmRequestType) {
3527         case UT_READ_DEVICE:
3528                 switch (req->bRequest) {
3529                 case UR_GET_DESCRIPTOR:
3530                         goto tr_handle_get_descriptor;
3531                 case UR_GET_CONFIG:
3532                         goto tr_handle_get_config;
3533                 case UR_GET_STATUS:
3534                         goto tr_handle_get_status;
3535                 default:
3536                         goto tr_stalled;
3537                 }
3538                 break;
3539
3540         case UT_WRITE_DEVICE:
3541                 switch (req->bRequest) {
3542                 case UR_SET_ADDRESS:
3543                         goto tr_handle_set_address;
3544                 case UR_SET_CONFIG:
3545                         goto tr_handle_set_config;
3546                 case UR_CLEAR_FEATURE:
3547                         goto tr_valid;  /* nop */
3548                 case UR_SET_DESCRIPTOR:
3549                         goto tr_valid;  /* nop */
3550                 case UR_SET_FEATURE:
3551                 default:
3552                         goto tr_stalled;
3553                 }
3554                 break;
3555
3556         case UT_WRITE_ENDPOINT:
3557                 switch (req->bRequest) {
3558                 case UR_CLEAR_FEATURE:
3559                         switch (UGETW(req->wValue)) {
3560                         case UF_ENDPOINT_HALT:
3561                                 goto tr_handle_clear_halt;
3562                         case UF_DEVICE_REMOTE_WAKEUP:
3563                                 goto tr_handle_clear_wakeup;
3564                         default:
3565                                 goto tr_stalled;
3566                         }
3567                         break;
3568                 case UR_SET_FEATURE:
3569                         switch (UGETW(req->wValue)) {
3570                         case UF_ENDPOINT_HALT:
3571                                 goto tr_handle_set_halt;
3572                         case UF_DEVICE_REMOTE_WAKEUP:
3573                                 goto tr_handle_set_wakeup;
3574                         default:
3575                                 goto tr_stalled;
3576                         }
3577                         break;
3578                 case UR_SYNCH_FRAME:
3579                         goto tr_valid;  /* nop */
3580                 default:
3581                         goto tr_stalled;
3582                 }
3583                 break;
3584
3585         case UT_READ_ENDPOINT:
3586                 switch (req->bRequest) {
3587                 case UR_GET_STATUS:
3588                         goto tr_handle_get_ep_status;
3589                 default:
3590                         goto tr_stalled;
3591                 }
3592                 break;
3593
3594         case UT_WRITE_INTERFACE:
3595                 switch (req->bRequest) {
3596                 case UR_SET_INTERFACE:
3597                         goto tr_handle_set_interface;
3598                 case UR_CLEAR_FEATURE:
3599                         goto tr_valid;  /* nop */
3600                 case UR_SET_FEATURE:
3601                 default:
3602                         goto tr_stalled;
3603                 }
3604                 break;
3605
3606         case UT_READ_INTERFACE:
3607                 switch (req->bRequest) {
3608                 case UR_GET_INTERFACE:
3609                         goto tr_handle_get_interface;
3610                 case UR_GET_STATUS:
3611                         goto tr_handle_get_iface_status;
3612                 default:
3613                         goto tr_stalled;
3614                 }
3615                 break;
3616
3617         case UT_WRITE_CLASS_INTERFACE:
3618         case UT_WRITE_VENDOR_INTERFACE:
3619                 /* XXX forward */
3620                 break;
3621
3622         case UT_READ_CLASS_INTERFACE:
3623         case UT_READ_VENDOR_INTERFACE:
3624                 /* XXX forward */
3625                 break;
3626
3627         case UT_WRITE_CLASS_DEVICE:
3628                 switch (req->bRequest) {
3629                 case UR_CLEAR_FEATURE:
3630                         goto tr_valid;
3631                 case UR_SET_DESCRIPTOR:
3632                 case UR_SET_FEATURE:
3633                         break;
3634                 default:
3635                         goto tr_stalled;
3636                 }
3637                 break;
3638
3639         case UT_WRITE_CLASS_OTHER:
3640                 switch (req->bRequest) {
3641                 case UR_CLEAR_FEATURE:
3642                         goto tr_handle_clear_port_feature;
3643                 case UR_SET_FEATURE:
3644                         goto tr_handle_set_port_feature;
3645                 case UR_CLEAR_TT_BUFFER:
3646                 case UR_RESET_TT:
3647                 case UR_STOP_TT:
3648                         goto tr_valid;
3649
3650                 default:
3651                         goto tr_stalled;
3652                 }
3653                 break;
3654
3655         case UT_READ_CLASS_OTHER:
3656                 switch (req->bRequest) {
3657                 case UR_GET_TT_STATE:
3658                         goto tr_handle_get_tt_state;
3659                 case UR_GET_STATUS:
3660                         goto tr_handle_get_port_status;
3661                 default:
3662                         goto tr_stalled;
3663                 }
3664                 break;
3665
3666         case UT_READ_CLASS_DEVICE:
3667                 switch (req->bRequest) {
3668                 case UR_GET_DESCRIPTOR:
3669                         goto tr_handle_get_class_descriptor;
3670                 case UR_GET_STATUS:
3671                         goto tr_handle_get_class_status;
3672
3673                 default:
3674                         goto tr_stalled;
3675                 }
3676                 break;
3677         default:
3678                 goto tr_stalled;
3679         }
3680         goto tr_valid;
3681
3682 tr_handle_get_descriptor:
3683         switch (value >> 8) {
3684         case UDESC_DEVICE:
3685                 if (value & 0xff) {
3686                         goto tr_stalled;
3687                 }
3688                 len = sizeof(dwc_otg_devd);
3689                 ptr = (const void *)&dwc_otg_devd;
3690                 goto tr_valid;
3691         case UDESC_CONFIG:
3692                 if (value & 0xff) {
3693                         goto tr_stalled;
3694                 }
3695                 len = sizeof(dwc_otg_confd);
3696                 ptr = (const void *)&dwc_otg_confd;
3697                 goto tr_valid;
3698         case UDESC_STRING:
3699                 switch (value & 0xff) {
3700                 case 0:         /* Language table */
3701                         len = sizeof(usb_string_lang_en);
3702                         ptr = (const void *)&usb_string_lang_en;
3703                         goto tr_valid;
3704
3705                 case 1:         /* Vendor */
3706                         len = sizeof(dwc_otg_vendor);
3707                         ptr = (const void *)&dwc_otg_vendor;
3708                         goto tr_valid;
3709
3710                 case 2:         /* Product */
3711                         len = sizeof(dwc_otg_product);
3712                         ptr = (const void *)&dwc_otg_product;
3713                         goto tr_valid;
3714                 default:
3715                         break;
3716                 }
3717                 break;
3718         default:
3719                 goto tr_stalled;
3720         }
3721         goto tr_stalled;
3722
3723 tr_handle_get_config:
3724         len = 1;
3725         sc->sc_hub_temp.wValue[0] = sc->sc_conf;
3726         goto tr_valid;
3727
3728 tr_handle_get_status:
3729         len = 2;
3730         USETW(sc->sc_hub_temp.wValue, UDS_SELF_POWERED);
3731         goto tr_valid;
3732
3733 tr_handle_set_address:
3734         if (value & 0xFF00) {
3735                 goto tr_stalled;
3736         }
3737         sc->sc_rt_addr = value;
3738         goto tr_valid;
3739
3740 tr_handle_set_config:
3741         if (value >= 2) {
3742                 goto tr_stalled;
3743         }
3744         sc->sc_conf = value;
3745         goto tr_valid;
3746
3747 tr_handle_get_interface:
3748         len = 1;
3749         sc->sc_hub_temp.wValue[0] = 0;
3750         goto tr_valid;
3751
3752 tr_handle_get_tt_state:
3753 tr_handle_get_class_status:
3754 tr_handle_get_iface_status:
3755 tr_handle_get_ep_status:
3756         len = 2;
3757         USETW(sc->sc_hub_temp.wValue, 0);
3758         goto tr_valid;
3759
3760 tr_handle_set_halt:
3761 tr_handle_set_interface:
3762 tr_handle_set_wakeup:
3763 tr_handle_clear_wakeup:
3764 tr_handle_clear_halt:
3765         goto tr_valid;
3766
3767 tr_handle_clear_port_feature:
3768         if (index != 1)
3769                 goto tr_stalled;
3770
3771         DPRINTFN(9, "UR_CLEAR_PORT_FEATURE on port %d\n", index);
3772
3773         switch (value) {
3774         case UHF_PORT_SUSPEND:
3775                 dwc_otg_wakeup_peer(sc);
3776                 break;
3777
3778         case UHF_PORT_ENABLE:
3779                 if (sc->sc_flags.status_device_mode == 0) {
3780                         DWC_OTG_WRITE_4(sc, DOTG_HPRT,
3781                             sc->sc_hprt_val | HPRT_PRTENA);
3782                 }
3783                 sc->sc_flags.port_enabled = 0;
3784                 break;
3785
3786         case UHF_C_PORT_RESET:
3787                 sc->sc_flags.change_reset = 0;
3788                 break;
3789
3790         case UHF_C_PORT_ENABLE:
3791                 sc->sc_flags.change_enabled = 0;
3792                 break;
3793
3794         case UHF_C_PORT_OVER_CURRENT:
3795                 sc->sc_flags.change_over_current = 0;
3796                 break;
3797
3798         case UHF_PORT_TEST:
3799         case UHF_PORT_INDICATOR:
3800                 /* nops */
3801                 break;
3802
3803         case UHF_PORT_POWER:
3804                 sc->sc_flags.port_powered = 0;
3805                 if (sc->sc_mode == DWC_MODE_HOST || sc->sc_mode == DWC_MODE_OTG) {
3806                         sc->sc_hprt_val = 0;
3807                         DWC_OTG_WRITE_4(sc, DOTG_HPRT, HPRT_PRTENA);
3808                 }
3809                 dwc_otg_pull_down(sc);
3810                 dwc_otg_clocks_off(sc);
3811                 break;
3812
3813         case UHF_C_PORT_CONNECTION:
3814                 /* clear connect change flag */
3815                 sc->sc_flags.change_connect = 0;
3816                 break;
3817
3818         case UHF_C_PORT_SUSPEND:
3819                 sc->sc_flags.change_suspend = 0;
3820                 break;
3821
3822         default:
3823                 err = USB_ERR_IOERROR;
3824                 goto done;
3825         }
3826         goto tr_valid;
3827
3828 tr_handle_set_port_feature:
3829         if (index != 1) {
3830                 goto tr_stalled;
3831         }
3832         DPRINTFN(9, "UR_SET_PORT_FEATURE\n");
3833
3834         switch (value) {
3835         case UHF_PORT_ENABLE:
3836                 break;
3837
3838         case UHF_PORT_SUSPEND:
3839                 if (sc->sc_flags.status_device_mode == 0) {
3840                         /* set suspend BIT */
3841                         sc->sc_hprt_val |= HPRT_PRTSUSP;
3842                         DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
3843
3844                         /* generate HUB suspend event */
3845                         dwc_otg_suspend_irq(sc);
3846                 }
3847                 break;
3848
3849         case UHF_PORT_RESET:
3850                 if (sc->sc_flags.status_device_mode == 0) {
3851
3852                         DPRINTF("PORT RESET\n");
3853
3854                         /* enable PORT reset */
3855                         DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val | HPRT_PRTRST);
3856
3857                         /* Wait 62.5ms for reset to complete */
3858                         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 16);
3859
3860                         DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
3861
3862                         /* Wait 62.5ms for reset to complete */
3863                         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 16);
3864
3865                         /* reset FIFOs */
3866                         dwc_otg_init_fifo(sc, DWC_MODE_HOST);
3867
3868                         sc->sc_flags.change_reset = 1;
3869                 } else {
3870                         err = USB_ERR_IOERROR;
3871                 }
3872                 break;
3873
3874         case UHF_PORT_TEST:
3875         case UHF_PORT_INDICATOR:
3876                 /* nops */
3877                 break;
3878         case UHF_PORT_POWER:
3879                 if (sc->sc_mode == DWC_MODE_HOST || sc->sc_mode == DWC_MODE_OTG) {
3880                         sc->sc_hprt_val |= HPRT_PRTPWR;
3881                         DWC_OTG_WRITE_4(sc, DOTG_HPRT, sc->sc_hprt_val);
3882                 }
3883                 sc->sc_flags.port_powered = 1;
3884                 break;
3885         default:
3886                 err = USB_ERR_IOERROR;
3887                 goto done;
3888         }
3889         goto tr_valid;
3890
3891 tr_handle_get_port_status:
3892
3893         DPRINTFN(9, "UR_GET_PORT_STATUS\n");
3894
3895         if (index != 1)
3896                 goto tr_stalled;
3897
3898         if (sc->sc_flags.status_vbus)
3899                 dwc_otg_clocks_on(sc);
3900         else
3901                 dwc_otg_clocks_off(sc);
3902
3903         /* Select Device Side Mode */
3904
3905         if (sc->sc_flags.status_device_mode) {
3906                 value = UPS_PORT_MODE_DEVICE;
3907                 dwc_otg_timer_stop(sc);
3908         } else {
3909                 value = 0;
3910                 dwc_otg_timer_start(sc);
3911         }
3912
3913         if (sc->sc_flags.status_high_speed)
3914                 value |= UPS_HIGH_SPEED;
3915         else if (sc->sc_flags.status_low_speed)
3916                 value |= UPS_LOW_SPEED;
3917
3918         if (sc->sc_flags.port_powered)
3919                 value |= UPS_PORT_POWER;
3920
3921         if (sc->sc_flags.port_enabled)
3922                 value |= UPS_PORT_ENABLED;
3923
3924         if (sc->sc_flags.port_over_current)
3925                 value |= UPS_OVERCURRENT_INDICATOR;
3926
3927         if (sc->sc_flags.status_vbus &&
3928             sc->sc_flags.status_bus_reset)
3929                 value |= UPS_CURRENT_CONNECT_STATUS;
3930
3931         if (sc->sc_flags.status_suspend)
3932                 value |= UPS_SUSPEND;
3933
3934         USETW(sc->sc_hub_temp.ps.wPortStatus, value);
3935
3936         value = 0;
3937
3938         if (sc->sc_flags.change_connect)
3939                 value |= UPS_C_CONNECT_STATUS;
3940         if (sc->sc_flags.change_suspend)
3941                 value |= UPS_C_SUSPEND;
3942         if (sc->sc_flags.change_reset)
3943                 value |= UPS_C_PORT_RESET;
3944         if (sc->sc_flags.change_over_current)
3945                 value |= UPS_C_OVERCURRENT_INDICATOR;
3946
3947         USETW(sc->sc_hub_temp.ps.wPortChange, value);
3948         len = sizeof(sc->sc_hub_temp.ps);
3949         goto tr_valid;
3950
3951 tr_handle_get_class_descriptor:
3952         if (value & 0xFF) {
3953                 goto tr_stalled;
3954         }
3955         ptr = (const void *)&dwc_otg_hubd;
3956         len = sizeof(dwc_otg_hubd);
3957         goto tr_valid;
3958
3959 tr_stalled:
3960         err = USB_ERR_STALLED;
3961 tr_valid:
3962 done:
3963         *plength = len;
3964         *pptr = ptr;
3965         return (err);
3966 }
3967
3968 static void
3969 dwc_otg_xfer_setup(struct usb_setup_params *parm)
3970 {
3971         struct usb_xfer *xfer;
3972         void *last_obj;
3973         uint32_t ntd;
3974         uint32_t n;
3975         uint8_t ep_no;
3976
3977         xfer = parm->curr_xfer;
3978
3979         /*
3980          * NOTE: This driver does not use any of the parameters that
3981          * are computed from the following values. Just set some
3982          * reasonable dummies:
3983          */
3984         parm->hc_max_packet_size = 0x500;
3985         parm->hc_max_packet_count = 1;
3986         parm->hc_max_frame_size = 0x500;
3987
3988         usbd_transfer_setup_sub(parm);
3989
3990         /*
3991          * compute maximum number of TDs
3992          */
3993         if ((xfer->endpoint->edesc->bmAttributes & UE_XFERTYPE) == UE_CONTROL) {
3994
3995                 ntd = xfer->nframes + 1 /* STATUS */ + 1 /* SYNC 1 */
3996                     + 1 /* SYNC 2 */ + 1 /* SYNC 3 */;
3997         } else {
3998
3999                 ntd = xfer->nframes + 1 /* SYNC */ ;
4000         }
4001
4002         /*
4003          * check if "usbd_transfer_setup_sub" set an error
4004          */
4005         if (parm->err)
4006                 return;
4007
4008         /*
4009          * allocate transfer descriptors
4010          */
4011         last_obj = NULL;
4012
4013         ep_no = xfer->endpointno & UE_ADDR;
4014
4015         /*
4016          * Check for a valid endpoint profile in USB device mode:
4017          */
4018         if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) {
4019                 const struct usb_hw_ep_profile *pf;
4020
4021                 dwc_otg_get_hw_ep_profile(parm->udev, &pf, ep_no);
4022
4023                 if (pf == NULL) {
4024                         /* should not happen */
4025                         parm->err = USB_ERR_INVAL;
4026                         return;
4027                 }
4028         }
4029
4030         /* align data */
4031         parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
4032
4033         for (n = 0; n != ntd; n++) {
4034
4035                 struct dwc_otg_td *td;
4036
4037                 if (parm->buf) {
4038
4039                         td = USB_ADD_BYTES(parm->buf, parm->size[0]);
4040
4041                         /* init TD */
4042                         td->max_packet_size = xfer->max_packet_size;
4043                         td->ep_no = ep_no;
4044                         td->obj_next = last_obj;
4045
4046                         last_obj = td;
4047                 }
4048                 parm->size[0] += sizeof(*td);
4049         }
4050
4051         xfer->td_start[0] = last_obj;
4052 }
4053
4054 static void
4055 dwc_otg_xfer_unsetup(struct usb_xfer *xfer)
4056 {
4057         return;
4058 }
4059
4060 static void
4061 dwc_otg_ep_init(struct usb_device *udev, struct usb_endpoint_descriptor *edesc,
4062     struct usb_endpoint *ep)
4063 {
4064         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(udev->bus);
4065
4066         DPRINTFN(2, "endpoint=%p, addr=%d, endpt=%d, mode=%d (%d,%d)\n",
4067             ep, udev->address,
4068             edesc->bEndpointAddress, udev->flags.usb_mode,
4069             sc->sc_rt_addr, udev->device_index);
4070
4071         if (udev->device_index != sc->sc_rt_addr) {
4072
4073                 if (udev->flags.usb_mode == USB_MODE_DEVICE) {
4074                         if (udev->speed != USB_SPEED_FULL &&
4075                             udev->speed != USB_SPEED_HIGH) {
4076                                 /* not supported */
4077                                 return;
4078                         }
4079                 } else {
4080                         uint16_t mps;
4081
4082                         mps = UGETW(edesc->wMaxPacketSize);
4083
4084                         /* Apply limitations of our USB host driver */
4085
4086                         switch (udev->speed) {
4087                         case USB_SPEED_HIGH:
4088                                 if (mps > 512) {
4089                                         DPRINTF("wMaxPacketSize=0x%04x"
4090                                             "is not supported\n", (int)mps);
4091                                         /* not supported */
4092                                         return;
4093                                 }
4094                                 break;
4095
4096                         case USB_SPEED_FULL:
4097                         case USB_SPEED_LOW:
4098                                 if (mps > 188) {
4099                                         DPRINTF("wMaxPacketSize=0x%04x"
4100                                             "is not supported\n", (int)mps);
4101                                         /* not supported */
4102                                         return;
4103                                 }
4104                                 break;
4105
4106                         default:
4107                                 DPRINTF("Invalid device speed\n");
4108                                 /* not supported */
4109                                 return;
4110                         }
4111                 }
4112
4113                 if ((edesc->bmAttributes & UE_XFERTYPE) == UE_ISOCHRONOUS)
4114                         ep->methods = &dwc_otg_device_isoc_methods;
4115                 else
4116                         ep->methods = &dwc_otg_device_non_isoc_methods;
4117         }
4118 }
4119
4120 static void
4121 dwc_otg_set_hw_power_sleep(struct usb_bus *bus, uint32_t state)
4122 {
4123         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(bus);
4124
4125         switch (state) {
4126         case USB_HW_POWER_SUSPEND:
4127                 dwc_otg_suspend(sc);
4128                 break;
4129         case USB_HW_POWER_SHUTDOWN:
4130                 dwc_otg_uninit(sc);
4131                 break;
4132         case USB_HW_POWER_RESUME:
4133                 dwc_otg_resume(sc);
4134                 break;
4135         default:
4136                 break;
4137         }
4138 }
4139
4140 static void
4141 dwc_otg_get_dma_delay(struct usb_device *udev, uint32_t *pus)
4142 {
4143         /* DMA delay - wait until any use of memory is finished */
4144         *pus = (2125);                  /* microseconds */
4145 }
4146
4147 static void
4148 dwc_otg_device_resume(struct usb_device *udev)
4149 {
4150         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(udev->bus);
4151         struct usb_xfer *xfer;
4152         struct dwc_otg_td *td;
4153
4154         DPRINTF("\n");
4155
4156         /* Enable relevant Host channels before resuming */
4157
4158         USB_BUS_LOCK(udev->bus);
4159
4160         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
4161
4162                 if (xfer->xroot->udev == udev) {
4163
4164                         td = xfer->td_transfer_cache;
4165                         if (td != NULL &&
4166                             td->channel < DWC_OTG_MAX_CHANNELS)
4167                                 sc->sc_chan_state[td->channel].suspended = 0;
4168                 }
4169         }
4170
4171         USB_BUS_UNLOCK(udev->bus);
4172
4173         /* poll all transfers again to restart resumed ones */
4174         dwc_otg_do_poll(udev->bus);
4175 }
4176
4177 static void
4178 dwc_otg_device_suspend(struct usb_device *udev)
4179 {
4180         struct dwc_otg_softc *sc = DWC_OTG_BUS2SC(udev->bus);
4181         struct usb_xfer *xfer;
4182         struct dwc_otg_td *td;
4183
4184         DPRINTF("\n");
4185
4186         /* Disable relevant Host channels before going to suspend */
4187
4188         USB_BUS_LOCK(udev->bus);
4189
4190         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
4191
4192                 if (xfer->xroot->udev == udev) {
4193
4194                         td = xfer->td_transfer_cache;
4195                         if (td != NULL &&
4196                             td->channel < DWC_OTG_MAX_CHANNELS)
4197                                 sc->sc_chan_state[td->channel].suspended = 1;
4198                 }
4199         }
4200
4201         USB_BUS_UNLOCK(udev->bus);
4202 }
4203
4204 struct usb_bus_methods dwc_otg_bus_methods =
4205 {
4206         .endpoint_init = &dwc_otg_ep_init,
4207         .xfer_setup = &dwc_otg_xfer_setup,
4208         .xfer_unsetup = &dwc_otg_xfer_unsetup,
4209         .get_hw_ep_profile = &dwc_otg_get_hw_ep_profile,
4210         .xfer_stall = &dwc_otg_xfer_stall,
4211         .set_stall = &dwc_otg_set_stall,
4212         .clear_stall = &dwc_otg_clear_stall,
4213         .roothub_exec = &dwc_otg_roothub_exec,
4214         .xfer_poll = &dwc_otg_do_poll,
4215         .device_state_change = &dwc_otg_device_state_change,
4216         .set_hw_power_sleep = &dwc_otg_set_hw_power_sleep,
4217         .get_dma_delay = &dwc_otg_get_dma_delay,
4218         .device_resume = &dwc_otg_device_resume,
4219         .device_suspend = &dwc_otg_device_suspend,
4220 };