sys/dev/usb/controller/ehci.c

   1 /*-
   2  * Copyright (c) 2008 Hans Petter Selasky. All rights reserved.
   3  * Copyright (c) 2004 The NetBSD Foundation, Inc. All rights reserved.
   4  * Copyright (c) 2004 Lennart Augustsson. All rights reserved.
   5  * Copyright (c) 2004 Charles M. Hannum. All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  26  * SUCH DAMAGE.
  27  */
  28
  29 /*
  30  * USB Enhanced Host Controller Driver, a.k.a. USB 2.0 controller.
  31  *
  32  * The EHCI 0.96 spec can be found at
  33  * http://developer.intel.com/technology/usb/download/ehci-r096.pdf
  34  * The EHCI 1.0 spec can be found at
  35  * http://developer.intel.com/technology/usb/download/ehci-r10.pdf
  36  * and the USB 2.0 spec at
  37  * http://www.usb.org/developers/docs/usb_20.zip
  38  *
  39  */
  40
  41 /*
  42  * TODO:
  43  * 1) command failures are not recovered correctly
  44  */
  45
  46 #include <sys/cdefs.h>
  47 __FBSDID("$FreeBSD$");
  48
  49 #include <sys/stdint.h>
  50 #include <sys/stddef.h>
  51 #include <sys/param.h>
  52 #include <sys/queue.h>
  53 #include <sys/types.h>
  54 #include <sys/systm.h>
  55 #include <sys/kernel.h>
  56 #include <sys/bus.h>
  57 #include <sys/linker_set.h>
  58 #include <sys/module.h>
  59 #include <sys/lock.h>
  60 #include <sys/mutex.h>
  61 #include <sys/condvar.h>
  62 #include <sys/sysctl.h>
  63 #include <sys/sx.h>
  64 #include <sys/unistd.h>
  65 #include <sys/callout.h>
  66 #include <sys/malloc.h>
  67 #include <sys/priv.h>
  68
  69 #include <dev/usb/usb.h>
  70 #include <dev/usb/usbdi.h>
  71
  72 #define USB_DEBUG_VAR ehcidebug
  73
  74 #include <dev/usb/usb_core.h>
  75 #include <dev/usb/usb_debug.h>
  76 #include <dev/usb/usb_busdma.h>
  77 #include <dev/usb/usb_process.h>
  78 #include <dev/usb/usb_transfer.h>
  79 #include <dev/usb/usb_device.h>
  80 #include <dev/usb/usb_hub.h>
  81 #include <dev/usb/usb_util.h>
  82
  83 #include <dev/usb/usb_controller.h>
  84 #include <dev/usb/usb_bus.h>
  85 #include <dev/usb/controller/ehci.h>
  86
  87 #define EHCI_BUS2SC(bus) \
  88    ((ehci_softc_t *)(((uint8_t *)(bus)) - \
  89     ((uint8_t *)&(((ehci_softc_t *)0)->sc_bus))))
  90
  91 #if USB_DEBUG
  92 static int ehcidebug = 0;
  93 static int ehcinohighspeed = 0;
  94
  95 SYSCTL_NODE(_hw_usb, OID_AUTO, ehci, CTLFLAG_RW, 0, "USB ehci");
  96 SYSCTL_INT(_hw_usb_ehci, OID_AUTO, debug, CTLFLAG_RW,
  97     &ehcidebug, 0, "Debug level");
  98 SYSCTL_INT(_hw_usb_ehci, OID_AUTO, no_hs, CTLFLAG_RW,
  99     &ehcinohighspeed, 0, "Disable High Speed USB");
 100
 101 static void ehci_dump_regs(ehci_softc_t *sc);
 102 static void ehci_dump_sqh(ehci_softc_t *sc, ehci_qh_t *sqh);
 103
 104 #endif
 105
 106 #define EHCI_INTR_ENDPT 1
 107
 108 extern struct usb_bus_methods ehci_bus_methods;
 109 extern struct usb_pipe_methods ehci_device_bulk_methods;
 110 extern struct usb_pipe_methods ehci_device_ctrl_methods;
 111 extern struct usb_pipe_methods ehci_device_intr_methods;
 112 extern struct usb_pipe_methods ehci_device_isoc_fs_methods;
 113 extern struct usb_pipe_methods ehci_device_isoc_hs_methods;
 114
 115 static void ehci_do_poll(struct usb_bus *bus);
 116 static void ehci_device_done(struct usb_xfer *xfer, usb_error_t error);
 117 static uint8_t ehci_check_transfer(struct usb_xfer *xfer);
 118 static void ehci_timeout(void *arg);
 119 static void ehci_root_intr(ehci_softc_t *sc);
 120
 121 struct ehci_std_temp {
 122         ehci_softc_t *sc;
 123         struct usb_page_cache *pc;
 124         ehci_qtd_t *td;
 125         ehci_qtd_t *td_next;
 126         uint32_t average;
 127         uint32_t qtd_status;
 128         uint32_t len;
 129         uint16_t max_frame_size;
 130         uint8_t shortpkt;
 131         uint8_t auto_data_toggle;
 132         uint8_t setup_alt_next;
 133         uint8_t last_frame;
 134 };
 135
 136 void
 137 ehci_iterate_hw_softc(struct usb_bus *bus, usb_bus_mem_sub_cb_t *cb)
 138 {
 139         ehci_softc_t *sc = EHCI_BUS2SC(bus);
 140         uint32_t i;
 141
 142         cb(bus, &sc->sc_hw.pframes_pc, &sc->sc_hw.pframes_pg,
 143             sizeof(uint32_t) * EHCI_FRAMELIST_COUNT, EHCI_FRAMELIST_ALIGN);
 144
 145         cb(bus, &sc->sc_hw.async_start_pc, &sc->sc_hw.async_start_pg,
 146             sizeof(ehci_qh_t), EHCI_QH_ALIGN);
 147
 148         for (i = 0; i != EHCI_VIRTUAL_FRAMELIST_COUNT; i++) {
 149                 cb(bus, sc->sc_hw.intr_start_pc + i,
 150                     sc->sc_hw.intr_start_pg + i,
 151                     sizeof(ehci_qh_t), EHCI_QH_ALIGN);
 152         }
 153
 154         for (i = 0; i != EHCI_VIRTUAL_FRAMELIST_COUNT; i++) {
 155                 cb(bus, sc->sc_hw.isoc_hs_start_pc + i,
 156                     sc->sc_hw.isoc_hs_start_pg + i,
 157                     sizeof(ehci_itd_t), EHCI_ITD_ALIGN);
 158         }
 159
 160         for (i = 0; i != EHCI_VIRTUAL_FRAMELIST_COUNT; i++) {
 161                 cb(bus, sc->sc_hw.isoc_fs_start_pc + i,
 162                     sc->sc_hw.isoc_fs_start_pg + i,
 163                     sizeof(ehci_sitd_t), EHCI_SITD_ALIGN);
 164         }
 165 }
 166
 167 usb_error_t
 168 ehci_reset(ehci_softc_t *sc)
 169 {
 170         uint32_t hcr;
 171         int i;
 172
 173         EOWRITE4(sc, EHCI_USBCMD, EHCI_CMD_HCRESET);
 174         for (i = 0; i < 100; i++) {
 175                 usb_pause_mtx(NULL, hz / 1000);
 176                 hcr = EOREAD4(sc, EHCI_USBCMD) & EHCI_CMD_HCRESET;
 177                 if (!hcr) {
 178                         if (sc->sc_flags & (EHCI_SCFLG_SETMODE | EHCI_SCFLG_BIGEMMIO)) {
 179                                 /*
 180                                  * Force USBMODE as requested.  Controllers
 181                                  * may have multiple operating modes.
 182                                  */
 183                                 uint32_t usbmode = EOREAD4(sc, EHCI_USBMODE);
 184                                 if (sc->sc_flags & EHCI_SCFLG_SETMODE) {
 185                                         usbmode = (usbmode &~ EHCI_UM_CM) | EHCI_UM_CM_HOST;
 186                                         device_printf(sc->sc_bus.bdev,
 187                                             "set host controller mode\n");
 188                                 }
 189                                 if (sc->sc_flags & EHCI_SCFLG_BIGEMMIO) {
 190                                         usbmode = (usbmode &~ EHCI_UM_ES) | EHCI_UM_ES_BE;
 191                                         device_printf(sc->sc_bus.bdev,
 192                                             "set big-endian mode\n");
 193                                 }
 194                                 EOWRITE4(sc,  EHCI_USBMODE, usbmode);
 195                         }
 196                         return (0);
 197                 }
 198         }
 199         device_printf(sc->sc_bus.bdev, "reset timeout\n");
 200         return (USB_ERR_IOERROR);
 201 }
 202
 203 static usb_error_t
 204 ehci_hcreset(ehci_softc_t *sc)
 205 {
 206         uint32_t hcr;
 207         int i;
 208
 209         EOWRITE4(sc, EHCI_USBCMD, 0);   /* Halt controller */
 210         for (i = 0; i < 100; i++) {
 211                 usb_pause_mtx(NULL, hz / 1000);
 212                 hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
 213                 if (hcr)
 214                         break;
 215         }
 216         if (!hcr)
 217                 /*
 218                  * Fall through and try reset anyway even though
 219                  * Table 2-9 in the EHCI spec says this will result
 220                  * in undefined behavior.
 221                  */
 222                 device_printf(sc->sc_bus.bdev, "stop timeout\n");
 223
 224         return ehci_reset(sc);
 225 }
 226
 227 usb_error_t
 228 ehci_init(ehci_softc_t *sc)
 229 {
 230         struct usb_page_search buf_res;
 231         uint32_t version;
 232         uint32_t sparams;
 233         uint32_t cparams;
 234         uint32_t hcr;
 235         uint16_t i;
 236         uint16_t x;
 237         uint16_t y;
 238         uint16_t bit;
 239         usb_error_t err = 0;
 240
 241         DPRINTF("start\n");
 242
 243         usb_callout_init_mtx(&sc->sc_tmo_pcd, &sc->sc_bus.bus_mtx, 0);
 244
 245 #if USB_DEBUG
 246         if (ehcidebug > 2) {
 247                 ehci_dump_regs(sc);
 248         }
 249 #endif
 250
 251         sc->sc_offs = EREAD1(sc, EHCI_CAPLENGTH);
 252
 253         version = EREAD2(sc, EHCI_HCIVERSION);
 254         device_printf(sc->sc_bus.bdev, "EHCI version %x.%x\n",
 255             version >> 8, version & 0xff);
 256
 257         sparams = EREAD4(sc, EHCI_HCSPARAMS);
 258         DPRINTF("sparams=0x%x\n", sparams);
 259
 260         sc->sc_noport = EHCI_HCS_N_PORTS(sparams);
 261         cparams = EREAD4(sc, EHCI_HCCPARAMS);
 262         DPRINTF("cparams=0x%x\n", cparams);
 263
 264         if (EHCI_HCC_64BIT(cparams)) {
 265                 DPRINTF("HCC uses 64-bit structures\n");
 266
 267                 /* MUST clear segment register if 64 bit capable */
 268                 EWRITE4(sc, EHCI_CTRLDSSEGMENT, 0);
 269         }
 270         sc->sc_bus.usbrev = USB_REV_2_0;
 271
 272         /* Reset the controller */
 273         DPRINTF("%s: resetting\n", device_get_nameunit(sc->sc_bus.bdev));
 274
 275         err = ehci_hcreset(sc);
 276         if (err) {
 277                 device_printf(sc->sc_bus.bdev, "reset timeout\n");
 278                 return (err);
 279         }
 280         /*
 281          * use current frame-list-size selection 0: 1024*4 bytes 1:  512*4
 282          * bytes 2:  256*4 bytes 3:      unknown
 283          */
 284         if (EHCI_CMD_FLS(EOREAD4(sc, EHCI_USBCMD)) == 3) {
 285                 device_printf(sc->sc_bus.bdev, "invalid frame-list-size\n");
 286                 return (USB_ERR_IOERROR);
 287         }
 288         /* set up the bus struct */
 289         sc->sc_bus.methods = &ehci_bus_methods;
 290
 291         sc->sc_eintrs = EHCI_NORMAL_INTRS;
 292
 293         for (i = 0; i < EHCI_VIRTUAL_FRAMELIST_COUNT; i++) {
 294                 ehci_qh_t *qh;
 295
 296                 usbd_get_page(sc->sc_hw.intr_start_pc + i, 0, &buf_res);
 297
 298                 qh = buf_res.buffer;
 299
 300                 /* initialize page cache pointer */
 301
 302                 qh->page_cache = sc->sc_hw.intr_start_pc + i;
 303
 304                 /* store a pointer to queue head */
 305
 306                 sc->sc_intr_p_last[i] = qh;
 307
 308                 qh->qh_self =
 309                     htohc32(sc, buf_res.physaddr) |
 310                     htohc32(sc, EHCI_LINK_QH);
 311
 312                 qh->qh_endp =
 313                     htohc32(sc, EHCI_QH_SET_EPS(EHCI_QH_SPEED_HIGH));
 314                 qh->qh_endphub =
 315                     htohc32(sc, EHCI_QH_SET_MULT(1));
 316                 qh->qh_curqtd = 0;
 317
 318                 qh->qh_qtd.qtd_next =
 319                     htohc32(sc, EHCI_LINK_TERMINATE);
 320                 qh->qh_qtd.qtd_altnext =
 321                     htohc32(sc, EHCI_LINK_TERMINATE);
 322                 qh->qh_qtd.qtd_status =
 323                     htohc32(sc, EHCI_QTD_HALTED);
 324         }
 325
 326         /*
 327          * the QHs are arranged to give poll intervals that are
 328          * powers of 2 times 1ms
 329          */
 330         bit = EHCI_VIRTUAL_FRAMELIST_COUNT / 2;
 331         while (bit) {
 332                 x = bit;
 333                 while (x & bit) {
 334                         ehci_qh_t *qh_x;
 335                         ehci_qh_t *qh_y;
 336
 337                         y = (x ^ bit) | (bit / 2);
 338
 339                         qh_x = sc->sc_intr_p_last[x];
 340                         qh_y = sc->sc_intr_p_last[y];
 341
 342                         /*
 343                          * the next QH has half the poll interval
 344                          */
 345                         qh_x->qh_link = qh_y->qh_self;
 346
 347                         x++;
 348                 }
 349                 bit >>= 1;
 350         }
 351
 352         if (1) {
 353                 ehci_qh_t *qh;
 354
 355                 qh = sc->sc_intr_p_last[0];
 356
 357                 /* the last (1ms) QH terminates */
 358                 qh->qh_link = htohc32(sc, EHCI_LINK_TERMINATE);
 359         }
 360         for (i = 0; i < EHCI_VIRTUAL_FRAMELIST_COUNT; i++) {
 361                 ehci_sitd_t *sitd;
 362                 ehci_itd_t *itd;
 363
 364                 usbd_get_page(sc->sc_hw.isoc_fs_start_pc + i, 0, &buf_res);
 365
 366                 sitd = buf_res.buffer;
 367
 368                 /* initialize page cache pointer */
 369
 370                 sitd->page_cache = sc->sc_hw.isoc_fs_start_pc + i;
 371
 372                 /* store a pointer to the transfer descriptor */
 373
 374                 sc->sc_isoc_fs_p_last[i] = sitd;
 375
 376                 /* initialize full speed isochronous */
 377
 378                 sitd->sitd_self =
 379                     htohc32(sc, buf_res.physaddr) |
 380                     htohc32(sc, EHCI_LINK_SITD);
 381
 382                 sitd->sitd_back =
 383                     htohc32(sc, EHCI_LINK_TERMINATE);
 384
 385                 sitd->sitd_next =
 386                     sc->sc_intr_p_last[i | (EHCI_VIRTUAL_FRAMELIST_COUNT / 2)]->qh_self;
 387
 388
 389                 usbd_get_page(sc->sc_hw.isoc_hs_start_pc + i, 0, &buf_res);
 390
 391                 itd = buf_res.buffer;
 392
 393                 /* initialize page cache pointer */
 394
 395                 itd->page_cache = sc->sc_hw.isoc_hs_start_pc + i;
 396
 397                 /* store a pointer to the transfer descriptor */
 398
 399                 sc->sc_isoc_hs_p_last[i] = itd;
 400
 401                 /* initialize high speed isochronous */
 402
 403                 itd->itd_self =
 404                     htohc32(sc, buf_res.physaddr) |
 405                     htohc32(sc, EHCI_LINK_ITD);
 406
 407                 itd->itd_next =
 408                     sitd->sitd_self;
 409         }
 410
 411         usbd_get_page(&sc->sc_hw.pframes_pc, 0, &buf_res);
 412
 413         if (1) {
 414                 uint32_t *pframes;
 415
 416                 pframes = buf_res.buffer;
 417
 418                 /*
 419                  * execution order:
 420                  * pframes -> high speed isochronous ->
 421                  *    full speed isochronous -> interrupt QH's
 422                  */
 423                 for (i = 0; i < EHCI_FRAMELIST_COUNT; i++) {
 424                         pframes[i] = sc->sc_isoc_hs_p_last
 425                             [i & (EHCI_VIRTUAL_FRAMELIST_COUNT - 1)]->itd_self;
 426                 }
 427         }
 428         /* setup sync list pointer */
 429         EOWRITE4(sc, EHCI_PERIODICLISTBASE, buf_res.physaddr);
 430
 431         usbd_get_page(&sc->sc_hw.async_start_pc, 0, &buf_res);
 432
 433         if (1) {
 434
 435                 ehci_qh_t *qh;
 436
 437                 qh = buf_res.buffer;
 438
 439                 /* initialize page cache pointer */
 440
 441                 qh->page_cache = &sc->sc_hw.async_start_pc;
 442
 443                 /* store a pointer to the queue head */
 444
 445                 sc->sc_async_p_last = qh;
 446
 447                 /* init dummy QH that starts the async list */
 448
 449                 qh->qh_self =
 450                     htohc32(sc, buf_res.physaddr) |
 451                     htohc32(sc, EHCI_LINK_QH);
 452
 453                 /* fill the QH */
 454                 qh->qh_endp =
 455                     htohc32(sc, EHCI_QH_SET_EPS(EHCI_QH_SPEED_HIGH) | EHCI_QH_HRECL);
 456                 qh->qh_endphub = htohc32(sc, EHCI_QH_SET_MULT(1));
 457                 qh->qh_link = qh->qh_self;
 458                 qh->qh_curqtd = 0;
 459
 460                 /* fill the overlay qTD */
 461                 qh->qh_qtd.qtd_next = htohc32(sc, EHCI_LINK_TERMINATE);
 462                 qh->qh_qtd.qtd_altnext = htohc32(sc, EHCI_LINK_TERMINATE);
 463                 qh->qh_qtd.qtd_status = htohc32(sc, EHCI_QTD_HALTED);
 464         }
 465         /* flush all cache into memory */
 466
 467         usb_bus_mem_flush_all(&sc->sc_bus, &ehci_iterate_hw_softc);
 468
 469 #if USB_DEBUG
 470         if (ehcidebug) {
 471                 ehci_dump_sqh(sc, sc->sc_async_p_last);
 472         }
 473 #endif
 474
 475         /* setup async list pointer */
 476         EOWRITE4(sc, EHCI_ASYNCLISTADDR, buf_res.physaddr | EHCI_LINK_QH);
 477
 478
 479         /* enable interrupts */
 480         EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
 481
 482         /* turn on controller */
 483         EOWRITE4(sc, EHCI_USBCMD,
 484             EHCI_CMD_ITC_1 |            /* 1 microframes interrupt delay */
 485             (EOREAD4(sc, EHCI_USBCMD) & EHCI_CMD_FLS_M) |
 486             EHCI_CMD_ASE |
 487             EHCI_CMD_PSE |
 488             EHCI_CMD_RS);
 489
 490         /* Take over port ownership */
 491         EOWRITE4(sc, EHCI_CONFIGFLAG, EHCI_CONF_CF);
 492
 493         for (i = 0; i < 100; i++) {
 494                 usb_pause_mtx(NULL, hz / 1000);
 495                 hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
 496                 if (!hcr) {
 497                         break;
 498                 }
 499         }
 500         if (hcr) {
 501                 device_printf(sc->sc_bus.bdev, "run timeout\n");
 502                 return (USB_ERR_IOERROR);
 503         }
 504
 505         if (!err) {
 506                 /* catch any lost interrupts */
 507                 ehci_do_poll(&sc->sc_bus);
 508         }
 509         return (err);
 510 }
 511
 512 /*
 513  * shut down the controller when the system is going down
 514  */
 515 void
 516 ehci_detach(ehci_softc_t *sc)
 517 {
 518         USB_BUS_LOCK(&sc->sc_bus);
 519
 520         usb_callout_stop(&sc->sc_tmo_pcd);
 521
 522         EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
 523         USB_BUS_UNLOCK(&sc->sc_bus);
 524
 525         if (ehci_hcreset(sc)) {
 526                 DPRINTF("reset failed!\n");
 527         }
 528
 529         /* XXX let stray task complete */
 530         usb_pause_mtx(NULL, hz / 20);
 531
 532         usb_callout_drain(&sc->sc_tmo_pcd);
 533 }
 534
 535 void
 536 ehci_suspend(ehci_softc_t *sc)
 537 {
 538         uint32_t cmd;
 539         uint32_t hcr;
 540         uint8_t i;
 541
 542         USB_BUS_LOCK(&sc->sc_bus);
 543
 544         for (i = 1; i <= sc->sc_noport; i++) {
 545                 cmd = EOREAD4(sc, EHCI_PORTSC(i));
 546                 if (((cmd & EHCI_PS_PO) == 0) &&
 547                     ((cmd & EHCI_PS_PE) == EHCI_PS_PE)) {
 548                         EOWRITE4(sc, EHCI_PORTSC(i),
 549                             cmd | EHCI_PS_SUSP);
 550                 }
 551         }
 552
 553         sc->sc_cmd = EOREAD4(sc, EHCI_USBCMD);
 554
 555         cmd = sc->sc_cmd & ~(EHCI_CMD_ASE | EHCI_CMD_PSE);
 556         EOWRITE4(sc, EHCI_USBCMD, cmd);
 557
 558         for (i = 0; i < 100; i++) {
 559                 hcr = EOREAD4(sc, EHCI_USBSTS) &
 560                     (EHCI_STS_ASS | EHCI_STS_PSS);
 561
 562                 if (hcr == 0) {
 563                         break;
 564                 }
 565                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 1000);
 566         }
 567
 568         if (hcr != 0) {
 569                 device_printf(sc->sc_bus.bdev, "reset timeout\n");
 570         }
 571         cmd &= ~EHCI_CMD_RS;
 572         EOWRITE4(sc, EHCI_USBCMD, cmd);
 573
 574         for (i = 0; i < 100; i++) {
 575                 hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
 576                 if (hcr == EHCI_STS_HCH) {
 577                         break;
 578                 }
 579                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 1000);
 580         }
 581
 582         if (hcr != EHCI_STS_HCH) {
 583                 device_printf(sc->sc_bus.bdev,
 584                     "config timeout\n");
 585         }
 586         USB_BUS_UNLOCK(&sc->sc_bus);
 587 }
 588
 589 void
 590 ehci_resume(ehci_softc_t *sc)
 591 {
 592         struct usb_page_search buf_res;
 593         uint32_t cmd;
 594         uint32_t hcr;
 595         uint8_t i;
 596
 597         USB_BUS_LOCK(&sc->sc_bus);
 598
 599         /* restore things in case the bios doesn't */
 600         EOWRITE4(sc, EHCI_CTRLDSSEGMENT, 0);
 601
 602         usbd_get_page(&sc->sc_hw.pframes_pc, 0, &buf_res);
 603         EOWRITE4(sc, EHCI_PERIODICLISTBASE, buf_res.physaddr);
 604
 605         usbd_get_page(&sc->sc_hw.async_start_pc, 0, &buf_res);
 606         EOWRITE4(sc, EHCI_ASYNCLISTADDR, buf_res.physaddr | EHCI_LINK_QH);
 607
 608         EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
 609
 610         hcr = 0;
 611         for (i = 1; i <= sc->sc_noport; i++) {
 612                 cmd = EOREAD4(sc, EHCI_PORTSC(i));
 613                 if (((cmd & EHCI_PS_PO) == 0) &&
 614                     ((cmd & EHCI_PS_SUSP) == EHCI_PS_SUSP)) {
 615                         EOWRITE4(sc, EHCI_PORTSC(i),
 616                             cmd | EHCI_PS_FPR);
 617                         hcr = 1;
 618                 }
 619         }
 620
 621         if (hcr) {
 622                 usb_pause_mtx(&sc->sc_bus.bus_mtx,
 623                     USB_MS_TO_TICKS(USB_RESUME_WAIT));
 624
 625                 for (i = 1; i <= sc->sc_noport; i++) {
 626                         cmd = EOREAD4(sc, EHCI_PORTSC(i));
 627                         if (((cmd & EHCI_PS_PO) == 0) &&
 628                             ((cmd & EHCI_PS_SUSP) == EHCI_PS_SUSP)) {
 629                                 EOWRITE4(sc, EHCI_PORTSC(i),
 630                                     cmd & ~EHCI_PS_FPR);
 631                         }
 632                 }
 633         }
 634         EOWRITE4(sc, EHCI_USBCMD, sc->sc_cmd);
 635
 636         for (i = 0; i < 100; i++) {
 637                 hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
 638                 if (hcr != EHCI_STS_HCH) {
 639                         break;
 640                 }
 641                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 1000);
 642         }
 643         if (hcr == EHCI_STS_HCH) {
 644                 device_printf(sc->sc_bus.bdev, "config timeout\n");
 645         }
 646
 647         USB_BUS_UNLOCK(&sc->sc_bus);
 648
 649         usb_pause_mtx(NULL,
 650             USB_MS_TO_TICKS(USB_RESUME_WAIT));
 651
 652         /* catch any lost interrupts */
 653         ehci_do_poll(&sc->sc_bus);
 654 }
 655
 656 void
 657 ehci_shutdown(ehci_softc_t *sc)
 658 {
 659         DPRINTF("stopping the HC\n");
 660
 661         if (ehci_hcreset(sc)) {
 662                 DPRINTF("reset failed!\n");
 663         }
 664 }
 665
 666 #if USB_DEBUG
 667 static void
 668 ehci_dump_regs(ehci_softc_t *sc)
 669 {
 670         uint32_t i;
 671
 672         i = EOREAD4(sc, EHCI_USBCMD);
 673         printf("cmd=0x%08x\n", i);
 674
 675         if (i & EHCI_CMD_ITC_1)
 676                 printf(" EHCI_CMD_ITC_1\n");
 677         if (i & EHCI_CMD_ITC_2)
 678                 printf(" EHCI_CMD_ITC_2\n");
 679         if (i & EHCI_CMD_ITC_4)
 680                 printf(" EHCI_CMD_ITC_4\n");
 681         if (i & EHCI_CMD_ITC_8)
 682                 printf(" EHCI_CMD_ITC_8\n");
 683         if (i & EHCI_CMD_ITC_16)
 684                 printf(" EHCI_CMD_ITC_16\n");
 685         if (i & EHCI_CMD_ITC_32)
 686                 printf(" EHCI_CMD_ITC_32\n");
 687         if (i & EHCI_CMD_ITC_64)
 688                 printf(" EHCI_CMD_ITC_64\n");
 689         if (i & EHCI_CMD_ASPME)
 690                 printf(" EHCI_CMD_ASPME\n");
 691         if (i & EHCI_CMD_ASPMC)
 692                 printf(" EHCI_CMD_ASPMC\n");
 693         if (i & EHCI_CMD_LHCR)
 694                 printf(" EHCI_CMD_LHCR\n");
 695         if (i & EHCI_CMD_IAAD)
 696                 printf(" EHCI_CMD_IAAD\n");
 697         if (i & EHCI_CMD_ASE)
 698                 printf(" EHCI_CMD_ASE\n");
 699         if (i & EHCI_CMD_PSE)
 700                 printf(" EHCI_CMD_PSE\n");
 701         if (i & EHCI_CMD_FLS_M)
 702                 printf(" EHCI_CMD_FLS_M\n");
 703         if (i & EHCI_CMD_HCRESET)
 704                 printf(" EHCI_CMD_HCRESET\n");
 705         if (i & EHCI_CMD_RS)
 706                 printf(" EHCI_CMD_RS\n");
 707
 708         i = EOREAD4(sc, EHCI_USBSTS);
 709
 710         printf("sts=0x%08x\n", i);
 711
 712         if (i & EHCI_STS_ASS)
 713                 printf(" EHCI_STS_ASS\n");
 714         if (i & EHCI_STS_PSS)
 715                 printf(" EHCI_STS_PSS\n");
 716         if (i & EHCI_STS_REC)
 717                 printf(" EHCI_STS_REC\n");
 718         if (i & EHCI_STS_HCH)
 719                 printf(" EHCI_STS_HCH\n");
 720         if (i & EHCI_STS_IAA)
 721                 printf(" EHCI_STS_IAA\n");
 722         if (i & EHCI_STS_HSE)
 723                 printf(" EHCI_STS_HSE\n");
 724         if (i & EHCI_STS_FLR)
 725                 printf(" EHCI_STS_FLR\n");
 726         if (i & EHCI_STS_PCD)
 727                 printf(" EHCI_STS_PCD\n");
 728         if (i & EHCI_STS_ERRINT)
 729                 printf(" EHCI_STS_ERRINT\n");
 730         if (i & EHCI_STS_INT)
 731                 printf(" EHCI_STS_INT\n");
 732
 733         printf("ien=0x%08x\n",
 734             EOREAD4(sc, EHCI_USBINTR));
 735         printf("frindex=0x%08x ctrdsegm=0x%08x periodic=0x%08x async=0x%08x\n",
 736             EOREAD4(sc, EHCI_FRINDEX),
 737             EOREAD4(sc, EHCI_CTRLDSSEGMENT),
 738             EOREAD4(sc, EHCI_PERIODICLISTBASE),
 739             EOREAD4(sc, EHCI_ASYNCLISTADDR));
 740         for (i = 1; i <= sc->sc_noport; i++) {
 741                 printf("port %d status=0x%08x\n", i,
 742                     EOREAD4(sc, EHCI_PORTSC(i)));
 743         }
 744 }
 745
 746 static void
 747 ehci_dump_link(ehci_softc_t *sc, uint32_t link, int type)
 748 {
 749         link = hc32toh(sc, link);
 750         printf("0x%08x", link);
 751         if (link & EHCI_LINK_TERMINATE)
 752                 printf("<T>");
 753         else {
 754                 printf("<");
 755                 if (type) {
 756                         switch (EHCI_LINK_TYPE(link)) {
 757                         case EHCI_LINK_ITD:
 758                                 printf("ITD");
 759                                 break;
 760                         case EHCI_LINK_QH:
 761                                 printf("QH");
 762                                 break;
 763                         case EHCI_LINK_SITD:
 764                                 printf("SITD");
 765                                 break;
 766                         case EHCI_LINK_FSTN:
 767                                 printf("FSTN");
 768                                 break;
 769                         }
 770                 }
 771                 printf(">");
 772         }
 773 }
 774
 775 static void
 776 ehci_dump_qtd(ehci_softc_t *sc, ehci_qtd_t *qtd)
 777 {
 778         uint32_t s;
 779
 780         printf("  next=");
 781         ehci_dump_link(sc, qtd->qtd_next, 0);
 782         printf(" altnext=");
 783         ehci_dump_link(sc, qtd->qtd_altnext, 0);
 784         printf("\n");
 785         s = hc32toh(sc, qtd->qtd_status);
 786         printf("  status=0x%08x: toggle=%d bytes=0x%x ioc=%d c_page=0x%x\n",
 787             s, EHCI_QTD_GET_TOGGLE(s), EHCI_QTD_GET_BYTES(s),
 788             EHCI_QTD_GET_IOC(s), EHCI_QTD_GET_C_PAGE(s));
 789         printf("    cerr=%d pid=%d stat=%s%s%s%s%s%s%s%s\n",
 790             EHCI_QTD_GET_CERR(s), EHCI_QTD_GET_PID(s),
 791             (s & EHCI_QTD_ACTIVE) ? "ACTIVE" : "NOT_ACTIVE",
 792             (s & EHCI_QTD_HALTED) ? "-HALTED" : "",
 793             (s & EHCI_QTD_BUFERR) ? "-BUFERR" : "",
 794             (s & EHCI_QTD_BABBLE) ? "-BABBLE" : "",
 795             (s & EHCI_QTD_XACTERR) ? "-XACTERR" : "",
 796             (s & EHCI_QTD_MISSEDMICRO) ? "-MISSED" : "",
 797             (s & EHCI_QTD_SPLITXSTATE) ? "-SPLIT" : "",
 798             (s & EHCI_QTD_PINGSTATE) ? "-PING" : "");
 799
 800         for (s = 0; s < 5; s++) {
 801                 printf("  buffer[%d]=0x%08x\n", s,
 802                     hc32toh(sc, qtd->qtd_buffer[s]));
 803         }
 804         for (s = 0; s < 5; s++) {
 805                 printf("  buffer_hi[%d]=0x%08x\n", s,
 806                     hc32toh(sc, qtd->qtd_buffer_hi[s]));
 807         }
 808 }
 809
 810 static uint8_t
 811 ehci_dump_sqtd(ehci_softc_t *sc, ehci_qtd_t *sqtd)
 812 {
 813         uint8_t temp;
 814
 815         usb_pc_cpu_invalidate(sqtd->page_cache);
 816         printf("QTD(%p) at 0x%08x:\n", sqtd, hc32toh(sc, sqtd->qtd_self));
 817         ehci_dump_qtd(sc, sqtd);
 818         temp = (sqtd->qtd_next & htohc32(sc, EHCI_LINK_TERMINATE)) ? 1 : 0;
 819         return (temp);
 820 }
 821
 822 static void
 823 ehci_dump_sqtds(ehci_softc_t *sc, ehci_qtd_t *sqtd)
 824 {
 825         uint16_t i;
 826         uint8_t stop;
 827
 828         stop = 0;
 829         for (i = 0; sqtd && (i < 20) && !stop; sqtd = sqtd->obj_next, i++) {
 830                 stop = ehci_dump_sqtd(sc, sqtd);
 831         }
 832         if (sqtd) {
 833                 printf("dump aborted, too many TDs\n");
 834         }
 835 }
 836
 837 static void
 838 ehci_dump_sqh(ehci_softc_t *sc, ehci_qh_t *qh)
 839 {
 840         uint32_t endp;
 841         uint32_t endphub;
 842
 843         usb_pc_cpu_invalidate(qh->page_cache);
 844         printf("QH(%p) at 0x%08x:\n", qh, hc32toh(sc, qh->qh_self) & ~0x1F);
 845         printf("  link=");
 846         ehci_dump_link(sc, qh->qh_link, 1);
 847         printf("\n");
 848         endp = hc32toh(sc, qh->qh_endp);
 849         printf("  endp=0x%08x\n", endp);
 850         printf("    addr=0x%02x inact=%d endpt=%d eps=%d dtc=%d hrecl=%d\n",
 851             EHCI_QH_GET_ADDR(endp), EHCI_QH_GET_INACT(endp),
 852             EHCI_QH_GET_ENDPT(endp), EHCI_QH_GET_EPS(endp),
 853             EHCI_QH_GET_DTC(endp), EHCI_QH_GET_HRECL(endp));
 854         printf("    mpl=0x%x ctl=%d nrl=%d\n",
 855             EHCI_QH_GET_MPL(endp), EHCI_QH_GET_CTL(endp),
 856             EHCI_QH_GET_NRL(endp));
 857         endphub = hc32toh(sc, qh->qh_endphub);
 858         printf("  endphub=0x%08x\n", endphub);
 859         printf("    smask=0x%02x cmask=0x%02x huba=0x%02x port=%d mult=%d\n",
 860             EHCI_QH_GET_SMASK(endphub), EHCI_QH_GET_CMASK(endphub),
 861             EHCI_QH_GET_HUBA(endphub), EHCI_QH_GET_PORT(endphub),
 862             EHCI_QH_GET_MULT(endphub));
 863         printf("  curqtd=");
 864         ehci_dump_link(sc, qh->qh_curqtd, 0);
 865         printf("\n");
 866         printf("Overlay qTD:\n");
 867         ehci_dump_qtd(sc, (void *)&qh->qh_qtd);
 868 }
 869
 870 static void
 871 ehci_dump_sitd(ehci_softc_t *sc, ehci_sitd_t *sitd)
 872 {
 873         usb_pc_cpu_invalidate(sitd->page_cache);
 874         printf("SITD(%p) at 0x%08x\n", sitd, hc32toh(sc, sitd->sitd_self) & ~0x1F);
 875         printf(" next=0x%08x\n", hc32toh(sc, sitd->sitd_next));
 876         printf(" portaddr=0x%08x dir=%s addr=%d endpt=0x%x port=0x%x huba=0x%x\n",
 877             hc32toh(sc, sitd->sitd_portaddr),
 878             (sitd->sitd_portaddr & htohc32(sc, EHCI_SITD_SET_DIR_IN))
 879             ? "in" : "out",
 880             EHCI_SITD_GET_ADDR(hc32toh(sc, sitd->sitd_portaddr)),
 881             EHCI_SITD_GET_ENDPT(hc32toh(sc, sitd->sitd_portaddr)),
 882             EHCI_SITD_GET_PORT(hc32toh(sc, sitd->sitd_portaddr)),
 883             EHCI_SITD_GET_HUBA(hc32toh(sc, sitd->sitd_portaddr)));
 884         printf(" mask=0x%08x\n", hc32toh(sc, sitd->sitd_mask));
 885         printf(" status=0x%08x <%s> len=0x%x\n", hc32toh(sc, sitd->sitd_status),
 886             (sitd->sitd_status & htohc32(sc, EHCI_SITD_ACTIVE)) ? "ACTIVE" : "",
 887             EHCI_SITD_GET_LEN(hc32toh(sc, sitd->sitd_status)));
 888         printf(" back=0x%08x, bp=0x%08x,0x%08x,0x%08x,0x%08x\n",
 889             hc32toh(sc, sitd->sitd_back),
 890             hc32toh(sc, sitd->sitd_bp[0]),
 891             hc32toh(sc, sitd->sitd_bp[1]),
 892             hc32toh(sc, sitd->sitd_bp_hi[0]),
 893             hc32toh(sc, sitd->sitd_bp_hi[1]));
 894 }
 895
 896 static void
 897 ehci_dump_itd(ehci_softc_t *sc, ehci_itd_t *itd)
 898 {
 899         usb_pc_cpu_invalidate(itd->page_cache);
 900         printf("ITD(%p) at 0x%08x\n", itd, hc32toh(sc, itd->itd_self) & ~0x1F);
 901         printf(" next=0x%08x\n", hc32toh(sc, itd->itd_next));
 902         printf(" status[0]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[0]),
 903             (itd->itd_status[0] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
 904         printf(" status[1]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[1]),
 905             (itd->itd_status[1] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
 906         printf(" status[2]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[2]),
 907             (itd->itd_status[2] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
 908         printf(" status[3]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[3]),
 909             (itd->itd_status[3] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
 910         printf(" status[4]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[4]),
 911             (itd->itd_status[4] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
 912         printf(" status[5]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[5]),
 913             (itd->itd_status[5] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
 914         printf(" status[6]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[6]),
 915             (itd->itd_status[6] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
 916         printf(" status[7]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[7]),
 917             (itd->itd_status[7] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
 918         printf(" bp[0]=0x%08x\n", hc32toh(sc, itd->itd_bp[0]));
 919         printf("  addr=0x%02x; endpt=0x%01x\n",
 920             EHCI_ITD_GET_ADDR(hc32toh(sc, itd->itd_bp[0])),
 921             EHCI_ITD_GET_ENDPT(hc32toh(sc, itd->itd_bp[0])));
 922         printf(" bp[1]=0x%08x\n", hc32toh(sc, itd->itd_bp[1]));
 923         printf(" dir=%s; mpl=0x%02x\n",
 924             (hc32toh(sc, itd->itd_bp[1]) & EHCI_ITD_SET_DIR_IN) ? "in" : "out",
 925             EHCI_ITD_GET_MPL(hc32toh(sc, itd->itd_bp[1])));
 926         printf(" bp[2..6]=0x%08x,0x%08x,0x%08x,0x%08x,0x%08x\n",
 927             hc32toh(sc, itd->itd_bp[2]),
 928             hc32toh(sc, itd->itd_bp[3]),
 929             hc32toh(sc, itd->itd_bp[4]),
 930             hc32toh(sc, itd->itd_bp[5]),
 931             hc32toh(sc, itd->itd_bp[6]));
 932         printf(" bp_hi=0x%08x,0x%08x,0x%08x,0x%08x,\n"
 933             "       0x%08x,0x%08x,0x%08x\n",
 934             hc32toh(sc, itd->itd_bp_hi[0]),
 935             hc32toh(sc, itd->itd_bp_hi[1]),
 936             hc32toh(sc, itd->itd_bp_hi[2]),
 937             hc32toh(sc, itd->itd_bp_hi[3]),
 938             hc32toh(sc, itd->itd_bp_hi[4]),
 939             hc32toh(sc, itd->itd_bp_hi[5]),
 940             hc32toh(sc, itd->itd_bp_hi[6]));
 941 }
 942
 943 static void
 944 ehci_dump_isoc(ehci_softc_t *sc)
 945 {
 946         ehci_itd_t *itd;
 947         ehci_sitd_t *sitd;
 948         uint16_t max = 1000;
 949         uint16_t pos;
 950
 951         pos = (EOREAD4(sc, EHCI_FRINDEX) / 8) &
 952             (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
 953
 954         printf("%s: isochronous dump from frame 0x%03x:\n",
 955             __FUNCTION__, pos);
 956
 957         itd = sc->sc_isoc_hs_p_last[pos];
 958         sitd = sc->sc_isoc_fs_p_last[pos];
 959
 960         while (itd && max && max--) {
 961                 ehci_dump_itd(sc, itd);
 962                 itd = itd->prev;
 963         }
 964
 965         while (sitd && max && max--) {
 966                 ehci_dump_sitd(sc, sitd);
 967                 sitd = sitd->prev;
 968         }
 969 }
 970
 971 #endif
 972
 973 static void
 974 ehci_transfer_intr_enqueue(struct usb_xfer *xfer)
 975 {
 976         /* check for early completion */
 977         if (ehci_check_transfer(xfer)) {
 978                 return;
 979         }
 980         /* put transfer on interrupt queue */
 981         usbd_transfer_enqueue(&xfer->xroot->bus->intr_q, xfer);
 982
 983         /* start timeout, if any */
 984         if (xfer->timeout != 0) {
 985                 usbd_transfer_timeout_ms(xfer, &ehci_timeout, xfer->timeout);
 986         }
 987 }
 988
 989 #define EHCI_APPEND_FS_TD(std,last) (last) = _ehci_append_fs_td(std,last)
 990 static ehci_sitd_t *
 991 _ehci_append_fs_td(ehci_sitd_t *std, ehci_sitd_t *last)
 992 {
 993         DPRINTFN(11, "%p to %p\n", std, last);
 994
 995         /* (sc->sc_bus.mtx) must be locked */
 996
 997         std->next = last->next;
 998         std->sitd_next = last->sitd_next;
 999
1000         std->prev = last;
1001
1002         usb_pc_cpu_flush(std->page_cache);
1003
1004         /*
1005          * the last->next->prev is never followed: std->next->prev = std;
1006          */
1007         last->next = std;
1008         last->sitd_next = std->sitd_self;
1009
1010         usb_pc_cpu_flush(last->page_cache);
1011
1012         return (std);
1013 }
1014
1015 #define EHCI_APPEND_HS_TD(std,last) (last) = _ehci_append_hs_td(std,last)
1016 static ehci_itd_t *
1017 _ehci_append_hs_td(ehci_itd_t *std, ehci_itd_t *last)
1018 {
1019         DPRINTFN(11, "%p to %p\n", std, last);
1020
1021         /* (sc->sc_bus.mtx) must be locked */
1022
1023         std->next = last->next;
1024         std->itd_next = last->itd_next;
1025
1026         std->prev = last;
1027
1028         usb_pc_cpu_flush(std->page_cache);
1029
1030         /*
1031          * the last->next->prev is never followed: std->next->prev = std;
1032          */
1033         last->next = std;
1034         last->itd_next = std->itd_self;
1035
1036         usb_pc_cpu_flush(last->page_cache);
1037
1038         return (std);
1039 }
1040
1041 #define EHCI_APPEND_QH(sqh,last) (last) = _ehci_append_qh(sqh,last)
1042 static ehci_qh_t *
1043 _ehci_append_qh(ehci_qh_t *sqh, ehci_qh_t *last)
1044 {
1045         DPRINTFN(11, "%p to %p\n", sqh, last);
1046
1047         if (sqh->prev != NULL) {
1048                 /* should not happen */
1049                 DPRINTFN(0, "QH already linked!\n");
1050                 return (last);
1051         }
1052         /* (sc->sc_bus.mtx) must be locked */
1053
1054         sqh->next = last->next;
1055         sqh->qh_link = last->qh_link;
1056
1057         sqh->prev = last;
1058
1059         usb_pc_cpu_flush(sqh->page_cache);
1060
1061         /*
1062          * the last->next->prev is never followed: sqh->next->prev = sqh;
1063          */
1064
1065         last->next = sqh;
1066         last->qh_link = sqh->qh_self;
1067
1068         usb_pc_cpu_flush(last->page_cache);
1069
1070         return (sqh);
1071 }
1072
1073 #define EHCI_REMOVE_FS_TD(std,last) (last) = _ehci_remove_fs_td(std,last)
1074 static ehci_sitd_t *
1075 _ehci_remove_fs_td(ehci_sitd_t *std, ehci_sitd_t *last)
1076 {
1077         DPRINTFN(11, "%p from %p\n", std, last);
1078
1079         /* (sc->sc_bus.mtx) must be locked */
1080
1081         std->prev->next = std->next;
1082         std->prev->sitd_next = std->sitd_next;
1083
1084         usb_pc_cpu_flush(std->prev->page_cache);
1085
1086         if (std->next) {
1087                 std->next->prev = std->prev;
1088                 usb_pc_cpu_flush(std->next->page_cache);
1089         }
1090         return ((last == std) ? std->prev : last);
1091 }
1092
1093 #define EHCI_REMOVE_HS_TD(std,last) (last) = _ehci_remove_hs_td(std,last)
1094 static ehci_itd_t *
1095 _ehci_remove_hs_td(ehci_itd_t *std, ehci_itd_t *last)
1096 {
1097         DPRINTFN(11, "%p from %p\n", std, last);
1098
1099         /* (sc->sc_bus.mtx) must be locked */
1100
1101         std->prev->next = std->next;
1102         std->prev->itd_next = std->itd_next;
1103
1104         usb_pc_cpu_flush(std->prev->page_cache);
1105
1106         if (std->next) {
1107                 std->next->prev = std->prev;
1108                 usb_pc_cpu_flush(std->next->page_cache);
1109         }
1110         return ((last == std) ? std->prev : last);
1111 }
1112
1113 #define EHCI_REMOVE_QH(sqh,last) (last) = _ehci_remove_qh(sqh,last)
1114 static ehci_qh_t *
1115 _ehci_remove_qh(ehci_qh_t *sqh, ehci_qh_t *last)
1116 {
1117         DPRINTFN(11, "%p from %p\n", sqh, last);
1118
1119         /* (sc->sc_bus.mtx) must be locked */
1120
1121         /* only remove if not removed from a queue */
1122         if (sqh->prev) {
1123
1124                 sqh->prev->next = sqh->next;
1125                 sqh->prev->qh_link = sqh->qh_link;
1126
1127                 usb_pc_cpu_flush(sqh->prev->page_cache);
1128
1129                 if (sqh->next) {
1130                         sqh->next->prev = sqh->prev;
1131                         usb_pc_cpu_flush(sqh->next->page_cache);
1132                 }
1133                 last = ((last == sqh) ? sqh->prev : last);
1134
1135                 sqh->prev = 0;
1136
1137                 usb_pc_cpu_flush(sqh->page_cache);
1138         }
1139         return (last);
1140 }
1141
1142 static usb_error_t
1143 ehci_non_isoc_done_sub(struct usb_xfer *xfer)
1144 {
1145         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
1146         ehci_qtd_t *td;
1147         ehci_qtd_t *td_alt_next;
1148         uint32_t status;
1149         uint16_t len;
1150
1151         td = xfer->td_transfer_cache;
1152         td_alt_next = td->alt_next;
1153
1154         if (xfer->aframes != xfer->nframes) {
1155                 usbd_xfer_set_frame_len(xfer, xfer->aframes, 0);
1156         }
1157         while (1) {
1158
1159                 usb_pc_cpu_invalidate(td->page_cache);
1160                 status = hc32toh(sc, td->qtd_status);
1161
1162                 len = EHCI_QTD_GET_BYTES(status);
1163
1164                 /*
1165                  * Verify the status length and
1166                  * add the length to "frlengths[]":
1167                  */
1168                 if (len > td->len) {
1169                         /* should not happen */
1170                         DPRINTF("Invalid status length, "
1171                             "0x%04x/0x%04x bytes\n", len, td->len);
1172                         status |= EHCI_QTD_HALTED;
1173                 } else if (xfer->aframes != xfer->nframes) {
1174                         xfer->frlengths[xfer->aframes] += td->len - len;
1175                 }
1176                 /* Check for last transfer */
1177                 if (((void *)td) == xfer->td_transfer_last) {
1178                         td = NULL;
1179                         break;
1180                 }
1181                 /* Check for transfer error */
1182                 if (status & EHCI_QTD_HALTED) {
1183                         /* the transfer is finished */
1184                         td = NULL;
1185                         break;
1186                 }
1187                 /* Check for short transfer */
1188                 if (len > 0) {
1189                         if (xfer->flags_int.short_frames_ok) {
1190                                 /* follow alt next */
1191                                 td = td->alt_next;
1192                         } else {
1193                                 /* the transfer is finished */
1194                                 td = NULL;
1195                         }
1196                         break;
1197                 }
1198                 td = td->obj_next;
1199
1200                 if (td->alt_next != td_alt_next) {
1201                         /* this USB frame is complete */
1202                         break;
1203                 }
1204         }
1205
1206         /* update transfer cache */
1207
1208         xfer->td_transfer_cache = td;
1209
1210         /* update data toggle */
1211
1212         xfer->endpoint->toggle_next =
1213             (status & EHCI_QTD_TOGGLE_MASK) ? 1 : 0;
1214
1215 #if USB_DEBUG
1216         if (status & EHCI_QTD_STATERRS) {
1217                 DPRINTFN(11, "error, addr=%d, endpt=0x%02x, frame=0x%02x"
1218                     "status=%s%s%s%s%s%s%s%s\n",
1219                     xfer->address, xfer->endpointno, xfer->aframes,
1220                     (status & EHCI_QTD_ACTIVE) ? "[ACTIVE]" : "[NOT_ACTIVE]",
1221                     (status & EHCI_QTD_HALTED) ? "[HALTED]" : "",
1222                     (status & EHCI_QTD_BUFERR) ? "[BUFERR]" : "",
1223                     (status & EHCI_QTD_BABBLE) ? "[BABBLE]" : "",
1224                     (status & EHCI_QTD_XACTERR) ? "[XACTERR]" : "",
1225                     (status & EHCI_QTD_MISSEDMICRO) ? "[MISSED]" : "",
1226                     (status & EHCI_QTD_SPLITXSTATE) ? "[SPLIT]" : "",
1227                     (status & EHCI_QTD_PINGSTATE) ? "[PING]" : "");
1228         }
1229 #endif
1230
1231         return ((status & EHCI_QTD_HALTED) ?
1232             USB_ERR_STALLED : USB_ERR_NORMAL_COMPLETION);
1233 }
1234
1235 static void
1236 ehci_non_isoc_done(struct usb_xfer *xfer)
1237 {
1238         usb_error_t err = 0;
1239
1240         DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
1241             xfer, xfer->endpoint);
1242
1243 #if USB_DEBUG
1244         if (ehcidebug > 10) {
1245                 ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
1246
1247                 ehci_dump_sqtds(sc, xfer->td_transfer_first);
1248         }
1249 #endif
1250
1251         /* reset scanner */
1252
1253         xfer->td_transfer_cache = xfer->td_transfer_first;
1254
1255         if (xfer->flags_int.control_xfr) {
1256
1257                 if (xfer->flags_int.control_hdr) {
1258
1259                         err = ehci_non_isoc_done_sub(xfer);
1260                 }
1261                 xfer->aframes = 1;
1262
1263                 if (xfer->td_transfer_cache == NULL) {
1264                         goto done;
1265                 }
1266         }
1267         while (xfer->aframes != xfer->nframes) {
1268
1269                 err = ehci_non_isoc_done_sub(xfer);
1270                 xfer->aframes++;
1271
1272                 if (xfer->td_transfer_cache == NULL) {
1273                         goto done;
1274                 }
1275         }
1276
1277         if (xfer->flags_int.control_xfr &&
1278             !xfer->flags_int.control_act) {
1279
1280                 err = ehci_non_isoc_done_sub(xfer);
1281         }
1282 done:
1283         ehci_device_done(xfer, err);
1284 }
1285
1286 /*------------------------------------------------------------------------*
1287  *      ehci_check_transfer
1288  *
1289  * Return values:
1290  *    0: USB transfer is not finished
1291  * Else: USB transfer is finished
1292  *------------------------------------------------------------------------*/
1293 static uint8_t
1294 ehci_check_transfer(struct usb_xfer *xfer)
1295 {
1296         struct usb_pipe_methods *methods = xfer->endpoint->methods;
1297         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
1298
1299         uint32_t status;
1300
1301         DPRINTFN(13, "xfer=%p checking transfer\n", xfer);
1302
1303         if (methods == &ehci_device_isoc_fs_methods) {
1304                 ehci_sitd_t *td;
1305
1306                 /* isochronous full speed transfer */
1307
1308                 td = xfer->td_transfer_last;
1309                 usb_pc_cpu_invalidate(td->page_cache);
1310                 status = hc32toh(sc, td->sitd_status);
1311
1312                 /* also check if first is complete */
1313
1314                 td = xfer->td_transfer_first;
1315                 usb_pc_cpu_invalidate(td->page_cache);
1316                 status |= hc32toh(sc, td->sitd_status);
1317
1318                 if (!(status & EHCI_SITD_ACTIVE)) {
1319                         ehci_device_done(xfer, USB_ERR_NORMAL_COMPLETION);
1320                         goto transferred;
1321                 }
1322         } else if (methods == &ehci_device_isoc_hs_methods) {
1323                 ehci_itd_t *td;
1324
1325                 /* isochronous high speed transfer */
1326
1327                 td = xfer->td_transfer_last;
1328                 usb_pc_cpu_invalidate(td->page_cache);
1329                 status =
1330                     td->itd_status[0] | td->itd_status[1] |
1331                     td->itd_status[2] | td->itd_status[3] |
1332                     td->itd_status[4] | td->itd_status[5] |
1333                     td->itd_status[6] | td->itd_status[7];
1334
1335                 /* also check first transfer */
1336                 td = xfer->td_transfer_first;
1337                 usb_pc_cpu_invalidate(td->page_cache);
1338                 status |=
1339                     td->itd_status[0] | td->itd_status[1] |
1340                     td->itd_status[2] | td->itd_status[3] |
1341                     td->itd_status[4] | td->itd_status[5] |
1342                     td->itd_status[6] | td->itd_status[7];
1343
1344                 /* if no transactions are active we continue */
1345                 if (!(status & htohc32(sc, EHCI_ITD_ACTIVE))) {
1346                         ehci_device_done(xfer, USB_ERR_NORMAL_COMPLETION);
1347                         goto transferred;
1348                 }
1349         } else {
1350                 ehci_qtd_t *td;
1351
1352                 /* non-isochronous transfer */
1353
1354                 /*
1355                  * check whether there is an error somewhere in the middle,
1356                  * or whether there was a short packet (SPD and not ACTIVE)
1357                  */
1358                 td = xfer->td_transfer_cache;
1359
1360                 while (1) {
1361                         usb_pc_cpu_invalidate(td->page_cache);
1362                         status = hc32toh(sc, td->qtd_status);
1363
1364                         /*
1365                          * if there is an active TD the transfer isn't done
1366                          */
1367                         if (status & EHCI_QTD_ACTIVE) {
1368                                 /* update cache */
1369                                 xfer->td_transfer_cache = td;
1370                                 goto done;
1371                         }
1372                         /*
1373                          * last transfer descriptor makes the transfer done
1374                          */
1375                         if (((void *)td) == xfer->td_transfer_last) {
1376                                 break;
1377                         }
1378                         /*
1379                          * any kind of error makes the transfer done
1380                          */
1381                         if (status & EHCI_QTD_HALTED) {
1382                                 break;
1383                         }
1384                         /*
1385                          * if there is no alternate next transfer, a short
1386                          * packet also makes the transfer done
1387                          */
1388                         if (EHCI_QTD_GET_BYTES(status)) {
1389                                 if (xfer->flags_int.short_frames_ok) {
1390                                         /* follow alt next */
1391                                         if (td->alt_next) {
1392                                                 td = td->alt_next;
1393                                                 continue;
1394                                         }
1395                                 }
1396                                 /* transfer is done */
1397                                 break;
1398                         }
1399                         td = td->obj_next;
1400                 }
1401                 ehci_non_isoc_done(xfer);
1402                 goto transferred;
1403         }
1404
1405 done:
1406         DPRINTFN(13, "xfer=%p is still active\n", xfer);
1407         return (0);
1408
1409 transferred:
1410         return (1);
1411 }
1412
1413 static void
1414 ehci_pcd_enable(ehci_softc_t *sc)
1415 {
1416         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
1417
1418         sc->sc_eintrs |= EHCI_STS_PCD;
1419         EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
1420
1421         /* acknowledge any PCD interrupt */
1422         EOWRITE4(sc, EHCI_USBSTS, EHCI_STS_PCD);
1423
1424         ehci_root_intr(sc);
1425 }
1426
1427 static void
1428 ehci_interrupt_poll(ehci_softc_t *sc)
1429 {
1430         struct usb_xfer *xfer;
1431
1432 repeat:
1433         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
1434                 /*
1435                  * check if transfer is transferred
1436                  */
1437                 if (ehci_check_transfer(xfer)) {
1438                         /* queue has been modified */
1439                         goto repeat;
1440                 }
1441         }
1442 }
1443
1444 /*------------------------------------------------------------------------*
1445  *      ehci_interrupt - EHCI interrupt handler
1446  *
1447  * NOTE: Do not access "sc->sc_bus.bdev" inside the interrupt handler,
1448  * hence the interrupt handler will be setup before "sc->sc_bus.bdev"
1449  * is present !
1450  *------------------------------------------------------------------------*/
1451 void
1452 ehci_interrupt(ehci_softc_t *sc)
1453 {
1454         uint32_t status;
1455
1456         USB_BUS_LOCK(&sc->sc_bus);
1457
1458         DPRINTFN(16, "real interrupt\n");
1459
1460 #if USB_DEBUG
1461         if (ehcidebug > 15) {
1462                 ehci_dump_regs(sc);
1463         }
1464 #endif
1465
1466         status = EHCI_STS_INTRS(EOREAD4(sc, EHCI_USBSTS));
1467         if (status == 0) {
1468                 /* the interrupt was not for us */
1469                 goto done;
1470         }
1471         if (!(status & sc->sc_eintrs)) {
1472                 goto done;
1473         }
1474         EOWRITE4(sc, EHCI_USBSTS, status);      /* acknowledge */
1475
1476         status &= sc->sc_eintrs;
1477
1478         if (status & EHCI_STS_HSE) {
1479                 printf("%s: unrecoverable error, "
1480                     "controller halted\n", __FUNCTION__);
1481 #if USB_DEBUG
1482                 ehci_dump_regs(sc);
1483                 ehci_dump_isoc(sc);
1484 #endif
1485         }
1486         if (status & EHCI_STS_PCD) {
1487                 /*
1488                  * Disable PCD interrupt for now, because it will be
1489                  * on until the port has been reset.
1490                  */
1491                 sc->sc_eintrs &= ~EHCI_STS_PCD;
1492                 EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
1493
1494                 ehci_root_intr(sc);
1495
1496                 /* do not allow RHSC interrupts > 1 per second */
1497                 usb_callout_reset(&sc->sc_tmo_pcd, hz,
1498                     (void *)&ehci_pcd_enable, sc);
1499         }
1500         status &= ~(EHCI_STS_INT | EHCI_STS_ERRINT | EHCI_STS_PCD | EHCI_STS_IAA);
1501
1502         if (status != 0) {
1503                 /* block unprocessed interrupts */
1504                 sc->sc_eintrs &= ~status;
1505                 EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
1506                 printf("%s: blocking interrupts 0x%x\n", __FUNCTION__, status);
1507         }
1508         /* poll all the USB transfers */
1509         ehci_interrupt_poll(sc);
1510
1511 done:
1512         USB_BUS_UNLOCK(&sc->sc_bus);
1513 }
1514
1515 /*
1516  * called when a request does not complete
1517  */
1518 static void
1519 ehci_timeout(void *arg)
1520 {
1521         struct usb_xfer *xfer = arg;
1522
1523         DPRINTF("xfer=%p\n", xfer);
1524
1525         USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
1526
1527         /* transfer is transferred */
1528         ehci_device_done(xfer, USB_ERR_TIMEOUT);
1529 }
1530
1531 static void
1532 ehci_do_poll(struct usb_bus *bus)
1533 {
1534         ehci_softc_t *sc = EHCI_BUS2SC(bus);
1535
1536         USB_BUS_LOCK(&sc->sc_bus);
1537         ehci_interrupt_poll(sc);
1538         USB_BUS_UNLOCK(&sc->sc_bus);
1539 }
1540
1541 static void
1542 ehci_setup_standard_chain_sub(struct ehci_std_temp *temp)
1543 {
1544         struct usb_page_search buf_res;
1545         ehci_qtd_t *td;
1546         ehci_qtd_t *td_next;
1547         ehci_qtd_t *td_alt_next;
1548         uint32_t qtd_altnext;
1549         uint32_t buf_offset;
1550         uint32_t average;
1551         uint32_t len_old;
1552         uint32_t terminate;
1553         uint8_t shortpkt_old;
1554         uint8_t precompute;
1555
1556         terminate = htohc32(temp->sc, EHCI_LINK_TERMINATE);
1557         qtd_altnext = terminate;
1558         td_alt_next = NULL;
1559         buf_offset = 0;
1560         shortpkt_old = temp->shortpkt;
1561         len_old = temp->len;
1562         precompute = 1;
1563
1564 restart:
1565
1566         td = temp->td;
1567         td_next = temp->td_next;
1568
1569         while (1) {
1570
1571                 if (temp->len == 0) {
1572
1573                         if (temp->shortpkt) {
1574                                 break;
1575                         }
1576                         /* send a Zero Length Packet, ZLP, last */
1577
1578                         temp->shortpkt = 1;
1579                         average = 0;
1580
1581                 } else {
1582
1583                         average = temp->average;
1584
1585                         if (temp->len < average) {
1586                                 if (temp->len % temp->max_frame_size) {
1587                                         temp->shortpkt = 1;
1588                                 }
1589                                 average = temp->len;
1590                         }
1591                 }
1592
1593                 if (td_next == NULL) {
1594                         panic("%s: out of EHCI transfer descriptors!", __FUNCTION__);
1595                 }
1596                 /* get next TD */
1597
1598                 td = td_next;
1599                 td_next = td->obj_next;
1600
1601                 /* check if we are pre-computing */
1602
1603                 if (precompute) {
1604
1605                         /* update remaining length */
1606
1607                         temp->len -= average;
1608
1609                         continue;
1610                 }
1611                 /* fill out current TD */
1612
1613                 td->qtd_status =
1614                     temp->qtd_status |
1615                     htohc32(temp->sc, EHCI_QTD_SET_BYTES(average));
1616
1617                 if (average == 0) {
1618
1619                         if (temp->auto_data_toggle == 0) {
1620
1621                                 /* update data toggle, ZLP case */
1622
1623                                 temp->qtd_status ^=
1624                                     htohc32(temp->sc, EHCI_QTD_TOGGLE_MASK);
1625                         }
1626                         td->len = 0;
1627
1628                         td->qtd_buffer[0] = 0;
1629                         td->qtd_buffer_hi[0] = 0;
1630
1631                         td->qtd_buffer[1] = 0;
1632                         td->qtd_buffer_hi[1] = 0;
1633
1634                 } else {
1635
1636                         uint8_t x;
1637
1638                         if (temp->auto_data_toggle == 0) {
1639
1640                                 /* update data toggle */
1641
1642                                 if (((average + temp->max_frame_size - 1) /
1643                                     temp->max_frame_size) & 1) {
1644                                         temp->qtd_status ^=
1645                                             htohc32(temp->sc, EHCI_QTD_TOGGLE_MASK);
1646                                 }
1647                         }
1648                         td->len = average;
1649
1650                         /* update remaining length */
1651
1652                         temp->len -= average;
1653
1654                         /* fill out buffer pointers */
1655
1656                         usbd_get_page(temp->pc, buf_offset, &buf_res);
1657                         td->qtd_buffer[0] =
1658                             htohc32(temp->sc, buf_res.physaddr);
1659                         td->qtd_buffer_hi[0] = 0;
1660
1661                         x = 1;
1662
1663                         while (average > EHCI_PAGE_SIZE) {
1664                                 average -= EHCI_PAGE_SIZE;
1665                                 buf_offset += EHCI_PAGE_SIZE;
1666                                 usbd_get_page(temp->pc, buf_offset, &buf_res);
1667                                 td->qtd_buffer[x] =
1668                                     htohc32(temp->sc,
1669                                     buf_res.physaddr & (~0xFFF));
1670                                 td->qtd_buffer_hi[x] = 0;
1671                                 x++;
1672                         }
1673
1674                         /*
1675                          * NOTE: The "average" variable is never zero after
1676                          * exiting the loop above !
1677                          *
1678                          * NOTE: We have to subtract one from the offset to
1679                          * ensure that we are computing the physical address
1680                          * of a valid page !
1681                          */
1682                         buf_offset += average;
1683                         usbd_get_page(temp->pc, buf_offset - 1, &buf_res);
1684                         td->qtd_buffer[x] =
1685                             htohc32(temp->sc,
1686                             buf_res.physaddr & (~0xFFF));
1687                         td->qtd_buffer_hi[x] = 0;
1688                 }
1689
1690                 if (td_next) {
1691                         /* link the current TD with the next one */
1692                         td->qtd_next = td_next->qtd_self;
1693                 }
1694                 td->qtd_altnext = qtd_altnext;
1695                 td->alt_next = td_alt_next;
1696
1697                 usb_pc_cpu_flush(td->page_cache);
1698         }
1699
1700         if (precompute) {
1701                 precompute = 0;
1702
1703                 /* setup alt next pointer, if any */
1704                 if (temp->last_frame) {
1705                         td_alt_next = NULL;
1706                         qtd_altnext = terminate;
1707                 } else {
1708                         /* we use this field internally */
1709                         td_alt_next = td_next;
1710                         if (temp->setup_alt_next) {
1711                                 qtd_altnext = td_next->qtd_self;
1712                         } else {
1713                                 qtd_altnext = terminate;
1714                         }
1715                 }
1716
1717                 /* restore */
1718                 temp->shortpkt = shortpkt_old;
1719                 temp->len = len_old;
1720                 goto restart;
1721         }
1722         temp->td = td;
1723         temp->td_next = td_next;
1724 }
1725
1726 static void
1727 ehci_setup_standard_chain(struct usb_xfer *xfer, ehci_qh_t **qh_last)
1728 {
1729         struct ehci_std_temp temp;
1730         struct usb_pipe_methods *methods;
1731         ehci_qh_t *qh;
1732         ehci_qtd_t *td;
1733         uint32_t qh_endp;
1734         uint32_t qh_endphub;
1735         uint32_t x;
1736
1737         DPRINTFN(9, "addr=%d endpt=%d sumlen=%d speed=%d\n",
1738             xfer->address, UE_GET_ADDR(xfer->endpointno),
1739             xfer->sumlen, usbd_get_speed(xfer->xroot->udev));
1740
1741         temp.average = xfer->max_hc_frame_size;
1742         temp.max_frame_size = xfer->max_frame_size;
1743         temp.sc = EHCI_BUS2SC(xfer->xroot->bus);
1744
1745         /* toggle the DMA set we are using */
1746         xfer->flags_int.curr_dma_set ^= 1;
1747
1748         /* get next DMA set */
1749         td = xfer->td_start[xfer->flags_int.curr_dma_set];
1750
1751         xfer->td_transfer_first = td;
1752         xfer->td_transfer_cache = td;
1753
1754         temp.td = NULL;
1755         temp.td_next = td;
1756         temp.qtd_status = 0;
1757         temp.last_frame = 0;
1758         temp.setup_alt_next = xfer->flags_int.short_frames_ok;
1759
1760         if (xfer->flags_int.control_xfr) {
1761                 if (xfer->endpoint->toggle_next) {
1762                         /* DATA1 is next */
1763                         temp.qtd_status |=
1764                             htohc32(temp.sc, EHCI_QTD_SET_TOGGLE(1));
1765                 }
1766                 temp.auto_data_toggle = 0;
1767         } else {
1768                 temp.auto_data_toggle = 1;
1769         }
1770
1771         if (usbd_get_speed(xfer->xroot->udev) != USB_SPEED_HIGH) {
1772                 /* max 3 retries */
1773                 temp.qtd_status |=
1774                     htohc32(temp.sc, EHCI_QTD_SET_CERR(3));
1775         }
1776         /* check if we should prepend a setup message */
1777
1778         if (xfer->flags_int.control_xfr) {
1779                 if (xfer->flags_int.control_hdr) {
1780
1781                         temp.qtd_status &=
1782                             htohc32(temp.sc, EHCI_QTD_SET_CERR(3));
1783                         temp.qtd_status |= htohc32(temp.sc,
1784                             EHCI_QTD_ACTIVE |
1785                             EHCI_QTD_SET_PID(EHCI_QTD_PID_SETUP) |
1786                             EHCI_QTD_SET_TOGGLE(0));
1787
1788                         temp.len = xfer->frlengths[0];
1789                         temp.pc = xfer->frbuffers + 0;
1790                         temp.shortpkt = temp.len ? 1 : 0;
1791                         /* check for last frame */
1792                         if (xfer->nframes == 1) {
1793                                 /* no STATUS stage yet, SETUP is last */
1794                                 if (xfer->flags_int.control_act) {
1795                                         temp.last_frame = 1;
1796                                         temp.setup_alt_next = 0;
1797                                 }
1798                         }
1799                         ehci_setup_standard_chain_sub(&temp);
1800                 }
1801                 x = 1;
1802         } else {
1803                 x = 0;
1804         }
1805
1806         while (x != xfer->nframes) {
1807
1808                 /* DATA0 / DATA1 message */
1809
1810                 temp.len = xfer->frlengths[x];
1811                 temp.pc = xfer->frbuffers + x;
1812
1813                 x++;
1814
1815                 if (x == xfer->nframes) {
1816                         if (xfer->flags_int.control_xfr) {
1817                                 /* no STATUS stage yet, DATA is last */
1818                                 if (xfer->flags_int.control_act) {
1819                                         temp.last_frame = 1;
1820                                         temp.setup_alt_next = 0;
1821                                 }
1822                         } else {
1823                                 temp.last_frame = 1;
1824                                 temp.setup_alt_next = 0;
1825                         }
1826                 }
1827                 /* keep previous data toggle and error count */
1828
1829                 temp.qtd_status &=
1830                     htohc32(temp.sc, EHCI_QTD_SET_CERR(3) |
1831                     EHCI_QTD_SET_TOGGLE(1));
1832
1833                 if (temp.len == 0) {
1834
1835                         /* make sure that we send an USB packet */
1836
1837                         temp.shortpkt = 0;
1838
1839                 } else {
1840
1841                         /* regular data transfer */
1842
1843                         temp.shortpkt = (xfer->flags.force_short_xfer) ? 0 : 1;
1844                 }
1845
1846                 /* set endpoint direction */
1847
1848                 temp.qtd_status |=
1849                     (UE_GET_DIR(xfer->endpointno) == UE_DIR_IN) ?
1850                     htohc32(temp.sc, EHCI_QTD_ACTIVE |
1851                     EHCI_QTD_SET_PID(EHCI_QTD_PID_IN)) :
1852                     htohc32(temp.sc, EHCI_QTD_ACTIVE |
1853                     EHCI_QTD_SET_PID(EHCI_QTD_PID_OUT));
1854
1855                 ehci_setup_standard_chain_sub(&temp);
1856         }
1857
1858         /* check if we should append a status stage */
1859
1860         if (xfer->flags_int.control_xfr &&
1861             !xfer->flags_int.control_act) {
1862
1863                 /*
1864                  * Send a DATA1 message and invert the current endpoint
1865                  * direction.
1866                  */
1867
1868                 temp.qtd_status &= htohc32(temp.sc, EHCI_QTD_SET_CERR(3) |
1869                     EHCI_QTD_SET_TOGGLE(1));
1870                 temp.qtd_status |=
1871                     (UE_GET_DIR(xfer->endpointno) == UE_DIR_OUT) ?
1872                     htohc32(temp.sc, EHCI_QTD_ACTIVE |
1873                     EHCI_QTD_SET_PID(EHCI_QTD_PID_IN) |
1874                     EHCI_QTD_SET_TOGGLE(1)) :
1875                     htohc32(temp.sc, EHCI_QTD_ACTIVE |
1876                     EHCI_QTD_SET_PID(EHCI_QTD_PID_OUT) |
1877                     EHCI_QTD_SET_TOGGLE(1));
1878
1879                 temp.len = 0;
1880                 temp.pc = NULL;
1881                 temp.shortpkt = 0;
1882                 temp.last_frame = 1;
1883                 temp.setup_alt_next = 0;
1884
1885                 ehci_setup_standard_chain_sub(&temp);
1886         }
1887         td = temp.td;
1888
1889         /* the last TD terminates the transfer: */
1890         td->qtd_next = htohc32(temp.sc, EHCI_LINK_TERMINATE);
1891         td->qtd_altnext = htohc32(temp.sc, EHCI_LINK_TERMINATE);
1892         td->qtd_status |= htohc32(temp.sc, EHCI_QTD_IOC);
1893
1894         usb_pc_cpu_flush(td->page_cache);
1895
1896         /* must have at least one frame! */
1897
1898         xfer->td_transfer_last = td;
1899
1900 #if USB_DEBUG
1901         if (ehcidebug > 8) {
1902                 DPRINTF("nexttog=%d; data before transfer:\n",
1903                     xfer->endpoint->toggle_next);
1904                 ehci_dump_sqtds(temp.sc,
1905                     xfer->td_transfer_first);
1906         }
1907 #endif
1908
1909         methods = xfer->endpoint->methods;
1910
1911         qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
1912
1913         /* the "qh_link" field is filled when the QH is added */
1914
1915         qh_endp =
1916             (EHCI_QH_SET_ADDR(xfer->address) |
1917             EHCI_QH_SET_ENDPT(UE_GET_ADDR(xfer->endpointno)) |
1918             EHCI_QH_SET_MPL(xfer->max_packet_size));
1919
1920         if (usbd_get_speed(xfer->xroot->udev) == USB_SPEED_HIGH) {
1921                 qh_endp |= (EHCI_QH_SET_EPS(EHCI_QH_SPEED_HIGH) |
1922                     EHCI_QH_DTC);
1923                 if (methods != &ehci_device_intr_methods)
1924                         qh_endp |= EHCI_QH_SET_NRL(8);
1925         } else {
1926
1927                 if (usbd_get_speed(xfer->xroot->udev) == USB_SPEED_FULL) {
1928                         qh_endp |= (EHCI_QH_SET_EPS(EHCI_QH_SPEED_FULL) |
1929                             EHCI_QH_DTC);
1930                 } else {
1931                         qh_endp |= (EHCI_QH_SET_EPS(EHCI_QH_SPEED_LOW) |
1932                             EHCI_QH_DTC);
1933                 }
1934
1935                 if (methods == &ehci_device_ctrl_methods) {
1936                         qh_endp |= EHCI_QH_CTL;
1937                 }
1938                 if (methods != &ehci_device_intr_methods) {
1939                         /* Only try one time per microframe! */
1940                         qh_endp |= EHCI_QH_SET_NRL(1);
1941                 }
1942         }
1943
1944         qh->qh_endp = htohc32(temp.sc, qh_endp);
1945
1946         qh_endphub =
1947             (EHCI_QH_SET_MULT(xfer->max_packet_count & 3) |
1948             EHCI_QH_SET_CMASK(xfer->usb_cmask) |
1949             EHCI_QH_SET_SMASK(xfer->usb_smask) |
1950             EHCI_QH_SET_HUBA(xfer->xroot->udev->hs_hub_addr) |
1951             EHCI_QH_SET_PORT(xfer->xroot->udev->hs_port_no));
1952
1953         qh->qh_endphub = htohc32(temp.sc, qh_endphub);
1954         qh->qh_curqtd = htohc32(temp.sc, 0);
1955
1956         /* fill the overlay qTD */
1957         qh->qh_qtd.qtd_status = htohc32(temp.sc, 0);
1958
1959         if (temp.auto_data_toggle) {
1960
1961                 /* let the hardware compute the data toggle */
1962
1963                 qh->qh_endp &= htohc32(temp.sc, ~EHCI_QH_DTC);
1964
1965                 if (xfer->endpoint->toggle_next) {
1966                         /* DATA1 is next */
1967                         qh->qh_qtd.qtd_status |=
1968                             htohc32(temp.sc, EHCI_QTD_SET_TOGGLE(1));
1969                 }
1970         }
1971         td = xfer->td_transfer_first;
1972
1973         qh->qh_qtd.qtd_next = td->qtd_self;
1974         qh->qh_qtd.qtd_altnext =
1975             htohc32(temp.sc, EHCI_LINK_TERMINATE);
1976
1977         usb_pc_cpu_flush(qh->page_cache);
1978
1979         if (xfer->xroot->udev->flags.self_suspended == 0) {
1980                 EHCI_APPEND_QH(qh, *qh_last);
1981         }
1982 }
1983
1984 static void
1985 ehci_root_intr(ehci_softc_t *sc)
1986 {
1987         uint16_t i;
1988         uint16_t m;
1989
1990         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
1991
1992         /* clear any old interrupt data */
1993         memset(sc->sc_hub_idata, 0, sizeof(sc->sc_hub_idata));
1994
1995         /* set bits */
1996         m = (sc->sc_noport + 1);
1997         if (m > (8 * sizeof(sc->sc_hub_idata))) {
1998                 m = (8 * sizeof(sc->sc_hub_idata));
1999         }
2000         for (i = 1; i < m; i++) {
2001                 /* pick out CHANGE bits from the status register */
2002                 if (EOREAD4(sc, EHCI_PORTSC(i)) & EHCI_PS_CLEAR) {
2003                         sc->sc_hub_idata[i / 8] |= 1 << (i % 8);
2004                         DPRINTF("port %d changed\n", i);
2005                 }
2006         }
2007         uhub_root_intr(&sc->sc_bus, sc->sc_hub_idata,
2008             sizeof(sc->sc_hub_idata));
2009 }
2010
2011 static void
2012 ehci_isoc_fs_done(ehci_softc_t *sc, struct usb_xfer *xfer)
2013 {
2014         uint32_t nframes = xfer->nframes;
2015         uint32_t status;
2016         uint32_t *plen = xfer->frlengths;
2017         uint16_t len = 0;
2018         ehci_sitd_t *td = xfer->td_transfer_first;
2019         ehci_sitd_t **pp_last = &sc->sc_isoc_fs_p_last[xfer->qh_pos];
2020
2021         DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
2022             xfer, xfer->endpoint);
2023
2024         while (nframes--) {
2025                 if (td == NULL) {
2026                         panic("%s:%d: out of TD's\n",
2027                             __FUNCTION__, __LINE__);
2028                 }
2029                 if (pp_last >= &sc->sc_isoc_fs_p_last[EHCI_VIRTUAL_FRAMELIST_COUNT]) {
2030                         pp_last = &sc->sc_isoc_fs_p_last[0];
2031                 }
2032 #if USB_DEBUG
2033                 if (ehcidebug > 15) {
2034                         DPRINTF("isoc FS-TD\n");
2035                         ehci_dump_sitd(sc, td);
2036                 }
2037 #endif
2038                 usb_pc_cpu_invalidate(td->page_cache);
2039                 status = hc32toh(sc, td->sitd_status);
2040
2041                 len = EHCI_SITD_GET_LEN(status);
2042
2043                 DPRINTFN(2, "status=0x%08x, rem=%u\n", status, len);
2044
2045                 if (*plen >= len) {
2046                         len = *plen - len;
2047                 } else {
2048                         len = 0;
2049                 }
2050
2051                 *plen = len;
2052
2053                 /* remove FS-TD from schedule */
2054                 EHCI_REMOVE_FS_TD(td, *pp_last);
2055
2056                 pp_last++;
2057                 plen++;
2058                 td = td->obj_next;
2059         }
2060
2061         xfer->aframes = xfer->nframes;
2062 }
2063
2064 static void
2065 ehci_isoc_hs_done(ehci_softc_t *sc, struct usb_xfer *xfer)
2066 {
2067         uint32_t nframes = xfer->nframes;
2068         uint32_t status;
2069         uint32_t *plen = xfer->frlengths;
2070         uint16_t len = 0;
2071         uint8_t td_no = 0;
2072         ehci_itd_t *td = xfer->td_transfer_first;
2073         ehci_itd_t **pp_last = &sc->sc_isoc_hs_p_last[xfer->qh_pos];
2074
2075         DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
2076             xfer, xfer->endpoint);
2077
2078         while (nframes--) {
2079                 if (td == NULL) {
2080                         panic("%s:%d: out of TD's\n",
2081                             __FUNCTION__, __LINE__);
2082                 }
2083                 if (pp_last >= &sc->sc_isoc_hs_p_last[EHCI_VIRTUAL_FRAMELIST_COUNT]) {
2084                         pp_last = &sc->sc_isoc_hs_p_last[0];
2085                 }
2086 #if USB_DEBUG
2087                 if (ehcidebug > 15) {
2088                         DPRINTF("isoc HS-TD\n");
2089                         ehci_dump_itd(sc, td);
2090                 }
2091 #endif
2092
2093                 usb_pc_cpu_invalidate(td->page_cache);
2094                 status = hc32toh(sc, td->itd_status[td_no]);
2095
2096                 len = EHCI_ITD_GET_LEN(status);
2097
2098                 DPRINTFN(2, "status=0x%08x, len=%u\n", status, len);
2099
2100                 if (*plen >= len) {
2101                         /*
2102                          * The length is valid. NOTE: The complete
2103                          * length is written back into the status
2104                          * field, and not the remainder like with
2105                          * other transfer descriptor types.
2106                          */
2107                 } else {
2108                         /* Invalid length - truncate */
2109                         len = 0;
2110                 }
2111
2112                 *plen = len;
2113
2114                 plen++;
2115                 td_no++;
2116
2117                 if ((td_no == 8) || (nframes == 0)) {
2118                         /* remove HS-TD from schedule */
2119                         EHCI_REMOVE_HS_TD(td, *pp_last);
2120                         pp_last++;
2121
2122                         td_no = 0;
2123                         td = td->obj_next;
2124                 }
2125         }
2126         xfer->aframes = xfer->nframes;
2127 }
2128
2129 /* NOTE: "done" can be run two times in a row,
2130  * from close and from interrupt
2131  */
2132 static void
2133 ehci_device_done(struct usb_xfer *xfer, usb_error_t error)
2134 {
2135         struct usb_pipe_methods *methods = xfer->endpoint->methods;
2136         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2137
2138         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
2139
2140         DPRINTFN(2, "xfer=%p, endpoint=%p, error=%d\n",
2141             xfer, xfer->endpoint, error);
2142
2143         if ((methods == &ehci_device_bulk_methods) ||
2144             (methods == &ehci_device_ctrl_methods)) {
2145 #if USB_DEBUG
2146                 if (ehcidebug > 8) {
2147                         DPRINTF("nexttog=%d; data after transfer:\n",
2148                             xfer->endpoint->toggle_next);
2149                         ehci_dump_sqtds(sc,
2150                             xfer->td_transfer_first);
2151                 }
2152 #endif
2153
2154                 EHCI_REMOVE_QH(xfer->qh_start[xfer->flags_int.curr_dma_set],
2155                     sc->sc_async_p_last);
2156         }
2157         if (methods == &ehci_device_intr_methods) {
2158                 EHCI_REMOVE_QH(xfer->qh_start[xfer->flags_int.curr_dma_set],
2159                     sc->sc_intr_p_last[xfer->qh_pos]);
2160         }
2161         /*
2162          * Only finish isochronous transfers once which will update
2163          * "xfer->frlengths".
2164          */
2165         if (xfer->td_transfer_first &&
2166             xfer->td_transfer_last) {
2167                 if (methods == &ehci_device_isoc_fs_methods) {
2168                         ehci_isoc_fs_done(sc, xfer);
2169                 }
2170                 if (methods == &ehci_device_isoc_hs_methods) {
2171                         ehci_isoc_hs_done(sc, xfer);
2172                 }
2173                 xfer->td_transfer_first = NULL;
2174                 xfer->td_transfer_last = NULL;
2175         }
2176         /* dequeue transfer and start next transfer */
2177         usbd_transfer_done(xfer, error);
2178 }
2179
2180 /*------------------------------------------------------------------------*
2181  * ehci bulk support
2182  *------------------------------------------------------------------------*/
2183 static void
2184 ehci_device_bulk_open(struct usb_xfer *xfer)
2185 {
2186         return;
2187 }
2188
2189 static void
2190 ehci_device_bulk_close(struct usb_xfer *xfer)
2191 {
2192         ehci_device_done(xfer, USB_ERR_CANCELLED);
2193 }
2194
2195 static void
2196 ehci_device_bulk_enter(struct usb_xfer *xfer)
2197 {
2198         return;
2199 }
2200
2201 static void
2202 ehci_device_bulk_start(struct usb_xfer *xfer)
2203 {
2204         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2205         uint32_t temp;
2206
2207         /* setup TD's and QH */
2208         ehci_setup_standard_chain(xfer, &sc->sc_async_p_last);
2209
2210         /* put transfer on interrupt queue */
2211         ehci_transfer_intr_enqueue(xfer);
2212
2213         /* XXX Performance quirk: Some Host Controllers have a too low
2214          * interrupt rate. Issue an IAAD to stimulate the Host
2215          * Controller after queueing the BULK transfer.
2216          */
2217         temp = EOREAD4(sc, EHCI_USBCMD);
2218         if (!(temp & EHCI_CMD_IAAD))
2219                 EOWRITE4(sc, EHCI_USBCMD, temp | EHCI_CMD_IAAD);
2220 }
2221
2222 struct usb_pipe_methods ehci_device_bulk_methods =
2223 {
2224         .open = ehci_device_bulk_open,
2225         .close = ehci_device_bulk_close,
2226         .enter = ehci_device_bulk_enter,
2227         .start = ehci_device_bulk_start,
2228 };
2229
2230 /*------------------------------------------------------------------------*
2231  * ehci control support
2232  *------------------------------------------------------------------------*/
2233 static void
2234 ehci_device_ctrl_open(struct usb_xfer *xfer)
2235 {
2236         return;
2237 }
2238
2239 static void
2240 ehci_device_ctrl_close(struct usb_xfer *xfer)
2241 {
2242         ehci_device_done(xfer, USB_ERR_CANCELLED);
2243 }
2244
2245 static void
2246 ehci_device_ctrl_enter(struct usb_xfer *xfer)
2247 {
2248         return;
2249 }
2250
2251 static void
2252 ehci_device_ctrl_start(struct usb_xfer *xfer)
2253 {
2254         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2255
2256         /* setup TD's and QH */
2257         ehci_setup_standard_chain(xfer, &sc->sc_async_p_last);
2258
2259         /* put transfer on interrupt queue */
2260         ehci_transfer_intr_enqueue(xfer);
2261 }
2262
2263 struct usb_pipe_methods ehci_device_ctrl_methods =
2264 {
2265         .open = ehci_device_ctrl_open,
2266         .close = ehci_device_ctrl_close,
2267         .enter = ehci_device_ctrl_enter,
2268         .start = ehci_device_ctrl_start,
2269 };
2270
2271 /*------------------------------------------------------------------------*
2272  * ehci interrupt support
2273  *------------------------------------------------------------------------*/
2274 static void
2275 ehci_device_intr_open(struct usb_xfer *xfer)
2276 {
2277         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2278         uint16_t best;
2279         uint16_t bit;
2280         uint16_t x;
2281         uint8_t slot;
2282
2283         /* Allocate a microframe slot first: */
2284
2285         slot = usb_intr_schedule_adjust
2286             (xfer->xroot->udev, xfer->max_frame_size, USB_HS_MICRO_FRAMES_MAX);
2287
2288         if (usbd_get_speed(xfer->xroot->udev) == USB_SPEED_HIGH) {
2289                 xfer->usb_uframe = slot;
2290                 xfer->usb_smask = (1 << slot) & 0xFF;
2291                 xfer->usb_cmask = 0;
2292         } else {
2293                 xfer->usb_uframe = slot;
2294                 xfer->usb_smask = (1 << slot) & 0x3F;
2295                 xfer->usb_cmask = (-(4 << slot)) & 0xFE;
2296         }
2297
2298         /*
2299          * Find the best QH position corresponding to the given interval:
2300          */
2301
2302         best = 0;
2303         bit = EHCI_VIRTUAL_FRAMELIST_COUNT / 2;
2304         while (bit) {
2305                 if (xfer->interval >= bit) {
2306                         x = bit;
2307                         best = bit;
2308                         while (x & bit) {
2309                                 if (sc->sc_intr_stat[x] <
2310                                     sc->sc_intr_stat[best]) {
2311                                         best = x;
2312                                 }
2313                                 x++;
2314                         }
2315                         break;
2316                 }
2317                 bit >>= 1;
2318         }
2319
2320         sc->sc_intr_stat[best]++;
2321         xfer->qh_pos = best;
2322
2323         DPRINTFN(3, "best=%d interval=%d\n",
2324             best, xfer->interval);
2325 }
2326
2327 static void
2328 ehci_device_intr_close(struct usb_xfer *xfer)
2329 {
2330         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2331         uint8_t slot;
2332
2333         slot = usb_intr_schedule_adjust
2334             (xfer->xroot->udev, -(xfer->max_frame_size), xfer->usb_uframe);
2335
2336         sc->sc_intr_stat[xfer->qh_pos]--;
2337
2338         ehci_device_done(xfer, USB_ERR_CANCELLED);
2339 }
2340
2341 static void
2342 ehci_device_intr_enter(struct usb_xfer *xfer)
2343 {
2344         return;
2345 }
2346
2347 static void
2348 ehci_device_intr_start(struct usb_xfer *xfer)
2349 {
2350         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2351
2352         /* setup TD's and QH */
2353         ehci_setup_standard_chain(xfer, &sc->sc_intr_p_last[xfer->qh_pos]);
2354
2355         /* put transfer on interrupt queue */
2356         ehci_transfer_intr_enqueue(xfer);
2357 }
2358
2359 struct usb_pipe_methods ehci_device_intr_methods =
2360 {
2361         .open = ehci_device_intr_open,
2362         .close = ehci_device_intr_close,
2363         .enter = ehci_device_intr_enter,
2364         .start = ehci_device_intr_start,
2365 };
2366
2367 /*------------------------------------------------------------------------*
2368  * ehci full speed isochronous support
2369  *------------------------------------------------------------------------*/
2370 static void
2371 ehci_device_isoc_fs_open(struct usb_xfer *xfer)
2372 {
2373         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2374         ehci_sitd_t *td;
2375         uint32_t sitd_portaddr;
2376         uint8_t ds;
2377
2378         sitd_portaddr =
2379             EHCI_SITD_SET_ADDR(xfer->address) |
2380             EHCI_SITD_SET_ENDPT(UE_GET_ADDR(xfer->endpointno)) |
2381             EHCI_SITD_SET_HUBA(xfer->xroot->udev->hs_hub_addr) |
2382             EHCI_SITD_SET_PORT(xfer->xroot->udev->hs_port_no);
2383
2384         if (UE_GET_DIR(xfer->endpointno) == UE_DIR_IN) {
2385                 sitd_portaddr |= EHCI_SITD_SET_DIR_IN;
2386         }
2387         sitd_portaddr = htohc32(sc, sitd_portaddr);
2388
2389         /* initialize all TD's */
2390
2391         for (ds = 0; ds != 2; ds++) {
2392
2393                 for (td = xfer->td_start[ds]; td; td = td->obj_next) {
2394
2395                         td->sitd_portaddr = sitd_portaddr;
2396
2397                         /*
2398                          * TODO: make some kind of automatic
2399                          * SMASK/CMASK selection based on micro-frame
2400                          * usage
2401                          *
2402                          * micro-frame usage (8 microframes per 1ms)
2403                          */
2404                         td->sitd_back = htohc32(sc, EHCI_LINK_TERMINATE);
2405
2406                         usb_pc_cpu_flush(td->page_cache);
2407                 }
2408         }
2409 }
2410
2411 static void
2412 ehci_device_isoc_fs_close(struct usb_xfer *xfer)
2413 {
2414         ehci_device_done(xfer, USB_ERR_CANCELLED);
2415 }
2416
2417 static void
2418 ehci_device_isoc_fs_enter(struct usb_xfer *xfer)
2419 {
2420         struct usb_page_search buf_res;
2421         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2422         struct usb_fs_isoc_schedule *fss_start;
2423         struct usb_fs_isoc_schedule *fss_end;
2424         struct usb_fs_isoc_schedule *fss;
2425         ehci_sitd_t *td;
2426         ehci_sitd_t *td_last = NULL;
2427         ehci_sitd_t **pp_last;
2428         uint32_t *plen;
2429         uint32_t buf_offset;
2430         uint32_t nframes;
2431         uint32_t temp;
2432         uint32_t sitd_mask;
2433         uint16_t tlen;
2434         uint8_t sa;
2435         uint8_t sb;
2436         uint8_t error;
2437
2438 #if USB_DEBUG
2439         uint8_t once = 1;
2440
2441 #endif
2442
2443         DPRINTFN(6, "xfer=%p next=%d nframes=%d\n",
2444             xfer, xfer->endpoint->isoc_next, xfer->nframes);
2445
2446         /* get the current frame index */
2447
2448         nframes = EOREAD4(sc, EHCI_FRINDEX) / 8;
2449
2450         /*
2451          * check if the frame index is within the window where the frames
2452          * will be inserted
2453          */
2454         buf_offset = (nframes - xfer->endpoint->isoc_next) &
2455             (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2456
2457         if ((xfer->endpoint->is_synced == 0) ||
2458             (buf_offset < xfer->nframes)) {
2459                 /*
2460                  * If there is data underflow or the pipe queue is empty we
2461                  * schedule the transfer a few frames ahead of the current
2462                  * frame position. Else two isochronous transfers might
2463                  * overlap.
2464                  */
2465                 xfer->endpoint->isoc_next = (nframes + 3) &
2466                     (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2467                 xfer->endpoint->is_synced = 1;
2468                 DPRINTFN(3, "start next=%d\n", xfer->endpoint->isoc_next);
2469         }
2470         /*
2471          * compute how many milliseconds the insertion is ahead of the
2472          * current frame position:
2473          */
2474         buf_offset = (xfer->endpoint->isoc_next - nframes) &
2475             (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2476
2477         /*
2478          * pre-compute when the isochronous transfer will be finished:
2479          */
2480         xfer->isoc_time_complete =
2481             usbd_fs_isoc_schedule_isoc_time_expand
2482             (xfer->xroot->udev, &fss_start, &fss_end, nframes) + buf_offset +
2483             xfer->nframes;
2484
2485         /* get the real number of frames */
2486
2487         nframes = xfer->nframes;
2488
2489         buf_offset = 0;
2490
2491         plen = xfer->frlengths;
2492
2493         /* toggle the DMA set we are using */
2494         xfer->flags_int.curr_dma_set ^= 1;
2495
2496         /* get next DMA set */
2497         td = xfer->td_start[xfer->flags_int.curr_dma_set];
2498         xfer->td_transfer_first = td;
2499
2500         pp_last = &sc->sc_isoc_fs_p_last[xfer->endpoint->isoc_next];
2501
2502         /* store starting position */
2503
2504         xfer->qh_pos = xfer->endpoint->isoc_next;
2505
2506         fss = fss_start + (xfer->qh_pos % USB_ISOC_TIME_MAX);
2507
2508         while (nframes--) {
2509                 if (td == NULL) {
2510                         panic("%s:%d: out of TD's\n",
2511                             __FUNCTION__, __LINE__);
2512                 }
2513                 if (pp_last >= &sc->sc_isoc_fs_p_last[EHCI_VIRTUAL_FRAMELIST_COUNT]) {
2514                         pp_last = &sc->sc_isoc_fs_p_last[0];
2515                 }
2516                 if (fss >= fss_end) {
2517                         fss = fss_start;
2518                 }
2519                 /* reuse sitd_portaddr and sitd_back from last transfer */
2520
2521                 if (*plen > xfer->max_frame_size) {
2522 #if USB_DEBUG
2523                         if (once) {
2524                                 once = 0;
2525                                 printf("%s: frame length(%d) exceeds %d "
2526                                     "bytes (frame truncated)\n",
2527                                     __FUNCTION__, *plen,
2528                                     xfer->max_frame_size);
2529                         }
2530 #endif
2531                         *plen = xfer->max_frame_size;
2532                 }
2533                 /*
2534                  * We currently don't care if the ISOCHRONOUS schedule is
2535                  * full!
2536                  */
2537                 error = usbd_fs_isoc_schedule_alloc(fss, &sa, *plen);
2538                 if (error) {
2539                         /*
2540                          * The FULL speed schedule is FULL! Set length
2541                          * to zero.
2542                          */
2543                         *plen = 0;
2544                 }
2545                 if (*plen) {
2546                         /*
2547                          * only call "usbd_get_page()" when we have a
2548                          * non-zero length
2549                          */
2550                         usbd_get_page(xfer->frbuffers, buf_offset, &buf_res);
2551                         td->sitd_bp[0] = htohc32(sc, buf_res.physaddr);
2552                         buf_offset += *plen;
2553                         /*
2554                          * NOTE: We need to subtract one from the offset so
2555                          * that we are on a valid page!
2556                          */
2557                         usbd_get_page(xfer->frbuffers, buf_offset - 1,
2558                             &buf_res);
2559                         temp = buf_res.physaddr & ~0xFFF;
2560                 } else {
2561                         td->sitd_bp[0] = 0;
2562                         temp = 0;
2563                 }
2564
2565                 if (UE_GET_DIR(xfer->endpointno) == UE_DIR_OUT) {
2566                         tlen = *plen;
2567                         if (tlen <= 188) {
2568                                 temp |= 1;      /* T-count = 1, TP = ALL */
2569                                 tlen = 1;
2570                         } else {
2571                                 tlen += 187;
2572                                 tlen /= 188;
2573                                 temp |= tlen;   /* T-count = [1..6] */
2574                                 temp |= 8;      /* TP = Begin */
2575                         }
2576
2577                         tlen += sa;
2578
2579                         if (tlen >= 8) {
2580                                 sb = 0;
2581                         } else {
2582                                 sb = (1 << tlen);
2583                         }
2584
2585                         sa = (1 << sa);
2586                         sa = (sb - sa) & 0x3F;
2587                         sb = 0;
2588                 } else {
2589                         sb = (-(4 << sa)) & 0xFE;
2590                         sa = (1 << sa) & 0x3F;
2591                 }
2592
2593                 sitd_mask = (EHCI_SITD_SET_SMASK(sa) |
2594                     EHCI_SITD_SET_CMASK(sb));
2595
2596                 td->sitd_bp[1] = htohc32(sc, temp);
2597
2598                 td->sitd_mask = htohc32(sc, sitd_mask);
2599
2600                 if (nframes == 0) {
2601                         td->sitd_status = htohc32(sc,
2602                             EHCI_SITD_IOC |
2603                             EHCI_SITD_ACTIVE |
2604                             EHCI_SITD_SET_LEN(*plen));
2605                 } else {
2606                         td->sitd_status = htohc32(sc,
2607                             EHCI_SITD_ACTIVE |
2608                             EHCI_SITD_SET_LEN(*plen));
2609                 }
2610                 usb_pc_cpu_flush(td->page_cache);
2611
2612 #if USB_DEBUG
2613                 if (ehcidebug > 15) {
2614                         DPRINTF("FS-TD %d\n", nframes);
2615                         ehci_dump_sitd(sc, td);
2616                 }
2617 #endif
2618                 /* insert TD into schedule */
2619                 EHCI_APPEND_FS_TD(td, *pp_last);
2620                 pp_last++;
2621
2622                 plen++;
2623                 fss++;
2624                 td_last = td;
2625                 td = td->obj_next;
2626         }
2627
2628         xfer->td_transfer_last = td_last;
2629
2630         /* update isoc_next */
2631         xfer->endpoint->isoc_next = (pp_last - &sc->sc_isoc_fs_p_last[0]) &
2632             (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2633 }
2634
2635 static void
2636 ehci_device_isoc_fs_start(struct usb_xfer *xfer)
2637 {
2638         /* put transfer on interrupt queue */
2639         ehci_transfer_intr_enqueue(xfer);
2640 }
2641
2642 struct usb_pipe_methods ehci_device_isoc_fs_methods =
2643 {
2644         .open = ehci_device_isoc_fs_open,
2645         .close = ehci_device_isoc_fs_close,
2646         .enter = ehci_device_isoc_fs_enter,
2647         .start = ehci_device_isoc_fs_start,
2648 };
2649
2650 /*------------------------------------------------------------------------*
2651  * ehci high speed isochronous support
2652  *------------------------------------------------------------------------*/
2653 static void
2654 ehci_device_isoc_hs_open(struct usb_xfer *xfer)
2655 {
2656         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2657         ehci_itd_t *td;
2658         uint32_t temp;
2659         uint8_t ds;
2660
2661         /* initialize all TD's */
2662
2663         for (ds = 0; ds != 2; ds++) {
2664
2665                 for (td = xfer->td_start[ds]; td; td = td->obj_next) {
2666
2667                         /* set TD inactive */
2668                         td->itd_status[0] = 0;
2669                         td->itd_status[1] = 0;
2670                         td->itd_status[2] = 0;
2671                         td->itd_status[3] = 0;
2672                         td->itd_status[4] = 0;
2673                         td->itd_status[5] = 0;
2674                         td->itd_status[6] = 0;
2675                         td->itd_status[7] = 0;
2676
2677                         /* set endpoint and address */
2678                         td->itd_bp[0] = htohc32(sc,
2679                             EHCI_ITD_SET_ADDR(xfer->address) |
2680                             EHCI_ITD_SET_ENDPT(UE_GET_ADDR(xfer->endpointno)));
2681
2682                         temp =
2683                             EHCI_ITD_SET_MPL(xfer->max_packet_size & 0x7FF);
2684
2685                         /* set direction */
2686                         if (UE_GET_DIR(xfer->endpointno) == UE_DIR_IN) {
2687                                 temp |= EHCI_ITD_SET_DIR_IN;
2688                         }
2689                         /* set maximum packet size */
2690                         td->itd_bp[1] = htohc32(sc, temp);
2691
2692                         /* set transfer multiplier */
2693                         td->itd_bp[2] = htohc32(sc, xfer->max_packet_count & 3);
2694
2695                         usb_pc_cpu_flush(td->page_cache);
2696                 }
2697         }
2698 }
2699
2700 static void
2701 ehci_device_isoc_hs_close(struct usb_xfer *xfer)
2702 {
2703         ehci_device_done(xfer, USB_ERR_CANCELLED);
2704 }
2705
2706 static void
2707 ehci_device_isoc_hs_enter(struct usb_xfer *xfer)
2708 {
2709         struct usb_page_search buf_res;
2710         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2711         ehci_itd_t *td;
2712         ehci_itd_t *td_last = NULL;
2713         ehci_itd_t **pp_last;
2714         bus_size_t page_addr;
2715         uint32_t *plen;
2716         uint32_t status;
2717         uint32_t buf_offset;
2718         uint32_t nframes;
2719         uint32_t itd_offset[8 + 1];
2720         uint8_t x;
2721         uint8_t td_no;
2722         uint8_t page_no;
2723
2724 #if USB_DEBUG
2725         uint8_t once = 1;
2726
2727 #endif
2728
2729         DPRINTFN(6, "xfer=%p next=%d nframes=%d\n",
2730             xfer, xfer->endpoint->isoc_next, xfer->nframes);
2731
2732         /* get the current frame index */
2733
2734         nframes = EOREAD4(sc, EHCI_FRINDEX) / 8;
2735
2736         /*
2737          * check if the frame index is within the window where the frames
2738          * will be inserted
2739          */
2740         buf_offset = (nframes - xfer->endpoint->isoc_next) &
2741             (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2742
2743         if ((xfer->endpoint->is_synced == 0) ||
2744             (buf_offset < ((xfer->nframes + 7) / 8))) {
2745                 /*
2746                  * If there is data underflow or the pipe queue is empty we
2747                  * schedule the transfer a few frames ahead of the current
2748                  * frame position. Else two isochronous transfers might
2749                  * overlap.
2750                  */
2751                 xfer->endpoint->isoc_next = (nframes + 3) &
2752                     (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2753                 xfer->endpoint->is_synced = 1;
2754                 DPRINTFN(3, "start next=%d\n", xfer->endpoint->isoc_next);
2755         }
2756         /*
2757          * compute how many milliseconds the insertion is ahead of the
2758          * current frame position:
2759          */
2760         buf_offset = (xfer->endpoint->isoc_next - nframes) &
2761             (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2762
2763         /*
2764          * pre-compute when the isochronous transfer will be finished:
2765          */
2766         xfer->isoc_time_complete =
2767             usb_isoc_time_expand(&sc->sc_bus, nframes) + buf_offset +
2768             ((xfer->nframes + 7) / 8);
2769
2770         /* get the real number of frames */
2771
2772         nframes = xfer->nframes;
2773
2774         buf_offset = 0;
2775         td_no = 0;
2776
2777         plen = xfer->frlengths;
2778
2779         /* toggle the DMA set we are using */
2780         xfer->flags_int.curr_dma_set ^= 1;
2781
2782         /* get next DMA set */
2783         td = xfer->td_start[xfer->flags_int.curr_dma_set];
2784         xfer->td_transfer_first = td;
2785
2786         pp_last = &sc->sc_isoc_hs_p_last[xfer->endpoint->isoc_next];
2787
2788         /* store starting position */
2789
2790         xfer->qh_pos = xfer->endpoint->isoc_next;
2791
2792         while (nframes--) {
2793                 if (td == NULL) {
2794                         panic("%s:%d: out of TD's\n",
2795                             __FUNCTION__, __LINE__);
2796                 }
2797                 if (pp_last >= &sc->sc_isoc_hs_p_last[EHCI_VIRTUAL_FRAMELIST_COUNT]) {
2798                         pp_last = &sc->sc_isoc_hs_p_last[0];
2799                 }
2800                 /* range check */
2801                 if (*plen > xfer->max_frame_size) {
2802 #if USB_DEBUG
2803                         if (once) {
2804                                 once = 0;
2805                                 printf("%s: frame length(%d) exceeds %d bytes "
2806                                     "(frame truncated)\n",
2807                                     __FUNCTION__, *plen, xfer->max_frame_size);
2808                         }
2809 #endif
2810                         *plen = xfer->max_frame_size;
2811                 }
2812                 status = (EHCI_ITD_SET_LEN(*plen) |
2813                     EHCI_ITD_ACTIVE |
2814                     EHCI_ITD_SET_PG(0));
2815                 td->itd_status[td_no] = htohc32(sc, status);
2816                 itd_offset[td_no] = buf_offset;
2817                 buf_offset += *plen;
2818                 plen++;
2819                 td_no++;
2820
2821                 if ((td_no == 8) || (nframes == 0)) {
2822
2823                         /* the rest of the transfers are not active, if any */
2824                         for (x = td_no; x != 8; x++) {
2825                                 td->itd_status[x] = 0;  /* not active */
2826                         }
2827
2828                         /* check if there is any data to be transferred */
2829                         if (itd_offset[0] != buf_offset) {
2830                                 page_no = 0;
2831                                 itd_offset[td_no] = buf_offset;
2832
2833                                 /* get first page offset */
2834                                 usbd_get_page(xfer->frbuffers, itd_offset[0], &buf_res);
2835                                 /* get page address */
2836                                 page_addr = buf_res.physaddr & ~0xFFF;
2837                                 /* update page address */
2838                                 td->itd_bp[0] &= htohc32(sc, 0xFFF);
2839                                 td->itd_bp[0] |= htohc32(sc, page_addr);
2840
2841                                 for (x = 0; x != td_no; x++) {
2842                                         /* set page number and page offset */
2843                                         status = (EHCI_ITD_SET_PG(page_no) |
2844                                             (buf_res.physaddr & 0xFFF));
2845                                         td->itd_status[x] |= htohc32(sc, status);
2846
2847                                         /* get next page offset */
2848                                         if (itd_offset[x + 1] == buf_offset) {
2849                                                 /*
2850                                                  * We subtract one so that
2851                                                  * we don't go off the last
2852                                                  * page!
2853                                                  */
2854                                                 usbd_get_page(xfer->frbuffers, buf_offset - 1, &buf_res);
2855                                         } else {
2856                                                 usbd_get_page(xfer->frbuffers, itd_offset[x + 1], &buf_res);
2857                                         }
2858
2859                                         /* check if we need a new page */
2860                                         if ((buf_res.physaddr ^ page_addr) & ~0xFFF) {
2861                                                 /* new page needed */
2862                                                 page_addr = buf_res.physaddr & ~0xFFF;
2863                                                 if (page_no == 6) {
2864                                                         panic("%s: too many pages\n", __FUNCTION__);
2865                                                 }
2866                                                 page_no++;
2867                                                 /* update page address */
2868                                                 td->itd_bp[page_no] &= htohc32(sc, 0xFFF);
2869                                                 td->itd_bp[page_no] |= htohc32(sc, page_addr);
2870                                         }
2871                                 }
2872                         }
2873                         /* set IOC bit if we are complete */
2874                         if (nframes == 0) {
2875                                 td->itd_status[7] |= htohc32(sc, EHCI_ITD_IOC);
2876                         }
2877                         usb_pc_cpu_flush(td->page_cache);
2878 #if USB_DEBUG
2879                         if (ehcidebug > 15) {
2880                                 DPRINTF("HS-TD %d\n", nframes);
2881                                 ehci_dump_itd(sc, td);
2882                         }
2883 #endif
2884                         /* insert TD into schedule */
2885                         EHCI_APPEND_HS_TD(td, *pp_last);
2886                         pp_last++;
2887
2888                         td_no = 0;
2889                         td_last = td;
2890                         td = td->obj_next;
2891                 }
2892         }
2893
2894         xfer->td_transfer_last = td_last;
2895
2896         /* update isoc_next */
2897         xfer->endpoint->isoc_next = (pp_last - &sc->sc_isoc_hs_p_last[0]) &
2898             (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2899 }
2900
2901 static void
2902 ehci_device_isoc_hs_start(struct usb_xfer *xfer)
2903 {
2904         /* put transfer on interrupt queue */
2905         ehci_transfer_intr_enqueue(xfer);
2906 }
2907
2908 struct usb_pipe_methods ehci_device_isoc_hs_methods =
2909 {
2910         .open = ehci_device_isoc_hs_open,
2911         .close = ehci_device_isoc_hs_close,
2912         .enter = ehci_device_isoc_hs_enter,
2913         .start = ehci_device_isoc_hs_start,
2914 };
2915
2916 /*------------------------------------------------------------------------*
2917  * ehci root control support
2918  *------------------------------------------------------------------------*
2919  * Simulate a hardware hub by handling all the necessary requests.
2920  *------------------------------------------------------------------------*/
2921
2922 static const
2923 struct usb_device_descriptor ehci_devd =
2924 {
2925         sizeof(struct usb_device_descriptor),
2926         UDESC_DEVICE,                   /* type */
2927         {0x00, 0x02},                   /* USB version */
2928         UDCLASS_HUB,                    /* class */
2929         UDSUBCLASS_HUB,                 /* subclass */
2930         UDPROTO_HSHUBSTT,               /* protocol */
2931         64,                             /* max packet */
2932         {0}, {0}, {0x00, 0x01},         /* device id */
2933         1, 2, 0,                        /* string indicies */
2934         1                               /* # of configurations */
2935 };
2936
2937 static const
2938 struct usb_device_qualifier ehci_odevd =
2939 {
2940         sizeof(struct usb_device_qualifier),
2941         UDESC_DEVICE_QUALIFIER,         /* type */
2942         {0x00, 0x02},                   /* USB version */
2943         UDCLASS_HUB,                    /* class */
2944         UDSUBCLASS_HUB,                 /* subclass */
2945         UDPROTO_FSHUB,                  /* protocol */
2946         0,                              /* max packet */
2947         0,                              /* # of configurations */
2948         0
2949 };
2950
2951 static const struct ehci_config_desc ehci_confd = {
2952         .confd = {
2953                 .bLength = sizeof(struct usb_config_descriptor),
2954                 .bDescriptorType = UDESC_CONFIG,
2955                 .wTotalLength[0] = sizeof(ehci_confd),
2956                 .bNumInterface = 1,
2957                 .bConfigurationValue = 1,
2958                 .iConfiguration = 0,
2959                 .bmAttributes = UC_SELF_POWERED,
2960                 .bMaxPower = 0          /* max power */
2961         },
2962         .ifcd = {
2963                 .bLength = sizeof(struct usb_interface_descriptor),
2964                 .bDescriptorType = UDESC_INTERFACE,
2965                 .bNumEndpoints = 1,
2966                 .bInterfaceClass = UICLASS_HUB,
2967                 .bInterfaceSubClass = UISUBCLASS_HUB,
2968                 .bInterfaceProtocol = UIPROTO_HSHUBSTT,
2969                 0
2970         },
2971         .endpd = {
2972                 .bLength = sizeof(struct usb_endpoint_descriptor),
2973                 .bDescriptorType = UDESC_ENDPOINT,
2974                 .bEndpointAddress = UE_DIR_IN | EHCI_INTR_ENDPT,
2975                 .bmAttributes = UE_INTERRUPT,
2976                 .wMaxPacketSize[0] = 8, /* max packet (63 ports) */
2977                 .bInterval = 255,
2978         },
2979 };
2980
2981 static const
2982 struct usb_hub_descriptor ehci_hubd =
2983 {
2984         0,                              /* dynamic length */
2985         UDESC_HUB,
2986         0,
2987         {0, 0},
2988         0,
2989         0,
2990         {0},
2991 };
2992
2993 static void
2994 ehci_disown(ehci_softc_t *sc, uint16_t index, uint8_t lowspeed)
2995 {
2996         uint32_t port;
2997         uint32_t v;
2998
2999         DPRINTF("index=%d lowspeed=%d\n", index, lowspeed);
3000
3001         port = EHCI_PORTSC(index);
3002         v = EOREAD4(sc, port) & ~EHCI_PS_CLEAR;
3003         EOWRITE4(sc, port, v | EHCI_PS_PO);
3004 }
3005
3006 static usb_error_t
3007 ehci_roothub_exec(struct usb_device *udev,
3008     struct usb_device_request *req, const void **pptr, uint16_t *plength)
3009 {
3010         ehci_softc_t *sc = EHCI_BUS2SC(udev->bus);
3011         const char *str_ptr;
3012         const void *ptr;
3013         uint32_t port;
3014         uint32_t v;
3015         uint16_t len;
3016         uint16_t i;
3017         uint16_t value;
3018         uint16_t index;
3019         uint8_t l;
3020         usb_error_t err;
3021
3022         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
3023
3024         /* buffer reset */
3025         ptr = (const void *)&sc->sc_hub_desc;
3026         len = 0;
3027         err = 0;
3028
3029         value = UGETW(req->wValue);
3030         index = UGETW(req->wIndex);
3031
3032         DPRINTFN(3, "type=0x%02x request=0x%02x wLen=0x%04x "
3033             "wValue=0x%04x wIndex=0x%04x\n",
3034             req->bmRequestType, req->bRequest,
3035             UGETW(req->wLength), value, index);
3036
3037 #define C(x,y) ((x) | ((y) << 8))
3038         switch (C(req->bRequest, req->bmRequestType)) {
3039         case C(UR_CLEAR_FEATURE, UT_WRITE_DEVICE):
3040         case C(UR_CLEAR_FEATURE, UT_WRITE_INTERFACE):
3041         case C(UR_CLEAR_FEATURE, UT_WRITE_ENDPOINT):
3042                 /*
3043                  * DEVICE_REMOTE_WAKEUP and ENDPOINT_HALT are no-ops
3044                  * for the integrated root hub.
3045                  */
3046                 break;
3047         case C(UR_GET_CONFIG, UT_READ_DEVICE):
3048                 len = 1;
3049                 sc->sc_hub_desc.temp[0] = sc->sc_conf;
3050                 break;
3051         case C(UR_GET_DESCRIPTOR, UT_READ_DEVICE):
3052                 switch (value >> 8) {
3053                 case UDESC_DEVICE:
3054                         if ((value & 0xff) != 0) {
3055                                 err = USB_ERR_IOERROR;
3056                                 goto done;
3057                         }
3058                         len = sizeof(ehci_devd);
3059                         ptr = (const void *)&ehci_devd;
3060                         break;
3061                         /*
3062                          * We can't really operate at another speed,
3063                          * but the specification says we need this
3064                          * descriptor:
3065                          */
3066                 case UDESC_DEVICE_QUALIFIER:
3067                         if ((value & 0xff) != 0) {
3068                                 err = USB_ERR_IOERROR;
3069                                 goto done;
3070                         }
3071                         len = sizeof(ehci_odevd);
3072                         ptr = (const void *)&ehci_odevd;
3073                         break;
3074
3075                 case UDESC_CONFIG:
3076                         if ((value & 0xff) != 0) {
3077                                 err = USB_ERR_IOERROR;
3078                                 goto done;
3079                         }
3080                         len = sizeof(ehci_confd);
3081                         ptr = (const void *)&ehci_confd;
3082                         break;
3083
3084                 case UDESC_STRING:
3085                         switch (value & 0xff) {
3086                         case 0: /* Language table */
3087                                 str_ptr = "\001";
3088                                 break;
3089
3090                         case 1: /* Vendor */
3091                                 str_ptr = sc->sc_vendor;
3092                                 break;
3093
3094                         case 2: /* Product */
3095                                 str_ptr = "EHCI root HUB";
3096                                 break;
3097
3098                         default:
3099                                 str_ptr = "";
3100                                 break;
3101                         }
3102
3103                         len = usb_make_str_desc(
3104                             sc->sc_hub_desc.temp,
3105                             sizeof(sc->sc_hub_desc.temp),
3106                             str_ptr);
3107                         break;
3108                 default:
3109                         err = USB_ERR_IOERROR;
3110                         goto done;
3111                 }
3112                 break;
3113         case C(UR_GET_INTERFACE, UT_READ_INTERFACE):
3114                 len = 1;
3115                 sc->sc_hub_desc.temp[0] = 0;
3116                 break;
3117         case C(UR_GET_STATUS, UT_READ_DEVICE):
3118                 len = 2;
3119                 USETW(sc->sc_hub_desc.stat.wStatus, UDS_SELF_POWERED);
3120                 break;
3121         case C(UR_GET_STATUS, UT_READ_INTERFACE):
3122         case C(UR_GET_STATUS, UT_READ_ENDPOINT):
3123                 len = 2;
3124                 USETW(sc->sc_hub_desc.stat.wStatus, 0);
3125                 break;
3126         case C(UR_SET_ADDRESS, UT_WRITE_DEVICE):
3127                 if (value >= EHCI_MAX_DEVICES) {
3128                         err = USB_ERR_IOERROR;
3129                         goto done;
3130                 }
3131                 sc->sc_addr = value;
3132                 break;
3133         case C(UR_SET_CONFIG, UT_WRITE_DEVICE):
3134                 if ((value != 0) && (value != 1)) {
3135                         err = USB_ERR_IOERROR;
3136                         goto done;
3137                 }
3138                 sc->sc_conf = value;
3139                 break;
3140         case C(UR_SET_DESCRIPTOR, UT_WRITE_DEVICE):
3141                 break;
3142         case C(UR_SET_FEATURE, UT_WRITE_DEVICE):
3143         case C(UR_SET_FEATURE, UT_WRITE_INTERFACE):
3144         case C(UR_SET_FEATURE, UT_WRITE_ENDPOINT):
3145                 err = USB_ERR_IOERROR;
3146                 goto done;
3147         case C(UR_SET_INTERFACE, UT_WRITE_INTERFACE):
3148                 break;
3149         case C(UR_SYNCH_FRAME, UT_WRITE_ENDPOINT):
3150                 break;
3151                 /* Hub requests */
3152         case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_DEVICE):
3153                 break;
3154         case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_OTHER):
3155                 DPRINTFN(9, "UR_CLEAR_PORT_FEATURE\n");
3156
3157                 if ((index < 1) ||
3158                     (index > sc->sc_noport)) {
3159                         err = USB_ERR_IOERROR;
3160                         goto done;
3161                 }
3162                 port = EHCI_PORTSC(index);
3163                 v = EOREAD4(sc, port) & ~EHCI_PS_CLEAR;
3164                 switch (value) {
3165                 case UHF_PORT_ENABLE:
3166                         EOWRITE4(sc, port, v & ~EHCI_PS_PE);
3167                         break;
3168                 case UHF_PORT_SUSPEND:
3169                         if ((v & EHCI_PS_SUSP) && (!(v & EHCI_PS_FPR))) {
3170
3171                                 /*
3172                                  * waking up a High Speed device is rather
3173                                  * complicated if
3174                                  */
3175                                 EOWRITE4(sc, port, v | EHCI_PS_FPR);
3176                         }
3177                         /* wait 20ms for resume sequence to complete */
3178                         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 50);
3179
3180                         EOWRITE4(sc, port, v & ~(EHCI_PS_SUSP |
3181                             EHCI_PS_FPR | (3 << 10) /* High Speed */ ));
3182
3183                         /* 4ms settle time */
3184                         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 250);
3185                         break;
3186                 case UHF_PORT_POWER:
3187                         EOWRITE4(sc, port, v & ~EHCI_PS_PP);
3188                         break;
3189                 case UHF_PORT_TEST:
3190                         DPRINTFN(3, "clear port test "
3191                             "%d\n", index);
3192                         break;
3193                 case UHF_PORT_INDICATOR:
3194                         DPRINTFN(3, "clear port ind "
3195                             "%d\n", index);
3196                         EOWRITE4(sc, port, v & ~EHCI_PS_PIC);
3197                         break;
3198                 case UHF_C_PORT_CONNECTION:
3199                         EOWRITE4(sc, port, v | EHCI_PS_CSC);
3200                         break;
3201                 case UHF_C_PORT_ENABLE:
3202                         EOWRITE4(sc, port, v | EHCI_PS_PEC);
3203                         break;
3204                 case UHF_C_PORT_SUSPEND:
3205                         EOWRITE4(sc, port, v | EHCI_PS_SUSP);
3206                         break;
3207                 case UHF_C_PORT_OVER_CURRENT:
3208                         EOWRITE4(sc, port, v | EHCI_PS_OCC);
3209                         break;
3210                 case UHF_C_PORT_RESET:
3211                         sc->sc_isreset = 0;
3212                         break;
3213                 default:
3214                         err = USB_ERR_IOERROR;
3215                         goto done;
3216                 }
3217                 break;
3218         case C(UR_GET_DESCRIPTOR, UT_READ_CLASS_DEVICE):
3219                 if ((value & 0xff) != 0) {
3220                         err = USB_ERR_IOERROR;
3221                         goto done;
3222                 }
3223                 v = EOREAD4(sc, EHCI_HCSPARAMS);
3224
3225                 sc->sc_hub_desc.hubd = ehci_hubd;
3226                 sc->sc_hub_desc.hubd.bNbrPorts = sc->sc_noport;
3227                 USETW(sc->sc_hub_desc.hubd.wHubCharacteristics,
3228                     (EHCI_HCS_PPC(v) ? UHD_PWR_INDIVIDUAL : UHD_PWR_NO_SWITCH) |
3229                     (EHCI_HCS_P_INDICATOR(EREAD4(sc, EHCI_HCSPARAMS)) ?
3230                     UHD_PORT_IND : 0));
3231                 /* XXX can't find out? */
3232                 sc->sc_hub_desc.hubd.bPwrOn2PwrGood = 200;
3233                 for (l = 0; l < sc->sc_noport; l++) {
3234                         /* XXX can't find out? */
3235                         sc->sc_hub_desc.hubd.DeviceRemovable[l / 8] &= ~(1 << (l % 8));
3236                 }
3237                 sc->sc_hub_desc.hubd.bDescLength =
3238                     8 + ((sc->sc_noport + 7) / 8);
3239                 len = sc->sc_hub_desc.hubd.bDescLength;
3240                 break;
3241         case C(UR_GET_STATUS, UT_READ_CLASS_DEVICE):
3242                 len = 16;
3243                 bzero(sc->sc_hub_desc.temp, 16);
3244                 break;
3245         case C(UR_GET_STATUS, UT_READ_CLASS_OTHER):
3246                 DPRINTFN(9, "get port status i=%d\n",
3247                     index);
3248                 if ((index < 1) ||
3249                     (index > sc->sc_noport)) {
3250                         err = USB_ERR_IOERROR;
3251                         goto done;
3252                 }
3253                 v = EOREAD4(sc, EHCI_PORTSC(index));
3254                 DPRINTFN(9, "port status=0x%04x\n", v);
3255                 if (sc->sc_flags & (EHCI_SCFLG_FORCESPEED | EHCI_SCFLG_TT)) {
3256                         if ((v & 0xc000000) == 0x8000000)
3257                                 i = UPS_HIGH_SPEED;
3258                         else if ((v & 0xc000000) == 0x4000000)
3259                                 i = UPS_LOW_SPEED;
3260                         else
3261                                 i = 0;
3262                 } else {
3263                         i = UPS_HIGH_SPEED;
3264                 }
3265                 if (v & EHCI_PS_CS)
3266                         i |= UPS_CURRENT_CONNECT_STATUS;
3267                 if (v & EHCI_PS_PE)
3268                         i |= UPS_PORT_ENABLED;
3269                 if ((v & EHCI_PS_SUSP) && !(v & EHCI_PS_FPR))
3270                         i |= UPS_SUSPEND;
3271                 if (v & EHCI_PS_OCA)
3272                         i |= UPS_OVERCURRENT_INDICATOR;
3273                 if (v & EHCI_PS_PR)
3274                         i |= UPS_RESET;
3275                 if (v & EHCI_PS_PP)
3276                         i |= UPS_PORT_POWER;
3277                 USETW(sc->sc_hub_desc.ps.wPortStatus, i);
3278                 i = 0;
3279                 if (v & EHCI_PS_CSC)
3280                         i |= UPS_C_CONNECT_STATUS;
3281                 if (v & EHCI_PS_PEC)
3282                         i |= UPS_C_PORT_ENABLED;
3283                 if (v & EHCI_PS_OCC)
3284                         i |= UPS_C_OVERCURRENT_INDICATOR;
3285                 if (v & EHCI_PS_FPR)
3286                         i |= UPS_C_SUSPEND;
3287                 if (sc->sc_isreset)
3288                         i |= UPS_C_PORT_RESET;
3289                 USETW(sc->sc_hub_desc.ps.wPortChange, i);
3290                 len = sizeof(sc->sc_hub_desc.ps);
3291                 break;
3292         case C(UR_SET_DESCRIPTOR, UT_WRITE_CLASS_DEVICE):
3293                 err = USB_ERR_IOERROR;
3294                 goto done;
3295         case C(UR_SET_FEATURE, UT_WRITE_CLASS_DEVICE):
3296                 break;
3297         case C(UR_SET_FEATURE, UT_WRITE_CLASS_OTHER):
3298                 if ((index < 1) ||
3299                     (index > sc->sc_noport)) {
3300                         err = USB_ERR_IOERROR;
3301                         goto done;
3302                 }
3303                 port = EHCI_PORTSC(index);
3304                 v = EOREAD4(sc, port) & ~EHCI_PS_CLEAR;
3305                 switch (value) {
3306                 case UHF_PORT_ENABLE:
3307                         EOWRITE4(sc, port, v | EHCI_PS_PE);
3308                         break;
3309                 case UHF_PORT_SUSPEND:
3310                         EOWRITE4(sc, port, v | EHCI_PS_SUSP);
3311                         break;
3312                 case UHF_PORT_RESET:
3313                         DPRINTFN(6, "reset port %d\n", index);
3314 #if USB_DEBUG
3315                         if (ehcinohighspeed) {
3316                                 /*
3317                                  * Connect USB device to companion
3318                                  * controller.
3319                                  */
3320                                 ehci_disown(sc, index, 1);
3321                                 break;
3322                         }
3323 #endif
3324                         if (EHCI_PS_IS_LOWSPEED(v) &&
3325                             (sc->sc_flags & EHCI_SCFLG_TT) == 0) {
3326                                 /* Low speed device, give up ownership. */
3327                                 ehci_disown(sc, index, 1);
3328                                 break;
3329                         }
3330                         /* Start reset sequence. */
3331                         v &= ~(EHCI_PS_PE | EHCI_PS_PR);
3332                         EOWRITE4(sc, port, v | EHCI_PS_PR);
3333
3334                         /* Wait for reset to complete. */
3335                         usb_pause_mtx(&sc->sc_bus.bus_mtx,
3336                             USB_MS_TO_TICKS(USB_PORT_ROOT_RESET_DELAY));
3337
3338                         /* Terminate reset sequence. */
3339                         if (!(sc->sc_flags & EHCI_SCFLG_NORESTERM))
3340                                 EOWRITE4(sc, port, v);
3341
3342                         /* Wait for HC to complete reset. */
3343                         usb_pause_mtx(&sc->sc_bus.bus_mtx,
3344                             USB_MS_TO_TICKS(EHCI_PORT_RESET_COMPLETE));
3345
3346                         v = EOREAD4(sc, port);
3347                         DPRINTF("ehci after reset, status=0x%08x\n", v);
3348                         if (v & EHCI_PS_PR) {
3349                                 device_printf(sc->sc_bus.bdev,
3350                                     "port reset timeout\n");
3351                                 err = USB_ERR_TIMEOUT;
3352                                 goto done;
3353                         }
3354                         if (!(v & EHCI_PS_PE) &&
3355                             (sc->sc_flags & EHCI_SCFLG_TT) == 0) {
3356                                 /* Not a high speed device, give up ownership.*/
3357                                 ehci_disown(sc, index, 0);
3358                                 break;
3359                         }
3360                         sc->sc_isreset = 1;
3361                         DPRINTF("ehci port %d reset, status = 0x%08x\n",
3362                             index, v);
3363                         break;
3364
3365                 case UHF_PORT_POWER:
3366                         DPRINTFN(3, "set port power %d\n", index);
3367                         EOWRITE4(sc, port, v | EHCI_PS_PP);
3368                         break;
3369
3370                 case UHF_PORT_TEST:
3371                         DPRINTFN(3, "set port test %d\n", index);
3372                         break;
3373
3374                 case UHF_PORT_INDICATOR:
3375                         DPRINTFN(3, "set port ind %d\n", index);
3376                         EOWRITE4(sc, port, v | EHCI_PS_PIC);
3377                         break;
3378
3379                 default:
3380                         err = USB_ERR_IOERROR;
3381                         goto done;
3382                 }
3383                 break;
3384         case C(UR_CLEAR_TT_BUFFER, UT_WRITE_CLASS_OTHER):
3385         case C(UR_RESET_TT, UT_WRITE_CLASS_OTHER):
3386         case C(UR_GET_TT_STATE, UT_READ_CLASS_OTHER):
3387         case C(UR_STOP_TT, UT_WRITE_CLASS_OTHER):
3388                 break;
3389         default:
3390                 err = USB_ERR_IOERROR;
3391                 goto done;
3392         }
3393 done:
3394         *plength = len;
3395         *pptr = ptr;
3396         return (err);
3397 }
3398
3399 static void
3400 ehci_xfer_setup(struct usb_setup_params *parm)
3401 {
3402         struct usb_page_search page_info;
3403         struct usb_page_cache *pc;
3404         ehci_softc_t *sc;
3405         struct usb_xfer *xfer;
3406         void *last_obj;
3407         uint32_t nqtd;
3408         uint32_t nqh;
3409         uint32_t nsitd;
3410         uint32_t nitd;
3411         uint32_t n;
3412
3413         sc = EHCI_BUS2SC(parm->udev->bus);
3414         xfer = parm->curr_xfer;
3415
3416         nqtd = 0;
3417         nqh = 0;
3418         nsitd = 0;
3419         nitd = 0;
3420
3421         /*
3422          * compute maximum number of some structures
3423          */
3424         if (parm->methods == &ehci_device_ctrl_methods) {
3425
3426                 /*
3427                  * The proof for the "nqtd" formula is illustrated like
3428                  * this:
3429                  *
3430                  * +------------------------------------+
3431                  * |                                    |
3432                  * |         |remainder ->              |
3433                  * |   +-----+---+                      |
3434                  * |   | xxx | x | frm 0                |
3435                  * |   +-----+---++                     |
3436                  * |   | xxx | xx | frm 1               |
3437                  * |   +-----+----+                     |
3438                  * |            ...                     |
3439                  * +------------------------------------+
3440                  *
3441                  * "xxx" means a completely full USB transfer descriptor
3442                  *
3443                  * "x" and "xx" means a short USB packet
3444                  *
3445                  * For the remainder of an USB transfer modulo
3446                  * "max_data_length" we need two USB transfer descriptors.
3447                  * One to transfer the remaining data and one to finalise
3448                  * with a zero length packet in case the "force_short_xfer"
3449                  * flag is set. We only need two USB transfer descriptors in
3450                  * the case where the transfer length of the first one is a
3451                  * factor of "max_frame_size". The rest of the needed USB
3452                  * transfer descriptors is given by the buffer size divided
3453                  * by the maximum data payload.
3454                  */
3455                 parm->hc_max_packet_size = 0x400;
3456                 parm->hc_max_packet_count = 1;
3457                 parm->hc_max_frame_size = EHCI_QTD_PAYLOAD_MAX;
3458                 xfer->flags_int.bdma_enable = 1;
3459
3460                 usbd_transfer_setup_sub(parm);
3461
3462                 nqh = 1;
3463                 nqtd = ((2 * xfer->nframes) + 1 /* STATUS */
3464                     + (xfer->max_data_length / xfer->max_hc_frame_size));
3465
3466         } else if (parm->methods == &ehci_device_bulk_methods) {
3467
3468                 parm->hc_max_packet_size = 0x400;
3469                 parm->hc_max_packet_count = 1;
3470                 parm->hc_max_frame_size = EHCI_QTD_PAYLOAD_MAX;
3471                 xfer->flags_int.bdma_enable = 1;
3472
3473                 usbd_transfer_setup_sub(parm);
3474
3475                 nqh = 1;
3476                 nqtd = ((2 * xfer->nframes)
3477                     + (xfer->max_data_length / xfer->max_hc_frame_size));
3478
3479         } else if (parm->methods == &ehci_device_intr_methods) {
3480
3481                 if (parm->speed == USB_SPEED_HIGH) {
3482                         parm->hc_max_packet_size = 0x400;
3483                         parm->hc_max_packet_count = 3;
3484                 } else if (parm->speed == USB_SPEED_FULL) {
3485                         parm->hc_max_packet_size = USB_FS_BYTES_PER_HS_UFRAME;
3486                         parm->hc_max_packet_count = 1;
3487                 } else {
3488                         parm->hc_max_packet_size = USB_FS_BYTES_PER_HS_UFRAME / 8;
3489                         parm->hc_max_packet_count = 1;
3490                 }
3491
3492                 parm->hc_max_frame_size = EHCI_QTD_PAYLOAD_MAX;
3493                 xfer->flags_int.bdma_enable = 1;
3494
3495                 usbd_transfer_setup_sub(parm);
3496
3497                 nqh = 1;
3498                 nqtd = ((2 * xfer->nframes)
3499                     + (xfer->max_data_length / xfer->max_hc_frame_size));
3500
3501         } else if (parm->methods == &ehci_device_isoc_fs_methods) {
3502
3503                 parm->hc_max_packet_size = 0x3FF;
3504                 parm->hc_max_packet_count = 1;
3505                 parm->hc_max_frame_size = 0x3FF;
3506                 xfer->flags_int.bdma_enable = 1;
3507
3508                 usbd_transfer_setup_sub(parm);
3509
3510                 nsitd = xfer->nframes;
3511
3512         } else if (parm->methods == &ehci_device_isoc_hs_methods) {
3513
3514                 parm->hc_max_packet_size = 0x400;
3515                 parm->hc_max_packet_count = 3;
3516                 parm->hc_max_frame_size = 0xC00;
3517                 xfer->flags_int.bdma_enable = 1;
3518
3519                 usbd_transfer_setup_sub(parm);
3520
3521                 nitd = (xfer->nframes + 7) / 8;
3522
3523         } else {
3524
3525                 parm->hc_max_packet_size = 0x400;
3526                 parm->hc_max_packet_count = 1;
3527                 parm->hc_max_frame_size = 0x400;
3528
3529                 usbd_transfer_setup_sub(parm);
3530         }
3531
3532 alloc_dma_set:
3533
3534         if (parm->err) {
3535                 return;
3536         }
3537         /*
3538          * Allocate queue heads and transfer descriptors
3539          */
3540         last_obj = NULL;
3541
3542         if (usbd_transfer_setup_sub_malloc(
3543             parm, &pc, sizeof(ehci_itd_t),
3544             EHCI_ITD_ALIGN, nitd)) {
3545                 parm->err = USB_ERR_NOMEM;
3546                 return;
3547         }
3548         if (parm->buf) {
3549                 for (n = 0; n != nitd; n++) {
3550                         ehci_itd_t *td;
3551
3552                         usbd_get_page(pc + n, 0, &page_info);
3553
3554                         td = page_info.buffer;
3555
3556                         /* init TD */
3557                         td->itd_self = htohc32(sc, page_info.physaddr | EHCI_LINK_ITD);
3558                         td->obj_next = last_obj;
3559                         td->page_cache = pc + n;
3560
3561                         last_obj = td;
3562
3563                         usb_pc_cpu_flush(pc + n);
3564                 }
3565         }
3566         if (usbd_transfer_setup_sub_malloc(
3567             parm, &pc, sizeof(ehci_sitd_t),
3568             EHCI_SITD_ALIGN, nsitd)) {
3569                 parm->err = USB_ERR_NOMEM;
3570                 return;
3571         }
3572         if (parm->buf) {
3573                 for (n = 0; n != nsitd; n++) {
3574                         ehci_sitd_t *td;
3575
3576                         usbd_get_page(pc + n, 0, &page_info);
3577
3578                         td = page_info.buffer;
3579
3580                         /* init TD */
3581                         td->sitd_self = htohc32(sc, page_info.physaddr | EHCI_LINK_SITD);
3582                         td->obj_next = last_obj;
3583                         td->page_cache = pc + n;
3584
3585                         last_obj = td;
3586
3587                         usb_pc_cpu_flush(pc + n);
3588                 }
3589         }
3590         if (usbd_transfer_setup_sub_malloc(
3591             parm, &pc, sizeof(ehci_qtd_t),
3592             EHCI_QTD_ALIGN, nqtd)) {
3593                 parm->err = USB_ERR_NOMEM;
3594                 return;
3595         }
3596         if (parm->buf) {
3597                 for (n = 0; n != nqtd; n++) {
3598                         ehci_qtd_t *qtd;
3599
3600                         usbd_get_page(pc + n, 0, &page_info);
3601
3602                         qtd = page_info.buffer;
3603
3604                         /* init TD */
3605                         qtd->qtd_self = htohc32(sc, page_info.physaddr);
3606                         qtd->obj_next = last_obj;
3607                         qtd->page_cache = pc + n;
3608
3609                         last_obj = qtd;
3610
3611                         usb_pc_cpu_flush(pc + n);
3612                 }
3613         }
3614         xfer->td_start[xfer->flags_int.curr_dma_set] = last_obj;
3615
3616         last_obj = NULL;
3617
3618         if (usbd_transfer_setup_sub_malloc(
3619             parm, &pc, sizeof(ehci_qh_t),
3620             EHCI_QH_ALIGN, nqh)) {
3621                 parm->err = USB_ERR_NOMEM;
3622                 return;
3623         }
3624         if (parm->buf) {
3625                 for (n = 0; n != nqh; n++) {
3626                         ehci_qh_t *qh;
3627
3628                         usbd_get_page(pc + n, 0, &page_info);
3629
3630                         qh = page_info.buffer;
3631
3632                         /* init QH */
3633                         qh->qh_self = htohc32(sc, page_info.physaddr | EHCI_LINK_QH);
3634                         qh->obj_next = last_obj;
3635                         qh->page_cache = pc + n;
3636
3637                         last_obj = qh;
3638
3639                         usb_pc_cpu_flush(pc + n);
3640                 }
3641         }
3642         xfer->qh_start[xfer->flags_int.curr_dma_set] = last_obj;
3643
3644         if (!xfer->flags_int.curr_dma_set) {
3645                 xfer->flags_int.curr_dma_set = 1;
3646                 goto alloc_dma_set;
3647         }
3648 }
3649
3650 static void
3651 ehci_xfer_unsetup(struct usb_xfer *xfer)
3652 {
3653         return;
3654 }
3655
3656 static void
3657 ehci_ep_init(struct usb_device *udev, struct usb_endpoint_descriptor *edesc,
3658     struct usb_endpoint *ep)
3659 {
3660         ehci_softc_t *sc = EHCI_BUS2SC(udev->bus);
3661
3662         DPRINTFN(2, "endpoint=%p, addr=%d, endpt=%d, mode=%d (%d)\n",
3663             ep, udev->address,
3664             edesc->bEndpointAddress, udev->flags.usb_mode,
3665             sc->sc_addr);
3666
3667         if (udev->flags.usb_mode != USB_MODE_HOST) {
3668                 /* not supported */
3669                 return;
3670         }
3671         if (udev->device_index != sc->sc_addr) {
3672
3673                 if ((udev->speed != USB_SPEED_HIGH) &&
3674                     ((udev->hs_hub_addr == 0) ||
3675                     (udev->hs_port_no == 0) ||
3676                     (udev->parent_hs_hub == NULL) ||
3677                     (udev->parent_hs_hub->hub == NULL))) {
3678                         /* We need a transaction translator */
3679                         goto done;
3680                 }
3681                 switch (edesc->bmAttributes & UE_XFERTYPE) {
3682                 case UE_CONTROL:
3683                         ep->methods = &ehci_device_ctrl_methods;
3684                         break;
3685                 case UE_INTERRUPT:
3686                         ep->methods = &ehci_device_intr_methods;
3687                         break;
3688                 case UE_ISOCHRONOUS:
3689                         if (udev->speed == USB_SPEED_HIGH) {
3690                                 ep->methods = &ehci_device_isoc_hs_methods;
3691                         } else if (udev->speed == USB_SPEED_FULL) {
3692                                 ep->methods = &ehci_device_isoc_fs_methods;
3693                         }
3694                         break;
3695                 case UE_BULK:
3696                         if (udev->speed != USB_SPEED_LOW) {
3697                                 ep->methods = &ehci_device_bulk_methods;
3698                         }
3699                         break;
3700                 default:
3701                         /* do nothing */
3702                         break;
3703                 }
3704         }
3705 done:
3706         return;
3707 }
3708
3709 static void
3710 ehci_get_dma_delay(struct usb_bus *bus, uint32_t *pus)
3711 {
3712         /*
3713          * Wait until the hardware has finished any possible use of
3714          * the transfer descriptor(s) and QH
3715          */
3716         *pus = (188);                   /* microseconds */
3717 }
3718
3719 static void
3720 ehci_device_resume(struct usb_device *udev)
3721 {
3722         ehci_softc_t *sc = EHCI_BUS2SC(udev->bus);
3723         struct usb_xfer *xfer;
3724         struct usb_pipe_methods *methods;
3725
3726         DPRINTF("\n");
3727
3728         USB_BUS_LOCK(udev->bus);
3729
3730         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
3731
3732                 if (xfer->xroot->udev == udev) {
3733
3734                         methods = xfer->endpoint->methods;
3735
3736                         if ((methods == &ehci_device_bulk_methods) ||
3737                             (methods == &ehci_device_ctrl_methods)) {
3738                                 EHCI_APPEND_QH(xfer->qh_start[xfer->flags_int.curr_dma_set],
3739                                     sc->sc_async_p_last);
3740                         }
3741                         if (methods == &ehci_device_intr_methods) {
3742                                 EHCI_APPEND_QH(xfer->qh_start[xfer->flags_int.curr_dma_set],
3743                                     sc->sc_intr_p_last[xfer->qh_pos]);
3744                         }
3745                 }
3746         }
3747
3748         USB_BUS_UNLOCK(udev->bus);
3749
3750         return;
3751 }
3752
3753 static void
3754 ehci_device_suspend(struct usb_device *udev)
3755 {
3756         ehci_softc_t *sc = EHCI_BUS2SC(udev->bus);
3757         struct usb_xfer *xfer;
3758         struct usb_pipe_methods *methods;
3759
3760         DPRINTF("\n");
3761
3762         USB_BUS_LOCK(udev->bus);
3763
3764         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
3765
3766                 if (xfer->xroot->udev == udev) {
3767
3768                         methods = xfer->endpoint->methods;
3769
3770                         if ((methods == &ehci_device_bulk_methods) ||
3771                             (methods == &ehci_device_ctrl_methods)) {
3772                                 EHCI_REMOVE_QH(xfer->qh_start[xfer->flags_int.curr_dma_set],
3773                                     sc->sc_async_p_last);
3774                         }
3775                         if (methods == &ehci_device_intr_methods) {
3776                                 EHCI_REMOVE_QH(xfer->qh_start[xfer->flags_int.curr_dma_set],
3777                                     sc->sc_intr_p_last[xfer->qh_pos]);
3778                         }
3779                 }
3780         }
3781
3782         USB_BUS_UNLOCK(udev->bus);
3783
3784         return;
3785 }
3786
3787 static void
3788 ehci_set_hw_power(struct usb_bus *bus)
3789 {
3790         ehci_softc_t *sc = EHCI_BUS2SC(bus);
3791         uint32_t temp;
3792         uint32_t flags;
3793
3794         DPRINTF("\n");
3795
3796         USB_BUS_LOCK(bus);
3797
3798         flags = bus->hw_power_state;
3799
3800         temp = EOREAD4(sc, EHCI_USBCMD);
3801
3802         temp &= ~(EHCI_CMD_ASE | EHCI_CMD_PSE);
3803
3804         if (flags & (USB_HW_POWER_CONTROL |
3805             USB_HW_POWER_BULK)) {
3806                 DPRINTF("Async is active\n");
3807                 temp |= EHCI_CMD_ASE;
3808         }
3809         if (flags & (USB_HW_POWER_INTERRUPT |
3810             USB_HW_POWER_ISOC)) {
3811                 DPRINTF("Periodic is active\n");
3812                 temp |= EHCI_CMD_PSE;
3813         }
3814         EOWRITE4(sc, EHCI_USBCMD, temp);
3815
3816         USB_BUS_UNLOCK(bus);
3817
3818         return;
3819 }
3820
3821 struct usb_bus_methods ehci_bus_methods =
3822 {
3823         .endpoint_init = ehci_ep_init,
3824         .xfer_setup = ehci_xfer_setup,
3825         .xfer_unsetup = ehci_xfer_unsetup,
3826         .get_dma_delay = ehci_get_dma_delay,
3827         .device_resume = ehci_device_resume,
3828         .device_suspend = ehci_device_suspend,
3829         .set_hw_power = ehci_set_hw_power,
3830         .roothub_exec = ehci_roothub_exec,
3831         .xfer_poll = ehci_do_poll,
3832 };