sys/dev/usb/controller/ehci.c

   1 /*-
   2  * Copyright (c) 2008 Hans Petter Selasky. All rights reserved.
   3  * Copyright (c) 2004 The NetBSD Foundation, Inc. All rights reserved.
   4  * Copyright (c) 2004 Lennart Augustsson. All rights reserved.
   5  * Copyright (c) 2004 Charles M. Hannum. All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  26  * SUCH DAMAGE.
  27  */
  28
  29 /*
  30  * USB Enhanced Host Controller Driver, a.k.a. USB 2.0 controller.
  31  *
  32  * The EHCI 0.96 spec can be found at
  33  * http://developer.intel.com/technology/usb/download/ehci-r096.pdf
  34  * The EHCI 1.0 spec can be found at
  35  * http://developer.intel.com/technology/usb/download/ehci-r10.pdf
  36  * and the USB 2.0 spec at
  37  * http://www.usb.org/developers/docs/usb_20.zip
  38  *
  39  */
  40
  41 /*
  42  * TODO:
  43  * 1) command failures are not recovered correctly
  44  */
  45
  46 #include <sys/cdefs.h>
  47 __FBSDID("$FreeBSD$");
  48
  49 #include <sys/stdint.h>
  50 #include <sys/stddef.h>
  51 #include <sys/param.h>
  52 #include <sys/queue.h>
  53 #include <sys/types.h>
  54 #include <sys/systm.h>
  55 #include <sys/kernel.h>
  56 #include <sys/bus.h>
  57 #include <sys/linker_set.h>
  58 #include <sys/module.h>
  59 #include <sys/lock.h>
  60 #include <sys/mutex.h>
  61 #include <sys/condvar.h>
  62 #include <sys/sysctl.h>
  63 #include <sys/sx.h>
  64 #include <sys/unistd.h>
  65 #include <sys/callout.h>
  66 #include <sys/malloc.h>
  67 #include <sys/priv.h>
  68
  69 #include <dev/usb/usb.h>
  70 #include <dev/usb/usbdi.h>
  71
  72 #define USB_DEBUG_VAR ehcidebug
  73
  74 #include <dev/usb/usb_core.h>
  75 #include <dev/usb/usb_debug.h>
  76 #include <dev/usb/usb_busdma.h>
  77 #include <dev/usb/usb_process.h>
  78 #include <dev/usb/usb_transfer.h>
  79 #include <dev/usb/usb_device.h>
  80 #include <dev/usb/usb_hub.h>
  81 #include <dev/usb/usb_util.h>
  82
  83 #include <dev/usb/usb_controller.h>
  84 #include <dev/usb/usb_bus.h>
  85 #include <dev/usb/controller/ehci.h>
  86 #include <dev/usb/controller/ehcireg.h>
  87
  88 #define EHCI_BUS2SC(bus) \
  89    ((ehci_softc_t *)(((uint8_t *)(bus)) - \
  90     ((uint8_t *)&(((ehci_softc_t *)0)->sc_bus))))
  91
  92 #ifdef USB_DEBUG
  93 static int ehcidebug = 0;
  94 static int ehcinohighspeed = 0;
  95 static int ehciiaadbug = 0;
  96 static int ehcilostintrbug = 0;
  97
  98 SYSCTL_NODE(_hw_usb, OID_AUTO, ehci, CTLFLAG_RW, 0, "USB ehci");
  99 SYSCTL_INT(_hw_usb_ehci, OID_AUTO, debug, CTLFLAG_RW,
 100     &ehcidebug, 0, "Debug level");
 101 SYSCTL_INT(_hw_usb_ehci, OID_AUTO, no_hs, CTLFLAG_RW,
 102     &ehcinohighspeed, 0, "Disable High Speed USB");
 103 SYSCTL_INT(_hw_usb_ehci, OID_AUTO, iaadbug, CTLFLAG_RW,
 104     &ehciiaadbug, 0, "Enable doorbell bug workaround");
 105 SYSCTL_INT(_hw_usb_ehci, OID_AUTO, lostintrbug, CTLFLAG_RW,
 106     &ehcilostintrbug, 0, "Enable lost interrupt bug workaround");
 107
 108 TUNABLE_INT("hw.usb.ehci.debug", &ehcidebug);
 109 TUNABLE_INT("hw.usb.ehci.no_hs", &ehcinohighspeed);
 110 TUNABLE_INT("hw.usb.ehci.iaadbug", &ehciiaadbug);
 111 TUNABLE_INT("hw.usb.ehci.lostintrbug", &ehcilostintrbug);
 112
 113 static void ehci_dump_regs(ehci_softc_t *sc);
 114 static void ehci_dump_sqh(ehci_softc_t *sc, ehci_qh_t *sqh);
 115
 116 #endif
 117
 118 #define EHCI_INTR_ENDPT 1
 119
 120 extern struct usb_bus_methods ehci_bus_methods;
 121 extern struct usb_pipe_methods ehci_device_bulk_methods;
 122 extern struct usb_pipe_methods ehci_device_ctrl_methods;
 123 extern struct usb_pipe_methods ehci_device_intr_methods;
 124 extern struct usb_pipe_methods ehci_device_isoc_fs_methods;
 125 extern struct usb_pipe_methods ehci_device_isoc_hs_methods;
 126
 127 static void ehci_do_poll(struct usb_bus *);
 128 static void ehci_device_done(struct usb_xfer *, usb_error_t);
 129 static uint8_t ehci_check_transfer(struct usb_xfer *);
 130 static void ehci_timeout(void *);
 131 static void ehci_poll_timeout(void *);
 132
 133 static void ehci_root_intr(ehci_softc_t *sc);
 134
 135 struct ehci_std_temp {
 136         ehci_softc_t *sc;
 137         struct usb_page_cache *pc;
 138         ehci_qtd_t *td;
 139         ehci_qtd_t *td_next;
 140         uint32_t average;
 141         uint32_t qtd_status;
 142         uint32_t len;
 143         uint16_t max_frame_size;
 144         uint8_t shortpkt;
 145         uint8_t auto_data_toggle;
 146         uint8_t setup_alt_next;
 147         uint8_t last_frame;
 148         uint8_t can_use_next;
 149 };
 150
 151 void
 152 ehci_iterate_hw_softc(struct usb_bus *bus, usb_bus_mem_sub_cb_t *cb)
 153 {
 154         ehci_softc_t *sc = EHCI_BUS2SC(bus);
 155         uint32_t i;
 156
 157         cb(bus, &sc->sc_hw.pframes_pc, &sc->sc_hw.pframes_pg,
 158             sizeof(uint32_t) * EHCI_FRAMELIST_COUNT, EHCI_FRAMELIST_ALIGN);
 159
 160         cb(bus, &sc->sc_hw.async_start_pc, &sc->sc_hw.async_start_pg,
 161             sizeof(ehci_qh_t), EHCI_QH_ALIGN);
 162
 163         for (i = 0; i != EHCI_VIRTUAL_FRAMELIST_COUNT; i++) {
 164                 cb(bus, sc->sc_hw.intr_start_pc + i,
 165                     sc->sc_hw.intr_start_pg + i,
 166                     sizeof(ehci_qh_t), EHCI_QH_ALIGN);
 167         }
 168
 169         for (i = 0; i != EHCI_VIRTUAL_FRAMELIST_COUNT; i++) {
 170                 cb(bus, sc->sc_hw.isoc_hs_start_pc + i,
 171                     sc->sc_hw.isoc_hs_start_pg + i,
 172                     sizeof(ehci_itd_t), EHCI_ITD_ALIGN);
 173         }
 174
 175         for (i = 0; i != EHCI_VIRTUAL_FRAMELIST_COUNT; i++) {
 176                 cb(bus, sc->sc_hw.isoc_fs_start_pc + i,
 177                     sc->sc_hw.isoc_fs_start_pg + i,
 178                     sizeof(ehci_sitd_t), EHCI_SITD_ALIGN);
 179         }
 180 }
 181
 182 usb_error_t
 183 ehci_reset(ehci_softc_t *sc)
 184 {
 185         uint32_t hcr;
 186         int i;
 187
 188         EOWRITE4(sc, EHCI_USBCMD, EHCI_CMD_HCRESET);
 189         for (i = 0; i < 100; i++) {
 190                 usb_pause_mtx(NULL, hz / 1000);
 191                 hcr = EOREAD4(sc, EHCI_USBCMD) & EHCI_CMD_HCRESET;
 192                 if (!hcr) {
 193                         if (sc->sc_flags & (EHCI_SCFLG_SETMODE | EHCI_SCFLG_BIGEMMIO)) {
 194                                 /*
 195                                  * Force USBMODE as requested.  Controllers
 196                                  * may have multiple operating modes.
 197                                  */
 198                                 uint32_t usbmode = EOREAD4(sc, EHCI_USBMODE);
 199                                 if (sc->sc_flags & EHCI_SCFLG_SETMODE) {
 200                                         usbmode = (usbmode &~ EHCI_UM_CM) | EHCI_UM_CM_HOST;
 201                                         device_printf(sc->sc_bus.bdev,
 202                                             "set host controller mode\n");
 203                                 }
 204                                 if (sc->sc_flags & EHCI_SCFLG_BIGEMMIO) {
 205                                         usbmode = (usbmode &~ EHCI_UM_ES) | EHCI_UM_ES_BE;
 206                                         device_printf(sc->sc_bus.bdev,
 207                                             "set big-endian mode\n");
 208                                 }
 209                                 EOWRITE4(sc,  EHCI_USBMODE, usbmode);
 210                         }
 211                         return (0);
 212                 }
 213         }
 214         device_printf(sc->sc_bus.bdev, "reset timeout\n");
 215         return (USB_ERR_IOERROR);
 216 }
 217
 218 static usb_error_t
 219 ehci_hcreset(ehci_softc_t *sc)
 220 {
 221         uint32_t hcr;
 222         int i;
 223
 224         EOWRITE4(sc, EHCI_USBCMD, 0);   /* Halt controller */
 225         for (i = 0; i < 100; i++) {
 226                 usb_pause_mtx(NULL, hz / 1000);
 227                 hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
 228                 if (hcr)
 229                         break;
 230         }
 231         if (!hcr)
 232                 /*
 233                  * Fall through and try reset anyway even though
 234                  * Table 2-9 in the EHCI spec says this will result
 235                  * in undefined behavior.
 236                  */
 237                 device_printf(sc->sc_bus.bdev, "stop timeout\n");
 238
 239         return ehci_reset(sc);
 240 }
 241
 242 usb_error_t
 243 ehci_init(ehci_softc_t *sc)
 244 {
 245         struct usb_page_search buf_res;
 246         uint32_t version;
 247         uint32_t sparams;
 248         uint32_t cparams;
 249         uint32_t hcr;
 250         uint16_t i;
 251         uint16_t x;
 252         uint16_t y;
 253         uint16_t bit;
 254         usb_error_t err = 0;
 255
 256         DPRINTF("start\n");
 257
 258         usb_callout_init_mtx(&sc->sc_tmo_pcd, &sc->sc_bus.bus_mtx, 0);
 259         usb_callout_init_mtx(&sc->sc_tmo_poll, &sc->sc_bus.bus_mtx, 0);
 260
 261 #ifdef USB_DEBUG
 262         if (ehciiaadbug)
 263                 sc->sc_flags |= EHCI_SCFLG_IAADBUG;
 264         if (ehcilostintrbug)
 265                 sc->sc_flags |= EHCI_SCFLG_LOSTINTRBUG;
 266         if (ehcidebug > 2) {
 267                 ehci_dump_regs(sc);
 268         }
 269 #endif
 270
 271         sc->sc_offs = EREAD1(sc, EHCI_CAPLENGTH);
 272
 273         version = EREAD2(sc, EHCI_HCIVERSION);
 274         device_printf(sc->sc_bus.bdev, "EHCI version %x.%x\n",
 275             version >> 8, version & 0xff);
 276
 277         sparams = EREAD4(sc, EHCI_HCSPARAMS);
 278         DPRINTF("sparams=0x%x\n", sparams);
 279
 280         sc->sc_noport = EHCI_HCS_N_PORTS(sparams);
 281         cparams = EREAD4(sc, EHCI_HCCPARAMS);
 282         DPRINTF("cparams=0x%x\n", cparams);
 283
 284         if (EHCI_HCC_64BIT(cparams)) {
 285                 DPRINTF("HCC uses 64-bit structures\n");
 286
 287                 /* MUST clear segment register if 64 bit capable */
 288                 EWRITE4(sc, EHCI_CTRLDSSEGMENT, 0);
 289         }
 290         sc->sc_bus.usbrev = USB_REV_2_0;
 291
 292         /* Reset the controller */
 293         DPRINTF("%s: resetting\n", device_get_nameunit(sc->sc_bus.bdev));
 294
 295         err = ehci_hcreset(sc);
 296         if (err) {
 297                 device_printf(sc->sc_bus.bdev, "reset timeout\n");
 298                 return (err);
 299         }
 300         /*
 301          * use current frame-list-size selection 0: 1024*4 bytes 1:  512*4
 302          * bytes 2:  256*4 bytes 3:      unknown
 303          */
 304         if (EHCI_CMD_FLS(EOREAD4(sc, EHCI_USBCMD)) == 3) {
 305                 device_printf(sc->sc_bus.bdev, "invalid frame-list-size\n");
 306                 return (USB_ERR_IOERROR);
 307         }
 308         /* set up the bus struct */
 309         sc->sc_bus.methods = &ehci_bus_methods;
 310
 311         sc->sc_eintrs = EHCI_NORMAL_INTRS;
 312
 313         for (i = 0; i < EHCI_VIRTUAL_FRAMELIST_COUNT; i++) {
 314                 ehci_qh_t *qh;
 315
 316                 usbd_get_page(sc->sc_hw.intr_start_pc + i, 0, &buf_res);
 317
 318                 qh = buf_res.buffer;
 319
 320                 /* initialize page cache pointer */
 321
 322                 qh->page_cache = sc->sc_hw.intr_start_pc + i;
 323
 324                 /* store a pointer to queue head */
 325
 326                 sc->sc_intr_p_last[i] = qh;
 327
 328                 qh->qh_self =
 329                     htohc32(sc, buf_res.physaddr) |
 330                     htohc32(sc, EHCI_LINK_QH);
 331
 332                 qh->qh_endp =
 333                     htohc32(sc, EHCI_QH_SET_EPS(EHCI_QH_SPEED_HIGH));
 334                 qh->qh_endphub =
 335                     htohc32(sc, EHCI_QH_SET_MULT(1));
 336                 qh->qh_curqtd = 0;
 337
 338                 qh->qh_qtd.qtd_next =
 339                     htohc32(sc, EHCI_LINK_TERMINATE);
 340                 qh->qh_qtd.qtd_altnext =
 341                     htohc32(sc, EHCI_LINK_TERMINATE);
 342                 qh->qh_qtd.qtd_status =
 343                     htohc32(sc, EHCI_QTD_HALTED);
 344         }
 345
 346         /*
 347          * the QHs are arranged to give poll intervals that are
 348          * powers of 2 times 1ms
 349          */
 350         bit = EHCI_VIRTUAL_FRAMELIST_COUNT / 2;
 351         while (bit) {
 352                 x = bit;
 353                 while (x & bit) {
 354                         ehci_qh_t *qh_x;
 355                         ehci_qh_t *qh_y;
 356
 357                         y = (x ^ bit) | (bit / 2);
 358
 359                         qh_x = sc->sc_intr_p_last[x];
 360                         qh_y = sc->sc_intr_p_last[y];
 361
 362                         /*
 363                          * the next QH has half the poll interval
 364                          */
 365                         qh_x->qh_link = qh_y->qh_self;
 366
 367                         x++;
 368                 }
 369                 bit >>= 1;
 370         }
 371
 372         if (1) {
 373                 ehci_qh_t *qh;
 374
 375                 qh = sc->sc_intr_p_last[0];
 376
 377                 /* the last (1ms) QH terminates */
 378                 qh->qh_link = htohc32(sc, EHCI_LINK_TERMINATE);
 379         }
 380         for (i = 0; i < EHCI_VIRTUAL_FRAMELIST_COUNT; i++) {
 381                 ehci_sitd_t *sitd;
 382                 ehci_itd_t *itd;
 383
 384                 usbd_get_page(sc->sc_hw.isoc_fs_start_pc + i, 0, &buf_res);
 385
 386                 sitd = buf_res.buffer;
 387
 388                 /* initialize page cache pointer */
 389
 390                 sitd->page_cache = sc->sc_hw.isoc_fs_start_pc + i;
 391
 392                 /* store a pointer to the transfer descriptor */
 393
 394                 sc->sc_isoc_fs_p_last[i] = sitd;
 395
 396                 /* initialize full speed isochronous */
 397
 398                 sitd->sitd_self =
 399                     htohc32(sc, buf_res.physaddr) |
 400                     htohc32(sc, EHCI_LINK_SITD);
 401
 402                 sitd->sitd_back =
 403                     htohc32(sc, EHCI_LINK_TERMINATE);
 404
 405                 sitd->sitd_next =
 406                     sc->sc_intr_p_last[i | (EHCI_VIRTUAL_FRAMELIST_COUNT / 2)]->qh_self;
 407
 408
 409                 usbd_get_page(sc->sc_hw.isoc_hs_start_pc + i, 0, &buf_res);
 410
 411                 itd = buf_res.buffer;
 412
 413                 /* initialize page cache pointer */
 414
 415                 itd->page_cache = sc->sc_hw.isoc_hs_start_pc + i;
 416
 417                 /* store a pointer to the transfer descriptor */
 418
 419                 sc->sc_isoc_hs_p_last[i] = itd;
 420
 421                 /* initialize high speed isochronous */
 422
 423                 itd->itd_self =
 424                     htohc32(sc, buf_res.physaddr) |
 425                     htohc32(sc, EHCI_LINK_ITD);
 426
 427                 itd->itd_next =
 428                     sitd->sitd_self;
 429         }
 430
 431         usbd_get_page(&sc->sc_hw.pframes_pc, 0, &buf_res);
 432
 433         if (1) {
 434                 uint32_t *pframes;
 435
 436                 pframes = buf_res.buffer;
 437
 438                 /*
 439                  * execution order:
 440                  * pframes -> high speed isochronous ->
 441                  *    full speed isochronous -> interrupt QH's
 442                  */
 443                 for (i = 0; i < EHCI_FRAMELIST_COUNT; i++) {
 444                         pframes[i] = sc->sc_isoc_hs_p_last
 445                             [i & (EHCI_VIRTUAL_FRAMELIST_COUNT - 1)]->itd_self;
 446                 }
 447         }
 448         /* setup sync list pointer */
 449         EOWRITE4(sc, EHCI_PERIODICLISTBASE, buf_res.physaddr);
 450
 451         usbd_get_page(&sc->sc_hw.async_start_pc, 0, &buf_res);
 452
 453         if (1) {
 454
 455                 ehci_qh_t *qh;
 456
 457                 qh = buf_res.buffer;
 458
 459                 /* initialize page cache pointer */
 460
 461                 qh->page_cache = &sc->sc_hw.async_start_pc;
 462
 463                 /* store a pointer to the queue head */
 464
 465                 sc->sc_async_p_last = qh;
 466
 467                 /* init dummy QH that starts the async list */
 468
 469                 qh->qh_self =
 470                     htohc32(sc, buf_res.physaddr) |
 471                     htohc32(sc, EHCI_LINK_QH);
 472
 473                 /* fill the QH */
 474                 qh->qh_endp =
 475                     htohc32(sc, EHCI_QH_SET_EPS(EHCI_QH_SPEED_HIGH) | EHCI_QH_HRECL);
 476                 qh->qh_endphub = htohc32(sc, EHCI_QH_SET_MULT(1));
 477                 qh->qh_link = qh->qh_self;
 478                 qh->qh_curqtd = 0;
 479
 480                 /* fill the overlay qTD */
 481                 qh->qh_qtd.qtd_next = htohc32(sc, EHCI_LINK_TERMINATE);
 482                 qh->qh_qtd.qtd_altnext = htohc32(sc, EHCI_LINK_TERMINATE);
 483                 qh->qh_qtd.qtd_status = htohc32(sc, EHCI_QTD_HALTED);
 484         }
 485         /* flush all cache into memory */
 486
 487         usb_bus_mem_flush_all(&sc->sc_bus, &ehci_iterate_hw_softc);
 488
 489 #ifdef USB_DEBUG
 490         if (ehcidebug) {
 491                 ehci_dump_sqh(sc, sc->sc_async_p_last);
 492         }
 493 #endif
 494
 495         /* setup async list pointer */
 496         EOWRITE4(sc, EHCI_ASYNCLISTADDR, buf_res.physaddr | EHCI_LINK_QH);
 497
 498
 499         /* enable interrupts */
 500         EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
 501
 502         /* turn on controller */
 503         EOWRITE4(sc, EHCI_USBCMD,
 504             EHCI_CMD_ITC_1 |            /* 1 microframes interrupt delay */
 505             (EOREAD4(sc, EHCI_USBCMD) & EHCI_CMD_FLS_M) |
 506             EHCI_CMD_ASE |
 507             EHCI_CMD_PSE |
 508             EHCI_CMD_RS);
 509
 510         /* Take over port ownership */
 511         EOWRITE4(sc, EHCI_CONFIGFLAG, EHCI_CONF_CF);
 512
 513         for (i = 0; i < 100; i++) {
 514                 usb_pause_mtx(NULL, hz / 1000);
 515                 hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
 516                 if (!hcr) {
 517                         break;
 518                 }
 519         }
 520         if (hcr) {
 521                 device_printf(sc->sc_bus.bdev, "run timeout\n");
 522                 return (USB_ERR_IOERROR);
 523         }
 524
 525         if (!err) {
 526                 /* catch any lost interrupts */
 527                 ehci_do_poll(&sc->sc_bus);
 528         }
 529         return (err);
 530 }
 531
 532 /*
 533  * shut down the controller when the system is going down
 534  */
 535 void
 536 ehci_detach(ehci_softc_t *sc)
 537 {
 538         USB_BUS_LOCK(&sc->sc_bus);
 539
 540         usb_callout_stop(&sc->sc_tmo_pcd);
 541         usb_callout_stop(&sc->sc_tmo_poll);
 542
 543         EOWRITE4(sc, EHCI_USBINTR, 0);
 544         USB_BUS_UNLOCK(&sc->sc_bus);
 545
 546         if (ehci_hcreset(sc)) {
 547                 DPRINTF("reset failed!\n");
 548         }
 549
 550         /* XXX let stray task complete */
 551         usb_pause_mtx(NULL, hz / 20);
 552
 553         usb_callout_drain(&sc->sc_tmo_pcd);
 554         usb_callout_drain(&sc->sc_tmo_poll);
 555 }
 556
 557 void
 558 ehci_suspend(ehci_softc_t *sc)
 559 {
 560         uint32_t cmd;
 561         uint32_t hcr;
 562         uint8_t i;
 563
 564         USB_BUS_LOCK(&sc->sc_bus);
 565
 566         for (i = 1; i <= sc->sc_noport; i++) {
 567                 cmd = EOREAD4(sc, EHCI_PORTSC(i));
 568                 if (((cmd & EHCI_PS_PO) == 0) &&
 569                     ((cmd & EHCI_PS_PE) == EHCI_PS_PE)) {
 570                         EOWRITE4(sc, EHCI_PORTSC(i),
 571                             cmd | EHCI_PS_SUSP);
 572                 }
 573         }
 574
 575         sc->sc_cmd = EOREAD4(sc, EHCI_USBCMD);
 576
 577         cmd = sc->sc_cmd & ~(EHCI_CMD_ASE | EHCI_CMD_PSE);
 578         EOWRITE4(sc, EHCI_USBCMD, cmd);
 579
 580         for (i = 0; i < 100; i++) {
 581                 hcr = EOREAD4(sc, EHCI_USBSTS) &
 582                     (EHCI_STS_ASS | EHCI_STS_PSS);
 583
 584                 if (hcr == 0) {
 585                         break;
 586                 }
 587                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 1000);
 588         }
 589
 590         if (hcr != 0) {
 591                 device_printf(sc->sc_bus.bdev, "reset timeout\n");
 592         }
 593         cmd &= ~EHCI_CMD_RS;
 594         EOWRITE4(sc, EHCI_USBCMD, cmd);
 595
 596         for (i = 0; i < 100; i++) {
 597                 hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
 598                 if (hcr == EHCI_STS_HCH) {
 599                         break;
 600                 }
 601                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 1000);
 602         }
 603
 604         if (hcr != EHCI_STS_HCH) {
 605                 device_printf(sc->sc_bus.bdev,
 606                     "config timeout\n");
 607         }
 608         USB_BUS_UNLOCK(&sc->sc_bus);
 609 }
 610
 611 void
 612 ehci_resume(ehci_softc_t *sc)
 613 {
 614         struct usb_page_search buf_res;
 615         uint32_t cmd;
 616         uint32_t hcr;
 617         uint8_t i;
 618
 619         USB_BUS_LOCK(&sc->sc_bus);
 620
 621         /* restore things in case the bios doesn't */
 622         EOWRITE4(sc, EHCI_CTRLDSSEGMENT, 0);
 623
 624         usbd_get_page(&sc->sc_hw.pframes_pc, 0, &buf_res);
 625         EOWRITE4(sc, EHCI_PERIODICLISTBASE, buf_res.physaddr);
 626
 627         usbd_get_page(&sc->sc_hw.async_start_pc, 0, &buf_res);
 628         EOWRITE4(sc, EHCI_ASYNCLISTADDR, buf_res.physaddr | EHCI_LINK_QH);
 629
 630         EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
 631
 632         hcr = 0;
 633         for (i = 1; i <= sc->sc_noport; i++) {
 634                 cmd = EOREAD4(sc, EHCI_PORTSC(i));
 635                 if (((cmd & EHCI_PS_PO) == 0) &&
 636                     ((cmd & EHCI_PS_SUSP) == EHCI_PS_SUSP)) {
 637                         EOWRITE4(sc, EHCI_PORTSC(i),
 638                             cmd | EHCI_PS_FPR);
 639                         hcr = 1;
 640                 }
 641         }
 642
 643         if (hcr) {
 644                 usb_pause_mtx(&sc->sc_bus.bus_mtx,
 645                     USB_MS_TO_TICKS(USB_RESUME_WAIT));
 646
 647                 for (i = 1; i <= sc->sc_noport; i++) {
 648                         cmd = EOREAD4(sc, EHCI_PORTSC(i));
 649                         if (((cmd & EHCI_PS_PO) == 0) &&
 650                             ((cmd & EHCI_PS_SUSP) == EHCI_PS_SUSP)) {
 651                                 EOWRITE4(sc, EHCI_PORTSC(i),
 652                                     cmd & ~EHCI_PS_FPR);
 653                         }
 654                 }
 655         }
 656         EOWRITE4(sc, EHCI_USBCMD, sc->sc_cmd);
 657
 658         for (i = 0; i < 100; i++) {
 659                 hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
 660                 if (hcr != EHCI_STS_HCH) {
 661                         break;
 662                 }
 663                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 1000);
 664         }
 665         if (hcr == EHCI_STS_HCH) {
 666                 device_printf(sc->sc_bus.bdev, "config timeout\n");
 667         }
 668
 669         USB_BUS_UNLOCK(&sc->sc_bus);
 670
 671         usb_pause_mtx(NULL,
 672             USB_MS_TO_TICKS(USB_RESUME_WAIT));
 673
 674         /* catch any lost interrupts */
 675         ehci_do_poll(&sc->sc_bus);
 676 }
 677
 678 void
 679 ehci_shutdown(ehci_softc_t *sc)
 680 {
 681         DPRINTF("stopping the HC\n");
 682
 683         if (ehci_hcreset(sc)) {
 684                 DPRINTF("reset failed!\n");
 685         }
 686 }
 687
 688 #ifdef USB_DEBUG
 689 static void
 690 ehci_dump_regs(ehci_softc_t *sc)
 691 {
 692         uint32_t i;
 693
 694         i = EOREAD4(sc, EHCI_USBCMD);
 695         printf("cmd=0x%08x\n", i);
 696
 697         if (i & EHCI_CMD_ITC_1)
 698                 printf(" EHCI_CMD_ITC_1\n");
 699         if (i & EHCI_CMD_ITC_2)
 700                 printf(" EHCI_CMD_ITC_2\n");
 701         if (i & EHCI_CMD_ITC_4)
 702                 printf(" EHCI_CMD_ITC_4\n");
 703         if (i & EHCI_CMD_ITC_8)
 704                 printf(" EHCI_CMD_ITC_8\n");
 705         if (i & EHCI_CMD_ITC_16)
 706                 printf(" EHCI_CMD_ITC_16\n");
 707         if (i & EHCI_CMD_ITC_32)
 708                 printf(" EHCI_CMD_ITC_32\n");
 709         if (i & EHCI_CMD_ITC_64)
 710                 printf(" EHCI_CMD_ITC_64\n");
 711         if (i & EHCI_CMD_ASPME)
 712                 printf(" EHCI_CMD_ASPME\n");
 713         if (i & EHCI_CMD_ASPMC)
 714                 printf(" EHCI_CMD_ASPMC\n");
 715         if (i & EHCI_CMD_LHCR)
 716                 printf(" EHCI_CMD_LHCR\n");
 717         if (i & EHCI_CMD_IAAD)
 718                 printf(" EHCI_CMD_IAAD\n");
 719         if (i & EHCI_CMD_ASE)
 720                 printf(" EHCI_CMD_ASE\n");
 721         if (i & EHCI_CMD_PSE)
 722                 printf(" EHCI_CMD_PSE\n");
 723         if (i & EHCI_CMD_FLS_M)
 724                 printf(" EHCI_CMD_FLS_M\n");
 725         if (i & EHCI_CMD_HCRESET)
 726                 printf(" EHCI_CMD_HCRESET\n");
 727         if (i & EHCI_CMD_RS)
 728                 printf(" EHCI_CMD_RS\n");
 729
 730         i = EOREAD4(sc, EHCI_USBSTS);
 731
 732         printf("sts=0x%08x\n", i);
 733
 734         if (i & EHCI_STS_ASS)
 735                 printf(" EHCI_STS_ASS\n");
 736         if (i & EHCI_STS_PSS)
 737                 printf(" EHCI_STS_PSS\n");
 738         if (i & EHCI_STS_REC)
 739                 printf(" EHCI_STS_REC\n");
 740         if (i & EHCI_STS_HCH)
 741                 printf(" EHCI_STS_HCH\n");
 742         if (i & EHCI_STS_IAA)
 743                 printf(" EHCI_STS_IAA\n");
 744         if (i & EHCI_STS_HSE)
 745                 printf(" EHCI_STS_HSE\n");
 746         if (i & EHCI_STS_FLR)
 747                 printf(" EHCI_STS_FLR\n");
 748         if (i & EHCI_STS_PCD)
 749                 printf(" EHCI_STS_PCD\n");
 750         if (i & EHCI_STS_ERRINT)
 751                 printf(" EHCI_STS_ERRINT\n");
 752         if (i & EHCI_STS_INT)
 753                 printf(" EHCI_STS_INT\n");
 754
 755         printf("ien=0x%08x\n",
 756             EOREAD4(sc, EHCI_USBINTR));
 757         printf("frindex=0x%08x ctrdsegm=0x%08x periodic=0x%08x async=0x%08x\n",
 758             EOREAD4(sc, EHCI_FRINDEX),
 759             EOREAD4(sc, EHCI_CTRLDSSEGMENT),
 760             EOREAD4(sc, EHCI_PERIODICLISTBASE),
 761             EOREAD4(sc, EHCI_ASYNCLISTADDR));
 762         for (i = 1; i <= sc->sc_noport; i++) {
 763                 printf("port %d status=0x%08x\n", i,
 764                     EOREAD4(sc, EHCI_PORTSC(i)));
 765         }
 766 }
 767
 768 static void
 769 ehci_dump_link(ehci_softc_t *sc, uint32_t link, int type)
 770 {
 771         link = hc32toh(sc, link);
 772         printf("0x%08x", link);
 773         if (link & EHCI_LINK_TERMINATE)
 774                 printf("<T>");
 775         else {
 776                 printf("<");
 777                 if (type) {
 778                         switch (EHCI_LINK_TYPE(link)) {
 779                         case EHCI_LINK_ITD:
 780                                 printf("ITD");
 781                                 break;
 782                         case EHCI_LINK_QH:
 783                                 printf("QH");
 784                                 break;
 785                         case EHCI_LINK_SITD:
 786                                 printf("SITD");
 787                                 break;
 788                         case EHCI_LINK_FSTN:
 789                                 printf("FSTN");
 790                                 break;
 791                         }
 792                 }
 793                 printf(">");
 794         }
 795 }
 796
 797 static void
 798 ehci_dump_qtd(ehci_softc_t *sc, ehci_qtd_t *qtd)
 799 {
 800         uint32_t s;
 801
 802         printf("  next=");
 803         ehci_dump_link(sc, qtd->qtd_next, 0);
 804         printf(" altnext=");
 805         ehci_dump_link(sc, qtd->qtd_altnext, 0);
 806         printf("\n");
 807         s = hc32toh(sc, qtd->qtd_status);
 808         printf("  status=0x%08x: toggle=%d bytes=0x%x ioc=%d c_page=0x%x\n",
 809             s, EHCI_QTD_GET_TOGGLE(s), EHCI_QTD_GET_BYTES(s),
 810             EHCI_QTD_GET_IOC(s), EHCI_QTD_GET_C_PAGE(s));
 811         printf("    cerr=%d pid=%d stat=%s%s%s%s%s%s%s%s\n",
 812             EHCI_QTD_GET_CERR(s), EHCI_QTD_GET_PID(s),
 813             (s & EHCI_QTD_ACTIVE) ? "ACTIVE" : "NOT_ACTIVE",
 814             (s & EHCI_QTD_HALTED) ? "-HALTED" : "",
 815             (s & EHCI_QTD_BUFERR) ? "-BUFERR" : "",
 816             (s & EHCI_QTD_BABBLE) ? "-BABBLE" : "",
 817             (s & EHCI_QTD_XACTERR) ? "-XACTERR" : "",
 818             (s & EHCI_QTD_MISSEDMICRO) ? "-MISSED" : "",
 819             (s & EHCI_QTD_SPLITXSTATE) ? "-SPLIT" : "",
 820             (s & EHCI_QTD_PINGSTATE) ? "-PING" : "");
 821
 822         for (s = 0; s < 5; s++) {
 823                 printf("  buffer[%d]=0x%08x\n", s,
 824                     hc32toh(sc, qtd->qtd_buffer[s]));
 825         }
 826         for (s = 0; s < 5; s++) {
 827                 printf("  buffer_hi[%d]=0x%08x\n", s,
 828                     hc32toh(sc, qtd->qtd_buffer_hi[s]));
 829         }
 830 }
 831
 832 static uint8_t
 833 ehci_dump_sqtd(ehci_softc_t *sc, ehci_qtd_t *sqtd)
 834 {
 835         uint8_t temp;
 836
 837         usb_pc_cpu_invalidate(sqtd->page_cache);
 838         printf("QTD(%p) at 0x%08x:\n", sqtd, hc32toh(sc, sqtd->qtd_self));
 839         ehci_dump_qtd(sc, sqtd);
 840         temp = (sqtd->qtd_next & htohc32(sc, EHCI_LINK_TERMINATE)) ? 1 : 0;
 841         return (temp);
 842 }
 843
 844 static void
 845 ehci_dump_sqtds(ehci_softc_t *sc, ehci_qtd_t *sqtd)
 846 {
 847         uint16_t i;
 848         uint8_t stop;
 849
 850         stop = 0;
 851         for (i = 0; sqtd && (i < 20) && !stop; sqtd = sqtd->obj_next, i++) {
 852                 stop = ehci_dump_sqtd(sc, sqtd);
 853         }
 854         if (sqtd) {
 855                 printf("dump aborted, too many TDs\n");
 856         }
 857 }
 858
 859 static void
 860 ehci_dump_sqh(ehci_softc_t *sc, ehci_qh_t *qh)
 861 {
 862         uint32_t endp;
 863         uint32_t endphub;
 864
 865         usb_pc_cpu_invalidate(qh->page_cache);
 866         printf("QH(%p) at 0x%08x:\n", qh, hc32toh(sc, qh->qh_self) & ~0x1F);
 867         printf("  link=");
 868         ehci_dump_link(sc, qh->qh_link, 1);
 869         printf("\n");
 870         endp = hc32toh(sc, qh->qh_endp);
 871         printf("  endp=0x%08x\n", endp);
 872         printf("    addr=0x%02x inact=%d endpt=%d eps=%d dtc=%d hrecl=%d\n",
 873             EHCI_QH_GET_ADDR(endp), EHCI_QH_GET_INACT(endp),
 874             EHCI_QH_GET_ENDPT(endp), EHCI_QH_GET_EPS(endp),
 875             EHCI_QH_GET_DTC(endp), EHCI_QH_GET_HRECL(endp));
 876         printf("    mpl=0x%x ctl=%d nrl=%d\n",
 877             EHCI_QH_GET_MPL(endp), EHCI_QH_GET_CTL(endp),
 878             EHCI_QH_GET_NRL(endp));
 879         endphub = hc32toh(sc, qh->qh_endphub);
 880         printf("  endphub=0x%08x\n", endphub);
 881         printf("    smask=0x%02x cmask=0x%02x huba=0x%02x port=%d mult=%d\n",
 882             EHCI_QH_GET_SMASK(endphub), EHCI_QH_GET_CMASK(endphub),
 883             EHCI_QH_GET_HUBA(endphub), EHCI_QH_GET_PORT(endphub),
 884             EHCI_QH_GET_MULT(endphub));
 885         printf("  curqtd=");
 886         ehci_dump_link(sc, qh->qh_curqtd, 0);
 887         printf("\n");
 888         printf("Overlay qTD:\n");
 889         ehci_dump_qtd(sc, (void *)&qh->qh_qtd);
 890 }
 891
 892 static void
 893 ehci_dump_sitd(ehci_softc_t *sc, ehci_sitd_t *sitd)
 894 {
 895         usb_pc_cpu_invalidate(sitd->page_cache);
 896         printf("SITD(%p) at 0x%08x\n", sitd, hc32toh(sc, sitd->sitd_self) & ~0x1F);
 897         printf(" next=0x%08x\n", hc32toh(sc, sitd->sitd_next));
 898         printf(" portaddr=0x%08x dir=%s addr=%d endpt=0x%x port=0x%x huba=0x%x\n",
 899             hc32toh(sc, sitd->sitd_portaddr),
 900             (sitd->sitd_portaddr & htohc32(sc, EHCI_SITD_SET_DIR_IN))
 901             ? "in" : "out",
 902             EHCI_SITD_GET_ADDR(hc32toh(sc, sitd->sitd_portaddr)),
 903             EHCI_SITD_GET_ENDPT(hc32toh(sc, sitd->sitd_portaddr)),
 904             EHCI_SITD_GET_PORT(hc32toh(sc, sitd->sitd_portaddr)),
 905             EHCI_SITD_GET_HUBA(hc32toh(sc, sitd->sitd_portaddr)));
 906         printf(" mask=0x%08x\n", hc32toh(sc, sitd->sitd_mask));
 907         printf(" status=0x%08x <%s> len=0x%x\n", hc32toh(sc, sitd->sitd_status),
 908             (sitd->sitd_status & htohc32(sc, EHCI_SITD_ACTIVE)) ? "ACTIVE" : "",
 909             EHCI_SITD_GET_LEN(hc32toh(sc, sitd->sitd_status)));
 910         printf(" back=0x%08x, bp=0x%08x,0x%08x,0x%08x,0x%08x\n",
 911             hc32toh(sc, sitd->sitd_back),
 912             hc32toh(sc, sitd->sitd_bp[0]),
 913             hc32toh(sc, sitd->sitd_bp[1]),
 914             hc32toh(sc, sitd->sitd_bp_hi[0]),
 915             hc32toh(sc, sitd->sitd_bp_hi[1]));
 916 }
 917
 918 static void
 919 ehci_dump_itd(ehci_softc_t *sc, ehci_itd_t *itd)
 920 {
 921         usb_pc_cpu_invalidate(itd->page_cache);
 922         printf("ITD(%p) at 0x%08x\n", itd, hc32toh(sc, itd->itd_self) & ~0x1F);
 923         printf(" next=0x%08x\n", hc32toh(sc, itd->itd_next));
 924         printf(" status[0]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[0]),
 925             (itd->itd_status[0] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
 926         printf(" status[1]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[1]),
 927             (itd->itd_status[1] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
 928         printf(" status[2]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[2]),
 929             (itd->itd_status[2] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
 930         printf(" status[3]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[3]),
 931             (itd->itd_status[3] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
 932         printf(" status[4]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[4]),
 933             (itd->itd_status[4] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
 934         printf(" status[5]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[5]),
 935             (itd->itd_status[5] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
 936         printf(" status[6]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[6]),
 937             (itd->itd_status[6] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
 938         printf(" status[7]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[7]),
 939             (itd->itd_status[7] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
 940         printf(" bp[0]=0x%08x\n", hc32toh(sc, itd->itd_bp[0]));
 941         printf("  addr=0x%02x; endpt=0x%01x\n",
 942             EHCI_ITD_GET_ADDR(hc32toh(sc, itd->itd_bp[0])),
 943             EHCI_ITD_GET_ENDPT(hc32toh(sc, itd->itd_bp[0])));
 944         printf(" bp[1]=0x%08x\n", hc32toh(sc, itd->itd_bp[1]));
 945         printf(" dir=%s; mpl=0x%02x\n",
 946             (hc32toh(sc, itd->itd_bp[1]) & EHCI_ITD_SET_DIR_IN) ? "in" : "out",
 947             EHCI_ITD_GET_MPL(hc32toh(sc, itd->itd_bp[1])));
 948         printf(" bp[2..6]=0x%08x,0x%08x,0x%08x,0x%08x,0x%08x\n",
 949             hc32toh(sc, itd->itd_bp[2]),
 950             hc32toh(sc, itd->itd_bp[3]),
 951             hc32toh(sc, itd->itd_bp[4]),
 952             hc32toh(sc, itd->itd_bp[5]),
 953             hc32toh(sc, itd->itd_bp[6]));
 954         printf(" bp_hi=0x%08x,0x%08x,0x%08x,0x%08x,\n"
 955             "       0x%08x,0x%08x,0x%08x\n",
 956             hc32toh(sc, itd->itd_bp_hi[0]),
 957             hc32toh(sc, itd->itd_bp_hi[1]),
 958             hc32toh(sc, itd->itd_bp_hi[2]),
 959             hc32toh(sc, itd->itd_bp_hi[3]),
 960             hc32toh(sc, itd->itd_bp_hi[4]),
 961             hc32toh(sc, itd->itd_bp_hi[5]),
 962             hc32toh(sc, itd->itd_bp_hi[6]));
 963 }
 964
 965 static void
 966 ehci_dump_isoc(ehci_softc_t *sc)
 967 {
 968         ehci_itd_t *itd;
 969         ehci_sitd_t *sitd;
 970         uint16_t max = 1000;
 971         uint16_t pos;
 972
 973         pos = (EOREAD4(sc, EHCI_FRINDEX) / 8) &
 974             (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
 975
 976         printf("%s: isochronous dump from frame 0x%03x:\n",
 977             __FUNCTION__, pos);
 978
 979         itd = sc->sc_isoc_hs_p_last[pos];
 980         sitd = sc->sc_isoc_fs_p_last[pos];
 981
 982         while (itd && max && max--) {
 983                 ehci_dump_itd(sc, itd);
 984                 itd = itd->prev;
 985         }
 986
 987         while (sitd && max && max--) {
 988                 ehci_dump_sitd(sc, sitd);
 989                 sitd = sitd->prev;
 990         }
 991 }
 992
 993 #endif
 994
 995 static void
 996 ehci_transfer_intr_enqueue(struct usb_xfer *xfer)
 997 {
 998         /* check for early completion */
 999         if (ehci_check_transfer(xfer)) {
1000                 return;
1001         }
1002         /* put transfer on interrupt queue */
1003         usbd_transfer_enqueue(&xfer->xroot->bus->intr_q, xfer);
1004
1005         /* start timeout, if any */
1006         if (xfer->timeout != 0) {
1007                 usbd_transfer_timeout_ms(xfer, &ehci_timeout, xfer->timeout);
1008         }
1009 }
1010
1011 #define EHCI_APPEND_FS_TD(std,last) (last) = _ehci_append_fs_td(std,last)
1012 static ehci_sitd_t *
1013 _ehci_append_fs_td(ehci_sitd_t *std, ehci_sitd_t *last)
1014 {
1015         DPRINTFN(11, "%p to %p\n", std, last);
1016
1017         /* (sc->sc_bus.mtx) must be locked */
1018
1019         std->next = last->next;
1020         std->sitd_next = last->sitd_next;
1021
1022         std->prev = last;
1023
1024         usb_pc_cpu_flush(std->page_cache);
1025
1026         /*
1027          * the last->next->prev is never followed: std->next->prev = std;
1028          */
1029         last->next = std;
1030         last->sitd_next = std->sitd_self;
1031
1032         usb_pc_cpu_flush(last->page_cache);
1033
1034         return (std);
1035 }
1036
1037 #define EHCI_APPEND_HS_TD(std,last) (last) = _ehci_append_hs_td(std,last)
1038 static ehci_itd_t *
1039 _ehci_append_hs_td(ehci_itd_t *std, ehci_itd_t *last)
1040 {
1041         DPRINTFN(11, "%p to %p\n", std, last);
1042
1043         /* (sc->sc_bus.mtx) must be locked */
1044
1045         std->next = last->next;
1046         std->itd_next = last->itd_next;
1047
1048         std->prev = last;
1049
1050         usb_pc_cpu_flush(std->page_cache);
1051
1052         /*
1053          * the last->next->prev is never followed: std->next->prev = std;
1054          */
1055         last->next = std;
1056         last->itd_next = std->itd_self;
1057
1058         usb_pc_cpu_flush(last->page_cache);
1059
1060         return (std);
1061 }
1062
1063 #define EHCI_APPEND_QH(sqh,last) (last) = _ehci_append_qh(sqh,last)
1064 static ehci_qh_t *
1065 _ehci_append_qh(ehci_qh_t *sqh, ehci_qh_t *last)
1066 {
1067         DPRINTFN(11, "%p to %p\n", sqh, last);
1068
1069         if (sqh->prev != NULL) {
1070                 /* should not happen */
1071                 DPRINTFN(0, "QH already linked!\n");
1072                 return (last);
1073         }
1074         /* (sc->sc_bus.mtx) must be locked */
1075
1076         sqh->next = last->next;
1077         sqh->qh_link = last->qh_link;
1078
1079         sqh->prev = last;
1080
1081         usb_pc_cpu_flush(sqh->page_cache);
1082
1083         /*
1084          * the last->next->prev is never followed: sqh->next->prev = sqh;
1085          */
1086
1087         last->next = sqh;
1088         last->qh_link = sqh->qh_self;
1089
1090         usb_pc_cpu_flush(last->page_cache);
1091
1092         return (sqh);
1093 }
1094
1095 #define EHCI_REMOVE_FS_TD(std,last) (last) = _ehci_remove_fs_td(std,last)
1096 static ehci_sitd_t *
1097 _ehci_remove_fs_td(ehci_sitd_t *std, ehci_sitd_t *last)
1098 {
1099         DPRINTFN(11, "%p from %p\n", std, last);
1100
1101         /* (sc->sc_bus.mtx) must be locked */
1102
1103         std->prev->next = std->next;
1104         std->prev->sitd_next = std->sitd_next;
1105
1106         usb_pc_cpu_flush(std->prev->page_cache);
1107
1108         if (std->next) {
1109                 std->next->prev = std->prev;
1110                 usb_pc_cpu_flush(std->next->page_cache);
1111         }
1112         return ((last == std) ? std->prev : last);
1113 }
1114
1115 #define EHCI_REMOVE_HS_TD(std,last) (last) = _ehci_remove_hs_td(std,last)
1116 static ehci_itd_t *
1117 _ehci_remove_hs_td(ehci_itd_t *std, ehci_itd_t *last)
1118 {
1119         DPRINTFN(11, "%p from %p\n", std, last);
1120
1121         /* (sc->sc_bus.mtx) must be locked */
1122
1123         std->prev->next = std->next;
1124         std->prev->itd_next = std->itd_next;
1125
1126         usb_pc_cpu_flush(std->prev->page_cache);
1127
1128         if (std->next) {
1129                 std->next->prev = std->prev;
1130                 usb_pc_cpu_flush(std->next->page_cache);
1131         }
1132         return ((last == std) ? std->prev : last);
1133 }
1134
1135 #define EHCI_REMOVE_QH(sqh,last) (last) = _ehci_remove_qh(sqh,last)
1136 static ehci_qh_t *
1137 _ehci_remove_qh(ehci_qh_t *sqh, ehci_qh_t *last)
1138 {
1139         DPRINTFN(11, "%p from %p\n", sqh, last);
1140
1141         /* (sc->sc_bus.mtx) must be locked */
1142
1143         /* only remove if not removed from a queue */
1144         if (sqh->prev) {
1145
1146                 sqh->prev->next = sqh->next;
1147                 sqh->prev->qh_link = sqh->qh_link;
1148
1149                 usb_pc_cpu_flush(sqh->prev->page_cache);
1150
1151                 if (sqh->next) {
1152                         sqh->next->prev = sqh->prev;
1153                         usb_pc_cpu_flush(sqh->next->page_cache);
1154                 }
1155                 last = ((last == sqh) ? sqh->prev : last);
1156
1157                 sqh->prev = 0;
1158
1159                 usb_pc_cpu_flush(sqh->page_cache);
1160         }
1161         return (last);
1162 }
1163
1164 static usb_error_t
1165 ehci_non_isoc_done_sub(struct usb_xfer *xfer)
1166 {
1167         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
1168         ehci_qtd_t *td;
1169         ehci_qtd_t *td_alt_next;
1170         uint32_t status;
1171         uint16_t len;
1172
1173         td = xfer->td_transfer_cache;
1174         td_alt_next = td->alt_next;
1175
1176         if (xfer->aframes != xfer->nframes) {
1177                 usbd_xfer_set_frame_len(xfer, xfer->aframes, 0);
1178         }
1179         while (1) {
1180
1181                 usb_pc_cpu_invalidate(td->page_cache);
1182                 status = hc32toh(sc, td->qtd_status);
1183
1184                 len = EHCI_QTD_GET_BYTES(status);
1185
1186                 /*
1187                  * Verify the status length and
1188                  * add the length to "frlengths[]":
1189                  */
1190                 if (len > td->len) {
1191                         /* should not happen */
1192                         DPRINTF("Invalid status length, "
1193                             "0x%04x/0x%04x bytes\n", len, td->len);
1194                         status |= EHCI_QTD_HALTED;
1195                 } else if (xfer->aframes != xfer->nframes) {
1196                         xfer->frlengths[xfer->aframes] += td->len - len;
1197                 }
1198                 /* Check for last transfer */
1199                 if (((void *)td) == xfer->td_transfer_last) {
1200                         td = NULL;
1201                         break;
1202                 }
1203                 /* Check for transfer error */
1204                 if (status & EHCI_QTD_HALTED) {
1205                         /* the transfer is finished */
1206                         td = NULL;
1207                         break;
1208                 }
1209                 /* Check for short transfer */
1210                 if (len > 0) {
1211                         if (xfer->flags_int.short_frames_ok) {
1212                                 /* follow alt next */
1213                                 td = td->alt_next;
1214                         } else {
1215                                 /* the transfer is finished */
1216                                 td = NULL;
1217                         }
1218                         break;
1219                 }
1220                 td = td->obj_next;
1221
1222                 if (td->alt_next != td_alt_next) {
1223                         /* this USB frame is complete */
1224                         break;
1225                 }
1226         }
1227
1228         /* update transfer cache */
1229
1230         xfer->td_transfer_cache = td;
1231
1232 #ifdef USB_DEBUG
1233         if (status & EHCI_QTD_STATERRS) {
1234                 DPRINTFN(11, "error, addr=%d, endpt=0x%02x, frame=0x%02x"
1235                     "status=%s%s%s%s%s%s%s%s\n",
1236                     xfer->address, xfer->endpointno, xfer->aframes,
1237                     (status & EHCI_QTD_ACTIVE) ? "[ACTIVE]" : "[NOT_ACTIVE]",
1238                     (status & EHCI_QTD_HALTED) ? "[HALTED]" : "",
1239                     (status & EHCI_QTD_BUFERR) ? "[BUFERR]" : "",
1240                     (status & EHCI_QTD_BABBLE) ? "[BABBLE]" : "",
1241                     (status & EHCI_QTD_XACTERR) ? "[XACTERR]" : "",
1242                     (status & EHCI_QTD_MISSEDMICRO) ? "[MISSED]" : "",
1243                     (status & EHCI_QTD_SPLITXSTATE) ? "[SPLIT]" : "",
1244                     (status & EHCI_QTD_PINGSTATE) ? "[PING]" : "");
1245         }
1246 #endif
1247
1248         return ((status & EHCI_QTD_HALTED) ?
1249             USB_ERR_STALLED : USB_ERR_NORMAL_COMPLETION);
1250 }
1251
1252 static void
1253 ehci_non_isoc_done(struct usb_xfer *xfer)
1254 {
1255         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
1256         ehci_qh_t *qh;
1257         uint32_t status;
1258         usb_error_t err = 0;
1259
1260         DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
1261             xfer, xfer->endpoint);
1262
1263 #ifdef USB_DEBUG
1264         if (ehcidebug > 10) {
1265                 ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
1266
1267                 ehci_dump_sqtds(sc, xfer->td_transfer_first);
1268         }
1269 #endif
1270
1271         /* extract data toggle directly from the QH's overlay area */
1272
1273         qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
1274
1275         usb_pc_cpu_invalidate(qh->page_cache);
1276
1277         status = hc32toh(sc, qh->qh_qtd.qtd_status);
1278
1279         xfer->endpoint->toggle_next =
1280             (status & EHCI_QTD_TOGGLE_MASK) ? 1 : 0;
1281
1282         /* reset scanner */
1283
1284         xfer->td_transfer_cache = xfer->td_transfer_first;
1285
1286         if (xfer->flags_int.control_xfr) {
1287
1288                 if (xfer->flags_int.control_hdr) {
1289
1290                         err = ehci_non_isoc_done_sub(xfer);
1291                 }
1292                 xfer->aframes = 1;
1293
1294                 if (xfer->td_transfer_cache == NULL) {
1295                         goto done;
1296                 }
1297         }
1298         while (xfer->aframes != xfer->nframes) {
1299
1300                 err = ehci_non_isoc_done_sub(xfer);
1301                 xfer->aframes++;
1302
1303                 if (xfer->td_transfer_cache == NULL) {
1304                         goto done;
1305                 }
1306         }
1307
1308         if (xfer->flags_int.control_xfr &&
1309             !xfer->flags_int.control_act) {
1310
1311                 err = ehci_non_isoc_done_sub(xfer);
1312         }
1313 done:
1314         ehci_device_done(xfer, err);
1315 }
1316
1317 /*------------------------------------------------------------------------*
1318  *      ehci_check_transfer
1319  *
1320  * Return values:
1321  *    0: USB transfer is not finished
1322  * Else: USB transfer is finished
1323  *------------------------------------------------------------------------*/
1324 static uint8_t
1325 ehci_check_transfer(struct usb_xfer *xfer)
1326 {
1327         struct usb_pipe_methods *methods = xfer->endpoint->methods;
1328         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
1329
1330         uint32_t status;
1331
1332         DPRINTFN(13, "xfer=%p checking transfer\n", xfer);
1333
1334         if (methods == &ehci_device_isoc_fs_methods) {
1335                 ehci_sitd_t *td;
1336
1337                 /* isochronous full speed transfer */
1338
1339                 td = xfer->td_transfer_last;
1340                 usb_pc_cpu_invalidate(td->page_cache);
1341                 status = hc32toh(sc, td->sitd_status);
1342
1343                 /* also check if first is complete */
1344
1345                 td = xfer->td_transfer_first;
1346                 usb_pc_cpu_invalidate(td->page_cache);
1347                 status |= hc32toh(sc, td->sitd_status);
1348
1349                 if (!(status & EHCI_SITD_ACTIVE)) {
1350                         ehci_device_done(xfer, USB_ERR_NORMAL_COMPLETION);
1351                         goto transferred;
1352                 }
1353         } else if (methods == &ehci_device_isoc_hs_methods) {
1354                 ehci_itd_t *td;
1355
1356                 /* isochronous high speed transfer */
1357
1358                 /* check last transfer */
1359                 td = xfer->td_transfer_last;
1360                 usb_pc_cpu_invalidate(td->page_cache);
1361                 status = td->itd_status[0];
1362                 status |= td->itd_status[1];
1363                 status |= td->itd_status[2];
1364                 status |= td->itd_status[3];
1365                 status |= td->itd_status[4];
1366                 status |= td->itd_status[5];
1367                 status |= td->itd_status[6];
1368                 status |= td->itd_status[7];
1369
1370                 /* also check first transfer */
1371                 td = xfer->td_transfer_first;
1372                 usb_pc_cpu_invalidate(td->page_cache);
1373                 status |= td->itd_status[0];
1374                 status |= td->itd_status[1];
1375                 status |= td->itd_status[2];
1376                 status |= td->itd_status[3];
1377                 status |= td->itd_status[4];
1378                 status |= td->itd_status[5];
1379                 status |= td->itd_status[6];
1380                 status |= td->itd_status[7];
1381
1382                 /* if no transactions are active we continue */
1383                 if (!(status & htohc32(sc, EHCI_ITD_ACTIVE))) {
1384                         ehci_device_done(xfer, USB_ERR_NORMAL_COMPLETION);
1385                         goto transferred;
1386                 }
1387         } else {
1388                 ehci_qtd_t *td;
1389                 ehci_qh_t *qh;
1390
1391                 /* non-isochronous transfer */
1392
1393                 /*
1394                  * check whether there is an error somewhere in the middle,
1395                  * or whether there was a short packet (SPD and not ACTIVE)
1396                  */
1397                 td = xfer->td_transfer_cache;
1398
1399                 qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
1400
1401                 usb_pc_cpu_invalidate(qh->page_cache);
1402
1403                 status = hc32toh(sc, qh->qh_qtd.qtd_status);
1404                 if (status & EHCI_QTD_ACTIVE) {
1405                         /* transfer is pending */
1406                         goto done;
1407                 }
1408
1409                 while (1) {
1410                         usb_pc_cpu_invalidate(td->page_cache);
1411                         status = hc32toh(sc, td->qtd_status);
1412
1413                         /*
1414                          * Check if there is an active TD which
1415                          * indicates that the transfer isn't done.
1416                          */
1417                         if (status & EHCI_QTD_ACTIVE) {
1418                                 /* update cache */
1419                                 if (xfer->td_transfer_cache != td) {
1420                                         xfer->td_transfer_cache = td;
1421                                         if (qh->qh_qtd.qtd_next &
1422                                             htohc32(sc, EHCI_LINK_TERMINATE)) {
1423                                                 /* XXX - manually advance to next frame */
1424                                                 qh->qh_qtd.qtd_next = td->qtd_self;
1425                                                 usb_pc_cpu_flush(td->page_cache);
1426                                         }
1427                                 }
1428                                 goto done;
1429                         }
1430                         /*
1431                          * last transfer descriptor makes the transfer done
1432                          */
1433                         if (((void *)td) == xfer->td_transfer_last) {
1434                                 break;
1435                         }
1436                         /*
1437                          * any kind of error makes the transfer done
1438                          */
1439                         if (status & EHCI_QTD_HALTED) {
1440                                 break;
1441                         }
1442                         /*
1443                          * if there is no alternate next transfer, a short
1444                          * packet also makes the transfer done
1445                          */
1446                         if (EHCI_QTD_GET_BYTES(status)) {
1447                                 if (xfer->flags_int.short_frames_ok) {
1448                                         /* follow alt next */
1449                                         if (td->alt_next) {
1450                                                 td = td->alt_next;
1451                                                 continue;
1452                                         }
1453                                 }
1454                                 /* transfer is done */
1455                                 break;
1456                         }
1457                         td = td->obj_next;
1458                 }
1459                 ehci_non_isoc_done(xfer);
1460                 goto transferred;
1461         }
1462
1463 done:
1464         DPRINTFN(13, "xfer=%p is still active\n", xfer);
1465         return (0);
1466
1467 transferred:
1468         return (1);
1469 }
1470
1471 static void
1472 ehci_pcd_enable(ehci_softc_t *sc)
1473 {
1474         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
1475
1476         sc->sc_eintrs |= EHCI_STS_PCD;
1477         EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
1478
1479         /* acknowledge any PCD interrupt */
1480         EOWRITE4(sc, EHCI_USBSTS, EHCI_STS_PCD);
1481
1482         ehci_root_intr(sc);
1483 }
1484
1485 static void
1486 ehci_interrupt_poll(ehci_softc_t *sc)
1487 {
1488         struct usb_xfer *xfer;
1489
1490 repeat:
1491         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
1492                 /*
1493                  * check if transfer is transferred
1494                  */
1495                 if (ehci_check_transfer(xfer)) {
1496                         /* queue has been modified */
1497                         goto repeat;
1498                 }
1499         }
1500 }
1501
1502 /*
1503  * Some EHCI chips from VIA / ATI seem to trigger interrupts before
1504  * writing back the qTD status, or miss signalling occasionally under
1505  * heavy load.  If the host machine is too fast, we can miss
1506  * transaction completion - when we scan the active list the
1507  * transaction still seems to be active. This generally exhibits
1508  * itself as a umass stall that never recovers.
1509  *
1510  * We work around this behaviour by setting up this callback after any
1511  * softintr that completes with transactions still pending, giving us
1512  * another chance to check for completion after the writeback has
1513  * taken place.
1514  */
1515 static void
1516 ehci_poll_timeout(void *arg)
1517 {
1518         ehci_softc_t *sc = arg;
1519
1520         DPRINTFN(3, "\n");
1521         ehci_interrupt_poll(sc);
1522 }
1523
1524 /*------------------------------------------------------------------------*
1525  *      ehci_interrupt - EHCI interrupt handler
1526  *
1527  * NOTE: Do not access "sc->sc_bus.bdev" inside the interrupt handler,
1528  * hence the interrupt handler will be setup before "sc->sc_bus.bdev"
1529  * is present !
1530  *------------------------------------------------------------------------*/
1531 void
1532 ehci_interrupt(ehci_softc_t *sc)
1533 {
1534         uint32_t status;
1535
1536         USB_BUS_LOCK(&sc->sc_bus);
1537
1538         DPRINTFN(16, "real interrupt\n");
1539
1540 #ifdef USB_DEBUG
1541         if (ehcidebug > 15) {
1542                 ehci_dump_regs(sc);
1543         }
1544 #endif
1545
1546         status = EHCI_STS_INTRS(EOREAD4(sc, EHCI_USBSTS));
1547         if (status == 0) {
1548                 /* the interrupt was not for us */
1549                 goto done;
1550         }
1551         if (!(status & sc->sc_eintrs)) {
1552                 goto done;
1553         }
1554         EOWRITE4(sc, EHCI_USBSTS, status);      /* acknowledge */
1555
1556         status &= sc->sc_eintrs;
1557
1558         if (status & EHCI_STS_HSE) {
1559                 printf("%s: unrecoverable error, "
1560                     "controller halted\n", __FUNCTION__);
1561 #ifdef USB_DEBUG
1562                 ehci_dump_regs(sc);
1563                 ehci_dump_isoc(sc);
1564 #endif
1565         }
1566         if (status & EHCI_STS_PCD) {
1567                 /*
1568                  * Disable PCD interrupt for now, because it will be
1569                  * on until the port has been reset.
1570                  */
1571                 sc->sc_eintrs &= ~EHCI_STS_PCD;
1572                 EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
1573
1574                 ehci_root_intr(sc);
1575
1576                 /* do not allow RHSC interrupts > 1 per second */
1577                 usb_callout_reset(&sc->sc_tmo_pcd, hz,
1578                     (void *)&ehci_pcd_enable, sc);
1579         }
1580         status &= ~(EHCI_STS_INT | EHCI_STS_ERRINT | EHCI_STS_PCD | EHCI_STS_IAA);
1581
1582         if (status != 0) {
1583                 /* block unprocessed interrupts */
1584                 sc->sc_eintrs &= ~status;
1585                 EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
1586                 printf("%s: blocking interrupts 0x%x\n", __FUNCTION__, status);
1587         }
1588         /* poll all the USB transfers */
1589         ehci_interrupt_poll(sc);
1590
1591         if (sc->sc_flags & EHCI_SCFLG_LOSTINTRBUG) {
1592                 usb_callout_reset(&sc->sc_tmo_poll, hz / 128,
1593                     (void *)&ehci_poll_timeout, sc);
1594         }
1595
1596 done:
1597         USB_BUS_UNLOCK(&sc->sc_bus);
1598 }
1599
1600 /*
1601  * called when a request does not complete
1602  */
1603 static void
1604 ehci_timeout(void *arg)
1605 {
1606         struct usb_xfer *xfer = arg;
1607
1608         DPRINTF("xfer=%p\n", xfer);
1609
1610         USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
1611
1612         /* transfer is transferred */
1613         ehci_device_done(xfer, USB_ERR_TIMEOUT);
1614 }
1615
1616 static void
1617 ehci_do_poll(struct usb_bus *bus)
1618 {
1619         ehci_softc_t *sc = EHCI_BUS2SC(bus);
1620
1621         USB_BUS_LOCK(&sc->sc_bus);
1622         ehci_interrupt_poll(sc);
1623         USB_BUS_UNLOCK(&sc->sc_bus);
1624 }
1625
1626 static void
1627 ehci_setup_standard_chain_sub(struct ehci_std_temp *temp)
1628 {
1629         struct usb_page_search buf_res;
1630         ehci_qtd_t *td;
1631         ehci_qtd_t *td_next;
1632         ehci_qtd_t *td_alt_next;
1633         uint32_t buf_offset;
1634         uint32_t average;
1635         uint32_t len_old;
1636         uint32_t terminate;
1637         uint8_t shortpkt_old;
1638         uint8_t precompute;
1639
1640         terminate = htohc32(temp->sc, EHCI_LINK_TERMINATE);
1641         td_alt_next = NULL;
1642         buf_offset = 0;
1643         shortpkt_old = temp->shortpkt;
1644         len_old = temp->len;
1645         precompute = 1;
1646
1647 restart:
1648
1649         td = temp->td;
1650         td_next = temp->td_next;
1651
1652         while (1) {
1653
1654                 if (temp->len == 0) {
1655
1656                         if (temp->shortpkt) {
1657                                 break;
1658                         }
1659                         /* send a Zero Length Packet, ZLP, last */
1660
1661                         temp->shortpkt = 1;
1662                         average = 0;
1663
1664                 } else {
1665
1666                         average = temp->average;
1667
1668                         if (temp->len < average) {
1669                                 if (temp->len % temp->max_frame_size) {
1670                                         temp->shortpkt = 1;
1671                                 }
1672                                 average = temp->len;
1673                         }
1674                 }
1675
1676                 if (td_next == NULL) {
1677                         panic("%s: out of EHCI transfer descriptors!", __FUNCTION__);
1678                 }
1679                 /* get next TD */
1680
1681                 td = td_next;
1682                 td_next = td->obj_next;
1683
1684                 /* check if we are pre-computing */
1685
1686                 if (precompute) {
1687
1688                         /* update remaining length */
1689
1690                         temp->len -= average;
1691
1692                         continue;
1693                 }
1694                 /* fill out current TD */
1695
1696                 td->qtd_status =
1697                     temp->qtd_status |
1698                     htohc32(temp->sc, EHCI_QTD_IOC |
1699                         EHCI_QTD_SET_BYTES(average));
1700
1701                 if (average == 0) {
1702
1703                         if (temp->auto_data_toggle == 0) {
1704
1705                                 /* update data toggle, ZLP case */
1706
1707                                 temp->qtd_status ^=
1708                                     htohc32(temp->sc, EHCI_QTD_TOGGLE_MASK);
1709                         }
1710                         td->len = 0;
1711
1712                         td->qtd_buffer[0] = 0;
1713                         td->qtd_buffer_hi[0] = 0;
1714
1715                         td->qtd_buffer[1] = 0;
1716                         td->qtd_buffer_hi[1] = 0;
1717
1718                 } else {
1719
1720                         uint8_t x;
1721
1722                         if (temp->auto_data_toggle == 0) {
1723
1724                                 /* update data toggle */
1725
1726                                 if (((average + temp->max_frame_size - 1) /
1727                                     temp->max_frame_size) & 1) {
1728                                         temp->qtd_status ^=
1729                                             htohc32(temp->sc, EHCI_QTD_TOGGLE_MASK);
1730                                 }
1731                         }
1732                         td->len = average;
1733
1734                         /* update remaining length */
1735
1736                         temp->len -= average;
1737
1738                         /* fill out buffer pointers */
1739
1740                         usbd_get_page(temp->pc, buf_offset, &buf_res);
1741                         td->qtd_buffer[0] =
1742                             htohc32(temp->sc, buf_res.physaddr);
1743                         td->qtd_buffer_hi[0] = 0;
1744
1745                         x = 1;
1746
1747                         while (average > EHCI_PAGE_SIZE) {
1748                                 average -= EHCI_PAGE_SIZE;
1749                                 buf_offset += EHCI_PAGE_SIZE;
1750                                 usbd_get_page(temp->pc, buf_offset, &buf_res);
1751                                 td->qtd_buffer[x] =
1752                                     htohc32(temp->sc,
1753                                     buf_res.physaddr & (~0xFFF));
1754                                 td->qtd_buffer_hi[x] = 0;
1755                                 x++;
1756                         }
1757
1758                         /*
1759                          * NOTE: The "average" variable is never zero after
1760                          * exiting the loop above !
1761                          *
1762                          * NOTE: We have to subtract one from the offset to
1763                          * ensure that we are computing the physical address
1764                          * of a valid page !
1765                          */
1766                         buf_offset += average;
1767                         usbd_get_page(temp->pc, buf_offset - 1, &buf_res);
1768                         td->qtd_buffer[x] =
1769                             htohc32(temp->sc,
1770                             buf_res.physaddr & (~0xFFF));
1771                         td->qtd_buffer_hi[x] = 0;
1772                 }
1773
1774                 if (temp->can_use_next) {
1775                         if (td_next) {
1776                                 /* link the current TD with the next one */
1777                                 td->qtd_next = td_next->qtd_self;
1778                         }
1779                 } else {
1780                         /*
1781                          * BUG WARNING: The EHCI HW can use the
1782                          * qtd_next field instead of qtd_altnext when
1783                          * a short packet is received! We work this
1784                          * around in software by not queueing more
1785                          * than one job/TD at a time!
1786                          */
1787                         td->qtd_next = terminate;
1788                 }
1789
1790                 td->qtd_altnext = terminate;
1791                 td->alt_next = td_alt_next;
1792
1793                 usb_pc_cpu_flush(td->page_cache);
1794         }
1795
1796         if (precompute) {
1797                 precompute = 0;
1798
1799                 /* setup alt next pointer, if any */
1800                 if (temp->last_frame) {
1801                         td_alt_next = NULL;
1802                 } else {
1803                         /* we use this field internally */
1804                         td_alt_next = td_next;
1805                 }
1806
1807                 /* restore */
1808                 temp->shortpkt = shortpkt_old;
1809                 temp->len = len_old;
1810                 goto restart;
1811         }
1812         temp->td = td;
1813         temp->td_next = td_next;
1814 }
1815
1816 static void
1817 ehci_setup_standard_chain(struct usb_xfer *xfer, ehci_qh_t **qh_last)
1818 {
1819         struct ehci_std_temp temp;
1820         struct usb_pipe_methods *methods;
1821         ehci_qh_t *qh;
1822         ehci_qtd_t *td;
1823         uint32_t qh_endp;
1824         uint32_t qh_endphub;
1825         uint32_t x;
1826
1827         DPRINTFN(9, "addr=%d endpt=%d sumlen=%d speed=%d\n",
1828             xfer->address, UE_GET_ADDR(xfer->endpointno),
1829             xfer->sumlen, usbd_get_speed(xfer->xroot->udev));
1830
1831         temp.average = xfer->max_hc_frame_size;
1832         temp.max_frame_size = xfer->max_frame_size;
1833         temp.sc = EHCI_BUS2SC(xfer->xroot->bus);
1834
1835         /* toggle the DMA set we are using */
1836         xfer->flags_int.curr_dma_set ^= 1;
1837
1838         /* get next DMA set */
1839         td = xfer->td_start[xfer->flags_int.curr_dma_set];
1840
1841         xfer->td_transfer_first = td;
1842         xfer->td_transfer_cache = td;
1843
1844         temp.td = NULL;
1845         temp.td_next = td;
1846         temp.qtd_status = 0;
1847         temp.last_frame = 0;
1848         temp.setup_alt_next = xfer->flags_int.short_frames_ok;
1849         temp.can_use_next = (xfer->flags_int.control_xfr ||
1850             (UE_GET_DIR(xfer->endpointno) == UE_DIR_OUT));
1851
1852         if (xfer->flags_int.control_xfr) {
1853                 if (xfer->endpoint->toggle_next) {
1854                         /* DATA1 is next */
1855                         temp.qtd_status |=
1856                             htohc32(temp.sc, EHCI_QTD_SET_TOGGLE(1));
1857                 }
1858                 temp.auto_data_toggle = 0;
1859         } else {
1860                 temp.auto_data_toggle = 1;
1861         }
1862
1863         if (usbd_get_speed(xfer->xroot->udev) != USB_SPEED_HIGH) {
1864                 /* max 3 retries */
1865                 temp.qtd_status |=
1866                     htohc32(temp.sc, EHCI_QTD_SET_CERR(3));
1867         }
1868         /* check if we should prepend a setup message */
1869
1870         if (xfer->flags_int.control_xfr) {
1871                 if (xfer->flags_int.control_hdr) {
1872
1873                         temp.qtd_status &=
1874                             htohc32(temp.sc, EHCI_QTD_SET_CERR(3));
1875                         temp.qtd_status |= htohc32(temp.sc,
1876                             EHCI_QTD_ACTIVE |
1877                             EHCI_QTD_SET_PID(EHCI_QTD_PID_SETUP) |
1878                             EHCI_QTD_SET_TOGGLE(0));
1879
1880                         temp.len = xfer->frlengths[0];
1881                         temp.pc = xfer->frbuffers + 0;
1882                         temp.shortpkt = temp.len ? 1 : 0;
1883                         /* check for last frame */
1884                         if (xfer->nframes == 1) {
1885                                 /* no STATUS stage yet, SETUP is last */
1886                                 if (xfer->flags_int.control_act) {
1887                                         temp.last_frame = 1;
1888                                         temp.setup_alt_next = 0;
1889                                 }
1890                         }
1891                         ehci_setup_standard_chain_sub(&temp);
1892                 }
1893                 x = 1;
1894         } else {
1895                 x = 0;
1896         }
1897
1898         while (x != xfer->nframes) {
1899
1900                 /* DATA0 / DATA1 message */
1901
1902                 temp.len = xfer->frlengths[x];
1903                 temp.pc = xfer->frbuffers + x;
1904
1905                 x++;
1906
1907                 if (x == xfer->nframes) {
1908                         if (xfer->flags_int.control_xfr) {
1909                                 /* no STATUS stage yet, DATA is last */
1910                                 if (xfer->flags_int.control_act) {
1911                                         temp.last_frame = 1;
1912                                         temp.setup_alt_next = 0;
1913                                 }
1914                         } else {
1915                                 temp.last_frame = 1;
1916                                 temp.setup_alt_next = 0;
1917                         }
1918                 }
1919                 /* keep previous data toggle and error count */
1920
1921                 temp.qtd_status &=
1922                     htohc32(temp.sc, EHCI_QTD_SET_CERR(3) |
1923                     EHCI_QTD_SET_TOGGLE(1));
1924
1925                 if (temp.len == 0) {
1926
1927                         /* make sure that we send an USB packet */
1928
1929                         temp.shortpkt = 0;
1930
1931                 } else {
1932
1933                         /* regular data transfer */
1934
1935                         temp.shortpkt = (xfer->flags.force_short_xfer) ? 0 : 1;
1936                 }
1937
1938                 /* set endpoint direction */
1939
1940                 temp.qtd_status |=
1941                     (UE_GET_DIR(xfer->endpointno) == UE_DIR_IN) ?
1942                     htohc32(temp.sc, EHCI_QTD_ACTIVE |
1943                     EHCI_QTD_SET_PID(EHCI_QTD_PID_IN)) :
1944                     htohc32(temp.sc, EHCI_QTD_ACTIVE |
1945                     EHCI_QTD_SET_PID(EHCI_QTD_PID_OUT));
1946
1947                 ehci_setup_standard_chain_sub(&temp);
1948         }
1949
1950         /* check if we should append a status stage */
1951
1952         if (xfer->flags_int.control_xfr &&
1953             !xfer->flags_int.control_act) {
1954
1955                 /*
1956                  * Send a DATA1 message and invert the current endpoint
1957                  * direction.
1958                  */
1959
1960                 temp.qtd_status &= htohc32(temp.sc, EHCI_QTD_SET_CERR(3) |
1961                     EHCI_QTD_SET_TOGGLE(1));
1962                 temp.qtd_status |=
1963                     (UE_GET_DIR(xfer->endpointno) == UE_DIR_OUT) ?
1964                     htohc32(temp.sc, EHCI_QTD_ACTIVE |
1965                     EHCI_QTD_SET_PID(EHCI_QTD_PID_IN) |
1966                     EHCI_QTD_SET_TOGGLE(1)) :
1967                     htohc32(temp.sc, EHCI_QTD_ACTIVE |
1968                     EHCI_QTD_SET_PID(EHCI_QTD_PID_OUT) |
1969                     EHCI_QTD_SET_TOGGLE(1));
1970
1971                 temp.len = 0;
1972                 temp.pc = NULL;
1973                 temp.shortpkt = 0;
1974                 temp.last_frame = 1;
1975                 temp.setup_alt_next = 0;
1976
1977                 ehci_setup_standard_chain_sub(&temp);
1978         }
1979         td = temp.td;
1980
1981         /* the last TD terminates the transfer: */
1982         td->qtd_next = htohc32(temp.sc, EHCI_LINK_TERMINATE);
1983         td->qtd_altnext = htohc32(temp.sc, EHCI_LINK_TERMINATE);
1984
1985         usb_pc_cpu_flush(td->page_cache);
1986
1987         /* must have at least one frame! */
1988
1989         xfer->td_transfer_last = td;
1990
1991 #ifdef USB_DEBUG
1992         if (ehcidebug > 8) {
1993                 DPRINTF("nexttog=%d; data before transfer:\n",
1994                     xfer->endpoint->toggle_next);
1995                 ehci_dump_sqtds(temp.sc,
1996                     xfer->td_transfer_first);
1997         }
1998 #endif
1999
2000         methods = xfer->endpoint->methods;
2001
2002         qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
2003
2004         /* the "qh_link" field is filled when the QH is added */
2005
2006         qh_endp =
2007             (EHCI_QH_SET_ADDR(xfer->address) |
2008             EHCI_QH_SET_ENDPT(UE_GET_ADDR(xfer->endpointno)) |
2009             EHCI_QH_SET_MPL(xfer->max_packet_size));
2010
2011         if (usbd_get_speed(xfer->xroot->udev) == USB_SPEED_HIGH) {
2012                 qh_endp |= EHCI_QH_SET_EPS(EHCI_QH_SPEED_HIGH);
2013                 if (methods != &ehci_device_intr_methods)
2014                         qh_endp |= EHCI_QH_SET_NRL(8);
2015         } else {
2016
2017                 if (usbd_get_speed(xfer->xroot->udev) == USB_SPEED_FULL) {
2018                         qh_endp |= EHCI_QH_SET_EPS(EHCI_QH_SPEED_FULL);
2019                 } else {
2020                         qh_endp |= EHCI_QH_SET_EPS(EHCI_QH_SPEED_LOW);
2021                 }
2022
2023                 if (methods == &ehci_device_ctrl_methods) {
2024                         qh_endp |= EHCI_QH_CTL;
2025                 }
2026                 if (methods != &ehci_device_intr_methods) {
2027                         /* Only try one time per microframe! */
2028                         qh_endp |= EHCI_QH_SET_NRL(1);
2029                 }
2030         }
2031
2032         if (temp.auto_data_toggle == 0) {
2033                 /* software computes the data toggle */
2034                 qh_endp |= EHCI_QH_DTC;
2035         }
2036
2037         qh->qh_endp = htohc32(temp.sc, qh_endp);
2038
2039         qh_endphub =
2040             (EHCI_QH_SET_MULT(xfer->max_packet_count & 3) |
2041             EHCI_QH_SET_CMASK(xfer->endpoint->usb_cmask) |
2042             EHCI_QH_SET_SMASK(xfer->endpoint->usb_smask) |
2043             EHCI_QH_SET_HUBA(xfer->xroot->udev->hs_hub_addr) |
2044             EHCI_QH_SET_PORT(xfer->xroot->udev->hs_port_no));
2045
2046         qh->qh_endphub = htohc32(temp.sc, qh_endphub);
2047         qh->qh_curqtd = 0;
2048
2049         /* fill the overlay qTD */
2050
2051         if (temp.auto_data_toggle && xfer->endpoint->toggle_next) {
2052                 /* DATA1 is next */
2053                 qh->qh_qtd.qtd_status = htohc32(temp.sc, EHCI_QTD_SET_TOGGLE(1));
2054         } else {
2055                 qh->qh_qtd.qtd_status = 0;
2056         }
2057
2058         td = xfer->td_transfer_first;
2059
2060         qh->qh_qtd.qtd_next = td->qtd_self;
2061         qh->qh_qtd.qtd_altnext =
2062             htohc32(temp.sc, EHCI_LINK_TERMINATE);
2063
2064         usb_pc_cpu_flush(qh->page_cache);
2065
2066         if (xfer->xroot->udev->flags.self_suspended == 0) {
2067                 EHCI_APPEND_QH(qh, *qh_last);
2068         }
2069 }
2070
2071 static void
2072 ehci_root_intr(ehci_softc_t *sc)
2073 {
2074         uint16_t i;
2075         uint16_t m;
2076
2077         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
2078
2079         /* clear any old interrupt data */
2080         memset(sc->sc_hub_idata, 0, sizeof(sc->sc_hub_idata));
2081
2082         /* set bits */
2083         m = (sc->sc_noport + 1);
2084         if (m > (8 * sizeof(sc->sc_hub_idata))) {
2085                 m = (8 * sizeof(sc->sc_hub_idata));
2086         }
2087         for (i = 1; i < m; i++) {
2088                 /* pick out CHANGE bits from the status register */
2089                 if (EOREAD4(sc, EHCI_PORTSC(i)) & EHCI_PS_CLEAR) {
2090                         sc->sc_hub_idata[i / 8] |= 1 << (i % 8);
2091                         DPRINTF("port %d changed\n", i);
2092                 }
2093         }
2094         uhub_root_intr(&sc->sc_bus, sc->sc_hub_idata,
2095             sizeof(sc->sc_hub_idata));
2096 }
2097
2098 static void
2099 ehci_isoc_fs_done(ehci_softc_t *sc, struct usb_xfer *xfer)
2100 {
2101         uint32_t nframes = xfer->nframes;
2102         uint32_t status;
2103         uint32_t *plen = xfer->frlengths;
2104         uint16_t len = 0;
2105         ehci_sitd_t *td = xfer->td_transfer_first;
2106         ehci_sitd_t **pp_last = &sc->sc_isoc_fs_p_last[xfer->qh_pos];
2107
2108         DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
2109             xfer, xfer->endpoint);
2110
2111         while (nframes--) {
2112                 if (td == NULL) {
2113                         panic("%s:%d: out of TD's\n",
2114                             __FUNCTION__, __LINE__);
2115                 }
2116                 if (pp_last >= &sc->sc_isoc_fs_p_last[EHCI_VIRTUAL_FRAMELIST_COUNT]) {
2117                         pp_last = &sc->sc_isoc_fs_p_last[0];
2118                 }
2119 #ifdef USB_DEBUG
2120                 if (ehcidebug > 15) {
2121                         DPRINTF("isoc FS-TD\n");
2122                         ehci_dump_sitd(sc, td);
2123                 }
2124 #endif
2125                 usb_pc_cpu_invalidate(td->page_cache);
2126                 status = hc32toh(sc, td->sitd_status);
2127
2128                 len = EHCI_SITD_GET_LEN(status);
2129
2130                 DPRINTFN(2, "status=0x%08x, rem=%u\n", status, len);
2131
2132                 if (*plen >= len) {
2133                         len = *plen - len;
2134                 } else {
2135                         len = 0;
2136                 }
2137
2138                 *plen = len;
2139
2140                 /* remove FS-TD from schedule */
2141                 EHCI_REMOVE_FS_TD(td, *pp_last);
2142
2143                 pp_last++;
2144                 plen++;
2145                 td = td->obj_next;
2146         }
2147
2148         xfer->aframes = xfer->nframes;
2149 }
2150
2151 static void
2152 ehci_isoc_hs_done(ehci_softc_t *sc, struct usb_xfer *xfer)
2153 {
2154         uint32_t nframes = xfer->nframes;
2155         uint32_t status;
2156         uint32_t *plen = xfer->frlengths;
2157         uint16_t len = 0;
2158         uint8_t td_no = 0;
2159         ehci_itd_t *td = xfer->td_transfer_first;
2160         ehci_itd_t **pp_last = &sc->sc_isoc_hs_p_last[xfer->qh_pos];
2161
2162         DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
2163             xfer, xfer->endpoint);
2164
2165         while (nframes) {
2166                 if (td == NULL) {
2167                         panic("%s:%d: out of TD's\n",
2168                             __FUNCTION__, __LINE__);
2169                 }
2170                 if (pp_last >= &sc->sc_isoc_hs_p_last[EHCI_VIRTUAL_FRAMELIST_COUNT]) {
2171                         pp_last = &sc->sc_isoc_hs_p_last[0];
2172                 }
2173 #ifdef USB_DEBUG
2174                 if (ehcidebug > 15) {
2175                         DPRINTF("isoc HS-TD\n");
2176                         ehci_dump_itd(sc, td);
2177                 }
2178 #endif
2179
2180                 usb_pc_cpu_invalidate(td->page_cache);
2181                 status = hc32toh(sc, td->itd_status[td_no]);
2182
2183                 len = EHCI_ITD_GET_LEN(status);
2184
2185                 DPRINTFN(2, "status=0x%08x, len=%u\n", status, len);
2186
2187                 if (xfer->endpoint->usb_smask & (1 << td_no)) {
2188
2189                         if (*plen >= len) {
2190                                 /*
2191                                  * The length is valid. NOTE: The
2192                                  * complete length is written back
2193                                  * into the status field, and not the
2194                                  * remainder like with other transfer
2195                                  * descriptor types.
2196                                  */
2197                         } else {
2198                                 /* Invalid length - truncate */
2199                                 len = 0;
2200                         }
2201
2202                         *plen = len;
2203                         plen++;
2204                         nframes--;
2205                 }
2206
2207                 td_no++;
2208
2209                 if ((td_no == 8) || (nframes == 0)) {
2210                         /* remove HS-TD from schedule */
2211                         EHCI_REMOVE_HS_TD(td, *pp_last);
2212                         pp_last++;
2213
2214                         td_no = 0;
2215                         td = td->obj_next;
2216                 }
2217         }
2218         xfer->aframes = xfer->nframes;
2219 }
2220
2221 /* NOTE: "done" can be run two times in a row,
2222  * from close and from interrupt
2223  */
2224 static void
2225 ehci_device_done(struct usb_xfer *xfer, usb_error_t error)
2226 {
2227         struct usb_pipe_methods *methods = xfer->endpoint->methods;
2228         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2229
2230         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
2231
2232         DPRINTFN(2, "xfer=%p, endpoint=%p, error=%d\n",
2233             xfer, xfer->endpoint, error);
2234
2235         if ((methods == &ehci_device_bulk_methods) ||
2236             (methods == &ehci_device_ctrl_methods)) {
2237 #ifdef USB_DEBUG
2238                 if (ehcidebug > 8) {
2239                         DPRINTF("nexttog=%d; data after transfer:\n",
2240                             xfer->endpoint->toggle_next);
2241                         ehci_dump_sqtds(sc,
2242                             xfer->td_transfer_first);
2243                 }
2244 #endif
2245
2246                 EHCI_REMOVE_QH(xfer->qh_start[xfer->flags_int.curr_dma_set],
2247                     sc->sc_async_p_last);
2248         }
2249         if (methods == &ehci_device_intr_methods) {
2250                 EHCI_REMOVE_QH(xfer->qh_start[xfer->flags_int.curr_dma_set],
2251                     sc->sc_intr_p_last[xfer->qh_pos]);
2252         }
2253         /*
2254          * Only finish isochronous transfers once which will update
2255          * "xfer->frlengths".
2256          */
2257         if (xfer->td_transfer_first &&
2258             xfer->td_transfer_last) {
2259                 if (methods == &ehci_device_isoc_fs_methods) {
2260                         ehci_isoc_fs_done(sc, xfer);
2261                 }
2262                 if (methods == &ehci_device_isoc_hs_methods) {
2263                         ehci_isoc_hs_done(sc, xfer);
2264                 }
2265                 xfer->td_transfer_first = NULL;
2266                 xfer->td_transfer_last = NULL;
2267         }
2268         /* dequeue transfer and start next transfer */
2269         usbd_transfer_done(xfer, error);
2270 }
2271
2272 /*------------------------------------------------------------------------*
2273  * ehci bulk support
2274  *------------------------------------------------------------------------*/
2275 static void
2276 ehci_device_bulk_open(struct usb_xfer *xfer)
2277 {
2278         return;
2279 }
2280
2281 static void
2282 ehci_device_bulk_close(struct usb_xfer *xfer)
2283 {
2284         ehci_device_done(xfer, USB_ERR_CANCELLED);
2285 }
2286
2287 static void
2288 ehci_device_bulk_enter(struct usb_xfer *xfer)
2289 {
2290         return;
2291 }
2292
2293 static void
2294 ehci_device_bulk_start(struct usb_xfer *xfer)
2295 {
2296         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2297         uint32_t temp;
2298
2299         /* setup TD's and QH */
2300         ehci_setup_standard_chain(xfer, &sc->sc_async_p_last);
2301
2302         /* put transfer on interrupt queue */
2303         ehci_transfer_intr_enqueue(xfer);
2304
2305         /*
2306          * XXX Certain nVidia chipsets choke when using the IAAD
2307          * feature too frequently.
2308          */
2309         if (sc->sc_flags & EHCI_SCFLG_IAADBUG)
2310                 return;
2311
2312         /* XXX Performance quirk: Some Host Controllers have a too low
2313          * interrupt rate. Issue an IAAD to stimulate the Host
2314          * Controller after queueing the BULK transfer.
2315          */
2316         temp = EOREAD4(sc, EHCI_USBCMD);
2317         if (!(temp & EHCI_CMD_IAAD))
2318                 EOWRITE4(sc, EHCI_USBCMD, temp | EHCI_CMD_IAAD);
2319 }
2320
2321 struct usb_pipe_methods ehci_device_bulk_methods =
2322 {
2323         .open = ehci_device_bulk_open,
2324         .close = ehci_device_bulk_close,
2325         .enter = ehci_device_bulk_enter,
2326         .start = ehci_device_bulk_start,
2327 };
2328
2329 /*------------------------------------------------------------------------*
2330  * ehci control support
2331  *------------------------------------------------------------------------*/
2332 static void
2333 ehci_device_ctrl_open(struct usb_xfer *xfer)
2334 {
2335         return;
2336 }
2337
2338 static void
2339 ehci_device_ctrl_close(struct usb_xfer *xfer)
2340 {
2341         ehci_device_done(xfer, USB_ERR_CANCELLED);
2342 }
2343
2344 static void
2345 ehci_device_ctrl_enter(struct usb_xfer *xfer)
2346 {
2347         return;
2348 }
2349
2350 static void
2351 ehci_device_ctrl_start(struct usb_xfer *xfer)
2352 {
2353         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2354
2355         /* setup TD's and QH */
2356         ehci_setup_standard_chain(xfer, &sc->sc_async_p_last);
2357
2358         /* put transfer on interrupt queue */
2359         ehci_transfer_intr_enqueue(xfer);
2360 }
2361
2362 struct usb_pipe_methods ehci_device_ctrl_methods =
2363 {
2364         .open = ehci_device_ctrl_open,
2365         .close = ehci_device_ctrl_close,
2366         .enter = ehci_device_ctrl_enter,
2367         .start = ehci_device_ctrl_start,
2368 };
2369
2370 /*------------------------------------------------------------------------*
2371  * ehci interrupt support
2372  *------------------------------------------------------------------------*/
2373 static void
2374 ehci_device_intr_open(struct usb_xfer *xfer)
2375 {
2376         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2377         uint16_t best;
2378         uint16_t bit;
2379         uint16_t x;
2380
2381         usb_hs_bandwidth_alloc(xfer);
2382
2383         /*
2384          * Find the best QH position corresponding to the given interval:
2385          */
2386
2387         best = 0;
2388         bit = EHCI_VIRTUAL_FRAMELIST_COUNT / 2;
2389         while (bit) {
2390                 if (xfer->interval >= bit) {
2391                         x = bit;
2392                         best = bit;
2393                         while (x & bit) {
2394                                 if (sc->sc_intr_stat[x] <
2395                                     sc->sc_intr_stat[best]) {
2396                                         best = x;
2397                                 }
2398                                 x++;
2399                         }
2400                         break;
2401                 }
2402                 bit >>= 1;
2403         }
2404
2405         sc->sc_intr_stat[best]++;
2406         xfer->qh_pos = best;
2407
2408         DPRINTFN(3, "best=%d interval=%d\n",
2409             best, xfer->interval);
2410 }
2411
2412 static void
2413 ehci_device_intr_close(struct usb_xfer *xfer)
2414 {
2415         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2416
2417         sc->sc_intr_stat[xfer->qh_pos]--;
2418
2419         ehci_device_done(xfer, USB_ERR_CANCELLED);
2420
2421         /* bandwidth must be freed after device done */
2422         usb_hs_bandwidth_free(xfer);
2423 }
2424
2425 static void
2426 ehci_device_intr_enter(struct usb_xfer *xfer)
2427 {
2428         return;
2429 }
2430
2431 static void
2432 ehci_device_intr_start(struct usb_xfer *xfer)
2433 {
2434         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2435
2436         /* setup TD's and QH */
2437         ehci_setup_standard_chain(xfer, &sc->sc_intr_p_last[xfer->qh_pos]);
2438
2439         /* put transfer on interrupt queue */
2440         ehci_transfer_intr_enqueue(xfer);
2441 }
2442
2443 struct usb_pipe_methods ehci_device_intr_methods =
2444 {
2445         .open = ehci_device_intr_open,
2446         .close = ehci_device_intr_close,
2447         .enter = ehci_device_intr_enter,
2448         .start = ehci_device_intr_start,
2449 };
2450
2451 /*------------------------------------------------------------------------*
2452  * ehci full speed isochronous support
2453  *------------------------------------------------------------------------*/
2454 static void
2455 ehci_device_isoc_fs_open(struct usb_xfer *xfer)
2456 {
2457         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2458         ehci_sitd_t *td;
2459         uint32_t sitd_portaddr;
2460         uint8_t ds;
2461
2462         sitd_portaddr =
2463             EHCI_SITD_SET_ADDR(xfer->address) |
2464             EHCI_SITD_SET_ENDPT(UE_GET_ADDR(xfer->endpointno)) |
2465             EHCI_SITD_SET_HUBA(xfer->xroot->udev->hs_hub_addr) |
2466             EHCI_SITD_SET_PORT(xfer->xroot->udev->hs_port_no);
2467
2468         if (UE_GET_DIR(xfer->endpointno) == UE_DIR_IN) {
2469                 sitd_portaddr |= EHCI_SITD_SET_DIR_IN;
2470         }
2471         sitd_portaddr = htohc32(sc, sitd_portaddr);
2472
2473         /* initialize all TD's */
2474
2475         for (ds = 0; ds != 2; ds++) {
2476
2477                 for (td = xfer->td_start[ds]; td; td = td->obj_next) {
2478
2479                         td->sitd_portaddr = sitd_portaddr;
2480
2481                         /*
2482                          * TODO: make some kind of automatic
2483                          * SMASK/CMASK selection based on micro-frame
2484                          * usage
2485                          *
2486                          * micro-frame usage (8 microframes per 1ms)
2487                          */
2488                         td->sitd_back = htohc32(sc, EHCI_LINK_TERMINATE);
2489
2490                         usb_pc_cpu_flush(td->page_cache);
2491                 }
2492         }
2493 }
2494
2495 static void
2496 ehci_device_isoc_fs_close(struct usb_xfer *xfer)
2497 {
2498         ehci_device_done(xfer, USB_ERR_CANCELLED);
2499 }
2500
2501 static void
2502 ehci_device_isoc_fs_enter(struct usb_xfer *xfer)
2503 {
2504         struct usb_page_search buf_res;
2505         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2506         struct usb_fs_isoc_schedule *fss_start;
2507         struct usb_fs_isoc_schedule *fss_end;
2508         struct usb_fs_isoc_schedule *fss;
2509         ehci_sitd_t *td;
2510         ehci_sitd_t *td_last = NULL;
2511         ehci_sitd_t **pp_last;
2512         uint32_t *plen;
2513         uint32_t buf_offset;
2514         uint32_t nframes;
2515         uint32_t temp;
2516         uint32_t sitd_mask;
2517         uint16_t tlen;
2518         uint8_t sa;
2519         uint8_t sb;
2520         uint8_t error;
2521
2522 #ifdef USB_DEBUG
2523         uint8_t once = 1;
2524
2525 #endif
2526
2527         DPRINTFN(6, "xfer=%p next=%d nframes=%d\n",
2528             xfer, xfer->endpoint->isoc_next, xfer->nframes);
2529
2530         /* get the current frame index */
2531
2532         nframes = EOREAD4(sc, EHCI_FRINDEX) / 8;
2533
2534         /*
2535          * check if the frame index is within the window where the frames
2536          * will be inserted
2537          */
2538         buf_offset = (nframes - xfer->endpoint->isoc_next) &
2539             (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2540
2541         if ((xfer->endpoint->is_synced == 0) ||
2542             (buf_offset < xfer->nframes)) {
2543                 /*
2544                  * If there is data underflow or the pipe queue is empty we
2545                  * schedule the transfer a few frames ahead of the current
2546                  * frame position. Else two isochronous transfers might
2547                  * overlap.
2548                  */
2549                 xfer->endpoint->isoc_next = (nframes + 3) &
2550                     (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2551                 xfer->endpoint->is_synced = 1;
2552                 DPRINTFN(3, "start next=%d\n", xfer->endpoint->isoc_next);
2553         }
2554         /*
2555          * compute how many milliseconds the insertion is ahead of the
2556          * current frame position:
2557          */
2558         buf_offset = (xfer->endpoint->isoc_next - nframes) &
2559             (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2560
2561         /*
2562          * pre-compute when the isochronous transfer will be finished:
2563          */
2564         xfer->isoc_time_complete =
2565             usbd_fs_isoc_schedule_isoc_time_expand
2566             (xfer->xroot->udev, &fss_start, &fss_end, nframes) + buf_offset +
2567             xfer->nframes;
2568
2569         /* get the real number of frames */
2570
2571         nframes = xfer->nframes;
2572
2573         buf_offset = 0;
2574
2575         plen = xfer->frlengths;
2576
2577         /* toggle the DMA set we are using */
2578         xfer->flags_int.curr_dma_set ^= 1;
2579
2580         /* get next DMA set */
2581         td = xfer->td_start[xfer->flags_int.curr_dma_set];
2582         xfer->td_transfer_first = td;
2583
2584         pp_last = &sc->sc_isoc_fs_p_last[xfer->endpoint->isoc_next];
2585
2586         /* store starting position */
2587
2588         xfer->qh_pos = xfer->endpoint->isoc_next;
2589
2590         fss = fss_start + (xfer->qh_pos % USB_ISOC_TIME_MAX);
2591
2592         while (nframes--) {
2593                 if (td == NULL) {
2594                         panic("%s:%d: out of TD's\n",
2595                             __FUNCTION__, __LINE__);
2596                 }
2597                 if (pp_last >= &sc->sc_isoc_fs_p_last[EHCI_VIRTUAL_FRAMELIST_COUNT]) {
2598                         pp_last = &sc->sc_isoc_fs_p_last[0];
2599                 }
2600                 if (fss >= fss_end) {
2601                         fss = fss_start;
2602                 }
2603                 /* reuse sitd_portaddr and sitd_back from last transfer */
2604
2605                 if (*plen > xfer->max_frame_size) {
2606 #ifdef USB_DEBUG
2607                         if (once) {
2608                                 once = 0;
2609                                 printf("%s: frame length(%d) exceeds %d "
2610                                     "bytes (frame truncated)\n",
2611                                     __FUNCTION__, *plen,
2612                                     xfer->max_frame_size);
2613                         }
2614 #endif
2615                         *plen = xfer->max_frame_size;
2616                 }
2617                 /*
2618                  * We currently don't care if the ISOCHRONOUS schedule is
2619                  * full!
2620                  */
2621                 error = usbd_fs_isoc_schedule_alloc(fss, &sa, *plen);
2622                 if (error) {
2623                         /*
2624                          * The FULL speed schedule is FULL! Set length
2625                          * to zero.
2626                          */
2627                         *plen = 0;
2628                 }
2629                 if (*plen) {
2630                         /*
2631                          * only call "usbd_get_page()" when we have a
2632                          * non-zero length
2633                          */
2634                         usbd_get_page(xfer->frbuffers, buf_offset, &buf_res);
2635                         td->sitd_bp[0] = htohc32(sc, buf_res.physaddr);
2636                         buf_offset += *plen;
2637                         /*
2638                          * NOTE: We need to subtract one from the offset so
2639                          * that we are on a valid page!
2640                          */
2641                         usbd_get_page(xfer->frbuffers, buf_offset - 1,
2642                             &buf_res);
2643                         temp = buf_res.physaddr & ~0xFFF;
2644                 } else {
2645                         td->sitd_bp[0] = 0;
2646                         temp = 0;
2647                 }
2648
2649                 if (UE_GET_DIR(xfer->endpointno) == UE_DIR_OUT) {
2650                         tlen = *plen;
2651                         if (tlen <= 188) {
2652                                 temp |= 1;      /* T-count = 1, TP = ALL */
2653                                 tlen = 1;
2654                         } else {
2655                                 tlen += 187;
2656                                 tlen /= 188;
2657                                 temp |= tlen;   /* T-count = [1..6] */
2658                                 temp |= 8;      /* TP = Begin */
2659                         }
2660
2661                         tlen += sa;
2662
2663                         if (tlen >= 8) {
2664                                 sb = 0;
2665                         } else {
2666                                 sb = (1 << tlen);
2667                         }
2668
2669                         sa = (1 << sa);
2670                         sa = (sb - sa) & 0x3F;
2671                         sb = 0;
2672                 } else {
2673                         sb = (-(4 << sa)) & 0xFE;
2674                         sa = (1 << sa) & 0x3F;
2675                 }
2676
2677                 sitd_mask = (EHCI_SITD_SET_SMASK(sa) |
2678                     EHCI_SITD_SET_CMASK(sb));
2679
2680                 td->sitd_bp[1] = htohc32(sc, temp);
2681
2682                 td->sitd_mask = htohc32(sc, sitd_mask);
2683
2684                 if (nframes == 0) {
2685                         td->sitd_status = htohc32(sc,
2686                             EHCI_SITD_IOC |
2687                             EHCI_SITD_ACTIVE |
2688                             EHCI_SITD_SET_LEN(*plen));
2689                 } else {
2690                         td->sitd_status = htohc32(sc,
2691                             EHCI_SITD_ACTIVE |
2692                             EHCI_SITD_SET_LEN(*plen));
2693                 }
2694                 usb_pc_cpu_flush(td->page_cache);
2695
2696 #ifdef USB_DEBUG
2697                 if (ehcidebug > 15) {
2698                         DPRINTF("FS-TD %d\n", nframes);
2699                         ehci_dump_sitd(sc, td);
2700                 }
2701 #endif
2702                 /* insert TD into schedule */
2703                 EHCI_APPEND_FS_TD(td, *pp_last);
2704                 pp_last++;
2705
2706                 plen++;
2707                 fss++;
2708                 td_last = td;
2709                 td = td->obj_next;
2710         }
2711
2712         xfer->td_transfer_last = td_last;
2713
2714         /* update isoc_next */
2715         xfer->endpoint->isoc_next = (pp_last - &sc->sc_isoc_fs_p_last[0]) &
2716             (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2717 }
2718
2719 static void
2720 ehci_device_isoc_fs_start(struct usb_xfer *xfer)
2721 {
2722         /* put transfer on interrupt queue */
2723         ehci_transfer_intr_enqueue(xfer);
2724 }
2725
2726 struct usb_pipe_methods ehci_device_isoc_fs_methods =
2727 {
2728         .open = ehci_device_isoc_fs_open,
2729         .close = ehci_device_isoc_fs_close,
2730         .enter = ehci_device_isoc_fs_enter,
2731         .start = ehci_device_isoc_fs_start,
2732 };
2733
2734 /*------------------------------------------------------------------------*
2735  * ehci high speed isochronous support
2736  *------------------------------------------------------------------------*/
2737 static void
2738 ehci_device_isoc_hs_open(struct usb_xfer *xfer)
2739 {
2740         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2741         ehci_itd_t *td;
2742         uint32_t temp;
2743         uint8_t ds;
2744
2745         usb_hs_bandwidth_alloc(xfer);
2746
2747         /* initialize all TD's */
2748
2749         for (ds = 0; ds != 2; ds++) {
2750
2751                 for (td = xfer->td_start[ds]; td; td = td->obj_next) {
2752
2753                         /* set TD inactive */
2754                         td->itd_status[0] = 0;
2755                         td->itd_status[1] = 0;
2756                         td->itd_status[2] = 0;
2757                         td->itd_status[3] = 0;
2758                         td->itd_status[4] = 0;
2759                         td->itd_status[5] = 0;
2760                         td->itd_status[6] = 0;
2761                         td->itd_status[7] = 0;
2762
2763                         /* set endpoint and address */
2764                         td->itd_bp[0] = htohc32(sc,
2765                             EHCI_ITD_SET_ADDR(xfer->address) |
2766                             EHCI_ITD_SET_ENDPT(UE_GET_ADDR(xfer->endpointno)));
2767
2768                         temp =
2769                             EHCI_ITD_SET_MPL(xfer->max_packet_size & 0x7FF);
2770
2771                         /* set direction */
2772                         if (UE_GET_DIR(xfer->endpointno) == UE_DIR_IN) {
2773                                 temp |= EHCI_ITD_SET_DIR_IN;
2774                         }
2775                         /* set maximum packet size */
2776                         td->itd_bp[1] = htohc32(sc, temp);
2777
2778                         /* set transfer multiplier */
2779                         td->itd_bp[2] = htohc32(sc, xfer->max_packet_count & 3);
2780
2781                         usb_pc_cpu_flush(td->page_cache);
2782                 }
2783         }
2784 }
2785
2786 static void
2787 ehci_device_isoc_hs_close(struct usb_xfer *xfer)
2788 {
2789         ehci_device_done(xfer, USB_ERR_CANCELLED);
2790
2791         /* bandwidth must be freed after device done */
2792         usb_hs_bandwidth_free(xfer);
2793 }
2794
2795 static void
2796 ehci_device_isoc_hs_enter(struct usb_xfer *xfer)
2797 {
2798         struct usb_page_search buf_res;
2799         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2800         ehci_itd_t *td;
2801         ehci_itd_t *td_last = NULL;
2802         ehci_itd_t **pp_last;
2803         bus_size_t page_addr;
2804         uint32_t *plen;
2805         uint32_t status;
2806         uint32_t buf_offset;
2807         uint32_t nframes;
2808         uint32_t itd_offset[8 + 1];
2809         uint8_t x;
2810         uint8_t td_no;
2811         uint8_t page_no;
2812         uint8_t shift = usbd_xfer_get_fps_shift(xfer);
2813
2814 #ifdef USB_DEBUG
2815         uint8_t once = 1;
2816
2817 #endif
2818
2819         DPRINTFN(6, "xfer=%p next=%d nframes=%d shift=%d\n",
2820             xfer, xfer->endpoint->isoc_next, xfer->nframes, (int)shift);
2821
2822         /* get the current frame index */
2823
2824         nframes = EOREAD4(sc, EHCI_FRINDEX) / 8;
2825
2826         /*
2827          * check if the frame index is within the window where the frames
2828          * will be inserted
2829          */
2830         buf_offset = (nframes - xfer->endpoint->isoc_next) &
2831             (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2832
2833         if ((xfer->endpoint->is_synced == 0) ||
2834             (buf_offset < (((xfer->nframes << shift) + 7) / 8))) {
2835                 /*
2836                  * If there is data underflow or the pipe queue is empty we
2837                  * schedule the transfer a few frames ahead of the current
2838                  * frame position. Else two isochronous transfers might
2839                  * overlap.
2840                  */
2841                 xfer->endpoint->isoc_next = (nframes + 3) &
2842                     (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2843                 xfer->endpoint->is_synced = 1;
2844                 DPRINTFN(3, "start next=%d\n", xfer->endpoint->isoc_next);
2845         }
2846         /*
2847          * compute how many milliseconds the insertion is ahead of the
2848          * current frame position:
2849          */
2850         buf_offset = (xfer->endpoint->isoc_next - nframes) &
2851             (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2852
2853         /*
2854          * pre-compute when the isochronous transfer will be finished:
2855          */
2856         xfer->isoc_time_complete =
2857             usb_isoc_time_expand(&sc->sc_bus, nframes) + buf_offset +
2858             (((xfer->nframes << shift) + 7) / 8);
2859
2860         /* get the real number of frames */
2861
2862         nframes = xfer->nframes;
2863
2864         buf_offset = 0;
2865         td_no = 0;
2866
2867         plen = xfer->frlengths;
2868
2869         /* toggle the DMA set we are using */
2870         xfer->flags_int.curr_dma_set ^= 1;
2871
2872         /* get next DMA set */
2873         td = xfer->td_start[xfer->flags_int.curr_dma_set];
2874         xfer->td_transfer_first = td;
2875
2876         pp_last = &sc->sc_isoc_hs_p_last[xfer->endpoint->isoc_next];
2877
2878         /* store starting position */
2879
2880         xfer->qh_pos = xfer->endpoint->isoc_next;
2881
2882         while (nframes) {
2883                 if (td == NULL) {
2884                         panic("%s:%d: out of TD's\n",
2885                             __FUNCTION__, __LINE__);
2886                 }
2887                 if (pp_last >= &sc->sc_isoc_hs_p_last[EHCI_VIRTUAL_FRAMELIST_COUNT]) {
2888                         pp_last = &sc->sc_isoc_hs_p_last[0];
2889                 }
2890                 /* range check */
2891                 if (*plen > xfer->max_frame_size) {
2892 #ifdef USB_DEBUG
2893                         if (once) {
2894                                 once = 0;
2895                                 printf("%s: frame length(%d) exceeds %d bytes "
2896                                     "(frame truncated)\n",
2897                                     __FUNCTION__, *plen, xfer->max_frame_size);
2898                         }
2899 #endif
2900                         *plen = xfer->max_frame_size;
2901                 }
2902
2903                 if (xfer->endpoint->usb_smask & (1 << td_no)) {
2904                         status = (EHCI_ITD_SET_LEN(*plen) |
2905                             EHCI_ITD_ACTIVE |
2906                             EHCI_ITD_SET_PG(0));
2907                         td->itd_status[td_no] = htohc32(sc, status);
2908                         itd_offset[td_no] = buf_offset;
2909                         buf_offset += *plen;
2910                         plen++;
2911                         nframes --;
2912                 } else {
2913                         td->itd_status[td_no] = 0;      /* not active */
2914                         itd_offset[td_no] = buf_offset;
2915                 }
2916
2917                 td_no++;
2918
2919                 if ((td_no == 8) || (nframes == 0)) {
2920
2921                         /* the rest of the transfers are not active, if any */
2922                         for (x = td_no; x != 8; x++) {
2923                                 td->itd_status[x] = 0;  /* not active */
2924                         }
2925
2926                         /* check if there is any data to be transferred */
2927                         if (itd_offset[0] != buf_offset) {
2928                                 page_no = 0;
2929                                 itd_offset[td_no] = buf_offset;
2930
2931                                 /* get first page offset */
2932                                 usbd_get_page(xfer->frbuffers, itd_offset[0], &buf_res);
2933                                 /* get page address */
2934                                 page_addr = buf_res.physaddr & ~0xFFF;
2935                                 /* update page address */
2936                                 td->itd_bp[0] &= htohc32(sc, 0xFFF);
2937                                 td->itd_bp[0] |= htohc32(sc, page_addr);
2938
2939                                 for (x = 0; x != td_no; x++) {
2940                                         /* set page number and page offset */
2941                                         status = (EHCI_ITD_SET_PG(page_no) |
2942                                             (buf_res.physaddr & 0xFFF));
2943                                         td->itd_status[x] |= htohc32(sc, status);
2944
2945                                         /* get next page offset */
2946                                         if (itd_offset[x + 1] == buf_offset) {
2947                                                 /*
2948                                                  * We subtract one so that
2949                                                  * we don't go off the last
2950                                                  * page!
2951                                                  */
2952                                                 usbd_get_page(xfer->frbuffers, buf_offset - 1, &buf_res);
2953                                         } else {
2954                                                 usbd_get_page(xfer->frbuffers, itd_offset[x + 1], &buf_res);
2955                                         }
2956
2957                                         /* check if we need a new page */
2958                                         if ((buf_res.physaddr ^ page_addr) & ~0xFFF) {
2959                                                 /* new page needed */
2960                                                 page_addr = buf_res.physaddr & ~0xFFF;
2961                                                 if (page_no == 6) {
2962                                                         panic("%s: too many pages\n", __FUNCTION__);
2963                                                 }
2964                                                 page_no++;
2965                                                 /* update page address */
2966                                                 td->itd_bp[page_no] &= htohc32(sc, 0xFFF);
2967                                                 td->itd_bp[page_no] |= htohc32(sc, page_addr);
2968                                         }
2969                                 }
2970                         }
2971                         /* set IOC bit if we are complete */
2972                         if (nframes == 0) {
2973                                 td->itd_status[td_no - 1] |= htohc32(sc, EHCI_ITD_IOC);
2974                         }
2975                         usb_pc_cpu_flush(td->page_cache);
2976 #ifdef USB_DEBUG
2977                         if (ehcidebug > 15) {
2978                                 DPRINTF("HS-TD %d\n", nframes);
2979                                 ehci_dump_itd(sc, td);
2980                         }
2981 #endif
2982                         /* insert TD into schedule */
2983                         EHCI_APPEND_HS_TD(td, *pp_last);
2984                         pp_last++;
2985
2986                         td_no = 0;
2987                         td_last = td;
2988                         td = td->obj_next;
2989                 }
2990         }
2991
2992         xfer->td_transfer_last = td_last;
2993
2994         /* update isoc_next */
2995         xfer->endpoint->isoc_next = (pp_last - &sc->sc_isoc_hs_p_last[0]) &
2996             (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2997 }
2998
2999 static void
3000 ehci_device_isoc_hs_start(struct usb_xfer *xfer)
3001 {
3002         /* put transfer on interrupt queue */
3003         ehci_transfer_intr_enqueue(xfer);
3004 }
3005
3006 struct usb_pipe_methods ehci_device_isoc_hs_methods =
3007 {
3008         .open = ehci_device_isoc_hs_open,
3009         .close = ehci_device_isoc_hs_close,
3010         .enter = ehci_device_isoc_hs_enter,
3011         .start = ehci_device_isoc_hs_start,
3012 };
3013
3014 /*------------------------------------------------------------------------*
3015  * ehci root control support
3016  *------------------------------------------------------------------------*
3017  * Simulate a hardware hub by handling all the necessary requests.
3018  *------------------------------------------------------------------------*/
3019
3020 static const
3021 struct usb_device_descriptor ehci_devd =
3022 {
3023         sizeof(struct usb_device_descriptor),
3024         UDESC_DEVICE,                   /* type */
3025         {0x00, 0x02},                   /* USB version */
3026         UDCLASS_HUB,                    /* class */
3027         UDSUBCLASS_HUB,                 /* subclass */
3028         UDPROTO_HSHUBSTT,               /* protocol */
3029         64,                             /* max packet */
3030         {0}, {0}, {0x00, 0x01},         /* device id */
3031         1, 2, 0,                        /* string indicies */
3032         1                               /* # of configurations */
3033 };
3034
3035 static const
3036 struct usb_device_qualifier ehci_odevd =
3037 {
3038         sizeof(struct usb_device_qualifier),
3039         UDESC_DEVICE_QUALIFIER,         /* type */
3040         {0x00, 0x02},                   /* USB version */
3041         UDCLASS_HUB,                    /* class */
3042         UDSUBCLASS_HUB,                 /* subclass */
3043         UDPROTO_FSHUB,                  /* protocol */
3044         0,                              /* max packet */
3045         0,                              /* # of configurations */
3046         0
3047 };
3048
3049 static const struct ehci_config_desc ehci_confd = {
3050         .confd = {
3051                 .bLength = sizeof(struct usb_config_descriptor),
3052                 .bDescriptorType = UDESC_CONFIG,
3053                 .wTotalLength[0] = sizeof(ehci_confd),
3054                 .bNumInterface = 1,
3055                 .bConfigurationValue = 1,
3056                 .iConfiguration = 0,
3057                 .bmAttributes = UC_SELF_POWERED,
3058                 .bMaxPower = 0          /* max power */
3059         },
3060         .ifcd = {
3061                 .bLength = sizeof(struct usb_interface_descriptor),
3062                 .bDescriptorType = UDESC_INTERFACE,
3063                 .bNumEndpoints = 1,
3064                 .bInterfaceClass = UICLASS_HUB,
3065                 .bInterfaceSubClass = UISUBCLASS_HUB,
3066                 .bInterfaceProtocol = UIPROTO_HSHUBSTT,
3067                 0
3068         },
3069         .endpd = {
3070                 .bLength = sizeof(struct usb_endpoint_descriptor),
3071                 .bDescriptorType = UDESC_ENDPOINT,
3072                 .bEndpointAddress = UE_DIR_IN | EHCI_INTR_ENDPT,
3073                 .bmAttributes = UE_INTERRUPT,
3074                 .wMaxPacketSize[0] = 8, /* max packet (63 ports) */
3075                 .bInterval = 255,
3076         },
3077 };
3078
3079 static const
3080 struct usb_hub_descriptor ehci_hubd =
3081 {
3082         0,                              /* dynamic length */
3083         UDESC_HUB,
3084         0,
3085         {0, 0},
3086         0,
3087         0,
3088         {0},
3089 };
3090
3091 static void
3092 ehci_disown(ehci_softc_t *sc, uint16_t index, uint8_t lowspeed)
3093 {
3094         uint32_t port;
3095         uint32_t v;
3096
3097         DPRINTF("index=%d lowspeed=%d\n", index, lowspeed);
3098
3099         port = EHCI_PORTSC(index);
3100         v = EOREAD4(sc, port) & ~EHCI_PS_CLEAR;
3101         EOWRITE4(sc, port, v | EHCI_PS_PO);
3102 }
3103
3104 static usb_error_t
3105 ehci_roothub_exec(struct usb_device *udev,
3106     struct usb_device_request *req, const void **pptr, uint16_t *plength)
3107 {
3108         ehci_softc_t *sc = EHCI_BUS2SC(udev->bus);
3109         const char *str_ptr;
3110         const void *ptr;
3111         uint32_t port;
3112         uint32_t v;
3113         uint16_t len;
3114         uint16_t i;
3115         uint16_t value;
3116         uint16_t index;
3117         uint8_t l;
3118         usb_error_t err;
3119
3120         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
3121
3122         /* buffer reset */
3123         ptr = (const void *)&sc->sc_hub_desc;
3124         len = 0;
3125         err = 0;
3126
3127         value = UGETW(req->wValue);
3128         index = UGETW(req->wIndex);
3129
3130         DPRINTFN(3, "type=0x%02x request=0x%02x wLen=0x%04x "
3131             "wValue=0x%04x wIndex=0x%04x\n",
3132             req->bmRequestType, req->bRequest,
3133             UGETW(req->wLength), value, index);
3134
3135 #define C(x,y) ((x) | ((y) << 8))
3136         switch (C(req->bRequest, req->bmRequestType)) {
3137         case C(UR_CLEAR_FEATURE, UT_WRITE_DEVICE):
3138         case C(UR_CLEAR_FEATURE, UT_WRITE_INTERFACE):
3139         case C(UR_CLEAR_FEATURE, UT_WRITE_ENDPOINT):
3140                 /*
3141                  * DEVICE_REMOTE_WAKEUP and ENDPOINT_HALT are no-ops
3142                  * for the integrated root hub.
3143                  */
3144                 break;
3145         case C(UR_GET_CONFIG, UT_READ_DEVICE):
3146                 len = 1;
3147                 sc->sc_hub_desc.temp[0] = sc->sc_conf;
3148                 break;
3149         case C(UR_GET_DESCRIPTOR, UT_READ_DEVICE):
3150                 switch (value >> 8) {
3151                 case UDESC_DEVICE:
3152                         if ((value & 0xff) != 0) {
3153                                 err = USB_ERR_IOERROR;
3154                                 goto done;
3155                         }
3156                         len = sizeof(ehci_devd);
3157                         ptr = (const void *)&ehci_devd;
3158                         break;
3159                         /*
3160                          * We can't really operate at another speed,
3161                          * but the specification says we need this
3162                          * descriptor:
3163                          */
3164                 case UDESC_DEVICE_QUALIFIER:
3165                         if ((value & 0xff) != 0) {
3166                                 err = USB_ERR_IOERROR;
3167                                 goto done;
3168                         }
3169                         len = sizeof(ehci_odevd);
3170                         ptr = (const void *)&ehci_odevd;
3171                         break;
3172
3173                 case UDESC_CONFIG:
3174                         if ((value & 0xff) != 0) {
3175                                 err = USB_ERR_IOERROR;
3176                                 goto done;
3177                         }
3178                         len = sizeof(ehci_confd);
3179                         ptr = (const void *)&ehci_confd;
3180                         break;
3181
3182                 case UDESC_STRING:
3183                         switch (value & 0xff) {
3184                         case 0: /* Language table */
3185                                 str_ptr = "\001";
3186                                 break;
3187
3188                         case 1: /* Vendor */
3189                                 str_ptr = sc->sc_vendor;
3190                                 break;
3191
3192                         case 2: /* Product */
3193                                 str_ptr = "EHCI root HUB";
3194                                 break;
3195
3196                         default:
3197                                 str_ptr = "";
3198                                 break;
3199                         }
3200
3201                         len = usb_make_str_desc(
3202                             sc->sc_hub_desc.temp,
3203                             sizeof(sc->sc_hub_desc.temp),
3204                             str_ptr);
3205                         break;
3206                 default:
3207                         err = USB_ERR_IOERROR;
3208                         goto done;
3209                 }
3210                 break;
3211         case C(UR_GET_INTERFACE, UT_READ_INTERFACE):
3212                 len = 1;
3213                 sc->sc_hub_desc.temp[0] = 0;
3214                 break;
3215         case C(UR_GET_STATUS, UT_READ_DEVICE):
3216                 len = 2;
3217                 USETW(sc->sc_hub_desc.stat.wStatus, UDS_SELF_POWERED);
3218                 break;
3219         case C(UR_GET_STATUS, UT_READ_INTERFACE):
3220         case C(UR_GET_STATUS, UT_READ_ENDPOINT):
3221                 len = 2;
3222                 USETW(sc->sc_hub_desc.stat.wStatus, 0);
3223                 break;
3224         case C(UR_SET_ADDRESS, UT_WRITE_DEVICE):
3225                 if (value >= EHCI_MAX_DEVICES) {
3226                         err = USB_ERR_IOERROR;
3227                         goto done;
3228                 }
3229                 sc->sc_addr = value;
3230                 break;
3231         case C(UR_SET_CONFIG, UT_WRITE_DEVICE):
3232                 if ((value != 0) && (value != 1)) {
3233                         err = USB_ERR_IOERROR;
3234                         goto done;
3235                 }
3236                 sc->sc_conf = value;
3237                 break;
3238         case C(UR_SET_DESCRIPTOR, UT_WRITE_DEVICE):
3239                 break;
3240         case C(UR_SET_FEATURE, UT_WRITE_DEVICE):
3241         case C(UR_SET_FEATURE, UT_WRITE_INTERFACE):
3242         case C(UR_SET_FEATURE, UT_WRITE_ENDPOINT):
3243                 err = USB_ERR_IOERROR;
3244                 goto done;
3245         case C(UR_SET_INTERFACE, UT_WRITE_INTERFACE):
3246                 break;
3247         case C(UR_SYNCH_FRAME, UT_WRITE_ENDPOINT):
3248                 break;
3249                 /* Hub requests */
3250         case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_DEVICE):
3251                 break;
3252         case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_OTHER):
3253                 DPRINTFN(9, "UR_CLEAR_PORT_FEATURE\n");
3254
3255                 if ((index < 1) ||
3256                     (index > sc->sc_noport)) {
3257                         err = USB_ERR_IOERROR;
3258                         goto done;
3259                 }
3260                 port = EHCI_PORTSC(index);
3261                 v = EOREAD4(sc, port) & ~EHCI_PS_CLEAR;
3262                 switch (value) {
3263                 case UHF_PORT_ENABLE:
3264                         EOWRITE4(sc, port, v & ~EHCI_PS_PE);
3265                         break;
3266                 case UHF_PORT_SUSPEND:
3267                         if ((v & EHCI_PS_SUSP) && (!(v & EHCI_PS_FPR))) {
3268
3269                                 /*
3270                                  * waking up a High Speed device is rather
3271                                  * complicated if
3272                                  */
3273                                 EOWRITE4(sc, port, v | EHCI_PS_FPR);
3274                         }
3275                         /* wait 20ms for resume sequence to complete */
3276                         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 50);
3277
3278                         EOWRITE4(sc, port, v & ~(EHCI_PS_SUSP |
3279                             EHCI_PS_FPR | (3 << 10) /* High Speed */ ));
3280
3281                         /* 4ms settle time */
3282                         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 250);
3283                         break;
3284                 case UHF_PORT_POWER:
3285                         EOWRITE4(sc, port, v & ~EHCI_PS_PP);
3286                         break;
3287                 case UHF_PORT_TEST:
3288                         DPRINTFN(3, "clear port test "
3289                             "%d\n", index);
3290                         break;
3291                 case UHF_PORT_INDICATOR:
3292                         DPRINTFN(3, "clear port ind "
3293                             "%d\n", index);
3294                         EOWRITE4(sc, port, v & ~EHCI_PS_PIC);
3295                         break;
3296                 case UHF_C_PORT_CONNECTION:
3297                         EOWRITE4(sc, port, v | EHCI_PS_CSC);
3298                         break;
3299                 case UHF_C_PORT_ENABLE:
3300                         EOWRITE4(sc, port, v | EHCI_PS_PEC);
3301                         break;
3302                 case UHF_C_PORT_SUSPEND:
3303                         EOWRITE4(sc, port, v | EHCI_PS_SUSP);
3304                         break;
3305                 case UHF_C_PORT_OVER_CURRENT:
3306                         EOWRITE4(sc, port, v | EHCI_PS_OCC);
3307                         break;
3308                 case UHF_C_PORT_RESET:
3309                         sc->sc_isreset = 0;
3310                         break;
3311                 default:
3312                         err = USB_ERR_IOERROR;
3313                         goto done;
3314                 }
3315                 break;
3316         case C(UR_GET_DESCRIPTOR, UT_READ_CLASS_DEVICE):
3317                 if ((value & 0xff) != 0) {
3318                         err = USB_ERR_IOERROR;
3319                         goto done;
3320                 }
3321                 v = EOREAD4(sc, EHCI_HCSPARAMS);
3322
3323                 sc->sc_hub_desc.hubd = ehci_hubd;
3324                 sc->sc_hub_desc.hubd.bNbrPorts = sc->sc_noport;
3325                 USETW(sc->sc_hub_desc.hubd.wHubCharacteristics,
3326                     (EHCI_HCS_PPC(v) ? UHD_PWR_INDIVIDUAL : UHD_PWR_NO_SWITCH) |
3327                     (EHCI_HCS_P_INDICATOR(EREAD4(sc, EHCI_HCSPARAMS)) ?
3328                     UHD_PORT_IND : 0));
3329                 /* XXX can't find out? */
3330                 sc->sc_hub_desc.hubd.bPwrOn2PwrGood = 200;
3331                 for (l = 0; l < sc->sc_noport; l++) {
3332                         /* XXX can't find out? */
3333                         sc->sc_hub_desc.hubd.DeviceRemovable[l / 8] &= ~(1 << (l % 8));
3334                 }
3335                 sc->sc_hub_desc.hubd.bDescLength =
3336                     8 + ((sc->sc_noport + 7) / 8);
3337                 len = sc->sc_hub_desc.hubd.bDescLength;
3338                 break;
3339         case C(UR_GET_STATUS, UT_READ_CLASS_DEVICE):
3340                 len = 16;
3341                 bzero(sc->sc_hub_desc.temp, 16);
3342                 break;
3343         case C(UR_GET_STATUS, UT_READ_CLASS_OTHER):
3344                 DPRINTFN(9, "get port status i=%d\n",
3345                     index);
3346                 if ((index < 1) ||
3347                     (index > sc->sc_noport)) {
3348                         err = USB_ERR_IOERROR;
3349                         goto done;
3350                 }
3351                 v = EOREAD4(sc, EHCI_PORTSC(index));
3352                 DPRINTFN(9, "port status=0x%04x\n", v);
3353                 if (sc->sc_flags & (EHCI_SCFLG_FORCESPEED | EHCI_SCFLG_TT)) {
3354                         if ((v & 0xc000000) == 0x8000000)
3355                                 i = UPS_HIGH_SPEED;
3356                         else if ((v & 0xc000000) == 0x4000000)
3357                                 i = UPS_LOW_SPEED;
3358                         else
3359                                 i = 0;
3360                 } else {
3361                         i = UPS_HIGH_SPEED;
3362                 }
3363                 if (v & EHCI_PS_CS)
3364                         i |= UPS_CURRENT_CONNECT_STATUS;
3365                 if (v & EHCI_PS_PE)
3366                         i |= UPS_PORT_ENABLED;
3367                 if ((v & EHCI_PS_SUSP) && !(v & EHCI_PS_FPR))
3368                         i |= UPS_SUSPEND;
3369                 if (v & EHCI_PS_OCA)
3370                         i |= UPS_OVERCURRENT_INDICATOR;
3371                 if (v & EHCI_PS_PR)
3372                         i |= UPS_RESET;
3373                 if (v & EHCI_PS_PP)
3374                         i |= UPS_PORT_POWER;
3375                 USETW(sc->sc_hub_desc.ps.wPortStatus, i);
3376                 i = 0;
3377                 if (v & EHCI_PS_CSC)
3378                         i |= UPS_C_CONNECT_STATUS;
3379                 if (v & EHCI_PS_PEC)
3380                         i |= UPS_C_PORT_ENABLED;
3381                 if (v & EHCI_PS_OCC)
3382                         i |= UPS_C_OVERCURRENT_INDICATOR;
3383                 if (v & EHCI_PS_FPR)
3384                         i |= UPS_C_SUSPEND;
3385                 if (sc->sc_isreset)
3386                         i |= UPS_C_PORT_RESET;
3387                 USETW(sc->sc_hub_desc.ps.wPortChange, i);
3388                 len = sizeof(sc->sc_hub_desc.ps);
3389                 break;
3390         case C(UR_SET_DESCRIPTOR, UT_WRITE_CLASS_DEVICE):
3391                 err = USB_ERR_IOERROR;
3392                 goto done;
3393         case C(UR_SET_FEATURE, UT_WRITE_CLASS_DEVICE):
3394                 break;
3395         case C(UR_SET_FEATURE, UT_WRITE_CLASS_OTHER):
3396                 if ((index < 1) ||
3397                     (index > sc->sc_noport)) {
3398                         err = USB_ERR_IOERROR;
3399                         goto done;
3400                 }
3401                 port = EHCI_PORTSC(index);
3402                 v = EOREAD4(sc, port) & ~EHCI_PS_CLEAR;
3403                 switch (value) {
3404                 case UHF_PORT_ENABLE:
3405                         EOWRITE4(sc, port, v | EHCI_PS_PE);
3406                         break;
3407                 case UHF_PORT_SUSPEND:
3408                         EOWRITE4(sc, port, v | EHCI_PS_SUSP);
3409                         break;
3410                 case UHF_PORT_RESET:
3411                         DPRINTFN(6, "reset port %d\n", index);
3412 #ifdef USB_DEBUG
3413                         if (ehcinohighspeed) {
3414                                 /*
3415                                  * Connect USB device to companion
3416                                  * controller.
3417                                  */
3418                                 ehci_disown(sc, index, 1);
3419                                 break;
3420                         }
3421 #endif
3422                         if (EHCI_PS_IS_LOWSPEED(v) &&
3423                             (sc->sc_flags & EHCI_SCFLG_TT) == 0) {
3424                                 /* Low speed device, give up ownership. */
3425                                 ehci_disown(sc, index, 1);
3426                                 break;
3427                         }
3428                         /* Start reset sequence. */
3429                         v &= ~(EHCI_PS_PE | EHCI_PS_PR);
3430                         EOWRITE4(sc, port, v | EHCI_PS_PR);
3431
3432                         /* Wait for reset to complete. */
3433                         usb_pause_mtx(&sc->sc_bus.bus_mtx,
3434                             USB_MS_TO_TICKS(USB_PORT_ROOT_RESET_DELAY));
3435
3436                         /* Terminate reset sequence. */
3437                         if (!(sc->sc_flags & EHCI_SCFLG_NORESTERM))
3438                                 EOWRITE4(sc, port, v);
3439
3440                         /* Wait for HC to complete reset. */
3441                         usb_pause_mtx(&sc->sc_bus.bus_mtx,
3442                             USB_MS_TO_TICKS(EHCI_PORT_RESET_COMPLETE));
3443
3444                         v = EOREAD4(sc, port);
3445                         DPRINTF("ehci after reset, status=0x%08x\n", v);
3446                         if (v & EHCI_PS_PR) {
3447                                 device_printf(sc->sc_bus.bdev,
3448                                     "port reset timeout\n");
3449                                 err = USB_ERR_TIMEOUT;
3450                                 goto done;
3451                         }
3452                         if (!(v & EHCI_PS_PE) &&
3453                             (sc->sc_flags & EHCI_SCFLG_TT) == 0) {
3454                                 /* Not a high speed device, give up ownership.*/
3455                                 ehci_disown(sc, index, 0);
3456                                 break;
3457                         }
3458                         sc->sc_isreset = 1;
3459                         DPRINTF("ehci port %d reset, status = 0x%08x\n",
3460                             index, v);
3461                         break;
3462
3463                 case UHF_PORT_POWER:
3464                         DPRINTFN(3, "set port power %d\n", index);
3465                         EOWRITE4(sc, port, v | EHCI_PS_PP);
3466                         break;
3467
3468                 case UHF_PORT_TEST:
3469                         DPRINTFN(3, "set port test %d\n", index);
3470                         break;
3471
3472                 case UHF_PORT_INDICATOR:
3473                         DPRINTFN(3, "set port ind %d\n", index);
3474                         EOWRITE4(sc, port, v | EHCI_PS_PIC);
3475                         break;
3476
3477                 default:
3478                         err = USB_ERR_IOERROR;
3479                         goto done;
3480                 }
3481                 break;
3482         case C(UR_CLEAR_TT_BUFFER, UT_WRITE_CLASS_OTHER):
3483         case C(UR_RESET_TT, UT_WRITE_CLASS_OTHER):
3484         case C(UR_GET_TT_STATE, UT_READ_CLASS_OTHER):
3485         case C(UR_STOP_TT, UT_WRITE_CLASS_OTHER):
3486                 break;
3487         default:
3488                 err = USB_ERR_IOERROR;
3489                 goto done;
3490         }
3491 done:
3492         *plength = len;
3493         *pptr = ptr;
3494         return (err);
3495 }
3496
3497 static void
3498 ehci_xfer_setup(struct usb_setup_params *parm)
3499 {
3500         struct usb_page_search page_info;
3501         struct usb_page_cache *pc;
3502         ehci_softc_t *sc;
3503         struct usb_xfer *xfer;
3504         void *last_obj;
3505         uint32_t nqtd;
3506         uint32_t nqh;
3507         uint32_t nsitd;
3508         uint32_t nitd;
3509         uint32_t n;
3510
3511         sc = EHCI_BUS2SC(parm->udev->bus);
3512         xfer = parm->curr_xfer;
3513
3514         nqtd = 0;
3515         nqh = 0;
3516         nsitd = 0;
3517         nitd = 0;
3518
3519         /*
3520          * compute maximum number of some structures
3521          */
3522         if (parm->methods == &ehci_device_ctrl_methods) {
3523
3524                 /*
3525                  * The proof for the "nqtd" formula is illustrated like
3526                  * this:
3527                  *
3528                  * +------------------------------------+
3529                  * |                                    |
3530                  * |         |remainder ->              |
3531                  * |   +-----+---+                      |
3532                  * |   | xxx | x | frm 0                |
3533                  * |   +-----+---++                     |
3534                  * |   | xxx | xx | frm 1               |
3535                  * |   +-----+----+                     |
3536                  * |            ...                     |
3537                  * +------------------------------------+
3538                  *
3539                  * "xxx" means a completely full USB transfer descriptor
3540                  *
3541                  * "x" and "xx" means a short USB packet
3542                  *
3543                  * For the remainder of an USB transfer modulo
3544                  * "max_data_length" we need two USB transfer descriptors.
3545                  * One to transfer the remaining data and one to finalise
3546                  * with a zero length packet in case the "force_short_xfer"
3547                  * flag is set. We only need two USB transfer descriptors in
3548                  * the case where the transfer length of the first one is a
3549                  * factor of "max_frame_size". The rest of the needed USB
3550                  * transfer descriptors is given by the buffer size divided
3551                  * by the maximum data payload.
3552                  */
3553                 parm->hc_max_packet_size = 0x400;
3554                 parm->hc_max_packet_count = 1;
3555                 parm->hc_max_frame_size = EHCI_QTD_PAYLOAD_MAX;
3556                 xfer->flags_int.bdma_enable = 1;
3557
3558                 usbd_transfer_setup_sub(parm);
3559
3560                 nqh = 1;
3561                 nqtd = ((2 * xfer->nframes) + 1 /* STATUS */
3562                     + (xfer->max_data_length / xfer->max_hc_frame_size));
3563
3564         } else if (parm->methods == &ehci_device_bulk_methods) {
3565
3566                 parm->hc_max_packet_size = 0x400;
3567                 parm->hc_max_packet_count = 1;
3568                 parm->hc_max_frame_size = EHCI_QTD_PAYLOAD_MAX;
3569                 xfer->flags_int.bdma_enable = 1;
3570
3571                 usbd_transfer_setup_sub(parm);
3572
3573                 nqh = 1;
3574                 nqtd = ((2 * xfer->nframes)
3575                     + (xfer->max_data_length / xfer->max_hc_frame_size));
3576
3577         } else if (parm->methods == &ehci_device_intr_methods) {
3578
3579                 if (parm->speed == USB_SPEED_HIGH) {
3580                         parm->hc_max_packet_size = 0x400;
3581                         parm->hc_max_packet_count = 3;
3582                 } else if (parm->speed == USB_SPEED_FULL) {
3583                         parm->hc_max_packet_size = USB_FS_BYTES_PER_HS_UFRAME;
3584                         parm->hc_max_packet_count = 1;
3585                 } else {
3586                         parm->hc_max_packet_size = USB_FS_BYTES_PER_HS_UFRAME / 8;
3587                         parm->hc_max_packet_count = 1;
3588                 }
3589
3590                 parm->hc_max_frame_size = EHCI_QTD_PAYLOAD_MAX;
3591                 xfer->flags_int.bdma_enable = 1;
3592
3593                 usbd_transfer_setup_sub(parm);
3594
3595                 nqh = 1;
3596                 nqtd = ((2 * xfer->nframes)
3597                     + (xfer->max_data_length / xfer->max_hc_frame_size));
3598
3599         } else if (parm->methods == &ehci_device_isoc_fs_methods) {
3600
3601                 parm->hc_max_packet_size = 0x3FF;
3602                 parm->hc_max_packet_count = 1;
3603                 parm->hc_max_frame_size = 0x3FF;
3604                 xfer->flags_int.bdma_enable = 1;
3605
3606                 usbd_transfer_setup_sub(parm);
3607
3608                 nsitd = xfer->nframes;
3609
3610         } else if (parm->methods == &ehci_device_isoc_hs_methods) {
3611
3612                 parm->hc_max_packet_size = 0x400;
3613                 parm->hc_max_packet_count = 3;
3614                 parm->hc_max_frame_size = 0xC00;
3615                 xfer->flags_int.bdma_enable = 1;
3616
3617                 usbd_transfer_setup_sub(parm);
3618
3619                 nitd = ((xfer->nframes + 7) / 8) <<
3620                     usbd_xfer_get_fps_shift(xfer);
3621
3622         } else {
3623
3624                 parm->hc_max_packet_size = 0x400;
3625                 parm->hc_max_packet_count = 1;
3626                 parm->hc_max_frame_size = 0x400;
3627
3628                 usbd_transfer_setup_sub(parm);
3629         }
3630
3631 alloc_dma_set:
3632
3633         if (parm->err) {
3634                 return;
3635         }
3636         /*
3637          * Allocate queue heads and transfer descriptors
3638          */
3639         last_obj = NULL;
3640
3641         if (usbd_transfer_setup_sub_malloc(
3642             parm, &pc, sizeof(ehci_itd_t),
3643             EHCI_ITD_ALIGN, nitd)) {
3644                 parm->err = USB_ERR_NOMEM;
3645                 return;
3646         }
3647         if (parm->buf) {
3648                 for (n = 0; n != nitd; n++) {
3649                         ehci_itd_t *td;
3650
3651                         usbd_get_page(pc + n, 0, &page_info);
3652
3653                         td = page_info.buffer;
3654
3655                         /* init TD */
3656                         td->itd_self = htohc32(sc, page_info.physaddr | EHCI_LINK_ITD);
3657                         td->obj_next = last_obj;
3658                         td->page_cache = pc + n;
3659
3660                         last_obj = td;
3661
3662                         usb_pc_cpu_flush(pc + n);
3663                 }
3664         }
3665         if (usbd_transfer_setup_sub_malloc(
3666             parm, &pc, sizeof(ehci_sitd_t),
3667             EHCI_SITD_ALIGN, nsitd)) {
3668                 parm->err = USB_ERR_NOMEM;
3669                 return;
3670         }
3671         if (parm->buf) {
3672                 for (n = 0; n != nsitd; n++) {
3673                         ehci_sitd_t *td;
3674
3675                         usbd_get_page(pc + n, 0, &page_info);
3676
3677                         td = page_info.buffer;
3678
3679                         /* init TD */
3680                         td->sitd_self = htohc32(sc, page_info.physaddr | EHCI_LINK_SITD);
3681                         td->obj_next = last_obj;
3682                         td->page_cache = pc + n;
3683
3684                         last_obj = td;
3685
3686                         usb_pc_cpu_flush(pc + n);
3687                 }
3688         }
3689         if (usbd_transfer_setup_sub_malloc(
3690             parm, &pc, sizeof(ehci_qtd_t),
3691             EHCI_QTD_ALIGN, nqtd)) {
3692                 parm->err = USB_ERR_NOMEM;
3693                 return;
3694         }
3695         if (parm->buf) {
3696                 for (n = 0; n != nqtd; n++) {
3697                         ehci_qtd_t *qtd;
3698
3699                         usbd_get_page(pc + n, 0, &page_info);
3700
3701                         qtd = page_info.buffer;
3702
3703                         /* init TD */
3704                         qtd->qtd_self = htohc32(sc, page_info.physaddr);
3705                         qtd->obj_next = last_obj;
3706                         qtd->page_cache = pc + n;
3707
3708                         last_obj = qtd;
3709
3710                         usb_pc_cpu_flush(pc + n);
3711                 }
3712         }
3713         xfer->td_start[xfer->flags_int.curr_dma_set] = last_obj;
3714
3715         last_obj = NULL;
3716
3717         if (usbd_transfer_setup_sub_malloc(
3718             parm, &pc, sizeof(ehci_qh_t),
3719             EHCI_QH_ALIGN, nqh)) {
3720                 parm->err = USB_ERR_NOMEM;
3721                 return;
3722         }
3723         if (parm->buf) {
3724                 for (n = 0; n != nqh; n++) {
3725                         ehci_qh_t *qh;
3726
3727                         usbd_get_page(pc + n, 0, &page_info);
3728
3729                         qh = page_info.buffer;
3730
3731                         /* init QH */
3732                         qh->qh_self = htohc32(sc, page_info.physaddr | EHCI_LINK_QH);
3733                         qh->obj_next = last_obj;
3734                         qh->page_cache = pc + n;
3735
3736                         last_obj = qh;
3737
3738                         usb_pc_cpu_flush(pc + n);
3739                 }
3740         }
3741         xfer->qh_start[xfer->flags_int.curr_dma_set] = last_obj;
3742
3743         if (!xfer->flags_int.curr_dma_set) {
3744                 xfer->flags_int.curr_dma_set = 1;
3745                 goto alloc_dma_set;
3746         }
3747 }
3748
3749 static void
3750 ehci_xfer_unsetup(struct usb_xfer *xfer)
3751 {
3752         return;
3753 }
3754
3755 static void
3756 ehci_ep_init(struct usb_device *udev, struct usb_endpoint_descriptor *edesc,
3757     struct usb_endpoint *ep)
3758 {
3759         ehci_softc_t *sc = EHCI_BUS2SC(udev->bus);
3760
3761         DPRINTFN(2, "endpoint=%p, addr=%d, endpt=%d, mode=%d (%d)\n",
3762             ep, udev->address,
3763             edesc->bEndpointAddress, udev->flags.usb_mode,
3764             sc->sc_addr);
3765
3766         if (udev->flags.usb_mode != USB_MODE_HOST) {
3767                 /* not supported */
3768                 return;
3769         }
3770         if (udev->device_index != sc->sc_addr) {
3771
3772                 if ((udev->speed != USB_SPEED_HIGH) &&
3773                     ((udev->hs_hub_addr == 0) ||
3774                     (udev->hs_port_no == 0) ||
3775                     (udev->parent_hs_hub == NULL) ||
3776                     (udev->parent_hs_hub->hub == NULL))) {
3777                         /* We need a transaction translator */
3778                         goto done;
3779                 }
3780                 switch (edesc->bmAttributes & UE_XFERTYPE) {
3781                 case UE_CONTROL:
3782                         ep->methods = &ehci_device_ctrl_methods;
3783                         break;
3784                 case UE_INTERRUPT:
3785                         ep->methods = &ehci_device_intr_methods;
3786                         break;
3787                 case UE_ISOCHRONOUS:
3788                         if (udev->speed == USB_SPEED_HIGH) {
3789                                 ep->methods = &ehci_device_isoc_hs_methods;
3790                         } else if (udev->speed == USB_SPEED_FULL) {
3791                                 ep->methods = &ehci_device_isoc_fs_methods;
3792                         }
3793                         break;
3794                 case UE_BULK:
3795                         if (udev->speed != USB_SPEED_LOW) {
3796                                 ep->methods = &ehci_device_bulk_methods;
3797                         }
3798                         break;
3799                 default:
3800                         /* do nothing */
3801                         break;
3802                 }
3803         }
3804 done:
3805         return;
3806 }
3807
3808 static void
3809 ehci_get_dma_delay(struct usb_bus *bus, uint32_t *pus)
3810 {
3811         /*
3812          * Wait until the hardware has finished any possible use of
3813          * the transfer descriptor(s) and QH
3814          */
3815         *pus = (188);                   /* microseconds */
3816 }
3817
3818 static void
3819 ehci_device_resume(struct usb_device *udev)
3820 {
3821         ehci_softc_t *sc = EHCI_BUS2SC(udev->bus);
3822         struct usb_xfer *xfer;
3823         struct usb_pipe_methods *methods;
3824
3825         DPRINTF("\n");
3826
3827         USB_BUS_LOCK(udev->bus);
3828
3829         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
3830
3831                 if (xfer->xroot->udev == udev) {
3832
3833                         methods = xfer->endpoint->methods;
3834
3835                         if ((methods == &ehci_device_bulk_methods) ||
3836                             (methods == &ehci_device_ctrl_methods)) {
3837                                 EHCI_APPEND_QH(xfer->qh_start[xfer->flags_int.curr_dma_set],
3838                                     sc->sc_async_p_last);
3839                         }
3840                         if (methods == &ehci_device_intr_methods) {
3841                                 EHCI_APPEND_QH(xfer->qh_start[xfer->flags_int.curr_dma_set],
3842                                     sc->sc_intr_p_last[xfer->qh_pos]);
3843                         }
3844                 }
3845         }
3846
3847         USB_BUS_UNLOCK(udev->bus);
3848
3849         return;
3850 }
3851
3852 static void
3853 ehci_device_suspend(struct usb_device *udev)
3854 {
3855         ehci_softc_t *sc = EHCI_BUS2SC(udev->bus);
3856         struct usb_xfer *xfer;
3857         struct usb_pipe_methods *methods;
3858
3859         DPRINTF("\n");
3860
3861         USB_BUS_LOCK(udev->bus);
3862
3863         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
3864
3865                 if (xfer->xroot->udev == udev) {
3866
3867                         methods = xfer->endpoint->methods;
3868
3869                         if ((methods == &ehci_device_bulk_methods) ||
3870                             (methods == &ehci_device_ctrl_methods)) {
3871                                 EHCI_REMOVE_QH(xfer->qh_start[xfer->flags_int.curr_dma_set],
3872                                     sc->sc_async_p_last);
3873                         }
3874                         if (methods == &ehci_device_intr_methods) {
3875                                 EHCI_REMOVE_QH(xfer->qh_start[xfer->flags_int.curr_dma_set],
3876                                     sc->sc_intr_p_last[xfer->qh_pos]);
3877                         }
3878                 }
3879         }
3880
3881         USB_BUS_UNLOCK(udev->bus);
3882
3883         return;
3884 }
3885
3886 static void
3887 ehci_set_hw_power(struct usb_bus *bus)
3888 {
3889         ehci_softc_t *sc = EHCI_BUS2SC(bus);
3890         uint32_t temp;
3891         uint32_t flags;
3892
3893         DPRINTF("\n");
3894
3895         USB_BUS_LOCK(bus);
3896
3897         flags = bus->hw_power_state;
3898
3899         temp = EOREAD4(sc, EHCI_USBCMD);
3900
3901         temp &= ~(EHCI_CMD_ASE | EHCI_CMD_PSE);
3902
3903         if (flags & (USB_HW_POWER_CONTROL |
3904             USB_HW_POWER_BULK)) {
3905                 DPRINTF("Async is active\n");
3906                 temp |= EHCI_CMD_ASE;
3907         }
3908         if (flags & (USB_HW_POWER_INTERRUPT |
3909             USB_HW_POWER_ISOC)) {
3910                 DPRINTF("Periodic is active\n");
3911                 temp |= EHCI_CMD_PSE;
3912         }
3913         EOWRITE4(sc, EHCI_USBCMD, temp);
3914
3915         USB_BUS_UNLOCK(bus);
3916
3917         return;
3918 }
3919
3920 struct usb_bus_methods ehci_bus_methods =
3921 {
3922         .endpoint_init = ehci_ep_init,
3923         .xfer_setup = ehci_xfer_setup,
3924         .xfer_unsetup = ehci_xfer_unsetup,
3925         .get_dma_delay = ehci_get_dma_delay,
3926         .device_resume = ehci_device_resume,
3927         .device_suspend = ehci_device_suspend,
3928         .set_hw_power = ehci_set_hw_power,
3929         .roothub_exec = ehci_roothub_exec,
3930         .xfer_poll = ehci_do_poll,
3931 };