sys/dev/usb/controller/ehci.c

   1 /*-
   2  * Copyright (c) 2008 Hans Petter Selasky. All rights reserved.
   3  * Copyright (c) 2004 The NetBSD Foundation, Inc. All rights reserved.
   4  * Copyright (c) 2004 Lennart Augustsson. All rights reserved.
   5  * Copyright (c) 2004 Charles M. Hannum. All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  26  * SUCH DAMAGE.
  27  */
  28
  29 /*
  30  * USB Enhanced Host Controller Driver, a.k.a. USB 2.0 controller.
  31  *
  32  * The EHCI 0.96 spec can be found at
  33  * http://developer.intel.com/technology/usb/download/ehci-r096.pdf
  34  * The EHCI 1.0 spec can be found at
  35  * http://developer.intel.com/technology/usb/download/ehci-r10.pdf
  36  * and the USB 2.0 spec at
  37  * http://www.usb.org/developers/docs/usb_20.zip
  38  *
  39  */
  40
  41 /*
  42  * TODO:
  43  * 1) command failures are not recovered correctly
  44  */
  45
  46 #include <sys/cdefs.h>
  47 __FBSDID("$FreeBSD$");
  48
  49 #include <sys/stdint.h>
  50 #include <sys/stddef.h>
  51 #include <sys/param.h>
  52 #include <sys/queue.h>
  53 #include <sys/types.h>
  54 #include <sys/systm.h>
  55 #include <sys/kernel.h>
  56 #include <sys/bus.h>
  57 #include <sys/linker_set.h>
  58 #include <sys/module.h>
  59 #include <sys/lock.h>
  60 #include <sys/mutex.h>
  61 #include <sys/condvar.h>
  62 #include <sys/sysctl.h>
  63 #include <sys/sx.h>
  64 #include <sys/unistd.h>
  65 #include <sys/callout.h>
  66 #include <sys/malloc.h>
  67 #include <sys/priv.h>
  68
  69 #include <dev/usb/usb.h>
  70 #include <dev/usb/usbdi.h>
  71
  72 #define USB_DEBUG_VAR ehcidebug
  73
  74 #include <dev/usb/usb_core.h>
  75 #include <dev/usb/usb_debug.h>
  76 #include <dev/usb/usb_busdma.h>
  77 #include <dev/usb/usb_process.h>
  78 #include <dev/usb/usb_transfer.h>
  79 #include <dev/usb/usb_device.h>
  80 #include <dev/usb/usb_hub.h>
  81 #include <dev/usb/usb_util.h>
  82
  83 #include <dev/usb/usb_controller.h>
  84 #include <dev/usb/usb_bus.h>
  85 #include <dev/usb/controller/ehci.h>
  86 #include <dev/usb/controller/ehcireg.h>
  87
  88 #define EHCI_BUS2SC(bus) \
  89    ((ehci_softc_t *)(((uint8_t *)(bus)) - \
  90     ((uint8_t *)&(((ehci_softc_t *)0)->sc_bus))))
  91
  92 #if USB_DEBUG
  93 static int ehcidebug = 0;
  94 static int ehcinohighspeed = 0;
  95 static int ehciiaadbug = 0;
  96 static int ehcilostintrbug = 0;
  97
  98 SYSCTL_NODE(_hw_usb, OID_AUTO, ehci, CTLFLAG_RW, 0, "USB ehci");
  99 SYSCTL_INT(_hw_usb_ehci, OID_AUTO, debug, CTLFLAG_RW,
 100     &ehcidebug, 0, "Debug level");
 101 SYSCTL_INT(_hw_usb_ehci, OID_AUTO, no_hs, CTLFLAG_RW,
 102     &ehcinohighspeed, 0, "Disable High Speed USB");
 103 SYSCTL_INT(_hw_usb_ehci, OID_AUTO, iaadbug, CTLFLAG_RW,
 104     &ehciiaadbug, 0, "Enable doorbell bug workaround");
 105 SYSCTL_INT(_hw_usb_ehci, OID_AUTO, lostintrbug, CTLFLAG_RW,
 106     &ehcilostintrbug, 0, "Enable lost interrupt bug workaround");
 107
 108 TUNABLE_INT("hw.usb.ehci.debug", &ehcidebug);
 109 TUNABLE_INT("hw.usb.ehci.no_hs", &ehcinohighspeed);
 110 TUNABLE_INT("hw.usb.ehci.iaadbug", &ehciiaadbug);
 111 TUNABLE_INT("hw.usb.ehci.lostintrbug", &ehcilostintrbug);
 112
 113 static void ehci_dump_regs(ehci_softc_t *sc);
 114 static void ehci_dump_sqh(ehci_softc_t *sc, ehci_qh_t *sqh);
 115
 116 #endif
 117
 118 #define EHCI_INTR_ENDPT 1
 119
 120 extern struct usb_bus_methods ehci_bus_methods;
 121 extern struct usb_pipe_methods ehci_device_bulk_methods;
 122 extern struct usb_pipe_methods ehci_device_ctrl_methods;
 123 extern struct usb_pipe_methods ehci_device_intr_methods;
 124 extern struct usb_pipe_methods ehci_device_isoc_fs_methods;
 125 extern struct usb_pipe_methods ehci_device_isoc_hs_methods;
 126
 127 static void ehci_do_poll(struct usb_bus *);
 128 static void ehci_device_done(struct usb_xfer *, usb_error_t);
 129 static uint8_t ehci_check_transfer(struct usb_xfer *);
 130 static void ehci_timeout(void *);
 131 static void ehci_poll_timeout(void *);
 132
 133 static void ehci_root_intr(ehci_softc_t *sc);
 134
 135 struct ehci_std_temp {
 136         ehci_softc_t *sc;
 137         struct usb_page_cache *pc;
 138         ehci_qtd_t *td;
 139         ehci_qtd_t *td_next;
 140         uint32_t average;
 141         uint32_t qtd_status;
 142         uint32_t len;
 143         uint16_t max_frame_size;
 144         uint8_t shortpkt;
 145         uint8_t auto_data_toggle;
 146         uint8_t setup_alt_next;
 147         uint8_t last_frame;
 148         uint8_t can_use_next;
 149 };
 150
 151 void
 152 ehci_iterate_hw_softc(struct usb_bus *bus, usb_bus_mem_sub_cb_t *cb)
 153 {
 154         ehci_softc_t *sc = EHCI_BUS2SC(bus);
 155         uint32_t i;
 156
 157         cb(bus, &sc->sc_hw.pframes_pc, &sc->sc_hw.pframes_pg,
 158             sizeof(uint32_t) * EHCI_FRAMELIST_COUNT, EHCI_FRAMELIST_ALIGN);
 159
 160         cb(bus, &sc->sc_hw.async_start_pc, &sc->sc_hw.async_start_pg,
 161             sizeof(ehci_qh_t), EHCI_QH_ALIGN);
 162
 163         for (i = 0; i != EHCI_VIRTUAL_FRAMELIST_COUNT; i++) {
 164                 cb(bus, sc->sc_hw.intr_start_pc + i,
 165                     sc->sc_hw.intr_start_pg + i,
 166                     sizeof(ehci_qh_t), EHCI_QH_ALIGN);
 167         }
 168
 169         for (i = 0; i != EHCI_VIRTUAL_FRAMELIST_COUNT; i++) {
 170                 cb(bus, sc->sc_hw.isoc_hs_start_pc + i,
 171                     sc->sc_hw.isoc_hs_start_pg + i,
 172                     sizeof(ehci_itd_t), EHCI_ITD_ALIGN);
 173         }
 174
 175         for (i = 0; i != EHCI_VIRTUAL_FRAMELIST_COUNT; i++) {
 176                 cb(bus, sc->sc_hw.isoc_fs_start_pc + i,
 177                     sc->sc_hw.isoc_fs_start_pg + i,
 178                     sizeof(ehci_sitd_t), EHCI_SITD_ALIGN);
 179         }
 180 }
 181
 182 usb_error_t
 183 ehci_reset(ehci_softc_t *sc)
 184 {
 185         uint32_t hcr;
 186         int i;
 187
 188         EOWRITE4(sc, EHCI_USBCMD, EHCI_CMD_HCRESET);
 189         for (i = 0; i < 100; i++) {
 190                 usb_pause_mtx(NULL, hz / 1000);
 191                 hcr = EOREAD4(sc, EHCI_USBCMD) & EHCI_CMD_HCRESET;
 192                 if (!hcr) {
 193                         if (sc->sc_flags & (EHCI_SCFLG_SETMODE | EHCI_SCFLG_BIGEMMIO)) {
 194                                 /*
 195                                  * Force USBMODE as requested.  Controllers
 196                                  * may have multiple operating modes.
 197                                  */
 198                                 uint32_t usbmode = EOREAD4(sc, EHCI_USBMODE);
 199                                 if (sc->sc_flags & EHCI_SCFLG_SETMODE) {
 200                                         usbmode = (usbmode &~ EHCI_UM_CM) | EHCI_UM_CM_HOST;
 201                                         device_printf(sc->sc_bus.bdev,
 202                                             "set host controller mode\n");
 203                                 }
 204                                 if (sc->sc_flags & EHCI_SCFLG_BIGEMMIO) {
 205                                         usbmode = (usbmode &~ EHCI_UM_ES) | EHCI_UM_ES_BE;
 206                                         device_printf(sc->sc_bus.bdev,
 207                                             "set big-endian mode\n");
 208                                 }
 209                                 EOWRITE4(sc,  EHCI_USBMODE, usbmode);
 210                         }
 211                         return (0);
 212                 }
 213         }
 214         device_printf(sc->sc_bus.bdev, "reset timeout\n");
 215         return (USB_ERR_IOERROR);
 216 }
 217
 218 static usb_error_t
 219 ehci_hcreset(ehci_softc_t *sc)
 220 {
 221         uint32_t hcr;
 222         int i;
 223
 224         EOWRITE4(sc, EHCI_USBCMD, 0);   /* Halt controller */
 225         for (i = 0; i < 100; i++) {
 226                 usb_pause_mtx(NULL, hz / 1000);
 227                 hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
 228                 if (hcr)
 229                         break;
 230         }
 231         if (!hcr)
 232                 /*
 233                  * Fall through and try reset anyway even though
 234                  * Table 2-9 in the EHCI spec says this will result
 235                  * in undefined behavior.
 236                  */
 237                 device_printf(sc->sc_bus.bdev, "stop timeout\n");
 238
 239         return ehci_reset(sc);
 240 }
 241
 242 usb_error_t
 243 ehci_init(ehci_softc_t *sc)
 244 {
 245         struct usb_page_search buf_res;
 246         uint32_t version;
 247         uint32_t sparams;
 248         uint32_t cparams;
 249         uint32_t hcr;
 250         uint16_t i;
 251         uint16_t x;
 252         uint16_t y;
 253         uint16_t bit;
 254         usb_error_t err = 0;
 255
 256         DPRINTF("start\n");
 257
 258         usb_callout_init_mtx(&sc->sc_tmo_pcd, &sc->sc_bus.bus_mtx, 0);
 259         usb_callout_init_mtx(&sc->sc_tmo_poll, &sc->sc_bus.bus_mtx, 0);
 260
 261 #if USB_DEBUG
 262         if (ehciiaadbug)
 263                 sc->sc_flags |= EHCI_SCFLG_IAADBUG;
 264         if (ehcilostintrbug)
 265                 sc->sc_flags |= EHCI_SCFLG_LOSTINTRBUG;
 266         if (ehcidebug > 2) {
 267                 ehci_dump_regs(sc);
 268         }
 269 #endif
 270
 271         sc->sc_offs = EREAD1(sc, EHCI_CAPLENGTH);
 272
 273         version = EREAD2(sc, EHCI_HCIVERSION);
 274         device_printf(sc->sc_bus.bdev, "EHCI version %x.%x\n",
 275             version >> 8, version & 0xff);
 276
 277         sparams = EREAD4(sc, EHCI_HCSPARAMS);
 278         DPRINTF("sparams=0x%x\n", sparams);
 279
 280         sc->sc_noport = EHCI_HCS_N_PORTS(sparams);
 281         cparams = EREAD4(sc, EHCI_HCCPARAMS);
 282         DPRINTF("cparams=0x%x\n", cparams);
 283
 284         if (EHCI_HCC_64BIT(cparams)) {
 285                 DPRINTF("HCC uses 64-bit structures\n");
 286
 287                 /* MUST clear segment register if 64 bit capable */
 288                 EWRITE4(sc, EHCI_CTRLDSSEGMENT, 0);
 289         }
 290         sc->sc_bus.usbrev = USB_REV_2_0;
 291
 292         /* Reset the controller */
 293         DPRINTF("%s: resetting\n", device_get_nameunit(sc->sc_bus.bdev));
 294
 295         err = ehci_hcreset(sc);
 296         if (err) {
 297                 device_printf(sc->sc_bus.bdev, "reset timeout\n");
 298                 return (err);
 299         }
 300         /*
 301          * use current frame-list-size selection 0: 1024*4 bytes 1:  512*4
 302          * bytes 2:  256*4 bytes 3:      unknown
 303          */
 304         if (EHCI_CMD_FLS(EOREAD4(sc, EHCI_USBCMD)) == 3) {
 305                 device_printf(sc->sc_bus.bdev, "invalid frame-list-size\n");
 306                 return (USB_ERR_IOERROR);
 307         }
 308         /* set up the bus struct */
 309         sc->sc_bus.methods = &ehci_bus_methods;
 310
 311         sc->sc_eintrs = EHCI_NORMAL_INTRS;
 312
 313         for (i = 0; i < EHCI_VIRTUAL_FRAMELIST_COUNT; i++) {
 314                 ehci_qh_t *qh;
 315
 316                 usbd_get_page(sc->sc_hw.intr_start_pc + i, 0, &buf_res);
 317
 318                 qh = buf_res.buffer;
 319
 320                 /* initialize page cache pointer */
 321
 322                 qh->page_cache = sc->sc_hw.intr_start_pc + i;
 323
 324                 /* store a pointer to queue head */
 325
 326                 sc->sc_intr_p_last[i] = qh;
 327
 328                 qh->qh_self =
 329                     htohc32(sc, buf_res.physaddr) |
 330                     htohc32(sc, EHCI_LINK_QH);
 331
 332                 qh->qh_endp =
 333                     htohc32(sc, EHCI_QH_SET_EPS(EHCI_QH_SPEED_HIGH));
 334                 qh->qh_endphub =
 335                     htohc32(sc, EHCI_QH_SET_MULT(1));
 336                 qh->qh_curqtd = 0;
 337
 338                 qh->qh_qtd.qtd_next =
 339                     htohc32(sc, EHCI_LINK_TERMINATE);
 340                 qh->qh_qtd.qtd_altnext =
 341                     htohc32(sc, EHCI_LINK_TERMINATE);
 342                 qh->qh_qtd.qtd_status =
 343                     htohc32(sc, EHCI_QTD_HALTED);
 344         }
 345
 346         /*
 347          * the QHs are arranged to give poll intervals that are
 348          * powers of 2 times 1ms
 349          */
 350         bit = EHCI_VIRTUAL_FRAMELIST_COUNT / 2;
 351         while (bit) {
 352                 x = bit;
 353                 while (x & bit) {
 354                         ehci_qh_t *qh_x;
 355                         ehci_qh_t *qh_y;
 356
 357                         y = (x ^ bit) | (bit / 2);
 358
 359                         qh_x = sc->sc_intr_p_last[x];
 360                         qh_y = sc->sc_intr_p_last[y];
 361
 362                         /*
 363                          * the next QH has half the poll interval
 364                          */
 365                         qh_x->qh_link = qh_y->qh_self;
 366
 367                         x++;
 368                 }
 369                 bit >>= 1;
 370         }
 371
 372         if (1) {
 373                 ehci_qh_t *qh;
 374
 375                 qh = sc->sc_intr_p_last[0];
 376
 377                 /* the last (1ms) QH terminates */
 378                 qh->qh_link = htohc32(sc, EHCI_LINK_TERMINATE);
 379         }
 380         for (i = 0; i < EHCI_VIRTUAL_FRAMELIST_COUNT; i++) {
 381                 ehci_sitd_t *sitd;
 382                 ehci_itd_t *itd;
 383
 384                 usbd_get_page(sc->sc_hw.isoc_fs_start_pc + i, 0, &buf_res);
 385
 386                 sitd = buf_res.buffer;
 387
 388                 /* initialize page cache pointer */
 389
 390                 sitd->page_cache = sc->sc_hw.isoc_fs_start_pc + i;
 391
 392                 /* store a pointer to the transfer descriptor */
 393
 394                 sc->sc_isoc_fs_p_last[i] = sitd;
 395
 396                 /* initialize full speed isochronous */
 397
 398                 sitd->sitd_self =
 399                     htohc32(sc, buf_res.physaddr) |
 400                     htohc32(sc, EHCI_LINK_SITD);
 401
 402                 sitd->sitd_back =
 403                     htohc32(sc, EHCI_LINK_TERMINATE);
 404
 405                 sitd->sitd_next =
 406                     sc->sc_intr_p_last[i | (EHCI_VIRTUAL_FRAMELIST_COUNT / 2)]->qh_self;
 407
 408
 409                 usbd_get_page(sc->sc_hw.isoc_hs_start_pc + i, 0, &buf_res);
 410
 411                 itd = buf_res.buffer;
 412
 413                 /* initialize page cache pointer */
 414
 415                 itd->page_cache = sc->sc_hw.isoc_hs_start_pc + i;
 416
 417                 /* store a pointer to the transfer descriptor */
 418
 419                 sc->sc_isoc_hs_p_last[i] = itd;
 420
 421                 /* initialize high speed isochronous */
 422
 423                 itd->itd_self =
 424                     htohc32(sc, buf_res.physaddr) |
 425                     htohc32(sc, EHCI_LINK_ITD);
 426
 427                 itd->itd_next =
 428                     sitd->sitd_self;
 429         }
 430
 431         usbd_get_page(&sc->sc_hw.pframes_pc, 0, &buf_res);
 432
 433         if (1) {
 434                 uint32_t *pframes;
 435
 436                 pframes = buf_res.buffer;
 437
 438                 /*
 439                  * execution order:
 440                  * pframes -> high speed isochronous ->
 441                  *    full speed isochronous -> interrupt QH's
 442                  */
 443                 for (i = 0; i < EHCI_FRAMELIST_COUNT; i++) {
 444                         pframes[i] = sc->sc_isoc_hs_p_last
 445                             [i & (EHCI_VIRTUAL_FRAMELIST_COUNT - 1)]->itd_self;
 446                 }
 447         }
 448         /* setup sync list pointer */
 449         EOWRITE4(sc, EHCI_PERIODICLISTBASE, buf_res.physaddr);
 450
 451         usbd_get_page(&sc->sc_hw.async_start_pc, 0, &buf_res);
 452
 453         if (1) {
 454
 455                 ehci_qh_t *qh;
 456
 457                 qh = buf_res.buffer;
 458
 459                 /* initialize page cache pointer */
 460
 461                 qh->page_cache = &sc->sc_hw.async_start_pc;
 462
 463                 /* store a pointer to the queue head */
 464
 465                 sc->sc_async_p_last = qh;
 466
 467                 /* init dummy QH that starts the async list */
 468
 469                 qh->qh_self =
 470                     htohc32(sc, buf_res.physaddr) |
 471                     htohc32(sc, EHCI_LINK_QH);
 472
 473                 /* fill the QH */
 474                 qh->qh_endp =
 475                     htohc32(sc, EHCI_QH_SET_EPS(EHCI_QH_SPEED_HIGH) | EHCI_QH_HRECL);
 476                 qh->qh_endphub = htohc32(sc, EHCI_QH_SET_MULT(1));
 477                 qh->qh_link = qh->qh_self;
 478                 qh->qh_curqtd = 0;
 479
 480                 /* fill the overlay qTD */
 481                 qh->qh_qtd.qtd_next = htohc32(sc, EHCI_LINK_TERMINATE);
 482                 qh->qh_qtd.qtd_altnext = htohc32(sc, EHCI_LINK_TERMINATE);
 483                 qh->qh_qtd.qtd_status = htohc32(sc, EHCI_QTD_HALTED);
 484         }
 485         /* flush all cache into memory */
 486
 487         usb_bus_mem_flush_all(&sc->sc_bus, &ehci_iterate_hw_softc);
 488
 489 #if USB_DEBUG
 490         if (ehcidebug) {
 491                 ehci_dump_sqh(sc, sc->sc_async_p_last);
 492         }
 493 #endif
 494
 495         /* setup async list pointer */
 496         EOWRITE4(sc, EHCI_ASYNCLISTADDR, buf_res.physaddr | EHCI_LINK_QH);
 497
 498
 499         /* enable interrupts */
 500         EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
 501
 502         /* turn on controller */
 503         EOWRITE4(sc, EHCI_USBCMD,
 504             EHCI_CMD_ITC_1 |            /* 1 microframes interrupt delay */
 505             (EOREAD4(sc, EHCI_USBCMD) & EHCI_CMD_FLS_M) |
 506             EHCI_CMD_ASE |
 507             EHCI_CMD_PSE |
 508             EHCI_CMD_RS);
 509
 510         /* Take over port ownership */
 511         EOWRITE4(sc, EHCI_CONFIGFLAG, EHCI_CONF_CF);
 512
 513         for (i = 0; i < 100; i++) {
 514                 usb_pause_mtx(NULL, hz / 1000);
 515                 hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
 516                 if (!hcr) {
 517                         break;
 518                 }
 519         }
 520         if (hcr) {
 521                 device_printf(sc->sc_bus.bdev, "run timeout\n");
 522                 return (USB_ERR_IOERROR);
 523         }
 524
 525         if (!err) {
 526                 /* catch any lost interrupts */
 527                 ehci_do_poll(&sc->sc_bus);
 528         }
 529         return (err);
 530 }
 531
 532 /*
 533  * shut down the controller when the system is going down
 534  */
 535 void
 536 ehci_detach(ehci_softc_t *sc)
 537 {
 538         USB_BUS_LOCK(&sc->sc_bus);
 539
 540         usb_callout_stop(&sc->sc_tmo_pcd);
 541         usb_callout_stop(&sc->sc_tmo_poll);
 542
 543         EOWRITE4(sc, EHCI_USBINTR, 0);
 544         USB_BUS_UNLOCK(&sc->sc_bus);
 545
 546         if (ehci_hcreset(sc)) {
 547                 DPRINTF("reset failed!\n");
 548         }
 549
 550         /* XXX let stray task complete */
 551         usb_pause_mtx(NULL, hz / 20);
 552
 553         usb_callout_drain(&sc->sc_tmo_pcd);
 554         usb_callout_drain(&sc->sc_tmo_poll);
 555 }
 556
 557 void
 558 ehci_suspend(ehci_softc_t *sc)
 559 {
 560         uint32_t cmd;
 561         uint32_t hcr;
 562         uint8_t i;
 563
 564         USB_BUS_LOCK(&sc->sc_bus);
 565
 566         for (i = 1; i <= sc->sc_noport; i++) {
 567                 cmd = EOREAD4(sc, EHCI_PORTSC(i));
 568                 if (((cmd & EHCI_PS_PO) == 0) &&
 569                     ((cmd & EHCI_PS_PE) == EHCI_PS_PE)) {
 570                         EOWRITE4(sc, EHCI_PORTSC(i),
 571                             cmd | EHCI_PS_SUSP);
 572                 }
 573         }
 574
 575         sc->sc_cmd = EOREAD4(sc, EHCI_USBCMD);
 576
 577         cmd = sc->sc_cmd & ~(EHCI_CMD_ASE | EHCI_CMD_PSE);
 578         EOWRITE4(sc, EHCI_USBCMD, cmd);
 579
 580         for (i = 0; i < 100; i++) {
 581                 hcr = EOREAD4(sc, EHCI_USBSTS) &
 582                     (EHCI_STS_ASS | EHCI_STS_PSS);
 583
 584                 if (hcr == 0) {
 585                         break;
 586                 }
 587                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 1000);
 588         }
 589
 590         if (hcr != 0) {
 591                 device_printf(sc->sc_bus.bdev, "reset timeout\n");
 592         }
 593         cmd &= ~EHCI_CMD_RS;
 594         EOWRITE4(sc, EHCI_USBCMD, cmd);
 595
 596         for (i = 0; i < 100; i++) {
 597                 hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
 598                 if (hcr == EHCI_STS_HCH) {
 599                         break;
 600                 }
 601                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 1000);
 602         }
 603
 604         if (hcr != EHCI_STS_HCH) {
 605                 device_printf(sc->sc_bus.bdev,
 606                     "config timeout\n");
 607         }
 608         USB_BUS_UNLOCK(&sc->sc_bus);
 609 }
 610
 611 void
 612 ehci_resume(ehci_softc_t *sc)
 613 {
 614         struct usb_page_search buf_res;
 615         uint32_t cmd;
 616         uint32_t hcr;
 617         uint8_t i;
 618
 619         USB_BUS_LOCK(&sc->sc_bus);
 620
 621         /* restore things in case the bios doesn't */
 622         EOWRITE4(sc, EHCI_CTRLDSSEGMENT, 0);
 623
 624         usbd_get_page(&sc->sc_hw.pframes_pc, 0, &buf_res);
 625         EOWRITE4(sc, EHCI_PERIODICLISTBASE, buf_res.physaddr);
 626
 627         usbd_get_page(&sc->sc_hw.async_start_pc, 0, &buf_res);
 628         EOWRITE4(sc, EHCI_ASYNCLISTADDR, buf_res.physaddr | EHCI_LINK_QH);
 629
 630         EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
 631
 632         hcr = 0;
 633         for (i = 1; i <= sc->sc_noport; i++) {
 634                 cmd = EOREAD4(sc, EHCI_PORTSC(i));
 635                 if (((cmd & EHCI_PS_PO) == 0) &&
 636                     ((cmd & EHCI_PS_SUSP) == EHCI_PS_SUSP)) {
 637                         EOWRITE4(sc, EHCI_PORTSC(i),
 638                             cmd | EHCI_PS_FPR);
 639                         hcr = 1;
 640                 }
 641         }
 642
 643         if (hcr) {
 644                 usb_pause_mtx(&sc->sc_bus.bus_mtx,
 645                     USB_MS_TO_TICKS(USB_RESUME_WAIT));
 646
 647                 for (i = 1; i <= sc->sc_noport; i++) {
 648                         cmd = EOREAD4(sc, EHCI_PORTSC(i));
 649                         if (((cmd & EHCI_PS_PO) == 0) &&
 650                             ((cmd & EHCI_PS_SUSP) == EHCI_PS_SUSP)) {
 651                                 EOWRITE4(sc, EHCI_PORTSC(i),
 652                                     cmd & ~EHCI_PS_FPR);
 653                         }
 654                 }
 655         }
 656         EOWRITE4(sc, EHCI_USBCMD, sc->sc_cmd);
 657
 658         for (i = 0; i < 100; i++) {
 659                 hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
 660                 if (hcr != EHCI_STS_HCH) {
 661                         break;
 662                 }
 663                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 1000);
 664         }
 665         if (hcr == EHCI_STS_HCH) {
 666                 device_printf(sc->sc_bus.bdev, "config timeout\n");
 667         }
 668
 669         USB_BUS_UNLOCK(&sc->sc_bus);
 670
 671         usb_pause_mtx(NULL,
 672             USB_MS_TO_TICKS(USB_RESUME_WAIT));
 673
 674         /* catch any lost interrupts */
 675         ehci_do_poll(&sc->sc_bus);
 676 }
 677
 678 void
 679 ehci_shutdown(ehci_softc_t *sc)
 680 {
 681         DPRINTF("stopping the HC\n");
 682
 683         if (ehci_hcreset(sc)) {
 684                 DPRINTF("reset failed!\n");
 685         }
 686 }
 687
 688 #if USB_DEBUG
 689 static void
 690 ehci_dump_regs(ehci_softc_t *sc)
 691 {
 692         uint32_t i;
 693
 694         i = EOREAD4(sc, EHCI_USBCMD);
 695         printf("cmd=0x%08x\n", i);
 696
 697         if (i & EHCI_CMD_ITC_1)
 698                 printf(" EHCI_CMD_ITC_1\n");
 699         if (i & EHCI_CMD_ITC_2)
 700                 printf(" EHCI_CMD_ITC_2\n");
 701         if (i & EHCI_CMD_ITC_4)
 702                 printf(" EHCI_CMD_ITC_4\n");
 703         if (i & EHCI_CMD_ITC_8)
 704                 printf(" EHCI_CMD_ITC_8\n");
 705         if (i & EHCI_CMD_ITC_16)
 706                 printf(" EHCI_CMD_ITC_16\n");
 707         if (i & EHCI_CMD_ITC_32)
 708                 printf(" EHCI_CMD_ITC_32\n");
 709         if (i & EHCI_CMD_ITC_64)
 710                 printf(" EHCI_CMD_ITC_64\n");
 711         if (i & EHCI_CMD_ASPME)
 712                 printf(" EHCI_CMD_ASPME\n");
 713         if (i & EHCI_CMD_ASPMC)
 714                 printf(" EHCI_CMD_ASPMC\n");
 715         if (i & EHCI_CMD_LHCR)
 716                 printf(" EHCI_CMD_LHCR\n");
 717         if (i & EHCI_CMD_IAAD)
 718                 printf(" EHCI_CMD_IAAD\n");
 719         if (i & EHCI_CMD_ASE)
 720                 printf(" EHCI_CMD_ASE\n");
 721         if (i & EHCI_CMD_PSE)
 722                 printf(" EHCI_CMD_PSE\n");
 723         if (i & EHCI_CMD_FLS_M)
 724                 printf(" EHCI_CMD_FLS_M\n");
 725         if (i & EHCI_CMD_HCRESET)
 726                 printf(" EHCI_CMD_HCRESET\n");
 727         if (i & EHCI_CMD_RS)
 728                 printf(" EHCI_CMD_RS\n");
 729
 730         i = EOREAD4(sc, EHCI_USBSTS);
 731
 732         printf("sts=0x%08x\n", i);
 733
 734         if (i & EHCI_STS_ASS)
 735                 printf(" EHCI_STS_ASS\n");
 736         if (i & EHCI_STS_PSS)
 737                 printf(" EHCI_STS_PSS\n");
 738         if (i & EHCI_STS_REC)
 739                 printf(" EHCI_STS_REC\n");
 740         if (i & EHCI_STS_HCH)
 741                 printf(" EHCI_STS_HCH\n");
 742         if (i & EHCI_STS_IAA)
 743                 printf(" EHCI_STS_IAA\n");
 744         if (i & EHCI_STS_HSE)
 745                 printf(" EHCI_STS_HSE\n");
 746         if (i & EHCI_STS_FLR)
 747                 printf(" EHCI_STS_FLR\n");
 748         if (i & EHCI_STS_PCD)
 749                 printf(" EHCI_STS_PCD\n");
 750         if (i & EHCI_STS_ERRINT)
 751                 printf(" EHCI_STS_ERRINT\n");
 752         if (i & EHCI_STS_INT)
 753                 printf(" EHCI_STS_INT\n");
 754
 755         printf("ien=0x%08x\n",
 756             EOREAD4(sc, EHCI_USBINTR));
 757         printf("frindex=0x%08x ctrdsegm=0x%08x periodic=0x%08x async=0x%08x\n",
 758             EOREAD4(sc, EHCI_FRINDEX),
 759             EOREAD4(sc, EHCI_CTRLDSSEGMENT),
 760             EOREAD4(sc, EHCI_PERIODICLISTBASE),
 761             EOREAD4(sc, EHCI_ASYNCLISTADDR));
 762         for (i = 1; i <= sc->sc_noport; i++) {
 763                 printf("port %d status=0x%08x\n", i,
 764                     EOREAD4(sc, EHCI_PORTSC(i)));
 765         }
 766 }
 767
 768 static void
 769 ehci_dump_link(ehci_softc_t *sc, uint32_t link, int type)
 770 {
 771         link = hc32toh(sc, link);
 772         printf("0x%08x", link);
 773         if (link & EHCI_LINK_TERMINATE)
 774                 printf("<T>");
 775         else {
 776                 printf("<");
 777                 if (type) {
 778                         switch (EHCI_LINK_TYPE(link)) {
 779                         case EHCI_LINK_ITD:
 780                                 printf("ITD");
 781                                 break;
 782                         case EHCI_LINK_QH:
 783                                 printf("QH");
 784                                 break;
 785                         case EHCI_LINK_SITD:
 786                                 printf("SITD");
 787                                 break;
 788                         case EHCI_LINK_FSTN:
 789                                 printf("FSTN");
 790                                 break;
 791                         }
 792                 }
 793                 printf(">");
 794         }
 795 }
 796
 797 static void
 798 ehci_dump_qtd(ehci_softc_t *sc, ehci_qtd_t *qtd)
 799 {
 800         uint32_t s;
 801
 802         printf("  next=");
 803         ehci_dump_link(sc, qtd->qtd_next, 0);
 804         printf(" altnext=");
 805         ehci_dump_link(sc, qtd->qtd_altnext, 0);
 806         printf("\n");
 807         s = hc32toh(sc, qtd->qtd_status);
 808         printf("  status=0x%08x: toggle=%d bytes=0x%x ioc=%d c_page=0x%x\n",
 809             s, EHCI_QTD_GET_TOGGLE(s), EHCI_QTD_GET_BYTES(s),
 810             EHCI_QTD_GET_IOC(s), EHCI_QTD_GET_C_PAGE(s));
 811         printf("    cerr=%d pid=%d stat=%s%s%s%s%s%s%s%s\n",
 812             EHCI_QTD_GET_CERR(s), EHCI_QTD_GET_PID(s),
 813             (s & EHCI_QTD_ACTIVE) ? "ACTIVE" : "NOT_ACTIVE",
 814             (s & EHCI_QTD_HALTED) ? "-HALTED" : "",
 815             (s & EHCI_QTD_BUFERR) ? "-BUFERR" : "",
 816             (s & EHCI_QTD_BABBLE) ? "-BABBLE" : "",
 817             (s & EHCI_QTD_XACTERR) ? "-XACTERR" : "",
 818             (s & EHCI_QTD_MISSEDMICRO) ? "-MISSED" : "",
 819             (s & EHCI_QTD_SPLITXSTATE) ? "-SPLIT" : "",
 820             (s & EHCI_QTD_PINGSTATE) ? "-PING" : "");
 821
 822         for (s = 0; s < 5; s++) {
 823                 printf("  buffer[%d]=0x%08x\n", s,
 824                     hc32toh(sc, qtd->qtd_buffer[s]));
 825         }
 826         for (s = 0; s < 5; s++) {
 827                 printf("  buffer_hi[%d]=0x%08x\n", s,
 828                     hc32toh(sc, qtd->qtd_buffer_hi[s]));
 829         }
 830 }
 831
 832 static uint8_t
 833 ehci_dump_sqtd(ehci_softc_t *sc, ehci_qtd_t *sqtd)
 834 {
 835         uint8_t temp;
 836
 837         usb_pc_cpu_invalidate(sqtd->page_cache);
 838         printf("QTD(%p) at 0x%08x:\n", sqtd, hc32toh(sc, sqtd->qtd_self));
 839         ehci_dump_qtd(sc, sqtd);
 840         temp = (sqtd->qtd_next & htohc32(sc, EHCI_LINK_TERMINATE)) ? 1 : 0;
 841         return (temp);
 842 }
 843
 844 static void
 845 ehci_dump_sqtds(ehci_softc_t *sc, ehci_qtd_t *sqtd)
 846 {
 847         uint16_t i;
 848         uint8_t stop;
 849
 850         stop = 0;
 851         for (i = 0; sqtd && (i < 20) && !stop; sqtd = sqtd->obj_next, i++) {
 852                 stop = ehci_dump_sqtd(sc, sqtd);
 853         }
 854         if (sqtd) {
 855                 printf("dump aborted, too many TDs\n");
 856         }
 857 }
 858
 859 static void
 860 ehci_dump_sqh(ehci_softc_t *sc, ehci_qh_t *qh)
 861 {
 862         uint32_t endp;
 863         uint32_t endphub;
 864
 865         usb_pc_cpu_invalidate(qh->page_cache);
 866         printf("QH(%p) at 0x%08x:\n", qh, hc32toh(sc, qh->qh_self) & ~0x1F);
 867         printf("  link=");
 868         ehci_dump_link(sc, qh->qh_link, 1);
 869         printf("\n");
 870         endp = hc32toh(sc, qh->qh_endp);
 871         printf("  endp=0x%08x\n", endp);
 872         printf("    addr=0x%02x inact=%d endpt=%d eps=%d dtc=%d hrecl=%d\n",
 873             EHCI_QH_GET_ADDR(endp), EHCI_QH_GET_INACT(endp),
 874             EHCI_QH_GET_ENDPT(endp), EHCI_QH_GET_EPS(endp),
 875             EHCI_QH_GET_DTC(endp), EHCI_QH_GET_HRECL(endp));
 876         printf("    mpl=0x%x ctl=%d nrl=%d\n",
 877             EHCI_QH_GET_MPL(endp), EHCI_QH_GET_CTL(endp),
 878             EHCI_QH_GET_NRL(endp));
 879         endphub = hc32toh(sc, qh->qh_endphub);
 880         printf("  endphub=0x%08x\n", endphub);
 881         printf("    smask=0x%02x cmask=0x%02x huba=0x%02x port=%d mult=%d\n",
 882             EHCI_QH_GET_SMASK(endphub), EHCI_QH_GET_CMASK(endphub),
 883             EHCI_QH_GET_HUBA(endphub), EHCI_QH_GET_PORT(endphub),
 884             EHCI_QH_GET_MULT(endphub));
 885         printf("  curqtd=");
 886         ehci_dump_link(sc, qh->qh_curqtd, 0);
 887         printf("\n");
 888         printf("Overlay qTD:\n");
 889         ehci_dump_qtd(sc, (void *)&qh->qh_qtd);
 890 }
 891
 892 static void
 893 ehci_dump_sitd(ehci_softc_t *sc, ehci_sitd_t *sitd)
 894 {
 895         usb_pc_cpu_invalidate(sitd->page_cache);
 896         printf("SITD(%p) at 0x%08x\n", sitd, hc32toh(sc, sitd->sitd_self) & ~0x1F);
 897         printf(" next=0x%08x\n", hc32toh(sc, sitd->sitd_next));
 898         printf(" portaddr=0x%08x dir=%s addr=%d endpt=0x%x port=0x%x huba=0x%x\n",
 899             hc32toh(sc, sitd->sitd_portaddr),
 900             (sitd->sitd_portaddr & htohc32(sc, EHCI_SITD_SET_DIR_IN))
 901             ? "in" : "out",
 902             EHCI_SITD_GET_ADDR(hc32toh(sc, sitd->sitd_portaddr)),
 903             EHCI_SITD_GET_ENDPT(hc32toh(sc, sitd->sitd_portaddr)),
 904             EHCI_SITD_GET_PORT(hc32toh(sc, sitd->sitd_portaddr)),
 905             EHCI_SITD_GET_HUBA(hc32toh(sc, sitd->sitd_portaddr)));
 906         printf(" mask=0x%08x\n", hc32toh(sc, sitd->sitd_mask));
 907         printf(" status=0x%08x <%s> len=0x%x\n", hc32toh(sc, sitd->sitd_status),
 908             (sitd->sitd_status & htohc32(sc, EHCI_SITD_ACTIVE)) ? "ACTIVE" : "",
 909             EHCI_SITD_GET_LEN(hc32toh(sc, sitd->sitd_status)));
 910         printf(" back=0x%08x, bp=0x%08x,0x%08x,0x%08x,0x%08x\n",
 911             hc32toh(sc, sitd->sitd_back),
 912             hc32toh(sc, sitd->sitd_bp[0]),
 913             hc32toh(sc, sitd->sitd_bp[1]),
 914             hc32toh(sc, sitd->sitd_bp_hi[0]),
 915             hc32toh(sc, sitd->sitd_bp_hi[1]));
 916 }
 917
 918 static void
 919 ehci_dump_itd(ehci_softc_t *sc, ehci_itd_t *itd)
 920 {
 921         usb_pc_cpu_invalidate(itd->page_cache);
 922         printf("ITD(%p) at 0x%08x\n", itd, hc32toh(sc, itd->itd_self) & ~0x1F);
 923         printf(" next=0x%08x\n", hc32toh(sc, itd->itd_next));
 924         printf(" status[0]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[0]),
 925             (itd->itd_status[0] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
 926         printf(" status[1]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[1]),
 927             (itd->itd_status[1] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
 928         printf(" status[2]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[2]),
 929             (itd->itd_status[2] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
 930         printf(" status[3]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[3]),
 931             (itd->itd_status[3] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
 932         printf(" status[4]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[4]),
 933             (itd->itd_status[4] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
 934         printf(" status[5]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[5]),
 935             (itd->itd_status[5] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
 936         printf(" status[6]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[6]),
 937             (itd->itd_status[6] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
 938         printf(" status[7]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[7]),
 939             (itd->itd_status[7] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
 940         printf(" bp[0]=0x%08x\n", hc32toh(sc, itd->itd_bp[0]));
 941         printf("  addr=0x%02x; endpt=0x%01x\n",
 942             EHCI_ITD_GET_ADDR(hc32toh(sc, itd->itd_bp[0])),
 943             EHCI_ITD_GET_ENDPT(hc32toh(sc, itd->itd_bp[0])));
 944         printf(" bp[1]=0x%08x\n", hc32toh(sc, itd->itd_bp[1]));
 945         printf(" dir=%s; mpl=0x%02x\n",
 946             (hc32toh(sc, itd->itd_bp[1]) & EHCI_ITD_SET_DIR_IN) ? "in" : "out",
 947             EHCI_ITD_GET_MPL(hc32toh(sc, itd->itd_bp[1])));
 948         printf(" bp[2..6]=0x%08x,0x%08x,0x%08x,0x%08x,0x%08x\n",
 949             hc32toh(sc, itd->itd_bp[2]),
 950             hc32toh(sc, itd->itd_bp[3]),
 951             hc32toh(sc, itd->itd_bp[4]),
 952             hc32toh(sc, itd->itd_bp[5]),
 953             hc32toh(sc, itd->itd_bp[6]));
 954         printf(" bp_hi=0x%08x,0x%08x,0x%08x,0x%08x,\n"
 955             "       0x%08x,0x%08x,0x%08x\n",
 956             hc32toh(sc, itd->itd_bp_hi[0]),
 957             hc32toh(sc, itd->itd_bp_hi[1]),
 958             hc32toh(sc, itd->itd_bp_hi[2]),
 959             hc32toh(sc, itd->itd_bp_hi[3]),
 960             hc32toh(sc, itd->itd_bp_hi[4]),
 961             hc32toh(sc, itd->itd_bp_hi[5]),
 962             hc32toh(sc, itd->itd_bp_hi[6]));
 963 }
 964
 965 static void
 966 ehci_dump_isoc(ehci_softc_t *sc)
 967 {
 968         ehci_itd_t *itd;
 969         ehci_sitd_t *sitd;
 970         uint16_t max = 1000;
 971         uint16_t pos;
 972
 973         pos = (EOREAD4(sc, EHCI_FRINDEX) / 8) &
 974             (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
 975
 976         printf("%s: isochronous dump from frame 0x%03x:\n",
 977             __FUNCTION__, pos);
 978
 979         itd = sc->sc_isoc_hs_p_last[pos];
 980         sitd = sc->sc_isoc_fs_p_last[pos];
 981
 982         while (itd && max && max--) {
 983                 ehci_dump_itd(sc, itd);
 984                 itd = itd->prev;
 985         }
 986
 987         while (sitd && max && max--) {
 988                 ehci_dump_sitd(sc, sitd);
 989                 sitd = sitd->prev;
 990         }
 991 }
 992
 993 #endif
 994
 995 static void
 996 ehci_transfer_intr_enqueue(struct usb_xfer *xfer)
 997 {
 998         /* check for early completion */
 999         if (ehci_check_transfer(xfer)) {
1000                 return;
1001         }
1002         /* put transfer on interrupt queue */
1003         usbd_transfer_enqueue(&xfer->xroot->bus->intr_q, xfer);
1004
1005         /* start timeout, if any */
1006         if (xfer->timeout != 0) {
1007                 usbd_transfer_timeout_ms(xfer, &ehci_timeout, xfer->timeout);
1008         }
1009 }
1010
1011 #define EHCI_APPEND_FS_TD(std,last) (last) = _ehci_append_fs_td(std,last)
1012 static ehci_sitd_t *
1013 _ehci_append_fs_td(ehci_sitd_t *std, ehci_sitd_t *last)
1014 {
1015         DPRINTFN(11, "%p to %p\n", std, last);
1016
1017         /* (sc->sc_bus.mtx) must be locked */
1018
1019         std->next = last->next;
1020         std->sitd_next = last->sitd_next;
1021
1022         std->prev = last;
1023
1024         usb_pc_cpu_flush(std->page_cache);
1025
1026         /*
1027          * the last->next->prev is never followed: std->next->prev = std;
1028          */
1029         last->next = std;
1030         last->sitd_next = std->sitd_self;
1031
1032         usb_pc_cpu_flush(last->page_cache);
1033
1034         return (std);
1035 }
1036
1037 #define EHCI_APPEND_HS_TD(std,last) (last) = _ehci_append_hs_td(std,last)
1038 static ehci_itd_t *
1039 _ehci_append_hs_td(ehci_itd_t *std, ehci_itd_t *last)
1040 {
1041         DPRINTFN(11, "%p to %p\n", std, last);
1042
1043         /* (sc->sc_bus.mtx) must be locked */
1044
1045         std->next = last->next;
1046         std->itd_next = last->itd_next;
1047
1048         std->prev = last;
1049
1050         usb_pc_cpu_flush(std->page_cache);
1051
1052         /*
1053          * the last->next->prev is never followed: std->next->prev = std;
1054          */
1055         last->next = std;
1056         last->itd_next = std->itd_self;
1057
1058         usb_pc_cpu_flush(last->page_cache);
1059
1060         return (std);
1061 }
1062
1063 #define EHCI_APPEND_QH(sqh,last) (last) = _ehci_append_qh(sqh,last)
1064 static ehci_qh_t *
1065 _ehci_append_qh(ehci_qh_t *sqh, ehci_qh_t *last)
1066 {
1067         DPRINTFN(11, "%p to %p\n", sqh, last);
1068
1069         if (sqh->prev != NULL) {
1070                 /* should not happen */
1071                 DPRINTFN(0, "QH already linked!\n");
1072                 return (last);
1073         }
1074         /* (sc->sc_bus.mtx) must be locked */
1075
1076         sqh->next = last->next;
1077         sqh->qh_link = last->qh_link;
1078
1079         sqh->prev = last;
1080
1081         usb_pc_cpu_flush(sqh->page_cache);
1082
1083         /*
1084          * the last->next->prev is never followed: sqh->next->prev = sqh;
1085          */
1086
1087         last->next = sqh;
1088         last->qh_link = sqh->qh_self;
1089
1090         usb_pc_cpu_flush(last->page_cache);
1091
1092         return (sqh);
1093 }
1094
1095 #define EHCI_REMOVE_FS_TD(std,last) (last) = _ehci_remove_fs_td(std,last)
1096 static ehci_sitd_t *
1097 _ehci_remove_fs_td(ehci_sitd_t *std, ehci_sitd_t *last)
1098 {
1099         DPRINTFN(11, "%p from %p\n", std, last);
1100
1101         /* (sc->sc_bus.mtx) must be locked */
1102
1103         std->prev->next = std->next;
1104         std->prev->sitd_next = std->sitd_next;
1105
1106         usb_pc_cpu_flush(std->prev->page_cache);
1107
1108         if (std->next) {
1109                 std->next->prev = std->prev;
1110                 usb_pc_cpu_flush(std->next->page_cache);
1111         }
1112         return ((last == std) ? std->prev : last);
1113 }
1114
1115 #define EHCI_REMOVE_HS_TD(std,last) (last) = _ehci_remove_hs_td(std,last)
1116 static ehci_itd_t *
1117 _ehci_remove_hs_td(ehci_itd_t *std, ehci_itd_t *last)
1118 {
1119         DPRINTFN(11, "%p from %p\n", std, last);
1120
1121         /* (sc->sc_bus.mtx) must be locked */
1122
1123         std->prev->next = std->next;
1124         std->prev->itd_next = std->itd_next;
1125
1126         usb_pc_cpu_flush(std->prev->page_cache);
1127
1128         if (std->next) {
1129                 std->next->prev = std->prev;
1130                 usb_pc_cpu_flush(std->next->page_cache);
1131         }
1132         return ((last == std) ? std->prev : last);
1133 }
1134
1135 #define EHCI_REMOVE_QH(sqh,last) (last) = _ehci_remove_qh(sqh,last)
1136 static ehci_qh_t *
1137 _ehci_remove_qh(ehci_qh_t *sqh, ehci_qh_t *last)
1138 {
1139         DPRINTFN(11, "%p from %p\n", sqh, last);
1140
1141         /* (sc->sc_bus.mtx) must be locked */
1142
1143         /* only remove if not removed from a queue */
1144         if (sqh->prev) {
1145
1146                 sqh->prev->next = sqh->next;
1147                 sqh->prev->qh_link = sqh->qh_link;
1148
1149                 usb_pc_cpu_flush(sqh->prev->page_cache);
1150
1151                 if (sqh->next) {
1152                         sqh->next->prev = sqh->prev;
1153                         usb_pc_cpu_flush(sqh->next->page_cache);
1154                 }
1155                 last = ((last == sqh) ? sqh->prev : last);
1156
1157                 sqh->prev = 0;
1158
1159                 usb_pc_cpu_flush(sqh->page_cache);
1160         }
1161         return (last);
1162 }
1163
1164 static usb_error_t
1165 ehci_non_isoc_done_sub(struct usb_xfer *xfer)
1166 {
1167         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
1168         ehci_qtd_t *td;
1169         ehci_qtd_t *td_alt_next;
1170         uint32_t status;
1171         uint16_t len;
1172
1173         td = xfer->td_transfer_cache;
1174         td_alt_next = td->alt_next;
1175
1176         if (xfer->aframes != xfer->nframes) {
1177                 usbd_xfer_set_frame_len(xfer, xfer->aframes, 0);
1178         }
1179         while (1) {
1180
1181                 usb_pc_cpu_invalidate(td->page_cache);
1182                 status = hc32toh(sc, td->qtd_status);
1183
1184                 len = EHCI_QTD_GET_BYTES(status);
1185
1186                 /*
1187                  * Verify the status length and
1188                  * add the length to "frlengths[]":
1189                  */
1190                 if (len > td->len) {
1191                         /* should not happen */
1192                         DPRINTF("Invalid status length, "
1193                             "0x%04x/0x%04x bytes\n", len, td->len);
1194                         status |= EHCI_QTD_HALTED;
1195                 } else if (xfer->aframes != xfer->nframes) {
1196                         xfer->frlengths[xfer->aframes] += td->len - len;
1197                 }
1198                 /* Check for last transfer */
1199                 if (((void *)td) == xfer->td_transfer_last) {
1200                         td = NULL;
1201                         break;
1202                 }
1203                 /* Check for transfer error */
1204                 if (status & EHCI_QTD_HALTED) {
1205                         /* the transfer is finished */
1206                         td = NULL;
1207                         break;
1208                 }
1209                 /* Check for short transfer */
1210                 if (len > 0) {
1211                         if (xfer->flags_int.short_frames_ok) {
1212                                 /* follow alt next */
1213                                 td = td->alt_next;
1214                         } else {
1215                                 /* the transfer is finished */
1216                                 td = NULL;
1217                         }
1218                         break;
1219                 }
1220                 td = td->obj_next;
1221
1222                 if (td->alt_next != td_alt_next) {
1223                         /* this USB frame is complete */
1224                         break;
1225                 }
1226         }
1227
1228         /* update transfer cache */
1229
1230         xfer->td_transfer_cache = td;
1231
1232 #if USB_DEBUG
1233         if (status & EHCI_QTD_STATERRS) {
1234                 DPRINTFN(11, "error, addr=%d, endpt=0x%02x, frame=0x%02x"
1235                     "status=%s%s%s%s%s%s%s%s\n",
1236                     xfer->address, xfer->endpointno, xfer->aframes,
1237                     (status & EHCI_QTD_ACTIVE) ? "[ACTIVE]" : "[NOT_ACTIVE]",
1238                     (status & EHCI_QTD_HALTED) ? "[HALTED]" : "",
1239                     (status & EHCI_QTD_BUFERR) ? "[BUFERR]" : "",
1240                     (status & EHCI_QTD_BABBLE) ? "[BABBLE]" : "",
1241                     (status & EHCI_QTD_XACTERR) ? "[XACTERR]" : "",
1242                     (status & EHCI_QTD_MISSEDMICRO) ? "[MISSED]" : "",
1243                     (status & EHCI_QTD_SPLITXSTATE) ? "[SPLIT]" : "",
1244                     (status & EHCI_QTD_PINGSTATE) ? "[PING]" : "");
1245         }
1246 #endif
1247
1248         return ((status & EHCI_QTD_HALTED) ?
1249             USB_ERR_STALLED : USB_ERR_NORMAL_COMPLETION);
1250 }
1251
1252 static void
1253 ehci_non_isoc_done(struct usb_xfer *xfer)
1254 {
1255         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
1256         ehci_qh_t *qh;
1257         uint32_t status;
1258         usb_error_t err = 0;
1259
1260         DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
1261             xfer, xfer->endpoint);
1262
1263 #if USB_DEBUG
1264         if (ehcidebug > 10) {
1265                 ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
1266
1267                 ehci_dump_sqtds(sc, xfer->td_transfer_first);
1268         }
1269 #endif
1270
1271         /* extract data toggle directly from the QH's overlay area */
1272
1273         qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
1274
1275         usb_pc_cpu_invalidate(qh->page_cache);
1276
1277         status = hc32toh(sc, qh->qh_qtd.qtd_status);
1278
1279         xfer->endpoint->toggle_next =
1280             (status & EHCI_QTD_TOGGLE_MASK) ? 1 : 0;
1281
1282         /* reset scanner */
1283
1284         xfer->td_transfer_cache = xfer->td_transfer_first;
1285
1286         if (xfer->flags_int.control_xfr) {
1287
1288                 if (xfer->flags_int.control_hdr) {
1289
1290                         err = ehci_non_isoc_done_sub(xfer);
1291                 }
1292                 xfer->aframes = 1;
1293
1294                 if (xfer->td_transfer_cache == NULL) {
1295                         goto done;
1296                 }
1297         }
1298         while (xfer->aframes != xfer->nframes) {
1299
1300                 err = ehci_non_isoc_done_sub(xfer);
1301                 xfer->aframes++;
1302
1303                 if (xfer->td_transfer_cache == NULL) {
1304                         goto done;
1305                 }
1306         }
1307
1308         if (xfer->flags_int.control_xfr &&
1309             !xfer->flags_int.control_act) {
1310
1311                 err = ehci_non_isoc_done_sub(xfer);
1312         }
1313 done:
1314         ehci_device_done(xfer, err);
1315 }
1316
1317 /*------------------------------------------------------------------------*
1318  *      ehci_check_transfer
1319  *
1320  * Return values:
1321  *    0: USB transfer is not finished
1322  * Else: USB transfer is finished
1323  *------------------------------------------------------------------------*/
1324 static uint8_t
1325 ehci_check_transfer(struct usb_xfer *xfer)
1326 {
1327         struct usb_pipe_methods *methods = xfer->endpoint->methods;
1328         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
1329
1330         uint32_t status;
1331
1332         DPRINTFN(13, "xfer=%p checking transfer\n", xfer);
1333
1334         if (methods == &ehci_device_isoc_fs_methods) {
1335                 ehci_sitd_t *td;
1336
1337                 /* isochronous full speed transfer */
1338
1339                 td = xfer->td_transfer_last;
1340                 usb_pc_cpu_invalidate(td->page_cache);
1341                 status = hc32toh(sc, td->sitd_status);
1342
1343                 /* also check if first is complete */
1344
1345                 td = xfer->td_transfer_first;
1346                 usb_pc_cpu_invalidate(td->page_cache);
1347                 status |= hc32toh(sc, td->sitd_status);
1348
1349                 if (!(status & EHCI_SITD_ACTIVE)) {
1350                         ehci_device_done(xfer, USB_ERR_NORMAL_COMPLETION);
1351                         goto transferred;
1352                 }
1353         } else if (methods == &ehci_device_isoc_hs_methods) {
1354                 ehci_itd_t *td;
1355                 uint8_t n = (xfer->nframes & 7);
1356
1357                 /* isochronous high speed transfer */
1358
1359                 /* check last transfer */
1360                 td = xfer->td_transfer_last;
1361                 usb_pc_cpu_invalidate(td->page_cache);
1362                 if (n == 0)
1363                         status = td->itd_status[7];
1364                 else
1365                         status = td->itd_status[n-1];
1366
1367                 /* also check first transfer */
1368                 td = xfer->td_transfer_first;
1369                 usb_pc_cpu_invalidate(td->page_cache);
1370                 status |= td->itd_status[0];
1371
1372                 /* if no transactions are active we continue */
1373                 if (!(status & htohc32(sc, EHCI_ITD_ACTIVE))) {
1374                         ehci_device_done(xfer, USB_ERR_NORMAL_COMPLETION);
1375                         goto transferred;
1376                 }
1377         } else {
1378                 ehci_qtd_t *td;
1379                 ehci_qh_t *qh;
1380
1381                 /* non-isochronous transfer */
1382
1383                 /*
1384                  * check whether there is an error somewhere in the middle,
1385                  * or whether there was a short packet (SPD and not ACTIVE)
1386                  */
1387                 td = xfer->td_transfer_cache;
1388
1389                 qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
1390
1391                 usb_pc_cpu_invalidate(qh->page_cache);
1392
1393                 status = hc32toh(sc, qh->qh_qtd.qtd_status);
1394                 if (status & EHCI_QTD_ACTIVE) {
1395                         /* transfer is pending */
1396                         goto done;
1397                 }
1398
1399                 while (1) {
1400                         usb_pc_cpu_invalidate(td->page_cache);
1401                         status = hc32toh(sc, td->qtd_status);
1402
1403                         /*
1404                          * Check if there is an active TD which
1405                          * indicates that the transfer isn't done.
1406                          */
1407                         if (status & EHCI_QTD_ACTIVE) {
1408                                 /* update cache */
1409                                 if (xfer->td_transfer_cache != td) {
1410                                         xfer->td_transfer_cache = td;
1411                                         if (qh->qh_qtd.qtd_next &
1412                                             htohc32(sc, EHCI_LINK_TERMINATE)) {
1413                                                 /* XXX - manually advance to next frame */
1414                                                 qh->qh_qtd.qtd_next = td->qtd_self;
1415                                                 usb_pc_cpu_flush(td->page_cache);
1416                                         }
1417                                 }
1418                                 goto done;
1419                         }
1420                         /*
1421                          * last transfer descriptor makes the transfer done
1422                          */
1423                         if (((void *)td) == xfer->td_transfer_last) {
1424                                 break;
1425                         }
1426                         /*
1427                          * any kind of error makes the transfer done
1428                          */
1429                         if (status & EHCI_QTD_HALTED) {
1430                                 break;
1431                         }
1432                         /*
1433                          * if there is no alternate next transfer, a short
1434                          * packet also makes the transfer done
1435                          */
1436                         if (EHCI_QTD_GET_BYTES(status)) {
1437                                 if (xfer->flags_int.short_frames_ok) {
1438                                         /* follow alt next */
1439                                         if (td->alt_next) {
1440                                                 td = td->alt_next;
1441                                                 continue;
1442                                         }
1443                                 }
1444                                 /* transfer is done */
1445                                 break;
1446                         }
1447                         td = td->obj_next;
1448                 }
1449                 ehci_non_isoc_done(xfer);
1450                 goto transferred;
1451         }
1452
1453 done:
1454         DPRINTFN(13, "xfer=%p is still active\n", xfer);
1455         return (0);
1456
1457 transferred:
1458         return (1);
1459 }
1460
1461 static void
1462 ehci_pcd_enable(ehci_softc_t *sc)
1463 {
1464         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
1465
1466         sc->sc_eintrs |= EHCI_STS_PCD;
1467         EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
1468
1469         /* acknowledge any PCD interrupt */
1470         EOWRITE4(sc, EHCI_USBSTS, EHCI_STS_PCD);
1471
1472         ehci_root_intr(sc);
1473 }
1474
1475 static void
1476 ehci_interrupt_poll(ehci_softc_t *sc)
1477 {
1478         struct usb_xfer *xfer;
1479
1480 repeat:
1481         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
1482                 /*
1483                  * check if transfer is transferred
1484                  */
1485                 if (ehci_check_transfer(xfer)) {
1486                         /* queue has been modified */
1487                         goto repeat;
1488                 }
1489         }
1490 }
1491
1492 /*
1493  * Some EHCI chips from VIA / ATI seem to trigger interrupts before
1494  * writing back the qTD status, or miss signalling occasionally under
1495  * heavy load.  If the host machine is too fast, we can miss
1496  * transaction completion - when we scan the active list the
1497  * transaction still seems to be active. This generally exhibits
1498  * itself as a umass stall that never recovers.
1499  *
1500  * We work around this behaviour by setting up this callback after any
1501  * softintr that completes with transactions still pending, giving us
1502  * another chance to check for completion after the writeback has
1503  * taken place.
1504  */
1505 static void
1506 ehci_poll_timeout(void *arg)
1507 {
1508         ehci_softc_t *sc = arg;
1509
1510         DPRINTFN(3, "\n");
1511         ehci_interrupt_poll(sc);
1512 }
1513
1514 /*------------------------------------------------------------------------*
1515  *      ehci_interrupt - EHCI interrupt handler
1516  *
1517  * NOTE: Do not access "sc->sc_bus.bdev" inside the interrupt handler,
1518  * hence the interrupt handler will be setup before "sc->sc_bus.bdev"
1519  * is present !
1520  *------------------------------------------------------------------------*/
1521 void
1522 ehci_interrupt(ehci_softc_t *sc)
1523 {
1524         uint32_t status;
1525
1526         USB_BUS_LOCK(&sc->sc_bus);
1527
1528         DPRINTFN(16, "real interrupt\n");
1529
1530 #if USB_DEBUG
1531         if (ehcidebug > 15) {
1532                 ehci_dump_regs(sc);
1533         }
1534 #endif
1535
1536         status = EHCI_STS_INTRS(EOREAD4(sc, EHCI_USBSTS));
1537         if (status == 0) {
1538                 /* the interrupt was not for us */
1539                 goto done;
1540         }
1541         if (!(status & sc->sc_eintrs)) {
1542                 goto done;
1543         }
1544         EOWRITE4(sc, EHCI_USBSTS, status);      /* acknowledge */
1545
1546         status &= sc->sc_eintrs;
1547
1548         if (status & EHCI_STS_HSE) {
1549                 printf("%s: unrecoverable error, "
1550                     "controller halted\n", __FUNCTION__);
1551 #if USB_DEBUG
1552                 ehci_dump_regs(sc);
1553                 ehci_dump_isoc(sc);
1554 #endif
1555         }
1556         if (status & EHCI_STS_PCD) {
1557                 /*
1558                  * Disable PCD interrupt for now, because it will be
1559                  * on until the port has been reset.
1560                  */
1561                 sc->sc_eintrs &= ~EHCI_STS_PCD;
1562                 EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
1563
1564                 ehci_root_intr(sc);
1565
1566                 /* do not allow RHSC interrupts > 1 per second */
1567                 usb_callout_reset(&sc->sc_tmo_pcd, hz,
1568                     (void *)&ehci_pcd_enable, sc);
1569         }
1570         status &= ~(EHCI_STS_INT | EHCI_STS_ERRINT | EHCI_STS_PCD | EHCI_STS_IAA);
1571
1572         if (status != 0) {
1573                 /* block unprocessed interrupts */
1574                 sc->sc_eintrs &= ~status;
1575                 EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
1576                 printf("%s: blocking interrupts 0x%x\n", __FUNCTION__, status);
1577         }
1578         /* poll all the USB transfers */
1579         ehci_interrupt_poll(sc);
1580
1581         if (sc->sc_flags & EHCI_SCFLG_LOSTINTRBUG) {
1582                 usb_callout_reset(&sc->sc_tmo_poll, hz / 128,
1583                     (void *)&ehci_poll_timeout, sc);
1584         }
1585
1586 done:
1587         USB_BUS_UNLOCK(&sc->sc_bus);
1588 }
1589
1590 /*
1591  * called when a request does not complete
1592  */
1593 static void
1594 ehci_timeout(void *arg)
1595 {
1596         struct usb_xfer *xfer = arg;
1597
1598         DPRINTF("xfer=%p\n", xfer);
1599
1600         USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
1601
1602         /* transfer is transferred */
1603         ehci_device_done(xfer, USB_ERR_TIMEOUT);
1604 }
1605
1606 static void
1607 ehci_do_poll(struct usb_bus *bus)
1608 {
1609         ehci_softc_t *sc = EHCI_BUS2SC(bus);
1610
1611         USB_BUS_LOCK(&sc->sc_bus);
1612         ehci_interrupt_poll(sc);
1613         USB_BUS_UNLOCK(&sc->sc_bus);
1614 }
1615
1616 static void
1617 ehci_setup_standard_chain_sub(struct ehci_std_temp *temp)
1618 {
1619         struct usb_page_search buf_res;
1620         ehci_qtd_t *td;
1621         ehci_qtd_t *td_next;
1622         ehci_qtd_t *td_alt_next;
1623         uint32_t buf_offset;
1624         uint32_t average;
1625         uint32_t len_old;
1626         uint32_t terminate;
1627         uint8_t shortpkt_old;
1628         uint8_t precompute;
1629
1630         terminate = htohc32(temp->sc, EHCI_LINK_TERMINATE);
1631         td_alt_next = NULL;
1632         buf_offset = 0;
1633         shortpkt_old = temp->shortpkt;
1634         len_old = temp->len;
1635         precompute = 1;
1636
1637 restart:
1638
1639         td = temp->td;
1640         td_next = temp->td_next;
1641
1642         while (1) {
1643
1644                 if (temp->len == 0) {
1645
1646                         if (temp->shortpkt) {
1647                                 break;
1648                         }
1649                         /* send a Zero Length Packet, ZLP, last */
1650
1651                         temp->shortpkt = 1;
1652                         average = 0;
1653
1654                 } else {
1655
1656                         average = temp->average;
1657
1658                         if (temp->len < average) {
1659                                 if (temp->len % temp->max_frame_size) {
1660                                         temp->shortpkt = 1;
1661                                 }
1662                                 average = temp->len;
1663                         }
1664                 }
1665
1666                 if (td_next == NULL) {
1667                         panic("%s: out of EHCI transfer descriptors!", __FUNCTION__);
1668                 }
1669                 /* get next TD */
1670
1671                 td = td_next;
1672                 td_next = td->obj_next;
1673
1674                 /* check if we are pre-computing */
1675
1676                 if (precompute) {
1677
1678                         /* update remaining length */
1679
1680                         temp->len -= average;
1681
1682                         continue;
1683                 }
1684                 /* fill out current TD */
1685
1686                 td->qtd_status =
1687                     temp->qtd_status |
1688                     htohc32(temp->sc, EHCI_QTD_IOC |
1689                         EHCI_QTD_SET_BYTES(average));
1690
1691                 if (average == 0) {
1692
1693                         if (temp->auto_data_toggle == 0) {
1694
1695                                 /* update data toggle, ZLP case */
1696
1697                                 temp->qtd_status ^=
1698                                     htohc32(temp->sc, EHCI_QTD_TOGGLE_MASK);
1699                         }
1700                         td->len = 0;
1701
1702                         td->qtd_buffer[0] = 0;
1703                         td->qtd_buffer_hi[0] = 0;
1704
1705                         td->qtd_buffer[1] = 0;
1706                         td->qtd_buffer_hi[1] = 0;
1707
1708                 } else {
1709
1710                         uint8_t x;
1711
1712                         if (temp->auto_data_toggle == 0) {
1713
1714                                 /* update data toggle */
1715
1716                                 if (((average + temp->max_frame_size - 1) /
1717                                     temp->max_frame_size) & 1) {
1718                                         temp->qtd_status ^=
1719                                             htohc32(temp->sc, EHCI_QTD_TOGGLE_MASK);
1720                                 }
1721                         }
1722                         td->len = average;
1723
1724                         /* update remaining length */
1725
1726                         temp->len -= average;
1727
1728                         /* fill out buffer pointers */
1729
1730                         usbd_get_page(temp->pc, buf_offset, &buf_res);
1731                         td->qtd_buffer[0] =
1732                             htohc32(temp->sc, buf_res.physaddr);
1733                         td->qtd_buffer_hi[0] = 0;
1734
1735                         x = 1;
1736
1737                         while (average > EHCI_PAGE_SIZE) {
1738                                 average -= EHCI_PAGE_SIZE;
1739                                 buf_offset += EHCI_PAGE_SIZE;
1740                                 usbd_get_page(temp->pc, buf_offset, &buf_res);
1741                                 td->qtd_buffer[x] =
1742                                     htohc32(temp->sc,
1743                                     buf_res.physaddr & (~0xFFF));
1744                                 td->qtd_buffer_hi[x] = 0;
1745                                 x++;
1746                         }
1747
1748                         /*
1749                          * NOTE: The "average" variable is never zero after
1750                          * exiting the loop above !
1751                          *
1752                          * NOTE: We have to subtract one from the offset to
1753                          * ensure that we are computing the physical address
1754                          * of a valid page !
1755                          */
1756                         buf_offset += average;
1757                         usbd_get_page(temp->pc, buf_offset - 1, &buf_res);
1758                         td->qtd_buffer[x] =
1759                             htohc32(temp->sc,
1760                             buf_res.physaddr & (~0xFFF));
1761                         td->qtd_buffer_hi[x] = 0;
1762                 }
1763
1764                 if (temp->can_use_next) {
1765                         if (td_next) {
1766                                 /* link the current TD with the next one */
1767                                 td->qtd_next = td_next->qtd_self;
1768                         }
1769                 } else {
1770                         /*
1771                          * BUG WARNING: The EHCI HW can use the
1772                          * qtd_next field instead of qtd_altnext when
1773                          * a short packet is received! We work this
1774                          * around in software by not queueing more
1775                          * than one job/TD at a time!
1776                          */
1777                         td->qtd_next = terminate;
1778                 }
1779
1780                 td->qtd_altnext = terminate;
1781                 td->alt_next = td_alt_next;
1782
1783                 usb_pc_cpu_flush(td->page_cache);
1784         }
1785
1786         if (precompute) {
1787                 precompute = 0;
1788
1789                 /* setup alt next pointer, if any */
1790                 if (temp->last_frame) {
1791                         td_alt_next = NULL;
1792                 } else {
1793                         /* we use this field internally */
1794                         td_alt_next = td_next;
1795                 }
1796
1797                 /* restore */
1798                 temp->shortpkt = shortpkt_old;
1799                 temp->len = len_old;
1800                 goto restart;
1801         }
1802         temp->td = td;
1803         temp->td_next = td_next;
1804 }
1805
1806 static void
1807 ehci_setup_standard_chain(struct usb_xfer *xfer, ehci_qh_t **qh_last)
1808 {
1809         struct ehci_std_temp temp;
1810         struct usb_pipe_methods *methods;
1811         ehci_qh_t *qh;
1812         ehci_qtd_t *td;
1813         uint32_t qh_endp;
1814         uint32_t qh_endphub;
1815         uint32_t x;
1816
1817         DPRINTFN(9, "addr=%d endpt=%d sumlen=%d speed=%d\n",
1818             xfer->address, UE_GET_ADDR(xfer->endpointno),
1819             xfer->sumlen, usbd_get_speed(xfer->xroot->udev));
1820
1821         temp.average = xfer->max_hc_frame_size;
1822         temp.max_frame_size = xfer->max_frame_size;
1823         temp.sc = EHCI_BUS2SC(xfer->xroot->bus);
1824
1825         /* toggle the DMA set we are using */
1826         xfer->flags_int.curr_dma_set ^= 1;
1827
1828         /* get next DMA set */
1829         td = xfer->td_start[xfer->flags_int.curr_dma_set];
1830
1831         xfer->td_transfer_first = td;
1832         xfer->td_transfer_cache = td;
1833
1834         temp.td = NULL;
1835         temp.td_next = td;
1836         temp.qtd_status = 0;
1837         temp.last_frame = 0;
1838         temp.setup_alt_next = xfer->flags_int.short_frames_ok;
1839         temp.can_use_next = (xfer->flags_int.control_xfr ||
1840             (UE_GET_DIR(xfer->endpointno) == UE_DIR_OUT));
1841
1842         if (xfer->flags_int.control_xfr) {
1843                 if (xfer->endpoint->toggle_next) {
1844                         /* DATA1 is next */
1845                         temp.qtd_status |=
1846                             htohc32(temp.sc, EHCI_QTD_SET_TOGGLE(1));
1847                 }
1848                 temp.auto_data_toggle = 0;
1849         } else {
1850                 temp.auto_data_toggle = 1;
1851         }
1852
1853         if (usbd_get_speed(xfer->xroot->udev) != USB_SPEED_HIGH) {
1854                 /* max 3 retries */
1855                 temp.qtd_status |=
1856                     htohc32(temp.sc, EHCI_QTD_SET_CERR(3));
1857         }
1858         /* check if we should prepend a setup message */
1859
1860         if (xfer->flags_int.control_xfr) {
1861                 if (xfer->flags_int.control_hdr) {
1862
1863                         temp.qtd_status &=
1864                             htohc32(temp.sc, EHCI_QTD_SET_CERR(3));
1865                         temp.qtd_status |= htohc32(temp.sc,
1866                             EHCI_QTD_ACTIVE |
1867                             EHCI_QTD_SET_PID(EHCI_QTD_PID_SETUP) |
1868                             EHCI_QTD_SET_TOGGLE(0));
1869
1870                         temp.len = xfer->frlengths[0];
1871                         temp.pc = xfer->frbuffers + 0;
1872                         temp.shortpkt = temp.len ? 1 : 0;
1873                         /* check for last frame */
1874                         if (xfer->nframes == 1) {
1875                                 /* no STATUS stage yet, SETUP is last */
1876                                 if (xfer->flags_int.control_act) {
1877                                         temp.last_frame = 1;
1878                                         temp.setup_alt_next = 0;
1879                                 }
1880                         }
1881                         ehci_setup_standard_chain_sub(&temp);
1882                 }
1883                 x = 1;
1884         } else {
1885                 x = 0;
1886         }
1887
1888         while (x != xfer->nframes) {
1889
1890                 /* DATA0 / DATA1 message */
1891
1892                 temp.len = xfer->frlengths[x];
1893                 temp.pc = xfer->frbuffers + x;
1894
1895                 x++;
1896
1897                 if (x == xfer->nframes) {
1898                         if (xfer->flags_int.control_xfr) {
1899                                 /* no STATUS stage yet, DATA is last */
1900                                 if (xfer->flags_int.control_act) {
1901                                         temp.last_frame = 1;
1902                                         temp.setup_alt_next = 0;
1903                                 }
1904                         } else {
1905                                 temp.last_frame = 1;
1906                                 temp.setup_alt_next = 0;
1907                         }
1908                 }
1909                 /* keep previous data toggle and error count */
1910
1911                 temp.qtd_status &=
1912                     htohc32(temp.sc, EHCI_QTD_SET_CERR(3) |
1913                     EHCI_QTD_SET_TOGGLE(1));
1914
1915                 if (temp.len == 0) {
1916
1917                         /* make sure that we send an USB packet */
1918
1919                         temp.shortpkt = 0;
1920
1921                 } else {
1922
1923                         /* regular data transfer */
1924
1925                         temp.shortpkt = (xfer->flags.force_short_xfer) ? 0 : 1;
1926                 }
1927
1928                 /* set endpoint direction */
1929
1930                 temp.qtd_status |=
1931                     (UE_GET_DIR(xfer->endpointno) == UE_DIR_IN) ?
1932                     htohc32(temp.sc, EHCI_QTD_ACTIVE |
1933                     EHCI_QTD_SET_PID(EHCI_QTD_PID_IN)) :
1934                     htohc32(temp.sc, EHCI_QTD_ACTIVE |
1935                     EHCI_QTD_SET_PID(EHCI_QTD_PID_OUT));
1936
1937                 ehci_setup_standard_chain_sub(&temp);
1938         }
1939
1940         /* check if we should append a status stage */
1941
1942         if (xfer->flags_int.control_xfr &&
1943             !xfer->flags_int.control_act) {
1944
1945                 /*
1946                  * Send a DATA1 message and invert the current endpoint
1947                  * direction.
1948                  */
1949
1950                 temp.qtd_status &= htohc32(temp.sc, EHCI_QTD_SET_CERR(3) |
1951                     EHCI_QTD_SET_TOGGLE(1));
1952                 temp.qtd_status |=
1953                     (UE_GET_DIR(xfer->endpointno) == UE_DIR_OUT) ?
1954                     htohc32(temp.sc, EHCI_QTD_ACTIVE |
1955                     EHCI_QTD_SET_PID(EHCI_QTD_PID_IN) |
1956                     EHCI_QTD_SET_TOGGLE(1)) :
1957                     htohc32(temp.sc, EHCI_QTD_ACTIVE |
1958                     EHCI_QTD_SET_PID(EHCI_QTD_PID_OUT) |
1959                     EHCI_QTD_SET_TOGGLE(1));
1960
1961                 temp.len = 0;
1962                 temp.pc = NULL;
1963                 temp.shortpkt = 0;
1964                 temp.last_frame = 1;
1965                 temp.setup_alt_next = 0;
1966
1967                 ehci_setup_standard_chain_sub(&temp);
1968         }
1969         td = temp.td;
1970
1971         /* the last TD terminates the transfer: */
1972         td->qtd_next = htohc32(temp.sc, EHCI_LINK_TERMINATE);
1973         td->qtd_altnext = htohc32(temp.sc, EHCI_LINK_TERMINATE);
1974
1975         usb_pc_cpu_flush(td->page_cache);
1976
1977         /* must have at least one frame! */
1978
1979         xfer->td_transfer_last = td;
1980
1981 #if USB_DEBUG
1982         if (ehcidebug > 8) {
1983                 DPRINTF("nexttog=%d; data before transfer:\n",
1984                     xfer->endpoint->toggle_next);
1985                 ehci_dump_sqtds(temp.sc,
1986                     xfer->td_transfer_first);
1987         }
1988 #endif
1989
1990         methods = xfer->endpoint->methods;
1991
1992         qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
1993
1994         /* the "qh_link" field is filled when the QH is added */
1995
1996         qh_endp =
1997             (EHCI_QH_SET_ADDR(xfer->address) |
1998             EHCI_QH_SET_ENDPT(UE_GET_ADDR(xfer->endpointno)) |
1999             EHCI_QH_SET_MPL(xfer->max_packet_size));
2000
2001         if (usbd_get_speed(xfer->xroot->udev) == USB_SPEED_HIGH) {
2002                 qh_endp |= EHCI_QH_SET_EPS(EHCI_QH_SPEED_HIGH);
2003                 if (methods != &ehci_device_intr_methods)
2004                         qh_endp |= EHCI_QH_SET_NRL(8);
2005         } else {
2006
2007                 if (usbd_get_speed(xfer->xroot->udev) == USB_SPEED_FULL) {
2008                         qh_endp |= EHCI_QH_SET_EPS(EHCI_QH_SPEED_FULL);
2009                 } else {
2010                         qh_endp |= EHCI_QH_SET_EPS(EHCI_QH_SPEED_LOW);
2011                 }
2012
2013                 if (methods == &ehci_device_ctrl_methods) {
2014                         qh_endp |= EHCI_QH_CTL;
2015                 }
2016                 if (methods != &ehci_device_intr_methods) {
2017                         /* Only try one time per microframe! */
2018                         qh_endp |= EHCI_QH_SET_NRL(1);
2019                 }
2020         }
2021
2022         if (temp.auto_data_toggle == 0) {
2023                 /* software computes the data toggle */
2024                 qh_endp |= EHCI_QH_DTC;
2025         }
2026
2027         qh->qh_endp = htohc32(temp.sc, qh_endp);
2028
2029         qh_endphub =
2030             (EHCI_QH_SET_MULT(xfer->max_packet_count & 3) |
2031             EHCI_QH_SET_CMASK(xfer->endpoint->usb_cmask) |
2032             EHCI_QH_SET_SMASK(xfer->endpoint->usb_smask) |
2033             EHCI_QH_SET_HUBA(xfer->xroot->udev->hs_hub_addr) |
2034             EHCI_QH_SET_PORT(xfer->xroot->udev->hs_port_no));
2035
2036         qh->qh_endphub = htohc32(temp.sc, qh_endphub);
2037         qh->qh_curqtd = 0;
2038
2039         /* fill the overlay qTD */
2040
2041         if (temp.auto_data_toggle && xfer->endpoint->toggle_next) {
2042                 /* DATA1 is next */
2043                 qh->qh_qtd.qtd_status = htohc32(temp.sc, EHCI_QTD_SET_TOGGLE(1));
2044         } else {
2045                 qh->qh_qtd.qtd_status = 0;
2046         }
2047
2048         td = xfer->td_transfer_first;
2049
2050         qh->qh_qtd.qtd_next = td->qtd_self;
2051         qh->qh_qtd.qtd_altnext =
2052             htohc32(temp.sc, EHCI_LINK_TERMINATE);
2053
2054         usb_pc_cpu_flush(qh->page_cache);
2055
2056         if (xfer->xroot->udev->flags.self_suspended == 0) {
2057                 EHCI_APPEND_QH(qh, *qh_last);
2058         }
2059 }
2060
2061 static void
2062 ehci_root_intr(ehci_softc_t *sc)
2063 {
2064         uint16_t i;
2065         uint16_t m;
2066
2067         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
2068
2069         /* clear any old interrupt data */
2070         memset(sc->sc_hub_idata, 0, sizeof(sc->sc_hub_idata));
2071
2072         /* set bits */
2073         m = (sc->sc_noport + 1);
2074         if (m > (8 * sizeof(sc->sc_hub_idata))) {
2075                 m = (8 * sizeof(sc->sc_hub_idata));
2076         }
2077         for (i = 1; i < m; i++) {
2078                 /* pick out CHANGE bits from the status register */
2079                 if (EOREAD4(sc, EHCI_PORTSC(i)) & EHCI_PS_CLEAR) {
2080                         sc->sc_hub_idata[i / 8] |= 1 << (i % 8);
2081                         DPRINTF("port %d changed\n", i);
2082                 }
2083         }
2084         uhub_root_intr(&sc->sc_bus, sc->sc_hub_idata,
2085             sizeof(sc->sc_hub_idata));
2086 }
2087
2088 static void
2089 ehci_isoc_fs_done(ehci_softc_t *sc, struct usb_xfer *xfer)
2090 {
2091         uint32_t nframes = xfer->nframes;
2092         uint32_t status;
2093         uint32_t *plen = xfer->frlengths;
2094         uint16_t len = 0;
2095         ehci_sitd_t *td = xfer->td_transfer_first;
2096         ehci_sitd_t **pp_last = &sc->sc_isoc_fs_p_last[xfer->qh_pos];
2097
2098         DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
2099             xfer, xfer->endpoint);
2100
2101         while (nframes--) {
2102                 if (td == NULL) {
2103                         panic("%s:%d: out of TD's\n",
2104                             __FUNCTION__, __LINE__);
2105                 }
2106                 if (pp_last >= &sc->sc_isoc_fs_p_last[EHCI_VIRTUAL_FRAMELIST_COUNT]) {
2107                         pp_last = &sc->sc_isoc_fs_p_last[0];
2108                 }
2109 #if USB_DEBUG
2110                 if (ehcidebug > 15) {
2111                         DPRINTF("isoc FS-TD\n");
2112                         ehci_dump_sitd(sc, td);
2113                 }
2114 #endif
2115                 usb_pc_cpu_invalidate(td->page_cache);
2116                 status = hc32toh(sc, td->sitd_status);
2117
2118                 len = EHCI_SITD_GET_LEN(status);
2119
2120                 DPRINTFN(2, "status=0x%08x, rem=%u\n", status, len);
2121
2122                 if (*plen >= len) {
2123                         len = *plen - len;
2124                 } else {
2125                         len = 0;
2126                 }
2127
2128                 *plen = len;
2129
2130                 /* remove FS-TD from schedule */
2131                 EHCI_REMOVE_FS_TD(td, *pp_last);
2132
2133                 pp_last++;
2134                 plen++;
2135                 td = td->obj_next;
2136         }
2137
2138         xfer->aframes = xfer->nframes;
2139 }
2140
2141 static void
2142 ehci_isoc_hs_done(ehci_softc_t *sc, struct usb_xfer *xfer)
2143 {
2144         uint32_t nframes = xfer->nframes;
2145         uint32_t status;
2146         uint32_t *plen = xfer->frlengths;
2147         uint16_t len = 0;
2148         uint8_t td_no = 0;
2149         ehci_itd_t *td = xfer->td_transfer_first;
2150         ehci_itd_t **pp_last = &sc->sc_isoc_hs_p_last[xfer->qh_pos];
2151
2152         DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
2153             xfer, xfer->endpoint);
2154
2155         while (nframes) {
2156                 if (td == NULL) {
2157                         panic("%s:%d: out of TD's\n",
2158                             __FUNCTION__, __LINE__);
2159                 }
2160                 if (pp_last >= &sc->sc_isoc_hs_p_last[EHCI_VIRTUAL_FRAMELIST_COUNT]) {
2161                         pp_last = &sc->sc_isoc_hs_p_last[0];
2162                 }
2163 #if USB_DEBUG
2164                 if (ehcidebug > 15) {
2165                         DPRINTF("isoc HS-TD\n");
2166                         ehci_dump_itd(sc, td);
2167                 }
2168 #endif
2169
2170                 usb_pc_cpu_invalidate(td->page_cache);
2171                 status = hc32toh(sc, td->itd_status[td_no]);
2172
2173                 len = EHCI_ITD_GET_LEN(status);
2174
2175                 DPRINTFN(2, "status=0x%08x, len=%u\n", status, len);
2176
2177                 if (xfer->endpoint->usb_smask & (1 << td_no)) {
2178
2179                         if (*plen >= len) {
2180                                 /*
2181                                  * The length is valid. NOTE: The
2182                                  * complete length is written back
2183                                  * into the status field, and not the
2184                                  * remainder like with other transfer
2185                                  * descriptor types.
2186                                  */
2187                         } else {
2188                                 /* Invalid length - truncate */
2189                                 len = 0;
2190                         }
2191
2192                         *plen = len;
2193                         plen++;
2194                         nframes--;
2195                 }
2196
2197                 td_no++;
2198
2199                 if ((td_no == 8) || (nframes == 0)) {
2200                         /* remove HS-TD from schedule */
2201                         EHCI_REMOVE_HS_TD(td, *pp_last);
2202                         pp_last++;
2203
2204                         td_no = 0;
2205                         td = td->obj_next;
2206                 }
2207         }
2208         xfer->aframes = xfer->nframes;
2209 }
2210
2211 /* NOTE: "done" can be run two times in a row,
2212  * from close and from interrupt
2213  */
2214 static void
2215 ehci_device_done(struct usb_xfer *xfer, usb_error_t error)
2216 {
2217         struct usb_pipe_methods *methods = xfer->endpoint->methods;
2218         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2219
2220         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
2221
2222         DPRINTFN(2, "xfer=%p, endpoint=%p, error=%d\n",
2223             xfer, xfer->endpoint, error);
2224
2225         if ((methods == &ehci_device_bulk_methods) ||
2226             (methods == &ehci_device_ctrl_methods)) {
2227 #if USB_DEBUG
2228                 if (ehcidebug > 8) {
2229                         DPRINTF("nexttog=%d; data after transfer:\n",
2230                             xfer->endpoint->toggle_next);
2231                         ehci_dump_sqtds(sc,
2232                             xfer->td_transfer_first);
2233                 }
2234 #endif
2235
2236                 EHCI_REMOVE_QH(xfer->qh_start[xfer->flags_int.curr_dma_set],
2237                     sc->sc_async_p_last);
2238         }
2239         if (methods == &ehci_device_intr_methods) {
2240                 EHCI_REMOVE_QH(xfer->qh_start[xfer->flags_int.curr_dma_set],
2241                     sc->sc_intr_p_last[xfer->qh_pos]);
2242         }
2243         /*
2244          * Only finish isochronous transfers once which will update
2245          * "xfer->frlengths".
2246          */
2247         if (xfer->td_transfer_first &&
2248             xfer->td_transfer_last) {
2249                 if (methods == &ehci_device_isoc_fs_methods) {
2250                         ehci_isoc_fs_done(sc, xfer);
2251                 }
2252                 if (methods == &ehci_device_isoc_hs_methods) {
2253                         ehci_isoc_hs_done(sc, xfer);
2254                 }
2255                 xfer->td_transfer_first = NULL;
2256                 xfer->td_transfer_last = NULL;
2257         }
2258         /* dequeue transfer and start next transfer */
2259         usbd_transfer_done(xfer, error);
2260 }
2261
2262 /*------------------------------------------------------------------------*
2263  * ehci bulk support
2264  *------------------------------------------------------------------------*/
2265 static void
2266 ehci_device_bulk_open(struct usb_xfer *xfer)
2267 {
2268         return;
2269 }
2270
2271 static void
2272 ehci_device_bulk_close(struct usb_xfer *xfer)
2273 {
2274         ehci_device_done(xfer, USB_ERR_CANCELLED);
2275 }
2276
2277 static void
2278 ehci_device_bulk_enter(struct usb_xfer *xfer)
2279 {
2280         return;
2281 }
2282
2283 static void
2284 ehci_device_bulk_start(struct usb_xfer *xfer)
2285 {
2286         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2287         uint32_t temp;
2288
2289         /* setup TD's and QH */
2290         ehci_setup_standard_chain(xfer, &sc->sc_async_p_last);
2291
2292         /* put transfer on interrupt queue */
2293         ehci_transfer_intr_enqueue(xfer);
2294
2295         /*
2296          * XXX Certain nVidia chipsets choke when using the IAAD
2297          * feature too frequently.
2298          */
2299         if (sc->sc_flags & EHCI_SCFLG_IAADBUG)
2300                 return;
2301
2302         /* XXX Performance quirk: Some Host Controllers have a too low
2303          * interrupt rate. Issue an IAAD to stimulate the Host
2304          * Controller after queueing the BULK transfer.
2305          */
2306         temp = EOREAD4(sc, EHCI_USBCMD);
2307         if (!(temp & EHCI_CMD_IAAD))
2308                 EOWRITE4(sc, EHCI_USBCMD, temp | EHCI_CMD_IAAD);
2309 }
2310
2311 struct usb_pipe_methods ehci_device_bulk_methods =
2312 {
2313         .open = ehci_device_bulk_open,
2314         .close = ehci_device_bulk_close,
2315         .enter = ehci_device_bulk_enter,
2316         .start = ehci_device_bulk_start,
2317 };
2318
2319 /*------------------------------------------------------------------------*
2320  * ehci control support
2321  *------------------------------------------------------------------------*/
2322 static void
2323 ehci_device_ctrl_open(struct usb_xfer *xfer)
2324 {
2325         return;
2326 }
2327
2328 static void
2329 ehci_device_ctrl_close(struct usb_xfer *xfer)
2330 {
2331         ehci_device_done(xfer, USB_ERR_CANCELLED);
2332 }
2333
2334 static void
2335 ehci_device_ctrl_enter(struct usb_xfer *xfer)
2336 {
2337         return;
2338 }
2339
2340 static void
2341 ehci_device_ctrl_start(struct usb_xfer *xfer)
2342 {
2343         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2344
2345         /* setup TD's and QH */
2346         ehci_setup_standard_chain(xfer, &sc->sc_async_p_last);
2347
2348         /* put transfer on interrupt queue */
2349         ehci_transfer_intr_enqueue(xfer);
2350 }
2351
2352 struct usb_pipe_methods ehci_device_ctrl_methods =
2353 {
2354         .open = ehci_device_ctrl_open,
2355         .close = ehci_device_ctrl_close,
2356         .enter = ehci_device_ctrl_enter,
2357         .start = ehci_device_ctrl_start,
2358 };
2359
2360 /*------------------------------------------------------------------------*
2361  * ehci interrupt support
2362  *------------------------------------------------------------------------*/
2363 static void
2364 ehci_device_intr_open(struct usb_xfer *xfer)
2365 {
2366         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2367         uint16_t best;
2368         uint16_t bit;
2369         uint16_t x;
2370
2371         usb_hs_bandwidth_alloc(xfer);
2372
2373         /*
2374          * Find the best QH position corresponding to the given interval:
2375          */
2376
2377         best = 0;
2378         bit = EHCI_VIRTUAL_FRAMELIST_COUNT / 2;
2379         while (bit) {
2380                 if (xfer->interval >= bit) {
2381                         x = bit;
2382                         best = bit;
2383                         while (x & bit) {
2384                                 if (sc->sc_intr_stat[x] <
2385                                     sc->sc_intr_stat[best]) {
2386                                         best = x;
2387                                 }
2388                                 x++;
2389                         }
2390                         break;
2391                 }
2392                 bit >>= 1;
2393         }
2394
2395         sc->sc_intr_stat[best]++;
2396         xfer->qh_pos = best;
2397
2398         DPRINTFN(3, "best=%d interval=%d\n",
2399             best, xfer->interval);
2400 }
2401
2402 static void
2403 ehci_device_intr_close(struct usb_xfer *xfer)
2404 {
2405         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2406
2407         sc->sc_intr_stat[xfer->qh_pos]--;
2408
2409         ehci_device_done(xfer, USB_ERR_CANCELLED);
2410
2411         /* bandwidth must be freed after device done */
2412         usb_hs_bandwidth_free(xfer);
2413 }
2414
2415 static void
2416 ehci_device_intr_enter(struct usb_xfer *xfer)
2417 {
2418         return;
2419 }
2420
2421 static void
2422 ehci_device_intr_start(struct usb_xfer *xfer)
2423 {
2424         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2425
2426         /* setup TD's and QH */
2427         ehci_setup_standard_chain(xfer, &sc->sc_intr_p_last[xfer->qh_pos]);
2428
2429         /* put transfer on interrupt queue */
2430         ehci_transfer_intr_enqueue(xfer);
2431 }
2432
2433 struct usb_pipe_methods ehci_device_intr_methods =
2434 {
2435         .open = ehci_device_intr_open,
2436         .close = ehci_device_intr_close,
2437         .enter = ehci_device_intr_enter,
2438         .start = ehci_device_intr_start,
2439 };
2440
2441 /*------------------------------------------------------------------------*
2442  * ehci full speed isochronous support
2443  *------------------------------------------------------------------------*/
2444 static void
2445 ehci_device_isoc_fs_open(struct usb_xfer *xfer)
2446 {
2447         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2448         ehci_sitd_t *td;
2449         uint32_t sitd_portaddr;
2450         uint8_t ds;
2451
2452         sitd_portaddr =
2453             EHCI_SITD_SET_ADDR(xfer->address) |
2454             EHCI_SITD_SET_ENDPT(UE_GET_ADDR(xfer->endpointno)) |
2455             EHCI_SITD_SET_HUBA(xfer->xroot->udev->hs_hub_addr) |
2456             EHCI_SITD_SET_PORT(xfer->xroot->udev->hs_port_no);
2457
2458         if (UE_GET_DIR(xfer->endpointno) == UE_DIR_IN) {
2459                 sitd_portaddr |= EHCI_SITD_SET_DIR_IN;
2460         }
2461         sitd_portaddr = htohc32(sc, sitd_portaddr);
2462
2463         /* initialize all TD's */
2464
2465         for (ds = 0; ds != 2; ds++) {
2466
2467                 for (td = xfer->td_start[ds]; td; td = td->obj_next) {
2468
2469                         td->sitd_portaddr = sitd_portaddr;
2470
2471                         /*
2472                          * TODO: make some kind of automatic
2473                          * SMASK/CMASK selection based on micro-frame
2474                          * usage
2475                          *
2476                          * micro-frame usage (8 microframes per 1ms)
2477                          */
2478                         td->sitd_back = htohc32(sc, EHCI_LINK_TERMINATE);
2479
2480                         usb_pc_cpu_flush(td->page_cache);
2481                 }
2482         }
2483 }
2484
2485 static void
2486 ehci_device_isoc_fs_close(struct usb_xfer *xfer)
2487 {
2488         ehci_device_done(xfer, USB_ERR_CANCELLED);
2489 }
2490
2491 static void
2492 ehci_device_isoc_fs_enter(struct usb_xfer *xfer)
2493 {
2494         struct usb_page_search buf_res;
2495         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2496         struct usb_fs_isoc_schedule *fss_start;
2497         struct usb_fs_isoc_schedule *fss_end;
2498         struct usb_fs_isoc_schedule *fss;
2499         ehci_sitd_t *td;
2500         ehci_sitd_t *td_last = NULL;
2501         ehci_sitd_t **pp_last;
2502         uint32_t *plen;
2503         uint32_t buf_offset;
2504         uint32_t nframes;
2505         uint32_t temp;
2506         uint32_t sitd_mask;
2507         uint16_t tlen;
2508         uint8_t sa;
2509         uint8_t sb;
2510         uint8_t error;
2511
2512 #if USB_DEBUG
2513         uint8_t once = 1;
2514
2515 #endif
2516
2517         DPRINTFN(6, "xfer=%p next=%d nframes=%d\n",
2518             xfer, xfer->endpoint->isoc_next, xfer->nframes);
2519
2520         /* get the current frame index */
2521
2522         nframes = EOREAD4(sc, EHCI_FRINDEX) / 8;
2523
2524         /*
2525          * check if the frame index is within the window where the frames
2526          * will be inserted
2527          */
2528         buf_offset = (nframes - xfer->endpoint->isoc_next) &
2529             (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2530
2531         if ((xfer->endpoint->is_synced == 0) ||
2532             (buf_offset < xfer->nframes)) {
2533                 /*
2534                  * If there is data underflow or the pipe queue is empty we
2535                  * schedule the transfer a few frames ahead of the current
2536                  * frame position. Else two isochronous transfers might
2537                  * overlap.
2538                  */
2539                 xfer->endpoint->isoc_next = (nframes + 3) &
2540                     (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2541                 xfer->endpoint->is_synced = 1;
2542                 DPRINTFN(3, "start next=%d\n", xfer->endpoint->isoc_next);
2543         }
2544         /*
2545          * compute how many milliseconds the insertion is ahead of the
2546          * current frame position:
2547          */
2548         buf_offset = (xfer->endpoint->isoc_next - nframes) &
2549             (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2550
2551         /*
2552          * pre-compute when the isochronous transfer will be finished:
2553          */
2554         xfer->isoc_time_complete =
2555             usbd_fs_isoc_schedule_isoc_time_expand
2556             (xfer->xroot->udev, &fss_start, &fss_end, nframes) + buf_offset +
2557             xfer->nframes;
2558
2559         /* get the real number of frames */
2560
2561         nframes = xfer->nframes;
2562
2563         buf_offset = 0;
2564
2565         plen = xfer->frlengths;
2566
2567         /* toggle the DMA set we are using */
2568         xfer->flags_int.curr_dma_set ^= 1;
2569
2570         /* get next DMA set */
2571         td = xfer->td_start[xfer->flags_int.curr_dma_set];
2572         xfer->td_transfer_first = td;
2573
2574         pp_last = &sc->sc_isoc_fs_p_last[xfer->endpoint->isoc_next];
2575
2576         /* store starting position */
2577
2578         xfer->qh_pos = xfer->endpoint->isoc_next;
2579
2580         fss = fss_start + (xfer->qh_pos % USB_ISOC_TIME_MAX);
2581
2582         while (nframes--) {
2583                 if (td == NULL) {
2584                         panic("%s:%d: out of TD's\n",
2585                             __FUNCTION__, __LINE__);
2586                 }
2587                 if (pp_last >= &sc->sc_isoc_fs_p_last[EHCI_VIRTUAL_FRAMELIST_COUNT]) {
2588                         pp_last = &sc->sc_isoc_fs_p_last[0];
2589                 }
2590                 if (fss >= fss_end) {
2591                         fss = fss_start;
2592                 }
2593                 /* reuse sitd_portaddr and sitd_back from last transfer */
2594
2595                 if (*plen > xfer->max_frame_size) {
2596 #if USB_DEBUG
2597                         if (once) {
2598                                 once = 0;
2599                                 printf("%s: frame length(%d) exceeds %d "
2600                                     "bytes (frame truncated)\n",
2601                                     __FUNCTION__, *plen,
2602                                     xfer->max_frame_size);
2603                         }
2604 #endif
2605                         *plen = xfer->max_frame_size;
2606                 }
2607                 /*
2608                  * We currently don't care if the ISOCHRONOUS schedule is
2609                  * full!
2610                  */
2611                 error = usbd_fs_isoc_schedule_alloc(fss, &sa, *plen);
2612                 if (error) {
2613                         /*
2614                          * The FULL speed schedule is FULL! Set length
2615                          * to zero.
2616                          */
2617                         *plen = 0;
2618                 }
2619                 if (*plen) {
2620                         /*
2621                          * only call "usbd_get_page()" when we have a
2622                          * non-zero length
2623                          */
2624                         usbd_get_page(xfer->frbuffers, buf_offset, &buf_res);
2625                         td->sitd_bp[0] = htohc32(sc, buf_res.physaddr);
2626                         buf_offset += *plen;
2627                         /*
2628                          * NOTE: We need to subtract one from the offset so
2629                          * that we are on a valid page!
2630                          */
2631                         usbd_get_page(xfer->frbuffers, buf_offset - 1,
2632                             &buf_res);
2633                         temp = buf_res.physaddr & ~0xFFF;
2634                 } else {
2635                         td->sitd_bp[0] = 0;
2636                         temp = 0;
2637                 }
2638
2639                 if (UE_GET_DIR(xfer->endpointno) == UE_DIR_OUT) {
2640                         tlen = *plen;
2641                         if (tlen <= 188) {
2642                                 temp |= 1;      /* T-count = 1, TP = ALL */
2643                                 tlen = 1;
2644                         } else {
2645                                 tlen += 187;
2646                                 tlen /= 188;
2647                                 temp |= tlen;   /* T-count = [1..6] */
2648                                 temp |= 8;      /* TP = Begin */
2649                         }
2650
2651                         tlen += sa;
2652
2653                         if (tlen >= 8) {
2654                                 sb = 0;
2655                         } else {
2656                                 sb = (1 << tlen);
2657                         }
2658
2659                         sa = (1 << sa);
2660                         sa = (sb - sa) & 0x3F;
2661                         sb = 0;
2662                 } else {
2663                         sb = (-(4 << sa)) & 0xFE;
2664                         sa = (1 << sa) & 0x3F;
2665                 }
2666
2667                 sitd_mask = (EHCI_SITD_SET_SMASK(sa) |
2668                     EHCI_SITD_SET_CMASK(sb));
2669
2670                 td->sitd_bp[1] = htohc32(sc, temp);
2671
2672                 td->sitd_mask = htohc32(sc, sitd_mask);
2673
2674                 if (nframes == 0) {
2675                         td->sitd_status = htohc32(sc,
2676                             EHCI_SITD_IOC |
2677                             EHCI_SITD_ACTIVE |
2678                             EHCI_SITD_SET_LEN(*plen));
2679                 } else {
2680                         td->sitd_status = htohc32(sc,
2681                             EHCI_SITD_ACTIVE |
2682                             EHCI_SITD_SET_LEN(*plen));
2683                 }
2684                 usb_pc_cpu_flush(td->page_cache);
2685
2686 #if USB_DEBUG
2687                 if (ehcidebug > 15) {
2688                         DPRINTF("FS-TD %d\n", nframes);
2689                         ehci_dump_sitd(sc, td);
2690                 }
2691 #endif
2692                 /* insert TD into schedule */
2693                 EHCI_APPEND_FS_TD(td, *pp_last);
2694                 pp_last++;
2695
2696                 plen++;
2697                 fss++;
2698                 td_last = td;
2699                 td = td->obj_next;
2700         }
2701
2702         xfer->td_transfer_last = td_last;
2703
2704         /* update isoc_next */
2705         xfer->endpoint->isoc_next = (pp_last - &sc->sc_isoc_fs_p_last[0]) &
2706             (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2707 }
2708
2709 static void
2710 ehci_device_isoc_fs_start(struct usb_xfer *xfer)
2711 {
2712         /* put transfer on interrupt queue */
2713         ehci_transfer_intr_enqueue(xfer);
2714 }
2715
2716 struct usb_pipe_methods ehci_device_isoc_fs_methods =
2717 {
2718         .open = ehci_device_isoc_fs_open,
2719         .close = ehci_device_isoc_fs_close,
2720         .enter = ehci_device_isoc_fs_enter,
2721         .start = ehci_device_isoc_fs_start,
2722 };
2723
2724 /*------------------------------------------------------------------------*
2725  * ehci high speed isochronous support
2726  *------------------------------------------------------------------------*/
2727 static void
2728 ehci_device_isoc_hs_open(struct usb_xfer *xfer)
2729 {
2730         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2731         ehci_itd_t *td;
2732         uint32_t temp;
2733         uint8_t ds;
2734
2735         usb_hs_bandwidth_alloc(xfer);
2736
2737         /* initialize all TD's */
2738
2739         for (ds = 0; ds != 2; ds++) {
2740
2741                 for (td = xfer->td_start[ds]; td; td = td->obj_next) {
2742
2743                         /* set TD inactive */
2744                         td->itd_status[0] = 0;
2745                         td->itd_status[1] = 0;
2746                         td->itd_status[2] = 0;
2747                         td->itd_status[3] = 0;
2748                         td->itd_status[4] = 0;
2749                         td->itd_status[5] = 0;
2750                         td->itd_status[6] = 0;
2751                         td->itd_status[7] = 0;
2752
2753                         /* set endpoint and address */
2754                         td->itd_bp[0] = htohc32(sc,
2755                             EHCI_ITD_SET_ADDR(xfer->address) |
2756                             EHCI_ITD_SET_ENDPT(UE_GET_ADDR(xfer->endpointno)));
2757
2758                         temp =
2759                             EHCI_ITD_SET_MPL(xfer->max_packet_size & 0x7FF);
2760
2761                         /* set direction */
2762                         if (UE_GET_DIR(xfer->endpointno) == UE_DIR_IN) {
2763                                 temp |= EHCI_ITD_SET_DIR_IN;
2764                         }
2765                         /* set maximum packet size */
2766                         td->itd_bp[1] = htohc32(sc, temp);
2767
2768                         /* set transfer multiplier */
2769                         td->itd_bp[2] = htohc32(sc, xfer->max_packet_count & 3);
2770
2771                         usb_pc_cpu_flush(td->page_cache);
2772                 }
2773         }
2774 }
2775
2776 static void
2777 ehci_device_isoc_hs_close(struct usb_xfer *xfer)
2778 {
2779         ehci_device_done(xfer, USB_ERR_CANCELLED);
2780
2781         /* bandwidth must be freed after device done */
2782         usb_hs_bandwidth_free(xfer);
2783 }
2784
2785 static void
2786 ehci_device_isoc_hs_enter(struct usb_xfer *xfer)
2787 {
2788         struct usb_page_search buf_res;
2789         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2790         ehci_itd_t *td;
2791         ehci_itd_t *td_last = NULL;
2792         ehci_itd_t **pp_last;
2793         bus_size_t page_addr;
2794         uint32_t *plen;
2795         uint32_t status;
2796         uint32_t buf_offset;
2797         uint32_t nframes;
2798         uint32_t itd_offset[8 + 1];
2799         uint8_t x;
2800         uint8_t td_no;
2801         uint8_t page_no;
2802
2803 #if USB_DEBUG
2804         uint8_t once = 1;
2805
2806 #endif
2807
2808         DPRINTFN(6, "xfer=%p next=%d nframes=%d\n",
2809             xfer, xfer->endpoint->isoc_next, xfer->nframes);
2810
2811         /* get the current frame index */
2812
2813         nframes = EOREAD4(sc, EHCI_FRINDEX) / 8;
2814
2815         /*
2816          * check if the frame index is within the window where the frames
2817          * will be inserted
2818          */
2819         buf_offset = (nframes - xfer->endpoint->isoc_next) &
2820             (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2821
2822         if ((xfer->endpoint->is_synced == 0) ||
2823             (buf_offset < ((xfer->nframes + 7) / 8))) {
2824                 /*
2825                  * If there is data underflow or the pipe queue is empty we
2826                  * schedule the transfer a few frames ahead of the current
2827                  * frame position. Else two isochronous transfers might
2828                  * overlap.
2829                  */
2830                 xfer->endpoint->isoc_next = (nframes + 3) &
2831                     (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2832                 xfer->endpoint->is_synced = 1;
2833                 DPRINTFN(3, "start next=%d\n", xfer->endpoint->isoc_next);
2834         }
2835         /*
2836          * compute how many milliseconds the insertion is ahead of the
2837          * current frame position:
2838          */
2839         buf_offset = (xfer->endpoint->isoc_next - nframes) &
2840             (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2841
2842         /*
2843          * pre-compute when the isochronous transfer will be finished:
2844          */
2845         xfer->isoc_time_complete =
2846             usb_isoc_time_expand(&sc->sc_bus, nframes) + buf_offset +
2847             ((xfer->nframes + 7) / 8);
2848
2849         /* get the real number of frames */
2850
2851         nframes = xfer->nframes;
2852
2853         buf_offset = 0;
2854         td_no = 0;
2855
2856         plen = xfer->frlengths;
2857
2858         /* toggle the DMA set we are using */
2859         xfer->flags_int.curr_dma_set ^= 1;
2860
2861         /* get next DMA set */
2862         td = xfer->td_start[xfer->flags_int.curr_dma_set];
2863         xfer->td_transfer_first = td;
2864
2865         pp_last = &sc->sc_isoc_hs_p_last[xfer->endpoint->isoc_next];
2866
2867         /* store starting position */
2868
2869         xfer->qh_pos = xfer->endpoint->isoc_next;
2870
2871         while (nframes) {
2872                 if (td == NULL) {
2873                         panic("%s:%d: out of TD's\n",
2874                             __FUNCTION__, __LINE__);
2875                 }
2876                 if (pp_last >= &sc->sc_isoc_hs_p_last[EHCI_VIRTUAL_FRAMELIST_COUNT]) {
2877                         pp_last = &sc->sc_isoc_hs_p_last[0];
2878                 }
2879                 /* range check */
2880                 if (*plen > xfer->max_frame_size) {
2881 #if USB_DEBUG
2882                         if (once) {
2883                                 once = 0;
2884                                 printf("%s: frame length(%d) exceeds %d bytes "
2885                                     "(frame truncated)\n",
2886                                     __FUNCTION__, *plen, xfer->max_frame_size);
2887                         }
2888 #endif
2889                         *plen = xfer->max_frame_size;
2890                 }
2891
2892                 if (xfer->endpoint->usb_smask & (1 << td_no)) {
2893                         status = (EHCI_ITD_SET_LEN(*plen) |
2894                             EHCI_ITD_ACTIVE |
2895                             EHCI_ITD_SET_PG(0));
2896                         td->itd_status[td_no] = htohc32(sc, status);
2897                         itd_offset[td_no] = buf_offset;
2898                         buf_offset += *plen;
2899                         plen++;
2900                         nframes --;
2901                 } else {
2902                         td->itd_status[td_no] = 0;      /* not active */
2903                         itd_offset[td_no] = buf_offset;
2904                 }
2905
2906                 td_no++;
2907
2908                 if ((td_no == 8) || (nframes == 0)) {
2909
2910                         /* the rest of the transfers are not active, if any */
2911                         for (x = td_no; x != 8; x++) {
2912                                 td->itd_status[x] = 0;  /* not active */
2913                         }
2914
2915                         /* check if there is any data to be transferred */
2916                         if (itd_offset[0] != buf_offset) {
2917                                 page_no = 0;
2918                                 itd_offset[td_no] = buf_offset;
2919
2920                                 /* get first page offset */
2921                                 usbd_get_page(xfer->frbuffers, itd_offset[0], &buf_res);
2922                                 /* get page address */
2923                                 page_addr = buf_res.physaddr & ~0xFFF;
2924                                 /* update page address */
2925                                 td->itd_bp[0] &= htohc32(sc, 0xFFF);
2926                                 td->itd_bp[0] |= htohc32(sc, page_addr);
2927
2928                                 for (x = 0; x != td_no; x++) {
2929                                         /* set page number and page offset */
2930                                         status = (EHCI_ITD_SET_PG(page_no) |
2931                                             (buf_res.physaddr & 0xFFF));
2932                                         td->itd_status[x] |= htohc32(sc, status);
2933
2934                                         /* get next page offset */
2935                                         if (itd_offset[x + 1] == buf_offset) {
2936                                                 /*
2937                                                  * We subtract one so that
2938                                                  * we don't go off the last
2939                                                  * page!
2940                                                  */
2941                                                 usbd_get_page(xfer->frbuffers, buf_offset - 1, &buf_res);
2942                                         } else {
2943                                                 usbd_get_page(xfer->frbuffers, itd_offset[x + 1], &buf_res);
2944                                         }
2945
2946                                         /* check if we need a new page */
2947                                         if ((buf_res.physaddr ^ page_addr) & ~0xFFF) {
2948                                                 /* new page needed */
2949                                                 page_addr = buf_res.physaddr & ~0xFFF;
2950                                                 if (page_no == 6) {
2951                                                         panic("%s: too many pages\n", __FUNCTION__);
2952                                                 }
2953                                                 page_no++;
2954                                                 /* update page address */
2955                                                 td->itd_bp[page_no] &= htohc32(sc, 0xFFF);
2956                                                 td->itd_bp[page_no] |= htohc32(sc, page_addr);
2957                                         }
2958                                 }
2959                         }
2960                         /* set IOC bit if we are complete */
2961                         if (nframes == 0) {
2962                                 td->itd_status[td_no - 1] |= htohc32(sc, EHCI_ITD_IOC);
2963                         }
2964                         usb_pc_cpu_flush(td->page_cache);
2965 #if USB_DEBUG
2966                         if (ehcidebug > 15) {
2967                                 DPRINTF("HS-TD %d\n", nframes);
2968                                 ehci_dump_itd(sc, td);
2969                         }
2970 #endif
2971                         /* insert TD into schedule */
2972                         EHCI_APPEND_HS_TD(td, *pp_last);
2973                         pp_last++;
2974
2975                         td_no = 0;
2976                         td_last = td;
2977                         td = td->obj_next;
2978                 }
2979         }
2980
2981         xfer->td_transfer_last = td_last;
2982
2983         /* update isoc_next */
2984         xfer->endpoint->isoc_next = (pp_last - &sc->sc_isoc_hs_p_last[0]) &
2985             (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2986 }
2987
2988 static void
2989 ehci_device_isoc_hs_start(struct usb_xfer *xfer)
2990 {
2991         /* put transfer on interrupt queue */
2992         ehci_transfer_intr_enqueue(xfer);
2993 }
2994
2995 struct usb_pipe_methods ehci_device_isoc_hs_methods =
2996 {
2997         .open = ehci_device_isoc_hs_open,
2998         .close = ehci_device_isoc_hs_close,
2999         .enter = ehci_device_isoc_hs_enter,
3000         .start = ehci_device_isoc_hs_start,
3001 };
3002
3003 /*------------------------------------------------------------------------*
3004  * ehci root control support
3005  *------------------------------------------------------------------------*
3006  * Simulate a hardware hub by handling all the necessary requests.
3007  *------------------------------------------------------------------------*/
3008
3009 static const
3010 struct usb_device_descriptor ehci_devd =
3011 {
3012         sizeof(struct usb_device_descriptor),
3013         UDESC_DEVICE,                   /* type */
3014         {0x00, 0x02},                   /* USB version */
3015         UDCLASS_HUB,                    /* class */
3016         UDSUBCLASS_HUB,                 /* subclass */
3017         UDPROTO_HSHUBSTT,               /* protocol */
3018         64,                             /* max packet */
3019         {0}, {0}, {0x00, 0x01},         /* device id */
3020         1, 2, 0,                        /* string indicies */
3021         1                               /* # of configurations */
3022 };
3023
3024 static const
3025 struct usb_device_qualifier ehci_odevd =
3026 {
3027         sizeof(struct usb_device_qualifier),
3028         UDESC_DEVICE_QUALIFIER,         /* type */
3029         {0x00, 0x02},                   /* USB version */
3030         UDCLASS_HUB,                    /* class */
3031         UDSUBCLASS_HUB,                 /* subclass */
3032         UDPROTO_FSHUB,                  /* protocol */
3033         0,                              /* max packet */
3034         0,                              /* # of configurations */
3035         0
3036 };
3037
3038 static const struct ehci_config_desc ehci_confd = {
3039         .confd = {
3040                 .bLength = sizeof(struct usb_config_descriptor),
3041                 .bDescriptorType = UDESC_CONFIG,
3042                 .wTotalLength[0] = sizeof(ehci_confd),
3043                 .bNumInterface = 1,
3044                 .bConfigurationValue = 1,
3045                 .iConfiguration = 0,
3046                 .bmAttributes = UC_SELF_POWERED,
3047                 .bMaxPower = 0          /* max power */
3048         },
3049         .ifcd = {
3050                 .bLength = sizeof(struct usb_interface_descriptor),
3051                 .bDescriptorType = UDESC_INTERFACE,
3052                 .bNumEndpoints = 1,
3053                 .bInterfaceClass = UICLASS_HUB,
3054                 .bInterfaceSubClass = UISUBCLASS_HUB,
3055                 .bInterfaceProtocol = UIPROTO_HSHUBSTT,
3056                 0
3057         },
3058         .endpd = {
3059                 .bLength = sizeof(struct usb_endpoint_descriptor),
3060                 .bDescriptorType = UDESC_ENDPOINT,
3061                 .bEndpointAddress = UE_DIR_IN | EHCI_INTR_ENDPT,
3062                 .bmAttributes = UE_INTERRUPT,
3063                 .wMaxPacketSize[0] = 8, /* max packet (63 ports) */
3064                 .bInterval = 255,
3065         },
3066 };
3067
3068 static const
3069 struct usb_hub_descriptor ehci_hubd =
3070 {
3071         0,                              /* dynamic length */
3072         UDESC_HUB,
3073         0,
3074         {0, 0},
3075         0,
3076         0,
3077         {0},
3078 };
3079
3080 static void
3081 ehci_disown(ehci_softc_t *sc, uint16_t index, uint8_t lowspeed)
3082 {
3083         uint32_t port;
3084         uint32_t v;
3085
3086         DPRINTF("index=%d lowspeed=%d\n", index, lowspeed);
3087
3088         port = EHCI_PORTSC(index);
3089         v = EOREAD4(sc, port) & ~EHCI_PS_CLEAR;
3090         EOWRITE4(sc, port, v | EHCI_PS_PO);
3091 }
3092
3093 static usb_error_t
3094 ehci_roothub_exec(struct usb_device *udev,
3095     struct usb_device_request *req, const void **pptr, uint16_t *plength)
3096 {
3097         ehci_softc_t *sc = EHCI_BUS2SC(udev->bus);
3098         const char *str_ptr;
3099         const void *ptr;
3100         uint32_t port;
3101         uint32_t v;
3102         uint16_t len;
3103         uint16_t i;
3104         uint16_t value;
3105         uint16_t index;
3106         uint8_t l;
3107         usb_error_t err;
3108
3109         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
3110
3111         /* buffer reset */
3112         ptr = (const void *)&sc->sc_hub_desc;
3113         len = 0;
3114         err = 0;
3115
3116         value = UGETW(req->wValue);
3117         index = UGETW(req->wIndex);
3118
3119         DPRINTFN(3, "type=0x%02x request=0x%02x wLen=0x%04x "
3120             "wValue=0x%04x wIndex=0x%04x\n",
3121             req->bmRequestType, req->bRequest,
3122             UGETW(req->wLength), value, index);
3123
3124 #define C(x,y) ((x) | ((y) << 8))
3125         switch (C(req->bRequest, req->bmRequestType)) {
3126         case C(UR_CLEAR_FEATURE, UT_WRITE_DEVICE):
3127         case C(UR_CLEAR_FEATURE, UT_WRITE_INTERFACE):
3128         case C(UR_CLEAR_FEATURE, UT_WRITE_ENDPOINT):
3129                 /*
3130                  * DEVICE_REMOTE_WAKEUP and ENDPOINT_HALT are no-ops
3131                  * for the integrated root hub.
3132                  */
3133                 break;
3134         case C(UR_GET_CONFIG, UT_READ_DEVICE):
3135                 len = 1;
3136                 sc->sc_hub_desc.temp[0] = sc->sc_conf;
3137                 break;
3138         case C(UR_GET_DESCRIPTOR, UT_READ_DEVICE):
3139                 switch (value >> 8) {
3140                 case UDESC_DEVICE:
3141                         if ((value & 0xff) != 0) {
3142                                 err = USB_ERR_IOERROR;
3143                                 goto done;
3144                         }
3145                         len = sizeof(ehci_devd);
3146                         ptr = (const void *)&ehci_devd;
3147                         break;
3148                         /*
3149                          * We can't really operate at another speed,
3150                          * but the specification says we need this
3151                          * descriptor:
3152                          */
3153                 case UDESC_DEVICE_QUALIFIER:
3154                         if ((value & 0xff) != 0) {
3155                                 err = USB_ERR_IOERROR;
3156                                 goto done;
3157                         }
3158                         len = sizeof(ehci_odevd);
3159                         ptr = (const void *)&ehci_odevd;
3160                         break;
3161
3162                 case UDESC_CONFIG:
3163                         if ((value & 0xff) != 0) {
3164                                 err = USB_ERR_IOERROR;
3165                                 goto done;
3166                         }
3167                         len = sizeof(ehci_confd);
3168                         ptr = (const void *)&ehci_confd;
3169                         break;
3170
3171                 case UDESC_STRING:
3172                         switch (value & 0xff) {
3173                         case 0: /* Language table */
3174                                 str_ptr = "\001";
3175                                 break;
3176
3177                         case 1: /* Vendor */
3178                                 str_ptr = sc->sc_vendor;
3179                                 break;
3180
3181                         case 2: /* Product */
3182                                 str_ptr = "EHCI root HUB";
3183                                 break;
3184
3185                         default:
3186                                 str_ptr = "";
3187                                 break;
3188                         }
3189
3190                         len = usb_make_str_desc(
3191                             sc->sc_hub_desc.temp,
3192                             sizeof(sc->sc_hub_desc.temp),
3193                             str_ptr);
3194                         break;
3195                 default:
3196                         err = USB_ERR_IOERROR;
3197                         goto done;
3198                 }
3199                 break;
3200         case C(UR_GET_INTERFACE, UT_READ_INTERFACE):
3201                 len = 1;
3202                 sc->sc_hub_desc.temp[0] = 0;
3203                 break;
3204         case C(UR_GET_STATUS, UT_READ_DEVICE):
3205                 len = 2;
3206                 USETW(sc->sc_hub_desc.stat.wStatus, UDS_SELF_POWERED);
3207                 break;
3208         case C(UR_GET_STATUS, UT_READ_INTERFACE):
3209         case C(UR_GET_STATUS, UT_READ_ENDPOINT):
3210                 len = 2;
3211                 USETW(sc->sc_hub_desc.stat.wStatus, 0);
3212                 break;
3213         case C(UR_SET_ADDRESS, UT_WRITE_DEVICE):
3214                 if (value >= EHCI_MAX_DEVICES) {
3215                         err = USB_ERR_IOERROR;
3216                         goto done;
3217                 }
3218                 sc->sc_addr = value;
3219                 break;
3220         case C(UR_SET_CONFIG, UT_WRITE_DEVICE):
3221                 if ((value != 0) && (value != 1)) {
3222                         err = USB_ERR_IOERROR;
3223                         goto done;
3224                 }
3225                 sc->sc_conf = value;
3226                 break;
3227         case C(UR_SET_DESCRIPTOR, UT_WRITE_DEVICE):
3228                 break;
3229         case C(UR_SET_FEATURE, UT_WRITE_DEVICE):
3230         case C(UR_SET_FEATURE, UT_WRITE_INTERFACE):
3231         case C(UR_SET_FEATURE, UT_WRITE_ENDPOINT):
3232                 err = USB_ERR_IOERROR;
3233                 goto done;
3234         case C(UR_SET_INTERFACE, UT_WRITE_INTERFACE):
3235                 break;
3236         case C(UR_SYNCH_FRAME, UT_WRITE_ENDPOINT):
3237                 break;
3238                 /* Hub requests */
3239         case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_DEVICE):
3240                 break;
3241         case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_OTHER):
3242                 DPRINTFN(9, "UR_CLEAR_PORT_FEATURE\n");
3243
3244                 if ((index < 1) ||
3245                     (index > sc->sc_noport)) {
3246                         err = USB_ERR_IOERROR;
3247                         goto done;
3248                 }
3249                 port = EHCI_PORTSC(index);
3250                 v = EOREAD4(sc, port) & ~EHCI_PS_CLEAR;
3251                 switch (value) {
3252                 case UHF_PORT_ENABLE:
3253                         EOWRITE4(sc, port, v & ~EHCI_PS_PE);
3254                         break;
3255                 case UHF_PORT_SUSPEND:
3256                         if ((v & EHCI_PS_SUSP) && (!(v & EHCI_PS_FPR))) {
3257
3258                                 /*
3259                                  * waking up a High Speed device is rather
3260                                  * complicated if
3261                                  */
3262                                 EOWRITE4(sc, port, v | EHCI_PS_FPR);
3263                         }
3264                         /* wait 20ms for resume sequence to complete */
3265                         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 50);
3266
3267                         EOWRITE4(sc, port, v & ~(EHCI_PS_SUSP |
3268                             EHCI_PS_FPR | (3 << 10) /* High Speed */ ));
3269
3270                         /* 4ms settle time */
3271                         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 250);
3272                         break;
3273                 case UHF_PORT_POWER:
3274                         EOWRITE4(sc, port, v & ~EHCI_PS_PP);
3275                         break;
3276                 case UHF_PORT_TEST:
3277                         DPRINTFN(3, "clear port test "
3278                             "%d\n", index);
3279                         break;
3280                 case UHF_PORT_INDICATOR:
3281                         DPRINTFN(3, "clear port ind "
3282                             "%d\n", index);
3283                         EOWRITE4(sc, port, v & ~EHCI_PS_PIC);
3284                         break;
3285                 case UHF_C_PORT_CONNECTION:
3286                         EOWRITE4(sc, port, v | EHCI_PS_CSC);
3287                         break;
3288                 case UHF_C_PORT_ENABLE:
3289                         EOWRITE4(sc, port, v | EHCI_PS_PEC);
3290                         break;
3291                 case UHF_C_PORT_SUSPEND:
3292                         EOWRITE4(sc, port, v | EHCI_PS_SUSP);
3293                         break;
3294                 case UHF_C_PORT_OVER_CURRENT:
3295                         EOWRITE4(sc, port, v | EHCI_PS_OCC);
3296                         break;
3297                 case UHF_C_PORT_RESET:
3298                         sc->sc_isreset = 0;
3299                         break;
3300                 default:
3301                         err = USB_ERR_IOERROR;
3302                         goto done;
3303                 }
3304                 break;
3305         case C(UR_GET_DESCRIPTOR, UT_READ_CLASS_DEVICE):
3306                 if ((value & 0xff) != 0) {
3307                         err = USB_ERR_IOERROR;
3308                         goto done;
3309                 }
3310                 v = EOREAD4(sc, EHCI_HCSPARAMS);
3311
3312                 sc->sc_hub_desc.hubd = ehci_hubd;
3313                 sc->sc_hub_desc.hubd.bNbrPorts = sc->sc_noport;
3314                 USETW(sc->sc_hub_desc.hubd.wHubCharacteristics,
3315                     (EHCI_HCS_PPC(v) ? UHD_PWR_INDIVIDUAL : UHD_PWR_NO_SWITCH) |
3316                     (EHCI_HCS_P_INDICATOR(EREAD4(sc, EHCI_HCSPARAMS)) ?
3317                     UHD_PORT_IND : 0));
3318                 /* XXX can't find out? */
3319                 sc->sc_hub_desc.hubd.bPwrOn2PwrGood = 200;
3320                 for (l = 0; l < sc->sc_noport; l++) {
3321                         /* XXX can't find out? */
3322                         sc->sc_hub_desc.hubd.DeviceRemovable[l / 8] &= ~(1 << (l % 8));
3323                 }
3324                 sc->sc_hub_desc.hubd.bDescLength =
3325                     8 + ((sc->sc_noport + 7) / 8);
3326                 len = sc->sc_hub_desc.hubd.bDescLength;
3327                 break;
3328         case C(UR_GET_STATUS, UT_READ_CLASS_DEVICE):
3329                 len = 16;
3330                 bzero(sc->sc_hub_desc.temp, 16);
3331                 break;
3332         case C(UR_GET_STATUS, UT_READ_CLASS_OTHER):
3333                 DPRINTFN(9, "get port status i=%d\n",
3334                     index);
3335                 if ((index < 1) ||
3336                     (index > sc->sc_noport)) {
3337                         err = USB_ERR_IOERROR;
3338                         goto done;
3339                 }
3340                 v = EOREAD4(sc, EHCI_PORTSC(index));
3341                 DPRINTFN(9, "port status=0x%04x\n", v);
3342                 if (sc->sc_flags & (EHCI_SCFLG_FORCESPEED | EHCI_SCFLG_TT)) {
3343                         if ((v & 0xc000000) == 0x8000000)
3344                                 i = UPS_HIGH_SPEED;
3345                         else if ((v & 0xc000000) == 0x4000000)
3346                                 i = UPS_LOW_SPEED;
3347                         else
3348                                 i = 0;
3349                 } else {
3350                         i = UPS_HIGH_SPEED;
3351                 }
3352                 if (v & EHCI_PS_CS)
3353                         i |= UPS_CURRENT_CONNECT_STATUS;
3354                 if (v & EHCI_PS_PE)
3355                         i |= UPS_PORT_ENABLED;
3356                 if ((v & EHCI_PS_SUSP) && !(v & EHCI_PS_FPR))
3357                         i |= UPS_SUSPEND;
3358                 if (v & EHCI_PS_OCA)
3359                         i |= UPS_OVERCURRENT_INDICATOR;
3360                 if (v & EHCI_PS_PR)
3361                         i |= UPS_RESET;
3362                 if (v & EHCI_PS_PP)
3363                         i |= UPS_PORT_POWER;
3364                 USETW(sc->sc_hub_desc.ps.wPortStatus, i);
3365                 i = 0;
3366                 if (v & EHCI_PS_CSC)
3367                         i |= UPS_C_CONNECT_STATUS;
3368                 if (v & EHCI_PS_PEC)
3369                         i |= UPS_C_PORT_ENABLED;
3370                 if (v & EHCI_PS_OCC)
3371                         i |= UPS_C_OVERCURRENT_INDICATOR;
3372                 if (v & EHCI_PS_FPR)
3373                         i |= UPS_C_SUSPEND;
3374                 if (sc->sc_isreset)
3375                         i |= UPS_C_PORT_RESET;
3376                 USETW(sc->sc_hub_desc.ps.wPortChange, i);
3377                 len = sizeof(sc->sc_hub_desc.ps);
3378                 break;
3379         case C(UR_SET_DESCRIPTOR, UT_WRITE_CLASS_DEVICE):
3380                 err = USB_ERR_IOERROR;
3381                 goto done;
3382         case C(UR_SET_FEATURE, UT_WRITE_CLASS_DEVICE):
3383                 break;
3384         case C(UR_SET_FEATURE, UT_WRITE_CLASS_OTHER):
3385                 if ((index < 1) ||
3386                     (index > sc->sc_noport)) {
3387                         err = USB_ERR_IOERROR;
3388                         goto done;
3389                 }
3390                 port = EHCI_PORTSC(index);
3391                 v = EOREAD4(sc, port) & ~EHCI_PS_CLEAR;
3392                 switch (value) {
3393                 case UHF_PORT_ENABLE:
3394                         EOWRITE4(sc, port, v | EHCI_PS_PE);
3395                         break;
3396                 case UHF_PORT_SUSPEND:
3397                         EOWRITE4(sc, port, v | EHCI_PS_SUSP);
3398                         break;
3399                 case UHF_PORT_RESET:
3400                         DPRINTFN(6, "reset port %d\n", index);
3401 #if USB_DEBUG
3402                         if (ehcinohighspeed) {
3403                                 /*
3404                                  * Connect USB device to companion
3405                                  * controller.
3406                                  */
3407                                 ehci_disown(sc, index, 1);
3408                                 break;
3409                         }
3410 #endif
3411                         if (EHCI_PS_IS_LOWSPEED(v) &&
3412                             (sc->sc_flags & EHCI_SCFLG_TT) == 0) {
3413                                 /* Low speed device, give up ownership. */
3414                                 ehci_disown(sc, index, 1);
3415                                 break;
3416                         }
3417                         /* Start reset sequence. */
3418                         v &= ~(EHCI_PS_PE | EHCI_PS_PR);
3419                         EOWRITE4(sc, port, v | EHCI_PS_PR);
3420
3421                         /* Wait for reset to complete. */
3422                         usb_pause_mtx(&sc->sc_bus.bus_mtx,
3423                             USB_MS_TO_TICKS(USB_PORT_ROOT_RESET_DELAY));
3424
3425                         /* Terminate reset sequence. */
3426                         if (!(sc->sc_flags & EHCI_SCFLG_NORESTERM))
3427                                 EOWRITE4(sc, port, v);
3428
3429                         /* Wait for HC to complete reset. */
3430                         usb_pause_mtx(&sc->sc_bus.bus_mtx,
3431                             USB_MS_TO_TICKS(EHCI_PORT_RESET_COMPLETE));
3432
3433                         v = EOREAD4(sc, port);
3434                         DPRINTF("ehci after reset, status=0x%08x\n", v);
3435                         if (v & EHCI_PS_PR) {
3436                                 device_printf(sc->sc_bus.bdev,
3437                                     "port reset timeout\n");
3438                                 err = USB_ERR_TIMEOUT;
3439                                 goto done;
3440                         }
3441                         if (!(v & EHCI_PS_PE) &&
3442                             (sc->sc_flags & EHCI_SCFLG_TT) == 0) {
3443                                 /* Not a high speed device, give up ownership.*/
3444                                 ehci_disown(sc, index, 0);
3445                                 break;
3446                         }
3447                         sc->sc_isreset = 1;
3448                         DPRINTF("ehci port %d reset, status = 0x%08x\n",
3449                             index, v);
3450                         break;
3451
3452                 case UHF_PORT_POWER:
3453                         DPRINTFN(3, "set port power %d\n", index);
3454                         EOWRITE4(sc, port, v | EHCI_PS_PP);
3455                         break;
3456
3457                 case UHF_PORT_TEST:
3458                         DPRINTFN(3, "set port test %d\n", index);
3459                         break;
3460
3461                 case UHF_PORT_INDICATOR:
3462                         DPRINTFN(3, "set port ind %d\n", index);
3463                         EOWRITE4(sc, port, v | EHCI_PS_PIC);
3464                         break;
3465
3466                 default:
3467                         err = USB_ERR_IOERROR;
3468                         goto done;
3469                 }
3470                 break;
3471         case C(UR_CLEAR_TT_BUFFER, UT_WRITE_CLASS_OTHER):
3472         case C(UR_RESET_TT, UT_WRITE_CLASS_OTHER):
3473         case C(UR_GET_TT_STATE, UT_READ_CLASS_OTHER):
3474         case C(UR_STOP_TT, UT_WRITE_CLASS_OTHER):
3475                 break;
3476         default:
3477                 err = USB_ERR_IOERROR;
3478                 goto done;
3479         }
3480 done:
3481         *plength = len;
3482         *pptr = ptr;
3483         return (err);
3484 }
3485
3486 static void
3487 ehci_xfer_setup(struct usb_setup_params *parm)
3488 {
3489         struct usb_page_search page_info;
3490         struct usb_page_cache *pc;
3491         ehci_softc_t *sc;
3492         struct usb_xfer *xfer;
3493         void *last_obj;
3494         uint32_t nqtd;
3495         uint32_t nqh;
3496         uint32_t nsitd;
3497         uint32_t nitd;
3498         uint32_t n;
3499
3500         sc = EHCI_BUS2SC(parm->udev->bus);
3501         xfer = parm->curr_xfer;
3502
3503         nqtd = 0;
3504         nqh = 0;
3505         nsitd = 0;
3506         nitd = 0;
3507
3508         /*
3509          * compute maximum number of some structures
3510          */
3511         if (parm->methods == &ehci_device_ctrl_methods) {
3512
3513                 /*
3514                  * The proof for the "nqtd" formula is illustrated like
3515                  * this:
3516                  *
3517                  * +------------------------------------+
3518                  * |                                    |
3519                  * |         |remainder ->              |
3520                  * |   +-----+---+                      |
3521                  * |   | xxx | x | frm 0                |
3522                  * |   +-----+---++                     |
3523                  * |   | xxx | xx | frm 1               |
3524                  * |   +-----+----+                     |
3525                  * |            ...                     |
3526                  * +------------------------------------+
3527                  *
3528                  * "xxx" means a completely full USB transfer descriptor
3529                  *
3530                  * "x" and "xx" means a short USB packet
3531                  *
3532                  * For the remainder of an USB transfer modulo
3533                  * "max_data_length" we need two USB transfer descriptors.
3534                  * One to transfer the remaining data and one to finalise
3535                  * with a zero length packet in case the "force_short_xfer"
3536                  * flag is set. We only need two USB transfer descriptors in
3537                  * the case where the transfer length of the first one is a
3538                  * factor of "max_frame_size". The rest of the needed USB
3539                  * transfer descriptors is given by the buffer size divided
3540                  * by the maximum data payload.
3541                  */
3542                 parm->hc_max_packet_size = 0x400;
3543                 parm->hc_max_packet_count = 1;
3544                 parm->hc_max_frame_size = EHCI_QTD_PAYLOAD_MAX;
3545                 xfer->flags_int.bdma_enable = 1;
3546
3547                 usbd_transfer_setup_sub(parm);
3548
3549                 nqh = 1;
3550                 nqtd = ((2 * xfer->nframes) + 1 /* STATUS */
3551                     + (xfer->max_data_length / xfer->max_hc_frame_size));
3552
3553         } else if (parm->methods == &ehci_device_bulk_methods) {
3554
3555                 parm->hc_max_packet_size = 0x400;
3556                 parm->hc_max_packet_count = 1;
3557                 parm->hc_max_frame_size = EHCI_QTD_PAYLOAD_MAX;
3558                 xfer->flags_int.bdma_enable = 1;
3559
3560                 usbd_transfer_setup_sub(parm);
3561
3562                 nqh = 1;
3563                 nqtd = ((2 * xfer->nframes)
3564                     + (xfer->max_data_length / xfer->max_hc_frame_size));
3565
3566         } else if (parm->methods == &ehci_device_intr_methods) {
3567
3568                 if (parm->speed == USB_SPEED_HIGH) {
3569                         parm->hc_max_packet_size = 0x400;
3570                         parm->hc_max_packet_count = 3;
3571                 } else if (parm->speed == USB_SPEED_FULL) {
3572                         parm->hc_max_packet_size = USB_FS_BYTES_PER_HS_UFRAME;
3573                         parm->hc_max_packet_count = 1;
3574                 } else {
3575                         parm->hc_max_packet_size = USB_FS_BYTES_PER_HS_UFRAME / 8;
3576                         parm->hc_max_packet_count = 1;
3577                 }
3578
3579                 parm->hc_max_frame_size = EHCI_QTD_PAYLOAD_MAX;
3580                 xfer->flags_int.bdma_enable = 1;
3581
3582                 usbd_transfer_setup_sub(parm);
3583
3584                 nqh = 1;
3585                 nqtd = ((2 * xfer->nframes)
3586                     + (xfer->max_data_length / xfer->max_hc_frame_size));
3587
3588         } else if (parm->methods == &ehci_device_isoc_fs_methods) {
3589
3590                 parm->hc_max_packet_size = 0x3FF;
3591                 parm->hc_max_packet_count = 1;
3592                 parm->hc_max_frame_size = 0x3FF;
3593                 xfer->flags_int.bdma_enable = 1;
3594
3595                 usbd_transfer_setup_sub(parm);
3596
3597                 nsitd = xfer->nframes;
3598
3599         } else if (parm->methods == &ehci_device_isoc_hs_methods) {
3600
3601                 parm->hc_max_packet_size = 0x400;
3602                 parm->hc_max_packet_count = 3;
3603                 parm->hc_max_frame_size = 0xC00;
3604                 xfer->flags_int.bdma_enable = 1;
3605
3606                 usbd_transfer_setup_sub(parm);
3607
3608                 nitd = ((xfer->nframes + 7) / 8) <<
3609                     usbd_xfer_get_fps_shift(xfer);
3610
3611         } else {
3612
3613                 parm->hc_max_packet_size = 0x400;
3614                 parm->hc_max_packet_count = 1;
3615                 parm->hc_max_frame_size = 0x400;
3616
3617                 usbd_transfer_setup_sub(parm);
3618         }
3619
3620 alloc_dma_set:
3621
3622         if (parm->err) {
3623                 return;
3624         }
3625         /*
3626          * Allocate queue heads and transfer descriptors
3627          */
3628         last_obj = NULL;
3629
3630         if (usbd_transfer_setup_sub_malloc(
3631             parm, &pc, sizeof(ehci_itd_t),
3632             EHCI_ITD_ALIGN, nitd)) {
3633                 parm->err = USB_ERR_NOMEM;
3634                 return;
3635         }
3636         if (parm->buf) {
3637                 for (n = 0; n != nitd; n++) {
3638                         ehci_itd_t *td;
3639
3640                         usbd_get_page(pc + n, 0, &page_info);
3641
3642                         td = page_info.buffer;
3643
3644                         /* init TD */
3645                         td->itd_self = htohc32(sc, page_info.physaddr | EHCI_LINK_ITD);
3646                         td->obj_next = last_obj;
3647                         td->page_cache = pc + n;
3648
3649                         last_obj = td;
3650
3651                         usb_pc_cpu_flush(pc + n);
3652                 }
3653         }
3654         if (usbd_transfer_setup_sub_malloc(
3655             parm, &pc, sizeof(ehci_sitd_t),
3656             EHCI_SITD_ALIGN, nsitd)) {
3657                 parm->err = USB_ERR_NOMEM;
3658                 return;
3659         }
3660         if (parm->buf) {
3661                 for (n = 0; n != nsitd; n++) {
3662                         ehci_sitd_t *td;
3663
3664                         usbd_get_page(pc + n, 0, &page_info);
3665
3666                         td = page_info.buffer;
3667
3668                         /* init TD */
3669                         td->sitd_self = htohc32(sc, page_info.physaddr | EHCI_LINK_SITD);
3670                         td->obj_next = last_obj;
3671                         td->page_cache = pc + n;
3672
3673                         last_obj = td;
3674
3675                         usb_pc_cpu_flush(pc + n);
3676                 }
3677         }
3678         if (usbd_transfer_setup_sub_malloc(
3679             parm, &pc, sizeof(ehci_qtd_t),
3680             EHCI_QTD_ALIGN, nqtd)) {
3681                 parm->err = USB_ERR_NOMEM;
3682                 return;
3683         }
3684         if (parm->buf) {
3685                 for (n = 0; n != nqtd; n++) {
3686                         ehci_qtd_t *qtd;
3687
3688                         usbd_get_page(pc + n, 0, &page_info);
3689
3690                         qtd = page_info.buffer;
3691
3692                         /* init TD */
3693                         qtd->qtd_self = htohc32(sc, page_info.physaddr);
3694                         qtd->obj_next = last_obj;
3695                         qtd->page_cache = pc + n;
3696
3697                         last_obj = qtd;
3698
3699                         usb_pc_cpu_flush(pc + n);
3700                 }
3701         }
3702         xfer->td_start[xfer->flags_int.curr_dma_set] = last_obj;
3703
3704         last_obj = NULL;
3705
3706         if (usbd_transfer_setup_sub_malloc(
3707             parm, &pc, sizeof(ehci_qh_t),
3708             EHCI_QH_ALIGN, nqh)) {
3709                 parm->err = USB_ERR_NOMEM;
3710                 return;
3711         }
3712         if (parm->buf) {
3713                 for (n = 0; n != nqh; n++) {
3714                         ehci_qh_t *qh;
3715
3716                         usbd_get_page(pc + n, 0, &page_info);
3717
3718                         qh = page_info.buffer;
3719
3720                         /* init QH */
3721                         qh->qh_self = htohc32(sc, page_info.physaddr | EHCI_LINK_QH);
3722                         qh->obj_next = last_obj;
3723                         qh->page_cache = pc + n;
3724
3725                         last_obj = qh;
3726
3727                         usb_pc_cpu_flush(pc + n);
3728                 }
3729         }
3730         xfer->qh_start[xfer->flags_int.curr_dma_set] = last_obj;
3731
3732         if (!xfer->flags_int.curr_dma_set) {
3733                 xfer->flags_int.curr_dma_set = 1;
3734                 goto alloc_dma_set;
3735         }
3736 }
3737
3738 static void
3739 ehci_xfer_unsetup(struct usb_xfer *xfer)
3740 {
3741         return;
3742 }
3743
3744 static void
3745 ehci_ep_init(struct usb_device *udev, struct usb_endpoint_descriptor *edesc,
3746     struct usb_endpoint *ep)
3747 {
3748         ehci_softc_t *sc = EHCI_BUS2SC(udev->bus);
3749
3750         DPRINTFN(2, "endpoint=%p, addr=%d, endpt=%d, mode=%d (%d)\n",
3751             ep, udev->address,
3752             edesc->bEndpointAddress, udev->flags.usb_mode,
3753             sc->sc_addr);
3754
3755         if (udev->flags.usb_mode != USB_MODE_HOST) {
3756                 /* not supported */
3757                 return;
3758         }
3759         if (udev->device_index != sc->sc_addr) {
3760
3761                 if ((udev->speed != USB_SPEED_HIGH) &&
3762                     ((udev->hs_hub_addr == 0) ||
3763                     (udev->hs_port_no == 0) ||
3764                     (udev->parent_hs_hub == NULL) ||
3765                     (udev->parent_hs_hub->hub == NULL))) {
3766                         /* We need a transaction translator */
3767                         goto done;
3768                 }
3769                 switch (edesc->bmAttributes & UE_XFERTYPE) {
3770                 case UE_CONTROL:
3771                         ep->methods = &ehci_device_ctrl_methods;
3772                         break;
3773                 case UE_INTERRUPT:
3774                         ep->methods = &ehci_device_intr_methods;
3775                         break;
3776                 case UE_ISOCHRONOUS:
3777                         if (udev->speed == USB_SPEED_HIGH) {
3778                                 ep->methods = &ehci_device_isoc_hs_methods;
3779                         } else if (udev->speed == USB_SPEED_FULL) {
3780                                 ep->methods = &ehci_device_isoc_fs_methods;
3781                         }
3782                         break;
3783                 case UE_BULK:
3784                         if (udev->speed != USB_SPEED_LOW) {
3785                                 ep->methods = &ehci_device_bulk_methods;
3786                         }
3787                         break;
3788                 default:
3789                         /* do nothing */
3790                         break;
3791                 }
3792         }
3793 done:
3794         return;
3795 }
3796
3797 static void
3798 ehci_get_dma_delay(struct usb_bus *bus, uint32_t *pus)
3799 {
3800         /*
3801          * Wait until the hardware has finished any possible use of
3802          * the transfer descriptor(s) and QH
3803          */
3804         *pus = (188);                   /* microseconds */
3805 }
3806
3807 static void
3808 ehci_device_resume(struct usb_device *udev)
3809 {
3810         ehci_softc_t *sc = EHCI_BUS2SC(udev->bus);
3811         struct usb_xfer *xfer;
3812         struct usb_pipe_methods *methods;
3813
3814         DPRINTF("\n");
3815
3816         USB_BUS_LOCK(udev->bus);
3817
3818         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
3819
3820                 if (xfer->xroot->udev == udev) {
3821
3822                         methods = xfer->endpoint->methods;
3823
3824                         if ((methods == &ehci_device_bulk_methods) ||
3825                             (methods == &ehci_device_ctrl_methods)) {
3826                                 EHCI_APPEND_QH(xfer->qh_start[xfer->flags_int.curr_dma_set],
3827                                     sc->sc_async_p_last);
3828                         }
3829                         if (methods == &ehci_device_intr_methods) {
3830                                 EHCI_APPEND_QH(xfer->qh_start[xfer->flags_int.curr_dma_set],
3831                                     sc->sc_intr_p_last[xfer->qh_pos]);
3832                         }
3833                 }
3834         }
3835
3836         USB_BUS_UNLOCK(udev->bus);
3837
3838         return;
3839 }
3840
3841 static void
3842 ehci_device_suspend(struct usb_device *udev)
3843 {
3844         ehci_softc_t *sc = EHCI_BUS2SC(udev->bus);
3845         struct usb_xfer *xfer;
3846         struct usb_pipe_methods *methods;
3847
3848         DPRINTF("\n");
3849
3850         USB_BUS_LOCK(udev->bus);
3851
3852         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
3853
3854                 if (xfer->xroot->udev == udev) {
3855
3856                         methods = xfer->endpoint->methods;
3857
3858                         if ((methods == &ehci_device_bulk_methods) ||
3859                             (methods == &ehci_device_ctrl_methods)) {
3860                                 EHCI_REMOVE_QH(xfer->qh_start[xfer->flags_int.curr_dma_set],
3861                                     sc->sc_async_p_last);
3862                         }
3863                         if (methods == &ehci_device_intr_methods) {
3864                                 EHCI_REMOVE_QH(xfer->qh_start[xfer->flags_int.curr_dma_set],
3865                                     sc->sc_intr_p_last[xfer->qh_pos]);
3866                         }
3867                 }
3868         }
3869
3870         USB_BUS_UNLOCK(udev->bus);
3871
3872         return;
3873 }
3874
3875 static void
3876 ehci_set_hw_power(struct usb_bus *bus)
3877 {
3878         ehci_softc_t *sc = EHCI_BUS2SC(bus);
3879         uint32_t temp;
3880         uint32_t flags;
3881
3882         DPRINTF("\n");
3883
3884         USB_BUS_LOCK(bus);
3885
3886         flags = bus->hw_power_state;
3887
3888         temp = EOREAD4(sc, EHCI_USBCMD);
3889
3890         temp &= ~(EHCI_CMD_ASE | EHCI_CMD_PSE);
3891
3892         if (flags & (USB_HW_POWER_CONTROL |
3893             USB_HW_POWER_BULK)) {
3894                 DPRINTF("Async is active\n");
3895                 temp |= EHCI_CMD_ASE;
3896         }
3897         if (flags & (USB_HW_POWER_INTERRUPT |
3898             USB_HW_POWER_ISOC)) {
3899                 DPRINTF("Periodic is active\n");
3900                 temp |= EHCI_CMD_PSE;
3901         }
3902         EOWRITE4(sc, EHCI_USBCMD, temp);
3903
3904         USB_BUS_UNLOCK(bus);
3905
3906         return;
3907 }
3908
3909 struct usb_bus_methods ehci_bus_methods =
3910 {
3911         .endpoint_init = ehci_ep_init,
3912         .xfer_setup = ehci_xfer_setup,
3913         .xfer_unsetup = ehci_xfer_unsetup,
3914         .get_dma_delay = ehci_get_dma_delay,
3915         .device_resume = ehci_device_resume,
3916         .device_suspend = ehci_device_suspend,
3917         .set_hw_power = ehci_set_hw_power,
3918         .roothub_exec = ehci_roothub_exec,
3919         .xfer_poll = ehci_do_poll,
3920 };