sys/dev/usb/controller/ehci.c

   1 /*-
   2  * Copyright (c) 2008 Hans Petter Selasky. All rights reserved.
   3  * Copyright (c) 2004 The NetBSD Foundation, Inc. All rights reserved.
   4  * Copyright (c) 2004 Lennart Augustsson. All rights reserved.
   5  * Copyright (c) 2004 Charles M. Hannum. All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  26  * SUCH DAMAGE.
  27  */
  28
  29 /*
  30  * USB Enhanced Host Controller Driver, a.k.a. USB 2.0 controller.
  31  *
  32  * The EHCI 0.96 spec can be found at
  33  * http://developer.intel.com/technology/usb/download/ehci-r096.pdf
  34  * The EHCI 1.0 spec can be found at
  35  * http://developer.intel.com/technology/usb/download/ehci-r10.pdf
  36  * and the USB 2.0 spec at
  37  * http://www.usb.org/developers/docs/usb_20.zip
  38  *
  39  */
  40
  41 /*
  42  * TODO:
  43  * 1) command failures are not recovered correctly
  44  */
  45
  46 #include <sys/cdefs.h>
  47 __FBSDID("$FreeBSD$");
  48
  49 #include <sys/stdint.h>
  50 #include <sys/stddef.h>
  51 #include <sys/param.h>
  52 #include <sys/queue.h>
  53 #include <sys/types.h>
  54 #include <sys/systm.h>
  55 #include <sys/kernel.h>
  56 #include <sys/bus.h>
  57 #include <sys/linker_set.h>
  58 #include <sys/module.h>
  59 #include <sys/lock.h>
  60 #include <sys/mutex.h>
  61 #include <sys/condvar.h>
  62 #include <sys/sysctl.h>
  63 #include <sys/sx.h>
  64 #include <sys/unistd.h>
  65 #include <sys/callout.h>
  66 #include <sys/malloc.h>
  67 #include <sys/priv.h>
  68
  69 #include <dev/usb/usb.h>
  70 #include <dev/usb/usbdi.h>
  71
  72 #define USB_DEBUG_VAR ehcidebug
  73
  74 #include <dev/usb/usb_core.h>
  75 #include <dev/usb/usb_debug.h>
  76 #include <dev/usb/usb_busdma.h>
  77 #include <dev/usb/usb_process.h>
  78 #include <dev/usb/usb_transfer.h>
  79 #include <dev/usb/usb_device.h>
  80 #include <dev/usb/usb_hub.h>
  81 #include <dev/usb/usb_util.h>
  82
  83 #include <dev/usb/usb_controller.h>
  84 #include <dev/usb/usb_bus.h>
  85 #include <dev/usb/controller/ehci.h>
  86 #include <dev/usb/controller/ehcireg.h>
  87
  88 #define EHCI_BUS2SC(bus) \
  89    ((ehci_softc_t *)(((uint8_t *)(bus)) - \
  90     ((uint8_t *)&(((ehci_softc_t *)0)->sc_bus))))
  91
  92 #if USB_DEBUG
  93 static int ehcidebug = 0;
  94 static int ehcinohighspeed = 0;
  95
  96 SYSCTL_NODE(_hw_usb, OID_AUTO, ehci, CTLFLAG_RW, 0, "USB ehci");
  97 SYSCTL_INT(_hw_usb_ehci, OID_AUTO, debug, CTLFLAG_RW,
  98     &ehcidebug, 0, "Debug level");
  99 SYSCTL_INT(_hw_usb_ehci, OID_AUTO, no_hs, CTLFLAG_RW,
 100     &ehcinohighspeed, 0, "Disable High Speed USB");
 101
 102 static void ehci_dump_regs(ehci_softc_t *sc);
 103 static void ehci_dump_sqh(ehci_softc_t *sc, ehci_qh_t *sqh);
 104
 105 #endif
 106
 107 #define EHCI_INTR_ENDPT 1
 108
 109 extern struct usb_bus_methods ehci_bus_methods;
 110 extern struct usb_pipe_methods ehci_device_bulk_methods;
 111 extern struct usb_pipe_methods ehci_device_ctrl_methods;
 112 extern struct usb_pipe_methods ehci_device_intr_methods;
 113 extern struct usb_pipe_methods ehci_device_isoc_fs_methods;
 114 extern struct usb_pipe_methods ehci_device_isoc_hs_methods;
 115
 116 static void ehci_do_poll(struct usb_bus *);
 117 static void ehci_device_done(struct usb_xfer *, usb_error_t);
 118 static uint8_t ehci_check_transfer(struct usb_xfer *);
 119 static void ehci_timeout(void *);
 120 static void ehci_poll_timeout(void *);
 121
 122 static void ehci_root_intr(ehci_softc_t *sc);
 123
 124 struct ehci_std_temp {
 125         ehci_softc_t *sc;
 126         struct usb_page_cache *pc;
 127         ehci_qtd_t *td;
 128         ehci_qtd_t *td_next;
 129         uint32_t average;
 130         uint32_t qtd_status;
 131         uint32_t len;
 132         uint16_t max_frame_size;
 133         uint8_t shortpkt;
 134         uint8_t auto_data_toggle;
 135         uint8_t setup_alt_next;
 136         uint8_t last_frame;
 137         uint8_t can_use_next;
 138 };
 139
 140 void
 141 ehci_iterate_hw_softc(struct usb_bus *bus, usb_bus_mem_sub_cb_t *cb)
 142 {
 143         ehci_softc_t *sc = EHCI_BUS2SC(bus);
 144         uint32_t i;
 145
 146         cb(bus, &sc->sc_hw.pframes_pc, &sc->sc_hw.pframes_pg,
 147             sizeof(uint32_t) * EHCI_FRAMELIST_COUNT, EHCI_FRAMELIST_ALIGN);
 148
 149         cb(bus, &sc->sc_hw.async_start_pc, &sc->sc_hw.async_start_pg,
 150             sizeof(ehci_qh_t), EHCI_QH_ALIGN);
 151
 152         for (i = 0; i != EHCI_VIRTUAL_FRAMELIST_COUNT; i++) {
 153                 cb(bus, sc->sc_hw.intr_start_pc + i,
 154                     sc->sc_hw.intr_start_pg + i,
 155                     sizeof(ehci_qh_t), EHCI_QH_ALIGN);
 156         }
 157
 158         for (i = 0; i != EHCI_VIRTUAL_FRAMELIST_COUNT; i++) {
 159                 cb(bus, sc->sc_hw.isoc_hs_start_pc + i,
 160                     sc->sc_hw.isoc_hs_start_pg + i,
 161                     sizeof(ehci_itd_t), EHCI_ITD_ALIGN);
 162         }
 163
 164         for (i = 0; i != EHCI_VIRTUAL_FRAMELIST_COUNT; i++) {
 165                 cb(bus, sc->sc_hw.isoc_fs_start_pc + i,
 166                     sc->sc_hw.isoc_fs_start_pg + i,
 167                     sizeof(ehci_sitd_t), EHCI_SITD_ALIGN);
 168         }
 169 }
 170
 171 usb_error_t
 172 ehci_reset(ehci_softc_t *sc)
 173 {
 174         uint32_t hcr;
 175         int i;
 176
 177         EOWRITE4(sc, EHCI_USBCMD, EHCI_CMD_HCRESET);
 178         for (i = 0; i < 100; i++) {
 179                 usb_pause_mtx(NULL, hz / 1000);
 180                 hcr = EOREAD4(sc, EHCI_USBCMD) & EHCI_CMD_HCRESET;
 181                 if (!hcr) {
 182                         if (sc->sc_flags & (EHCI_SCFLG_SETMODE | EHCI_SCFLG_BIGEMMIO)) {
 183                                 /*
 184                                  * Force USBMODE as requested.  Controllers
 185                                  * may have multiple operating modes.
 186                                  */
 187                                 uint32_t usbmode = EOREAD4(sc, EHCI_USBMODE);
 188                                 if (sc->sc_flags & EHCI_SCFLG_SETMODE) {
 189                                         usbmode = (usbmode &~ EHCI_UM_CM) | EHCI_UM_CM_HOST;
 190                                         device_printf(sc->sc_bus.bdev,
 191                                             "set host controller mode\n");
 192                                 }
 193                                 if (sc->sc_flags & EHCI_SCFLG_BIGEMMIO) {
 194                                         usbmode = (usbmode &~ EHCI_UM_ES) | EHCI_UM_ES_BE;
 195                                         device_printf(sc->sc_bus.bdev,
 196                                             "set big-endian mode\n");
 197                                 }
 198                                 EOWRITE4(sc,  EHCI_USBMODE, usbmode);
 199                         }
 200                         return (0);
 201                 }
 202         }
 203         device_printf(sc->sc_bus.bdev, "reset timeout\n");
 204         return (USB_ERR_IOERROR);
 205 }
 206
 207 static usb_error_t
 208 ehci_hcreset(ehci_softc_t *sc)
 209 {
 210         uint32_t hcr;
 211         int i;
 212
 213         EOWRITE4(sc, EHCI_USBCMD, 0);   /* Halt controller */
 214         for (i = 0; i < 100; i++) {
 215                 usb_pause_mtx(NULL, hz / 1000);
 216                 hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
 217                 if (hcr)
 218                         break;
 219         }
 220         if (!hcr)
 221                 /*
 222                  * Fall through and try reset anyway even though
 223                  * Table 2-9 in the EHCI spec says this will result
 224                  * in undefined behavior.
 225                  */
 226                 device_printf(sc->sc_bus.bdev, "stop timeout\n");
 227
 228         return ehci_reset(sc);
 229 }
 230
 231 usb_error_t
 232 ehci_init(ehci_softc_t *sc)
 233 {
 234         struct usb_page_search buf_res;
 235         uint32_t version;
 236         uint32_t sparams;
 237         uint32_t cparams;
 238         uint32_t hcr;
 239         uint16_t i;
 240         uint16_t x;
 241         uint16_t y;
 242         uint16_t bit;
 243         usb_error_t err = 0;
 244
 245         DPRINTF("start\n");
 246
 247         usb_callout_init_mtx(&sc->sc_tmo_pcd, &sc->sc_bus.bus_mtx, 0);
 248         usb_callout_init_mtx(&sc->sc_tmo_poll, &sc->sc_bus.bus_mtx, 0);
 249
 250 #if USB_DEBUG
 251         if (ehcidebug > 2) {
 252                 ehci_dump_regs(sc);
 253         }
 254 #endif
 255
 256         sc->sc_offs = EREAD1(sc, EHCI_CAPLENGTH);
 257
 258         version = EREAD2(sc, EHCI_HCIVERSION);
 259         device_printf(sc->sc_bus.bdev, "EHCI version %x.%x\n",
 260             version >> 8, version & 0xff);
 261
 262         sparams = EREAD4(sc, EHCI_HCSPARAMS);
 263         DPRINTF("sparams=0x%x\n", sparams);
 264
 265         sc->sc_noport = EHCI_HCS_N_PORTS(sparams);
 266         cparams = EREAD4(sc, EHCI_HCCPARAMS);
 267         DPRINTF("cparams=0x%x\n", cparams);
 268
 269         if (EHCI_HCC_64BIT(cparams)) {
 270                 DPRINTF("HCC uses 64-bit structures\n");
 271
 272                 /* MUST clear segment register if 64 bit capable */
 273                 EWRITE4(sc, EHCI_CTRLDSSEGMENT, 0);
 274         }
 275         sc->sc_bus.usbrev = USB_REV_2_0;
 276
 277         /* Reset the controller */
 278         DPRINTF("%s: resetting\n", device_get_nameunit(sc->sc_bus.bdev));
 279
 280         err = ehci_hcreset(sc);
 281         if (err) {
 282                 device_printf(sc->sc_bus.bdev, "reset timeout\n");
 283                 return (err);
 284         }
 285         /*
 286          * use current frame-list-size selection 0: 1024*4 bytes 1:  512*4
 287          * bytes 2:  256*4 bytes 3:      unknown
 288          */
 289         if (EHCI_CMD_FLS(EOREAD4(sc, EHCI_USBCMD)) == 3) {
 290                 device_printf(sc->sc_bus.bdev, "invalid frame-list-size\n");
 291                 return (USB_ERR_IOERROR);
 292         }
 293         /* set up the bus struct */
 294         sc->sc_bus.methods = &ehci_bus_methods;
 295
 296         sc->sc_eintrs = EHCI_NORMAL_INTRS;
 297
 298         for (i = 0; i < EHCI_VIRTUAL_FRAMELIST_COUNT; i++) {
 299                 ehci_qh_t *qh;
 300
 301                 usbd_get_page(sc->sc_hw.intr_start_pc + i, 0, &buf_res);
 302
 303                 qh = buf_res.buffer;
 304
 305                 /* initialize page cache pointer */
 306
 307                 qh->page_cache = sc->sc_hw.intr_start_pc + i;
 308
 309                 /* store a pointer to queue head */
 310
 311                 sc->sc_intr_p_last[i] = qh;
 312
 313                 qh->qh_self =
 314                     htohc32(sc, buf_res.physaddr) |
 315                     htohc32(sc, EHCI_LINK_QH);
 316
 317                 qh->qh_endp =
 318                     htohc32(sc, EHCI_QH_SET_EPS(EHCI_QH_SPEED_HIGH));
 319                 qh->qh_endphub =
 320                     htohc32(sc, EHCI_QH_SET_MULT(1));
 321                 qh->qh_curqtd = 0;
 322
 323                 qh->qh_qtd.qtd_next =
 324                     htohc32(sc, EHCI_LINK_TERMINATE);
 325                 qh->qh_qtd.qtd_altnext =
 326                     htohc32(sc, EHCI_LINK_TERMINATE);
 327                 qh->qh_qtd.qtd_status =
 328                     htohc32(sc, EHCI_QTD_HALTED);
 329         }
 330
 331         /*
 332          * the QHs are arranged to give poll intervals that are
 333          * powers of 2 times 1ms
 334          */
 335         bit = EHCI_VIRTUAL_FRAMELIST_COUNT / 2;
 336         while (bit) {
 337                 x = bit;
 338                 while (x & bit) {
 339                         ehci_qh_t *qh_x;
 340                         ehci_qh_t *qh_y;
 341
 342                         y = (x ^ bit) | (bit / 2);
 343
 344                         qh_x = sc->sc_intr_p_last[x];
 345                         qh_y = sc->sc_intr_p_last[y];
 346
 347                         /*
 348                          * the next QH has half the poll interval
 349                          */
 350                         qh_x->qh_link = qh_y->qh_self;
 351
 352                         x++;
 353                 }
 354                 bit >>= 1;
 355         }
 356
 357         if (1) {
 358                 ehci_qh_t *qh;
 359
 360                 qh = sc->sc_intr_p_last[0];
 361
 362                 /* the last (1ms) QH terminates */
 363                 qh->qh_link = htohc32(sc, EHCI_LINK_TERMINATE);
 364         }
 365         for (i = 0; i < EHCI_VIRTUAL_FRAMELIST_COUNT; i++) {
 366                 ehci_sitd_t *sitd;
 367                 ehci_itd_t *itd;
 368
 369                 usbd_get_page(sc->sc_hw.isoc_fs_start_pc + i, 0, &buf_res);
 370
 371                 sitd = buf_res.buffer;
 372
 373                 /* initialize page cache pointer */
 374
 375                 sitd->page_cache = sc->sc_hw.isoc_fs_start_pc + i;
 376
 377                 /* store a pointer to the transfer descriptor */
 378
 379                 sc->sc_isoc_fs_p_last[i] = sitd;
 380
 381                 /* initialize full speed isochronous */
 382
 383                 sitd->sitd_self =
 384                     htohc32(sc, buf_res.physaddr) |
 385                     htohc32(sc, EHCI_LINK_SITD);
 386
 387                 sitd->sitd_back =
 388                     htohc32(sc, EHCI_LINK_TERMINATE);
 389
 390                 sitd->sitd_next =
 391                     sc->sc_intr_p_last[i | (EHCI_VIRTUAL_FRAMELIST_COUNT / 2)]->qh_self;
 392
 393
 394                 usbd_get_page(sc->sc_hw.isoc_hs_start_pc + i, 0, &buf_res);
 395
 396                 itd = buf_res.buffer;
 397
 398                 /* initialize page cache pointer */
 399
 400                 itd->page_cache = sc->sc_hw.isoc_hs_start_pc + i;
 401
 402                 /* store a pointer to the transfer descriptor */
 403
 404                 sc->sc_isoc_hs_p_last[i] = itd;
 405
 406                 /* initialize high speed isochronous */
 407
 408                 itd->itd_self =
 409                     htohc32(sc, buf_res.physaddr) |
 410                     htohc32(sc, EHCI_LINK_ITD);
 411
 412                 itd->itd_next =
 413                     sitd->sitd_self;
 414         }
 415
 416         usbd_get_page(&sc->sc_hw.pframes_pc, 0, &buf_res);
 417
 418         if (1) {
 419                 uint32_t *pframes;
 420
 421                 pframes = buf_res.buffer;
 422
 423                 /*
 424                  * execution order:
 425                  * pframes -> high speed isochronous ->
 426                  *    full speed isochronous -> interrupt QH's
 427                  */
 428                 for (i = 0; i < EHCI_FRAMELIST_COUNT; i++) {
 429                         pframes[i] = sc->sc_isoc_hs_p_last
 430                             [i & (EHCI_VIRTUAL_FRAMELIST_COUNT - 1)]->itd_self;
 431                 }
 432         }
 433         /* setup sync list pointer */
 434         EOWRITE4(sc, EHCI_PERIODICLISTBASE, buf_res.physaddr);
 435
 436         usbd_get_page(&sc->sc_hw.async_start_pc, 0, &buf_res);
 437
 438         if (1) {
 439
 440                 ehci_qh_t *qh;
 441
 442                 qh = buf_res.buffer;
 443
 444                 /* initialize page cache pointer */
 445
 446                 qh->page_cache = &sc->sc_hw.async_start_pc;
 447
 448                 /* store a pointer to the queue head */
 449
 450                 sc->sc_async_p_last = qh;
 451
 452                 /* init dummy QH that starts the async list */
 453
 454                 qh->qh_self =
 455                     htohc32(sc, buf_res.physaddr) |
 456                     htohc32(sc, EHCI_LINK_QH);
 457
 458                 /* fill the QH */
 459                 qh->qh_endp =
 460                     htohc32(sc, EHCI_QH_SET_EPS(EHCI_QH_SPEED_HIGH) | EHCI_QH_HRECL);
 461                 qh->qh_endphub = htohc32(sc, EHCI_QH_SET_MULT(1));
 462                 qh->qh_link = qh->qh_self;
 463                 qh->qh_curqtd = 0;
 464
 465                 /* fill the overlay qTD */
 466                 qh->qh_qtd.qtd_next = htohc32(sc, EHCI_LINK_TERMINATE);
 467                 qh->qh_qtd.qtd_altnext = htohc32(sc, EHCI_LINK_TERMINATE);
 468                 qh->qh_qtd.qtd_status = htohc32(sc, EHCI_QTD_HALTED);
 469         }
 470         /* flush all cache into memory */
 471
 472         usb_bus_mem_flush_all(&sc->sc_bus, &ehci_iterate_hw_softc);
 473
 474 #if USB_DEBUG
 475         if (ehcidebug) {
 476                 ehci_dump_sqh(sc, sc->sc_async_p_last);
 477         }
 478 #endif
 479
 480         /* setup async list pointer */
 481         EOWRITE4(sc, EHCI_ASYNCLISTADDR, buf_res.physaddr | EHCI_LINK_QH);
 482
 483
 484         /* enable interrupts */
 485         EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
 486
 487         /* turn on controller */
 488         EOWRITE4(sc, EHCI_USBCMD,
 489             EHCI_CMD_ITC_1 |            /* 1 microframes interrupt delay */
 490             (EOREAD4(sc, EHCI_USBCMD) & EHCI_CMD_FLS_M) |
 491             EHCI_CMD_ASE |
 492             EHCI_CMD_PSE |
 493             EHCI_CMD_RS);
 494
 495         /* Take over port ownership */
 496         EOWRITE4(sc, EHCI_CONFIGFLAG, EHCI_CONF_CF);
 497
 498         for (i = 0; i < 100; i++) {
 499                 usb_pause_mtx(NULL, hz / 1000);
 500                 hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
 501                 if (!hcr) {
 502                         break;
 503                 }
 504         }
 505         if (hcr) {
 506                 device_printf(sc->sc_bus.bdev, "run timeout\n");
 507                 return (USB_ERR_IOERROR);
 508         }
 509
 510         if (!err) {
 511                 /* catch any lost interrupts */
 512                 ehci_do_poll(&sc->sc_bus);
 513         }
 514         return (err);
 515 }
 516
 517 /*
 518  * shut down the controller when the system is going down
 519  */
 520 void
 521 ehci_detach(ehci_softc_t *sc)
 522 {
 523         USB_BUS_LOCK(&sc->sc_bus);
 524
 525         usb_callout_stop(&sc->sc_tmo_pcd);
 526         usb_callout_stop(&sc->sc_tmo_poll);
 527
 528         EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
 529         USB_BUS_UNLOCK(&sc->sc_bus);
 530
 531         if (ehci_hcreset(sc)) {
 532                 DPRINTF("reset failed!\n");
 533         }
 534
 535         /* XXX let stray task complete */
 536         usb_pause_mtx(NULL, hz / 20);
 537
 538         usb_callout_drain(&sc->sc_tmo_pcd);
 539         usb_callout_drain(&sc->sc_tmo_poll);
 540 }
 541
 542 void
 543 ehci_suspend(ehci_softc_t *sc)
 544 {
 545         uint32_t cmd;
 546         uint32_t hcr;
 547         uint8_t i;
 548
 549         USB_BUS_LOCK(&sc->sc_bus);
 550
 551         for (i = 1; i <= sc->sc_noport; i++) {
 552                 cmd = EOREAD4(sc, EHCI_PORTSC(i));
 553                 if (((cmd & EHCI_PS_PO) == 0) &&
 554                     ((cmd & EHCI_PS_PE) == EHCI_PS_PE)) {
 555                         EOWRITE4(sc, EHCI_PORTSC(i),
 556                             cmd | EHCI_PS_SUSP);
 557                 }
 558         }
 559
 560         sc->sc_cmd = EOREAD4(sc, EHCI_USBCMD);
 561
 562         cmd = sc->sc_cmd & ~(EHCI_CMD_ASE | EHCI_CMD_PSE);
 563         EOWRITE4(sc, EHCI_USBCMD, cmd);
 564
 565         for (i = 0; i < 100; i++) {
 566                 hcr = EOREAD4(sc, EHCI_USBSTS) &
 567                     (EHCI_STS_ASS | EHCI_STS_PSS);
 568
 569                 if (hcr == 0) {
 570                         break;
 571                 }
 572                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 1000);
 573         }
 574
 575         if (hcr != 0) {
 576                 device_printf(sc->sc_bus.bdev, "reset timeout\n");
 577         }
 578         cmd &= ~EHCI_CMD_RS;
 579         EOWRITE4(sc, EHCI_USBCMD, cmd);
 580
 581         for (i = 0; i < 100; i++) {
 582                 hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
 583                 if (hcr == EHCI_STS_HCH) {
 584                         break;
 585                 }
 586                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 1000);
 587         }
 588
 589         if (hcr != EHCI_STS_HCH) {
 590                 device_printf(sc->sc_bus.bdev,
 591                     "config timeout\n");
 592         }
 593         USB_BUS_UNLOCK(&sc->sc_bus);
 594 }
 595
 596 void
 597 ehci_resume(ehci_softc_t *sc)
 598 {
 599         struct usb_page_search buf_res;
 600         uint32_t cmd;
 601         uint32_t hcr;
 602         uint8_t i;
 603
 604         USB_BUS_LOCK(&sc->sc_bus);
 605
 606         /* restore things in case the bios doesn't */
 607         EOWRITE4(sc, EHCI_CTRLDSSEGMENT, 0);
 608
 609         usbd_get_page(&sc->sc_hw.pframes_pc, 0, &buf_res);
 610         EOWRITE4(sc, EHCI_PERIODICLISTBASE, buf_res.physaddr);
 611
 612         usbd_get_page(&sc->sc_hw.async_start_pc, 0, &buf_res);
 613         EOWRITE4(sc, EHCI_ASYNCLISTADDR, buf_res.physaddr | EHCI_LINK_QH);
 614
 615         EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
 616
 617         hcr = 0;
 618         for (i = 1; i <= sc->sc_noport; i++) {
 619                 cmd = EOREAD4(sc, EHCI_PORTSC(i));
 620                 if (((cmd & EHCI_PS_PO) == 0) &&
 621                     ((cmd & EHCI_PS_SUSP) == EHCI_PS_SUSP)) {
 622                         EOWRITE4(sc, EHCI_PORTSC(i),
 623                             cmd | EHCI_PS_FPR);
 624                         hcr = 1;
 625                 }
 626         }
 627
 628         if (hcr) {
 629                 usb_pause_mtx(&sc->sc_bus.bus_mtx,
 630                     USB_MS_TO_TICKS(USB_RESUME_WAIT));
 631
 632                 for (i = 1; i <= sc->sc_noport; i++) {
 633                         cmd = EOREAD4(sc, EHCI_PORTSC(i));
 634                         if (((cmd & EHCI_PS_PO) == 0) &&
 635                             ((cmd & EHCI_PS_SUSP) == EHCI_PS_SUSP)) {
 636                                 EOWRITE4(sc, EHCI_PORTSC(i),
 637                                     cmd & ~EHCI_PS_FPR);
 638                         }
 639                 }
 640         }
 641         EOWRITE4(sc, EHCI_USBCMD, sc->sc_cmd);
 642
 643         for (i = 0; i < 100; i++) {
 644                 hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
 645                 if (hcr != EHCI_STS_HCH) {
 646                         break;
 647                 }
 648                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 1000);
 649         }
 650         if (hcr == EHCI_STS_HCH) {
 651                 device_printf(sc->sc_bus.bdev, "config timeout\n");
 652         }
 653
 654         USB_BUS_UNLOCK(&sc->sc_bus);
 655
 656         usb_pause_mtx(NULL,
 657             USB_MS_TO_TICKS(USB_RESUME_WAIT));
 658
 659         /* catch any lost interrupts */
 660         ehci_do_poll(&sc->sc_bus);
 661 }
 662
 663 void
 664 ehci_shutdown(ehci_softc_t *sc)
 665 {
 666         DPRINTF("stopping the HC\n");
 667
 668         if (ehci_hcreset(sc)) {
 669                 DPRINTF("reset failed!\n");
 670         }
 671 }
 672
 673 #if USB_DEBUG
 674 static void
 675 ehci_dump_regs(ehci_softc_t *sc)
 676 {
 677         uint32_t i;
 678
 679         i = EOREAD4(sc, EHCI_USBCMD);
 680         printf("cmd=0x%08x\n", i);
 681
 682         if (i & EHCI_CMD_ITC_1)
 683                 printf(" EHCI_CMD_ITC_1\n");
 684         if (i & EHCI_CMD_ITC_2)
 685                 printf(" EHCI_CMD_ITC_2\n");
 686         if (i & EHCI_CMD_ITC_4)
 687                 printf(" EHCI_CMD_ITC_4\n");
 688         if (i & EHCI_CMD_ITC_8)
 689                 printf(" EHCI_CMD_ITC_8\n");
 690         if (i & EHCI_CMD_ITC_16)
 691                 printf(" EHCI_CMD_ITC_16\n");
 692         if (i & EHCI_CMD_ITC_32)
 693                 printf(" EHCI_CMD_ITC_32\n");
 694         if (i & EHCI_CMD_ITC_64)
 695                 printf(" EHCI_CMD_ITC_64\n");
 696         if (i & EHCI_CMD_ASPME)
 697                 printf(" EHCI_CMD_ASPME\n");
 698         if (i & EHCI_CMD_ASPMC)
 699                 printf(" EHCI_CMD_ASPMC\n");
 700         if (i & EHCI_CMD_LHCR)
 701                 printf(" EHCI_CMD_LHCR\n");
 702         if (i & EHCI_CMD_IAAD)
 703                 printf(" EHCI_CMD_IAAD\n");
 704         if (i & EHCI_CMD_ASE)
 705                 printf(" EHCI_CMD_ASE\n");
 706         if (i & EHCI_CMD_PSE)
 707                 printf(" EHCI_CMD_PSE\n");
 708         if (i & EHCI_CMD_FLS_M)
 709                 printf(" EHCI_CMD_FLS_M\n");
 710         if (i & EHCI_CMD_HCRESET)
 711                 printf(" EHCI_CMD_HCRESET\n");
 712         if (i & EHCI_CMD_RS)
 713                 printf(" EHCI_CMD_RS\n");
 714
 715         i = EOREAD4(sc, EHCI_USBSTS);
 716
 717         printf("sts=0x%08x\n", i);
 718
 719         if (i & EHCI_STS_ASS)
 720                 printf(" EHCI_STS_ASS\n");
 721         if (i & EHCI_STS_PSS)
 722                 printf(" EHCI_STS_PSS\n");
 723         if (i & EHCI_STS_REC)
 724                 printf(" EHCI_STS_REC\n");
 725         if (i & EHCI_STS_HCH)
 726                 printf(" EHCI_STS_HCH\n");
 727         if (i & EHCI_STS_IAA)
 728                 printf(" EHCI_STS_IAA\n");
 729         if (i & EHCI_STS_HSE)
 730                 printf(" EHCI_STS_HSE\n");
 731         if (i & EHCI_STS_FLR)
 732                 printf(" EHCI_STS_FLR\n");
 733         if (i & EHCI_STS_PCD)
 734                 printf(" EHCI_STS_PCD\n");
 735         if (i & EHCI_STS_ERRINT)
 736                 printf(" EHCI_STS_ERRINT\n");
 737         if (i & EHCI_STS_INT)
 738                 printf(" EHCI_STS_INT\n");
 739
 740         printf("ien=0x%08x\n",
 741             EOREAD4(sc, EHCI_USBINTR));
 742         printf("frindex=0x%08x ctrdsegm=0x%08x periodic=0x%08x async=0x%08x\n",
 743             EOREAD4(sc, EHCI_FRINDEX),
 744             EOREAD4(sc, EHCI_CTRLDSSEGMENT),
 745             EOREAD4(sc, EHCI_PERIODICLISTBASE),
 746             EOREAD4(sc, EHCI_ASYNCLISTADDR));
 747         for (i = 1; i <= sc->sc_noport; i++) {
 748                 printf("port %d status=0x%08x\n", i,
 749                     EOREAD4(sc, EHCI_PORTSC(i)));
 750         }
 751 }
 752
 753 static void
 754 ehci_dump_link(ehci_softc_t *sc, uint32_t link, int type)
 755 {
 756         link = hc32toh(sc, link);
 757         printf("0x%08x", link);
 758         if (link & EHCI_LINK_TERMINATE)
 759                 printf("<T>");
 760         else {
 761                 printf("<");
 762                 if (type) {
 763                         switch (EHCI_LINK_TYPE(link)) {
 764                         case EHCI_LINK_ITD:
 765                                 printf("ITD");
 766                                 break;
 767                         case EHCI_LINK_QH:
 768                                 printf("QH");
 769                                 break;
 770                         case EHCI_LINK_SITD:
 771                                 printf("SITD");
 772                                 break;
 773                         case EHCI_LINK_FSTN:
 774                                 printf("FSTN");
 775                                 break;
 776                         }
 777                 }
 778                 printf(">");
 779         }
 780 }
 781
 782 static void
 783 ehci_dump_qtd(ehci_softc_t *sc, ehci_qtd_t *qtd)
 784 {
 785         uint32_t s;
 786
 787         printf("  next=");
 788         ehci_dump_link(sc, qtd->qtd_next, 0);
 789         printf(" altnext=");
 790         ehci_dump_link(sc, qtd->qtd_altnext, 0);
 791         printf("\n");
 792         s = hc32toh(sc, qtd->qtd_status);
 793         printf("  status=0x%08x: toggle=%d bytes=0x%x ioc=%d c_page=0x%x\n",
 794             s, EHCI_QTD_GET_TOGGLE(s), EHCI_QTD_GET_BYTES(s),
 795             EHCI_QTD_GET_IOC(s), EHCI_QTD_GET_C_PAGE(s));
 796         printf("    cerr=%d pid=%d stat=%s%s%s%s%s%s%s%s\n",
 797             EHCI_QTD_GET_CERR(s), EHCI_QTD_GET_PID(s),
 798             (s & EHCI_QTD_ACTIVE) ? "ACTIVE" : "NOT_ACTIVE",
 799             (s & EHCI_QTD_HALTED) ? "-HALTED" : "",
 800             (s & EHCI_QTD_BUFERR) ? "-BUFERR" : "",
 801             (s & EHCI_QTD_BABBLE) ? "-BABBLE" : "",
 802             (s & EHCI_QTD_XACTERR) ? "-XACTERR" : "",
 803             (s & EHCI_QTD_MISSEDMICRO) ? "-MISSED" : "",
 804             (s & EHCI_QTD_SPLITXSTATE) ? "-SPLIT" : "",
 805             (s & EHCI_QTD_PINGSTATE) ? "-PING" : "");
 806
 807         for (s = 0; s < 5; s++) {
 808                 printf("  buffer[%d]=0x%08x\n", s,
 809                     hc32toh(sc, qtd->qtd_buffer[s]));
 810         }
 811         for (s = 0; s < 5; s++) {
 812                 printf("  buffer_hi[%d]=0x%08x\n", s,
 813                     hc32toh(sc, qtd->qtd_buffer_hi[s]));
 814         }
 815 }
 816
 817 static uint8_t
 818 ehci_dump_sqtd(ehci_softc_t *sc, ehci_qtd_t *sqtd)
 819 {
 820         uint8_t temp;
 821
 822         usb_pc_cpu_invalidate(sqtd->page_cache);
 823         printf("QTD(%p) at 0x%08x:\n", sqtd, hc32toh(sc, sqtd->qtd_self));
 824         ehci_dump_qtd(sc, sqtd);
 825         temp = (sqtd->qtd_next & htohc32(sc, EHCI_LINK_TERMINATE)) ? 1 : 0;
 826         return (temp);
 827 }
 828
 829 static void
 830 ehci_dump_sqtds(ehci_softc_t *sc, ehci_qtd_t *sqtd)
 831 {
 832         uint16_t i;
 833         uint8_t stop;
 834
 835         stop = 0;
 836         for (i = 0; sqtd && (i < 20) && !stop; sqtd = sqtd->obj_next, i++) {
 837                 stop = ehci_dump_sqtd(sc, sqtd);
 838         }
 839         if (sqtd) {
 840                 printf("dump aborted, too many TDs\n");
 841         }
 842 }
 843
 844 static void
 845 ehci_dump_sqh(ehci_softc_t *sc, ehci_qh_t *qh)
 846 {
 847         uint32_t endp;
 848         uint32_t endphub;
 849
 850         usb_pc_cpu_invalidate(qh->page_cache);
 851         printf("QH(%p) at 0x%08x:\n", qh, hc32toh(sc, qh->qh_self) & ~0x1F);
 852         printf("  link=");
 853         ehci_dump_link(sc, qh->qh_link, 1);
 854         printf("\n");
 855         endp = hc32toh(sc, qh->qh_endp);
 856         printf("  endp=0x%08x\n", endp);
 857         printf("    addr=0x%02x inact=%d endpt=%d eps=%d dtc=%d hrecl=%d\n",
 858             EHCI_QH_GET_ADDR(endp), EHCI_QH_GET_INACT(endp),
 859             EHCI_QH_GET_ENDPT(endp), EHCI_QH_GET_EPS(endp),
 860             EHCI_QH_GET_DTC(endp), EHCI_QH_GET_HRECL(endp));
 861         printf("    mpl=0x%x ctl=%d nrl=%d\n",
 862             EHCI_QH_GET_MPL(endp), EHCI_QH_GET_CTL(endp),
 863             EHCI_QH_GET_NRL(endp));
 864         endphub = hc32toh(sc, qh->qh_endphub);
 865         printf("  endphub=0x%08x\n", endphub);
 866         printf("    smask=0x%02x cmask=0x%02x huba=0x%02x port=%d mult=%d\n",
 867             EHCI_QH_GET_SMASK(endphub), EHCI_QH_GET_CMASK(endphub),
 868             EHCI_QH_GET_HUBA(endphub), EHCI_QH_GET_PORT(endphub),
 869             EHCI_QH_GET_MULT(endphub));
 870         printf("  curqtd=");
 871         ehci_dump_link(sc, qh->qh_curqtd, 0);
 872         printf("\n");
 873         printf("Overlay qTD:\n");
 874         ehci_dump_qtd(sc, (void *)&qh->qh_qtd);
 875 }
 876
 877 static void
 878 ehci_dump_sitd(ehci_softc_t *sc, ehci_sitd_t *sitd)
 879 {
 880         usb_pc_cpu_invalidate(sitd->page_cache);
 881         printf("SITD(%p) at 0x%08x\n", sitd, hc32toh(sc, sitd->sitd_self) & ~0x1F);
 882         printf(" next=0x%08x\n", hc32toh(sc, sitd->sitd_next));
 883         printf(" portaddr=0x%08x dir=%s addr=%d endpt=0x%x port=0x%x huba=0x%x\n",
 884             hc32toh(sc, sitd->sitd_portaddr),
 885             (sitd->sitd_portaddr & htohc32(sc, EHCI_SITD_SET_DIR_IN))
 886             ? "in" : "out",
 887             EHCI_SITD_GET_ADDR(hc32toh(sc, sitd->sitd_portaddr)),
 888             EHCI_SITD_GET_ENDPT(hc32toh(sc, sitd->sitd_portaddr)),
 889             EHCI_SITD_GET_PORT(hc32toh(sc, sitd->sitd_portaddr)),
 890             EHCI_SITD_GET_HUBA(hc32toh(sc, sitd->sitd_portaddr)));
 891         printf(" mask=0x%08x\n", hc32toh(sc, sitd->sitd_mask));
 892         printf(" status=0x%08x <%s> len=0x%x\n", hc32toh(sc, sitd->sitd_status),
 893             (sitd->sitd_status & htohc32(sc, EHCI_SITD_ACTIVE)) ? "ACTIVE" : "",
 894             EHCI_SITD_GET_LEN(hc32toh(sc, sitd->sitd_status)));
 895         printf(" back=0x%08x, bp=0x%08x,0x%08x,0x%08x,0x%08x\n",
 896             hc32toh(sc, sitd->sitd_back),
 897             hc32toh(sc, sitd->sitd_bp[0]),
 898             hc32toh(sc, sitd->sitd_bp[1]),
 899             hc32toh(sc, sitd->sitd_bp_hi[0]),
 900             hc32toh(sc, sitd->sitd_bp_hi[1]));
 901 }
 902
 903 static void
 904 ehci_dump_itd(ehci_softc_t *sc, ehci_itd_t *itd)
 905 {
 906         usb_pc_cpu_invalidate(itd->page_cache);
 907         printf("ITD(%p) at 0x%08x\n", itd, hc32toh(sc, itd->itd_self) & ~0x1F);
 908         printf(" next=0x%08x\n", hc32toh(sc, itd->itd_next));
 909         printf(" status[0]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[0]),
 910             (itd->itd_status[0] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
 911         printf(" status[1]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[1]),
 912             (itd->itd_status[1] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
 913         printf(" status[2]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[2]),
 914             (itd->itd_status[2] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
 915         printf(" status[3]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[3]),
 916             (itd->itd_status[3] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
 917         printf(" status[4]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[4]),
 918             (itd->itd_status[4] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
 919         printf(" status[5]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[5]),
 920             (itd->itd_status[5] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
 921         printf(" status[6]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[6]),
 922             (itd->itd_status[6] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
 923         printf(" status[7]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[7]),
 924             (itd->itd_status[7] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
 925         printf(" bp[0]=0x%08x\n", hc32toh(sc, itd->itd_bp[0]));
 926         printf("  addr=0x%02x; endpt=0x%01x\n",
 927             EHCI_ITD_GET_ADDR(hc32toh(sc, itd->itd_bp[0])),
 928             EHCI_ITD_GET_ENDPT(hc32toh(sc, itd->itd_bp[0])));
 929         printf(" bp[1]=0x%08x\n", hc32toh(sc, itd->itd_bp[1]));
 930         printf(" dir=%s; mpl=0x%02x\n",
 931             (hc32toh(sc, itd->itd_bp[1]) & EHCI_ITD_SET_DIR_IN) ? "in" : "out",
 932             EHCI_ITD_GET_MPL(hc32toh(sc, itd->itd_bp[1])));
 933         printf(" bp[2..6]=0x%08x,0x%08x,0x%08x,0x%08x,0x%08x\n",
 934             hc32toh(sc, itd->itd_bp[2]),
 935             hc32toh(sc, itd->itd_bp[3]),
 936             hc32toh(sc, itd->itd_bp[4]),
 937             hc32toh(sc, itd->itd_bp[5]),
 938             hc32toh(sc, itd->itd_bp[6]));
 939         printf(" bp_hi=0x%08x,0x%08x,0x%08x,0x%08x,\n"
 940             "       0x%08x,0x%08x,0x%08x\n",
 941             hc32toh(sc, itd->itd_bp_hi[0]),
 942             hc32toh(sc, itd->itd_bp_hi[1]),
 943             hc32toh(sc, itd->itd_bp_hi[2]),
 944             hc32toh(sc, itd->itd_bp_hi[3]),
 945             hc32toh(sc, itd->itd_bp_hi[4]),
 946             hc32toh(sc, itd->itd_bp_hi[5]),
 947             hc32toh(sc, itd->itd_bp_hi[6]));
 948 }
 949
 950 static void
 951 ehci_dump_isoc(ehci_softc_t *sc)
 952 {
 953         ehci_itd_t *itd;
 954         ehci_sitd_t *sitd;
 955         uint16_t max = 1000;
 956         uint16_t pos;
 957
 958         pos = (EOREAD4(sc, EHCI_FRINDEX) / 8) &
 959             (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
 960
 961         printf("%s: isochronous dump from frame 0x%03x:\n",
 962             __FUNCTION__, pos);
 963
 964         itd = sc->sc_isoc_hs_p_last[pos];
 965         sitd = sc->sc_isoc_fs_p_last[pos];
 966
 967         while (itd && max && max--) {
 968                 ehci_dump_itd(sc, itd);
 969                 itd = itd->prev;
 970         }
 971
 972         while (sitd && max && max--) {
 973                 ehci_dump_sitd(sc, sitd);
 974                 sitd = sitd->prev;
 975         }
 976 }
 977
 978 #endif
 979
 980 static void
 981 ehci_transfer_intr_enqueue(struct usb_xfer *xfer)
 982 {
 983         /* check for early completion */
 984         if (ehci_check_transfer(xfer)) {
 985                 return;
 986         }
 987         /* put transfer on interrupt queue */
 988         usbd_transfer_enqueue(&xfer->xroot->bus->intr_q, xfer);
 989
 990         /* start timeout, if any */
 991         if (xfer->timeout != 0) {
 992                 usbd_transfer_timeout_ms(xfer, &ehci_timeout, xfer->timeout);
 993         }
 994 }
 995
 996 #define EHCI_APPEND_FS_TD(std,last) (last) = _ehci_append_fs_td(std,last)
 997 static ehci_sitd_t *
 998 _ehci_append_fs_td(ehci_sitd_t *std, ehci_sitd_t *last)
 999 {
1000         DPRINTFN(11, "%p to %p\n", std, last);
1001
1002         /* (sc->sc_bus.mtx) must be locked */
1003
1004         std->next = last->next;
1005         std->sitd_next = last->sitd_next;
1006
1007         std->prev = last;
1008
1009         usb_pc_cpu_flush(std->page_cache);
1010
1011         /*
1012          * the last->next->prev is never followed: std->next->prev = std;
1013          */
1014         last->next = std;
1015         last->sitd_next = std->sitd_self;
1016
1017         usb_pc_cpu_flush(last->page_cache);
1018
1019         return (std);
1020 }
1021
1022 #define EHCI_APPEND_HS_TD(std,last) (last) = _ehci_append_hs_td(std,last)
1023 static ehci_itd_t *
1024 _ehci_append_hs_td(ehci_itd_t *std, ehci_itd_t *last)
1025 {
1026         DPRINTFN(11, "%p to %p\n", std, last);
1027
1028         /* (sc->sc_bus.mtx) must be locked */
1029
1030         std->next = last->next;
1031         std->itd_next = last->itd_next;
1032
1033         std->prev = last;
1034
1035         usb_pc_cpu_flush(std->page_cache);
1036
1037         /*
1038          * the last->next->prev is never followed: std->next->prev = std;
1039          */
1040         last->next = std;
1041         last->itd_next = std->itd_self;
1042
1043         usb_pc_cpu_flush(last->page_cache);
1044
1045         return (std);
1046 }
1047
1048 #define EHCI_APPEND_QH(sqh,last) (last) = _ehci_append_qh(sqh,last)
1049 static ehci_qh_t *
1050 _ehci_append_qh(ehci_qh_t *sqh, ehci_qh_t *last)
1051 {
1052         DPRINTFN(11, "%p to %p\n", sqh, last);
1053
1054         if (sqh->prev != NULL) {
1055                 /* should not happen */
1056                 DPRINTFN(0, "QH already linked!\n");
1057                 return (last);
1058         }
1059         /* (sc->sc_bus.mtx) must be locked */
1060
1061         sqh->next = last->next;
1062         sqh->qh_link = last->qh_link;
1063
1064         sqh->prev = last;
1065
1066         usb_pc_cpu_flush(sqh->page_cache);
1067
1068         /*
1069          * the last->next->prev is never followed: sqh->next->prev = sqh;
1070          */
1071
1072         last->next = sqh;
1073         last->qh_link = sqh->qh_self;
1074
1075         usb_pc_cpu_flush(last->page_cache);
1076
1077         return (sqh);
1078 }
1079
1080 #define EHCI_REMOVE_FS_TD(std,last) (last) = _ehci_remove_fs_td(std,last)
1081 static ehci_sitd_t *
1082 _ehci_remove_fs_td(ehci_sitd_t *std, ehci_sitd_t *last)
1083 {
1084         DPRINTFN(11, "%p from %p\n", std, last);
1085
1086         /* (sc->sc_bus.mtx) must be locked */
1087
1088         std->prev->next = std->next;
1089         std->prev->sitd_next = std->sitd_next;
1090
1091         usb_pc_cpu_flush(std->prev->page_cache);
1092
1093         if (std->next) {
1094                 std->next->prev = std->prev;
1095                 usb_pc_cpu_flush(std->next->page_cache);
1096         }
1097         return ((last == std) ? std->prev : last);
1098 }
1099
1100 #define EHCI_REMOVE_HS_TD(std,last) (last) = _ehci_remove_hs_td(std,last)
1101 static ehci_itd_t *
1102 _ehci_remove_hs_td(ehci_itd_t *std, ehci_itd_t *last)
1103 {
1104         DPRINTFN(11, "%p from %p\n", std, last);
1105
1106         /* (sc->sc_bus.mtx) must be locked */
1107
1108         std->prev->next = std->next;
1109         std->prev->itd_next = std->itd_next;
1110
1111         usb_pc_cpu_flush(std->prev->page_cache);
1112
1113         if (std->next) {
1114                 std->next->prev = std->prev;
1115                 usb_pc_cpu_flush(std->next->page_cache);
1116         }
1117         return ((last == std) ? std->prev : last);
1118 }
1119
1120 #define EHCI_REMOVE_QH(sqh,last) (last) = _ehci_remove_qh(sqh,last)
1121 static ehci_qh_t *
1122 _ehci_remove_qh(ehci_qh_t *sqh, ehci_qh_t *last)
1123 {
1124         DPRINTFN(11, "%p from %p\n", sqh, last);
1125
1126         /* (sc->sc_bus.mtx) must be locked */
1127
1128         /* only remove if not removed from a queue */
1129         if (sqh->prev) {
1130
1131                 sqh->prev->next = sqh->next;
1132                 sqh->prev->qh_link = sqh->qh_link;
1133
1134                 usb_pc_cpu_flush(sqh->prev->page_cache);
1135
1136                 if (sqh->next) {
1137                         sqh->next->prev = sqh->prev;
1138                         usb_pc_cpu_flush(sqh->next->page_cache);
1139                 }
1140                 last = ((last == sqh) ? sqh->prev : last);
1141
1142                 sqh->prev = 0;
1143
1144                 usb_pc_cpu_flush(sqh->page_cache);
1145         }
1146         return (last);
1147 }
1148
1149 static usb_error_t
1150 ehci_non_isoc_done_sub(struct usb_xfer *xfer)
1151 {
1152         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
1153         ehci_qtd_t *td;
1154         ehci_qtd_t *td_alt_next;
1155         uint32_t status;
1156         uint16_t len;
1157
1158         td = xfer->td_transfer_cache;
1159         td_alt_next = td->alt_next;
1160
1161         if (xfer->aframes != xfer->nframes) {
1162                 usbd_xfer_set_frame_len(xfer, xfer->aframes, 0);
1163         }
1164         while (1) {
1165
1166                 usb_pc_cpu_invalidate(td->page_cache);
1167                 status = hc32toh(sc, td->qtd_status);
1168
1169                 len = EHCI_QTD_GET_BYTES(status);
1170
1171                 /*
1172                  * Verify the status length and
1173                  * add the length to "frlengths[]":
1174                  */
1175                 if (len > td->len) {
1176                         /* should not happen */
1177                         DPRINTF("Invalid status length, "
1178                             "0x%04x/0x%04x bytes\n", len, td->len);
1179                         status |= EHCI_QTD_HALTED;
1180                 } else if (xfer->aframes != xfer->nframes) {
1181                         xfer->frlengths[xfer->aframes] += td->len - len;
1182                 }
1183                 /* Check for last transfer */
1184                 if (((void *)td) == xfer->td_transfer_last) {
1185                         td = NULL;
1186                         break;
1187                 }
1188                 /* Check for transfer error */
1189                 if (status & EHCI_QTD_HALTED) {
1190                         /* the transfer is finished */
1191                         td = NULL;
1192                         break;
1193                 }
1194                 /* Check for short transfer */
1195                 if (len > 0) {
1196                         if (xfer->flags_int.short_frames_ok) {
1197                                 /* follow alt next */
1198                                 td = td->alt_next;
1199                         } else {
1200                                 /* the transfer is finished */
1201                                 td = NULL;
1202                         }
1203                         break;
1204                 }
1205                 td = td->obj_next;
1206
1207                 if (td->alt_next != td_alt_next) {
1208                         /* this USB frame is complete */
1209                         break;
1210                 }
1211         }
1212
1213         /* update transfer cache */
1214
1215         xfer->td_transfer_cache = td;
1216
1217 #if USB_DEBUG
1218         if (status & EHCI_QTD_STATERRS) {
1219                 DPRINTFN(11, "error, addr=%d, endpt=0x%02x, frame=0x%02x"
1220                     "status=%s%s%s%s%s%s%s%s\n",
1221                     xfer->address, xfer->endpointno, xfer->aframes,
1222                     (status & EHCI_QTD_ACTIVE) ? "[ACTIVE]" : "[NOT_ACTIVE]",
1223                     (status & EHCI_QTD_HALTED) ? "[HALTED]" : "",
1224                     (status & EHCI_QTD_BUFERR) ? "[BUFERR]" : "",
1225                     (status & EHCI_QTD_BABBLE) ? "[BABBLE]" : "",
1226                     (status & EHCI_QTD_XACTERR) ? "[XACTERR]" : "",
1227                     (status & EHCI_QTD_MISSEDMICRO) ? "[MISSED]" : "",
1228                     (status & EHCI_QTD_SPLITXSTATE) ? "[SPLIT]" : "",
1229                     (status & EHCI_QTD_PINGSTATE) ? "[PING]" : "");
1230         }
1231 #endif
1232
1233         return ((status & EHCI_QTD_HALTED) ?
1234             USB_ERR_STALLED : USB_ERR_NORMAL_COMPLETION);
1235 }
1236
1237 static void
1238 ehci_non_isoc_done(struct usb_xfer *xfer)
1239 {
1240         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
1241         ehci_qh_t *qh;
1242         uint32_t status;
1243         usb_error_t err = 0;
1244
1245         DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
1246             xfer, xfer->endpoint);
1247
1248 #if USB_DEBUG
1249         if (ehcidebug > 10) {
1250                 ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
1251
1252                 ehci_dump_sqtds(sc, xfer->td_transfer_first);
1253         }
1254 #endif
1255
1256         /* extract data toggle directly from the QH's overlay area */
1257
1258         qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
1259
1260         usb_pc_cpu_invalidate(qh->page_cache);
1261
1262         status = hc32toh(sc, qh->qh_qtd.qtd_status);
1263
1264         xfer->endpoint->toggle_next =
1265             (status & EHCI_QTD_TOGGLE_MASK) ? 1 : 0;
1266
1267         /* reset scanner */
1268
1269         xfer->td_transfer_cache = xfer->td_transfer_first;
1270
1271         if (xfer->flags_int.control_xfr) {
1272
1273                 if (xfer->flags_int.control_hdr) {
1274
1275                         err = ehci_non_isoc_done_sub(xfer);
1276                 }
1277                 xfer->aframes = 1;
1278
1279                 if (xfer->td_transfer_cache == NULL) {
1280                         goto done;
1281                 }
1282         }
1283         while (xfer->aframes != xfer->nframes) {
1284
1285                 err = ehci_non_isoc_done_sub(xfer);
1286                 xfer->aframes++;
1287
1288                 if (xfer->td_transfer_cache == NULL) {
1289                         goto done;
1290                 }
1291         }
1292
1293         if (xfer->flags_int.control_xfr &&
1294             !xfer->flags_int.control_act) {
1295
1296                 err = ehci_non_isoc_done_sub(xfer);
1297         }
1298 done:
1299         ehci_device_done(xfer, err);
1300 }
1301
1302 /*------------------------------------------------------------------------*
1303  *      ehci_check_transfer
1304  *
1305  * Return values:
1306  *    0: USB transfer is not finished
1307  * Else: USB transfer is finished
1308  *------------------------------------------------------------------------*/
1309 static uint8_t
1310 ehci_check_transfer(struct usb_xfer *xfer)
1311 {
1312         struct usb_pipe_methods *methods = xfer->endpoint->methods;
1313         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
1314
1315         uint32_t status;
1316
1317         DPRINTFN(13, "xfer=%p checking transfer\n", xfer);
1318
1319         if (methods == &ehci_device_isoc_fs_methods) {
1320                 ehci_sitd_t *td;
1321
1322                 /* isochronous full speed transfer */
1323
1324                 td = xfer->td_transfer_last;
1325                 usb_pc_cpu_invalidate(td->page_cache);
1326                 status = hc32toh(sc, td->sitd_status);
1327
1328                 /* also check if first is complete */
1329
1330                 td = xfer->td_transfer_first;
1331                 usb_pc_cpu_invalidate(td->page_cache);
1332                 status |= hc32toh(sc, td->sitd_status);
1333
1334                 if (!(status & EHCI_SITD_ACTIVE)) {
1335                         ehci_device_done(xfer, USB_ERR_NORMAL_COMPLETION);
1336                         goto transferred;
1337                 }
1338         } else if (methods == &ehci_device_isoc_hs_methods) {
1339                 ehci_itd_t *td;
1340
1341                 /* isochronous high speed transfer */
1342
1343                 td = xfer->td_transfer_last;
1344                 usb_pc_cpu_invalidate(td->page_cache);
1345                 status =
1346                     td->itd_status[0] | td->itd_status[1] |
1347                     td->itd_status[2] | td->itd_status[3] |
1348                     td->itd_status[4] | td->itd_status[5] |
1349                     td->itd_status[6] | td->itd_status[7];
1350
1351                 /* also check first transfer */
1352                 td = xfer->td_transfer_first;
1353                 usb_pc_cpu_invalidate(td->page_cache);
1354                 status |=
1355                     td->itd_status[0] | td->itd_status[1] |
1356                     td->itd_status[2] | td->itd_status[3] |
1357                     td->itd_status[4] | td->itd_status[5] |
1358                     td->itd_status[6] | td->itd_status[7];
1359
1360                 /* if no transactions are active we continue */
1361                 if (!(status & htohc32(sc, EHCI_ITD_ACTIVE))) {
1362                         ehci_device_done(xfer, USB_ERR_NORMAL_COMPLETION);
1363                         goto transferred;
1364                 }
1365         } else {
1366                 ehci_qtd_t *td;
1367                 ehci_qh_t *qh;
1368
1369                 /* non-isochronous transfer */
1370
1371                 /*
1372                  * check whether there is an error somewhere in the middle,
1373                  * or whether there was a short packet (SPD and not ACTIVE)
1374                  */
1375                 td = xfer->td_transfer_cache;
1376
1377                 qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
1378
1379                 usb_pc_cpu_invalidate(qh->page_cache);
1380
1381                 status = hc32toh(sc, qh->qh_qtd.qtd_status);
1382                 if (status & EHCI_QTD_ACTIVE) {
1383                         /* transfer is pending */
1384                         goto done;
1385                 }
1386
1387                 while (1) {
1388                         usb_pc_cpu_invalidate(td->page_cache);
1389                         status = hc32toh(sc, td->qtd_status);
1390
1391                         /*
1392                          * Check if there is an active TD which
1393                          * indicates that the transfer isn't done.
1394                          */
1395                         if (status & EHCI_QTD_ACTIVE) {
1396                                 /* update cache */
1397                                 if (xfer->td_transfer_cache != td) {
1398                                         xfer->td_transfer_cache = td;
1399                                         if (qh->qh_qtd.qtd_next &
1400                                             htohc32(sc, EHCI_LINK_TERMINATE)) {
1401                                                 /* XXX - manually advance to next frame */
1402                                                 qh->qh_qtd.qtd_next = td->qtd_self;
1403                                                 usb_pc_cpu_flush(td->page_cache);
1404                                         }
1405                                 }
1406                                 goto done;
1407                         }
1408                         /*
1409                          * last transfer descriptor makes the transfer done
1410                          */
1411                         if (((void *)td) == xfer->td_transfer_last) {
1412                                 break;
1413                         }
1414                         /*
1415                          * any kind of error makes the transfer done
1416                          */
1417                         if (status & EHCI_QTD_HALTED) {
1418                                 break;
1419                         }
1420                         /*
1421                          * if there is no alternate next transfer, a short
1422                          * packet also makes the transfer done
1423                          */
1424                         if (EHCI_QTD_GET_BYTES(status)) {
1425                                 if (xfer->flags_int.short_frames_ok) {
1426                                         /* follow alt next */
1427                                         if (td->alt_next) {
1428                                                 td = td->alt_next;
1429                                                 continue;
1430                                         }
1431                                 }
1432                                 /* transfer is done */
1433                                 break;
1434                         }
1435                         td = td->obj_next;
1436                 }
1437                 ehci_non_isoc_done(xfer);
1438                 goto transferred;
1439         }
1440
1441 done:
1442         DPRINTFN(13, "xfer=%p is still active\n", xfer);
1443         return (0);
1444
1445 transferred:
1446         return (1);
1447 }
1448
1449 static void
1450 ehci_pcd_enable(ehci_softc_t *sc)
1451 {
1452         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
1453
1454         sc->sc_eintrs |= EHCI_STS_PCD;
1455         EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
1456
1457         /* acknowledge any PCD interrupt */
1458         EOWRITE4(sc, EHCI_USBSTS, EHCI_STS_PCD);
1459
1460         ehci_root_intr(sc);
1461 }
1462
1463 static void
1464 ehci_interrupt_poll(ehci_softc_t *sc)
1465 {
1466         struct usb_xfer *xfer;
1467
1468 repeat:
1469         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
1470                 /*
1471                  * check if transfer is transferred
1472                  */
1473                 if (ehci_check_transfer(xfer)) {
1474                         /* queue has been modified */
1475                         goto repeat;
1476                 }
1477         }
1478 }
1479
1480 /*
1481  * Some EHCI chips from VIA / ATI seem to trigger interrupts before
1482  * writing back the qTD status, or miss signalling occasionally under
1483  * heavy load.  If the host machine is too fast, we can miss
1484  * transaction completion - when we scan the active list the
1485  * transaction still seems to be active. This generally exhibits
1486  * itself as a umass stall that never recovers.
1487  *
1488  * We work around this behaviour by setting up this callback after any
1489  * softintr that completes with transactions still pending, giving us
1490  * another chance to check for completion after the writeback has
1491  * taken place.
1492  */
1493 static void
1494 ehci_poll_timeout(void *arg)
1495 {
1496         ehci_softc_t *sc = arg;
1497
1498         DPRINTFN(3, "\n");
1499         ehci_interrupt_poll(sc);
1500 }
1501
1502 /*------------------------------------------------------------------------*
1503  *      ehci_interrupt - EHCI interrupt handler
1504  *
1505  * NOTE: Do not access "sc->sc_bus.bdev" inside the interrupt handler,
1506  * hence the interrupt handler will be setup before "sc->sc_bus.bdev"
1507  * is present !
1508  *------------------------------------------------------------------------*/
1509 void
1510 ehci_interrupt(ehci_softc_t *sc)
1511 {
1512         uint32_t status;
1513
1514         USB_BUS_LOCK(&sc->sc_bus);
1515
1516         DPRINTFN(16, "real interrupt\n");
1517
1518 #if USB_DEBUG
1519         if (ehcidebug > 15) {
1520                 ehci_dump_regs(sc);
1521         }
1522 #endif
1523
1524         status = EHCI_STS_INTRS(EOREAD4(sc, EHCI_USBSTS));
1525         if (status == 0) {
1526                 /* the interrupt was not for us */
1527                 goto done;
1528         }
1529         if (!(status & sc->sc_eintrs)) {
1530                 goto done;
1531         }
1532         EOWRITE4(sc, EHCI_USBSTS, status);      /* acknowledge */
1533
1534         status &= sc->sc_eintrs;
1535
1536         if (status & EHCI_STS_HSE) {
1537                 printf("%s: unrecoverable error, "
1538                     "controller halted\n", __FUNCTION__);
1539 #if USB_DEBUG
1540                 ehci_dump_regs(sc);
1541                 ehci_dump_isoc(sc);
1542 #endif
1543         }
1544         if (status & EHCI_STS_PCD) {
1545                 /*
1546                  * Disable PCD interrupt for now, because it will be
1547                  * on until the port has been reset.
1548                  */
1549                 sc->sc_eintrs &= ~EHCI_STS_PCD;
1550                 EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
1551
1552                 ehci_root_intr(sc);
1553
1554                 /* do not allow RHSC interrupts > 1 per second */
1555                 usb_callout_reset(&sc->sc_tmo_pcd, hz,
1556                     (void *)&ehci_pcd_enable, sc);
1557         }
1558         status &= ~(EHCI_STS_INT | EHCI_STS_ERRINT | EHCI_STS_PCD | EHCI_STS_IAA);
1559
1560         if (status != 0) {
1561                 /* block unprocessed interrupts */
1562                 sc->sc_eintrs &= ~status;
1563                 EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
1564                 printf("%s: blocking interrupts 0x%x\n", __FUNCTION__, status);
1565         }
1566         /* poll all the USB transfers */
1567         ehci_interrupt_poll(sc);
1568
1569         if (sc->sc_flags & EHCI_SCFLG_LOSTINTRBUG) {
1570                 usb_callout_reset(&sc->sc_tmo_poll, hz / 128,
1571                     (void *)&ehci_poll_timeout, sc);
1572         }
1573
1574 done:
1575         USB_BUS_UNLOCK(&sc->sc_bus);
1576 }
1577
1578 /*
1579  * called when a request does not complete
1580  */
1581 static void
1582 ehci_timeout(void *arg)
1583 {
1584         struct usb_xfer *xfer = arg;
1585
1586         DPRINTF("xfer=%p\n", xfer);
1587
1588         USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
1589
1590         /* transfer is transferred */
1591         ehci_device_done(xfer, USB_ERR_TIMEOUT);
1592 }
1593
1594 static void
1595 ehci_do_poll(struct usb_bus *bus)
1596 {
1597         ehci_softc_t *sc = EHCI_BUS2SC(bus);
1598
1599         USB_BUS_LOCK(&sc->sc_bus);
1600         ehci_interrupt_poll(sc);
1601         USB_BUS_UNLOCK(&sc->sc_bus);
1602 }
1603
1604 static void
1605 ehci_setup_standard_chain_sub(struct ehci_std_temp *temp)
1606 {
1607         struct usb_page_search buf_res;
1608         ehci_qtd_t *td;
1609         ehci_qtd_t *td_next;
1610         ehci_qtd_t *td_alt_next;
1611         uint32_t buf_offset;
1612         uint32_t average;
1613         uint32_t len_old;
1614         uint32_t terminate;
1615         uint8_t shortpkt_old;
1616         uint8_t precompute;
1617
1618         terminate = htohc32(temp->sc, EHCI_LINK_TERMINATE);
1619         td_alt_next = NULL;
1620         buf_offset = 0;
1621         shortpkt_old = temp->shortpkt;
1622         len_old = temp->len;
1623         precompute = 1;
1624
1625 restart:
1626
1627         td = temp->td;
1628         td_next = temp->td_next;
1629
1630         while (1) {
1631
1632                 if (temp->len == 0) {
1633
1634                         if (temp->shortpkt) {
1635                                 break;
1636                         }
1637                         /* send a Zero Length Packet, ZLP, last */
1638
1639                         temp->shortpkt = 1;
1640                         average = 0;
1641
1642                 } else {
1643
1644                         average = temp->average;
1645
1646                         if (temp->len < average) {
1647                                 if (temp->len % temp->max_frame_size) {
1648                                         temp->shortpkt = 1;
1649                                 }
1650                                 average = temp->len;
1651                         }
1652                 }
1653
1654                 if (td_next == NULL) {
1655                         panic("%s: out of EHCI transfer descriptors!", __FUNCTION__);
1656                 }
1657                 /* get next TD */
1658
1659                 td = td_next;
1660                 td_next = td->obj_next;
1661
1662                 /* check if we are pre-computing */
1663
1664                 if (precompute) {
1665
1666                         /* update remaining length */
1667
1668                         temp->len -= average;
1669
1670                         continue;
1671                 }
1672                 /* fill out current TD */
1673
1674                 td->qtd_status =
1675                     temp->qtd_status |
1676                     htohc32(temp->sc, EHCI_QTD_IOC |
1677                         EHCI_QTD_SET_BYTES(average));
1678
1679                 if (average == 0) {
1680
1681                         if (temp->auto_data_toggle == 0) {
1682
1683                                 /* update data toggle, ZLP case */
1684
1685                                 temp->qtd_status ^=
1686                                     htohc32(temp->sc, EHCI_QTD_TOGGLE_MASK);
1687                         }
1688                         td->len = 0;
1689
1690                         td->qtd_buffer[0] = 0;
1691                         td->qtd_buffer_hi[0] = 0;
1692
1693                         td->qtd_buffer[1] = 0;
1694                         td->qtd_buffer_hi[1] = 0;
1695
1696                 } else {
1697
1698                         uint8_t x;
1699
1700                         if (temp->auto_data_toggle == 0) {
1701
1702                                 /* update data toggle */
1703
1704                                 if (((average + temp->max_frame_size - 1) /
1705                                     temp->max_frame_size) & 1) {
1706                                         temp->qtd_status ^=
1707                                             htohc32(temp->sc, EHCI_QTD_TOGGLE_MASK);
1708                                 }
1709                         }
1710                         td->len = average;
1711
1712                         /* update remaining length */
1713
1714                         temp->len -= average;
1715
1716                         /* fill out buffer pointers */
1717
1718                         usbd_get_page(temp->pc, buf_offset, &buf_res);
1719                         td->qtd_buffer[0] =
1720                             htohc32(temp->sc, buf_res.physaddr);
1721                         td->qtd_buffer_hi[0] = 0;
1722
1723                         x = 1;
1724
1725                         while (average > EHCI_PAGE_SIZE) {
1726                                 average -= EHCI_PAGE_SIZE;
1727                                 buf_offset += EHCI_PAGE_SIZE;
1728                                 usbd_get_page(temp->pc, buf_offset, &buf_res);
1729                                 td->qtd_buffer[x] =
1730                                     htohc32(temp->sc,
1731                                     buf_res.physaddr & (~0xFFF));
1732                                 td->qtd_buffer_hi[x] = 0;
1733                                 x++;
1734                         }
1735
1736                         /*
1737                          * NOTE: The "average" variable is never zero after
1738                          * exiting the loop above !
1739                          *
1740                          * NOTE: We have to subtract one from the offset to
1741                          * ensure that we are computing the physical address
1742                          * of a valid page !
1743                          */
1744                         buf_offset += average;
1745                         usbd_get_page(temp->pc, buf_offset - 1, &buf_res);
1746                         td->qtd_buffer[x] =
1747                             htohc32(temp->sc,
1748                             buf_res.physaddr & (~0xFFF));
1749                         td->qtd_buffer_hi[x] = 0;
1750                 }
1751
1752                 if (temp->can_use_next) {
1753                         if (td_next) {
1754                                 /* link the current TD with the next one */
1755                                 td->qtd_next = td_next->qtd_self;
1756                         }
1757                 } else {
1758                         /*
1759                          * BUG WARNING: The EHCI HW can use the
1760                          * qtd_next field instead of qtd_altnext when
1761                          * a short packet is received! We work this
1762                          * around in software by not queueing more
1763                          * than one job/TD at a time!
1764                          */
1765                         td->qtd_next = terminate;
1766                 }
1767
1768                 td->qtd_altnext = terminate;
1769                 td->alt_next = td_alt_next;
1770
1771                 usb_pc_cpu_flush(td->page_cache);
1772         }
1773
1774         if (precompute) {
1775                 precompute = 0;
1776
1777                 /* setup alt next pointer, if any */
1778                 if (temp->last_frame) {
1779                         td_alt_next = NULL;
1780                 } else {
1781                         /* we use this field internally */
1782                         td_alt_next = td_next;
1783                 }
1784
1785                 /* restore */
1786                 temp->shortpkt = shortpkt_old;
1787                 temp->len = len_old;
1788                 goto restart;
1789         }
1790         temp->td = td;
1791         temp->td_next = td_next;
1792 }
1793
1794 static void
1795 ehci_setup_standard_chain(struct usb_xfer *xfer, ehci_qh_t **qh_last)
1796 {
1797         struct ehci_std_temp temp;
1798         struct usb_pipe_methods *methods;
1799         ehci_qh_t *qh;
1800         ehci_qtd_t *td;
1801         uint32_t qh_endp;
1802         uint32_t qh_endphub;
1803         uint32_t x;
1804
1805         DPRINTFN(9, "addr=%d endpt=%d sumlen=%d speed=%d\n",
1806             xfer->address, UE_GET_ADDR(xfer->endpointno),
1807             xfer->sumlen, usbd_get_speed(xfer->xroot->udev));
1808
1809         temp.average = xfer->max_hc_frame_size;
1810         temp.max_frame_size = xfer->max_frame_size;
1811         temp.sc = EHCI_BUS2SC(xfer->xroot->bus);
1812
1813         /* toggle the DMA set we are using */
1814         xfer->flags_int.curr_dma_set ^= 1;
1815
1816         /* get next DMA set */
1817         td = xfer->td_start[xfer->flags_int.curr_dma_set];
1818
1819         xfer->td_transfer_first = td;
1820         xfer->td_transfer_cache = td;
1821
1822         temp.td = NULL;
1823         temp.td_next = td;
1824         temp.qtd_status = 0;
1825         temp.last_frame = 0;
1826         temp.setup_alt_next = xfer->flags_int.short_frames_ok;
1827         temp.can_use_next = (xfer->flags_int.control_xfr ||
1828             (UE_GET_DIR(xfer->endpointno) == UE_DIR_OUT));
1829
1830         if (xfer->flags_int.control_xfr) {
1831                 if (xfer->endpoint->toggle_next) {
1832                         /* DATA1 is next */
1833                         temp.qtd_status |=
1834                             htohc32(temp.sc, EHCI_QTD_SET_TOGGLE(1));
1835                 }
1836                 temp.auto_data_toggle = 0;
1837         } else {
1838                 temp.auto_data_toggle = 1;
1839         }
1840
1841         if (usbd_get_speed(xfer->xroot->udev) != USB_SPEED_HIGH) {
1842                 /* max 3 retries */
1843                 temp.qtd_status |=
1844                     htohc32(temp.sc, EHCI_QTD_SET_CERR(3));
1845         }
1846         /* check if we should prepend a setup message */
1847
1848         if (xfer->flags_int.control_xfr) {
1849                 if (xfer->flags_int.control_hdr) {
1850
1851                         temp.qtd_status &=
1852                             htohc32(temp.sc, EHCI_QTD_SET_CERR(3));
1853                         temp.qtd_status |= htohc32(temp.sc,
1854                             EHCI_QTD_ACTIVE |
1855                             EHCI_QTD_SET_PID(EHCI_QTD_PID_SETUP) |
1856                             EHCI_QTD_SET_TOGGLE(0));
1857
1858                         temp.len = xfer->frlengths[0];
1859                         temp.pc = xfer->frbuffers + 0;
1860                         temp.shortpkt = temp.len ? 1 : 0;
1861                         /* check for last frame */
1862                         if (xfer->nframes == 1) {
1863                                 /* no STATUS stage yet, SETUP is last */
1864                                 if (xfer->flags_int.control_act) {
1865                                         temp.last_frame = 1;
1866                                         temp.setup_alt_next = 0;
1867                                 }
1868                         }
1869                         ehci_setup_standard_chain_sub(&temp);
1870                 }
1871                 x = 1;
1872         } else {
1873                 x = 0;
1874         }
1875
1876         while (x != xfer->nframes) {
1877
1878                 /* DATA0 / DATA1 message */
1879
1880                 temp.len = xfer->frlengths[x];
1881                 temp.pc = xfer->frbuffers + x;
1882
1883                 x++;
1884
1885                 if (x == xfer->nframes) {
1886                         if (xfer->flags_int.control_xfr) {
1887                                 /* no STATUS stage yet, DATA is last */
1888                                 if (xfer->flags_int.control_act) {
1889                                         temp.last_frame = 1;
1890                                         temp.setup_alt_next = 0;
1891                                 }
1892                         } else {
1893                                 temp.last_frame = 1;
1894                                 temp.setup_alt_next = 0;
1895                         }
1896                 }
1897                 /* keep previous data toggle and error count */
1898
1899                 temp.qtd_status &=
1900                     htohc32(temp.sc, EHCI_QTD_SET_CERR(3) |
1901                     EHCI_QTD_SET_TOGGLE(1));
1902
1903                 if (temp.len == 0) {
1904
1905                         /* make sure that we send an USB packet */
1906
1907                         temp.shortpkt = 0;
1908
1909                 } else {
1910
1911                         /* regular data transfer */
1912
1913                         temp.shortpkt = (xfer->flags.force_short_xfer) ? 0 : 1;
1914                 }
1915
1916                 /* set endpoint direction */
1917
1918                 temp.qtd_status |=
1919                     (UE_GET_DIR(xfer->endpointno) == UE_DIR_IN) ?
1920                     htohc32(temp.sc, EHCI_QTD_ACTIVE |
1921                     EHCI_QTD_SET_PID(EHCI_QTD_PID_IN)) :
1922                     htohc32(temp.sc, EHCI_QTD_ACTIVE |
1923                     EHCI_QTD_SET_PID(EHCI_QTD_PID_OUT));
1924
1925                 ehci_setup_standard_chain_sub(&temp);
1926         }
1927
1928         /* check if we should append a status stage */
1929
1930         if (xfer->flags_int.control_xfr &&
1931             !xfer->flags_int.control_act) {
1932
1933                 /*
1934                  * Send a DATA1 message and invert the current endpoint
1935                  * direction.
1936                  */
1937
1938                 temp.qtd_status &= htohc32(temp.sc, EHCI_QTD_SET_CERR(3) |
1939                     EHCI_QTD_SET_TOGGLE(1));
1940                 temp.qtd_status |=
1941                     (UE_GET_DIR(xfer->endpointno) == UE_DIR_OUT) ?
1942                     htohc32(temp.sc, EHCI_QTD_ACTIVE |
1943                     EHCI_QTD_SET_PID(EHCI_QTD_PID_IN) |
1944                     EHCI_QTD_SET_TOGGLE(1)) :
1945                     htohc32(temp.sc, EHCI_QTD_ACTIVE |
1946                     EHCI_QTD_SET_PID(EHCI_QTD_PID_OUT) |
1947                     EHCI_QTD_SET_TOGGLE(1));
1948
1949                 temp.len = 0;
1950                 temp.pc = NULL;
1951                 temp.shortpkt = 0;
1952                 temp.last_frame = 1;
1953                 temp.setup_alt_next = 0;
1954
1955                 ehci_setup_standard_chain_sub(&temp);
1956         }
1957         td = temp.td;
1958
1959         /* the last TD terminates the transfer: */
1960         td->qtd_next = htohc32(temp.sc, EHCI_LINK_TERMINATE);
1961         td->qtd_altnext = htohc32(temp.sc, EHCI_LINK_TERMINATE);
1962
1963         usb_pc_cpu_flush(td->page_cache);
1964
1965         /* must have at least one frame! */
1966
1967         xfer->td_transfer_last = td;
1968
1969 #if USB_DEBUG
1970         if (ehcidebug > 8) {
1971                 DPRINTF("nexttog=%d; data before transfer:\n",
1972                     xfer->endpoint->toggle_next);
1973                 ehci_dump_sqtds(temp.sc,
1974                     xfer->td_transfer_first);
1975         }
1976 #endif
1977
1978         methods = xfer->endpoint->methods;
1979
1980         qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
1981
1982         /* the "qh_link" field is filled when the QH is added */
1983
1984         qh_endp =
1985             (EHCI_QH_SET_ADDR(xfer->address) |
1986             EHCI_QH_SET_ENDPT(UE_GET_ADDR(xfer->endpointno)) |
1987             EHCI_QH_SET_MPL(xfer->max_packet_size));
1988
1989         if (usbd_get_speed(xfer->xroot->udev) == USB_SPEED_HIGH) {
1990                 qh_endp |= EHCI_QH_SET_EPS(EHCI_QH_SPEED_HIGH);
1991                 if (methods != &ehci_device_intr_methods)
1992                         qh_endp |= EHCI_QH_SET_NRL(8);
1993         } else {
1994
1995                 if (usbd_get_speed(xfer->xroot->udev) == USB_SPEED_FULL) {
1996                         qh_endp |= EHCI_QH_SET_EPS(EHCI_QH_SPEED_FULL);
1997                 } else {
1998                         qh_endp |= EHCI_QH_SET_EPS(EHCI_QH_SPEED_LOW);
1999                 }
2000
2001                 if (methods == &ehci_device_ctrl_methods) {
2002                         qh_endp |= EHCI_QH_CTL;
2003                 }
2004                 if (methods != &ehci_device_intr_methods) {
2005                         /* Only try one time per microframe! */
2006                         qh_endp |= EHCI_QH_SET_NRL(1);
2007                 }
2008         }
2009
2010         if (temp.auto_data_toggle == 0) {
2011                 /* software computes the data toggle */
2012                 qh_endp |= EHCI_QH_DTC;
2013         }
2014
2015         qh->qh_endp = htohc32(temp.sc, qh_endp);
2016
2017         qh_endphub =
2018             (EHCI_QH_SET_MULT(xfer->max_packet_count & 3) |
2019             EHCI_QH_SET_CMASK(xfer->usb_cmask) |
2020             EHCI_QH_SET_SMASK(xfer->usb_smask) |
2021             EHCI_QH_SET_HUBA(xfer->xroot->udev->hs_hub_addr) |
2022             EHCI_QH_SET_PORT(xfer->xroot->udev->hs_port_no));
2023
2024         qh->qh_endphub = htohc32(temp.sc, qh_endphub);
2025         qh->qh_curqtd = 0;
2026
2027         /* fill the overlay qTD */
2028
2029         if (temp.auto_data_toggle && xfer->endpoint->toggle_next) {
2030                 /* DATA1 is next */
2031                 qh->qh_qtd.qtd_status = htohc32(temp.sc, EHCI_QTD_SET_TOGGLE(1));
2032         } else {
2033                 qh->qh_qtd.qtd_status = 0;
2034         }
2035
2036         td = xfer->td_transfer_first;
2037
2038         qh->qh_qtd.qtd_next = td->qtd_self;
2039         qh->qh_qtd.qtd_altnext =
2040             htohc32(temp.sc, EHCI_LINK_TERMINATE);
2041
2042         usb_pc_cpu_flush(qh->page_cache);
2043
2044         if (xfer->xroot->udev->flags.self_suspended == 0) {
2045                 EHCI_APPEND_QH(qh, *qh_last);
2046         }
2047 }
2048
2049 static void
2050 ehci_root_intr(ehci_softc_t *sc)
2051 {
2052         uint16_t i;
2053         uint16_t m;
2054
2055         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
2056
2057         /* clear any old interrupt data */
2058         memset(sc->sc_hub_idata, 0, sizeof(sc->sc_hub_idata));
2059
2060         /* set bits */
2061         m = (sc->sc_noport + 1);
2062         if (m > (8 * sizeof(sc->sc_hub_idata))) {
2063                 m = (8 * sizeof(sc->sc_hub_idata));
2064         }
2065         for (i = 1; i < m; i++) {
2066                 /* pick out CHANGE bits from the status register */
2067                 if (EOREAD4(sc, EHCI_PORTSC(i)) & EHCI_PS_CLEAR) {
2068                         sc->sc_hub_idata[i / 8] |= 1 << (i % 8);
2069                         DPRINTF("port %d changed\n", i);
2070                 }
2071         }
2072         uhub_root_intr(&sc->sc_bus, sc->sc_hub_idata,
2073             sizeof(sc->sc_hub_idata));
2074 }
2075
2076 static void
2077 ehci_isoc_fs_done(ehci_softc_t *sc, struct usb_xfer *xfer)
2078 {
2079         uint32_t nframes = xfer->nframes;
2080         uint32_t status;
2081         uint32_t *plen = xfer->frlengths;
2082         uint16_t len = 0;
2083         ehci_sitd_t *td = xfer->td_transfer_first;
2084         ehci_sitd_t **pp_last = &sc->sc_isoc_fs_p_last[xfer->qh_pos];
2085
2086         DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
2087             xfer, xfer->endpoint);
2088
2089         while (nframes--) {
2090                 if (td == NULL) {
2091                         panic("%s:%d: out of TD's\n",
2092                             __FUNCTION__, __LINE__);
2093                 }
2094                 if (pp_last >= &sc->sc_isoc_fs_p_last[EHCI_VIRTUAL_FRAMELIST_COUNT]) {
2095                         pp_last = &sc->sc_isoc_fs_p_last[0];
2096                 }
2097 #if USB_DEBUG
2098                 if (ehcidebug > 15) {
2099                         DPRINTF("isoc FS-TD\n");
2100                         ehci_dump_sitd(sc, td);
2101                 }
2102 #endif
2103                 usb_pc_cpu_invalidate(td->page_cache);
2104                 status = hc32toh(sc, td->sitd_status);
2105
2106                 len = EHCI_SITD_GET_LEN(status);
2107
2108                 DPRINTFN(2, "status=0x%08x, rem=%u\n", status, len);
2109
2110                 if (*plen >= len) {
2111                         len = *plen - len;
2112                 } else {
2113                         len = 0;
2114                 }
2115
2116                 *plen = len;
2117
2118                 /* remove FS-TD from schedule */
2119                 EHCI_REMOVE_FS_TD(td, *pp_last);
2120
2121                 pp_last++;
2122                 plen++;
2123                 td = td->obj_next;
2124         }
2125
2126         xfer->aframes = xfer->nframes;
2127 }
2128
2129 static void
2130 ehci_isoc_hs_done(ehci_softc_t *sc, struct usb_xfer *xfer)
2131 {
2132         uint32_t nframes = xfer->nframes;
2133         uint32_t status;
2134         uint32_t *plen = xfer->frlengths;
2135         uint16_t len = 0;
2136         uint8_t td_no = 0;
2137         ehci_itd_t *td = xfer->td_transfer_first;
2138         ehci_itd_t **pp_last = &sc->sc_isoc_hs_p_last[xfer->qh_pos];
2139
2140         DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
2141             xfer, xfer->endpoint);
2142
2143         while (nframes--) {
2144                 if (td == NULL) {
2145                         panic("%s:%d: out of TD's\n",
2146                             __FUNCTION__, __LINE__);
2147                 }
2148                 if (pp_last >= &sc->sc_isoc_hs_p_last[EHCI_VIRTUAL_FRAMELIST_COUNT]) {
2149                         pp_last = &sc->sc_isoc_hs_p_last[0];
2150                 }
2151 #if USB_DEBUG
2152                 if (ehcidebug > 15) {
2153                         DPRINTF("isoc HS-TD\n");
2154                         ehci_dump_itd(sc, td);
2155                 }
2156 #endif
2157
2158                 usb_pc_cpu_invalidate(td->page_cache);
2159                 status = hc32toh(sc, td->itd_status[td_no]);
2160
2161                 len = EHCI_ITD_GET_LEN(status);
2162
2163                 DPRINTFN(2, "status=0x%08x, len=%u\n", status, len);
2164
2165                 if (*plen >= len) {
2166                         /*
2167                          * The length is valid. NOTE: The complete
2168                          * length is written back into the status
2169                          * field, and not the remainder like with
2170                          * other transfer descriptor types.
2171                          */
2172                 } else {
2173                         /* Invalid length - truncate */
2174                         len = 0;
2175                 }
2176
2177                 *plen = len;
2178
2179                 plen++;
2180                 td_no++;
2181
2182                 if ((td_no == 8) || (nframes == 0)) {
2183                         /* remove HS-TD from schedule */
2184                         EHCI_REMOVE_HS_TD(td, *pp_last);
2185                         pp_last++;
2186
2187                         td_no = 0;
2188                         td = td->obj_next;
2189                 }
2190         }
2191         xfer->aframes = xfer->nframes;
2192 }
2193
2194 /* NOTE: "done" can be run two times in a row,
2195  * from close and from interrupt
2196  */
2197 static void
2198 ehci_device_done(struct usb_xfer *xfer, usb_error_t error)
2199 {
2200         struct usb_pipe_methods *methods = xfer->endpoint->methods;
2201         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2202
2203         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
2204
2205         DPRINTFN(2, "xfer=%p, endpoint=%p, error=%d\n",
2206             xfer, xfer->endpoint, error);
2207
2208         if ((methods == &ehci_device_bulk_methods) ||
2209             (methods == &ehci_device_ctrl_methods)) {
2210 #if USB_DEBUG
2211                 if (ehcidebug > 8) {
2212                         DPRINTF("nexttog=%d; data after transfer:\n",
2213                             xfer->endpoint->toggle_next);
2214                         ehci_dump_sqtds(sc,
2215                             xfer->td_transfer_first);
2216                 }
2217 #endif
2218
2219                 EHCI_REMOVE_QH(xfer->qh_start[xfer->flags_int.curr_dma_set],
2220                     sc->sc_async_p_last);
2221         }
2222         if (methods == &ehci_device_intr_methods) {
2223                 EHCI_REMOVE_QH(xfer->qh_start[xfer->flags_int.curr_dma_set],
2224                     sc->sc_intr_p_last[xfer->qh_pos]);
2225         }
2226         /*
2227          * Only finish isochronous transfers once which will update
2228          * "xfer->frlengths".
2229          */
2230         if (xfer->td_transfer_first &&
2231             xfer->td_transfer_last) {
2232                 if (methods == &ehci_device_isoc_fs_methods) {
2233                         ehci_isoc_fs_done(sc, xfer);
2234                 }
2235                 if (methods == &ehci_device_isoc_hs_methods) {
2236                         ehci_isoc_hs_done(sc, xfer);
2237                 }
2238                 xfer->td_transfer_first = NULL;
2239                 xfer->td_transfer_last = NULL;
2240         }
2241         /* dequeue transfer and start next transfer */
2242         usbd_transfer_done(xfer, error);
2243 }
2244
2245 /*------------------------------------------------------------------------*
2246  * ehci bulk support
2247  *------------------------------------------------------------------------*/
2248 static void
2249 ehci_device_bulk_open(struct usb_xfer *xfer)
2250 {
2251         return;
2252 }
2253
2254 static void
2255 ehci_device_bulk_close(struct usb_xfer *xfer)
2256 {
2257         ehci_device_done(xfer, USB_ERR_CANCELLED);
2258 }
2259
2260 static void
2261 ehci_device_bulk_enter(struct usb_xfer *xfer)
2262 {
2263         return;
2264 }
2265
2266 static void
2267 ehci_device_bulk_start(struct usb_xfer *xfer)
2268 {
2269         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2270         uint32_t temp;
2271
2272         /* setup TD's and QH */
2273         ehci_setup_standard_chain(xfer, &sc->sc_async_p_last);
2274
2275         /* put transfer on interrupt queue */
2276         ehci_transfer_intr_enqueue(xfer);
2277
2278         /* XXX Performance quirk: Some Host Controllers have a too low
2279          * interrupt rate. Issue an IAAD to stimulate the Host
2280          * Controller after queueing the BULK transfer.
2281          */
2282         temp = EOREAD4(sc, EHCI_USBCMD);
2283         if (!(temp & EHCI_CMD_IAAD))
2284                 EOWRITE4(sc, EHCI_USBCMD, temp | EHCI_CMD_IAAD);
2285 }
2286
2287 struct usb_pipe_methods ehci_device_bulk_methods =
2288 {
2289         .open = ehci_device_bulk_open,
2290         .close = ehci_device_bulk_close,
2291         .enter = ehci_device_bulk_enter,
2292         .start = ehci_device_bulk_start,
2293 };
2294
2295 /*------------------------------------------------------------------------*
2296  * ehci control support
2297  *------------------------------------------------------------------------*/
2298 static void
2299 ehci_device_ctrl_open(struct usb_xfer *xfer)
2300 {
2301         return;
2302 }
2303
2304 static void
2305 ehci_device_ctrl_close(struct usb_xfer *xfer)
2306 {
2307         ehci_device_done(xfer, USB_ERR_CANCELLED);
2308 }
2309
2310 static void
2311 ehci_device_ctrl_enter(struct usb_xfer *xfer)
2312 {
2313         return;
2314 }
2315
2316 static void
2317 ehci_device_ctrl_start(struct usb_xfer *xfer)
2318 {
2319         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2320
2321         /* setup TD's and QH */
2322         ehci_setup_standard_chain(xfer, &sc->sc_async_p_last);
2323
2324         /* put transfer on interrupt queue */
2325         ehci_transfer_intr_enqueue(xfer);
2326 }
2327
2328 struct usb_pipe_methods ehci_device_ctrl_methods =
2329 {
2330         .open = ehci_device_ctrl_open,
2331         .close = ehci_device_ctrl_close,
2332         .enter = ehci_device_ctrl_enter,
2333         .start = ehci_device_ctrl_start,
2334 };
2335
2336 /*------------------------------------------------------------------------*
2337  * ehci interrupt support
2338  *------------------------------------------------------------------------*/
2339 static void
2340 ehci_device_intr_open(struct usb_xfer *xfer)
2341 {
2342         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2343         uint16_t best;
2344         uint16_t bit;
2345         uint16_t x;
2346         uint8_t slot;
2347
2348         /* Allocate a microframe slot first: */
2349
2350         slot = usb_intr_schedule_adjust
2351             (xfer->xroot->udev, xfer->max_frame_size, USB_HS_MICRO_FRAMES_MAX);
2352
2353         if (usbd_get_speed(xfer->xroot->udev) == USB_SPEED_HIGH) {
2354                 xfer->usb_uframe = slot;
2355                 xfer->usb_smask = (1 << slot) & 0xFF;
2356                 xfer->usb_cmask = 0;
2357         } else {
2358                 xfer->usb_uframe = slot;
2359                 xfer->usb_smask = (1 << slot) & 0x3F;
2360                 xfer->usb_cmask = (-(4 << slot)) & 0xFE;
2361         }
2362
2363         /*
2364          * Find the best QH position corresponding to the given interval:
2365          */
2366
2367         best = 0;
2368         bit = EHCI_VIRTUAL_FRAMELIST_COUNT / 2;
2369         while (bit) {
2370                 if (xfer->interval >= bit) {
2371                         x = bit;
2372                         best = bit;
2373                         while (x & bit) {
2374                                 if (sc->sc_intr_stat[x] <
2375                                     sc->sc_intr_stat[best]) {
2376                                         best = x;
2377                                 }
2378                                 x++;
2379                         }
2380                         break;
2381                 }
2382                 bit >>= 1;
2383         }
2384
2385         sc->sc_intr_stat[best]++;
2386         xfer->qh_pos = best;
2387
2388         DPRINTFN(3, "best=%d interval=%d\n",
2389             best, xfer->interval);
2390 }
2391
2392 static void
2393 ehci_device_intr_close(struct usb_xfer *xfer)
2394 {
2395         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2396         uint8_t slot;
2397
2398         slot = usb_intr_schedule_adjust
2399             (xfer->xroot->udev, -(xfer->max_frame_size), xfer->usb_uframe);
2400
2401         sc->sc_intr_stat[xfer->qh_pos]--;
2402
2403         ehci_device_done(xfer, USB_ERR_CANCELLED);
2404 }
2405
2406 static void
2407 ehci_device_intr_enter(struct usb_xfer *xfer)
2408 {
2409         return;
2410 }
2411
2412 static void
2413 ehci_device_intr_start(struct usb_xfer *xfer)
2414 {
2415         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2416
2417         /* setup TD's and QH */
2418         ehci_setup_standard_chain(xfer, &sc->sc_intr_p_last[xfer->qh_pos]);
2419
2420         /* put transfer on interrupt queue */
2421         ehci_transfer_intr_enqueue(xfer);
2422 }
2423
2424 struct usb_pipe_methods ehci_device_intr_methods =
2425 {
2426         .open = ehci_device_intr_open,
2427         .close = ehci_device_intr_close,
2428         .enter = ehci_device_intr_enter,
2429         .start = ehci_device_intr_start,
2430 };
2431
2432 /*------------------------------------------------------------------------*
2433  * ehci full speed isochronous support
2434  *------------------------------------------------------------------------*/
2435 static void
2436 ehci_device_isoc_fs_open(struct usb_xfer *xfer)
2437 {
2438         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2439         ehci_sitd_t *td;
2440         uint32_t sitd_portaddr;
2441         uint8_t ds;
2442
2443         sitd_portaddr =
2444             EHCI_SITD_SET_ADDR(xfer->address) |
2445             EHCI_SITD_SET_ENDPT(UE_GET_ADDR(xfer->endpointno)) |
2446             EHCI_SITD_SET_HUBA(xfer->xroot->udev->hs_hub_addr) |
2447             EHCI_SITD_SET_PORT(xfer->xroot->udev->hs_port_no);
2448
2449         if (UE_GET_DIR(xfer->endpointno) == UE_DIR_IN) {
2450                 sitd_portaddr |= EHCI_SITD_SET_DIR_IN;
2451         }
2452         sitd_portaddr = htohc32(sc, sitd_portaddr);
2453
2454         /* initialize all TD's */
2455
2456         for (ds = 0; ds != 2; ds++) {
2457
2458                 for (td = xfer->td_start[ds]; td; td = td->obj_next) {
2459
2460                         td->sitd_portaddr = sitd_portaddr;
2461
2462                         /*
2463                          * TODO: make some kind of automatic
2464                          * SMASK/CMASK selection based on micro-frame
2465                          * usage
2466                          *
2467                          * micro-frame usage (8 microframes per 1ms)
2468                          */
2469                         td->sitd_back = htohc32(sc, EHCI_LINK_TERMINATE);
2470
2471                         usb_pc_cpu_flush(td->page_cache);
2472                 }
2473         }
2474 }
2475
2476 static void
2477 ehci_device_isoc_fs_close(struct usb_xfer *xfer)
2478 {
2479         ehci_device_done(xfer, USB_ERR_CANCELLED);
2480 }
2481
2482 static void
2483 ehci_device_isoc_fs_enter(struct usb_xfer *xfer)
2484 {
2485         struct usb_page_search buf_res;
2486         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2487         struct usb_fs_isoc_schedule *fss_start;
2488         struct usb_fs_isoc_schedule *fss_end;
2489         struct usb_fs_isoc_schedule *fss;
2490         ehci_sitd_t *td;
2491         ehci_sitd_t *td_last = NULL;
2492         ehci_sitd_t **pp_last;
2493         uint32_t *plen;
2494         uint32_t buf_offset;
2495         uint32_t nframes;
2496         uint32_t temp;
2497         uint32_t sitd_mask;
2498         uint16_t tlen;
2499         uint8_t sa;
2500         uint8_t sb;
2501         uint8_t error;
2502
2503 #if USB_DEBUG
2504         uint8_t once = 1;
2505
2506 #endif
2507
2508         DPRINTFN(6, "xfer=%p next=%d nframes=%d\n",
2509             xfer, xfer->endpoint->isoc_next, xfer->nframes);
2510
2511         /* get the current frame index */
2512
2513         nframes = EOREAD4(sc, EHCI_FRINDEX) / 8;
2514
2515         /*
2516          * check if the frame index is within the window where the frames
2517          * will be inserted
2518          */
2519         buf_offset = (nframes - xfer->endpoint->isoc_next) &
2520             (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2521
2522         if ((xfer->endpoint->is_synced == 0) ||
2523             (buf_offset < xfer->nframes)) {
2524                 /*
2525                  * If there is data underflow or the pipe queue is empty we
2526                  * schedule the transfer a few frames ahead of the current
2527                  * frame position. Else two isochronous transfers might
2528                  * overlap.
2529                  */
2530                 xfer->endpoint->isoc_next = (nframes + 3) &
2531                     (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2532                 xfer->endpoint->is_synced = 1;
2533                 DPRINTFN(3, "start next=%d\n", xfer->endpoint->isoc_next);
2534         }
2535         /*
2536          * compute how many milliseconds the insertion is ahead of the
2537          * current frame position:
2538          */
2539         buf_offset = (xfer->endpoint->isoc_next - nframes) &
2540             (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2541
2542         /*
2543          * pre-compute when the isochronous transfer will be finished:
2544          */
2545         xfer->isoc_time_complete =
2546             usbd_fs_isoc_schedule_isoc_time_expand
2547             (xfer->xroot->udev, &fss_start, &fss_end, nframes) + buf_offset +
2548             xfer->nframes;
2549
2550         /* get the real number of frames */
2551
2552         nframes = xfer->nframes;
2553
2554         buf_offset = 0;
2555
2556         plen = xfer->frlengths;
2557
2558         /* toggle the DMA set we are using */
2559         xfer->flags_int.curr_dma_set ^= 1;
2560
2561         /* get next DMA set */
2562         td = xfer->td_start[xfer->flags_int.curr_dma_set];
2563         xfer->td_transfer_first = td;
2564
2565         pp_last = &sc->sc_isoc_fs_p_last[xfer->endpoint->isoc_next];
2566
2567         /* store starting position */
2568
2569         xfer->qh_pos = xfer->endpoint->isoc_next;
2570
2571         fss = fss_start + (xfer->qh_pos % USB_ISOC_TIME_MAX);
2572
2573         while (nframes--) {
2574                 if (td == NULL) {
2575                         panic("%s:%d: out of TD's\n",
2576                             __FUNCTION__, __LINE__);
2577                 }
2578                 if (pp_last >= &sc->sc_isoc_fs_p_last[EHCI_VIRTUAL_FRAMELIST_COUNT]) {
2579                         pp_last = &sc->sc_isoc_fs_p_last[0];
2580                 }
2581                 if (fss >= fss_end) {
2582                         fss = fss_start;
2583                 }
2584                 /* reuse sitd_portaddr and sitd_back from last transfer */
2585
2586                 if (*plen > xfer->max_frame_size) {
2587 #if USB_DEBUG
2588                         if (once) {
2589                                 once = 0;
2590                                 printf("%s: frame length(%d) exceeds %d "
2591                                     "bytes (frame truncated)\n",
2592                                     __FUNCTION__, *plen,
2593                                     xfer->max_frame_size);
2594                         }
2595 #endif
2596                         *plen = xfer->max_frame_size;
2597                 }
2598                 /*
2599                  * We currently don't care if the ISOCHRONOUS schedule is
2600                  * full!
2601                  */
2602                 error = usbd_fs_isoc_schedule_alloc(fss, &sa, *plen);
2603                 if (error) {
2604                         /*
2605                          * The FULL speed schedule is FULL! Set length
2606                          * to zero.
2607                          */
2608                         *plen = 0;
2609                 }
2610                 if (*plen) {
2611                         /*
2612                          * only call "usbd_get_page()" when we have a
2613                          * non-zero length
2614                          */
2615                         usbd_get_page(xfer->frbuffers, buf_offset, &buf_res);
2616                         td->sitd_bp[0] = htohc32(sc, buf_res.physaddr);
2617                         buf_offset += *plen;
2618                         /*
2619                          * NOTE: We need to subtract one from the offset so
2620                          * that we are on a valid page!
2621                          */
2622                         usbd_get_page(xfer->frbuffers, buf_offset - 1,
2623                             &buf_res);
2624                         temp = buf_res.physaddr & ~0xFFF;
2625                 } else {
2626                         td->sitd_bp[0] = 0;
2627                         temp = 0;
2628                 }
2629
2630                 if (UE_GET_DIR(xfer->endpointno) == UE_DIR_OUT) {
2631                         tlen = *plen;
2632                         if (tlen <= 188) {
2633                                 temp |= 1;      /* T-count = 1, TP = ALL */
2634                                 tlen = 1;
2635                         } else {
2636                                 tlen += 187;
2637                                 tlen /= 188;
2638                                 temp |= tlen;   /* T-count = [1..6] */
2639                                 temp |= 8;      /* TP = Begin */
2640                         }
2641
2642                         tlen += sa;
2643
2644                         if (tlen >= 8) {
2645                                 sb = 0;
2646                         } else {
2647                                 sb = (1 << tlen);
2648                         }
2649
2650                         sa = (1 << sa);
2651                         sa = (sb - sa) & 0x3F;
2652                         sb = 0;
2653                 } else {
2654                         sb = (-(4 << sa)) & 0xFE;
2655                         sa = (1 << sa) & 0x3F;
2656                 }
2657
2658                 sitd_mask = (EHCI_SITD_SET_SMASK(sa) |
2659                     EHCI_SITD_SET_CMASK(sb));
2660
2661                 td->sitd_bp[1] = htohc32(sc, temp);
2662
2663                 td->sitd_mask = htohc32(sc, sitd_mask);
2664
2665                 if (nframes == 0) {
2666                         td->sitd_status = htohc32(sc,
2667                             EHCI_SITD_IOC |
2668                             EHCI_SITD_ACTIVE |
2669                             EHCI_SITD_SET_LEN(*plen));
2670                 } else {
2671                         td->sitd_status = htohc32(sc,
2672                             EHCI_SITD_ACTIVE |
2673                             EHCI_SITD_SET_LEN(*plen));
2674                 }
2675                 usb_pc_cpu_flush(td->page_cache);
2676
2677 #if USB_DEBUG
2678                 if (ehcidebug > 15) {
2679                         DPRINTF("FS-TD %d\n", nframes);
2680                         ehci_dump_sitd(sc, td);
2681                 }
2682 #endif
2683                 /* insert TD into schedule */
2684                 EHCI_APPEND_FS_TD(td, *pp_last);
2685                 pp_last++;
2686
2687                 plen++;
2688                 fss++;
2689                 td_last = td;
2690                 td = td->obj_next;
2691         }
2692
2693         xfer->td_transfer_last = td_last;
2694
2695         /* update isoc_next */
2696         xfer->endpoint->isoc_next = (pp_last - &sc->sc_isoc_fs_p_last[0]) &
2697             (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2698 }
2699
2700 static void
2701 ehci_device_isoc_fs_start(struct usb_xfer *xfer)
2702 {
2703         /* put transfer on interrupt queue */
2704         ehci_transfer_intr_enqueue(xfer);
2705 }
2706
2707 struct usb_pipe_methods ehci_device_isoc_fs_methods =
2708 {
2709         .open = ehci_device_isoc_fs_open,
2710         .close = ehci_device_isoc_fs_close,
2711         .enter = ehci_device_isoc_fs_enter,
2712         .start = ehci_device_isoc_fs_start,
2713 };
2714
2715 /*------------------------------------------------------------------------*
2716  * ehci high speed isochronous support
2717  *------------------------------------------------------------------------*/
2718 static void
2719 ehci_device_isoc_hs_open(struct usb_xfer *xfer)
2720 {
2721         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2722         ehci_itd_t *td;
2723         uint32_t temp;
2724         uint8_t ds;
2725
2726         /* initialize all TD's */
2727
2728         for (ds = 0; ds != 2; ds++) {
2729
2730                 for (td = xfer->td_start[ds]; td; td = td->obj_next) {
2731
2732                         /* set TD inactive */
2733                         td->itd_status[0] = 0;
2734                         td->itd_status[1] = 0;
2735                         td->itd_status[2] = 0;
2736                         td->itd_status[3] = 0;
2737                         td->itd_status[4] = 0;
2738                         td->itd_status[5] = 0;
2739                         td->itd_status[6] = 0;
2740                         td->itd_status[7] = 0;
2741
2742                         /* set endpoint and address */
2743                         td->itd_bp[0] = htohc32(sc,
2744                             EHCI_ITD_SET_ADDR(xfer->address) |
2745                             EHCI_ITD_SET_ENDPT(UE_GET_ADDR(xfer->endpointno)));
2746
2747                         temp =
2748                             EHCI_ITD_SET_MPL(xfer->max_packet_size & 0x7FF);
2749
2750                         /* set direction */
2751                         if (UE_GET_DIR(xfer->endpointno) == UE_DIR_IN) {
2752                                 temp |= EHCI_ITD_SET_DIR_IN;
2753                         }
2754                         /* set maximum packet size */
2755                         td->itd_bp[1] = htohc32(sc, temp);
2756
2757                         /* set transfer multiplier */
2758                         td->itd_bp[2] = htohc32(sc, xfer->max_packet_count & 3);
2759
2760                         usb_pc_cpu_flush(td->page_cache);
2761                 }
2762         }
2763 }
2764
2765 static void
2766 ehci_device_isoc_hs_close(struct usb_xfer *xfer)
2767 {
2768         ehci_device_done(xfer, USB_ERR_CANCELLED);
2769 }
2770
2771 static void
2772 ehci_device_isoc_hs_enter(struct usb_xfer *xfer)
2773 {
2774         struct usb_page_search buf_res;
2775         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2776         ehci_itd_t *td;
2777         ehci_itd_t *td_last = NULL;
2778         ehci_itd_t **pp_last;
2779         bus_size_t page_addr;
2780         uint32_t *plen;
2781         uint32_t status;
2782         uint32_t buf_offset;
2783         uint32_t nframes;
2784         uint32_t itd_offset[8 + 1];
2785         uint8_t x;
2786         uint8_t td_no;
2787         uint8_t page_no;
2788
2789 #if USB_DEBUG
2790         uint8_t once = 1;
2791
2792 #endif
2793
2794         DPRINTFN(6, "xfer=%p next=%d nframes=%d\n",
2795             xfer, xfer->endpoint->isoc_next, xfer->nframes);
2796
2797         /* get the current frame index */
2798
2799         nframes = EOREAD4(sc, EHCI_FRINDEX) / 8;
2800
2801         /*
2802          * check if the frame index is within the window where the frames
2803          * will be inserted
2804          */
2805         buf_offset = (nframes - xfer->endpoint->isoc_next) &
2806             (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2807
2808         if ((xfer->endpoint->is_synced == 0) ||
2809             (buf_offset < ((xfer->nframes + 7) / 8))) {
2810                 /*
2811                  * If there is data underflow or the pipe queue is empty we
2812                  * schedule the transfer a few frames ahead of the current
2813                  * frame position. Else two isochronous transfers might
2814                  * overlap.
2815                  */
2816                 xfer->endpoint->isoc_next = (nframes + 3) &
2817                     (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2818                 xfer->endpoint->is_synced = 1;
2819                 DPRINTFN(3, "start next=%d\n", xfer->endpoint->isoc_next);
2820         }
2821         /*
2822          * compute how many milliseconds the insertion is ahead of the
2823          * current frame position:
2824          */
2825         buf_offset = (xfer->endpoint->isoc_next - nframes) &
2826             (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2827
2828         /*
2829          * pre-compute when the isochronous transfer will be finished:
2830          */
2831         xfer->isoc_time_complete =
2832             usb_isoc_time_expand(&sc->sc_bus, nframes) + buf_offset +
2833             ((xfer->nframes + 7) / 8);
2834
2835         /* get the real number of frames */
2836
2837         nframes = xfer->nframes;
2838
2839         buf_offset = 0;
2840         td_no = 0;
2841
2842         plen = xfer->frlengths;
2843
2844         /* toggle the DMA set we are using */
2845         xfer->flags_int.curr_dma_set ^= 1;
2846
2847         /* get next DMA set */
2848         td = xfer->td_start[xfer->flags_int.curr_dma_set];
2849         xfer->td_transfer_first = td;
2850
2851         pp_last = &sc->sc_isoc_hs_p_last[xfer->endpoint->isoc_next];
2852
2853         /* store starting position */
2854
2855         xfer->qh_pos = xfer->endpoint->isoc_next;
2856
2857         while (nframes--) {
2858                 if (td == NULL) {
2859                         panic("%s:%d: out of TD's\n",
2860                             __FUNCTION__, __LINE__);
2861                 }
2862                 if (pp_last >= &sc->sc_isoc_hs_p_last[EHCI_VIRTUAL_FRAMELIST_COUNT]) {
2863                         pp_last = &sc->sc_isoc_hs_p_last[0];
2864                 }
2865                 /* range check */
2866                 if (*plen > xfer->max_frame_size) {
2867 #if USB_DEBUG
2868                         if (once) {
2869                                 once = 0;
2870                                 printf("%s: frame length(%d) exceeds %d bytes "
2871                                     "(frame truncated)\n",
2872                                     __FUNCTION__, *plen, xfer->max_frame_size);
2873                         }
2874 #endif
2875                         *plen = xfer->max_frame_size;
2876                 }
2877                 status = (EHCI_ITD_SET_LEN(*plen) |
2878                     EHCI_ITD_ACTIVE |
2879                     EHCI_ITD_SET_PG(0));
2880                 td->itd_status[td_no] = htohc32(sc, status);
2881                 itd_offset[td_no] = buf_offset;
2882                 buf_offset += *plen;
2883                 plen++;
2884                 td_no++;
2885
2886                 if ((td_no == 8) || (nframes == 0)) {
2887
2888                         /* the rest of the transfers are not active, if any */
2889                         for (x = td_no; x != 8; x++) {
2890                                 td->itd_status[x] = 0;  /* not active */
2891                         }
2892
2893                         /* check if there is any data to be transferred */
2894                         if (itd_offset[0] != buf_offset) {
2895                                 page_no = 0;
2896                                 itd_offset[td_no] = buf_offset;
2897
2898                                 /* get first page offset */
2899                                 usbd_get_page(xfer->frbuffers, itd_offset[0], &buf_res);
2900                                 /* get page address */
2901                                 page_addr = buf_res.physaddr & ~0xFFF;
2902                                 /* update page address */
2903                                 td->itd_bp[0] &= htohc32(sc, 0xFFF);
2904                                 td->itd_bp[0] |= htohc32(sc, page_addr);
2905
2906                                 for (x = 0; x != td_no; x++) {
2907                                         /* set page number and page offset */
2908                                         status = (EHCI_ITD_SET_PG(page_no) |
2909                                             (buf_res.physaddr & 0xFFF));
2910                                         td->itd_status[x] |= htohc32(sc, status);
2911
2912                                         /* get next page offset */
2913                                         if (itd_offset[x + 1] == buf_offset) {
2914                                                 /*
2915                                                  * We subtract one so that
2916                                                  * we don't go off the last
2917                                                  * page!
2918                                                  */
2919                                                 usbd_get_page(xfer->frbuffers, buf_offset - 1, &buf_res);
2920                                         } else {
2921                                                 usbd_get_page(xfer->frbuffers, itd_offset[x + 1], &buf_res);
2922                                         }
2923
2924                                         /* check if we need a new page */
2925                                         if ((buf_res.physaddr ^ page_addr) & ~0xFFF) {
2926                                                 /* new page needed */
2927                                                 page_addr = buf_res.physaddr & ~0xFFF;
2928                                                 if (page_no == 6) {
2929                                                         panic("%s: too many pages\n", __FUNCTION__);
2930                                                 }
2931                                                 page_no++;
2932                                                 /* update page address */
2933                                                 td->itd_bp[page_no] &= htohc32(sc, 0xFFF);
2934                                                 td->itd_bp[page_no] |= htohc32(sc, page_addr);
2935                                         }
2936                                 }
2937                         }
2938                         /* set IOC bit if we are complete */
2939                         if (nframes == 0) {
2940                                 td->itd_status[7] |= htohc32(sc, EHCI_ITD_IOC);
2941                         }
2942                         usb_pc_cpu_flush(td->page_cache);
2943 #if USB_DEBUG
2944                         if (ehcidebug > 15) {
2945                                 DPRINTF("HS-TD %d\n", nframes);
2946                                 ehci_dump_itd(sc, td);
2947                         }
2948 #endif
2949                         /* insert TD into schedule */
2950                         EHCI_APPEND_HS_TD(td, *pp_last);
2951                         pp_last++;
2952
2953                         td_no = 0;
2954                         td_last = td;
2955                         td = td->obj_next;
2956                 }
2957         }
2958
2959         xfer->td_transfer_last = td_last;
2960
2961         /* update isoc_next */
2962         xfer->endpoint->isoc_next = (pp_last - &sc->sc_isoc_hs_p_last[0]) &
2963             (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2964 }
2965
2966 static void
2967 ehci_device_isoc_hs_start(struct usb_xfer *xfer)
2968 {
2969         /* put transfer on interrupt queue */
2970         ehci_transfer_intr_enqueue(xfer);
2971 }
2972
2973 struct usb_pipe_methods ehci_device_isoc_hs_methods =
2974 {
2975         .open = ehci_device_isoc_hs_open,
2976         .close = ehci_device_isoc_hs_close,
2977         .enter = ehci_device_isoc_hs_enter,
2978         .start = ehci_device_isoc_hs_start,
2979 };
2980
2981 /*------------------------------------------------------------------------*
2982  * ehci root control support
2983  *------------------------------------------------------------------------*
2984  * Simulate a hardware hub by handling all the necessary requests.
2985  *------------------------------------------------------------------------*/
2986
2987 static const
2988 struct usb_device_descriptor ehci_devd =
2989 {
2990         sizeof(struct usb_device_descriptor),
2991         UDESC_DEVICE,                   /* type */
2992         {0x00, 0x02},                   /* USB version */
2993         UDCLASS_HUB,                    /* class */
2994         UDSUBCLASS_HUB,                 /* subclass */
2995         UDPROTO_HSHUBSTT,               /* protocol */
2996         64,                             /* max packet */
2997         {0}, {0}, {0x00, 0x01},         /* device id */
2998         1, 2, 0,                        /* string indicies */
2999         1                               /* # of configurations */
3000 };
3001
3002 static const
3003 struct usb_device_qualifier ehci_odevd =
3004 {
3005         sizeof(struct usb_device_qualifier),
3006         UDESC_DEVICE_QUALIFIER,         /* type */
3007         {0x00, 0x02},                   /* USB version */
3008         UDCLASS_HUB,                    /* class */
3009         UDSUBCLASS_HUB,                 /* subclass */
3010         UDPROTO_FSHUB,                  /* protocol */
3011         0,                              /* max packet */
3012         0,                              /* # of configurations */
3013         0
3014 };
3015
3016 static const struct ehci_config_desc ehci_confd = {
3017         .confd = {
3018                 .bLength = sizeof(struct usb_config_descriptor),
3019                 .bDescriptorType = UDESC_CONFIG,
3020                 .wTotalLength[0] = sizeof(ehci_confd),
3021                 .bNumInterface = 1,
3022                 .bConfigurationValue = 1,
3023                 .iConfiguration = 0,
3024                 .bmAttributes = UC_SELF_POWERED,
3025                 .bMaxPower = 0          /* max power */
3026         },
3027         .ifcd = {
3028                 .bLength = sizeof(struct usb_interface_descriptor),
3029                 .bDescriptorType = UDESC_INTERFACE,
3030                 .bNumEndpoints = 1,
3031                 .bInterfaceClass = UICLASS_HUB,
3032                 .bInterfaceSubClass = UISUBCLASS_HUB,
3033                 .bInterfaceProtocol = UIPROTO_HSHUBSTT,
3034                 0
3035         },
3036         .endpd = {
3037                 .bLength = sizeof(struct usb_endpoint_descriptor),
3038                 .bDescriptorType = UDESC_ENDPOINT,
3039                 .bEndpointAddress = UE_DIR_IN | EHCI_INTR_ENDPT,
3040                 .bmAttributes = UE_INTERRUPT,
3041                 .wMaxPacketSize[0] = 8, /* max packet (63 ports) */
3042                 .bInterval = 255,
3043         },
3044 };
3045
3046 static const
3047 struct usb_hub_descriptor ehci_hubd =
3048 {
3049         0,                              /* dynamic length */
3050         UDESC_HUB,
3051         0,
3052         {0, 0},
3053         0,
3054         0,
3055         {0},
3056 };
3057
3058 static void
3059 ehci_disown(ehci_softc_t *sc, uint16_t index, uint8_t lowspeed)
3060 {
3061         uint32_t port;
3062         uint32_t v;
3063
3064         DPRINTF("index=%d lowspeed=%d\n", index, lowspeed);
3065
3066         port = EHCI_PORTSC(index);
3067         v = EOREAD4(sc, port) & ~EHCI_PS_CLEAR;
3068         EOWRITE4(sc, port, v | EHCI_PS_PO);
3069 }
3070
3071 static usb_error_t
3072 ehci_roothub_exec(struct usb_device *udev,
3073     struct usb_device_request *req, const void **pptr, uint16_t *plength)
3074 {
3075         ehci_softc_t *sc = EHCI_BUS2SC(udev->bus);
3076         const char *str_ptr;
3077         const void *ptr;
3078         uint32_t port;
3079         uint32_t v;
3080         uint16_t len;
3081         uint16_t i;
3082         uint16_t value;
3083         uint16_t index;
3084         uint8_t l;
3085         usb_error_t err;
3086
3087         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
3088
3089         /* buffer reset */
3090         ptr = (const void *)&sc->sc_hub_desc;
3091         len = 0;
3092         err = 0;
3093
3094         value = UGETW(req->wValue);
3095         index = UGETW(req->wIndex);
3096
3097         DPRINTFN(3, "type=0x%02x request=0x%02x wLen=0x%04x "
3098             "wValue=0x%04x wIndex=0x%04x\n",
3099             req->bmRequestType, req->bRequest,
3100             UGETW(req->wLength), value, index);
3101
3102 #define C(x,y) ((x) | ((y) << 8))
3103         switch (C(req->bRequest, req->bmRequestType)) {
3104         case C(UR_CLEAR_FEATURE, UT_WRITE_DEVICE):
3105         case C(UR_CLEAR_FEATURE, UT_WRITE_INTERFACE):
3106         case C(UR_CLEAR_FEATURE, UT_WRITE_ENDPOINT):
3107                 /*
3108                  * DEVICE_REMOTE_WAKEUP and ENDPOINT_HALT are no-ops
3109                  * for the integrated root hub.
3110                  */
3111                 break;
3112         case C(UR_GET_CONFIG, UT_READ_DEVICE):
3113                 len = 1;
3114                 sc->sc_hub_desc.temp[0] = sc->sc_conf;
3115                 break;
3116         case C(UR_GET_DESCRIPTOR, UT_READ_DEVICE):
3117                 switch (value >> 8) {
3118                 case UDESC_DEVICE:
3119                         if ((value & 0xff) != 0) {
3120                                 err = USB_ERR_IOERROR;
3121                                 goto done;
3122                         }
3123                         len = sizeof(ehci_devd);
3124                         ptr = (const void *)&ehci_devd;
3125                         break;
3126                         /*
3127                          * We can't really operate at another speed,
3128                          * but the specification says we need this
3129                          * descriptor:
3130                          */
3131                 case UDESC_DEVICE_QUALIFIER:
3132                         if ((value & 0xff) != 0) {
3133                                 err = USB_ERR_IOERROR;
3134                                 goto done;
3135                         }
3136                         len = sizeof(ehci_odevd);
3137                         ptr = (const void *)&ehci_odevd;
3138                         break;
3139
3140                 case UDESC_CONFIG:
3141                         if ((value & 0xff) != 0) {
3142                                 err = USB_ERR_IOERROR;
3143                                 goto done;
3144                         }
3145                         len = sizeof(ehci_confd);
3146                         ptr = (const void *)&ehci_confd;
3147                         break;
3148
3149                 case UDESC_STRING:
3150                         switch (value & 0xff) {
3151                         case 0: /* Language table */
3152                                 str_ptr = "\001";
3153                                 break;
3154
3155                         case 1: /* Vendor */
3156                                 str_ptr = sc->sc_vendor;
3157                                 break;
3158
3159                         case 2: /* Product */
3160                                 str_ptr = "EHCI root HUB";
3161                                 break;
3162
3163                         default:
3164                                 str_ptr = "";
3165                                 break;
3166                         }
3167
3168                         len = usb_make_str_desc(
3169                             sc->sc_hub_desc.temp,
3170                             sizeof(sc->sc_hub_desc.temp),
3171                             str_ptr);
3172                         break;
3173                 default:
3174                         err = USB_ERR_IOERROR;
3175                         goto done;
3176                 }
3177                 break;
3178         case C(UR_GET_INTERFACE, UT_READ_INTERFACE):
3179                 len = 1;
3180                 sc->sc_hub_desc.temp[0] = 0;
3181                 break;
3182         case C(UR_GET_STATUS, UT_READ_DEVICE):
3183                 len = 2;
3184                 USETW(sc->sc_hub_desc.stat.wStatus, UDS_SELF_POWERED);
3185                 break;
3186         case C(UR_GET_STATUS, UT_READ_INTERFACE):
3187         case C(UR_GET_STATUS, UT_READ_ENDPOINT):
3188                 len = 2;
3189                 USETW(sc->sc_hub_desc.stat.wStatus, 0);
3190                 break;
3191         case C(UR_SET_ADDRESS, UT_WRITE_DEVICE):
3192                 if (value >= EHCI_MAX_DEVICES) {
3193                         err = USB_ERR_IOERROR;
3194                         goto done;
3195                 }
3196                 sc->sc_addr = value;
3197                 break;
3198         case C(UR_SET_CONFIG, UT_WRITE_DEVICE):
3199                 if ((value != 0) && (value != 1)) {
3200                         err = USB_ERR_IOERROR;
3201                         goto done;
3202                 }
3203                 sc->sc_conf = value;
3204                 break;
3205         case C(UR_SET_DESCRIPTOR, UT_WRITE_DEVICE):
3206                 break;
3207         case C(UR_SET_FEATURE, UT_WRITE_DEVICE):
3208         case C(UR_SET_FEATURE, UT_WRITE_INTERFACE):
3209         case C(UR_SET_FEATURE, UT_WRITE_ENDPOINT):
3210                 err = USB_ERR_IOERROR;
3211                 goto done;
3212         case C(UR_SET_INTERFACE, UT_WRITE_INTERFACE):
3213                 break;
3214         case C(UR_SYNCH_FRAME, UT_WRITE_ENDPOINT):
3215                 break;
3216                 /* Hub requests */
3217         case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_DEVICE):
3218                 break;
3219         case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_OTHER):
3220                 DPRINTFN(9, "UR_CLEAR_PORT_FEATURE\n");
3221
3222                 if ((index < 1) ||
3223                     (index > sc->sc_noport)) {
3224                         err = USB_ERR_IOERROR;
3225                         goto done;
3226                 }
3227                 port = EHCI_PORTSC(index);
3228                 v = EOREAD4(sc, port) & ~EHCI_PS_CLEAR;
3229                 switch (value) {
3230                 case UHF_PORT_ENABLE:
3231                         EOWRITE4(sc, port, v & ~EHCI_PS_PE);
3232                         break;
3233                 case UHF_PORT_SUSPEND:
3234                         if ((v & EHCI_PS_SUSP) && (!(v & EHCI_PS_FPR))) {
3235
3236                                 /*
3237                                  * waking up a High Speed device is rather
3238                                  * complicated if
3239                                  */
3240                                 EOWRITE4(sc, port, v | EHCI_PS_FPR);
3241                         }
3242                         /* wait 20ms for resume sequence to complete */
3243                         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 50);
3244
3245                         EOWRITE4(sc, port, v & ~(EHCI_PS_SUSP |
3246                             EHCI_PS_FPR | (3 << 10) /* High Speed */ ));
3247
3248                         /* 4ms settle time */
3249                         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 250);
3250                         break;
3251                 case UHF_PORT_POWER:
3252                         EOWRITE4(sc, port, v & ~EHCI_PS_PP);
3253                         break;
3254                 case UHF_PORT_TEST:
3255                         DPRINTFN(3, "clear port test "
3256                             "%d\n", index);
3257                         break;
3258                 case UHF_PORT_INDICATOR:
3259                         DPRINTFN(3, "clear port ind "
3260                             "%d\n", index);
3261                         EOWRITE4(sc, port, v & ~EHCI_PS_PIC);
3262                         break;
3263                 case UHF_C_PORT_CONNECTION:
3264                         EOWRITE4(sc, port, v | EHCI_PS_CSC);
3265                         break;
3266                 case UHF_C_PORT_ENABLE:
3267                         EOWRITE4(sc, port, v | EHCI_PS_PEC);
3268                         break;
3269                 case UHF_C_PORT_SUSPEND:
3270                         EOWRITE4(sc, port, v | EHCI_PS_SUSP);
3271                         break;
3272                 case UHF_C_PORT_OVER_CURRENT:
3273                         EOWRITE4(sc, port, v | EHCI_PS_OCC);
3274                         break;
3275                 case UHF_C_PORT_RESET:
3276                         sc->sc_isreset = 0;
3277                         break;
3278                 default:
3279                         err = USB_ERR_IOERROR;
3280                         goto done;
3281                 }
3282                 break;
3283         case C(UR_GET_DESCRIPTOR, UT_READ_CLASS_DEVICE):
3284                 if ((value & 0xff) != 0) {
3285                         err = USB_ERR_IOERROR;
3286                         goto done;
3287                 }
3288                 v = EOREAD4(sc, EHCI_HCSPARAMS);
3289
3290                 sc->sc_hub_desc.hubd = ehci_hubd;
3291                 sc->sc_hub_desc.hubd.bNbrPorts = sc->sc_noport;
3292                 USETW(sc->sc_hub_desc.hubd.wHubCharacteristics,
3293                     (EHCI_HCS_PPC(v) ? UHD_PWR_INDIVIDUAL : UHD_PWR_NO_SWITCH) |
3294                     (EHCI_HCS_P_INDICATOR(EREAD4(sc, EHCI_HCSPARAMS)) ?
3295                     UHD_PORT_IND : 0));
3296                 /* XXX can't find out? */
3297                 sc->sc_hub_desc.hubd.bPwrOn2PwrGood = 200;
3298                 for (l = 0; l < sc->sc_noport; l++) {
3299                         /* XXX can't find out? */
3300                         sc->sc_hub_desc.hubd.DeviceRemovable[l / 8] &= ~(1 << (l % 8));
3301                 }
3302                 sc->sc_hub_desc.hubd.bDescLength =
3303                     8 + ((sc->sc_noport + 7) / 8);
3304                 len = sc->sc_hub_desc.hubd.bDescLength;
3305                 break;
3306         case C(UR_GET_STATUS, UT_READ_CLASS_DEVICE):
3307                 len = 16;
3308                 bzero(sc->sc_hub_desc.temp, 16);
3309                 break;
3310         case C(UR_GET_STATUS, UT_READ_CLASS_OTHER):
3311                 DPRINTFN(9, "get port status i=%d\n",
3312                     index);
3313                 if ((index < 1) ||
3314                     (index > sc->sc_noport)) {
3315                         err = USB_ERR_IOERROR;
3316                         goto done;
3317                 }
3318                 v = EOREAD4(sc, EHCI_PORTSC(index));
3319                 DPRINTFN(9, "port status=0x%04x\n", v);
3320                 if (sc->sc_flags & (EHCI_SCFLG_FORCESPEED | EHCI_SCFLG_TT)) {
3321                         if ((v & 0xc000000) == 0x8000000)
3322                                 i = UPS_HIGH_SPEED;
3323                         else if ((v & 0xc000000) == 0x4000000)
3324                                 i = UPS_LOW_SPEED;
3325                         else
3326                                 i = 0;
3327                 } else {
3328                         i = UPS_HIGH_SPEED;
3329                 }
3330                 if (v & EHCI_PS_CS)
3331                         i |= UPS_CURRENT_CONNECT_STATUS;
3332                 if (v & EHCI_PS_PE)
3333                         i |= UPS_PORT_ENABLED;
3334                 if ((v & EHCI_PS_SUSP) && !(v & EHCI_PS_FPR))
3335                         i |= UPS_SUSPEND;
3336                 if (v & EHCI_PS_OCA)
3337                         i |= UPS_OVERCURRENT_INDICATOR;
3338                 if (v & EHCI_PS_PR)
3339                         i |= UPS_RESET;
3340                 if (v & EHCI_PS_PP)
3341                         i |= UPS_PORT_POWER;
3342                 USETW(sc->sc_hub_desc.ps.wPortStatus, i);
3343                 i = 0;
3344                 if (v & EHCI_PS_CSC)
3345                         i |= UPS_C_CONNECT_STATUS;
3346                 if (v & EHCI_PS_PEC)
3347                         i |= UPS_C_PORT_ENABLED;
3348                 if (v & EHCI_PS_OCC)
3349                         i |= UPS_C_OVERCURRENT_INDICATOR;
3350                 if (v & EHCI_PS_FPR)
3351                         i |= UPS_C_SUSPEND;
3352                 if (sc->sc_isreset)
3353                         i |= UPS_C_PORT_RESET;
3354                 USETW(sc->sc_hub_desc.ps.wPortChange, i);
3355                 len = sizeof(sc->sc_hub_desc.ps);
3356                 break;
3357         case C(UR_SET_DESCRIPTOR, UT_WRITE_CLASS_DEVICE):
3358                 err = USB_ERR_IOERROR;
3359                 goto done;
3360         case C(UR_SET_FEATURE, UT_WRITE_CLASS_DEVICE):
3361                 break;
3362         case C(UR_SET_FEATURE, UT_WRITE_CLASS_OTHER):
3363                 if ((index < 1) ||
3364                     (index > sc->sc_noport)) {
3365                         err = USB_ERR_IOERROR;
3366                         goto done;
3367                 }
3368                 port = EHCI_PORTSC(index);
3369                 v = EOREAD4(sc, port) & ~EHCI_PS_CLEAR;
3370                 switch (value) {
3371                 case UHF_PORT_ENABLE:
3372                         EOWRITE4(sc, port, v | EHCI_PS_PE);
3373                         break;
3374                 case UHF_PORT_SUSPEND:
3375                         EOWRITE4(sc, port, v | EHCI_PS_SUSP);
3376                         break;
3377                 case UHF_PORT_RESET:
3378                         DPRINTFN(6, "reset port %d\n", index);
3379 #if USB_DEBUG
3380                         if (ehcinohighspeed) {
3381                                 /*
3382                                  * Connect USB device to companion
3383                                  * controller.
3384                                  */
3385                                 ehci_disown(sc, index, 1);
3386                                 break;
3387                         }
3388 #endif
3389                         if (EHCI_PS_IS_LOWSPEED(v) &&
3390                             (sc->sc_flags & EHCI_SCFLG_TT) == 0) {
3391                                 /* Low speed device, give up ownership. */
3392                                 ehci_disown(sc, index, 1);
3393                                 break;
3394                         }
3395                         /* Start reset sequence. */
3396                         v &= ~(EHCI_PS_PE | EHCI_PS_PR);
3397                         EOWRITE4(sc, port, v | EHCI_PS_PR);
3398
3399                         /* Wait for reset to complete. */
3400                         usb_pause_mtx(&sc->sc_bus.bus_mtx,
3401                             USB_MS_TO_TICKS(USB_PORT_ROOT_RESET_DELAY));
3402
3403                         /* Terminate reset sequence. */
3404                         if (!(sc->sc_flags & EHCI_SCFLG_NORESTERM))
3405                                 EOWRITE4(sc, port, v);
3406
3407                         /* Wait for HC to complete reset. */
3408                         usb_pause_mtx(&sc->sc_bus.bus_mtx,
3409                             USB_MS_TO_TICKS(EHCI_PORT_RESET_COMPLETE));
3410
3411                         v = EOREAD4(sc, port);
3412                         DPRINTF("ehci after reset, status=0x%08x\n", v);
3413                         if (v & EHCI_PS_PR) {
3414                                 device_printf(sc->sc_bus.bdev,
3415                                     "port reset timeout\n");
3416                                 err = USB_ERR_TIMEOUT;
3417                                 goto done;
3418                         }
3419                         if (!(v & EHCI_PS_PE) &&
3420                             (sc->sc_flags & EHCI_SCFLG_TT) == 0) {
3421                                 /* Not a high speed device, give up ownership.*/
3422                                 ehci_disown(sc, index, 0);
3423                                 break;
3424                         }
3425                         sc->sc_isreset = 1;
3426                         DPRINTF("ehci port %d reset, status = 0x%08x\n",
3427                             index, v);
3428                         break;
3429
3430                 case UHF_PORT_POWER:
3431                         DPRINTFN(3, "set port power %d\n", index);
3432                         EOWRITE4(sc, port, v | EHCI_PS_PP);
3433                         break;
3434
3435                 case UHF_PORT_TEST:
3436                         DPRINTFN(3, "set port test %d\n", index);
3437                         break;
3438
3439                 case UHF_PORT_INDICATOR:
3440                         DPRINTFN(3, "set port ind %d\n", index);
3441                         EOWRITE4(sc, port, v | EHCI_PS_PIC);
3442                         break;
3443
3444                 default:
3445                         err = USB_ERR_IOERROR;
3446                         goto done;
3447                 }
3448                 break;
3449         case C(UR_CLEAR_TT_BUFFER, UT_WRITE_CLASS_OTHER):
3450         case C(UR_RESET_TT, UT_WRITE_CLASS_OTHER):
3451         case C(UR_GET_TT_STATE, UT_READ_CLASS_OTHER):
3452         case C(UR_STOP_TT, UT_WRITE_CLASS_OTHER):
3453                 break;
3454         default:
3455                 err = USB_ERR_IOERROR;
3456                 goto done;
3457         }
3458 done:
3459         *plength = len;
3460         *pptr = ptr;
3461         return (err);
3462 }
3463
3464 static void
3465 ehci_xfer_setup(struct usb_setup_params *parm)
3466 {
3467         struct usb_page_search page_info;
3468         struct usb_page_cache *pc;
3469         ehci_softc_t *sc;
3470         struct usb_xfer *xfer;
3471         void *last_obj;
3472         uint32_t nqtd;
3473         uint32_t nqh;
3474         uint32_t nsitd;
3475         uint32_t nitd;
3476         uint32_t n;
3477
3478         sc = EHCI_BUS2SC(parm->udev->bus);
3479         xfer = parm->curr_xfer;
3480
3481         nqtd = 0;
3482         nqh = 0;
3483         nsitd = 0;
3484         nitd = 0;
3485
3486         /*
3487          * compute maximum number of some structures
3488          */
3489         if (parm->methods == &ehci_device_ctrl_methods) {
3490
3491                 /*
3492                  * The proof for the "nqtd" formula is illustrated like
3493                  * this:
3494                  *
3495                  * +------------------------------------+
3496                  * |                                    |
3497                  * |         |remainder ->              |
3498                  * |   +-----+---+                      |
3499                  * |   | xxx | x | frm 0                |
3500                  * |   +-----+---++                     |
3501                  * |   | xxx | xx | frm 1               |
3502                  * |   +-----+----+                     |
3503                  * |            ...                     |
3504                  * +------------------------------------+
3505                  *
3506                  * "xxx" means a completely full USB transfer descriptor
3507                  *
3508                  * "x" and "xx" means a short USB packet
3509                  *
3510                  * For the remainder of an USB transfer modulo
3511                  * "max_data_length" we need two USB transfer descriptors.
3512                  * One to transfer the remaining data and one to finalise
3513                  * with a zero length packet in case the "force_short_xfer"
3514                  * flag is set. We only need two USB transfer descriptors in
3515                  * the case where the transfer length of the first one is a
3516                  * factor of "max_frame_size". The rest of the needed USB
3517                  * transfer descriptors is given by the buffer size divided
3518                  * by the maximum data payload.
3519                  */
3520                 parm->hc_max_packet_size = 0x400;
3521                 parm->hc_max_packet_count = 1;
3522                 parm->hc_max_frame_size = EHCI_QTD_PAYLOAD_MAX;
3523                 xfer->flags_int.bdma_enable = 1;
3524
3525                 usbd_transfer_setup_sub(parm);
3526
3527                 nqh = 1;
3528                 nqtd = ((2 * xfer->nframes) + 1 /* STATUS */
3529                     + (xfer->max_data_length / xfer->max_hc_frame_size));
3530
3531         } else if (parm->methods == &ehci_device_bulk_methods) {
3532
3533                 parm->hc_max_packet_size = 0x400;
3534                 parm->hc_max_packet_count = 1;
3535                 parm->hc_max_frame_size = EHCI_QTD_PAYLOAD_MAX;
3536                 xfer->flags_int.bdma_enable = 1;
3537
3538                 usbd_transfer_setup_sub(parm);
3539
3540                 nqh = 1;
3541                 nqtd = ((2 * xfer->nframes)
3542                     + (xfer->max_data_length / xfer->max_hc_frame_size));
3543
3544         } else if (parm->methods == &ehci_device_intr_methods) {
3545
3546                 if (parm->speed == USB_SPEED_HIGH) {
3547                         parm->hc_max_packet_size = 0x400;
3548                         parm->hc_max_packet_count = 3;
3549                 } else if (parm->speed == USB_SPEED_FULL) {
3550                         parm->hc_max_packet_size = USB_FS_BYTES_PER_HS_UFRAME;
3551                         parm->hc_max_packet_count = 1;
3552                 } else {
3553                         parm->hc_max_packet_size = USB_FS_BYTES_PER_HS_UFRAME / 8;
3554                         parm->hc_max_packet_count = 1;
3555                 }
3556
3557                 parm->hc_max_frame_size = EHCI_QTD_PAYLOAD_MAX;
3558                 xfer->flags_int.bdma_enable = 1;
3559
3560                 usbd_transfer_setup_sub(parm);
3561
3562                 nqh = 1;
3563                 nqtd = ((2 * xfer->nframes)
3564                     + (xfer->max_data_length / xfer->max_hc_frame_size));
3565
3566         } else if (parm->methods == &ehci_device_isoc_fs_methods) {
3567
3568                 parm->hc_max_packet_size = 0x3FF;
3569                 parm->hc_max_packet_count = 1;
3570                 parm->hc_max_frame_size = 0x3FF;
3571                 xfer->flags_int.bdma_enable = 1;
3572
3573                 usbd_transfer_setup_sub(parm);
3574
3575                 nsitd = xfer->nframes;
3576
3577         } else if (parm->methods == &ehci_device_isoc_hs_methods) {
3578
3579                 parm->hc_max_packet_size = 0x400;
3580                 parm->hc_max_packet_count = 3;
3581                 parm->hc_max_frame_size = 0xC00;
3582                 xfer->flags_int.bdma_enable = 1;
3583
3584                 usbd_transfer_setup_sub(parm);
3585
3586                 nitd = (xfer->nframes + 7) / 8;
3587
3588         } else {
3589
3590                 parm->hc_max_packet_size = 0x400;
3591                 parm->hc_max_packet_count = 1;
3592                 parm->hc_max_frame_size = 0x400;
3593
3594                 usbd_transfer_setup_sub(parm);
3595         }
3596
3597 alloc_dma_set:
3598
3599         if (parm->err) {
3600                 return;
3601         }
3602         /*
3603          * Allocate queue heads and transfer descriptors
3604          */
3605         last_obj = NULL;
3606
3607         if (usbd_transfer_setup_sub_malloc(
3608             parm, &pc, sizeof(ehci_itd_t),
3609             EHCI_ITD_ALIGN, nitd)) {
3610                 parm->err = USB_ERR_NOMEM;
3611                 return;
3612         }
3613         if (parm->buf) {
3614                 for (n = 0; n != nitd; n++) {
3615                         ehci_itd_t *td;
3616
3617                         usbd_get_page(pc + n, 0, &page_info);
3618
3619                         td = page_info.buffer;
3620
3621                         /* init TD */
3622                         td->itd_self = htohc32(sc, page_info.physaddr | EHCI_LINK_ITD);
3623                         td->obj_next = last_obj;
3624                         td->page_cache = pc + n;
3625
3626                         last_obj = td;
3627
3628                         usb_pc_cpu_flush(pc + n);
3629                 }
3630         }
3631         if (usbd_transfer_setup_sub_malloc(
3632             parm, &pc, sizeof(ehci_sitd_t),
3633             EHCI_SITD_ALIGN, nsitd)) {
3634                 parm->err = USB_ERR_NOMEM;
3635                 return;
3636         }
3637         if (parm->buf) {
3638                 for (n = 0; n != nsitd; n++) {
3639                         ehci_sitd_t *td;
3640
3641                         usbd_get_page(pc + n, 0, &page_info);
3642
3643                         td = page_info.buffer;
3644
3645                         /* init TD */
3646                         td->sitd_self = htohc32(sc, page_info.physaddr | EHCI_LINK_SITD);
3647                         td->obj_next = last_obj;
3648                         td->page_cache = pc + n;
3649
3650                         last_obj = td;
3651
3652                         usb_pc_cpu_flush(pc + n);
3653                 }
3654         }
3655         if (usbd_transfer_setup_sub_malloc(
3656             parm, &pc, sizeof(ehci_qtd_t),
3657             EHCI_QTD_ALIGN, nqtd)) {
3658                 parm->err = USB_ERR_NOMEM;
3659                 return;
3660         }
3661         if (parm->buf) {
3662                 for (n = 0; n != nqtd; n++) {
3663                         ehci_qtd_t *qtd;
3664
3665                         usbd_get_page(pc + n, 0, &page_info);
3666
3667                         qtd = page_info.buffer;
3668
3669                         /* init TD */
3670                         qtd->qtd_self = htohc32(sc, page_info.physaddr);
3671                         qtd->obj_next = last_obj;
3672                         qtd->page_cache = pc + n;
3673
3674                         last_obj = qtd;
3675
3676                         usb_pc_cpu_flush(pc + n);
3677                 }
3678         }
3679         xfer->td_start[xfer->flags_int.curr_dma_set] = last_obj;
3680
3681         last_obj = NULL;
3682
3683         if (usbd_transfer_setup_sub_malloc(
3684             parm, &pc, sizeof(ehci_qh_t),
3685             EHCI_QH_ALIGN, nqh)) {
3686                 parm->err = USB_ERR_NOMEM;
3687                 return;
3688         }
3689         if (parm->buf) {
3690                 for (n = 0; n != nqh; n++) {
3691                         ehci_qh_t *qh;
3692
3693                         usbd_get_page(pc + n, 0, &page_info);
3694
3695                         qh = page_info.buffer;
3696
3697                         /* init QH */
3698                         qh->qh_self = htohc32(sc, page_info.physaddr | EHCI_LINK_QH);
3699                         qh->obj_next = last_obj;
3700                         qh->page_cache = pc + n;
3701
3702                         last_obj = qh;
3703
3704                         usb_pc_cpu_flush(pc + n);
3705                 }
3706         }
3707         xfer->qh_start[xfer->flags_int.curr_dma_set] = last_obj;
3708
3709         if (!xfer->flags_int.curr_dma_set) {
3710                 xfer->flags_int.curr_dma_set = 1;
3711                 goto alloc_dma_set;
3712         }
3713 }
3714
3715 static void
3716 ehci_xfer_unsetup(struct usb_xfer *xfer)
3717 {
3718         return;
3719 }
3720
3721 static void
3722 ehci_ep_init(struct usb_device *udev, struct usb_endpoint_descriptor *edesc,
3723     struct usb_endpoint *ep)
3724 {
3725         ehci_softc_t *sc = EHCI_BUS2SC(udev->bus);
3726
3727         DPRINTFN(2, "endpoint=%p, addr=%d, endpt=%d, mode=%d (%d)\n",
3728             ep, udev->address,
3729             edesc->bEndpointAddress, udev->flags.usb_mode,
3730             sc->sc_addr);
3731
3732         if (udev->flags.usb_mode != USB_MODE_HOST) {
3733                 /* not supported */
3734                 return;
3735         }
3736         if (udev->device_index != sc->sc_addr) {
3737
3738                 if ((udev->speed != USB_SPEED_HIGH) &&
3739                     ((udev->hs_hub_addr == 0) ||
3740                     (udev->hs_port_no == 0) ||
3741                     (udev->parent_hs_hub == NULL) ||
3742                     (udev->parent_hs_hub->hub == NULL))) {
3743                         /* We need a transaction translator */
3744                         goto done;
3745                 }
3746                 switch (edesc->bmAttributes & UE_XFERTYPE) {
3747                 case UE_CONTROL:
3748                         ep->methods = &ehci_device_ctrl_methods;
3749                         break;
3750                 case UE_INTERRUPT:
3751                         ep->methods = &ehci_device_intr_methods;
3752                         break;
3753                 case UE_ISOCHRONOUS:
3754                         if (udev->speed == USB_SPEED_HIGH) {
3755                                 ep->methods = &ehci_device_isoc_hs_methods;
3756                         } else if (udev->speed == USB_SPEED_FULL) {
3757                                 ep->methods = &ehci_device_isoc_fs_methods;
3758                         }
3759                         break;
3760                 case UE_BULK:
3761                         if (udev->speed != USB_SPEED_LOW) {
3762                                 ep->methods = &ehci_device_bulk_methods;
3763                         }
3764                         break;
3765                 default:
3766                         /* do nothing */
3767                         break;
3768                 }
3769         }
3770 done:
3771         return;
3772 }
3773
3774 static void
3775 ehci_get_dma_delay(struct usb_bus *bus, uint32_t *pus)
3776 {
3777         /*
3778          * Wait until the hardware has finished any possible use of
3779          * the transfer descriptor(s) and QH
3780          */
3781         *pus = (188);                   /* microseconds */
3782 }
3783
3784 static void
3785 ehci_device_resume(struct usb_device *udev)
3786 {
3787         ehci_softc_t *sc = EHCI_BUS2SC(udev->bus);
3788         struct usb_xfer *xfer;
3789         struct usb_pipe_methods *methods;
3790
3791         DPRINTF("\n");
3792
3793         USB_BUS_LOCK(udev->bus);
3794
3795         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
3796
3797                 if (xfer->xroot->udev == udev) {
3798
3799                         methods = xfer->endpoint->methods;
3800
3801                         if ((methods == &ehci_device_bulk_methods) ||
3802                             (methods == &ehci_device_ctrl_methods)) {
3803                                 EHCI_APPEND_QH(xfer->qh_start[xfer->flags_int.curr_dma_set],
3804                                     sc->sc_async_p_last);
3805                         }
3806                         if (methods == &ehci_device_intr_methods) {
3807                                 EHCI_APPEND_QH(xfer->qh_start[xfer->flags_int.curr_dma_set],
3808                                     sc->sc_intr_p_last[xfer->qh_pos]);
3809                         }
3810                 }
3811         }
3812
3813         USB_BUS_UNLOCK(udev->bus);
3814
3815         return;
3816 }
3817
3818 static void
3819 ehci_device_suspend(struct usb_device *udev)
3820 {
3821         ehci_softc_t *sc = EHCI_BUS2SC(udev->bus);
3822         struct usb_xfer *xfer;
3823         struct usb_pipe_methods *methods;
3824
3825         DPRINTF("\n");
3826
3827         USB_BUS_LOCK(udev->bus);
3828
3829         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
3830
3831                 if (xfer->xroot->udev == udev) {
3832
3833                         methods = xfer->endpoint->methods;
3834
3835                         if ((methods == &ehci_device_bulk_methods) ||
3836                             (methods == &ehci_device_ctrl_methods)) {
3837                                 EHCI_REMOVE_QH(xfer->qh_start[xfer->flags_int.curr_dma_set],
3838                                     sc->sc_async_p_last);
3839                         }
3840                         if (methods == &ehci_device_intr_methods) {
3841                                 EHCI_REMOVE_QH(xfer->qh_start[xfer->flags_int.curr_dma_set],
3842                                     sc->sc_intr_p_last[xfer->qh_pos]);
3843                         }
3844                 }
3845         }
3846
3847         USB_BUS_UNLOCK(udev->bus);
3848
3849         return;
3850 }
3851
3852 static void
3853 ehci_set_hw_power(struct usb_bus *bus)
3854 {
3855         ehci_softc_t *sc = EHCI_BUS2SC(bus);
3856         uint32_t temp;
3857         uint32_t flags;
3858
3859         DPRINTF("\n");
3860
3861         USB_BUS_LOCK(bus);
3862
3863         flags = bus->hw_power_state;
3864
3865         temp = EOREAD4(sc, EHCI_USBCMD);
3866
3867         temp &= ~(EHCI_CMD_ASE | EHCI_CMD_PSE);
3868
3869         if (flags & (USB_HW_POWER_CONTROL |
3870             USB_HW_POWER_BULK)) {
3871                 DPRINTF("Async is active\n");
3872                 temp |= EHCI_CMD_ASE;
3873         }
3874         if (flags & (USB_HW_POWER_INTERRUPT |
3875             USB_HW_POWER_ISOC)) {
3876                 DPRINTF("Periodic is active\n");
3877                 temp |= EHCI_CMD_PSE;
3878         }
3879         EOWRITE4(sc, EHCI_USBCMD, temp);
3880
3881         USB_BUS_UNLOCK(bus);
3882
3883         return;
3884 }
3885
3886 struct usb_bus_methods ehci_bus_methods =
3887 {
3888         .endpoint_init = ehci_ep_init,
3889         .xfer_setup = ehci_xfer_setup,
3890         .xfer_unsetup = ehci_xfer_unsetup,
3891         .get_dma_delay = ehci_get_dma_delay,
3892         .device_resume = ehci_device_resume,
3893         .device_suspend = ehci_device_suspend,
3894         .set_hw_power = ehci_set_hw_power,
3895         .roothub_exec = ehci_roothub_exec,
3896         .xfer_poll = ehci_do_poll,
3897 };