sys/dev/usb/controller/ehci.c

   1 /* $FreeBSD$ */
   2 /*-
   3  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   4  *
   5  * Copyright (c) 2008 Hans Petter Selasky. All rights reserved.
   6  * Copyright (c) 2004 The NetBSD Foundation, Inc. All rights reserved.
   7  * Copyright (c) 2004 Lennart Augustsson. All rights reserved.
   8  * Copyright (c) 2004 Charles M. Hannum. All rights reserved.
   9  *
  10  * Redistribution and use in source and binary forms, with or without
  11  * modification, are permitted provided that the following conditions
  12  * are met:
  13  * 1. Redistributions of source code must retain the above copyright
  14  *    notice, this list of conditions and the following disclaimer.
  15  * 2. Redistributions in binary form must reproduce the above copyright
  16  *    notice, this list of conditions and the following disclaimer in the
  17  *    documentation and/or other materials provided with the distribution.
  18  *
  19  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  20  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  21  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  22  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  23  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  24  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  25  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  26  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  27  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  28  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  29  * SUCH DAMAGE.
  30  */
  31
  32 /*
  33  * USB Enhanced Host Controller Driver, a.k.a. USB 2.0 controller.
  34  *
  35  * The EHCI 0.96 spec can be found at
  36  * http://developer.intel.com/technology/usb/download/ehci-r096.pdf
  37  * The EHCI 1.0 spec can be found at
  38  * http://developer.intel.com/technology/usb/download/ehci-r10.pdf
  39  * and the USB 2.0 spec at
  40  * http://www.usb.org/developers/docs/usb_20.zip
  41  *
  42  */
  43
  44 /*
  45  * TODO:
  46  * 1) command failures are not recovered correctly
  47  */
  48
  49 #ifdef USB_GLOBAL_INCLUDE_FILE
  50 #include USB_GLOBAL_INCLUDE_FILE
  51 #else
  52 #include <sys/stdint.h>
  53 #include <sys/stddef.h>
  54 #include <sys/param.h>
  55 #include <sys/queue.h>
  56 #include <sys/types.h>
  57 #include <sys/systm.h>
  58 #include <sys/kernel.h>
  59 #include <sys/bus.h>
  60 #include <sys/module.h>
  61 #include <sys/lock.h>
  62 #include <sys/mutex.h>
  63 #include <sys/condvar.h>
  64 #include <sys/sysctl.h>
  65 #include <sys/sx.h>
  66 #include <sys/unistd.h>
  67 #include <sys/callout.h>
  68 #include <sys/malloc.h>
  69 #include <sys/priv.h>
  70
  71 #include <dev/usb/usb.h>
  72 #include <dev/usb/usbdi.h>
  73
  74 #define USB_DEBUG_VAR ehcidebug
  75
  76 #include <dev/usb/usb_core.h>
  77 #include <dev/usb/usb_debug.h>
  78 #include <dev/usb/usb_busdma.h>
  79 #include <dev/usb/usb_process.h>
  80 #include <dev/usb/usb_transfer.h>
  81 #include <dev/usb/usb_device.h>
  82 #include <dev/usb/usb_hub.h>
  83 #include <dev/usb/usb_util.h>
  84
  85 #include <dev/usb/usb_controller.h>
  86 #include <dev/usb/usb_bus.h>
  87 #endif                  /* USB_GLOBAL_INCLUDE_FILE */
  88
  89 #include <dev/usb/controller/ehci.h>
  90 #include <dev/usb/controller/ehcireg.h>
  91
  92 #define EHCI_BUS2SC(bus) \
  93    ((ehci_softc_t *)(((uint8_t *)(bus)) - \
  94     ((uint8_t *)&(((ehci_softc_t *)0)->sc_bus))))
  95
  96 #ifdef USB_DEBUG
  97 static int ehcidebug = 0;
  98 static int ehcinohighspeed = 0;
  99 static int ehciiaadbug = 0;
 100 static int ehcilostintrbug = 0;
 101
 102 static SYSCTL_NODE(_hw_usb, OID_AUTO, ehci, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
 103     "USB ehci");
 104 SYSCTL_INT(_hw_usb_ehci, OID_AUTO, debug, CTLFLAG_RWTUN,
 105     &ehcidebug, 0, "Debug level");
 106 SYSCTL_INT(_hw_usb_ehci, OID_AUTO, no_hs, CTLFLAG_RWTUN,
 107     &ehcinohighspeed, 0, "Disable High Speed USB");
 108 SYSCTL_INT(_hw_usb_ehci, OID_AUTO, iaadbug, CTLFLAG_RWTUN,
 109     &ehciiaadbug, 0, "Enable doorbell bug workaround");
 110 SYSCTL_INT(_hw_usb_ehci, OID_AUTO, lostintrbug, CTLFLAG_RWTUN,
 111     &ehcilostintrbug, 0, "Enable lost interrupt bug workaround");
 112
 113 static void ehci_dump_regs(ehci_softc_t *sc);
 114 static void ehci_dump_sqh(ehci_softc_t *sc, ehci_qh_t *sqh);
 115
 116 #endif
 117
 118 #define EHCI_INTR_ENDPT 1
 119
 120 static const struct usb_bus_methods ehci_bus_methods;
 121 static const struct usb_pipe_methods ehci_device_bulk_methods;
 122 static const struct usb_pipe_methods ehci_device_ctrl_methods;
 123 static const struct usb_pipe_methods ehci_device_intr_methods;
 124 static const struct usb_pipe_methods ehci_device_isoc_fs_methods;
 125 static const struct usb_pipe_methods ehci_device_isoc_hs_methods;
 126
 127 static void ehci_do_poll(struct usb_bus *);
 128 static void ehci_device_done(struct usb_xfer *, usb_error_t);
 129 static uint8_t ehci_check_transfer(struct usb_xfer *);
 130 static void ehci_timeout(void *);
 131 static void ehci_poll_timeout(void *);
 132
 133 static void ehci_root_intr(ehci_softc_t *sc);
 134
 135 struct ehci_std_temp {
 136         ehci_softc_t *sc;
 137         struct usb_page_cache *pc;
 138         ehci_qtd_t *td;
 139         ehci_qtd_t *td_next;
 140         uint32_t average;
 141         uint32_t qtd_status;
 142         uint32_t len;
 143         uint16_t max_frame_size;
 144         uint8_t shortpkt;
 145         uint8_t auto_data_toggle;
 146         uint8_t setup_alt_next;
 147         uint8_t last_frame;
 148 };
 149
 150 void
 151 ehci_iterate_hw_softc(struct usb_bus *bus, usb_bus_mem_sub_cb_t *cb)
 152 {
 153         ehci_softc_t *sc = EHCI_BUS2SC(bus);
 154         uint32_t i;
 155
 156         cb(bus, &sc->sc_hw.pframes_pc, &sc->sc_hw.pframes_pg,
 157             sizeof(uint32_t) * EHCI_FRAMELIST_COUNT, EHCI_FRAMELIST_ALIGN);
 158
 159         cb(bus, &sc->sc_hw.terminate_pc, &sc->sc_hw.terminate_pg,
 160             sizeof(struct ehci_qh_sub), EHCI_QH_ALIGN);
 161
 162         cb(bus, &sc->sc_hw.async_start_pc, &sc->sc_hw.async_start_pg,
 163             sizeof(ehci_qh_t), EHCI_QH_ALIGN);
 164
 165         for (i = 0; i != EHCI_VIRTUAL_FRAMELIST_COUNT; i++) {
 166                 cb(bus, sc->sc_hw.intr_start_pc + i,
 167                     sc->sc_hw.intr_start_pg + i,
 168                     sizeof(ehci_qh_t), EHCI_QH_ALIGN);
 169         }
 170
 171         for (i = 0; i != EHCI_VIRTUAL_FRAMELIST_COUNT; i++) {
 172                 cb(bus, sc->sc_hw.isoc_hs_start_pc + i,
 173                     sc->sc_hw.isoc_hs_start_pg + i,
 174                     sizeof(ehci_itd_t), EHCI_ITD_ALIGN);
 175         }
 176
 177         for (i = 0; i != EHCI_VIRTUAL_FRAMELIST_COUNT; i++) {
 178                 cb(bus, sc->sc_hw.isoc_fs_start_pc + i,
 179                     sc->sc_hw.isoc_fs_start_pg + i,
 180                     sizeof(ehci_sitd_t), EHCI_SITD_ALIGN);
 181         }
 182 }
 183
 184 usb_error_t
 185 ehci_reset(ehci_softc_t *sc)
 186 {
 187         uint32_t hcr;
 188         int i;
 189
 190         EOWRITE4(sc, EHCI_USBCMD, EHCI_CMD_HCRESET);
 191         for (i = 0; i < 100; i++) {
 192                 usb_pause_mtx(NULL, hz / 128);
 193                 hcr = EOREAD4(sc, EHCI_USBCMD) & EHCI_CMD_HCRESET;
 194                 if (!hcr) {
 195                         if (sc->sc_vendor_post_reset != NULL)
 196                                 sc->sc_vendor_post_reset(sc);
 197                         return (0);
 198                 }
 199         }
 200         device_printf(sc->sc_bus.bdev, "reset timeout\n");
 201         return (USB_ERR_IOERROR);
 202 }
 203
 204 static usb_error_t
 205 ehci_hcreset(ehci_softc_t *sc)
 206 {
 207         uint32_t hcr;
 208         int i;
 209
 210         EOWRITE4(sc, EHCI_USBCMD, 0);   /* Halt controller */
 211         for (i = 0; i < 100; i++) {
 212                 usb_pause_mtx(NULL, hz / 128);
 213                 hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
 214                 if (hcr)
 215                         break;
 216         }
 217         if (!hcr)
 218                 /*
 219                  * Fall through and try reset anyway even though
 220                  * Table 2-9 in the EHCI spec says this will result
 221                  * in undefined behavior.
 222                  */
 223                 device_printf(sc->sc_bus.bdev, "stop timeout\n");
 224
 225         return (ehci_reset(sc));
 226 }
 227
 228 static int
 229 ehci_init_sub(struct ehci_softc *sc)
 230 {
 231         struct usb_page_search buf_res;
 232         uint32_t cparams;
 233         uint32_t hcr;
 234         uint8_t i;
 235
 236         cparams = EREAD4(sc, EHCI_HCCPARAMS);
 237
 238         DPRINTF("cparams=0x%x\n", cparams);
 239
 240         if (EHCI_HCC_64BIT(cparams)) {
 241                 DPRINTF("HCC uses 64-bit structures\n");
 242
 243                 /* MUST clear segment register if 64 bit capable */
 244                 EOWRITE4(sc, EHCI_CTRLDSSEGMENT, 0);
 245         }
 246
 247         usbd_get_page(&sc->sc_hw.pframes_pc, 0, &buf_res);
 248         EOWRITE4(sc, EHCI_PERIODICLISTBASE, buf_res.physaddr);
 249
 250         usbd_get_page(&sc->sc_hw.async_start_pc, 0, &buf_res);
 251         EOWRITE4(sc, EHCI_ASYNCLISTADDR, buf_res.physaddr | EHCI_LINK_QH);
 252
 253         /* enable interrupts */
 254         EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
 255
 256         /* turn on controller */
 257         EOWRITE4(sc, EHCI_USBCMD,
 258             EHCI_CMD_ITC_1 |            /* 1 microframes interrupt delay */
 259             (EOREAD4(sc, EHCI_USBCMD) & EHCI_CMD_FLS_M) |
 260             EHCI_CMD_ASE |
 261             EHCI_CMD_PSE |
 262             EHCI_CMD_RS);
 263
 264         /* Take over port ownership */
 265         EOWRITE4(sc, EHCI_CONFIGFLAG, EHCI_CONF_CF);
 266
 267         for (i = 0; i < 100; i++) {
 268                 usb_pause_mtx(NULL, hz / 128);
 269                 hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
 270                 if (!hcr) {
 271                         break;
 272                 }
 273         }
 274         if (hcr) {
 275                 device_printf(sc->sc_bus.bdev, "run timeout\n");
 276                 return (USB_ERR_IOERROR);
 277         }
 278         return (USB_ERR_NORMAL_COMPLETION);
 279 }
 280
 281 usb_error_t
 282 ehci_init(ehci_softc_t *sc)
 283 {
 284         struct usb_page_search buf_res;
 285         uint32_t version;
 286         uint32_t sparams;
 287         uint16_t i;
 288         uint16_t x;
 289         uint16_t y;
 290         uint16_t bit;
 291         usb_error_t err = 0;
 292
 293         DPRINTF("start\n");
 294
 295         usb_callout_init_mtx(&sc->sc_tmo_pcd, &sc->sc_bus.bus_mtx, 0);
 296         usb_callout_init_mtx(&sc->sc_tmo_poll, &sc->sc_bus.bus_mtx, 0);
 297
 298         sc->sc_offs = EHCI_CAPLENGTH(EREAD4(sc, EHCI_CAPLEN_HCIVERSION));
 299
 300 #ifdef USB_DEBUG
 301         if (ehciiaadbug)
 302                 sc->sc_flags |= EHCI_SCFLG_IAADBUG;
 303         if (ehcilostintrbug)
 304                 sc->sc_flags |= EHCI_SCFLG_LOSTINTRBUG;
 305         if (ehcidebug > 2) {
 306                 ehci_dump_regs(sc);
 307         }
 308 #endif
 309
 310         version = EHCI_HCIVERSION(EREAD4(sc, EHCI_CAPLEN_HCIVERSION));
 311         device_printf(sc->sc_bus.bdev, "EHCI version %x.%x\n",
 312             version >> 8, version & 0xff);
 313
 314         sparams = EREAD4(sc, EHCI_HCSPARAMS);
 315         DPRINTF("sparams=0x%x\n", sparams);
 316
 317         sc->sc_noport = EHCI_HCS_N_PORTS(sparams);
 318         sc->sc_bus.usbrev = USB_REV_2_0;
 319
 320         if (!(sc->sc_flags & EHCI_SCFLG_DONTRESET)) {
 321                 /* Reset the controller */
 322                 DPRINTF("%s: resetting\n",
 323                     device_get_nameunit(sc->sc_bus.bdev));
 324
 325                 err = ehci_hcreset(sc);
 326                 if (err) {
 327                         device_printf(sc->sc_bus.bdev, "reset timeout\n");
 328                         return (err);
 329                 }
 330         }
 331
 332         /*
 333          * use current frame-list-size selection 0: 1024*4 bytes 1:  512*4
 334          * bytes 2:  256*4 bytes 3:      unknown
 335          */
 336         if (EHCI_CMD_FLS(EOREAD4(sc, EHCI_USBCMD)) == 3) {
 337                 device_printf(sc->sc_bus.bdev, "invalid frame-list-size\n");
 338                 return (USB_ERR_IOERROR);
 339         }
 340         /* set up the bus struct */
 341         sc->sc_bus.methods = &ehci_bus_methods;
 342
 343         sc->sc_eintrs = EHCI_NORMAL_INTRS;
 344
 345         if (1) {
 346                 struct ehci_qh_sub *qh;
 347
 348                 usbd_get_page(&sc->sc_hw.terminate_pc, 0, &buf_res);
 349
 350                 qh = buf_res.buffer;
 351
 352                 sc->sc_terminate_self = htohc32(sc, buf_res.physaddr);
 353
 354                 /* init terminate TD */
 355                 qh->qtd_next =
 356                     htohc32(sc, EHCI_LINK_TERMINATE);
 357                 qh->qtd_altnext =
 358                     htohc32(sc, EHCI_LINK_TERMINATE);
 359                 qh->qtd_status =
 360                     htohc32(sc, EHCI_QTD_HALTED);
 361         }
 362
 363         for (i = 0; i < EHCI_VIRTUAL_FRAMELIST_COUNT; i++) {
 364                 ehci_qh_t *qh;
 365
 366                 usbd_get_page(sc->sc_hw.intr_start_pc + i, 0, &buf_res);
 367
 368                 qh = buf_res.buffer;
 369
 370                 /* initialize page cache pointer */
 371
 372                 qh->page_cache = sc->sc_hw.intr_start_pc + i;
 373
 374                 /* store a pointer to queue head */
 375
 376                 sc->sc_intr_p_last[i] = qh;
 377
 378                 qh->qh_self =
 379                     htohc32(sc, buf_res.physaddr) |
 380                     htohc32(sc, EHCI_LINK_QH);
 381
 382                 qh->qh_endp =
 383                     htohc32(sc, EHCI_QH_SET_EPS(EHCI_QH_SPEED_HIGH));
 384                 qh->qh_endphub =
 385                     htohc32(sc, EHCI_QH_SET_MULT(1));
 386                 qh->qh_curqtd = 0;
 387
 388                 qh->qh_qtd.qtd_next =
 389                     htohc32(sc, EHCI_LINK_TERMINATE);
 390                 qh->qh_qtd.qtd_altnext =
 391                     htohc32(sc, EHCI_LINK_TERMINATE);
 392                 qh->qh_qtd.qtd_status =
 393                     htohc32(sc, EHCI_QTD_HALTED);
 394         }
 395
 396         /*
 397          * the QHs are arranged to give poll intervals that are
 398          * powers of 2 times 1ms
 399          */
 400         bit = EHCI_VIRTUAL_FRAMELIST_COUNT / 2;
 401         while (bit) {
 402                 x = bit;
 403                 while (x & bit) {
 404                         ehci_qh_t *qh_x;
 405                         ehci_qh_t *qh_y;
 406
 407                         y = (x ^ bit) | (bit / 2);
 408
 409                         qh_x = sc->sc_intr_p_last[x];
 410                         qh_y = sc->sc_intr_p_last[y];
 411
 412                         /*
 413                          * the next QH has half the poll interval
 414                          */
 415                         qh_x->qh_link = qh_y->qh_self;
 416
 417                         x++;
 418                 }
 419                 bit >>= 1;
 420         }
 421
 422         if (1) {
 423                 ehci_qh_t *qh;
 424
 425                 qh = sc->sc_intr_p_last[0];
 426
 427                 /* the last (1ms) QH terminates */
 428                 qh->qh_link = htohc32(sc, EHCI_LINK_TERMINATE);
 429         }
 430         for (i = 0; i < EHCI_VIRTUAL_FRAMELIST_COUNT; i++) {
 431                 ehci_sitd_t *sitd;
 432                 ehci_itd_t *itd;
 433
 434                 usbd_get_page(sc->sc_hw.isoc_fs_start_pc + i, 0, &buf_res);
 435
 436                 sitd = buf_res.buffer;
 437
 438                 /* initialize page cache pointer */
 439
 440                 sitd->page_cache = sc->sc_hw.isoc_fs_start_pc + i;
 441
 442                 /* store a pointer to the transfer descriptor */
 443
 444                 sc->sc_isoc_fs_p_last[i] = sitd;
 445
 446                 /* initialize full speed isochronous */
 447
 448                 sitd->sitd_self =
 449                     htohc32(sc, buf_res.physaddr) |
 450                     htohc32(sc, EHCI_LINK_SITD);
 451
 452                 sitd->sitd_back =
 453                     htohc32(sc, EHCI_LINK_TERMINATE);
 454
 455                 sitd->sitd_next =
 456                     sc->sc_intr_p_last[i | (EHCI_VIRTUAL_FRAMELIST_COUNT / 2)]->qh_self;
 457
 458                 usbd_get_page(sc->sc_hw.isoc_hs_start_pc + i, 0, &buf_res);
 459
 460                 itd = buf_res.buffer;
 461
 462                 /* initialize page cache pointer */
 463
 464                 itd->page_cache = sc->sc_hw.isoc_hs_start_pc + i;
 465
 466                 /* store a pointer to the transfer descriptor */
 467
 468                 sc->sc_isoc_hs_p_last[i] = itd;
 469
 470                 /* initialize high speed isochronous */
 471
 472                 itd->itd_self =
 473                     htohc32(sc, buf_res.physaddr) |
 474                     htohc32(sc, EHCI_LINK_ITD);
 475
 476                 itd->itd_next =
 477                     sitd->sitd_self;
 478         }
 479
 480         usbd_get_page(&sc->sc_hw.pframes_pc, 0, &buf_res);
 481
 482         if (1) {
 483                 uint32_t *pframes;
 484
 485                 pframes = buf_res.buffer;
 486
 487                 /*
 488                  * execution order:
 489                  * pframes -> high speed isochronous ->
 490                  *    full speed isochronous -> interrupt QH's
 491                  */
 492                 for (i = 0; i < EHCI_FRAMELIST_COUNT; i++) {
 493                         pframes[i] = sc->sc_isoc_hs_p_last
 494                             [i & (EHCI_VIRTUAL_FRAMELIST_COUNT - 1)]->itd_self;
 495                 }
 496         }
 497         usbd_get_page(&sc->sc_hw.async_start_pc, 0, &buf_res);
 498
 499         if (1) {
 500                 ehci_qh_t *qh;
 501
 502                 qh = buf_res.buffer;
 503
 504                 /* initialize page cache pointer */
 505
 506                 qh->page_cache = &sc->sc_hw.async_start_pc;
 507
 508                 /* store a pointer to the queue head */
 509
 510                 sc->sc_async_p_last = qh;
 511
 512                 /* init dummy QH that starts the async list */
 513
 514                 qh->qh_self =
 515                     htohc32(sc, buf_res.physaddr) |
 516                     htohc32(sc, EHCI_LINK_QH);
 517
 518                 /* fill the QH */
 519                 qh->qh_endp =
 520                     htohc32(sc, EHCI_QH_SET_EPS(EHCI_QH_SPEED_HIGH) | EHCI_QH_HRECL);
 521                 qh->qh_endphub = htohc32(sc, EHCI_QH_SET_MULT(1));
 522                 qh->qh_link = qh->qh_self;
 523                 qh->qh_curqtd = 0;
 524
 525                 /* fill the overlay qTD */
 526                 qh->qh_qtd.qtd_next = htohc32(sc, EHCI_LINK_TERMINATE);
 527                 qh->qh_qtd.qtd_altnext = htohc32(sc, EHCI_LINK_TERMINATE);
 528                 qh->qh_qtd.qtd_status = htohc32(sc, EHCI_QTD_HALTED);
 529         }
 530         /* flush all cache into memory */
 531
 532         usb_bus_mem_flush_all(&sc->sc_bus, &ehci_iterate_hw_softc);
 533
 534 #ifdef USB_DEBUG
 535         if (ehcidebug) {
 536                 ehci_dump_sqh(sc, sc->sc_async_p_last);
 537         }
 538 #endif
 539
 540         /* finial setup */
 541         err = ehci_init_sub(sc);
 542
 543         if (!err) {
 544                 /* catch any lost interrupts */
 545                 ehci_do_poll(&sc->sc_bus);
 546         }
 547         return (err);
 548 }
 549
 550 /*
 551  * shut down the controller when the system is going down
 552  */
 553 void
 554 ehci_detach(ehci_softc_t *sc)
 555 {
 556         USB_BUS_LOCK(&sc->sc_bus);
 557
 558         usb_callout_stop(&sc->sc_tmo_pcd);
 559         usb_callout_stop(&sc->sc_tmo_poll);
 560
 561         EOWRITE4(sc, EHCI_USBINTR, 0);
 562         USB_BUS_UNLOCK(&sc->sc_bus);
 563
 564         if (ehci_hcreset(sc)) {
 565                 DPRINTF("reset failed!\n");
 566         }
 567
 568         /* XXX let stray task complete */
 569         usb_pause_mtx(NULL, hz / 20);
 570
 571         usb_callout_drain(&sc->sc_tmo_pcd);
 572         usb_callout_drain(&sc->sc_tmo_poll);
 573 }
 574
 575 static void
 576 ehci_suspend(ehci_softc_t *sc)
 577 {
 578         DPRINTF("stopping the HC\n");
 579
 580         /* reset HC */
 581         ehci_hcreset(sc);
 582 }
 583
 584 static void
 585 ehci_resume(ehci_softc_t *sc)
 586 {
 587         /* reset HC */
 588         ehci_hcreset(sc);
 589
 590         /* setup HC */
 591         ehci_init_sub(sc);
 592
 593         /* catch any lost interrupts */
 594         ehci_do_poll(&sc->sc_bus);
 595 }
 596
 597 #ifdef USB_DEBUG
 598 static void
 599 ehci_dump_regs(ehci_softc_t *sc)
 600 {
 601         uint32_t i;
 602
 603         i = EOREAD4(sc, EHCI_USBCMD);
 604         printf("cmd=0x%08x\n", i);
 605
 606         if (i & EHCI_CMD_ITC_1)
 607                 printf(" EHCI_CMD_ITC_1\n");
 608         if (i & EHCI_CMD_ITC_2)
 609                 printf(" EHCI_CMD_ITC_2\n");
 610         if (i & EHCI_CMD_ITC_4)
 611                 printf(" EHCI_CMD_ITC_4\n");
 612         if (i & EHCI_CMD_ITC_8)
 613                 printf(" EHCI_CMD_ITC_8\n");
 614         if (i & EHCI_CMD_ITC_16)
 615                 printf(" EHCI_CMD_ITC_16\n");
 616         if (i & EHCI_CMD_ITC_32)
 617                 printf(" EHCI_CMD_ITC_32\n");
 618         if (i & EHCI_CMD_ITC_64)
 619                 printf(" EHCI_CMD_ITC_64\n");
 620         if (i & EHCI_CMD_ASPME)
 621                 printf(" EHCI_CMD_ASPME\n");
 622         if (i & EHCI_CMD_ASPMC)
 623                 printf(" EHCI_CMD_ASPMC\n");
 624         if (i & EHCI_CMD_LHCR)
 625                 printf(" EHCI_CMD_LHCR\n");
 626         if (i & EHCI_CMD_IAAD)
 627                 printf(" EHCI_CMD_IAAD\n");
 628         if (i & EHCI_CMD_ASE)
 629                 printf(" EHCI_CMD_ASE\n");
 630         if (i & EHCI_CMD_PSE)
 631                 printf(" EHCI_CMD_PSE\n");
 632         if (i & EHCI_CMD_FLS_M)
 633                 printf(" EHCI_CMD_FLS_M\n");
 634         if (i & EHCI_CMD_HCRESET)
 635                 printf(" EHCI_CMD_HCRESET\n");
 636         if (i & EHCI_CMD_RS)
 637                 printf(" EHCI_CMD_RS\n");
 638
 639         i = EOREAD4(sc, EHCI_USBSTS);
 640
 641         printf("sts=0x%08x\n", i);
 642
 643         if (i & EHCI_STS_ASS)
 644                 printf(" EHCI_STS_ASS\n");
 645         if (i & EHCI_STS_PSS)
 646                 printf(" EHCI_STS_PSS\n");
 647         if (i & EHCI_STS_REC)
 648                 printf(" EHCI_STS_REC\n");
 649         if (i & EHCI_STS_HCH)
 650                 printf(" EHCI_STS_HCH\n");
 651         if (i & EHCI_STS_IAA)
 652                 printf(" EHCI_STS_IAA\n");
 653         if (i & EHCI_STS_HSE)
 654                 printf(" EHCI_STS_HSE\n");
 655         if (i & EHCI_STS_FLR)
 656                 printf(" EHCI_STS_FLR\n");
 657         if (i & EHCI_STS_PCD)
 658                 printf(" EHCI_STS_PCD\n");
 659         if (i & EHCI_STS_ERRINT)
 660                 printf(" EHCI_STS_ERRINT\n");
 661         if (i & EHCI_STS_INT)
 662                 printf(" EHCI_STS_INT\n");
 663
 664         printf("ien=0x%08x\n",
 665             EOREAD4(sc, EHCI_USBINTR));
 666         printf("frindex=0x%08x ctrdsegm=0x%08x periodic=0x%08x async=0x%08x\n",
 667             EOREAD4(sc, EHCI_FRINDEX),
 668             EOREAD4(sc, EHCI_CTRLDSSEGMENT),
 669             EOREAD4(sc, EHCI_PERIODICLISTBASE),
 670             EOREAD4(sc, EHCI_ASYNCLISTADDR));
 671         for (i = 1; i <= sc->sc_noport; i++) {
 672                 printf("port %d status=0x%08x\n", i,
 673                     EOREAD4(sc, EHCI_PORTSC(i)));
 674         }
 675 }
 676
 677 static void
 678 ehci_dump_link(ehci_softc_t *sc, uint32_t link, int type)
 679 {
 680         link = hc32toh(sc, link);
 681         printf("0x%08x", link);
 682         if (link & EHCI_LINK_TERMINATE)
 683                 printf("<T>");
 684         else {
 685                 printf("<");
 686                 if (type) {
 687                         switch (EHCI_LINK_TYPE(link)) {
 688                         case EHCI_LINK_ITD:
 689                                 printf("ITD");
 690                                 break;
 691                         case EHCI_LINK_QH:
 692                                 printf("QH");
 693                                 break;
 694                         case EHCI_LINK_SITD:
 695                                 printf("SITD");
 696                                 break;
 697                         case EHCI_LINK_FSTN:
 698                                 printf("FSTN");
 699                                 break;
 700                         }
 701                 }
 702                 printf(">");
 703         }
 704 }
 705
 706 static void
 707 ehci_dump_qtd(ehci_softc_t *sc, ehci_qtd_t *qtd)
 708 {
 709         uint32_t s;
 710
 711         printf("  next=");
 712         ehci_dump_link(sc, qtd->qtd_next, 0);
 713         printf(" altnext=");
 714         ehci_dump_link(sc, qtd->qtd_altnext, 0);
 715         printf("\n");
 716         s = hc32toh(sc, qtd->qtd_status);
 717         printf("  status=0x%08x: toggle=%d bytes=0x%x ioc=%d c_page=0x%x\n",
 718             s, EHCI_QTD_GET_TOGGLE(s), EHCI_QTD_GET_BYTES(s),
 719             EHCI_QTD_GET_IOC(s), EHCI_QTD_GET_C_PAGE(s));
 720         printf("    cerr=%d pid=%d stat=%s%s%s%s%s%s%s%s\n",
 721             EHCI_QTD_GET_CERR(s), EHCI_QTD_GET_PID(s),
 722             (s & EHCI_QTD_ACTIVE) ? "ACTIVE" : "NOT_ACTIVE",
 723             (s & EHCI_QTD_HALTED) ? "-HALTED" : "",
 724             (s & EHCI_QTD_BUFERR) ? "-BUFERR" : "",
 725             (s & EHCI_QTD_BABBLE) ? "-BABBLE" : "",
 726             (s & EHCI_QTD_XACTERR) ? "-XACTERR" : "",
 727             (s & EHCI_QTD_MISSEDMICRO) ? "-MISSED" : "",
 728             (s & EHCI_QTD_SPLITXSTATE) ? "-SPLIT" : "",
 729             (s & EHCI_QTD_PINGSTATE) ? "-PING" : "");
 730
 731         for (s = 0; s < 5; s++) {
 732                 printf("  buffer[%d]=0x%08x\n", s,
 733                     hc32toh(sc, qtd->qtd_buffer[s]));
 734         }
 735         for (s = 0; s < 5; s++) {
 736                 printf("  buffer_hi[%d]=0x%08x\n", s,
 737                     hc32toh(sc, qtd->qtd_buffer_hi[s]));
 738         }
 739 }
 740
 741 static uint8_t
 742 ehci_dump_sqtd(ehci_softc_t *sc, ehci_qtd_t *sqtd)
 743 {
 744         uint8_t temp;
 745
 746         usb_pc_cpu_invalidate(sqtd->page_cache);
 747         printf("QTD(%p) at 0x%08x:\n", sqtd, hc32toh(sc, sqtd->qtd_self));
 748         ehci_dump_qtd(sc, sqtd);
 749         temp = (sqtd->qtd_next & htohc32(sc, EHCI_LINK_TERMINATE)) ? 1 : 0;
 750         return (temp);
 751 }
 752
 753 static void
 754 ehci_dump_sqtds(ehci_softc_t *sc, ehci_qtd_t *sqtd)
 755 {
 756         uint16_t i;
 757         uint8_t stop;
 758
 759         stop = 0;
 760         for (i = 0; sqtd && (i < 20) && !stop; sqtd = sqtd->obj_next, i++) {
 761                 stop = ehci_dump_sqtd(sc, sqtd);
 762         }
 763         if (sqtd) {
 764                 printf("dump aborted, too many TDs\n");
 765         }
 766 }
 767
 768 static void
 769 ehci_dump_sqh(ehci_softc_t *sc, ehci_qh_t *qh)
 770 {
 771         uint32_t endp;
 772         uint32_t endphub;
 773
 774         usb_pc_cpu_invalidate(qh->page_cache);
 775         printf("QH(%p) at 0x%08x:\n", qh, hc32toh(sc, qh->qh_self) & ~0x1F);
 776         printf("  link=");
 777         ehci_dump_link(sc, qh->qh_link, 1);
 778         printf("\n");
 779         endp = hc32toh(sc, qh->qh_endp);
 780         printf("  endp=0x%08x\n", endp);
 781         printf("    addr=0x%02x inact=%d endpt=%d eps=%d dtc=%d hrecl=%d\n",
 782             EHCI_QH_GET_ADDR(endp), EHCI_QH_GET_INACT(endp),
 783             EHCI_QH_GET_ENDPT(endp), EHCI_QH_GET_EPS(endp),
 784             EHCI_QH_GET_DTC(endp), EHCI_QH_GET_HRECL(endp));
 785         printf("    mpl=0x%x ctl=%d nrl=%d\n",
 786             EHCI_QH_GET_MPL(endp), EHCI_QH_GET_CTL(endp),
 787             EHCI_QH_GET_NRL(endp));
 788         endphub = hc32toh(sc, qh->qh_endphub);
 789         printf("  endphub=0x%08x\n", endphub);
 790         printf("    smask=0x%02x cmask=0x%02x huba=0x%02x port=%d mult=%d\n",
 791             EHCI_QH_GET_SMASK(endphub), EHCI_QH_GET_CMASK(endphub),
 792             EHCI_QH_GET_HUBA(endphub), EHCI_QH_GET_PORT(endphub),
 793             EHCI_QH_GET_MULT(endphub));
 794         printf("  curqtd=");
 795         ehci_dump_link(sc, qh->qh_curqtd, 0);
 796         printf("\n");
 797         printf("Overlay qTD:\n");
 798         ehci_dump_qtd(sc, (void *)&qh->qh_qtd);
 799 }
 800
 801 static void
 802 ehci_dump_sitd(ehci_softc_t *sc, ehci_sitd_t *sitd)
 803 {
 804         usb_pc_cpu_invalidate(sitd->page_cache);
 805         printf("SITD(%p) at 0x%08x\n", sitd, hc32toh(sc, sitd->sitd_self) & ~0x1F);
 806         printf(" next=0x%08x\n", hc32toh(sc, sitd->sitd_next));
 807         printf(" portaddr=0x%08x dir=%s addr=%d endpt=0x%x port=0x%x huba=0x%x\n",
 808             hc32toh(sc, sitd->sitd_portaddr),
 809             (sitd->sitd_portaddr & htohc32(sc, EHCI_SITD_SET_DIR_IN))
 810             ? "in" : "out",
 811             EHCI_SITD_GET_ADDR(hc32toh(sc, sitd->sitd_portaddr)),
 812             EHCI_SITD_GET_ENDPT(hc32toh(sc, sitd->sitd_portaddr)),
 813             EHCI_SITD_GET_PORT(hc32toh(sc, sitd->sitd_portaddr)),
 814             EHCI_SITD_GET_HUBA(hc32toh(sc, sitd->sitd_portaddr)));
 815         printf(" mask=0x%08x\n", hc32toh(sc, sitd->sitd_mask));
 816         printf(" status=0x%08x <%s> len=0x%x\n", hc32toh(sc, sitd->sitd_status),
 817             (sitd->sitd_status & htohc32(sc, EHCI_SITD_ACTIVE)) ? "ACTIVE" : "",
 818             EHCI_SITD_GET_LEN(hc32toh(sc, sitd->sitd_status)));
 819         printf(" back=0x%08x, bp=0x%08x,0x%08x,0x%08x,0x%08x\n",
 820             hc32toh(sc, sitd->sitd_back),
 821             hc32toh(sc, sitd->sitd_bp[0]),
 822             hc32toh(sc, sitd->sitd_bp[1]),
 823             hc32toh(sc, sitd->sitd_bp_hi[0]),
 824             hc32toh(sc, sitd->sitd_bp_hi[1]));
 825 }
 826
 827 static void
 828 ehci_dump_itd(ehci_softc_t *sc, ehci_itd_t *itd)
 829 {
 830         usb_pc_cpu_invalidate(itd->page_cache);
 831         printf("ITD(%p) at 0x%08x\n", itd, hc32toh(sc, itd->itd_self) & ~0x1F);
 832         printf(" next=0x%08x\n", hc32toh(sc, itd->itd_next));
 833         printf(" status[0]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[0]),
 834             (itd->itd_status[0] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
 835         printf(" status[1]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[1]),
 836             (itd->itd_status[1] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
 837         printf(" status[2]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[2]),
 838             (itd->itd_status[2] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
 839         printf(" status[3]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[3]),
 840             (itd->itd_status[3] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
 841         printf(" status[4]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[4]),
 842             (itd->itd_status[4] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
 843         printf(" status[5]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[5]),
 844             (itd->itd_status[5] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
 845         printf(" status[6]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[6]),
 846             (itd->itd_status[6] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
 847         printf(" status[7]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[7]),
 848             (itd->itd_status[7] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
 849         printf(" bp[0]=0x%08x\n", hc32toh(sc, itd->itd_bp[0]));
 850         printf("  addr=0x%02x; endpt=0x%01x\n",
 851             EHCI_ITD_GET_ADDR(hc32toh(sc, itd->itd_bp[0])),
 852             EHCI_ITD_GET_ENDPT(hc32toh(sc, itd->itd_bp[0])));
 853         printf(" bp[1]=0x%08x\n", hc32toh(sc, itd->itd_bp[1]));
 854         printf(" dir=%s; mpl=0x%02x\n",
 855             (hc32toh(sc, itd->itd_bp[1]) & EHCI_ITD_SET_DIR_IN) ? "in" : "out",
 856             EHCI_ITD_GET_MPL(hc32toh(sc, itd->itd_bp[1])));
 857         printf(" bp[2..6]=0x%08x,0x%08x,0x%08x,0x%08x,0x%08x\n",
 858             hc32toh(sc, itd->itd_bp[2]),
 859             hc32toh(sc, itd->itd_bp[3]),
 860             hc32toh(sc, itd->itd_bp[4]),
 861             hc32toh(sc, itd->itd_bp[5]),
 862             hc32toh(sc, itd->itd_bp[6]));
 863         printf(" bp_hi=0x%08x,0x%08x,0x%08x,0x%08x,\n"
 864             "       0x%08x,0x%08x,0x%08x\n",
 865             hc32toh(sc, itd->itd_bp_hi[0]),
 866             hc32toh(sc, itd->itd_bp_hi[1]),
 867             hc32toh(sc, itd->itd_bp_hi[2]),
 868             hc32toh(sc, itd->itd_bp_hi[3]),
 869             hc32toh(sc, itd->itd_bp_hi[4]),
 870             hc32toh(sc, itd->itd_bp_hi[5]),
 871             hc32toh(sc, itd->itd_bp_hi[6]));
 872 }
 873
 874 static void
 875 ehci_dump_isoc(ehci_softc_t *sc)
 876 {
 877         ehci_itd_t *itd;
 878         ehci_sitd_t *sitd;
 879         uint16_t max = 1000;
 880         uint16_t pos;
 881
 882         pos = (EOREAD4(sc, EHCI_FRINDEX) / 8) &
 883             (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
 884
 885         printf("%s: isochronous dump from frame 0x%03x:\n",
 886             __FUNCTION__, pos);
 887
 888         itd = sc->sc_isoc_hs_p_last[pos];
 889         sitd = sc->sc_isoc_fs_p_last[pos];
 890
 891         while (itd && max && max--) {
 892                 ehci_dump_itd(sc, itd);
 893                 itd = itd->prev;
 894         }
 895
 896         while (sitd && max && max--) {
 897                 ehci_dump_sitd(sc, sitd);
 898                 sitd = sitd->prev;
 899         }
 900 }
 901
 902 #endif
 903
 904 static void
 905 ehci_transfer_intr_enqueue(struct usb_xfer *xfer)
 906 {
 907         /* check for early completion */
 908         if (ehci_check_transfer(xfer)) {
 909                 return;
 910         }
 911         /* put transfer on interrupt queue */
 912         usbd_transfer_enqueue(&xfer->xroot->bus->intr_q, xfer);
 913
 914         /* start timeout, if any */
 915         if (xfer->timeout != 0) {
 916                 usbd_transfer_timeout_ms(xfer, &ehci_timeout, xfer->timeout);
 917         }
 918 }
 919
 920 #define EHCI_APPEND_FS_TD(std,last) (last) = _ehci_append_fs_td(std,last)
 921 static ehci_sitd_t *
 922 _ehci_append_fs_td(ehci_sitd_t *std, ehci_sitd_t *last)
 923 {
 924         DPRINTFN(11, "%p to %p\n", std, last);
 925
 926         /* (sc->sc_bus.mtx) must be locked */
 927
 928         std->next = last->next;
 929         std->sitd_next = last->sitd_next;
 930
 931         std->prev = last;
 932
 933         usb_pc_cpu_flush(std->page_cache);
 934
 935         /*
 936          * the last->next->prev is never followed: std->next->prev = std;
 937          */
 938         last->next = std;
 939         last->sitd_next = std->sitd_self;
 940
 941         usb_pc_cpu_flush(last->page_cache);
 942
 943         return (std);
 944 }
 945
 946 #define EHCI_APPEND_HS_TD(std,last) (last) = _ehci_append_hs_td(std,last)
 947 static ehci_itd_t *
 948 _ehci_append_hs_td(ehci_itd_t *std, ehci_itd_t *last)
 949 {
 950         DPRINTFN(11, "%p to %p\n", std, last);
 951
 952         /* (sc->sc_bus.mtx) must be locked */
 953
 954         std->next = last->next;
 955         std->itd_next = last->itd_next;
 956
 957         std->prev = last;
 958
 959         usb_pc_cpu_flush(std->page_cache);
 960
 961         /*
 962          * the last->next->prev is never followed: std->next->prev = std;
 963          */
 964         last->next = std;
 965         last->itd_next = std->itd_self;
 966
 967         usb_pc_cpu_flush(last->page_cache);
 968
 969         return (std);
 970 }
 971
 972 #define EHCI_APPEND_QH(sqh,last) (last) = _ehci_append_qh(sqh,last)
 973 static ehci_qh_t *
 974 _ehci_append_qh(ehci_qh_t *sqh, ehci_qh_t *last)
 975 {
 976         DPRINTFN(11, "%p to %p\n", sqh, last);
 977
 978         if (sqh->prev != NULL) {
 979                 /* should not happen */
 980                 DPRINTFN(0, "QH already linked!\n");
 981                 return (last);
 982         }
 983         /* (sc->sc_bus.mtx) must be locked */
 984
 985         sqh->next = last->next;
 986         sqh->qh_link = last->qh_link;
 987
 988         sqh->prev = last;
 989
 990         usb_pc_cpu_flush(sqh->page_cache);
 991
 992         /*
 993          * the last->next->prev is never followed: sqh->next->prev = sqh;
 994          */
 995
 996         last->next = sqh;
 997         last->qh_link = sqh->qh_self;
 998
 999         usb_pc_cpu_flush(last->page_cache);
1000
1001         return (sqh);
1002 }
1003
1004 #define EHCI_REMOVE_FS_TD(std,last) (last) = _ehci_remove_fs_td(std,last)
1005 static ehci_sitd_t *
1006 _ehci_remove_fs_td(ehci_sitd_t *std, ehci_sitd_t *last)
1007 {
1008         DPRINTFN(11, "%p from %p\n", std, last);
1009
1010         /* (sc->sc_bus.mtx) must be locked */
1011
1012         std->prev->next = std->next;
1013         std->prev->sitd_next = std->sitd_next;
1014
1015         usb_pc_cpu_flush(std->prev->page_cache);
1016
1017         if (std->next) {
1018                 std->next->prev = std->prev;
1019                 usb_pc_cpu_flush(std->next->page_cache);
1020         }
1021         return ((last == std) ? std->prev : last);
1022 }
1023
1024 #define EHCI_REMOVE_HS_TD(std,last) (last) = _ehci_remove_hs_td(std,last)
1025 static ehci_itd_t *
1026 _ehci_remove_hs_td(ehci_itd_t *std, ehci_itd_t *last)
1027 {
1028         DPRINTFN(11, "%p from %p\n", std, last);
1029
1030         /* (sc->sc_bus.mtx) must be locked */
1031
1032         std->prev->next = std->next;
1033         std->prev->itd_next = std->itd_next;
1034
1035         usb_pc_cpu_flush(std->prev->page_cache);
1036
1037         if (std->next) {
1038                 std->next->prev = std->prev;
1039                 usb_pc_cpu_flush(std->next->page_cache);
1040         }
1041         return ((last == std) ? std->prev : last);
1042 }
1043
1044 #define EHCI_REMOVE_QH(sqh,last) (last) = _ehci_remove_qh(sqh,last)
1045 static ehci_qh_t *
1046 _ehci_remove_qh(ehci_qh_t *sqh, ehci_qh_t *last)
1047 {
1048         DPRINTFN(11, "%p from %p\n", sqh, last);
1049
1050         /* (sc->sc_bus.mtx) must be locked */
1051
1052         /* only remove if not removed from a queue */
1053         if (sqh->prev) {
1054                 sqh->prev->next = sqh->next;
1055                 sqh->prev->qh_link = sqh->qh_link;
1056
1057                 usb_pc_cpu_flush(sqh->prev->page_cache);
1058
1059                 if (sqh->next) {
1060                         sqh->next->prev = sqh->prev;
1061                         usb_pc_cpu_flush(sqh->next->page_cache);
1062                 }
1063                 last = ((last == sqh) ? sqh->prev : last);
1064
1065                 sqh->prev = 0;
1066
1067                 usb_pc_cpu_flush(sqh->page_cache);
1068         }
1069         return (last);
1070 }
1071
1072 static void
1073 ehci_data_toggle_update(struct usb_xfer *xfer, uint16_t actlen, uint16_t xlen)
1074 {
1075         uint16_t rem;
1076         uint8_t dt;
1077
1078         /* count number of full packets */
1079         dt = (actlen / xfer->max_packet_size) & 1;
1080
1081         /* compute remainder */
1082         rem = actlen % xfer->max_packet_size;
1083
1084         if (rem > 0)
1085                 dt ^= 1;        /* short packet at the end */
1086         else if (actlen != xlen)
1087                 dt ^= 1;        /* zero length packet at the end */
1088         else if (xlen == 0)
1089                 dt ^= 1;        /* zero length transfer */
1090
1091         xfer->endpoint->toggle_next ^= dt;
1092 }
1093
1094 static usb_error_t
1095 ehci_non_isoc_done_sub(struct usb_xfer *xfer)
1096 {
1097         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
1098         ehci_qtd_t *td;
1099         ehci_qtd_t *td_alt_next;
1100         uint32_t status;
1101         uint16_t len;
1102
1103         td = xfer->td_transfer_cache;
1104         td_alt_next = td->alt_next;
1105
1106         if (xfer->aframes != xfer->nframes) {
1107                 usbd_xfer_set_frame_len(xfer, xfer->aframes, 0);
1108         }
1109         while (1) {
1110                 usb_pc_cpu_invalidate(td->page_cache);
1111                 status = hc32toh(sc, td->qtd_status);
1112
1113                 len = EHCI_QTD_GET_BYTES(status);
1114
1115                 /*
1116                  * Verify the status length and
1117                  * add the length to "frlengths[]":
1118                  */
1119                 if (len > td->len) {
1120                         /* should not happen */
1121                         DPRINTF("Invalid status length, "
1122                             "0x%04x/0x%04x bytes\n", len, td->len);
1123                         status |= EHCI_QTD_HALTED;
1124                 } else if (xfer->aframes != xfer->nframes) {
1125                         xfer->frlengths[xfer->aframes] += td->len - len;
1126                         /* manually update data toggle */
1127                         ehci_data_toggle_update(xfer, td->len - len, td->len);
1128                 }
1129
1130                 /* Check for last transfer */
1131                 if (((void *)td) == xfer->td_transfer_last) {
1132                         td = NULL;
1133                         break;
1134                 }
1135                 /* Check for transfer error */
1136                 if (status & EHCI_QTD_HALTED) {
1137                         /* the transfer is finished */
1138                         td = NULL;
1139                         break;
1140                 }
1141                 /* Check for short transfer */
1142                 if (len > 0) {
1143                         if (xfer->flags_int.short_frames_ok) {
1144                                 /* follow alt next */
1145                                 td = td->alt_next;
1146                         } else {
1147                                 /* the transfer is finished */
1148                                 td = NULL;
1149                         }
1150                         break;
1151                 }
1152                 td = td->obj_next;
1153
1154                 if (td->alt_next != td_alt_next) {
1155                         /* this USB frame is complete */
1156                         break;
1157                 }
1158         }
1159
1160         /* update transfer cache */
1161
1162         xfer->td_transfer_cache = td;
1163
1164 #ifdef USB_DEBUG
1165         if (status & EHCI_QTD_STATERRS) {
1166                 DPRINTFN(11, "error, addr=%d, endpt=0x%02x, frame=0x%02x"
1167                     "status=%s%s%s%s%s%s%s%s\n",
1168                     xfer->address, xfer->endpointno, xfer->aframes,
1169                     (status & EHCI_QTD_ACTIVE) ? "[ACTIVE]" : "[NOT_ACTIVE]",
1170                     (status & EHCI_QTD_HALTED) ? "[HALTED]" : "",
1171                     (status & EHCI_QTD_BUFERR) ? "[BUFERR]" : "",
1172                     (status & EHCI_QTD_BABBLE) ? "[BABBLE]" : "",
1173                     (status & EHCI_QTD_XACTERR) ? "[XACTERR]" : "",
1174                     (status & EHCI_QTD_MISSEDMICRO) ? "[MISSED]" : "",
1175                     (status & EHCI_QTD_SPLITXSTATE) ? "[SPLIT]" : "",
1176                     (status & EHCI_QTD_PINGSTATE) ? "[PING]" : "");
1177         }
1178 #endif
1179         if (status & EHCI_QTD_HALTED) {
1180                 if ((xfer->xroot->udev->parent_hs_hub != NULL) ||
1181                     (xfer->xroot->udev->address != 0)) {
1182                         /* try to separate I/O errors from STALL */
1183                         if (EHCI_QTD_GET_CERR(status) == 0)
1184                                 return (USB_ERR_IOERROR);
1185                 }
1186                 return (USB_ERR_STALLED);
1187         }
1188         return (USB_ERR_NORMAL_COMPLETION);
1189 }
1190
1191 static void
1192 ehci_non_isoc_done(struct usb_xfer *xfer)
1193 {
1194         ehci_qh_t *qh;
1195         usb_error_t err = 0;
1196
1197         DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
1198             xfer, xfer->endpoint);
1199
1200 #ifdef USB_DEBUG
1201         if (ehcidebug > 10) {
1202                 ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
1203
1204                 ehci_dump_sqtds(sc, xfer->td_transfer_first);
1205         }
1206 #endif
1207
1208         /* extract data toggle directly from the QH's overlay area */
1209
1210         qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
1211
1212         usb_pc_cpu_invalidate(qh->page_cache);
1213
1214         /* reset scanner */
1215
1216         xfer->td_transfer_cache = xfer->td_transfer_first;
1217
1218         if (xfer->flags_int.control_xfr) {
1219                 if (xfer->flags_int.control_hdr) {
1220                         err = ehci_non_isoc_done_sub(xfer);
1221                 }
1222                 xfer->aframes = 1;
1223
1224                 if (xfer->td_transfer_cache == NULL) {
1225                         goto done;
1226                 }
1227         }
1228         while (xfer->aframes != xfer->nframes) {
1229                 err = ehci_non_isoc_done_sub(xfer);
1230                 xfer->aframes++;
1231
1232                 if (xfer->td_transfer_cache == NULL) {
1233                         goto done;
1234                 }
1235         }
1236
1237         if (xfer->flags_int.control_xfr &&
1238             !xfer->flags_int.control_act) {
1239                 err = ehci_non_isoc_done_sub(xfer);
1240         }
1241 done:
1242         ehci_device_done(xfer, err);
1243 }
1244
1245 /*------------------------------------------------------------------------*
1246  *      ehci_check_transfer
1247  *
1248  * Return values:
1249  *    0: USB transfer is not finished
1250  * Else: USB transfer is finished
1251  *------------------------------------------------------------------------*/
1252 static uint8_t
1253 ehci_check_transfer(struct usb_xfer *xfer)
1254 {
1255         const struct usb_pipe_methods *methods = xfer->endpoint->methods;
1256         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
1257
1258         uint32_t status;
1259
1260         DPRINTFN(13, "xfer=%p checking transfer\n", xfer);
1261
1262         if (methods == &ehci_device_isoc_fs_methods) {
1263                 ehci_sitd_t *td;
1264
1265                 /* isochronous full speed transfer */
1266
1267                 td = xfer->td_transfer_last;
1268                 usb_pc_cpu_invalidate(td->page_cache);
1269                 status = hc32toh(sc, td->sitd_status);
1270
1271                 /* also check if first is complete */
1272
1273                 td = xfer->td_transfer_first;
1274                 usb_pc_cpu_invalidate(td->page_cache);
1275                 status |= hc32toh(sc, td->sitd_status);
1276
1277                 if (!(status & EHCI_SITD_ACTIVE)) {
1278                         ehci_device_done(xfer, USB_ERR_NORMAL_COMPLETION);
1279                         goto transferred;
1280                 }
1281         } else if (methods == &ehci_device_isoc_hs_methods) {
1282                 ehci_itd_t *td;
1283
1284                 /* isochronous high speed transfer */
1285
1286                 /* check last transfer */
1287                 td = xfer->td_transfer_last;
1288                 usb_pc_cpu_invalidate(td->page_cache);
1289                 status = td->itd_status[0];
1290                 status |= td->itd_status[1];
1291                 status |= td->itd_status[2];
1292                 status |= td->itd_status[3];
1293                 status |= td->itd_status[4];
1294                 status |= td->itd_status[5];
1295                 status |= td->itd_status[6];
1296                 status |= td->itd_status[7];
1297
1298                 /* also check first transfer */
1299                 td = xfer->td_transfer_first;
1300                 usb_pc_cpu_invalidate(td->page_cache);
1301                 status |= td->itd_status[0];
1302                 status |= td->itd_status[1];
1303                 status |= td->itd_status[2];
1304                 status |= td->itd_status[3];
1305                 status |= td->itd_status[4];
1306                 status |= td->itd_status[5];
1307                 status |= td->itd_status[6];
1308                 status |= td->itd_status[7];
1309
1310                 /* if no transactions are active we continue */
1311                 if (!(status & htohc32(sc, EHCI_ITD_ACTIVE))) {
1312                         ehci_device_done(xfer, USB_ERR_NORMAL_COMPLETION);
1313                         goto transferred;
1314                 }
1315         } else {
1316                 ehci_qtd_t *td;
1317                 ehci_qh_t *qh;
1318
1319                 /* non-isochronous transfer */
1320
1321                 /*
1322                  * check whether there is an error somewhere in the middle,
1323                  * or whether there was a short packet (SPD and not ACTIVE)
1324                  */
1325                 td = xfer->td_transfer_cache;
1326
1327                 qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
1328
1329                 usb_pc_cpu_invalidate(qh->page_cache);
1330
1331                 status = hc32toh(sc, qh->qh_qtd.qtd_status);
1332                 if (status & EHCI_QTD_ACTIVE) {
1333                         /* transfer is pending */
1334                         goto done;
1335                 }
1336
1337                 while (1) {
1338                         usb_pc_cpu_invalidate(td->page_cache);
1339                         status = hc32toh(sc, td->qtd_status);
1340
1341                         /*
1342                          * Check if there is an active TD which
1343                          * indicates that the transfer isn't done.
1344                          */
1345                         if (status & EHCI_QTD_ACTIVE) {
1346                                 /* update cache */
1347                                 xfer->td_transfer_cache = td;
1348                                 goto done;
1349                         }
1350                         /*
1351                          * last transfer descriptor makes the transfer done
1352                          */
1353                         if (((void *)td) == xfer->td_transfer_last) {
1354                                 break;
1355                         }
1356                         /*
1357                          * any kind of error makes the transfer done
1358                          */
1359                         if (status & EHCI_QTD_HALTED) {
1360                                 break;
1361                         }
1362                         /*
1363                          * if there is no alternate next transfer, a short
1364                          * packet also makes the transfer done
1365                          */
1366                         if (EHCI_QTD_GET_BYTES(status)) {
1367                                 if (xfer->flags_int.short_frames_ok) {
1368                                         /* follow alt next */
1369                                         if (td->alt_next) {
1370                                                 td = td->alt_next;
1371                                                 continue;
1372                                         }
1373                                 }
1374                                 /* transfer is done */
1375                                 break;
1376                         }
1377                         td = td->obj_next;
1378                 }
1379                 ehci_non_isoc_done(xfer);
1380                 goto transferred;
1381         }
1382
1383 done:
1384         DPRINTFN(13, "xfer=%p is still active\n", xfer);
1385         return (0);
1386
1387 transferred:
1388         return (1);
1389 }
1390
1391 static void
1392 ehci_pcd_enable(ehci_softc_t *sc)
1393 {
1394         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
1395
1396         sc->sc_eintrs |= EHCI_STS_PCD;
1397         EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
1398
1399         /* acknowledge any PCD interrupt */
1400         EOWRITE4(sc, EHCI_USBSTS, EHCI_STS_PCD);
1401
1402         ehci_root_intr(sc);
1403 }
1404
1405 static void
1406 ehci_interrupt_poll(ehci_softc_t *sc)
1407 {
1408         struct usb_xfer *xfer;
1409
1410 repeat:
1411         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
1412                 /*
1413                  * check if transfer is transferred
1414                  */
1415                 if (ehci_check_transfer(xfer)) {
1416                         /* queue has been modified */
1417                         goto repeat;
1418                 }
1419         }
1420 }
1421
1422 /*
1423  * Some EHCI chips from VIA / ATI seem to trigger interrupts before
1424  * writing back the qTD status, or miss signalling occasionally under
1425  * heavy load.  If the host machine is too fast, we can miss
1426  * transaction completion - when we scan the active list the
1427  * transaction still seems to be active. This generally exhibits
1428  * itself as a umass stall that never recovers.
1429  *
1430  * We work around this behaviour by setting up this callback after any
1431  * softintr that completes with transactions still pending, giving us
1432  * another chance to check for completion after the writeback has
1433  * taken place.
1434  */
1435 static void
1436 ehci_poll_timeout(void *arg)
1437 {
1438         ehci_softc_t *sc = arg;
1439
1440         DPRINTFN(3, "\n");
1441         ehci_interrupt_poll(sc);
1442 }
1443
1444 /*------------------------------------------------------------------------*
1445  *      ehci_interrupt - EHCI interrupt handler
1446  *
1447  * NOTE: Do not access "sc->sc_bus.bdev" inside the interrupt handler,
1448  * hence the interrupt handler will be setup before "sc->sc_bus.bdev"
1449  * is present !
1450  *------------------------------------------------------------------------*/
1451 void
1452 ehci_interrupt(ehci_softc_t *sc)
1453 {
1454         uint32_t status;
1455
1456         USB_BUS_LOCK(&sc->sc_bus);
1457
1458         DPRINTFN(16, "real interrupt\n");
1459
1460 #ifdef USB_DEBUG
1461         if (ehcidebug > 15) {
1462                 ehci_dump_regs(sc);
1463         }
1464 #endif
1465
1466         status = EHCI_STS_INTRS(EOREAD4(sc, EHCI_USBSTS));
1467         if (status == 0) {
1468                 /* the interrupt was not for us */
1469                 goto done;
1470         }
1471         if (!(status & sc->sc_eintrs)) {
1472                 goto done;
1473         }
1474         EOWRITE4(sc, EHCI_USBSTS, status);      /* acknowledge */
1475
1476         status &= sc->sc_eintrs;
1477
1478         if (status & EHCI_STS_HSE) {
1479                 printf("%s: unrecoverable error, "
1480                     "controller halted\n", __FUNCTION__);
1481 #ifdef USB_DEBUG
1482                 ehci_dump_regs(sc);
1483                 ehci_dump_isoc(sc);
1484 #endif
1485         }
1486         if (status & EHCI_STS_PCD) {
1487                 /*
1488                  * Disable PCD interrupt for now, because it will be
1489                  * on until the port has been reset.
1490                  */
1491                 sc->sc_eintrs &= ~EHCI_STS_PCD;
1492                 EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
1493
1494                 ehci_root_intr(sc);
1495
1496                 /* do not allow RHSC interrupts > 1 per second */
1497                 usb_callout_reset(&sc->sc_tmo_pcd, hz,
1498                     (void *)&ehci_pcd_enable, sc);
1499         }
1500         status &= ~(EHCI_STS_INT | EHCI_STS_ERRINT | EHCI_STS_PCD | EHCI_STS_IAA);
1501
1502         if (status != 0) {
1503                 /* block unprocessed interrupts */
1504                 sc->sc_eintrs &= ~status;
1505                 EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
1506                 printf("%s: blocking interrupts 0x%x\n", __FUNCTION__, status);
1507         }
1508         /* poll all the USB transfers */
1509         ehci_interrupt_poll(sc);
1510
1511         if (sc->sc_flags & EHCI_SCFLG_LOSTINTRBUG) {
1512                 usb_callout_reset(&sc->sc_tmo_poll, hz / 128,
1513                     (void *)&ehci_poll_timeout, sc);
1514         }
1515
1516 done:
1517         USB_BUS_UNLOCK(&sc->sc_bus);
1518 }
1519
1520 /*
1521  * called when a request does not complete
1522  */
1523 static void
1524 ehci_timeout(void *arg)
1525 {
1526         struct usb_xfer *xfer = arg;
1527
1528         DPRINTF("xfer=%p\n", xfer);
1529
1530         USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
1531
1532         /* transfer is transferred */
1533         ehci_device_done(xfer, USB_ERR_TIMEOUT);
1534 }
1535
1536 static void
1537 ehci_do_poll(struct usb_bus *bus)
1538 {
1539         ehci_softc_t *sc = EHCI_BUS2SC(bus);
1540
1541         USB_BUS_LOCK(&sc->sc_bus);
1542         ehci_interrupt_poll(sc);
1543         USB_BUS_UNLOCK(&sc->sc_bus);
1544 }
1545
1546 static void
1547 ehci_setup_standard_chain_sub(struct ehci_std_temp *temp)
1548 {
1549         struct usb_page_search buf_res;
1550         ehci_qtd_t *td;
1551         ehci_qtd_t *td_next;
1552         ehci_qtd_t *td_alt_next;
1553         uint32_t buf_offset;
1554         uint32_t average;
1555         uint32_t len_old;
1556         uint32_t terminate;
1557         uint32_t qtd_altnext;
1558         uint8_t shortpkt_old;
1559         uint8_t precompute;
1560
1561         terminate = temp->sc->sc_terminate_self;
1562         qtd_altnext = temp->sc->sc_terminate_self;
1563         td_alt_next = NULL;
1564         buf_offset = 0;
1565         shortpkt_old = temp->shortpkt;
1566         len_old = temp->len;
1567         precompute = 1;
1568
1569 restart:
1570
1571         td = temp->td;
1572         td_next = temp->td_next;
1573
1574         while (1) {
1575                 if (temp->len == 0) {
1576                         if (temp->shortpkt) {
1577                                 break;
1578                         }
1579                         /* send a Zero Length Packet, ZLP, last */
1580
1581                         temp->shortpkt = 1;
1582                         average = 0;
1583
1584                 } else {
1585                         average = temp->average;
1586
1587                         if (temp->len < average) {
1588                                 if (temp->len % temp->max_frame_size) {
1589                                         temp->shortpkt = 1;
1590                                 }
1591                                 average = temp->len;
1592                         }
1593                 }
1594
1595                 if (td_next == NULL) {
1596                         panic("%s: out of EHCI transfer descriptors!", __FUNCTION__);
1597                 }
1598                 /* get next TD */
1599
1600                 td = td_next;
1601                 td_next = td->obj_next;
1602
1603                 /* check if we are pre-computing */
1604
1605                 if (precompute) {
1606                         /* update remaining length */
1607
1608                         temp->len -= average;
1609
1610                         continue;
1611                 }
1612                 /* fill out current TD */
1613
1614                 td->qtd_status =
1615                     temp->qtd_status |
1616                     htohc32(temp->sc, EHCI_QTD_IOC |
1617                         EHCI_QTD_SET_BYTES(average));
1618
1619                 if (average == 0) {
1620                         if (temp->auto_data_toggle == 0) {
1621                                 /* update data toggle, ZLP case */
1622
1623                                 temp->qtd_status ^=
1624                                     htohc32(temp->sc, EHCI_QTD_TOGGLE_MASK);
1625                         }
1626                         td->len = 0;
1627
1628                         /* properly reset reserved fields */
1629                         td->qtd_buffer[0] = 0;
1630                         td->qtd_buffer[1] = 0;
1631                         td->qtd_buffer[2] = 0;
1632                         td->qtd_buffer[3] = 0;
1633                         td->qtd_buffer[4] = 0;
1634                         td->qtd_buffer_hi[0] = 0;
1635                         td->qtd_buffer_hi[1] = 0;
1636                         td->qtd_buffer_hi[2] = 0;
1637                         td->qtd_buffer_hi[3] = 0;
1638                         td->qtd_buffer_hi[4] = 0;
1639                 } else {
1640                         uint8_t x;
1641
1642                         if (temp->auto_data_toggle == 0) {
1643                                 /* update data toggle */
1644
1645                                 if (howmany(average, temp->max_frame_size) & 1) {
1646                                         temp->qtd_status ^=
1647                                             htohc32(temp->sc, EHCI_QTD_TOGGLE_MASK);
1648                                 }
1649                         }
1650                         td->len = average;
1651
1652                         /* update remaining length */
1653
1654                         temp->len -= average;
1655
1656                         /* fill out buffer pointers */
1657
1658                         usbd_get_page(temp->pc, buf_offset, &buf_res);
1659                         td->qtd_buffer[0] =
1660                             htohc32(temp->sc, buf_res.physaddr);
1661                         td->qtd_buffer_hi[0] = 0;
1662
1663                         x = 1;
1664
1665                         while (average > EHCI_PAGE_SIZE) {
1666                                 average -= EHCI_PAGE_SIZE;
1667                                 buf_offset += EHCI_PAGE_SIZE;
1668                                 usbd_get_page(temp->pc, buf_offset, &buf_res);
1669                                 td->qtd_buffer[x] =
1670                                     htohc32(temp->sc,
1671                                     buf_res.physaddr & (~0xFFF));
1672                                 td->qtd_buffer_hi[x] = 0;
1673                                 x++;
1674                         }
1675
1676                         /*
1677                          * NOTE: The "average" variable is never zero after
1678                          * exiting the loop above !
1679                          *
1680                          * NOTE: We have to subtract one from the offset to
1681                          * ensure that we are computing the physical address
1682                          * of a valid page !
1683                          */
1684                         buf_offset += average;
1685                         usbd_get_page(temp->pc, buf_offset - 1, &buf_res);
1686                         td->qtd_buffer[x] =
1687                             htohc32(temp->sc,
1688                             buf_res.physaddr & (~0xFFF));
1689                         td->qtd_buffer_hi[x] = 0;
1690
1691                         /* properly reset reserved fields */
1692                         while (++x < EHCI_QTD_NBUFFERS) {
1693                                 td->qtd_buffer[x] = 0;
1694                                 td->qtd_buffer_hi[x] = 0;
1695                         }
1696                 }
1697
1698                 if (td_next) {
1699                         /* link the current TD with the next one */
1700                         td->qtd_next = td_next->qtd_self;
1701                 }
1702                 td->qtd_altnext = qtd_altnext;
1703                 td->alt_next = td_alt_next;
1704
1705                 usb_pc_cpu_flush(td->page_cache);
1706         }
1707
1708         if (precompute) {
1709                 precompute = 0;
1710
1711                 /* setup alt next pointer, if any */
1712                 if (temp->last_frame) {
1713                         td_alt_next = NULL;
1714                         qtd_altnext = terminate;
1715                 } else {
1716                         /* we use this field internally */
1717                         td_alt_next = td_next;
1718                         if (temp->setup_alt_next) {
1719                                 qtd_altnext = td_next->qtd_self;
1720                         } else {
1721                                 qtd_altnext = terminate;
1722                         }
1723                 }
1724
1725                 /* restore */
1726                 temp->shortpkt = shortpkt_old;
1727                 temp->len = len_old;
1728                 goto restart;
1729         }
1730         temp->td = td;
1731         temp->td_next = td_next;
1732 }
1733
1734 static void
1735 ehci_setup_standard_chain(struct usb_xfer *xfer, ehci_qh_t **qh_last)
1736 {
1737         struct ehci_std_temp temp;
1738         const struct usb_pipe_methods *methods;
1739         ehci_qh_t *qh;
1740         ehci_qtd_t *td;
1741         uint32_t qh_endp;
1742         uint32_t qh_endphub;
1743         uint32_t x;
1744
1745         DPRINTFN(9, "addr=%d endpt=%d sumlen=%d speed=%d\n",
1746             xfer->address, UE_GET_ADDR(xfer->endpointno),
1747             xfer->sumlen, usbd_get_speed(xfer->xroot->udev));
1748
1749         temp.average = xfer->max_hc_frame_size;
1750         temp.max_frame_size = xfer->max_frame_size;
1751         temp.sc = EHCI_BUS2SC(xfer->xroot->bus);
1752
1753         /* toggle the DMA set we are using */
1754         xfer->flags_int.curr_dma_set ^= 1;
1755
1756         /* get next DMA set */
1757         td = xfer->td_start[xfer->flags_int.curr_dma_set];
1758
1759         xfer->td_transfer_first = td;
1760         xfer->td_transfer_cache = td;
1761
1762         temp.td = NULL;
1763         temp.td_next = td;
1764         temp.qtd_status = 0;
1765         temp.last_frame = 0;
1766         temp.setup_alt_next = xfer->flags_int.short_frames_ok;
1767
1768         if (xfer->flags_int.control_xfr) {
1769                 if (xfer->endpoint->toggle_next) {
1770                         /* DATA1 is next */
1771                         temp.qtd_status |=
1772                             htohc32(temp.sc, EHCI_QTD_SET_TOGGLE(1));
1773                 }
1774                 temp.auto_data_toggle = 0;
1775         } else {
1776                 temp.auto_data_toggle = 1;
1777         }
1778
1779         if ((xfer->xroot->udev->parent_hs_hub != NULL) ||
1780             (xfer->xroot->udev->address != 0)) {
1781                 /* max 3 retries */
1782                 temp.qtd_status |=
1783                     htohc32(temp.sc, EHCI_QTD_SET_CERR(3));
1784         }
1785         /* check if we should prepend a setup message */
1786
1787         if (xfer->flags_int.control_xfr) {
1788                 if (xfer->flags_int.control_hdr) {
1789                         xfer->endpoint->toggle_next = 0;
1790
1791                         temp.qtd_status &=
1792                             htohc32(temp.sc, EHCI_QTD_SET_CERR(3));
1793                         temp.qtd_status |= htohc32(temp.sc,
1794                             EHCI_QTD_ACTIVE |
1795                             EHCI_QTD_SET_PID(EHCI_QTD_PID_SETUP) |
1796                             EHCI_QTD_SET_TOGGLE(0));
1797
1798                         temp.len = xfer->frlengths[0];
1799                         temp.pc = xfer->frbuffers + 0;
1800                         temp.shortpkt = temp.len ? 1 : 0;
1801                         /* check for last frame */
1802                         if (xfer->nframes == 1) {
1803                                 /* no STATUS stage yet, SETUP is last */
1804                                 if (xfer->flags_int.control_act) {
1805                                         temp.last_frame = 1;
1806                                         temp.setup_alt_next = 0;
1807                                 }
1808                         }
1809                         ehci_setup_standard_chain_sub(&temp);
1810                 }
1811                 x = 1;
1812         } else {
1813                 x = 0;
1814         }
1815
1816         while (x != xfer->nframes) {
1817                 /* DATA0 / DATA1 message */
1818
1819                 temp.len = xfer->frlengths[x];
1820                 temp.pc = xfer->frbuffers + x;
1821
1822                 x++;
1823
1824                 if (x == xfer->nframes) {
1825                         if (xfer->flags_int.control_xfr) {
1826                                 /* no STATUS stage yet, DATA is last */
1827                                 if (xfer->flags_int.control_act) {
1828                                         temp.last_frame = 1;
1829                                         temp.setup_alt_next = 0;
1830                                 }
1831                         } else {
1832                                 temp.last_frame = 1;
1833                                 temp.setup_alt_next = 0;
1834                         }
1835                 }
1836                 /* keep previous data toggle and error count */
1837
1838                 temp.qtd_status &=
1839                     htohc32(temp.sc, EHCI_QTD_SET_CERR(3) |
1840                     EHCI_QTD_SET_TOGGLE(1));
1841
1842                 if (temp.len == 0) {
1843                         /* make sure that we send an USB packet */
1844
1845                         temp.shortpkt = 0;
1846
1847                 } else {
1848                         /* regular data transfer */
1849
1850                         temp.shortpkt = (xfer->flags.force_short_xfer) ? 0 : 1;
1851                 }
1852
1853                 /* set endpoint direction */
1854
1855                 temp.qtd_status |=
1856                     (UE_GET_DIR(xfer->endpointno) == UE_DIR_IN) ?
1857                     htohc32(temp.sc, EHCI_QTD_ACTIVE |
1858                     EHCI_QTD_SET_PID(EHCI_QTD_PID_IN)) :
1859                     htohc32(temp.sc, EHCI_QTD_ACTIVE |
1860                     EHCI_QTD_SET_PID(EHCI_QTD_PID_OUT));
1861
1862                 ehci_setup_standard_chain_sub(&temp);
1863         }
1864
1865         /* check if we should append a status stage */
1866
1867         if (xfer->flags_int.control_xfr &&
1868             !xfer->flags_int.control_act) {
1869                 /*
1870                  * Send a DATA1 message and invert the current endpoint
1871                  * direction.
1872                  */
1873
1874                 temp.qtd_status &= htohc32(temp.sc, EHCI_QTD_SET_CERR(3) |
1875                     EHCI_QTD_SET_TOGGLE(1));
1876                 temp.qtd_status |=
1877                     (UE_GET_DIR(xfer->endpointno) == UE_DIR_OUT) ?
1878                     htohc32(temp.sc, EHCI_QTD_ACTIVE |
1879                     EHCI_QTD_SET_PID(EHCI_QTD_PID_IN) |
1880                     EHCI_QTD_SET_TOGGLE(1)) :
1881                     htohc32(temp.sc, EHCI_QTD_ACTIVE |
1882                     EHCI_QTD_SET_PID(EHCI_QTD_PID_OUT) |
1883                     EHCI_QTD_SET_TOGGLE(1));
1884
1885                 temp.len = 0;
1886                 temp.pc = NULL;
1887                 temp.shortpkt = 0;
1888                 temp.last_frame = 1;
1889                 temp.setup_alt_next = 0;
1890
1891                 ehci_setup_standard_chain_sub(&temp);
1892         }
1893         td = temp.td;
1894
1895         /* the last TD terminates the transfer: */
1896         td->qtd_next = htohc32(temp.sc, EHCI_LINK_TERMINATE);
1897         td->qtd_altnext = htohc32(temp.sc, EHCI_LINK_TERMINATE);
1898
1899         usb_pc_cpu_flush(td->page_cache);
1900
1901         /* must have at least one frame! */
1902
1903         xfer->td_transfer_last = td;
1904
1905 #ifdef USB_DEBUG
1906         if (ehcidebug > 8) {
1907                 DPRINTF("nexttog=%d; data before transfer:\n",
1908                     xfer->endpoint->toggle_next);
1909                 ehci_dump_sqtds(temp.sc,
1910                     xfer->td_transfer_first);
1911         }
1912 #endif
1913
1914         methods = xfer->endpoint->methods;
1915
1916         qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
1917
1918         /* the "qh_link" field is filled when the QH is added */
1919
1920         qh_endp =
1921             (EHCI_QH_SET_ADDR(xfer->address) |
1922             EHCI_QH_SET_ENDPT(UE_GET_ADDR(xfer->endpointno)) |
1923             EHCI_QH_SET_MPL(xfer->max_packet_size));
1924
1925         if (usbd_get_speed(xfer->xroot->udev) == USB_SPEED_HIGH) {
1926                 qh_endp |= EHCI_QH_SET_EPS(EHCI_QH_SPEED_HIGH);
1927                 if (methods != &ehci_device_intr_methods)
1928                         qh_endp |= EHCI_QH_SET_NRL(8);
1929         } else {
1930                 if (usbd_get_speed(xfer->xroot->udev) == USB_SPEED_FULL) {
1931                         qh_endp |= EHCI_QH_SET_EPS(EHCI_QH_SPEED_FULL);
1932                 } else {
1933                         qh_endp |= EHCI_QH_SET_EPS(EHCI_QH_SPEED_LOW);
1934                 }
1935
1936                 if (methods == &ehci_device_ctrl_methods) {
1937                         qh_endp |= EHCI_QH_CTL;
1938                 }
1939                 if (methods != &ehci_device_intr_methods) {
1940                         /* Only try one time per microframe! */
1941                         qh_endp |= EHCI_QH_SET_NRL(1);
1942                 }
1943         }
1944
1945         if (temp.auto_data_toggle == 0) {
1946                 /* software computes the data toggle */
1947                 qh_endp |= EHCI_QH_DTC;
1948         }
1949
1950         qh->qh_endp = htohc32(temp.sc, qh_endp);
1951
1952         qh_endphub =
1953             (EHCI_QH_SET_MULT(xfer->max_packet_count & 3) |
1954             EHCI_QH_SET_CMASK(xfer->endpoint->usb_cmask) |
1955             EHCI_QH_SET_SMASK(xfer->endpoint->usb_smask) |
1956             EHCI_QH_SET_HUBA(xfer->xroot->udev->hs_hub_addr) |
1957             EHCI_QH_SET_PORT(xfer->xroot->udev->hs_port_no));
1958
1959         qh->qh_endphub = htohc32(temp.sc, qh_endphub);
1960         qh->qh_curqtd = 0;
1961
1962         /* fill the overlay qTD */
1963
1964         if (temp.auto_data_toggle && xfer->endpoint->toggle_next) {
1965                 /* DATA1 is next */
1966                 qh->qh_qtd.qtd_status = htohc32(temp.sc, EHCI_QTD_SET_TOGGLE(1));
1967         } else {
1968                 qh->qh_qtd.qtd_status = 0;
1969         }
1970
1971         td = xfer->td_transfer_first;
1972
1973         qh->qh_qtd.qtd_next = td->qtd_self;
1974         qh->qh_qtd.qtd_altnext =
1975             htohc32(temp.sc, EHCI_LINK_TERMINATE);
1976
1977         /* properly reset reserved fields */
1978         qh->qh_qtd.qtd_buffer[0] = 0;
1979         qh->qh_qtd.qtd_buffer[1] = 0;
1980         qh->qh_qtd.qtd_buffer[2] = 0;
1981         qh->qh_qtd.qtd_buffer[3] = 0;
1982         qh->qh_qtd.qtd_buffer[4] = 0;
1983         qh->qh_qtd.qtd_buffer_hi[0] = 0;
1984         qh->qh_qtd.qtd_buffer_hi[1] = 0;
1985         qh->qh_qtd.qtd_buffer_hi[2] = 0;
1986         qh->qh_qtd.qtd_buffer_hi[3] = 0;
1987         qh->qh_qtd.qtd_buffer_hi[4] = 0;
1988
1989         usb_pc_cpu_flush(qh->page_cache);
1990
1991         if (xfer->xroot->udev->flags.self_suspended == 0) {
1992                 EHCI_APPEND_QH(qh, *qh_last);
1993         }
1994 }
1995
1996 static void
1997 ehci_root_intr(ehci_softc_t *sc)
1998 {
1999         uint16_t i;
2000         uint16_t m;
2001
2002         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
2003
2004         /* clear any old interrupt data */
2005         memset(sc->sc_hub_idata, 0, sizeof(sc->sc_hub_idata));
2006
2007         /* set bits */
2008         m = (sc->sc_noport + 1);
2009         if (m > (8 * sizeof(sc->sc_hub_idata))) {
2010                 m = (8 * sizeof(sc->sc_hub_idata));
2011         }
2012         for (i = 1; i < m; i++) {
2013                 /* pick out CHANGE bits from the status register */
2014                 if (EOREAD4(sc, EHCI_PORTSC(i)) & EHCI_PS_CLEAR) {
2015                         sc->sc_hub_idata[i / 8] |= 1 << (i % 8);
2016                         DPRINTF("port %d changed\n", i);
2017                 }
2018         }
2019         uhub_root_intr(&sc->sc_bus, sc->sc_hub_idata,
2020             sizeof(sc->sc_hub_idata));
2021 }
2022
2023 static void
2024 ehci_isoc_fs_done(ehci_softc_t *sc, struct usb_xfer *xfer)
2025 {
2026         uint32_t nframes = xfer->nframes;
2027         uint32_t status;
2028         uint32_t *plen = xfer->frlengths;
2029         uint16_t len = 0;
2030         ehci_sitd_t *td = xfer->td_transfer_first;
2031         ehci_sitd_t **pp_last = &sc->sc_isoc_fs_p_last[xfer->qh_pos];
2032
2033         DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
2034             xfer, xfer->endpoint);
2035
2036         while (nframes--) {
2037                 if (td == NULL) {
2038                         panic("%s:%d: out of TD's\n",
2039                             __FUNCTION__, __LINE__);
2040                 }
2041                 if (pp_last >= &sc->sc_isoc_fs_p_last[EHCI_VIRTUAL_FRAMELIST_COUNT]) {
2042                         pp_last = &sc->sc_isoc_fs_p_last[0];
2043                 }
2044 #ifdef USB_DEBUG
2045                 if (ehcidebug > 15) {
2046                         DPRINTF("isoc FS-TD\n");
2047                         ehci_dump_sitd(sc, td);
2048                 }
2049 #endif
2050                 usb_pc_cpu_invalidate(td->page_cache);
2051                 status = hc32toh(sc, td->sitd_status);
2052
2053                 len = EHCI_SITD_GET_LEN(status);
2054
2055                 DPRINTFN(2, "status=0x%08x, rem=%u\n", status, len);
2056
2057                 if (*plen >= len) {
2058                         len = *plen - len;
2059                 } else {
2060                         len = 0;
2061                 }
2062
2063                 *plen = len;
2064
2065                 /* remove FS-TD from schedule */
2066                 EHCI_REMOVE_FS_TD(td, *pp_last);
2067
2068                 pp_last++;
2069                 plen++;
2070                 td = td->obj_next;
2071         }
2072
2073         xfer->aframes = xfer->nframes;
2074 }
2075
2076 static void
2077 ehci_isoc_hs_done(ehci_softc_t *sc, struct usb_xfer *xfer)
2078 {
2079         uint32_t nframes = xfer->nframes;
2080         uint32_t status;
2081         uint32_t *plen = xfer->frlengths;
2082         uint16_t len = 0;
2083         uint8_t td_no = 0;
2084         ehci_itd_t *td = xfer->td_transfer_first;
2085         ehci_itd_t **pp_last = &sc->sc_isoc_hs_p_last[xfer->qh_pos];
2086
2087         DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
2088             xfer, xfer->endpoint);
2089
2090         while (nframes) {
2091                 if (td == NULL) {
2092                         panic("%s:%d: out of TD's\n",
2093                             __FUNCTION__, __LINE__);
2094                 }
2095                 if (pp_last >= &sc->sc_isoc_hs_p_last[EHCI_VIRTUAL_FRAMELIST_COUNT]) {
2096                         pp_last = &sc->sc_isoc_hs_p_last[0];
2097                 }
2098 #ifdef USB_DEBUG
2099                 if (ehcidebug > 15) {
2100                         DPRINTF("isoc HS-TD\n");
2101                         ehci_dump_itd(sc, td);
2102                 }
2103 #endif
2104
2105                 usb_pc_cpu_invalidate(td->page_cache);
2106                 status = hc32toh(sc, td->itd_status[td_no]);
2107
2108                 len = EHCI_ITD_GET_LEN(status);
2109
2110                 DPRINTFN(2, "status=0x%08x, len=%u\n", status, len);
2111
2112                 if (xfer->endpoint->usb_smask & (1 << td_no)) {
2113                         if (*plen >= len) {
2114                                 /*
2115                                  * The length is valid. NOTE: The
2116                                  * complete length is written back
2117                                  * into the status field, and not the
2118                                  * remainder like with other transfer
2119                                  * descriptor types.
2120                                  */
2121                         } else {
2122                                 /* Invalid length - truncate */
2123                                 len = 0;
2124                         }
2125
2126                         *plen = len;
2127                         plen++;
2128                         nframes--;
2129                 }
2130
2131                 td_no++;
2132
2133                 if ((td_no == 8) || (nframes == 0)) {
2134                         /* remove HS-TD from schedule */
2135                         EHCI_REMOVE_HS_TD(td, *pp_last);
2136                         pp_last++;
2137
2138                         td_no = 0;
2139                         td = td->obj_next;
2140                 }
2141         }
2142         xfer->aframes = xfer->nframes;
2143 }
2144
2145 /* NOTE: "done" can be run two times in a row,
2146  * from close and from interrupt
2147  */
2148 static void
2149 ehci_device_done(struct usb_xfer *xfer, usb_error_t error)
2150 {
2151         const struct usb_pipe_methods *methods = xfer->endpoint->methods;
2152         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2153
2154         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
2155
2156         DPRINTFN(2, "xfer=%p, endpoint=%p, error=%d\n",
2157             xfer, xfer->endpoint, error);
2158
2159         if ((methods == &ehci_device_bulk_methods) ||
2160             (methods == &ehci_device_ctrl_methods)) {
2161 #ifdef USB_DEBUG
2162                 if (ehcidebug > 8) {
2163                         DPRINTF("nexttog=%d; data after transfer:\n",
2164                             xfer->endpoint->toggle_next);
2165                         ehci_dump_sqtds(sc,
2166                             xfer->td_transfer_first);
2167                 }
2168 #endif
2169
2170                 EHCI_REMOVE_QH(xfer->qh_start[xfer->flags_int.curr_dma_set],
2171                     sc->sc_async_p_last);
2172         }
2173         if (methods == &ehci_device_intr_methods) {
2174                 EHCI_REMOVE_QH(xfer->qh_start[xfer->flags_int.curr_dma_set],
2175                     sc->sc_intr_p_last[xfer->qh_pos]);
2176         }
2177         /*
2178          * Only finish isochronous transfers once which will update
2179          * "xfer->frlengths".
2180          */
2181         if (xfer->td_transfer_first &&
2182             xfer->td_transfer_last) {
2183                 if (methods == &ehci_device_isoc_fs_methods) {
2184                         ehci_isoc_fs_done(sc, xfer);
2185                 }
2186                 if (methods == &ehci_device_isoc_hs_methods) {
2187                         ehci_isoc_hs_done(sc, xfer);
2188                 }
2189                 xfer->td_transfer_first = NULL;
2190                 xfer->td_transfer_last = NULL;
2191         }
2192         /* dequeue transfer and start next transfer */
2193         usbd_transfer_done(xfer, error);
2194 }
2195
2196 /*------------------------------------------------------------------------*
2197  * ehci bulk support
2198  *------------------------------------------------------------------------*/
2199 static void
2200 ehci_device_bulk_open(struct usb_xfer *xfer)
2201 {
2202         return;
2203 }
2204
2205 static void
2206 ehci_device_bulk_close(struct usb_xfer *xfer)
2207 {
2208         ehci_device_done(xfer, USB_ERR_CANCELLED);
2209 }
2210
2211 static void
2212 ehci_device_bulk_enter(struct usb_xfer *xfer)
2213 {
2214         return;
2215 }
2216
2217 static void
2218 ehci_doorbell_async(struct ehci_softc *sc)
2219 {
2220         uint32_t temp;
2221
2222         /*
2223          * XXX Performance quirk: Some Host Controllers have a too low
2224          * interrupt rate. Issue an IAAD to stimulate the Host
2225          * Controller after queueing the BULK transfer.
2226          *
2227          * XXX Force the host controller to refresh any QH caches.
2228          */
2229         temp = EOREAD4(sc, EHCI_USBCMD);
2230         if (!(temp & EHCI_CMD_IAAD))
2231                 EOWRITE4(sc, EHCI_USBCMD, temp | EHCI_CMD_IAAD);
2232 }
2233
2234 static void
2235 ehci_device_bulk_start(struct usb_xfer *xfer)
2236 {
2237         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2238
2239         /* setup TD's and QH */
2240         ehci_setup_standard_chain(xfer, &sc->sc_async_p_last);
2241
2242         /* put transfer on interrupt queue */
2243         ehci_transfer_intr_enqueue(xfer);
2244
2245         /*
2246          * XXX Certain nVidia chipsets choke when using the IAAD
2247          * feature too frequently.
2248          */
2249         if (sc->sc_flags & EHCI_SCFLG_IAADBUG)
2250                 return;
2251
2252         ehci_doorbell_async(sc);
2253 }
2254
2255 static const struct usb_pipe_methods ehci_device_bulk_methods =
2256 {
2257         .open = ehci_device_bulk_open,
2258         .close = ehci_device_bulk_close,
2259         .enter = ehci_device_bulk_enter,
2260         .start = ehci_device_bulk_start,
2261 };
2262
2263 /*------------------------------------------------------------------------*
2264  * ehci control support
2265  *------------------------------------------------------------------------*/
2266 static void
2267 ehci_device_ctrl_open(struct usb_xfer *xfer)
2268 {
2269         return;
2270 }
2271
2272 static void
2273 ehci_device_ctrl_close(struct usb_xfer *xfer)
2274 {
2275         ehci_device_done(xfer, USB_ERR_CANCELLED);
2276 }
2277
2278 static void
2279 ehci_device_ctrl_enter(struct usb_xfer *xfer)
2280 {
2281         return;
2282 }
2283
2284 static void
2285 ehci_device_ctrl_start(struct usb_xfer *xfer)
2286 {
2287         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2288
2289         /* setup TD's and QH */
2290         ehci_setup_standard_chain(xfer, &sc->sc_async_p_last);
2291
2292         /* put transfer on interrupt queue */
2293         ehci_transfer_intr_enqueue(xfer);
2294 }
2295
2296 static const struct usb_pipe_methods ehci_device_ctrl_methods =
2297 {
2298         .open = ehci_device_ctrl_open,
2299         .close = ehci_device_ctrl_close,
2300         .enter = ehci_device_ctrl_enter,
2301         .start = ehci_device_ctrl_start,
2302 };
2303
2304 /*------------------------------------------------------------------------*
2305  * ehci interrupt support
2306  *------------------------------------------------------------------------*/
2307 static void
2308 ehci_device_intr_open(struct usb_xfer *xfer)
2309 {
2310         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2311         uint16_t best;
2312         uint16_t bit;
2313         uint16_t x;
2314
2315         usb_hs_bandwidth_alloc(xfer);
2316
2317         /*
2318          * Find the best QH position corresponding to the given interval:
2319          */
2320
2321         best = 0;
2322         bit = EHCI_VIRTUAL_FRAMELIST_COUNT / 2;
2323         while (bit) {
2324                 if (xfer->interval >= bit) {
2325                         x = bit;
2326                         best = bit;
2327                         while (x & bit) {
2328                                 if (sc->sc_intr_stat[x] <
2329                                     sc->sc_intr_stat[best]) {
2330                                         best = x;
2331                                 }
2332                                 x++;
2333                         }
2334                         break;
2335                 }
2336                 bit >>= 1;
2337         }
2338
2339         sc->sc_intr_stat[best]++;
2340         xfer->qh_pos = best;
2341
2342         DPRINTFN(3, "best=%d interval=%d\n",
2343             best, xfer->interval);
2344 }
2345
2346 static void
2347 ehci_device_intr_close(struct usb_xfer *xfer)
2348 {
2349         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2350
2351         sc->sc_intr_stat[xfer->qh_pos]--;
2352
2353         ehci_device_done(xfer, USB_ERR_CANCELLED);
2354
2355         /* bandwidth must be freed after device done */
2356         usb_hs_bandwidth_free(xfer);
2357 }
2358
2359 static void
2360 ehci_device_intr_enter(struct usb_xfer *xfer)
2361 {
2362         return;
2363 }
2364
2365 static void
2366 ehci_device_intr_start(struct usb_xfer *xfer)
2367 {
2368         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2369
2370         /* setup TD's and QH */
2371         ehci_setup_standard_chain(xfer, &sc->sc_intr_p_last[xfer->qh_pos]);
2372
2373         /* put transfer on interrupt queue */
2374         ehci_transfer_intr_enqueue(xfer);
2375 }
2376
2377 static const struct usb_pipe_methods ehci_device_intr_methods =
2378 {
2379         .open = ehci_device_intr_open,
2380         .close = ehci_device_intr_close,
2381         .enter = ehci_device_intr_enter,
2382         .start = ehci_device_intr_start,
2383 };
2384
2385 /*------------------------------------------------------------------------*
2386  * ehci full speed isochronous support
2387  *------------------------------------------------------------------------*/
2388 static void
2389 ehci_device_isoc_fs_open(struct usb_xfer *xfer)
2390 {
2391         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2392         ehci_sitd_t *td;
2393         uint32_t sitd_portaddr;
2394         uint8_t ds;
2395
2396         sitd_portaddr =
2397             EHCI_SITD_SET_ADDR(xfer->address) |
2398             EHCI_SITD_SET_ENDPT(UE_GET_ADDR(xfer->endpointno)) |
2399             EHCI_SITD_SET_HUBA(xfer->xroot->udev->hs_hub_addr) |
2400             EHCI_SITD_SET_PORT(xfer->xroot->udev->hs_port_no);
2401
2402         if (UE_GET_DIR(xfer->endpointno) == UE_DIR_IN)
2403                 sitd_portaddr |= EHCI_SITD_SET_DIR_IN;
2404
2405         sitd_portaddr = htohc32(sc, sitd_portaddr);
2406
2407         /* initialize all TD's */
2408
2409         for (ds = 0; ds != 2; ds++) {
2410                 for (td = xfer->td_start[ds]; td; td = td->obj_next) {
2411                         td->sitd_portaddr = sitd_portaddr;
2412
2413                         /*
2414                          * TODO: make some kind of automatic
2415                          * SMASK/CMASK selection based on micro-frame
2416                          * usage
2417                          *
2418                          * micro-frame usage (8 microframes per 1ms)
2419                          */
2420                         td->sitd_back = htohc32(sc, EHCI_LINK_TERMINATE);
2421
2422                         usb_pc_cpu_flush(td->page_cache);
2423                 }
2424         }
2425 }
2426
2427 static void
2428 ehci_device_isoc_fs_close(struct usb_xfer *xfer)
2429 {
2430         ehci_device_done(xfer, USB_ERR_CANCELLED);
2431 }
2432
2433 static void
2434 ehci_device_isoc_fs_enter(struct usb_xfer *xfer)
2435 {
2436         struct usb_page_search buf_res;
2437         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2438         ehci_sitd_t *td;
2439         ehci_sitd_t *td_last = NULL;
2440         ehci_sitd_t **pp_last;
2441         uint32_t *plen;
2442         uint32_t buf_offset;
2443         uint32_t nframes;
2444         uint32_t temp;
2445         uint32_t sitd_mask;
2446         uint16_t tlen;
2447         uint8_t sa;
2448         uint8_t sb;
2449
2450 #ifdef USB_DEBUG
2451         uint8_t once = 1;
2452
2453 #endif
2454
2455         DPRINTFN(6, "xfer=%p next=%d nframes=%d\n",
2456             xfer, xfer->endpoint->isoc_next, xfer->nframes);
2457
2458         /* get the current frame index */
2459
2460         nframes = EOREAD4(sc, EHCI_FRINDEX) / 8;
2461
2462         /*
2463          * check if the frame index is within the window where the frames
2464          * will be inserted
2465          */
2466         buf_offset = (nframes - xfer->endpoint->isoc_next) &
2467             (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2468
2469         if ((xfer->endpoint->is_synced == 0) ||
2470             (buf_offset < xfer->nframes)) {
2471                 /*
2472                  * If there is data underflow or the pipe queue is empty we
2473                  * schedule the transfer a few frames ahead of the current
2474                  * frame position. Else two isochronous transfers might
2475                  * overlap.
2476                  */
2477                 xfer->endpoint->isoc_next = (nframes + 3) &
2478                     (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2479                 xfer->endpoint->is_synced = 1;
2480                 DPRINTFN(3, "start next=%d\n", xfer->endpoint->isoc_next);
2481         }
2482         /*
2483          * compute how many milliseconds the insertion is ahead of the
2484          * current frame position:
2485          */
2486         buf_offset = (xfer->endpoint->isoc_next - nframes) &
2487             (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2488
2489         /*
2490          * pre-compute when the isochronous transfer will be finished:
2491          */
2492         xfer->isoc_time_complete =
2493             usb_isoc_time_expand(&sc->sc_bus, nframes) +
2494             buf_offset + xfer->nframes;
2495
2496         /* get the real number of frames */
2497
2498         nframes = xfer->nframes;
2499
2500         buf_offset = 0;
2501
2502         plen = xfer->frlengths;
2503
2504         /* toggle the DMA set we are using */
2505         xfer->flags_int.curr_dma_set ^= 1;
2506
2507         /* get next DMA set */
2508         td = xfer->td_start[xfer->flags_int.curr_dma_set];
2509         xfer->td_transfer_first = td;
2510
2511         pp_last = &sc->sc_isoc_fs_p_last[xfer->endpoint->isoc_next];
2512
2513         /* store starting position */
2514
2515         xfer->qh_pos = xfer->endpoint->isoc_next;
2516
2517         while (nframes--) {
2518                 if (td == NULL) {
2519                         panic("%s:%d: out of TD's\n",
2520                             __FUNCTION__, __LINE__);
2521                 }
2522                 if (pp_last >= &sc->sc_isoc_fs_p_last[EHCI_VIRTUAL_FRAMELIST_COUNT])
2523                         pp_last = &sc->sc_isoc_fs_p_last[0];
2524
2525                 /* reuse sitd_portaddr and sitd_back from last transfer */
2526
2527                 if (*plen > xfer->max_frame_size) {
2528 #ifdef USB_DEBUG
2529                         if (once) {
2530                                 once = 0;
2531                                 printf("%s: frame length(%d) exceeds %d "
2532                                     "bytes (frame truncated)\n",
2533                                     __FUNCTION__, *plen,
2534                                     xfer->max_frame_size);
2535                         }
2536 #endif
2537                         *plen = xfer->max_frame_size;
2538                 }
2539
2540                 /* allocate a slot */
2541
2542                 sa = usbd_fs_isoc_schedule_alloc_slot(xfer,
2543                     xfer->isoc_time_complete - nframes - 1);
2544
2545                 if (sa == 255) {
2546                         /*
2547                          * Schedule is FULL, set length to zero:
2548                          */
2549
2550                         *plen = 0;
2551                         sa = USB_FS_ISOC_UFRAME_MAX - 1;
2552                 }
2553                 if (*plen) {
2554                         /*
2555                          * only call "usbd_get_page()" when we have a
2556                          * non-zero length
2557                          */
2558                         usbd_get_page(xfer->frbuffers, buf_offset, &buf_res);
2559                         td->sitd_bp[0] = htohc32(sc, buf_res.physaddr);
2560                         buf_offset += *plen;
2561                         /*
2562                          * NOTE: We need to subtract one from the offset so
2563                          * that we are on a valid page!
2564                          */
2565                         usbd_get_page(xfer->frbuffers, buf_offset - 1,
2566                             &buf_res);
2567                         temp = buf_res.physaddr & ~0xFFF;
2568                 } else {
2569                         td->sitd_bp[0] = 0;
2570                         temp = 0;
2571                 }
2572
2573                 if (UE_GET_DIR(xfer->endpointno) == UE_DIR_OUT) {
2574                         tlen = *plen;
2575                         if (tlen <= 188) {
2576                                 temp |= 1;      /* T-count = 1, TP = ALL */
2577                                 tlen = 1;
2578                         } else {
2579                                 tlen += 187;
2580                                 tlen /= 188;
2581                                 temp |= tlen;   /* T-count = [1..6] */
2582                                 temp |= 8;      /* TP = Begin */
2583                         }
2584
2585                         tlen += sa;
2586
2587                         if (tlen >= 8) {
2588                                 sb = 0;
2589                         } else {
2590                                 sb = (1 << tlen);
2591                         }
2592
2593                         sa = (1 << sa);
2594                         sa = (sb - sa) & 0x3F;
2595                         sb = 0;
2596                 } else {
2597                         sb = (-(4 << sa)) & 0xFE;
2598                         sa = (1 << sa) & 0x3F;
2599                 }
2600
2601                 sitd_mask = (EHCI_SITD_SET_SMASK(sa) |
2602                     EHCI_SITD_SET_CMASK(sb));
2603
2604                 td->sitd_bp[1] = htohc32(sc, temp);
2605
2606                 td->sitd_mask = htohc32(sc, sitd_mask);
2607
2608                 if (nframes == 0) {
2609                         td->sitd_status = htohc32(sc,
2610                             EHCI_SITD_IOC |
2611                             EHCI_SITD_ACTIVE |
2612                             EHCI_SITD_SET_LEN(*plen));
2613                 } else {
2614                         td->sitd_status = htohc32(sc,
2615                             EHCI_SITD_ACTIVE |
2616                             EHCI_SITD_SET_LEN(*plen));
2617                 }
2618                 usb_pc_cpu_flush(td->page_cache);
2619
2620 #ifdef USB_DEBUG
2621                 if (ehcidebug > 15) {
2622                         DPRINTF("FS-TD %d\n", nframes);
2623                         ehci_dump_sitd(sc, td);
2624                 }
2625 #endif
2626                 /* insert TD into schedule */
2627                 EHCI_APPEND_FS_TD(td, *pp_last);
2628                 pp_last++;
2629
2630                 plen++;
2631                 td_last = td;
2632                 td = td->obj_next;
2633         }
2634
2635         xfer->td_transfer_last = td_last;
2636
2637         /* update isoc_next */
2638         xfer->endpoint->isoc_next = (pp_last - &sc->sc_isoc_fs_p_last[0]) &
2639             (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2640
2641         /*
2642          * We don't allow cancelling of the SPLIT transaction USB FULL
2643          * speed transfer, because it disturbs the bandwidth
2644          * computation algorithm.
2645          */
2646         xfer->flags_int.can_cancel_immed = 0;
2647 }
2648
2649 static void
2650 ehci_device_isoc_fs_start(struct usb_xfer *xfer)
2651 {
2652         /*
2653          * We don't allow cancelling of the SPLIT transaction USB FULL
2654          * speed transfer, because it disturbs the bandwidth
2655          * computation algorithm.
2656          */
2657         xfer->flags_int.can_cancel_immed = 0;
2658
2659         /* set a default timeout */
2660         if (xfer->timeout == 0)
2661                 xfer->timeout = 500; /* ms */
2662
2663         /* put transfer on interrupt queue */
2664         ehci_transfer_intr_enqueue(xfer);
2665 }
2666
2667 static const struct usb_pipe_methods ehci_device_isoc_fs_methods =
2668 {
2669         .open = ehci_device_isoc_fs_open,
2670         .close = ehci_device_isoc_fs_close,
2671         .enter = ehci_device_isoc_fs_enter,
2672         .start = ehci_device_isoc_fs_start,
2673 };
2674
2675 /*------------------------------------------------------------------------*
2676  * ehci high speed isochronous support
2677  *------------------------------------------------------------------------*/
2678 static void
2679 ehci_device_isoc_hs_open(struct usb_xfer *xfer)
2680 {
2681         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2682         ehci_itd_t *td;
2683         uint32_t temp;
2684         uint8_t ds;
2685
2686         usb_hs_bandwidth_alloc(xfer);
2687
2688         /* initialize all TD's */
2689
2690         for (ds = 0; ds != 2; ds++) {
2691                 for (td = xfer->td_start[ds]; td; td = td->obj_next) {
2692                         /* set TD inactive */
2693                         td->itd_status[0] = 0;
2694                         td->itd_status[1] = 0;
2695                         td->itd_status[2] = 0;
2696                         td->itd_status[3] = 0;
2697                         td->itd_status[4] = 0;
2698                         td->itd_status[5] = 0;
2699                         td->itd_status[6] = 0;
2700                         td->itd_status[7] = 0;
2701
2702                         /* set endpoint and address */
2703                         td->itd_bp[0] = htohc32(sc,
2704                             EHCI_ITD_SET_ADDR(xfer->address) |
2705                             EHCI_ITD_SET_ENDPT(UE_GET_ADDR(xfer->endpointno)));
2706
2707                         temp =
2708                             EHCI_ITD_SET_MPL(xfer->max_packet_size & 0x7FF);
2709
2710                         /* set direction */
2711                         if (UE_GET_DIR(xfer->endpointno) == UE_DIR_IN) {
2712                                 temp |= EHCI_ITD_SET_DIR_IN;
2713                         }
2714                         /* set maximum packet size */
2715                         td->itd_bp[1] = htohc32(sc, temp);
2716
2717                         /* set transfer multiplier */
2718                         td->itd_bp[2] = htohc32(sc, xfer->max_packet_count & 3);
2719
2720                         usb_pc_cpu_flush(td->page_cache);
2721                 }
2722         }
2723 }
2724
2725 static void
2726 ehci_device_isoc_hs_close(struct usb_xfer *xfer)
2727 {
2728         ehci_device_done(xfer, USB_ERR_CANCELLED);
2729
2730         /* bandwidth must be freed after device done */
2731         usb_hs_bandwidth_free(xfer);
2732 }
2733
2734 static void
2735 ehci_device_isoc_hs_enter(struct usb_xfer *xfer)
2736 {
2737         struct usb_page_search buf_res;
2738         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2739         ehci_itd_t *td;
2740         ehci_itd_t *td_last = NULL;
2741         ehci_itd_t **pp_last;
2742         bus_size_t page_addr;
2743         uint32_t *plen;
2744         uint32_t status;
2745         uint32_t buf_offset;
2746         uint32_t nframes;
2747         uint32_t itd_offset[8 + 1];
2748         uint8_t x;
2749         uint8_t td_no;
2750         uint8_t page_no;
2751         uint8_t shift = usbd_xfer_get_fps_shift(xfer);
2752
2753 #ifdef USB_DEBUG
2754         uint8_t once = 1;
2755
2756 #endif
2757
2758         DPRINTFN(6, "xfer=%p next=%d nframes=%d shift=%d\n",
2759             xfer, xfer->endpoint->isoc_next, xfer->nframes, (int)shift);
2760
2761         /* get the current frame index */
2762
2763         nframes = EOREAD4(sc, EHCI_FRINDEX) / 8;
2764
2765         /*
2766          * check if the frame index is within the window where the frames
2767          * will be inserted
2768          */
2769         buf_offset = (nframes - xfer->endpoint->isoc_next) &
2770             (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2771
2772         if ((xfer->endpoint->is_synced == 0) ||
2773             (buf_offset < (((xfer->nframes << shift) + 7) / 8))) {
2774                 /*
2775                  * If there is data underflow or the pipe queue is empty we
2776                  * schedule the transfer a few frames ahead of the current
2777                  * frame position. Else two isochronous transfers might
2778                  * overlap.
2779                  */
2780                 xfer->endpoint->isoc_next = (nframes + 3) &
2781                     (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2782                 xfer->endpoint->is_synced = 1;
2783                 DPRINTFN(3, "start next=%d\n", xfer->endpoint->isoc_next);
2784         }
2785         /*
2786          * compute how many milliseconds the insertion is ahead of the
2787          * current frame position:
2788          */
2789         buf_offset = (xfer->endpoint->isoc_next - nframes) &
2790             (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2791
2792         /*
2793          * pre-compute when the isochronous transfer will be finished:
2794          */
2795         xfer->isoc_time_complete =
2796             usb_isoc_time_expand(&sc->sc_bus, nframes) + buf_offset +
2797             (((xfer->nframes << shift) + 7) / 8);
2798
2799         /* get the real number of frames */
2800
2801         nframes = xfer->nframes;
2802
2803         buf_offset = 0;
2804         td_no = 0;
2805
2806         plen = xfer->frlengths;
2807
2808         /* toggle the DMA set we are using */
2809         xfer->flags_int.curr_dma_set ^= 1;
2810
2811         /* get next DMA set */
2812         td = xfer->td_start[xfer->flags_int.curr_dma_set];
2813         xfer->td_transfer_first = td;
2814
2815         pp_last = &sc->sc_isoc_hs_p_last[xfer->endpoint->isoc_next];
2816
2817         /* store starting position */
2818
2819         xfer->qh_pos = xfer->endpoint->isoc_next;
2820
2821         while (nframes) {
2822                 if (td == NULL) {
2823                         panic("%s:%d: out of TD's\n",
2824                             __FUNCTION__, __LINE__);
2825                 }
2826                 if (pp_last >= &sc->sc_isoc_hs_p_last[EHCI_VIRTUAL_FRAMELIST_COUNT]) {
2827                         pp_last = &sc->sc_isoc_hs_p_last[0];
2828                 }
2829                 /* range check */
2830                 if (*plen > xfer->max_frame_size) {
2831 #ifdef USB_DEBUG
2832                         if (once) {
2833                                 once = 0;
2834                                 printf("%s: frame length(%d) exceeds %d bytes "
2835                                     "(frame truncated)\n",
2836                                     __FUNCTION__, *plen, xfer->max_frame_size);
2837                         }
2838 #endif
2839                         *plen = xfer->max_frame_size;
2840                 }
2841
2842                 if (xfer->endpoint->usb_smask & (1 << td_no)) {
2843                         status = (EHCI_ITD_SET_LEN(*plen) |
2844                             EHCI_ITD_ACTIVE |
2845                             EHCI_ITD_SET_PG(0));
2846                         td->itd_status[td_no] = htohc32(sc, status);
2847                         itd_offset[td_no] = buf_offset;
2848                         buf_offset += *plen;
2849                         plen++;
2850                         nframes --;
2851                 } else {
2852                         td->itd_status[td_no] = 0;      /* not active */
2853                         itd_offset[td_no] = buf_offset;
2854                 }
2855
2856                 td_no++;
2857
2858                 if ((td_no == 8) || (nframes == 0)) {
2859                         /* the rest of the transfers are not active, if any */
2860                         for (x = td_no; x != 8; x++) {
2861                                 td->itd_status[x] = 0;  /* not active */
2862                         }
2863
2864                         /* check if there is any data to be transferred */
2865                         if (itd_offset[0] != buf_offset) {
2866                                 page_no = 0;
2867                                 itd_offset[td_no] = buf_offset;
2868
2869                                 /* get first page offset */
2870                                 usbd_get_page(xfer->frbuffers, itd_offset[0], &buf_res);
2871                                 /* get page address */
2872                                 page_addr = buf_res.physaddr & ~0xFFF;
2873                                 /* update page address */
2874                                 td->itd_bp[0] &= htohc32(sc, 0xFFF);
2875                                 td->itd_bp[0] |= htohc32(sc, page_addr);
2876
2877                                 for (x = 0; x != td_no; x++) {
2878                                         /* set page number and page offset */
2879                                         status = (EHCI_ITD_SET_PG(page_no) |
2880                                             (buf_res.physaddr & 0xFFF));
2881                                         td->itd_status[x] |= htohc32(sc, status);
2882
2883                                         /* get next page offset */
2884                                         if (itd_offset[x + 1] == buf_offset) {
2885                                                 /*
2886                                                  * We subtract one so that
2887                                                  * we don't go off the last
2888                                                  * page!
2889                                                  */
2890                                                 usbd_get_page(xfer->frbuffers, buf_offset - 1, &buf_res);
2891                                         } else {
2892                                                 usbd_get_page(xfer->frbuffers, itd_offset[x + 1], &buf_res);
2893                                         }
2894
2895                                         /* check if we need a new page */
2896                                         if ((buf_res.physaddr ^ page_addr) & ~0xFFF) {
2897                                                 /* new page needed */
2898                                                 page_addr = buf_res.physaddr & ~0xFFF;
2899                                                 if (page_no == 6) {
2900                                                         panic("%s: too many pages\n", __FUNCTION__);
2901                                                 }
2902                                                 page_no++;
2903                                                 /* update page address */
2904                                                 td->itd_bp[page_no] &= htohc32(sc, 0xFFF);
2905                                                 td->itd_bp[page_no] |= htohc32(sc, page_addr);
2906                                         }
2907                                 }
2908                         }
2909                         /* set IOC bit if we are complete */
2910                         if (nframes == 0) {
2911                                 td->itd_status[td_no - 1] |= htohc32(sc, EHCI_ITD_IOC);
2912                         }
2913                         usb_pc_cpu_flush(td->page_cache);
2914 #ifdef USB_DEBUG
2915                         if (ehcidebug > 15) {
2916                                 DPRINTF("HS-TD %d\n", nframes);
2917                                 ehci_dump_itd(sc, td);
2918                         }
2919 #endif
2920                         /* insert TD into schedule */
2921                         EHCI_APPEND_HS_TD(td, *pp_last);
2922                         pp_last++;
2923
2924                         td_no = 0;
2925                         td_last = td;
2926                         td = td->obj_next;
2927                 }
2928         }
2929
2930         xfer->td_transfer_last = td_last;
2931
2932         /* update isoc_next */
2933         xfer->endpoint->isoc_next = (pp_last - &sc->sc_isoc_hs_p_last[0]) &
2934             (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2935 }
2936
2937 static void
2938 ehci_device_isoc_hs_start(struct usb_xfer *xfer)
2939 {
2940         /* put transfer on interrupt queue */
2941         ehci_transfer_intr_enqueue(xfer);
2942 }
2943
2944 static const struct usb_pipe_methods ehci_device_isoc_hs_methods =
2945 {
2946         .open = ehci_device_isoc_hs_open,
2947         .close = ehci_device_isoc_hs_close,
2948         .enter = ehci_device_isoc_hs_enter,
2949         .start = ehci_device_isoc_hs_start,
2950 };
2951
2952 /*------------------------------------------------------------------------*
2953  * ehci root control support
2954  *------------------------------------------------------------------------*
2955  * Simulate a hardware hub by handling all the necessary requests.
2956  *------------------------------------------------------------------------*/
2957
2958 static const
2959 struct usb_device_descriptor ehci_devd =
2960 {
2961         sizeof(struct usb_device_descriptor),
2962         UDESC_DEVICE,                   /* type */
2963         {0x00, 0x02},                   /* USB version */
2964         UDCLASS_HUB,                    /* class */
2965         UDSUBCLASS_HUB,                 /* subclass */
2966         UDPROTO_HSHUBSTT,               /* protocol */
2967         64,                             /* max packet */
2968         {0}, {0}, {0x00, 0x01},         /* device id */
2969         1, 2, 0,                        /* string indexes */
2970         1                               /* # of configurations */
2971 };
2972
2973 static const
2974 struct usb_device_qualifier ehci_odevd =
2975 {
2976         sizeof(struct usb_device_qualifier),
2977         UDESC_DEVICE_QUALIFIER,         /* type */
2978         {0x00, 0x02},                   /* USB version */
2979         UDCLASS_HUB,                    /* class */
2980         UDSUBCLASS_HUB,                 /* subclass */
2981         UDPROTO_FSHUB,                  /* protocol */
2982         0,                              /* max packet */
2983         0,                              /* # of configurations */
2984         0
2985 };
2986
2987 static const struct ehci_config_desc ehci_confd = {
2988         .confd = {
2989                 .bLength = sizeof(struct usb_config_descriptor),
2990                 .bDescriptorType = UDESC_CONFIG,
2991                 .wTotalLength[0] = sizeof(ehci_confd),
2992                 .bNumInterface = 1,
2993                 .bConfigurationValue = 1,
2994                 .iConfiguration = 0,
2995                 .bmAttributes = UC_SELF_POWERED,
2996                 .bMaxPower = 0          /* max power */
2997         },
2998         .ifcd = {
2999                 .bLength = sizeof(struct usb_interface_descriptor),
3000                 .bDescriptorType = UDESC_INTERFACE,
3001                 .bNumEndpoints = 1,
3002                 .bInterfaceClass = UICLASS_HUB,
3003                 .bInterfaceSubClass = UISUBCLASS_HUB,
3004                 .bInterfaceProtocol = 0,
3005         },
3006         .endpd = {
3007                 .bLength = sizeof(struct usb_endpoint_descriptor),
3008                 .bDescriptorType = UDESC_ENDPOINT,
3009                 .bEndpointAddress = UE_DIR_IN | EHCI_INTR_ENDPT,
3010                 .bmAttributes = UE_INTERRUPT,
3011                 .wMaxPacketSize[0] = 8, /* max packet (63 ports) */
3012                 .bInterval = 255,
3013         },
3014 };
3015
3016 static const
3017 struct usb_hub_descriptor ehci_hubd =
3018 {
3019         .bDescLength = 0,               /* dynamic length */
3020         .bDescriptorType = UDESC_HUB,
3021 };
3022
3023 uint16_t
3024 ehci_get_port_speed_portsc(struct ehci_softc *sc, uint16_t index)
3025 {
3026         uint32_t v;
3027
3028         v = EOREAD4(sc, EHCI_PORTSC(index));
3029         v = (v >> EHCI_PORTSC_PSPD_SHIFT) & EHCI_PORTSC_PSPD_MASK;
3030
3031         if (v == EHCI_PORT_SPEED_HIGH)
3032                 return (UPS_HIGH_SPEED);
3033         if (v == EHCI_PORT_SPEED_LOW)
3034                 return (UPS_LOW_SPEED);
3035         return (0);
3036 }
3037
3038 uint16_t
3039 ehci_get_port_speed_hostc(struct ehci_softc *sc, uint16_t index)
3040 {
3041         uint32_t v;
3042
3043         v = EOREAD4(sc, EHCI_HOSTC(index));
3044         v = (v >> EHCI_HOSTC_PSPD_SHIFT) & EHCI_HOSTC_PSPD_MASK;
3045
3046         if (v == EHCI_PORT_SPEED_HIGH)
3047                 return (UPS_HIGH_SPEED);
3048         if (v == EHCI_PORT_SPEED_LOW)
3049                 return (UPS_LOW_SPEED);
3050         return (0);
3051 }
3052
3053 static void
3054 ehci_disown(ehci_softc_t *sc, uint16_t index, uint8_t lowspeed)
3055 {
3056         uint32_t port;
3057         uint32_t v;
3058
3059         DPRINTF("index=%d lowspeed=%d\n", index, lowspeed);
3060
3061         port = EHCI_PORTSC(index);
3062         v = EOREAD4(sc, port) & ~EHCI_PS_CLEAR;
3063         EOWRITE4(sc, port, v | EHCI_PS_PO);
3064 }
3065
3066 static usb_error_t
3067 ehci_roothub_exec(struct usb_device *udev,
3068     struct usb_device_request *req, const void **pptr, uint16_t *plength)
3069 {
3070         ehci_softc_t *sc = EHCI_BUS2SC(udev->bus);
3071         const char *str_ptr;
3072         const void *ptr;
3073         uint32_t port;
3074         uint32_t v;
3075         uint16_t len;
3076         uint16_t i;
3077         uint16_t value;
3078         uint16_t index;
3079         usb_error_t err;
3080
3081         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
3082
3083         /* buffer reset */
3084         ptr = (const void *)&sc->sc_hub_desc;
3085         len = 0;
3086         err = 0;
3087
3088         value = UGETW(req->wValue);
3089         index = UGETW(req->wIndex);
3090
3091         DPRINTFN(3, "type=0x%02x request=0x%02x wLen=0x%04x "
3092             "wValue=0x%04x wIndex=0x%04x\n",
3093             req->bmRequestType, req->bRequest,
3094             UGETW(req->wLength), value, index);
3095
3096 #define C(x,y) ((x) | ((y) << 8))
3097         switch (C(req->bRequest, req->bmRequestType)) {
3098         case C(UR_CLEAR_FEATURE, UT_WRITE_DEVICE):
3099         case C(UR_CLEAR_FEATURE, UT_WRITE_INTERFACE):
3100         case C(UR_CLEAR_FEATURE, UT_WRITE_ENDPOINT):
3101                 /*
3102                  * DEVICE_REMOTE_WAKEUP and ENDPOINT_HALT are no-ops
3103                  * for the integrated root hub.
3104                  */
3105                 break;
3106         case C(UR_GET_CONFIG, UT_READ_DEVICE):
3107                 len = 1;
3108                 sc->sc_hub_desc.temp[0] = sc->sc_conf;
3109                 break;
3110         case C(UR_GET_DESCRIPTOR, UT_READ_DEVICE):
3111                 switch (value >> 8) {
3112                 case UDESC_DEVICE:
3113                         if ((value & 0xff) != 0) {
3114                                 err = USB_ERR_IOERROR;
3115                                 goto done;
3116                         }
3117                         len = sizeof(ehci_devd);
3118                         ptr = (const void *)&ehci_devd;
3119                         break;
3120                         /*
3121                          * We can't really operate at another speed,
3122                          * but the specification says we need this
3123                          * descriptor:
3124                          */
3125                 case UDESC_DEVICE_QUALIFIER:
3126                         if ((value & 0xff) != 0) {
3127                                 err = USB_ERR_IOERROR;
3128                                 goto done;
3129                         }
3130                         len = sizeof(ehci_odevd);
3131                         ptr = (const void *)&ehci_odevd;
3132                         break;
3133
3134                 case UDESC_CONFIG:
3135                         if ((value & 0xff) != 0) {
3136                                 err = USB_ERR_IOERROR;
3137                                 goto done;
3138                         }
3139                         len = sizeof(ehci_confd);
3140                         ptr = (const void *)&ehci_confd;
3141                         break;
3142
3143                 case UDESC_STRING:
3144                         switch (value & 0xff) {
3145                         case 0: /* Language table */
3146                                 str_ptr = "\001";
3147                                 break;
3148
3149                         case 1: /* Vendor */
3150                                 str_ptr = sc->sc_vendor;
3151                                 break;
3152
3153                         case 2: /* Product */
3154                                 str_ptr = "EHCI root HUB";
3155                                 break;
3156
3157                         default:
3158                                 str_ptr = "";
3159                                 break;
3160                         }
3161
3162                         len = usb_make_str_desc(
3163                             sc->sc_hub_desc.temp,
3164                             sizeof(sc->sc_hub_desc.temp),
3165                             str_ptr);
3166                         break;
3167                 default:
3168                         err = USB_ERR_IOERROR;
3169                         goto done;
3170                 }
3171                 break;
3172         case C(UR_GET_INTERFACE, UT_READ_INTERFACE):
3173                 len = 1;
3174                 sc->sc_hub_desc.temp[0] = 0;
3175                 break;
3176         case C(UR_GET_STATUS, UT_READ_DEVICE):
3177                 len = 2;
3178                 USETW(sc->sc_hub_desc.stat.wStatus, UDS_SELF_POWERED);
3179                 break;
3180         case C(UR_GET_STATUS, UT_READ_INTERFACE):
3181         case C(UR_GET_STATUS, UT_READ_ENDPOINT):
3182                 len = 2;
3183                 USETW(sc->sc_hub_desc.stat.wStatus, 0);
3184                 break;
3185         case C(UR_SET_ADDRESS, UT_WRITE_DEVICE):
3186                 if (value >= EHCI_MAX_DEVICES) {
3187                         err = USB_ERR_IOERROR;
3188                         goto done;
3189                 }
3190                 sc->sc_addr = value;
3191                 break;
3192         case C(UR_SET_CONFIG, UT_WRITE_DEVICE):
3193                 if ((value != 0) && (value != 1)) {
3194                         err = USB_ERR_IOERROR;
3195                         goto done;
3196                 }
3197                 sc->sc_conf = value;
3198                 break;
3199         case C(UR_SET_DESCRIPTOR, UT_WRITE_DEVICE):
3200                 break;
3201         case C(UR_SET_FEATURE, UT_WRITE_DEVICE):
3202         case C(UR_SET_FEATURE, UT_WRITE_INTERFACE):
3203         case C(UR_SET_FEATURE, UT_WRITE_ENDPOINT):
3204                 err = USB_ERR_IOERROR;
3205                 goto done;
3206         case C(UR_SET_INTERFACE, UT_WRITE_INTERFACE):
3207                 break;
3208         case C(UR_SYNCH_FRAME, UT_WRITE_ENDPOINT):
3209                 break;
3210                 /* Hub requests */
3211         case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_DEVICE):
3212                 break;
3213         case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_OTHER):
3214                 DPRINTFN(9, "UR_CLEAR_PORT_FEATURE\n");
3215
3216                 if ((index < 1) ||
3217                     (index > sc->sc_noport)) {
3218                         err = USB_ERR_IOERROR;
3219                         goto done;
3220                 }
3221                 port = EHCI_PORTSC(index);
3222                 v = EOREAD4(sc, port) & ~EHCI_PS_CLEAR;
3223                 switch (value) {
3224                 case UHF_PORT_ENABLE:
3225                         EOWRITE4(sc, port, v & ~EHCI_PS_PE);
3226                         break;
3227                 case UHF_PORT_SUSPEND:
3228                         if ((v & EHCI_PS_SUSP) && (!(v & EHCI_PS_FPR))) {
3229                                 /*
3230                                  * waking up a High Speed device is rather
3231                                  * complicated if
3232                                  */
3233                                 EOWRITE4(sc, port, v | EHCI_PS_FPR);
3234                         }
3235                         /* wait 20ms for resume sequence to complete */
3236                         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 50);
3237
3238                         EOWRITE4(sc, port, v & ~(EHCI_PS_SUSP |
3239                             EHCI_PS_FPR | (3 << 10) /* High Speed */ ));
3240
3241                         /* 4ms settle time */
3242                         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 250);
3243                         break;
3244                 case UHF_PORT_POWER:
3245                         EOWRITE4(sc, port, v & ~EHCI_PS_PP);
3246                         break;
3247                 case UHF_PORT_TEST:
3248                         DPRINTFN(3, "clear port test "
3249                             "%d\n", index);
3250                         break;
3251                 case UHF_PORT_INDICATOR:
3252                         DPRINTFN(3, "clear port ind "
3253                             "%d\n", index);
3254                         EOWRITE4(sc, port, v & ~EHCI_PS_PIC);
3255                         break;
3256                 case UHF_C_PORT_CONNECTION:
3257                         EOWRITE4(sc, port, v | EHCI_PS_CSC);
3258                         break;
3259                 case UHF_C_PORT_ENABLE:
3260                         EOWRITE4(sc, port, v | EHCI_PS_PEC);
3261                         break;
3262                 case UHF_C_PORT_SUSPEND:
3263                         EOWRITE4(sc, port, v | EHCI_PS_SUSP);
3264                         break;
3265                 case UHF_C_PORT_OVER_CURRENT:
3266                         EOWRITE4(sc, port, v | EHCI_PS_OCC);
3267                         break;
3268                 case UHF_C_PORT_RESET:
3269                         sc->sc_isreset = 0;
3270                         break;
3271                 default:
3272                         err = USB_ERR_IOERROR;
3273                         goto done;
3274                 }
3275                 break;
3276         case C(UR_GET_DESCRIPTOR, UT_READ_CLASS_DEVICE):
3277                 if ((value & 0xff) != 0) {
3278                         err = USB_ERR_IOERROR;
3279                         goto done;
3280                 }
3281                 v = EREAD4(sc, EHCI_HCSPARAMS);
3282
3283                 sc->sc_hub_desc.hubd = ehci_hubd;
3284                 sc->sc_hub_desc.hubd.bNbrPorts = sc->sc_noport;
3285
3286                 if (EHCI_HCS_PPC(v))
3287                         i = UHD_PWR_INDIVIDUAL;
3288                 else
3289                         i = UHD_PWR_NO_SWITCH;
3290
3291                 if (EHCI_HCS_P_INDICATOR(v))
3292                         i |= UHD_PORT_IND;
3293
3294                 USETW(sc->sc_hub_desc.hubd.wHubCharacteristics, i);
3295                 /* XXX can't find out? */
3296                 sc->sc_hub_desc.hubd.bPwrOn2PwrGood = 200;
3297                 /* XXX don't know if ports are removable or not */
3298                 sc->sc_hub_desc.hubd.bDescLength =
3299                     8 + ((sc->sc_noport + 7) / 8);
3300                 len = sc->sc_hub_desc.hubd.bDescLength;
3301                 break;
3302         case C(UR_GET_STATUS, UT_READ_CLASS_DEVICE):
3303                 len = 16;
3304                 memset(sc->sc_hub_desc.temp, 0, 16);
3305                 break;
3306         case C(UR_GET_STATUS, UT_READ_CLASS_OTHER):
3307                 DPRINTFN(9, "get port status i=%d\n",
3308                     index);
3309                 if ((index < 1) ||
3310                     (index > sc->sc_noport)) {
3311                         err = USB_ERR_IOERROR;
3312                         goto done;
3313                 }
3314                 v = EOREAD4(sc, EHCI_PORTSC(index));
3315                 DPRINTFN(9, "port status=0x%04x\n", v);
3316                 if (sc->sc_flags & EHCI_SCFLG_TT) {
3317                         if (sc->sc_vendor_get_port_speed != NULL) {
3318                                 i = sc->sc_vendor_get_port_speed(sc, index);
3319                         } else {
3320                                 device_printf(sc->sc_bus.bdev,
3321                                     "EHCI_SCFLG_TT quirk is set but "
3322                                     "sc_vendor_get_hub_speed() is NULL\n");
3323                                 i = UPS_HIGH_SPEED;
3324                         }
3325                 } else {
3326                         i = UPS_HIGH_SPEED;
3327                 }
3328                 if (v & EHCI_PS_CS)
3329                         i |= UPS_CURRENT_CONNECT_STATUS;
3330                 if (v & EHCI_PS_PE)
3331                         i |= UPS_PORT_ENABLED;
3332                 if ((v & EHCI_PS_SUSP) && !(v & EHCI_PS_FPR))
3333                         i |= UPS_SUSPEND;
3334                 if (v & EHCI_PS_OCA)
3335                         i |= UPS_OVERCURRENT_INDICATOR;
3336                 if (v & EHCI_PS_PR)
3337                         i |= UPS_RESET;
3338                 if (v & EHCI_PS_PP)
3339                         i |= UPS_PORT_POWER;
3340                 USETW(sc->sc_hub_desc.ps.wPortStatus, i);
3341                 i = 0;
3342                 if (v & EHCI_PS_CSC)
3343                         i |= UPS_C_CONNECT_STATUS;
3344                 if (v & EHCI_PS_PEC)
3345                         i |= UPS_C_PORT_ENABLED;
3346                 if (v & EHCI_PS_OCC)
3347                         i |= UPS_C_OVERCURRENT_INDICATOR;
3348                 if (v & EHCI_PS_FPR)
3349                         i |= UPS_C_SUSPEND;
3350                 if (sc->sc_isreset)
3351                         i |= UPS_C_PORT_RESET;
3352                 USETW(sc->sc_hub_desc.ps.wPortChange, i);
3353                 len = sizeof(sc->sc_hub_desc.ps);
3354                 break;
3355         case C(UR_SET_DESCRIPTOR, UT_WRITE_CLASS_DEVICE):
3356                 err = USB_ERR_IOERROR;
3357                 goto done;
3358         case C(UR_SET_FEATURE, UT_WRITE_CLASS_DEVICE):
3359                 break;
3360         case C(UR_SET_FEATURE, UT_WRITE_CLASS_OTHER):
3361                 if ((index < 1) ||
3362                     (index > sc->sc_noport)) {
3363                         err = USB_ERR_IOERROR;
3364                         goto done;
3365                 }
3366                 port = EHCI_PORTSC(index);
3367                 v = EOREAD4(sc, port) & ~EHCI_PS_CLEAR;
3368                 switch (value) {
3369                 case UHF_PORT_ENABLE:
3370                         EOWRITE4(sc, port, v | EHCI_PS_PE);
3371                         break;
3372                 case UHF_PORT_SUSPEND:
3373                         EOWRITE4(sc, port, v | EHCI_PS_SUSP);
3374                         break;
3375                 case UHF_PORT_RESET:
3376                         DPRINTFN(6, "reset port %d\n", index);
3377 #ifdef USB_DEBUG
3378                         if (ehcinohighspeed) {
3379                                 /*
3380                                  * Connect USB device to companion
3381                                  * controller.
3382                                  */
3383                                 ehci_disown(sc, index, 1);
3384                                 break;
3385                         }
3386 #endif
3387                         if (EHCI_PS_IS_LOWSPEED(v) &&
3388                             (sc->sc_flags & EHCI_SCFLG_TT) == 0) {
3389                                 /* Low speed device, give up ownership. */
3390                                 ehci_disown(sc, index, 1);
3391                                 break;
3392                         }
3393                         /* Start reset sequence. */
3394                         v &= ~(EHCI_PS_PE | EHCI_PS_PR);
3395                         EOWRITE4(sc, port, v | EHCI_PS_PR);
3396
3397                         /* Wait for reset to complete. */
3398                         usb_pause_mtx(&sc->sc_bus.bus_mtx,
3399                             USB_MS_TO_TICKS(usb_port_root_reset_delay));
3400
3401                         /* Terminate reset sequence. */
3402                         if (!(sc->sc_flags & EHCI_SCFLG_NORESTERM))
3403                                 EOWRITE4(sc, port, v);
3404
3405                         /* Wait for HC to complete reset. */
3406                         usb_pause_mtx(&sc->sc_bus.bus_mtx,
3407                             USB_MS_TO_TICKS(EHCI_PORT_RESET_COMPLETE));
3408
3409                         v = EOREAD4(sc, port);
3410                         DPRINTF("ehci after reset, status=0x%08x\n", v);
3411                         if (v & EHCI_PS_PR) {
3412                                 device_printf(sc->sc_bus.bdev,
3413                                     "port reset timeout\n");
3414                                 err = USB_ERR_TIMEOUT;
3415                                 goto done;
3416                         }
3417                         if (!(v & EHCI_PS_PE) &&
3418                             (sc->sc_flags & EHCI_SCFLG_TT) == 0) {
3419                                 /* Not a high speed device, give up ownership.*/
3420                                 ehci_disown(sc, index, 0);
3421                                 break;
3422                         }
3423                         sc->sc_isreset = 1;
3424                         DPRINTF("ehci port %d reset, status = 0x%08x\n",
3425                             index, v);
3426                         break;
3427
3428                 case UHF_PORT_POWER:
3429                         DPRINTFN(3, "set port power %d\n", index);
3430                         EOWRITE4(sc, port, v | EHCI_PS_PP);
3431                         break;
3432
3433                 case UHF_PORT_TEST:
3434                         DPRINTFN(3, "set port test %d\n", index);
3435                         break;
3436
3437                 case UHF_PORT_INDICATOR:
3438                         DPRINTFN(3, "set port ind %d\n", index);
3439                         EOWRITE4(sc, port, v | EHCI_PS_PIC);
3440                         break;
3441
3442                 default:
3443                         err = USB_ERR_IOERROR;
3444                         goto done;
3445                 }
3446                 break;
3447         case C(UR_CLEAR_TT_BUFFER, UT_WRITE_CLASS_OTHER):
3448         case C(UR_RESET_TT, UT_WRITE_CLASS_OTHER):
3449         case C(UR_GET_TT_STATE, UT_READ_CLASS_OTHER):
3450         case C(UR_STOP_TT, UT_WRITE_CLASS_OTHER):
3451                 break;
3452         default:
3453                 err = USB_ERR_IOERROR;
3454                 goto done;
3455         }
3456 done:
3457         *plength = len;
3458         *pptr = ptr;
3459         return (err);
3460 }
3461
3462 static void
3463 ehci_xfer_setup(struct usb_setup_params *parm)
3464 {
3465         struct usb_page_search page_info;
3466         struct usb_page_cache *pc;
3467         ehci_softc_t *sc;
3468         struct usb_xfer *xfer;
3469         void *last_obj;
3470         uint32_t nqtd;
3471         uint32_t nqh;
3472         uint32_t nsitd;
3473         uint32_t nitd;
3474         uint32_t n;
3475
3476         sc = EHCI_BUS2SC(parm->udev->bus);
3477         xfer = parm->curr_xfer;
3478
3479         nqtd = 0;
3480         nqh = 0;
3481         nsitd = 0;
3482         nitd = 0;
3483
3484         /*
3485          * compute maximum number of some structures
3486          */
3487         if (parm->methods == &ehci_device_ctrl_methods) {
3488                 /*
3489                  * The proof for the "nqtd" formula is illustrated like
3490                  * this:
3491                  *
3492                  * +------------------------------------+
3493                  * |                                    |
3494                  * |         |remainder ->              |
3495                  * |   +-----+---+                      |
3496                  * |   | xxx | x | frm 0                |
3497                  * |   +-----+---++                     |
3498                  * |   | xxx | xx | frm 1               |
3499                  * |   +-----+----+                     |
3500                  * |            ...                     |
3501                  * +------------------------------------+
3502                  *
3503                  * "xxx" means a completely full USB transfer descriptor
3504                  *
3505                  * "x" and "xx" means a short USB packet
3506                  *
3507                  * For the remainder of an USB transfer modulo
3508                  * "max_data_length" we need two USB transfer descriptors.
3509                  * One to transfer the remaining data and one to finalise
3510                  * with a zero length packet in case the "force_short_xfer"
3511                  * flag is set. We only need two USB transfer descriptors in
3512                  * the case where the transfer length of the first one is a
3513                  * factor of "max_frame_size". The rest of the needed USB
3514                  * transfer descriptors is given by the buffer size divided
3515                  * by the maximum data payload.
3516                  */
3517                 parm->hc_max_packet_size = 0x400;
3518                 parm->hc_max_packet_count = 1;
3519                 parm->hc_max_frame_size = EHCI_QTD_PAYLOAD_MAX;
3520                 xfer->flags_int.bdma_enable = 1;
3521
3522                 usbd_transfer_setup_sub(parm);
3523
3524                 nqh = 1;
3525                 nqtd = ((2 * xfer->nframes) + 1 /* STATUS */
3526                     + (xfer->max_data_length / xfer->max_hc_frame_size));
3527
3528         } else if (parm->methods == &ehci_device_bulk_methods) {
3529                 parm->hc_max_packet_size = 0x400;
3530                 parm->hc_max_packet_count = 1;
3531                 parm->hc_max_frame_size = EHCI_QTD_PAYLOAD_MAX;
3532                 xfer->flags_int.bdma_enable = 1;
3533
3534                 usbd_transfer_setup_sub(parm);
3535
3536                 nqh = 1;
3537                 nqtd = ((2 * xfer->nframes)
3538                     + (xfer->max_data_length / xfer->max_hc_frame_size));
3539
3540         } else if (parm->methods == &ehci_device_intr_methods) {
3541                 if (parm->speed == USB_SPEED_HIGH) {
3542                         parm->hc_max_packet_size = 0x400;
3543                         parm->hc_max_packet_count = 3;
3544                 } else if (parm->speed == USB_SPEED_FULL) {
3545                         parm->hc_max_packet_size = USB_FS_BYTES_PER_HS_UFRAME;
3546                         parm->hc_max_packet_count = 1;
3547                 } else {
3548                         parm->hc_max_packet_size = USB_FS_BYTES_PER_HS_UFRAME / 8;
3549                         parm->hc_max_packet_count = 1;
3550                 }
3551
3552                 parm->hc_max_frame_size = EHCI_QTD_PAYLOAD_MAX;
3553                 xfer->flags_int.bdma_enable = 1;
3554
3555                 usbd_transfer_setup_sub(parm);
3556
3557                 nqh = 1;
3558                 nqtd = ((2 * xfer->nframes)
3559                     + (xfer->max_data_length / xfer->max_hc_frame_size));
3560
3561         } else if (parm->methods == &ehci_device_isoc_fs_methods) {
3562                 parm->hc_max_packet_size = 0x3FF;
3563                 parm->hc_max_packet_count = 1;
3564                 parm->hc_max_frame_size = 0x3FF;
3565                 xfer->flags_int.bdma_enable = 1;
3566
3567                 usbd_transfer_setup_sub(parm);
3568
3569                 nsitd = xfer->nframes;
3570
3571         } else if (parm->methods == &ehci_device_isoc_hs_methods) {
3572                 parm->hc_max_packet_size = 0x400;
3573                 parm->hc_max_packet_count = 3;
3574                 parm->hc_max_frame_size = 0xC00;
3575                 xfer->flags_int.bdma_enable = 1;
3576
3577                 usbd_transfer_setup_sub(parm);
3578
3579                 nitd = ((xfer->nframes + 7) / 8) <<
3580                     usbd_xfer_get_fps_shift(xfer);
3581
3582         } else {
3583                 parm->hc_max_packet_size = 0x400;
3584                 parm->hc_max_packet_count = 1;
3585                 parm->hc_max_frame_size = 0x400;
3586
3587                 usbd_transfer_setup_sub(parm);
3588         }
3589
3590 alloc_dma_set:
3591
3592         if (parm->err) {
3593                 return;
3594         }
3595         /*
3596          * Allocate queue heads and transfer descriptors
3597          */
3598         last_obj = NULL;
3599
3600         if (usbd_transfer_setup_sub_malloc(
3601             parm, &pc, sizeof(ehci_itd_t),
3602             EHCI_ITD_ALIGN, nitd)) {
3603                 parm->err = USB_ERR_NOMEM;
3604                 return;
3605         }
3606         if (parm->buf) {
3607                 for (n = 0; n != nitd; n++) {
3608                         ehci_itd_t *td;
3609
3610                         usbd_get_page(pc + n, 0, &page_info);
3611
3612                         td = page_info.buffer;
3613
3614                         /* init TD */
3615                         td->itd_self = htohc32(sc, page_info.physaddr | EHCI_LINK_ITD);
3616                         td->obj_next = last_obj;
3617                         td->page_cache = pc + n;
3618
3619                         last_obj = td;
3620
3621                         usb_pc_cpu_flush(pc + n);
3622                 }
3623         }
3624         if (usbd_transfer_setup_sub_malloc(
3625             parm, &pc, sizeof(ehci_sitd_t),
3626             EHCI_SITD_ALIGN, nsitd)) {
3627                 parm->err = USB_ERR_NOMEM;
3628                 return;
3629         }
3630         if (parm->buf) {
3631                 for (n = 0; n != nsitd; n++) {
3632                         ehci_sitd_t *td;
3633
3634                         usbd_get_page(pc + n, 0, &page_info);
3635
3636                         td = page_info.buffer;
3637
3638                         /* init TD */
3639                         td->sitd_self = htohc32(sc, page_info.physaddr | EHCI_LINK_SITD);
3640                         td->obj_next = last_obj;
3641                         td->page_cache = pc + n;
3642
3643                         last_obj = td;
3644
3645                         usb_pc_cpu_flush(pc + n);
3646                 }
3647         }
3648         if (usbd_transfer_setup_sub_malloc(
3649             parm, &pc, sizeof(ehci_qtd_t),
3650             EHCI_QTD_ALIGN, nqtd)) {
3651                 parm->err = USB_ERR_NOMEM;
3652                 return;
3653         }
3654         if (parm->buf) {
3655                 for (n = 0; n != nqtd; n++) {
3656                         ehci_qtd_t *qtd;
3657
3658                         usbd_get_page(pc + n, 0, &page_info);
3659
3660                         qtd = page_info.buffer;
3661
3662                         /* init TD */
3663                         qtd->qtd_self = htohc32(sc, page_info.physaddr);
3664                         qtd->obj_next = last_obj;
3665                         qtd->page_cache = pc + n;
3666
3667                         last_obj = qtd;
3668
3669                         usb_pc_cpu_flush(pc + n);
3670                 }
3671         }
3672         xfer->td_start[xfer->flags_int.curr_dma_set] = last_obj;
3673
3674         last_obj = NULL;
3675
3676         if (usbd_transfer_setup_sub_malloc(
3677             parm, &pc, sizeof(ehci_qh_t),
3678             EHCI_QH_ALIGN, nqh)) {
3679                 parm->err = USB_ERR_NOMEM;
3680                 return;
3681         }
3682         if (parm->buf) {
3683                 for (n = 0; n != nqh; n++) {
3684                         ehci_qh_t *qh;
3685
3686                         usbd_get_page(pc + n, 0, &page_info);
3687
3688                         qh = page_info.buffer;
3689
3690                         /* init QH */
3691                         qh->qh_self = htohc32(sc, page_info.physaddr | EHCI_LINK_QH);
3692                         qh->obj_next = last_obj;
3693                         qh->page_cache = pc + n;
3694
3695                         last_obj = qh;
3696
3697                         usb_pc_cpu_flush(pc + n);
3698                 }
3699         }
3700         xfer->qh_start[xfer->flags_int.curr_dma_set] = last_obj;
3701
3702         if (!xfer->flags_int.curr_dma_set) {
3703                 xfer->flags_int.curr_dma_set = 1;
3704                 goto alloc_dma_set;
3705         }
3706 }
3707
3708 static void
3709 ehci_xfer_unsetup(struct usb_xfer *xfer)
3710 {
3711         return;
3712 }
3713
3714 static void
3715 ehci_ep_init(struct usb_device *udev, struct usb_endpoint_descriptor *edesc,
3716     struct usb_endpoint *ep)
3717 {
3718         ehci_softc_t *sc = EHCI_BUS2SC(udev->bus);
3719
3720         DPRINTFN(2, "endpoint=%p, addr=%d, endpt=%d, mode=%d (%d)\n",
3721             ep, udev->address,
3722             edesc->bEndpointAddress, udev->flags.usb_mode,
3723             sc->sc_addr);
3724
3725         if (udev->device_index != sc->sc_addr) {
3726                 if ((udev->speed != USB_SPEED_HIGH) &&
3727                     ((udev->hs_hub_addr == 0) ||
3728                     (udev->hs_port_no == 0) ||
3729                     (udev->parent_hs_hub == NULL) ||
3730                     (udev->parent_hs_hub->hub == NULL))) {
3731                         /* We need a transaction translator */
3732                         goto done;
3733                 }
3734                 switch (edesc->bmAttributes & UE_XFERTYPE) {
3735                 case UE_CONTROL:
3736                         ep->methods = &ehci_device_ctrl_methods;
3737                         break;
3738                 case UE_INTERRUPT:
3739                         ep->methods = &ehci_device_intr_methods;
3740                         break;
3741                 case UE_ISOCHRONOUS:
3742                         if (udev->speed == USB_SPEED_HIGH) {
3743                                 ep->methods = &ehci_device_isoc_hs_methods;
3744                         } else if (udev->speed == USB_SPEED_FULL) {
3745                                 ep->methods = &ehci_device_isoc_fs_methods;
3746                         }
3747                         break;
3748                 case UE_BULK:
3749                         ep->methods = &ehci_device_bulk_methods;
3750                         break;
3751                 default:
3752                         /* do nothing */
3753                         break;
3754                 }
3755         }
3756 done:
3757         return;
3758 }
3759
3760 static void
3761 ehci_get_dma_delay(struct usb_device *udev, uint32_t *pus)
3762 {
3763         /*
3764          * Wait until the hardware has finished any possible use of
3765          * the transfer descriptor(s) and QH
3766          */
3767         *pus = (1125);                  /* microseconds */
3768 }
3769
3770 static void
3771 ehci_device_resume(struct usb_device *udev)
3772 {
3773         ehci_softc_t *sc = EHCI_BUS2SC(udev->bus);
3774         struct usb_xfer *xfer;
3775         const struct usb_pipe_methods *methods;
3776
3777         DPRINTF("\n");
3778
3779         USB_BUS_LOCK(udev->bus);
3780
3781         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
3782                 if (xfer->xroot->udev == udev) {
3783                         methods = xfer->endpoint->methods;
3784
3785                         if ((methods == &ehci_device_bulk_methods) ||
3786                             (methods == &ehci_device_ctrl_methods)) {
3787                                 EHCI_APPEND_QH(xfer->qh_start[xfer->flags_int.curr_dma_set],
3788                                     sc->sc_async_p_last);
3789                         }
3790                         if (methods == &ehci_device_intr_methods) {
3791                                 EHCI_APPEND_QH(xfer->qh_start[xfer->flags_int.curr_dma_set],
3792                                     sc->sc_intr_p_last[xfer->qh_pos]);
3793                         }
3794                 }
3795         }
3796
3797         USB_BUS_UNLOCK(udev->bus);
3798
3799         return;
3800 }
3801
3802 static void
3803 ehci_device_suspend(struct usb_device *udev)
3804 {
3805         ehci_softc_t *sc = EHCI_BUS2SC(udev->bus);
3806         struct usb_xfer *xfer;
3807         const struct usb_pipe_methods *methods;
3808
3809         DPRINTF("\n");
3810
3811         USB_BUS_LOCK(udev->bus);
3812
3813         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
3814                 if (xfer->xroot->udev == udev) {
3815                         methods = xfer->endpoint->methods;
3816
3817                         if ((methods == &ehci_device_bulk_methods) ||
3818                             (methods == &ehci_device_ctrl_methods)) {
3819                                 EHCI_REMOVE_QH(xfer->qh_start[xfer->flags_int.curr_dma_set],
3820                                     sc->sc_async_p_last);
3821                         }
3822                         if (methods == &ehci_device_intr_methods) {
3823                                 EHCI_REMOVE_QH(xfer->qh_start[xfer->flags_int.curr_dma_set],
3824                                     sc->sc_intr_p_last[xfer->qh_pos]);
3825                         }
3826                 }
3827         }
3828
3829         USB_BUS_UNLOCK(udev->bus);
3830 }
3831
3832 static void
3833 ehci_set_hw_power_sleep(struct usb_bus *bus, uint32_t state)
3834 {
3835         struct ehci_softc *sc = EHCI_BUS2SC(bus);
3836
3837         switch (state) {
3838         case USB_HW_POWER_SUSPEND:
3839         case USB_HW_POWER_SHUTDOWN:
3840                 ehci_suspend(sc);
3841                 break;
3842         case USB_HW_POWER_RESUME:
3843                 ehci_resume(sc);
3844                 break;
3845         default:
3846                 break;
3847         }
3848 }
3849
3850 static void
3851 ehci_set_hw_power(struct usb_bus *bus)
3852 {
3853         ehci_softc_t *sc = EHCI_BUS2SC(bus);
3854         uint32_t temp;
3855         uint32_t flags;
3856
3857         DPRINTF("\n");
3858
3859         USB_BUS_LOCK(bus);
3860
3861         flags = bus->hw_power_state;
3862
3863         temp = EOREAD4(sc, EHCI_USBCMD);
3864
3865         temp &= ~(EHCI_CMD_ASE | EHCI_CMD_PSE);
3866
3867         if (flags & (USB_HW_POWER_CONTROL |
3868             USB_HW_POWER_BULK)) {
3869                 DPRINTF("Async is active\n");
3870                 temp |= EHCI_CMD_ASE;
3871         }
3872         if (flags & (USB_HW_POWER_INTERRUPT |
3873             USB_HW_POWER_ISOC)) {
3874                 DPRINTF("Periodic is active\n");
3875                 temp |= EHCI_CMD_PSE;
3876         }
3877         EOWRITE4(sc, EHCI_USBCMD, temp);
3878
3879         USB_BUS_UNLOCK(bus);
3880
3881         return;
3882 }
3883
3884 static void
3885 ehci_start_dma_delay_second(struct usb_xfer *xfer)
3886 {
3887         struct ehci_softc *sc = EHCI_BUS2SC(xfer->xroot->bus);
3888
3889         DPRINTF("\n");
3890
3891         /* trigger doorbell */
3892         ehci_doorbell_async(sc);
3893
3894         /* give the doorbell 4ms */
3895         usbd_transfer_timeout_ms(xfer,
3896             (void (*)(void *))&usb_dma_delay_done_cb, 4);
3897 }
3898
3899 /*
3900  * Ring the doorbell twice before freeing any DMA descriptors. Some host
3901  * controllers apparently cache the QH descriptors and need a message
3902  * that the cache needs to be discarded.
3903  */
3904 static void
3905 ehci_start_dma_delay(struct usb_xfer *xfer)
3906 {
3907         struct ehci_softc *sc = EHCI_BUS2SC(xfer->xroot->bus);
3908
3909         DPRINTF("\n");
3910
3911         /* trigger doorbell */
3912         ehci_doorbell_async(sc);
3913
3914         /* give the doorbell 4ms */
3915         usbd_transfer_timeout_ms(xfer,
3916             (void (*)(void *))&ehci_start_dma_delay_second, 4);
3917 }
3918
3919 static const struct usb_bus_methods ehci_bus_methods =
3920 {
3921         .endpoint_init = ehci_ep_init,
3922         .xfer_setup = ehci_xfer_setup,
3923         .xfer_unsetup = ehci_xfer_unsetup,
3924         .get_dma_delay = ehci_get_dma_delay,
3925         .device_resume = ehci_device_resume,
3926         .device_suspend = ehci_device_suspend,
3927         .set_hw_power = ehci_set_hw_power,
3928         .set_hw_power_sleep = ehci_set_hw_power_sleep,
3929         .roothub_exec = ehci_roothub_exec,
3930         .xfer_poll = ehci_do_poll,
3931         .start_dma_delay = ehci_start_dma_delay,
3932 };