sys/dev/usb/ehci.c

   1 /*      $NetBSD: ehci.c,v 1.91 2005/02/27 00:27:51 perry Exp $ */
   2
   3 /*-
   4  * Copyright (c) 2004 The NetBSD Foundation, Inc.
   5  * All rights reserved.
   6  *
   7  * This code is derived from software contributed to The NetBSD Foundation
   8  * by Lennart Augustsson (lennart@augustsson.net) and by Charles M. Hannum.
   9  *
  10  * Redistribution and use in source and binary forms, with or without
  11  * modification, are permitted provided that the following conditions
  12  * are met:
  13  * 1. Redistributions of source code must retain the above copyright
  14  *    notice, this list of conditions and the following disclaimer.
  15  * 2. Redistributions in binary form must reproduce the above copyright
  16  *    notice, this list of conditions and the following disclaimer in the
  17  *    documentation and/or other materials provided with the distribution.
  18  * 3. All advertising materials mentioning features or use of this software
  19  *    must display the following acknowledgement:
  20  *        This product includes software developed by the NetBSD
  21  *        Foundation, Inc. and its contributors.
  22  * 4. Neither the name of The NetBSD Foundation nor the names of its
  23  *    contributors may be used to endorse or promote products derived
  24  *    from this software without specific prior written permission.
  25  *
  26  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
  27  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
  28  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  29  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
  30  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  31  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  32  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  33  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  34  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  35  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  36  * POSSIBILITY OF SUCH DAMAGE.
  37  */
  38
  39 /*
  40  * USB Enhanced Host Controller Driver, a.k.a. USB 2.0 controller.
  41  *
  42  * The EHCI 1.0 spec can be found at
  43  * http://developer.intel.com/technology/usb/download/ehci-r10.pdf
  44  * and the USB 2.0 spec at
  45  * http://www.usb.org/developers/docs/usb_20.zip
  46  *
  47  */
  48
  49 /*
  50  * TODO:
  51  * 1) The EHCI driver lacks support for isochronous transfers, so
  52  *    devices using them don't work.
  53  *
  54  * 2) Interrupt transfer scheduling does not manage the time available
  55  *    in each frame, so it is possible for the transfers to overrun
  56  *    the end of the frame.
  57  *
  58  * 3) Command failures are not recovered correctly.
  59  */
  60
  61 #include <sys/cdefs.h>
  62 __FBSDID("$FreeBSD$");
  63
  64 #include <sys/param.h>
  65 #include <sys/systm.h>
  66 #include <sys/malloc.h>
  67 #include <sys/kernel.h>
  68 #include <sys/endian.h>
  69 #include <sys/module.h>
  70 #include <sys/bus.h>
  71 #include <sys/lock.h>
  72 #include <sys/lockmgr.h>
  73 #if defined(DIAGNOSTIC) && defined(__i386__) && defined(__FreeBSD__)
  74 #include <machine/cpu.h>
  75 #endif
  76 #include <sys/proc.h>
  77 #include <sys/queue.h>
  78 #include <sys/sysctl.h>
  79
  80 #include <machine/bus.h>
  81 #include <machine/endian.h>
  82
  83 #include <dev/usb/usb.h>
  84 #include <dev/usb/usbdi.h>
  85 #include <dev/usb/usbdivar.h>
  86 #include <dev/usb/usb_mem.h>
  87 #include <dev/usb/usb_quirks.h>
  88
  89 #include <dev/usb/ehcireg.h>
  90 #include <dev/usb/ehcivar.h>
  91
  92 #define delay(d)                DELAY(d)
  93
  94 #ifdef USB_DEBUG
  95 #define EHCI_DEBUG USB_DEBUG
  96 #define DPRINTF(x)      do { if (ehcidebug) printf x; } while (0)
  97 #define DPRINTFN(n,x)   do { if (ehcidebug>(n)) printf x; } while (0)
  98 int ehcidebug = 0;
  99 SYSCTL_NODE(_hw_usb, OID_AUTO, ehci, CTLFLAG_RW, 0, "USB ehci");
 100 SYSCTL_INT(_hw_usb_ehci, OID_AUTO, debug, CTLFLAG_RW,
 101            &ehcidebug, 0, "ehci debug level");
 102 #define bitmask_snprintf(q,f,b,l) snprintf((b), (l), "%b", (q), (f))
 103 #else
 104 #define DPRINTF(x)
 105 #define DPRINTFN(n,x)
 106 #endif
 107
 108 struct ehci_pipe {
 109         struct usbd_pipe pipe;
 110
 111         ehci_soft_qh_t *sqh;
 112         union {
 113                 ehci_soft_qtd_t *qtd;
 114                 /* ehci_soft_itd_t *itd; */
 115         } tail;
 116         union {
 117                 /* Control pipe */
 118                 struct {
 119                         usb_dma_t reqdma;
 120                         u_int length;
 121                         /*ehci_soft_qtd_t *setup, *data, *stat;*/
 122                 } ctl;
 123                 /* Interrupt pipe */
 124                 struct {
 125                         u_int length;
 126                 } intr;
 127                 /* Bulk pipe */
 128                 struct {
 129                         u_int length;
 130                 } bulk;
 131                 /* Iso pipe */
 132                 /* XXX */
 133         } u;
 134 };
 135
 136 static usbd_status      ehci_open(usbd_pipe_handle);
 137 static void             ehci_poll(struct usbd_bus *);
 138 static void             ehci_softintr(void *);
 139 static int              ehci_intr1(ehci_softc_t *);
 140 static void             ehci_waitintr(ehci_softc_t *, usbd_xfer_handle);
 141 static void             ehci_check_intr(ehci_softc_t *, struct ehci_xfer *);
 142 static void             ehci_idone(struct ehci_xfer *);
 143 static void             ehci_timeout(void *);
 144 static void             ehci_timeout_task(void *);
 145 static void             ehci_intrlist_timeout(void *);
 146
 147 static usbd_status      ehci_allocm(struct usbd_bus *, usb_dma_t *, u_int32_t);
 148 static void             ehci_freem(struct usbd_bus *, usb_dma_t *);
 149
 150 static usbd_xfer_handle ehci_allocx(struct usbd_bus *);
 151 static void             ehci_freex(struct usbd_bus *, usbd_xfer_handle);
 152
 153 static usbd_status      ehci_root_ctrl_transfer(usbd_xfer_handle);
 154 static usbd_status      ehci_root_ctrl_start(usbd_xfer_handle);
 155 static void             ehci_root_ctrl_abort(usbd_xfer_handle);
 156 static void             ehci_root_ctrl_close(usbd_pipe_handle);
 157 static void             ehci_root_ctrl_done(usbd_xfer_handle);
 158
 159 static usbd_status      ehci_root_intr_transfer(usbd_xfer_handle);
 160 static usbd_status      ehci_root_intr_start(usbd_xfer_handle);
 161 static void             ehci_root_intr_abort(usbd_xfer_handle);
 162 static void             ehci_root_intr_close(usbd_pipe_handle);
 163 static void             ehci_root_intr_done(usbd_xfer_handle);
 164
 165 static usbd_status      ehci_device_ctrl_transfer(usbd_xfer_handle);
 166 static usbd_status      ehci_device_ctrl_start(usbd_xfer_handle);
 167 static void             ehci_device_ctrl_abort(usbd_xfer_handle);
 168 static void             ehci_device_ctrl_close(usbd_pipe_handle);
 169 static void             ehci_device_ctrl_done(usbd_xfer_handle);
 170
 171 static usbd_status      ehci_device_bulk_transfer(usbd_xfer_handle);
 172 static usbd_status      ehci_device_bulk_start(usbd_xfer_handle);
 173 static void             ehci_device_bulk_abort(usbd_xfer_handle);
 174 static void             ehci_device_bulk_close(usbd_pipe_handle);
 175 static void             ehci_device_bulk_done(usbd_xfer_handle);
 176
 177 static usbd_status      ehci_device_intr_transfer(usbd_xfer_handle);
 178 static usbd_status      ehci_device_intr_start(usbd_xfer_handle);
 179 static void             ehci_device_intr_abort(usbd_xfer_handle);
 180 static void             ehci_device_intr_close(usbd_pipe_handle);
 181 static void             ehci_device_intr_done(usbd_xfer_handle);
 182
 183 static usbd_status      ehci_device_isoc_transfer(usbd_xfer_handle);
 184 static usbd_status      ehci_device_isoc_start(usbd_xfer_handle);
 185 static void             ehci_device_isoc_abort(usbd_xfer_handle);
 186 static void             ehci_device_isoc_close(usbd_pipe_handle);
 187 static void             ehci_device_isoc_done(usbd_xfer_handle);
 188
 189 static void             ehci_device_clear_toggle(usbd_pipe_handle pipe);
 190 static void             ehci_noop(usbd_pipe_handle pipe);
 191
 192 static int              ehci_str(usb_string_descriptor_t *, int, char *);
 193 static void             ehci_pcd(ehci_softc_t *, usbd_xfer_handle);
 194 static void             ehci_pcd_able(ehci_softc_t *, int);
 195 static void             ehci_pcd_enable(void *);
 196 static void             ehci_disown(ehci_softc_t *, int, int);
 197
 198 static ehci_soft_qh_t  *ehci_alloc_sqh(ehci_softc_t *);
 199 static void             ehci_free_sqh(ehci_softc_t *, ehci_soft_qh_t *);
 200
 201 static ehci_soft_qtd_t  *ehci_alloc_sqtd(ehci_softc_t *);
 202 static void             ehci_free_sqtd(ehci_softc_t *, ehci_soft_qtd_t *);
 203 static usbd_status      ehci_alloc_sqtd_chain(struct ehci_pipe *,
 204                             ehci_softc_t *, int, int, usbd_xfer_handle,
 205                             ehci_soft_qtd_t *, ehci_soft_qtd_t *,
 206                             ehci_soft_qtd_t **, ehci_soft_qtd_t **);
 207 static void             ehci_free_sqtd_chain(ehci_softc_t *, ehci_soft_qh_t *,
 208                             ehci_soft_qtd_t *, ehci_soft_qtd_t *);
 209
 210 static usbd_status      ehci_device_request(usbd_xfer_handle xfer);
 211
 212 static usbd_status      ehci_device_setintr(ehci_softc_t *, ehci_soft_qh_t *,
 213                             int ival);
 214
 215 static void             ehci_add_qh(ehci_soft_qh_t *, ehci_soft_qh_t *);
 216 static void             ehci_rem_qh(ehci_softc_t *, ehci_soft_qh_t *,
 217                                     ehci_soft_qh_t *);
 218 static void             ehci_activate_qh(ehci_soft_qh_t *, ehci_soft_qtd_t *);
 219 static void             ehci_sync_hc(ehci_softc_t *);
 220
 221 static void             ehci_close_pipe(usbd_pipe_handle, ehci_soft_qh_t *);
 222 static void             ehci_abort_xfer(usbd_xfer_handle, usbd_status);
 223
 224 #ifdef EHCI_DEBUG
 225 static void             ehci_dump_regs(ehci_softc_t *);
 226 void                    ehci_dump(void);
 227 static ehci_softc_t     *theehci;
 228 static void             ehci_dump_link(ehci_link_t, int);
 229 static void             ehci_dump_sqtds(ehci_soft_qtd_t *);
 230 static void             ehci_dump_sqtd(ehci_soft_qtd_t *);
 231 static void             ehci_dump_qtd(ehci_qtd_t *);
 232 static void             ehci_dump_sqh(ehci_soft_qh_t *);
 233 #ifdef DIAGNOSTIC
 234 static void             ehci_dump_exfer(struct ehci_xfer *);
 235 #endif
 236 #endif
 237
 238 #define EHCI_NULL htole32(EHCI_LINK_TERMINATE)
 239
 240 #define EHCI_INTR_ENDPT 1
 241
 242 #define ehci_add_intr_list(sc, ex) \
 243         LIST_INSERT_HEAD(&(sc)->sc_intrhead, (ex), inext);
 244 #define ehci_del_intr_list(ex) \
 245         do { \
 246                 LIST_REMOVE((ex), inext); \
 247                 (ex)->inext.le_prev = NULL; \
 248         } while (0)
 249 #define ehci_active_intr_list(ex) ((ex)->inext.le_prev != NULL)
 250
 251 static struct usbd_bus_methods ehci_bus_methods = {
 252         ehci_open,
 253         ehci_softintr,
 254         ehci_poll,
 255         ehci_allocm,
 256         ehci_freem,
 257         ehci_allocx,
 258         ehci_freex,
 259 };
 260
 261 static struct usbd_pipe_methods ehci_root_ctrl_methods = {
 262         ehci_root_ctrl_transfer,
 263         ehci_root_ctrl_start,
 264         ehci_root_ctrl_abort,
 265         ehci_root_ctrl_close,
 266         ehci_noop,
 267         ehci_root_ctrl_done,
 268 };
 269
 270 static struct usbd_pipe_methods ehci_root_intr_methods = {
 271         ehci_root_intr_transfer,
 272         ehci_root_intr_start,
 273         ehci_root_intr_abort,
 274         ehci_root_intr_close,
 275         ehci_noop,
 276         ehci_root_intr_done,
 277 };
 278
 279 static struct usbd_pipe_methods ehci_device_ctrl_methods = {
 280         ehci_device_ctrl_transfer,
 281         ehci_device_ctrl_start,
 282         ehci_device_ctrl_abort,
 283         ehci_device_ctrl_close,
 284         ehci_noop,
 285         ehci_device_ctrl_done,
 286 };
 287
 288 static struct usbd_pipe_methods ehci_device_intr_methods = {
 289         ehci_device_intr_transfer,
 290         ehci_device_intr_start,
 291         ehci_device_intr_abort,
 292         ehci_device_intr_close,
 293         ehci_device_clear_toggle,
 294         ehci_device_intr_done,
 295 };
 296
 297 static struct usbd_pipe_methods ehci_device_bulk_methods = {
 298         ehci_device_bulk_transfer,
 299         ehci_device_bulk_start,
 300         ehci_device_bulk_abort,
 301         ehci_device_bulk_close,
 302         ehci_device_clear_toggle,
 303         ehci_device_bulk_done,
 304 };
 305
 306 static struct usbd_pipe_methods ehci_device_isoc_methods = {
 307         ehci_device_isoc_transfer,
 308         ehci_device_isoc_start,
 309         ehci_device_isoc_abort,
 310         ehci_device_isoc_close,
 311         ehci_noop,
 312         ehci_device_isoc_done,
 313 };
 314
 315 static usbd_status
 316 ehci_hcreset(ehci_softc_t *sc)
 317 {
 318         u_int32_t hcr;
 319         u_int i;
 320
 321         EOWRITE4(sc, EHCI_USBCMD, 0);   /* Halt controller */
 322         for (i = 0; i < 100; i++) {
 323                 usb_delay_ms(&sc->sc_bus, 1);
 324                 hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
 325                 if (hcr)
 326                         break;
 327         }
 328         if (!hcr)
 329                 /*
 330                  * Fall through and try reset anyway even though
 331                  * Table 2-9 in the EHCI spec says this will result
 332                  * in undefined behavior.
 333                  */
 334                 printf("%s: stop timeout\n",
 335                        device_get_nameunit(sc->sc_bus.bdev));
 336
 337         EOWRITE4(sc, EHCI_USBCMD, EHCI_CMD_HCRESET);
 338         for (i = 0; i < 100; i++) {
 339                 usb_delay_ms(&sc->sc_bus, 1);
 340                 hcr = EOREAD4(sc, EHCI_USBCMD) & EHCI_CMD_HCRESET;
 341                 if (!hcr)
 342                         return (USBD_NORMAL_COMPLETION);
 343         }
 344         printf("%s: reset timeout\n", device_get_nameunit(sc->sc_bus.bdev));
 345         return (USBD_IOERROR);
 346 }
 347
 348 usbd_status
 349 ehci_init(ehci_softc_t *sc)
 350 {
 351         u_int32_t version, sparams, cparams, hcr;
 352         u_int i;
 353         usbd_status err;
 354         ehci_soft_qh_t *sqh;
 355         u_int ncomp;
 356         int lev;
 357
 358         DPRINTF(("ehci_init: start\n"));
 359 #ifdef EHCI_DEBUG
 360         theehci = sc;
 361 #endif
 362
 363         sc->sc_offs = EREAD1(sc, EHCI_CAPLENGTH);
 364
 365         version = EREAD2(sc, EHCI_HCIVERSION);
 366         printf("%s: EHCI version %x.%x\n", device_get_nameunit(sc->sc_bus.bdev),
 367                version >> 8, version & 0xff);
 368
 369         sparams = EREAD4(sc, EHCI_HCSPARAMS);
 370         DPRINTF(("ehci_init: sparams=0x%x\n", sparams));
 371         sc->sc_npcomp = EHCI_HCS_N_PCC(sparams);
 372         ncomp = EHCI_HCS_N_CC(sparams);
 373         if (ncomp != sc->sc_ncomp) {
 374                 printf("%s: wrong number of companions (%d != %d)\n",
 375                        device_get_nameunit(sc->sc_bus.bdev),
 376                        ncomp, sc->sc_ncomp);
 377                 if (ncomp < sc->sc_ncomp)
 378                         sc->sc_ncomp = ncomp;
 379         }
 380         if (sc->sc_ncomp > 0) {
 381                 printf("%s: companion controller%s, %d port%s each:",
 382                     device_get_nameunit(sc->sc_bus.bdev), sc->sc_ncomp!=1 ? "s" : "",
 383                     EHCI_HCS_N_PCC(sparams),
 384                     EHCI_HCS_N_PCC(sparams)!=1 ? "s" : "");
 385                 for (i = 0; i < sc->sc_ncomp; i++)
 386                         printf(" %s", device_get_nameunit(sc->sc_comps[i]->bdev));
 387                 printf("\n");
 388         }
 389         sc->sc_noport = EHCI_HCS_N_PORTS(sparams);
 390         cparams = EREAD4(sc, EHCI_HCCPARAMS);
 391         DPRINTF(("ehci_init: cparams=0x%x\n", cparams));
 392
 393         if (EHCI_HCC_64BIT(cparams)) {
 394                 /* MUST clear segment register if 64 bit capable. */
 395                 EWRITE4(sc, EHCI_CTRLDSSEGMENT, 0);
 396         }
 397
 398         sc->sc_bus.usbrev = USBREV_2_0;
 399
 400         /* Reset the controller */
 401         DPRINTF(("%s: resetting\n", device_get_nameunit(sc->sc_bus.bdev)));
 402         err = ehci_hcreset(sc);
 403         if (err != USBD_NORMAL_COMPLETION)
 404                 return (err);
 405
 406         /* frame list size at default, read back what we got and use that */
 407         switch (EHCI_CMD_FLS(EOREAD4(sc, EHCI_USBCMD))) {
 408         case 0: sc->sc_flsize = 1024; break;
 409         case 1: sc->sc_flsize = 512; break;
 410         case 2: sc->sc_flsize = 256; break;
 411         case 3: return (USBD_IOERROR);
 412         }
 413         err = usb_allocmem(&sc->sc_bus, sc->sc_flsize * sizeof(ehci_link_t),
 414             EHCI_FLALIGN_ALIGN, &sc->sc_fldma);
 415         if (err)
 416                 return (err);
 417         DPRINTF(("%s: flsize=%d\n", device_get_nameunit(sc->sc_bus.bdev),sc->sc_flsize));
 418         sc->sc_flist = KERNADDR(&sc->sc_fldma, 0);
 419         EOWRITE4(sc, EHCI_PERIODICLISTBASE, DMAADDR(&sc->sc_fldma, 0));
 420
 421         /* Set up the bus struct. */
 422         sc->sc_bus.methods = &ehci_bus_methods;
 423         sc->sc_bus.pipe_size = sizeof(struct ehci_pipe);
 424
 425 #if defined(__NetBSD__) || defined(__OpenBSD__)
 426         sc->sc_powerhook = powerhook_establish(ehci_power, sc);
 427         sc->sc_shutdownhook = shutdownhook_establish(ehci_shutdown, sc);
 428 #endif
 429
 430         sc->sc_eintrs = EHCI_NORMAL_INTRS;
 431
 432         /*
 433          * Allocate the interrupt dummy QHs. These are arranged to give
 434          * poll intervals that are powers of 2 times 1ms.
 435          */
 436         for (i = 0; i < EHCI_INTRQHS; i++) {
 437                 sqh = ehci_alloc_sqh(sc);
 438                 if (sqh == NULL) {
 439                         err = USBD_NOMEM;
 440                         goto bad1;
 441                 }
 442                 sc->sc_islots[i].sqh = sqh;
 443         }
 444         lev = 0;
 445         for (i = 0; i < EHCI_INTRQHS; i++) {
 446                 if (i == EHCI_IQHIDX(lev + 1, 0))
 447                         lev++;
 448                 sqh = sc->sc_islots[i].sqh;
 449                 if (i == 0) {
 450                         /* The last (1ms) QH terminates. */
 451                         sqh->qh.qh_link = EHCI_NULL;
 452                         sqh->next = NULL;
 453                 } else {
 454                         /* Otherwise the next QH has half the poll interval */
 455                         sqh->next =
 456                             sc->sc_islots[EHCI_IQHIDX(lev - 1, i + 1)].sqh;
 457                         sqh->qh.qh_link = htole32(sqh->next->physaddr |
 458                             EHCI_LINK_QH);
 459                 }
 460                 sqh->qh.qh_endp = htole32(EHCI_QH_SET_EPS(EHCI_QH_SPEED_HIGH));
 461                 sqh->qh.qh_endphub = htole32(EHCI_QH_SET_MULT(1));
 462                 sqh->qh.qh_curqtd = EHCI_NULL;
 463                 sqh->qh.qh_qtd.qtd_next = EHCI_NULL;
 464                 sqh->qh.qh_qtd.qtd_altnext = EHCI_NULL;
 465                 sqh->qh.qh_qtd.qtd_status = htole32(EHCI_QTD_HALTED);
 466         }
 467         /* Point the frame list at the last level (128ms). */
 468         for (i = 0; i < sc->sc_flsize; i++) {
 469                 sc->sc_flist[i] = htole32(EHCI_LINK_QH |
 470                     sc->sc_islots[EHCI_IQHIDX(EHCI_IPOLLRATES - 1,
 471                     i)].sqh->physaddr);
 472         }
 473
 474         /* Allocate dummy QH that starts the async list. */
 475         sqh = ehci_alloc_sqh(sc);
 476         if (sqh == NULL) {
 477                 err = USBD_NOMEM;
 478                 goto bad1;
 479         }
 480         /* Fill the QH */
 481         sqh->qh.qh_endp =
 482             htole32(EHCI_QH_SET_EPS(EHCI_QH_SPEED_HIGH) | EHCI_QH_HRECL);
 483         sqh->qh.qh_link =
 484             htole32(sqh->physaddr | EHCI_LINK_QH);
 485         sqh->qh.qh_curqtd = EHCI_NULL;
 486         sqh->prev = sqh; /*It's a circular list.. */
 487         sqh->next = sqh;
 488         /* Fill the overlay qTD */
 489         sqh->qh.qh_qtd.qtd_next = EHCI_NULL;
 490         sqh->qh.qh_qtd.qtd_altnext = EHCI_NULL;
 491         sqh->qh.qh_qtd.qtd_status = htole32(0);
 492 #ifdef EHCI_DEBUG
 493         if (ehcidebug) {
 494                 ehci_dump_sqh(sqh);
 495         }
 496 #endif
 497
 498         /* Point to async list */
 499         sc->sc_async_head = sqh;
 500         EOWRITE4(sc, EHCI_ASYNCLISTADDR, sqh->physaddr | EHCI_LINK_QH);
 501
 502         callout_init(&sc->sc_tmo_pcd, 0);
 503         callout_init(&sc->sc_tmo_intrlist, 0);
 504
 505         lockinit(&sc->sc_doorbell_lock, PZERO, "ehcidb", 0, 0);
 506
 507         /* Enable interrupts */
 508         EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
 509
 510         /* Turn on controller */
 511         EOWRITE4(sc, EHCI_USBCMD,
 512                  EHCI_CMD_ITC_2 | /* 2 microframes interrupt delay */
 513                  (EOREAD4(sc, EHCI_USBCMD) & EHCI_CMD_FLS_M) |
 514                  EHCI_CMD_ASE |
 515                  EHCI_CMD_PSE |
 516                  EHCI_CMD_RS);
 517
 518         /* Take over port ownership */
 519         EOWRITE4(sc, EHCI_CONFIGFLAG, EHCI_CONF_CF);
 520
 521         for (i = 0; i < 100; i++) {
 522                 usb_delay_ms(&sc->sc_bus, 1);
 523                 hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
 524                 if (!hcr)
 525                         break;
 526         }
 527         if (hcr) {
 528                 printf("%s: run timeout\n", device_get_nameunit(sc->sc_bus.bdev));
 529                 return (USBD_IOERROR);
 530         }
 531
 532         return (USBD_NORMAL_COMPLETION);
 533
 534 #if 0
 535  bad2:
 536         ehci_free_sqh(sc, sc->sc_async_head);
 537 #endif
 538  bad1:
 539         usb_freemem(&sc->sc_bus, &sc->sc_fldma);
 540         return (err);
 541 }
 542
 543 int
 544 ehci_intr(void *v)
 545 {
 546         ehci_softc_t *sc = v;
 547
 548         if (sc == NULL || sc->sc_dying)
 549                 return (0);
 550
 551         /* If we get an interrupt while polling, then just ignore it. */
 552         if (sc->sc_bus.use_polling) {
 553                 u_int32_t intrs = EHCI_STS_INTRS(EOREAD4(sc, EHCI_USBSTS));
 554
 555                 if (intrs)
 556                         EOWRITE4(sc, EHCI_USBSTS, intrs); /* Acknowledge */
 557 #ifdef DIAGNOSTIC
 558                 DPRINTFN(16, ("ehci_intr: ignored interrupt while polling\n"));
 559 #endif
 560                 return (0);
 561         }
 562
 563         return (ehci_intr1(sc));
 564 }
 565
 566 static int
 567 ehci_intr1(ehci_softc_t *sc)
 568 {
 569         u_int32_t intrs, eintrs;
 570
 571         DPRINTFN(20,("ehci_intr1: enter\n"));
 572
 573         /* In case the interrupt occurs before initialization has completed. */
 574         if (sc == NULL) {
 575 #ifdef DIAGNOSTIC
 576                 printf("ehci_intr1: sc == NULL\n");
 577 #endif
 578                 return (0);
 579         }
 580
 581         intrs = EHCI_STS_INTRS(EOREAD4(sc, EHCI_USBSTS));
 582         if (!intrs)
 583                 return (0);
 584
 585         eintrs = intrs & sc->sc_eintrs;
 586         DPRINTFN(7, ("ehci_intr1: sc=%p intrs=0x%x(0x%x) eintrs=0x%x\n",
 587                      sc, (u_int)intrs, EOREAD4(sc, EHCI_USBSTS),
 588                      (u_int)eintrs));
 589         if (!eintrs)
 590                 return (0);
 591
 592         EOWRITE4(sc, EHCI_USBSTS, intrs); /* Acknowledge */
 593         sc->sc_bus.intr_context++;
 594         sc->sc_bus.no_intrs++;
 595         if (eintrs & EHCI_STS_IAA) {
 596                 DPRINTF(("ehci_intr1: door bell\n"));
 597                 wakeup(&sc->sc_async_head);
 598                 eintrs &= ~EHCI_STS_IAA;
 599         }
 600         if (eintrs & (EHCI_STS_INT | EHCI_STS_ERRINT)) {
 601                 DPRINTFN(5,("ehci_intr1: %s %s\n",
 602                             eintrs & EHCI_STS_INT ? "INT" : "",
 603                             eintrs & EHCI_STS_ERRINT ? "ERRINT" : ""));
 604                 usb_schedsoftintr(&sc->sc_bus);
 605                 eintrs &= ~(EHCI_STS_INT | EHCI_STS_ERRINT);
 606         }
 607         if (eintrs & EHCI_STS_HSE) {
 608                 printf("%s: unrecoverable error, controller halted\n",
 609                        device_get_nameunit(sc->sc_bus.bdev));
 610                 /* XXX what else */
 611         }
 612         if (eintrs & EHCI_STS_PCD) {
 613                 ehci_pcd(sc, sc->sc_intrxfer);
 614                 /*
 615                  * Disable PCD interrupt for now, because it will be
 616                  * on until the port has been reset.
 617                  */
 618                 ehci_pcd_able(sc, 0);
 619                 /* Do not allow RHSC interrupts > 1 per second */
 620                 callout_reset(&sc->sc_tmo_pcd, hz, ehci_pcd_enable, sc);
 621                 eintrs &= ~EHCI_STS_PCD;
 622         }
 623
 624         sc->sc_bus.intr_context--;
 625
 626         if (eintrs != 0) {
 627                 /* Block unprocessed interrupts. */
 628                 sc->sc_eintrs &= ~eintrs;
 629                 EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
 630                 printf("%s: blocking intrs 0x%x\n",
 631                        device_get_nameunit(sc->sc_bus.bdev), eintrs);
 632         }
 633
 634         return (1);
 635 }
 636
 637 void
 638 ehci_pcd_able(ehci_softc_t *sc, int on)
 639 {
 640         DPRINTFN(4, ("ehci_pcd_able: on=%d\n", on));
 641         if (on)
 642                 sc->sc_eintrs |= EHCI_STS_PCD;
 643         else
 644                 sc->sc_eintrs &= ~EHCI_STS_PCD;
 645         EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
 646 }
 647
 648 void
 649 ehci_pcd_enable(void *v_sc)
 650 {
 651         ehci_softc_t *sc = v_sc;
 652
 653         ehci_pcd_able(sc, 1);
 654 }
 655
 656 /*
 657  * XXX write back xfer data for architectures with a write-back
 658  *     data cache; this is a hack because usb is mis-architected
 659  *     in blindly mixing bus_dma w/ PIO.
 660  */
 661 static __inline void
 662 hacksync(usbd_xfer_handle xfer)
 663 {
 664         bus_dma_tag_t tag;
 665         struct usb_dma_mapping *dmap;
 666
 667         if (xfer->length == 0)
 668                 return;
 669         tag = xfer->pipe->device->bus->buffer_dmatag;
 670         dmap = &xfer->dmamap;
 671         bus_dmamap_sync(tag, dmap->map, BUS_DMASYNC_PREWRITE);
 672 }
 673
 674 void
 675 ehci_pcd(ehci_softc_t *sc, usbd_xfer_handle xfer)
 676 {
 677         usbd_pipe_handle pipe;
 678         u_char *p;
 679         int i, m;
 680
 681         if (xfer == NULL) {
 682                 /* Just ignore the change. */
 683                 return;
 684         }
 685
 686         pipe = xfer->pipe;
 687
 688         p = xfer->buffer;
 689         m = min(sc->sc_noport, xfer->length * 8 - 1);
 690         memset(p, 0, xfer->length);
 691         for (i = 1; i <= m; i++) {
 692                 /* Pick out CHANGE bits from the status reg. */
 693                 if (EOREAD4(sc, EHCI_PORTSC(i)) & EHCI_PS_CLEAR)
 694                         p[i/8] |= 1 << (i%8);
 695         }
 696         DPRINTF(("ehci_pcd: change=0x%02x\n", *p));
 697         xfer->actlen = xfer->length;
 698         xfer->status = USBD_NORMAL_COMPLETION;
 699
 700         hacksync(xfer); /* XXX to compensate for usb_transfer_complete */
 701         usb_transfer_complete(xfer);
 702 }
 703
 704 void
 705 ehci_softintr(void *v)
 706 {
 707         ehci_softc_t *sc = v;
 708         struct ehci_xfer *ex, *nextex;
 709
 710         DPRINTFN(10,("%s: ehci_softintr (%d)\n", device_get_nameunit(sc->sc_bus.bdev),
 711                      sc->sc_bus.intr_context));
 712
 713         sc->sc_bus.intr_context++;
 714
 715         /*
 716          * The only explanation I can think of for why EHCI is as brain dead
 717          * as UHCI interrupt-wise is that Intel was involved in both.
 718          * An interrupt just tells us that something is done, we have no
 719          * clue what, so we need to scan through all active transfers. :-(
 720          */
 721         for (ex = LIST_FIRST(&sc->sc_intrhead); ex; ex = nextex) {
 722                 nextex = LIST_NEXT(ex, inext);
 723                 ehci_check_intr(sc, ex);
 724         }
 725
 726         /* Schedule a callout to catch any dropped transactions. */
 727         if ((sc->sc_flags & EHCI_SCFLG_LOSTINTRBUG) &&
 728             !LIST_EMPTY(&sc->sc_intrhead))
 729                 callout_reset(&sc->sc_tmo_intrlist, hz / 5,
 730                     ehci_intrlist_timeout, sc);
 731
 732 #ifdef USB_USE_SOFTINTR
 733         if (sc->sc_softwake) {
 734                 sc->sc_softwake = 0;
 735                 wakeup(&sc->sc_softwake);
 736         }
 737 #endif /* USB_USE_SOFTINTR */
 738
 739         sc->sc_bus.intr_context--;
 740 }
 741
 742 /* Check for an interrupt. */
 743 void
 744 ehci_check_intr(ehci_softc_t *sc, struct ehci_xfer *ex)
 745 {
 746         ehci_soft_qtd_t *sqtd, *lsqtd;
 747         u_int32_t status;
 748
 749         DPRINTFN(/*15*/2, ("ehci_check_intr: ex=%p\n", ex));
 750
 751         if (ex->sqtdstart == NULL) {
 752                 printf("ehci_check_intr: sqtdstart=NULL\n");
 753                 return;
 754         }
 755         lsqtd = ex->sqtdend;
 756 #ifdef DIAGNOSTIC
 757         if (lsqtd == NULL) {
 758                 printf("ehci_check_intr: lsqtd==0\n");
 759                 return;
 760         }
 761 #endif
 762         /*
 763          * If the last TD is still active we need to check whether there
 764          * is a an error somewhere in the middle, or whether there was a
 765          * short packet (SPD and not ACTIVE).
 766          */
 767         if (le32toh(lsqtd->qtd.qtd_status) & EHCI_QTD_ACTIVE) {
 768                 DPRINTFN(12, ("ehci_check_intr: active ex=%p\n", ex));
 769                 for (sqtd = ex->sqtdstart; sqtd != lsqtd; sqtd=sqtd->nextqtd) {
 770                         status = le32toh(sqtd->qtd.qtd_status);
 771                         /* If there's an active QTD the xfer isn't done. */
 772                         if (status & EHCI_QTD_ACTIVE)
 773                                 break;
 774                         /* Any kind of error makes the xfer done. */
 775                         if (status & EHCI_QTD_HALTED)
 776                                 goto done;
 777                         /* We want short packets, and it is short: it's done */
 778                         if (EHCI_QTD_GET_BYTES(status) != 0)
 779                                 goto done;
 780                 }
 781                 DPRINTFN(12, ("ehci_check_intr: ex=%p std=%p still active\n",
 782                               ex, ex->sqtdstart));
 783                 return;
 784         }
 785  done:
 786         DPRINTFN(12, ("ehci_check_intr: ex=%p done\n", ex));
 787         callout_stop(&ex->xfer.timeout_handle);
 788         usb_rem_task(ex->xfer.pipe->device, &ex->abort_task);
 789         ehci_idone(ex);
 790 }
 791
 792 void
 793 ehci_idone(struct ehci_xfer *ex)
 794 {
 795         usbd_xfer_handle xfer = &ex->xfer;
 796         struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
 797         ehci_soft_qtd_t *sqtd, *lsqtd;
 798         u_int32_t status = 0, nstatus = 0;
 799         ehci_physaddr_t nextphys, altnextphys;
 800         int actlen, cerr;
 801
 802         DPRINTFN(/*12*/2, ("ehci_idone: ex=%p\n", ex));
 803 #ifdef DIAGNOSTIC
 804         {
 805                 int s = splhigh();
 806                 if (ex->isdone) {
 807                         splx(s);
 808 #ifdef EHCI_DEBUG
 809                         printf("ehci_idone: ex is done!\n   ");
 810                         ehci_dump_exfer(ex);
 811 #else
 812                         printf("ehci_idone: ex=%p is done!\n", ex);
 813 #endif
 814                         return;
 815                 }
 816                 ex->isdone = 1;
 817                 splx(s);
 818         }
 819 #endif
 820
 821         if (xfer->status == USBD_CANCELLED ||
 822             xfer->status == USBD_TIMEOUT) {
 823                 DPRINTF(("ehci_idone: aborted xfer=%p\n", xfer));
 824                 return;
 825         }
 826
 827 #ifdef EHCI_DEBUG
 828         DPRINTFN(/*10*/2, ("ehci_idone: xfer=%p, pipe=%p ready\n", xfer, epipe));
 829         if (ehcidebug > 10)
 830                 ehci_dump_sqtds(ex->sqtdstart);
 831 #endif
 832
 833         /*
 834          * Make sure that the QH overlay qTD does not reference any
 835          * of the qTDs we are about to free. This is probably only
 836          * necessary if the transfer is marked as HALTED.
 837          */
 838         nextphys = EHCI_LINK_ADDR(le32toh(epipe->sqh->qh.qh_qtd.qtd_next));
 839         altnextphys =
 840             EHCI_LINK_ADDR(le32toh(epipe->sqh->qh.qh_qtd.qtd_altnext));
 841         for (sqtd = ex->sqtdstart; sqtd != ex->sqtdend->nextqtd;
 842              sqtd = sqtd->nextqtd) {
 843                 if (sqtd->physaddr == nextphys) {
 844                         epipe->sqh->qh.qh_qtd.qtd_next =
 845                             htole32(ex->sqtdend->nextqtd->physaddr);
 846                         DPRINTFN(4, ("ehci_idone: updated overlay next ptr\n"));
 847
 848                 }
 849                 if (sqtd->physaddr == altnextphys) {
 850                         DPRINTFN(4,
 851                             ("ehci_idone: updated overlay altnext ptr\n"));
 852                         epipe->sqh->qh.qh_qtd.qtd_altnext =
 853                             htole32(ex->sqtdend->nextqtd->physaddr);
 854                 }
 855         }
 856
 857         /* The transfer is done, compute actual length and status. */
 858         lsqtd = ex->sqtdend;
 859         actlen = 0;
 860         for (sqtd = ex->sqtdstart; sqtd != lsqtd->nextqtd; sqtd=sqtd->nextqtd) {
 861                 nstatus = le32toh(sqtd->qtd.qtd_status);
 862                 if (nstatus & EHCI_QTD_ACTIVE)
 863                         break;
 864
 865                 status = nstatus;
 866                 /* halt is ok if descriptor is last, and complete */
 867                 if (sqtd == lsqtd && EHCI_QTD_GET_BYTES(status) == 0)
 868                         status &= ~EHCI_QTD_HALTED;
 869                 if (EHCI_QTD_GET_PID(status) != EHCI_QTD_PID_SETUP)
 870                         actlen += sqtd->len - EHCI_QTD_GET_BYTES(status);
 871         }
 872
 873         cerr = EHCI_QTD_GET_CERR(status);
 874         DPRINTFN(/*10*/2, ("ehci_idone: len=%d, actlen=%d, cerr=%d, "
 875             "status=0x%x\n", xfer->length, actlen, cerr, status));
 876         xfer->actlen = actlen;
 877         if ((status & EHCI_QTD_HALTED) != 0) {
 878 #ifdef EHCI_DEBUG
 879                 char sbuf[128];
 880
 881                 bitmask_snprintf((u_int32_t)status,
 882                     "\20\7HALTED\6BUFERR\5BABBLE\4XACTERR"
 883                     "\3MISSED\2SPLIT\1PING", sbuf, sizeof(sbuf));
 884
 885                 DPRINTFN(2,
 886                          ("ehci_idone: error, addr=%d, endpt=0x%02x, "
 887                           "status 0x%s\n",
 888                           xfer->pipe->device->address,
 889                           xfer->pipe->endpoint->edesc->bEndpointAddress,
 890                           sbuf));
 891                 if (ehcidebug > 2) {
 892                         ehci_dump_sqh(epipe->sqh);
 893                         ehci_dump_sqtds(ex->sqtdstart);
 894                 }
 895 #endif
 896                 if ((status & EHCI_QTD_BABBLE) == 0 && cerr > 0)
 897                         xfer->status = USBD_STALLED;
 898                 else
 899                         xfer->status = USBD_IOERROR; /* more info XXX */
 900         } else {
 901                 xfer->status = USBD_NORMAL_COMPLETION;
 902         }
 903
 904         usb_transfer_complete(xfer);
 905         DPRINTFN(/*12*/2, ("ehci_idone: ex=%p done\n", ex));
 906 }
 907
 908 /*
 909  * Wait here until controller claims to have an interrupt.
 910  * Then call ehci_intr and return.  Use timeout to avoid waiting
 911  * too long.
 912  */
 913 void
 914 ehci_waitintr(ehci_softc_t *sc, usbd_xfer_handle xfer)
 915 {
 916         int timo = xfer->timeout;
 917         int usecs;
 918         u_int32_t intrs;
 919
 920         xfer->status = USBD_IN_PROGRESS;
 921         for (usecs = timo * 1000000 / hz; usecs > 0; usecs -= 1000) {
 922                 usb_delay_ms(&sc->sc_bus, 1);
 923                 if (sc->sc_dying)
 924                         break;
 925                 intrs = EHCI_STS_INTRS(EOREAD4(sc, EHCI_USBSTS)) &
 926                         sc->sc_eintrs;
 927                 DPRINTFN(15,("ehci_waitintr: 0x%04x\n", intrs));
 928 #ifdef EHCI_DEBUG
 929                 if (ehcidebug > 15)
 930                         ehci_dump_regs(sc);
 931 #endif
 932                 if (intrs) {
 933                         ehci_intr1(sc);
 934                         if (xfer->status != USBD_IN_PROGRESS)
 935                                 return;
 936                 }
 937         }
 938
 939         /* Timeout */
 940         DPRINTF(("ehci_waitintr: timeout\n"));
 941         xfer->status = USBD_TIMEOUT;
 942         usb_transfer_complete(xfer);
 943         /* XXX should free TD */
 944 }
 945
 946 void
 947 ehci_poll(struct usbd_bus *bus)
 948 {
 949         ehci_softc_t *sc = (ehci_softc_t *)bus;
 950 #ifdef EHCI_DEBUG
 951         static int last;
 952         int new;
 953         new = EHCI_STS_INTRS(EOREAD4(sc, EHCI_USBSTS));
 954         if (new != last) {
 955                 DPRINTFN(10,("ehci_poll: intrs=0x%04x\n", new));
 956                 last = new;
 957         }
 958 #endif
 959
 960         if (EOREAD4(sc, EHCI_USBSTS) & sc->sc_eintrs)
 961                 ehci_intr1(sc);
 962 }
 963
 964 int
 965 ehci_detach(struct ehci_softc *sc, int flags)
 966 {
 967         int rv = 0;
 968
 969         sc->sc_dying = 1;
 970
 971         EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
 972         (void) ehci_hcreset(sc);
 973         callout_stop(&sc->sc_tmo_intrlist);
 974         callout_stop(&sc->sc_tmo_pcd);
 975
 976 #if defined(__NetBSD__) || defined(__OpenBSD__)
 977         if (sc->sc_powerhook != NULL)
 978                 powerhook_disestablish(sc->sc_powerhook);
 979         if (sc->sc_shutdownhook != NULL)
 980                 shutdownhook_disestablish(sc->sc_shutdownhook);
 981 #endif
 982         usb_delay_ms(&sc->sc_bus, 300); /* XXX let stray task complete */
 983
 984         usb_freemem(&sc->sc_bus, &sc->sc_fldma);
 985         /* XXX free other data structures XXX */
 986
 987         return (rv);
 988 }
 989
 990 /*
 991  * Handle suspend/resume.
 992  *
 993  * We need to switch to polling mode here, because this routine is
 994  * called from an interrupt context.  This is all right since we
 995  * are almost suspended anyway.
 996  */
 997 void
 998 ehci_power(int why, void *v)
 999 {
1000         ehci_softc_t *sc = v;
1001         u_int32_t cmd, hcr;
1002         int s, i;
1003
1004 #ifdef EHCI_DEBUG
1005         DPRINTF(("ehci_power: sc=%p, why=%d\n", sc, why));
1006         if (ehcidebug > 0)
1007                 ehci_dump_regs(sc);
1008 #endif
1009
1010         s = splhardusb();
1011         switch (why) {
1012         case PWR_SUSPEND:
1013         case PWR_STANDBY:
1014                 sc->sc_bus.use_polling++;
1015
1016                 for (i = 1; i <= sc->sc_noport; i++) {
1017                         cmd = EOREAD4(sc, EHCI_PORTSC(i));
1018                         if ((cmd & EHCI_PS_PO) == 0 &&
1019                             (cmd & EHCI_PS_PE) == EHCI_PS_PE)
1020                                 EOWRITE4(sc, EHCI_PORTSC(i),
1021                                     cmd | EHCI_PS_SUSP);
1022                 }
1023
1024                 sc->sc_cmd = EOREAD4(sc, EHCI_USBCMD);
1025
1026                 cmd = sc->sc_cmd & ~(EHCI_CMD_ASE | EHCI_CMD_PSE);
1027                 EOWRITE4(sc, EHCI_USBCMD, cmd);
1028
1029                 for (i = 0; i < 100; i++) {
1030                         hcr = EOREAD4(sc, EHCI_USBSTS) &
1031                             (EHCI_STS_ASS | EHCI_STS_PSS);
1032                         if (hcr == 0)
1033                                 break;
1034
1035                         usb_delay_ms(&sc->sc_bus, 1);
1036                 }
1037                 if (hcr != 0) {
1038                         printf("%s: reset timeout\n",
1039                             device_get_nameunit(sc->sc_bus.bdev));
1040                 }
1041
1042                 cmd &= ~EHCI_CMD_RS;
1043                 EOWRITE4(sc, EHCI_USBCMD, cmd);
1044
1045                 for (i = 0; i < 100; i++) {
1046                         hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
1047                         if (hcr == EHCI_STS_HCH)
1048                                 break;
1049
1050                         usb_delay_ms(&sc->sc_bus, 1);
1051                 }
1052                 if (hcr != EHCI_STS_HCH) {
1053                         printf("%s: config timeout\n",
1054                             device_get_nameunit(sc->sc_bus.bdev));
1055                 }
1056
1057                 sc->sc_bus.use_polling--;
1058                 break;
1059
1060         case PWR_RESUME:
1061                 sc->sc_bus.use_polling++;
1062
1063                 /* restore things in case the bios sucks */
1064                 EOWRITE4(sc, EHCI_CTRLDSSEGMENT, 0);
1065                 EOWRITE4(sc, EHCI_PERIODICLISTBASE, DMAADDR(&sc->sc_fldma, 0));
1066                 EOWRITE4(sc, EHCI_ASYNCLISTADDR,
1067                     sc->sc_async_head->physaddr | EHCI_LINK_QH);
1068                 EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
1069
1070                 hcr = 0;
1071                 for (i = 1; i <= sc->sc_noport; i++) {
1072                         cmd = EOREAD4(sc, EHCI_PORTSC(i));
1073                         if ((cmd & EHCI_PS_PO) == 0 &&
1074                             (cmd & EHCI_PS_SUSP) == EHCI_PS_SUSP) {
1075                                 EOWRITE4(sc, EHCI_PORTSC(i),
1076                                     cmd | EHCI_PS_FPR);
1077                                 hcr = 1;
1078                         }
1079                 }
1080
1081                 if (hcr) {
1082                         usb_delay_ms(&sc->sc_bus, USB_RESUME_WAIT);
1083
1084                         for (i = 1; i <= sc->sc_noport; i++) {
1085                                 cmd = EOREAD4(sc, EHCI_PORTSC(i));
1086                                 if ((cmd & EHCI_PS_PO) == 0 &&
1087                                     (cmd & EHCI_PS_SUSP) == EHCI_PS_SUSP)
1088                                         EOWRITE4(sc, EHCI_PORTSC(i),
1089                                             cmd & ~EHCI_PS_FPR);
1090                         }
1091                 }
1092
1093                 EOWRITE4(sc, EHCI_USBCMD, sc->sc_cmd);
1094
1095                 for (i = 0; i < 100; i++) {
1096                         hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
1097                         if (hcr != EHCI_STS_HCH)
1098                                 break;
1099
1100                         usb_delay_ms(&sc->sc_bus, 1);
1101                 }
1102                 if (hcr == EHCI_STS_HCH) {
1103                         printf("%s: config timeout\n",
1104                             device_get_nameunit(sc->sc_bus.bdev));
1105                 }
1106
1107                 usb_delay_ms(&sc->sc_bus, USB_RESUME_WAIT);
1108
1109                 sc->sc_bus.use_polling--;
1110                 break;
1111         case PWR_SOFTSUSPEND:
1112         case PWR_SOFTSTANDBY:
1113         case PWR_SOFTRESUME:
1114                 break;
1115         }
1116         splx(s);
1117
1118 #ifdef EHCI_DEBUG
1119         DPRINTF(("ehci_power: sc=%p\n", sc));
1120         if (ehcidebug > 0)
1121                 ehci_dump_regs(sc);
1122 #endif
1123 }
1124
1125 /*
1126  * Shut down the controller when the system is going down.
1127  */
1128 void
1129 ehci_shutdown(void *v)
1130 {
1131         ehci_softc_t *sc = v;
1132
1133         DPRINTF(("ehci_shutdown: stopping the HC\n"));
1134         (void) ehci_hcreset(sc);
1135 }
1136
1137 usbd_status
1138 ehci_allocm(struct usbd_bus *bus, usb_dma_t *dma, u_int32_t size)
1139 {
1140         usbd_status err;
1141
1142         err = usb_allocmem(bus, size, 0, dma);
1143 #ifdef EHCI_DEBUG
1144         if (err)
1145                 printf("ehci_allocm: usb_allocmem()=%d\n", err);
1146 #endif
1147         return (err);
1148 }
1149
1150 void
1151 ehci_freem(struct usbd_bus *bus, usb_dma_t *dma)
1152 {
1153         usb_freemem(bus, dma);
1154 }
1155
1156 usbd_xfer_handle
1157 ehci_allocx(struct usbd_bus *bus)
1158 {
1159         struct ehci_softc *sc = (struct ehci_softc *)bus;
1160         usbd_xfer_handle xfer;
1161
1162         xfer = STAILQ_FIRST(&sc->sc_free_xfers);
1163         if (xfer != NULL) {
1164                 STAILQ_REMOVE_HEAD(&sc->sc_free_xfers, next);
1165 #ifdef DIAGNOSTIC
1166                 if (xfer->busy_free != XFER_FREE) {
1167                         printf("ehci_allocx: xfer=%p not free, 0x%08x\n", xfer,
1168                                xfer->busy_free);
1169                 }
1170 #endif
1171         } else {
1172                 xfer = malloc(sizeof(struct ehci_xfer), M_USB, M_NOWAIT);
1173         }
1174         if (xfer != NULL) {
1175                 memset(xfer, 0, sizeof(struct ehci_xfer));
1176                 usb_init_task(&EXFER(xfer)->abort_task, ehci_timeout_task,
1177                     xfer);
1178                 EXFER(xfer)->ehci_xfer_flags = 0;
1179 #ifdef DIAGNOSTIC
1180                 EXFER(xfer)->isdone = 1;
1181                 xfer->busy_free = XFER_BUSY;
1182 #endif
1183         }
1184         return (xfer);
1185 }
1186
1187 void
1188 ehci_freex(struct usbd_bus *bus, usbd_xfer_handle xfer)
1189 {
1190         struct ehci_softc *sc = (struct ehci_softc *)bus;
1191
1192 #ifdef DIAGNOSTIC
1193         if (xfer->busy_free != XFER_BUSY) {
1194                 printf("ehci_freex: xfer=%p not busy, 0x%08x\n", xfer,
1195                        xfer->busy_free);
1196                 return;
1197         }
1198         xfer->busy_free = XFER_FREE;
1199         if (!EXFER(xfer)->isdone) {
1200                 printf("ehci_freex: !isdone\n");
1201                 return;
1202         }
1203 #endif
1204         STAILQ_INSERT_HEAD(&sc->sc_free_xfers, xfer, next);
1205 }
1206
1207 static void
1208 ehci_device_clear_toggle(usbd_pipe_handle pipe)
1209 {
1210         struct ehci_pipe *epipe = (struct ehci_pipe *)pipe;
1211
1212         DPRINTF(("ehci_device_clear_toggle: epipe=%p status=0x%x\n",
1213                  epipe, epipe->sqh->qh.qh_qtd.qtd_status));
1214 #ifdef USB_DEBUG
1215         if (ehcidebug)
1216                 usbd_dump_pipe(pipe);
1217 #endif
1218         KASSERT((epipe->sqh->qh.qh_qtd.qtd_status &
1219             htole32(EHCI_QTD_ACTIVE)) == 0,
1220             ("ehci_device_clear_toggle: queue active"));
1221         epipe->sqh->qh.qh_qtd.qtd_status &= htole32(~EHCI_QTD_TOGGLE_MASK);
1222 }
1223
1224 static void
1225 ehci_noop(usbd_pipe_handle pipe)
1226 {
1227 }
1228
1229 #ifdef EHCI_DEBUG
1230 void
1231 ehci_dump_regs(ehci_softc_t *sc)
1232 {
1233         int i;
1234         printf("cmd=0x%08x, sts=0x%08x, ien=0x%08x\n",
1235                EOREAD4(sc, EHCI_USBCMD),
1236                EOREAD4(sc, EHCI_USBSTS),
1237                EOREAD4(sc, EHCI_USBINTR));
1238         printf("frindex=0x%08x ctrdsegm=0x%08x periodic=0x%08x async=0x%08x\n",
1239                EOREAD4(sc, EHCI_FRINDEX),
1240                EOREAD4(sc, EHCI_CTRLDSSEGMENT),
1241                EOREAD4(sc, EHCI_PERIODICLISTBASE),
1242                EOREAD4(sc, EHCI_ASYNCLISTADDR));
1243         for (i = 1; i <= sc->sc_noport; i++)
1244                 printf("port %d status=0x%08x\n", i,
1245                        EOREAD4(sc, EHCI_PORTSC(i)));
1246 }
1247
1248 /*
1249  * Unused function - this is meant to be called from a kernel
1250  * debugger.
1251  */
1252 void
1253 ehci_dump()
1254 {
1255         ehci_dump_regs(theehci);
1256 }
1257
1258 void
1259 ehci_dump_link(ehci_link_t link, int type)
1260 {
1261         link = le32toh(link);
1262         printf("0x%08x", link);
1263         if (link & EHCI_LINK_TERMINATE)
1264                 printf("<T>");
1265         else {
1266                 printf("<");
1267                 if (type) {
1268                         switch (EHCI_LINK_TYPE(link)) {
1269                         case EHCI_LINK_ITD: printf("ITD"); break;
1270                         case EHCI_LINK_QH: printf("QH"); break;
1271                         case EHCI_LINK_SITD: printf("SITD"); break;
1272                         case EHCI_LINK_FSTN: printf("FSTN"); break;
1273                         }
1274                 }
1275                 printf(">");
1276         }
1277 }
1278
1279 void
1280 ehci_dump_sqtds(ehci_soft_qtd_t *sqtd)
1281 {
1282         int i;
1283         u_int32_t stop;
1284
1285         stop = 0;
1286         for (i = 0; sqtd && i < 20 && !stop; sqtd = sqtd->nextqtd, i++) {
1287                 ehci_dump_sqtd(sqtd);
1288                 stop = sqtd->qtd.qtd_next & htole32(EHCI_LINK_TERMINATE);
1289         }
1290         if (sqtd)
1291                 printf("dump aborted, too many TDs\n");
1292 }
1293
1294 void
1295 ehci_dump_sqtd(ehci_soft_qtd_t *sqtd)
1296 {
1297         printf("QTD(%p) at 0x%08x:\n", sqtd, sqtd->physaddr);
1298         ehci_dump_qtd(&sqtd->qtd);
1299 }
1300
1301 void
1302 ehci_dump_qtd(ehci_qtd_t *qtd)
1303 {
1304         u_int32_t s;
1305         char sbuf[128];
1306
1307         printf("  next="); ehci_dump_link(qtd->qtd_next, 0);
1308         printf(" altnext="); ehci_dump_link(qtd->qtd_altnext, 0);
1309         printf("\n");
1310         s = le32toh(qtd->qtd_status);
1311         bitmask_snprintf(EHCI_QTD_GET_STATUS(s),
1312                          "\20\10ACTIVE\7HALTED\6BUFERR\5BABBLE\4XACTERR"
1313                          "\3MISSED\2SPLIT\1PING", sbuf, sizeof(sbuf));
1314         printf("  status=0x%08x: toggle=%d bytes=0x%x ioc=%d c_page=0x%x\n",
1315                s, EHCI_QTD_GET_TOGGLE(s), EHCI_QTD_GET_BYTES(s),
1316                EHCI_QTD_GET_IOC(s), EHCI_QTD_GET_C_PAGE(s));
1317         printf("    cerr=%d pid=%d stat=0x%s\n", EHCI_QTD_GET_CERR(s),
1318                EHCI_QTD_GET_PID(s), sbuf);
1319         for (s = 0; s < 5; s++)
1320                 printf("  buffer[%d]=0x%08x\n", s, le32toh(qtd->qtd_buffer[s]));
1321 }
1322
1323 void
1324 ehci_dump_sqh(ehci_soft_qh_t *sqh)
1325 {
1326         ehci_qh_t *qh = &sqh->qh;
1327         u_int32_t endp, endphub;
1328
1329         printf("QH(%p) at 0x%08x:\n", sqh, sqh->physaddr);
1330         printf("  sqtd=%p inactivesqtd=%p\n", sqh->sqtd, sqh->inactivesqtd);
1331         printf("  link="); ehci_dump_link(qh->qh_link, 1); printf("\n");
1332         endp = le32toh(qh->qh_endp);
1333         printf("  endp=0x%08x\n", endp);
1334         printf("    addr=0x%02x inact=%d endpt=%d eps=%d dtc=%d hrecl=%d\n",
1335                EHCI_QH_GET_ADDR(endp), EHCI_QH_GET_INACT(endp),
1336                EHCI_QH_GET_ENDPT(endp),  EHCI_QH_GET_EPS(endp),
1337                EHCI_QH_GET_DTC(endp), EHCI_QH_GET_HRECL(endp));
1338         printf("    mpl=0x%x ctl=%d nrl=%d\n",
1339                EHCI_QH_GET_MPL(endp), EHCI_QH_GET_CTL(endp),
1340                EHCI_QH_GET_NRL(endp));
1341         endphub = le32toh(qh->qh_endphub);
1342         printf("  endphub=0x%08x\n", endphub);
1343         printf("    smask=0x%02x cmask=0x%02x huba=0x%02x port=%d mult=%d\n",
1344                EHCI_QH_GET_SMASK(endphub), EHCI_QH_GET_CMASK(endphub),
1345                EHCI_QH_GET_HUBA(endphub), EHCI_QH_GET_PORT(endphub),
1346                EHCI_QH_GET_MULT(endphub));
1347         printf("  curqtd="); ehci_dump_link(qh->qh_curqtd, 0); printf("\n");
1348         printf("Overlay qTD:\n");
1349         ehci_dump_qtd(&qh->qh_qtd);
1350 }
1351
1352 #ifdef DIAGNOSTIC
1353 static void
1354 ehci_dump_exfer(struct ehci_xfer *ex)
1355 {
1356         printf("ehci_dump_exfer: ex=%p\n", ex);
1357 }
1358 #endif
1359 #endif
1360
1361 usbd_status
1362 ehci_open(usbd_pipe_handle pipe)
1363 {
1364         usbd_device_handle dev = pipe->device;
1365         ehci_softc_t *sc = (ehci_softc_t *)dev->bus;
1366         usb_endpoint_descriptor_t *ed = pipe->endpoint->edesc;
1367         u_int8_t addr = dev->address;
1368         u_int8_t xfertype = ed->bmAttributes & UE_XFERTYPE;
1369         struct ehci_pipe *epipe = (struct ehci_pipe *)pipe;
1370         ehci_soft_qh_t *sqh;
1371         usbd_status err;
1372         int s;
1373         int ival, speed, naks;
1374         int hshubaddr, hshubport;
1375
1376         DPRINTFN(1, ("ehci_open: pipe=%p, addr=%d, endpt=%d (%d)\n",
1377                      pipe, addr, ed->bEndpointAddress, sc->sc_addr));
1378
1379         if (dev->myhsport) {
1380                 hshubaddr = dev->myhsport->parent->address;
1381                 hshubport = dev->myhsport->portno;
1382         } else {
1383                 hshubaddr = 0;
1384                 hshubport = 0;
1385         }
1386
1387         if (sc->sc_dying)
1388                 return (USBD_IOERROR);
1389
1390         if (addr == sc->sc_addr) {
1391                 switch (ed->bEndpointAddress) {
1392                 case USB_CONTROL_ENDPOINT:
1393                         pipe->methods = &ehci_root_ctrl_methods;
1394                         break;
1395                 case UE_DIR_IN | EHCI_INTR_ENDPT:
1396                         pipe->methods = &ehci_root_intr_methods;
1397                         break;
1398                 default:
1399                         return (USBD_INVAL);
1400                 }
1401                 return (USBD_NORMAL_COMPLETION);
1402         }
1403
1404         /* XXX All this stuff is only valid for async. */
1405         switch (dev->speed) {
1406         case USB_SPEED_LOW:  speed = EHCI_QH_SPEED_LOW;  break;
1407         case USB_SPEED_FULL: speed = EHCI_QH_SPEED_FULL; break;
1408         case USB_SPEED_HIGH: speed = EHCI_QH_SPEED_HIGH; break;
1409         default: panic("ehci_open: bad device speed %d", dev->speed);
1410         }
1411         if (speed != EHCI_QH_SPEED_HIGH && xfertype == UE_ISOCHRONOUS) {
1412                 printf("%s: *** WARNING: opening low/full speed device, this "
1413                        "does not work yet.\n",
1414                        device_get_nameunit(sc->sc_bus.bdev));
1415                 DPRINTFN(1,("ehci_open: hshubaddr=%d hshubport=%d\n",
1416                             hshubaddr, hshubport));
1417                 return USBD_INVAL;
1418         }
1419
1420         naks = 8;               /* XXX */
1421         sqh = ehci_alloc_sqh(sc);
1422         if (sqh == NULL)
1423                 goto bad0;
1424         /* qh_link filled when the QH is added */
1425         sqh->qh.qh_endp = htole32(
1426                 EHCI_QH_SET_ADDR(addr) |
1427                 EHCI_QH_SET_ENDPT(UE_GET_ADDR(ed->bEndpointAddress)) |
1428                 EHCI_QH_SET_EPS(speed) |
1429                 (xfertype == UE_CONTROL ? EHCI_QH_DTC : 0) |
1430                 EHCI_QH_SET_MPL(UGETW(ed->wMaxPacketSize)) |
1431                 (speed != EHCI_QH_SPEED_HIGH && xfertype == UE_CONTROL ?
1432                  EHCI_QH_CTL : 0) |
1433                 EHCI_QH_SET_NRL(naks)
1434                 );
1435         sqh->qh.qh_endphub = htole32(
1436                 EHCI_QH_SET_MULT(1) |
1437                 EHCI_QH_SET_HUBA(hshubaddr) |
1438                 EHCI_QH_SET_PORT(hshubport) |
1439                 EHCI_QH_SET_CMASK(0x1c) |
1440                 EHCI_QH_SET_SMASK(xfertype == UE_INTERRUPT ? 0x01 : 0)
1441                 );
1442         sqh->qh.qh_curqtd = EHCI_NULL;
1443         /* The overlay qTD was already set up by ehci_alloc_sqh(). */
1444         sqh->qh.qh_qtd.qtd_status =
1445             htole32(EHCI_QTD_SET_TOGGLE(pipe->endpoint->savedtoggle));
1446
1447         epipe->sqh = sqh;
1448
1449         switch (xfertype) {
1450         case UE_CONTROL:
1451                 err = usb_allocmem(&sc->sc_bus, sizeof(usb_device_request_t),
1452                                    0, &epipe->u.ctl.reqdma);
1453 #ifdef EHCI_DEBUG
1454                 if (err)
1455                         printf("ehci_open: usb_allocmem()=%d\n", err);
1456 #endif
1457                 if (err)
1458                         goto bad1;
1459                 pipe->methods = &ehci_device_ctrl_methods;
1460                 s = splusb();
1461                 ehci_add_qh(sqh, sc->sc_async_head);
1462                 splx(s);
1463                 break;
1464         case UE_BULK:
1465                 pipe->methods = &ehci_device_bulk_methods;
1466                 s = splusb();
1467                 ehci_add_qh(sqh, sc->sc_async_head);
1468                 splx(s);
1469                 break;
1470         case UE_INTERRUPT:
1471                 pipe->methods = &ehci_device_intr_methods;
1472                 ival = pipe->interval;
1473                 if (ival == USBD_DEFAULT_INTERVAL)
1474                         ival = ed->bInterval;
1475                 return (ehci_device_setintr(sc, sqh, ival));
1476         case UE_ISOCHRONOUS:
1477                 pipe->methods = &ehci_device_isoc_methods;
1478                 return (USBD_INVAL);
1479         default:
1480                 return (USBD_INVAL);
1481         }
1482         return (USBD_NORMAL_COMPLETION);
1483
1484  bad1:
1485         ehci_free_sqh(sc, sqh);
1486  bad0:
1487         return (USBD_NOMEM);
1488 }
1489
1490 /*
1491  * Add an ED to the schedule.  Called at splusb().
1492  * If in the async schedule, it will always have a next.
1493  * If in the intr schedule it may not.
1494  */
1495 void
1496 ehci_add_qh(ehci_soft_qh_t *sqh, ehci_soft_qh_t *head)
1497 {
1498         SPLUSBCHECK;
1499
1500         sqh->next = head->next;
1501         sqh->prev = head;
1502         sqh->qh.qh_link = head->qh.qh_link;
1503         head->next = sqh;
1504         if (sqh->next)
1505                 sqh->next->prev = sqh;
1506         head->qh.qh_link = htole32(sqh->physaddr | EHCI_LINK_QH);
1507
1508 #ifdef EHCI_DEBUG
1509         if (ehcidebug > 5) {
1510                 printf("ehci_add_qh:\n");
1511                 ehci_dump_sqh(sqh);
1512         }
1513 #endif
1514 }
1515
1516 /*
1517  * Remove an ED from the schedule.  Called at splusb().
1518  * Will always have a 'next' if it's in the async list as it's circular.
1519  */
1520 void
1521 ehci_rem_qh(ehci_softc_t *sc, ehci_soft_qh_t *sqh, ehci_soft_qh_t *head)
1522 {
1523         SPLUSBCHECK;
1524         /* XXX */
1525         sqh->prev->qh.qh_link = sqh->qh.qh_link;
1526         sqh->prev->next = sqh->next;
1527         if (sqh->next)
1528                 sqh->next->prev = sqh->prev;
1529         ehci_sync_hc(sc);
1530 }
1531
1532 /* Restart a QH following the addition of a qTD. */
1533 void
1534 ehci_activate_qh(ehci_soft_qh_t *sqh, ehci_soft_qtd_t *sqtd)
1535 {
1536         KASSERT((sqtd->qtd.qtd_status & htole32(EHCI_QTD_ACTIVE)) == 0,
1537             ("ehci_activate_qh: already active"));
1538
1539         /*
1540          * When a QH is idle, the overlay qTD should be marked as not
1541          * halted and not active. This causes the host controller to
1542          * retrieve the real qTD on each pass (rather than just examinig
1543          * the overlay), so it will notice when we activate the qTD.
1544          */
1545         if (sqtd == sqh->sqtd) {
1546                 /* Check that the hardware is in the state we expect. */
1547                 if (EHCI_LINK_ADDR(le32toh(sqh->qh.qh_qtd.qtd_next)) !=
1548                     sqtd->physaddr) {
1549 #ifdef EHCI_DEBUG
1550                         printf("ehci_activate_qh: unexpected next ptr\n");
1551                         ehci_dump_sqh(sqh);
1552                         ehci_dump_sqtds(sqh->sqtd);
1553 #endif
1554                         sqh->qh.qh_qtd.qtd_next = htole32(sqtd->physaddr);
1555                         sqh->qh.qh_qtd.qtd_altnext = EHCI_NULL;
1556                 }
1557                 /* Ensure the flags are correct. */
1558                 sqh->qh.qh_qtd.qtd_status &= htole32(EHCI_QTD_PINGSTATE |
1559                     EHCI_QTD_TOGGLE_MASK);
1560         }
1561
1562         /* Now activate the qTD. */
1563         sqtd->qtd.qtd_status |= htole32(EHCI_QTD_ACTIVE);
1564 }
1565
1566 /*
1567  * Ensure that the HC has released all references to the QH.  We do this
1568  * by asking for a Async Advance Doorbell interrupt and then we wait for
1569  * the interrupt.
1570  * To make this easier we first obtain exclusive use of the doorbell.
1571  */
1572 void
1573 ehci_sync_hc(ehci_softc_t *sc)
1574 {
1575         int s, error;
1576
1577         if (sc->sc_dying) {
1578                 DPRINTFN(2,("ehci_sync_hc: dying\n"));
1579                 return;
1580         }
1581         DPRINTFN(2,("ehci_sync_hc: enter\n"));
1582         /* get doorbell */
1583         lockmgr(&sc->sc_doorbell_lock, LK_EXCLUSIVE, NULL, NULL);
1584         s = splhardusb();
1585         /* ask for doorbell */
1586         EOWRITE4(sc, EHCI_USBCMD, EOREAD4(sc, EHCI_USBCMD) | EHCI_CMD_IAAD);
1587         DPRINTFN(1,("ehci_sync_hc: cmd=0x%08x sts=0x%08x\n",
1588                     EOREAD4(sc, EHCI_USBCMD), EOREAD4(sc, EHCI_USBSTS)));
1589         error = tsleep(&sc->sc_async_head, PZERO, "ehcidi", hz); /* bell wait */
1590         DPRINTFN(1,("ehci_sync_hc: cmd=0x%08x sts=0x%08x\n",
1591                     EOREAD4(sc, EHCI_USBCMD), EOREAD4(sc, EHCI_USBSTS)));
1592         splx(s);
1593         /* release doorbell */
1594         lockmgr(&sc->sc_doorbell_lock, LK_RELEASE, NULL, NULL);
1595 #ifdef DIAGNOSTIC
1596         if (error)
1597                 printf("ehci_sync_hc: tsleep() = %d\n", error);
1598 #endif
1599         DPRINTFN(2,("ehci_sync_hc: exit\n"));
1600 }
1601
1602 /***********/
1603
1604 /*
1605  * Data structures and routines to emulate the root hub.
1606  */
1607 static usb_device_descriptor_t ehci_devd = {
1608         USB_DEVICE_DESCRIPTOR_SIZE,
1609         UDESC_DEVICE,           /* type */
1610         {0x00, 0x02},           /* USB version */
1611         UDCLASS_HUB,            /* class */
1612         UDSUBCLASS_HUB,         /* subclass */
1613         UDPROTO_HSHUBSTT,       /* protocol */
1614         64,                     /* max packet */
1615         {0},{0},{0x00,0x01},    /* device id */
1616         1,2,0,                  /* string indicies */
1617         1                       /* # of configurations */
1618 };
1619
1620 static usb_device_qualifier_t ehci_odevd = {
1621         USB_DEVICE_DESCRIPTOR_SIZE,
1622         UDESC_DEVICE_QUALIFIER, /* type */
1623         {0x00, 0x02},           /* USB version */
1624         UDCLASS_HUB,            /* class */
1625         UDSUBCLASS_HUB,         /* subclass */
1626         UDPROTO_FSHUB,          /* protocol */
1627         64,                     /* max packet */
1628         1,                      /* # of configurations */
1629         0
1630 };
1631
1632 static usb_config_descriptor_t ehci_confd = {
1633         USB_CONFIG_DESCRIPTOR_SIZE,
1634         UDESC_CONFIG,
1635         {USB_CONFIG_DESCRIPTOR_SIZE +
1636          USB_INTERFACE_DESCRIPTOR_SIZE +
1637          USB_ENDPOINT_DESCRIPTOR_SIZE},
1638         1,
1639         1,
1640         0,
1641         UC_SELF_POWERED,
1642         0                       /* max power */
1643 };
1644
1645 static usb_interface_descriptor_t ehci_ifcd = {
1646         USB_INTERFACE_DESCRIPTOR_SIZE,
1647         UDESC_INTERFACE,
1648         0,
1649         0,
1650         1,
1651         UICLASS_HUB,
1652         UISUBCLASS_HUB,
1653         UIPROTO_HSHUBSTT,
1654         0
1655 };
1656
1657 static usb_endpoint_descriptor_t ehci_endpd = {
1658         USB_ENDPOINT_DESCRIPTOR_SIZE,
1659         UDESC_ENDPOINT,
1660         UE_DIR_IN | EHCI_INTR_ENDPT,
1661         UE_INTERRUPT,
1662         {8, 0},                 /* max packet */
1663         255
1664 };
1665
1666 static usb_hub_descriptor_t ehci_hubd = {
1667         USB_HUB_DESCRIPTOR_SIZE,
1668         UDESC_HUB,
1669         0,
1670         {0,0},
1671         0,
1672         0,
1673         {0},
1674 };
1675
1676 static int
1677 ehci_str(usb_string_descriptor_t *p, int l, char *s)
1678 {
1679         int i;
1680
1681         if (l == 0)
1682                 return (0);
1683         p->bLength = 2 * strlen(s) + 2;
1684         if (l == 1)
1685                 return (1);
1686         p->bDescriptorType = UDESC_STRING;
1687         l -= 2;
1688         for (i = 0; s[i] && l > 1; i++, l -= 2)
1689                 USETW2(p->bString[i], 0, s[i]);
1690         return (2*i+2);
1691 }
1692
1693 /*
1694  * Simulate a hardware hub by handling all the necessary requests.
1695  */
1696 static usbd_status
1697 ehci_root_ctrl_transfer(usbd_xfer_handle xfer)
1698 {
1699         usbd_status err;
1700
1701         /* Insert last in queue. */
1702         err = usb_insert_transfer(xfer);
1703         if (err)
1704                 return (err);
1705
1706         /* Pipe isn't running, start first */
1707         return (ehci_root_ctrl_start(STAILQ_FIRST(&xfer->pipe->queue)));
1708 }
1709
1710 static usbd_status
1711 ehci_root_ctrl_start(usbd_xfer_handle xfer)
1712 {
1713         ehci_softc_t *sc = (ehci_softc_t *)xfer->pipe->device->bus;
1714         usb_device_request_t *req;
1715         void *buf = NULL;
1716         int port, i;
1717         int s, len, value, index, l, totlen = 0;
1718         usb_port_status_t ps;
1719         usb_hub_descriptor_t hubd;
1720         usbd_status err;
1721         u_int32_t v;
1722
1723         if (sc->sc_dying)
1724                 return (USBD_IOERROR);
1725
1726 #ifdef DIAGNOSTIC
1727         if (!(xfer->rqflags & URQ_REQUEST))
1728                 /* XXX panic */
1729                 return (USBD_INVAL);
1730 #endif
1731         req = &xfer->request;
1732
1733         DPRINTFN(4,("ehci_root_ctrl_start: type=0x%02x request=%02x\n",
1734                     req->bmRequestType, req->bRequest));
1735
1736         len = UGETW(req->wLength);
1737         value = UGETW(req->wValue);
1738         index = UGETW(req->wIndex);
1739
1740         if (len != 0)
1741                 buf = xfer->buffer;
1742
1743 #define C(x,y) ((x) | ((y) << 8))
1744         switch(C(req->bRequest, req->bmRequestType)) {
1745         case C(UR_CLEAR_FEATURE, UT_WRITE_DEVICE):
1746         case C(UR_CLEAR_FEATURE, UT_WRITE_INTERFACE):
1747         case C(UR_CLEAR_FEATURE, UT_WRITE_ENDPOINT):
1748                 /*
1749                  * DEVICE_REMOTE_WAKEUP and ENDPOINT_HALT are no-ops
1750                  * for the integrated root hub.
1751                  */
1752                 break;
1753         case C(UR_GET_CONFIG, UT_READ_DEVICE):
1754                 if (len > 0) {
1755                         *(u_int8_t *)buf = sc->sc_conf;
1756                         totlen = 1;
1757                 }
1758                 break;
1759         case C(UR_GET_DESCRIPTOR, UT_READ_DEVICE):
1760                 DPRINTFN(8,("ehci_root_ctrl_start: wValue=0x%04x\n", value));
1761                 switch(value >> 8) {
1762                 case UDESC_DEVICE:
1763                         if ((value & 0xff) != 0) {
1764                                 err = USBD_IOERROR;
1765                                 goto ret;
1766                         }
1767                         totlen = l = min(len, USB_DEVICE_DESCRIPTOR_SIZE);
1768                         USETW(ehci_devd.idVendor, sc->sc_id_vendor);
1769                         memcpy(buf, &ehci_devd, l);
1770                         break;
1771                 /*
1772                  * We can't really operate at another speed, but the spec says
1773                  * we need this descriptor.
1774                  */
1775                 case UDESC_DEVICE_QUALIFIER:
1776                         if ((value & 0xff) != 0) {
1777                                 err = USBD_IOERROR;
1778                                 goto ret;
1779                         }
1780                         totlen = l = min(len, USB_DEVICE_DESCRIPTOR_SIZE);
1781                         memcpy(buf, &ehci_odevd, l);
1782                         break;
1783                 /*
1784                  * We can't really operate at another speed, but the spec says
1785                  * we need this descriptor.
1786                  */
1787                 case UDESC_OTHER_SPEED_CONFIGURATION:
1788                 case UDESC_CONFIG:
1789                         if ((value & 0xff) != 0) {
1790                                 err = USBD_IOERROR;
1791                                 goto ret;
1792                         }
1793                         totlen = l = min(len, USB_CONFIG_DESCRIPTOR_SIZE);
1794                         memcpy(buf, &ehci_confd, l);
1795                         ((usb_config_descriptor_t *)buf)->bDescriptorType =
1796                                 value >> 8;
1797                         buf = (char *)buf + l;
1798                         len -= l;
1799                         l = min(len, USB_INTERFACE_DESCRIPTOR_SIZE);
1800                         totlen += l;
1801                         memcpy(buf, &ehci_ifcd, l);
1802                         buf = (char *)buf + l;
1803                         len -= l;
1804                         l = min(len, USB_ENDPOINT_DESCRIPTOR_SIZE);
1805                         totlen += l;
1806                         memcpy(buf, &ehci_endpd, l);
1807                         break;
1808                 case UDESC_STRING:
1809                         if (len == 0)
1810                                 break;
1811                         *(u_int8_t *)buf = 0;
1812                         totlen = 1;
1813                         switch (value & 0xff) {
1814                         case 0: /* Language table */
1815                                 totlen = ehci_str(buf, len, "\001");
1816                                 break;
1817                         case 1: /* Vendor */
1818                                 totlen = ehci_str(buf, len, sc->sc_vendor);
1819                                 break;
1820                         case 2: /* Product */
1821                                 totlen = ehci_str(buf, len, "EHCI root hub");
1822                                 break;
1823                         }
1824                         break;
1825                 default:
1826                         err = USBD_IOERROR;
1827                         goto ret;
1828                 }
1829                 break;
1830         case C(UR_GET_INTERFACE, UT_READ_INTERFACE):
1831                 if (len > 0) {
1832                         *(u_int8_t *)buf = 0;
1833                         totlen = 1;
1834                 }
1835                 break;
1836         case C(UR_GET_STATUS, UT_READ_DEVICE):
1837                 if (len > 1) {
1838                         USETW(((usb_status_t *)buf)->wStatus,UDS_SELF_POWERED);
1839                         totlen = 2;
1840                 }
1841                 break;
1842         case C(UR_GET_STATUS, UT_READ_INTERFACE):
1843         case C(UR_GET_STATUS, UT_READ_ENDPOINT):
1844                 if (len > 1) {
1845                         USETW(((usb_status_t *)buf)->wStatus, 0);
1846                         totlen = 2;
1847                 }
1848                 break;
1849         case C(UR_SET_ADDRESS, UT_WRITE_DEVICE):
1850                 if (value >= USB_MAX_DEVICES) {
1851                         err = USBD_IOERROR;
1852                         goto ret;
1853                 }
1854                 sc->sc_addr = value;
1855                 break;
1856         case C(UR_SET_CONFIG, UT_WRITE_DEVICE):
1857                 if (value != 0 && value != 1) {
1858                         err = USBD_IOERROR;
1859                         goto ret;
1860                 }
1861                 sc->sc_conf = value;
1862                 break;
1863         case C(UR_SET_DESCRIPTOR, UT_WRITE_DEVICE):
1864                 break;
1865         case C(UR_SET_FEATURE, UT_WRITE_DEVICE):
1866         case C(UR_SET_FEATURE, UT_WRITE_INTERFACE):
1867         case C(UR_SET_FEATURE, UT_WRITE_ENDPOINT):
1868                 err = USBD_IOERROR;
1869                 goto ret;
1870         case C(UR_SET_INTERFACE, UT_WRITE_INTERFACE):
1871                 break;
1872         case C(UR_SYNCH_FRAME, UT_WRITE_ENDPOINT):
1873                 break;
1874         /* Hub requests */
1875         case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_DEVICE):
1876                 break;
1877         case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_OTHER):
1878                 DPRINTFN(8, ("ehci_root_ctrl_start: UR_CLEAR_PORT_FEATURE "
1879                              "port=%d feature=%d\n",
1880                              index, value));
1881                 if (index < 1 || index > sc->sc_noport) {
1882                         err = USBD_IOERROR;
1883                         goto ret;
1884                 }
1885                 port = EHCI_PORTSC(index);
1886                 v = EOREAD4(sc, port) &~ EHCI_PS_CLEAR;
1887                 switch(value) {
1888                 case UHF_PORT_ENABLE:
1889                         EOWRITE4(sc, port, v &~ EHCI_PS_PE);
1890                         break;
1891                 case UHF_PORT_SUSPEND:
1892                         EOWRITE4(sc, port, v &~ EHCI_PS_SUSP);
1893                         break;
1894                 case UHF_PORT_POWER:
1895                         EOWRITE4(sc, port, v &~ EHCI_PS_PP);
1896                         break;
1897                 case UHF_PORT_TEST:
1898                         DPRINTFN(2,("ehci_root_ctrl_start: clear port test "
1899                                     "%d\n", index));
1900                         break;
1901                 case UHF_PORT_INDICATOR:
1902                         DPRINTFN(2,("ehci_root_ctrl_start: clear port ind "
1903                                     "%d\n", index));
1904                         EOWRITE4(sc, port, v &~ EHCI_PS_PIC);
1905                         break;
1906                 case UHF_C_PORT_CONNECTION:
1907                         EOWRITE4(sc, port, v | EHCI_PS_CSC);
1908                         break;
1909                 case UHF_C_PORT_ENABLE:
1910                         EOWRITE4(sc, port, v | EHCI_PS_PEC);
1911                         break;
1912                 case UHF_C_PORT_SUSPEND:
1913                         /* how? */
1914                         break;
1915                 case UHF_C_PORT_OVER_CURRENT:
1916                         EOWRITE4(sc, port, v | EHCI_PS_OCC);
1917                         break;
1918                 case UHF_C_PORT_RESET:
1919                         sc->sc_isreset = 0;
1920                         break;
1921                 default:
1922                         err = USBD_IOERROR;
1923                         goto ret;
1924                 }
1925 #if 0
1926                 switch(value) {
1927                 case UHF_C_PORT_CONNECTION:
1928                 case UHF_C_PORT_ENABLE:
1929                 case UHF_C_PORT_SUSPEND:
1930                 case UHF_C_PORT_OVER_CURRENT:
1931                 case UHF_C_PORT_RESET:
1932                         /* Enable RHSC interrupt if condition is cleared. */
1933                         if ((OREAD4(sc, port) >> 16) == 0)
1934                                 ehci_pcd_able(sc, 1);
1935                         break;
1936                 default:
1937                         break;
1938                 }
1939 #endif
1940                 break;
1941         case C(UR_GET_DESCRIPTOR, UT_READ_CLASS_DEVICE):
1942                 if ((value & 0xff) != 0) {
1943                         err = USBD_IOERROR;
1944                         goto ret;
1945                 }
1946                 hubd = ehci_hubd;
1947                 hubd.bNbrPorts = sc->sc_noport;
1948                 v = EOREAD4(sc, EHCI_HCSPARAMS);
1949                 USETW(hubd.wHubCharacteristics,
1950                     EHCI_HCS_PPC(v) ? UHD_PWR_INDIVIDUAL : UHD_PWR_NO_SWITCH |
1951                     EHCI_HCS_P_INDICATOR(EREAD4(sc, EHCI_HCSPARAMS))
1952                         ? UHD_PORT_IND : 0);
1953                 hubd.bPwrOn2PwrGood = 200; /* XXX can't find out? */
1954                 for (i = 0, l = sc->sc_noport; l > 0; i++, l -= 8, v >>= 8)
1955                         hubd.DeviceRemovable[i++] = 0; /* XXX can't find out? */
1956                 hubd.bDescLength = USB_HUB_DESCRIPTOR_SIZE + i;
1957                 l = min(len, hubd.bDescLength);
1958                 totlen = l;
1959                 memcpy(buf, &hubd, l);
1960                 break;
1961         case C(UR_GET_STATUS, UT_READ_CLASS_DEVICE):
1962                 if (len != 4) {
1963                         err = USBD_IOERROR;
1964                         goto ret;
1965                 }
1966                 memset(buf, 0, len); /* ? XXX */
1967                 totlen = len;
1968                 break;
1969         case C(UR_GET_STATUS, UT_READ_CLASS_OTHER):
1970                 DPRINTFN(8,("ehci_root_ctrl_start: get port status i=%d\n",
1971                             index));
1972                 if (index < 1 || index > sc->sc_noport) {
1973                         err = USBD_IOERROR;
1974                         goto ret;
1975                 }
1976                 if (len != 4) {
1977                         err = USBD_IOERROR;
1978                         goto ret;
1979                 }
1980                 v = EOREAD4(sc, EHCI_PORTSC(index));
1981                 DPRINTFN(8,("ehci_root_ctrl_start: port status=0x%04x\n",
1982                             v));
1983                 i = UPS_HIGH_SPEED;
1984                 if (v & EHCI_PS_CS)     i |= UPS_CURRENT_CONNECT_STATUS;
1985                 if (v & EHCI_PS_PE)     i |= UPS_PORT_ENABLED;
1986                 if (v & EHCI_PS_SUSP)   i |= UPS_SUSPEND;
1987                 if (v & EHCI_PS_OCA)    i |= UPS_OVERCURRENT_INDICATOR;
1988                 if (v & EHCI_PS_PR)     i |= UPS_RESET;
1989                 if (v & EHCI_PS_PP)     i |= UPS_PORT_POWER;
1990                 USETW(ps.wPortStatus, i);
1991                 i = 0;
1992                 if (v & EHCI_PS_CSC)    i |= UPS_C_CONNECT_STATUS;
1993                 if (v & EHCI_PS_PEC)    i |= UPS_C_PORT_ENABLED;
1994                 if (v & EHCI_PS_OCC)    i |= UPS_C_OVERCURRENT_INDICATOR;
1995                 if (sc->sc_isreset)     i |= UPS_C_PORT_RESET;
1996                 USETW(ps.wPortChange, i);
1997                 l = min(len, sizeof ps);
1998                 memcpy(buf, &ps, l);
1999                 totlen = l;
2000                 break;
2001         case C(UR_SET_DESCRIPTOR, UT_WRITE_CLASS_DEVICE):
2002                 err = USBD_IOERROR;
2003                 goto ret;
2004         case C(UR_SET_FEATURE, UT_WRITE_CLASS_DEVICE):
2005                 break;
2006         case C(UR_SET_FEATURE, UT_WRITE_CLASS_OTHER):
2007                 if (index < 1 || index > sc->sc_noport) {
2008                         err = USBD_IOERROR;
2009                         goto ret;
2010                 }
2011                 port = EHCI_PORTSC(index);
2012                 v = EOREAD4(sc, port) &~ EHCI_PS_CLEAR;
2013                 switch(value) {
2014                 case UHF_PORT_ENABLE:
2015                         EOWRITE4(sc, port, v | EHCI_PS_PE);
2016                         break;
2017                 case UHF_PORT_SUSPEND:
2018                         EOWRITE4(sc, port, v | EHCI_PS_SUSP);
2019                         break;
2020                 case UHF_PORT_RESET:
2021                         DPRINTFN(5,("ehci_root_ctrl_start: reset port %d\n",
2022                                     index));
2023                         if (EHCI_PS_IS_LOWSPEED(v)) {
2024                                 /* Low speed device, give up ownership. */
2025                                 ehci_disown(sc, index, 1);
2026                                 break;
2027                         }
2028                         /* Start reset sequence. */
2029                         v &= ~ (EHCI_PS_PE | EHCI_PS_PR);
2030                         EOWRITE4(sc, port, v | EHCI_PS_PR);
2031                         /* Wait for reset to complete. */
2032                         usb_delay_ms(&sc->sc_bus, USB_PORT_ROOT_RESET_DELAY);
2033                         if (sc->sc_dying) {
2034                                 err = USBD_IOERROR;
2035                                 goto ret;
2036                         }
2037                         /* Terminate reset sequence. */
2038                         EOWRITE4(sc, port, v);
2039                         /* Wait for HC to complete reset. */
2040                         usb_delay_ms(&sc->sc_bus, EHCI_PORT_RESET_COMPLETE);
2041                         if (sc->sc_dying) {
2042                                 err = USBD_IOERROR;
2043                                 goto ret;
2044                         }
2045                         v = EOREAD4(sc, port);
2046                         DPRINTF(("ehci after reset, status=0x%08x\n", v));
2047                         if (v & EHCI_PS_PR) {
2048                                 printf("%s: port reset timeout\n",
2049                                        device_get_nameunit(sc->sc_bus.bdev));
2050                                 return (USBD_TIMEOUT);
2051                         }
2052                         if (!(v & EHCI_PS_PE)) {
2053                                 /* Not a high speed device, give up ownership.*/
2054                                 ehci_disown(sc, index, 0);
2055                                 break;
2056                         }
2057                         sc->sc_isreset = 1;
2058                         DPRINTF(("ehci port %d reset, status = 0x%08x\n",
2059                                  index, v));
2060                         break;
2061                 case UHF_PORT_POWER:
2062                         DPRINTFN(2,("ehci_root_ctrl_start: set port power "
2063                                     "%d\n", index));
2064                         EOWRITE4(sc, port, v | EHCI_PS_PP);
2065                         break;
2066                 case UHF_PORT_TEST:
2067                         DPRINTFN(2,("ehci_root_ctrl_start: set port test "
2068                                     "%d\n", index));
2069                         break;
2070                 case UHF_PORT_INDICATOR:
2071                         DPRINTFN(2,("ehci_root_ctrl_start: set port ind "
2072                                     "%d\n", index));
2073                         EOWRITE4(sc, port, v | EHCI_PS_PIC);
2074                         break;
2075                 default:
2076                         err = USBD_IOERROR;
2077                         goto ret;
2078                 }
2079                 break;
2080         case C(UR_CLEAR_TT_BUFFER, UT_WRITE_CLASS_OTHER):
2081         case C(UR_RESET_TT, UT_WRITE_CLASS_OTHER):
2082         case C(UR_GET_TT_STATE, UT_READ_CLASS_OTHER):
2083         case C(UR_STOP_TT, UT_WRITE_CLASS_OTHER):
2084                 break;
2085         default:
2086                 err = USBD_IOERROR;
2087                 goto ret;
2088         }
2089         xfer->actlen = totlen;
2090         err = USBD_NORMAL_COMPLETION;
2091  ret:
2092         xfer->status = err;
2093         s = splusb();
2094         hacksync(xfer); /* XXX to compensate for usb_transfer_complete */
2095         usb_transfer_complete(xfer);
2096         splx(s);
2097         return (USBD_IN_PROGRESS);
2098 }
2099
2100 void
2101 ehci_disown(ehci_softc_t *sc, int index, int lowspeed)
2102 {
2103         int port;
2104         u_int32_t v;
2105
2106         DPRINTF(("ehci_disown: index=%d lowspeed=%d\n", index, lowspeed));
2107 #ifdef DIAGNOSTIC
2108         if (sc->sc_npcomp != 0) {
2109                 int i = (index-1) / sc->sc_npcomp;
2110                 if (i >= sc->sc_ncomp)
2111                         printf("%s: strange port\n",
2112                                device_get_nameunit(sc->sc_bus.bdev));
2113                 else
2114                         printf("%s: handing over %s speed device on "
2115                                "port %d to %s\n",
2116                                device_get_nameunit(sc->sc_bus.bdev),
2117                                lowspeed ? "low" : "full",
2118                                index, device_get_nameunit(sc->sc_comps[i]->bdev));
2119         } else {
2120                 printf("%s: npcomp == 0\n", device_get_nameunit(sc->sc_bus.bdev));
2121         }
2122 #endif
2123         port = EHCI_PORTSC(index);
2124         v = EOREAD4(sc, port) &~ EHCI_PS_CLEAR;
2125         EOWRITE4(sc, port, v | EHCI_PS_PO);
2126 }
2127
2128 /* Abort a root control request. */
2129 static void
2130 ehci_root_ctrl_abort(usbd_xfer_handle xfer)
2131 {
2132         /* Nothing to do, all transfers are synchronous. */
2133 }
2134
2135 /* Close the root pipe. */
2136 static void
2137 ehci_root_ctrl_close(usbd_pipe_handle pipe)
2138 {
2139         DPRINTF(("ehci_root_ctrl_close\n"));
2140         /* Nothing to do. */
2141 }
2142
2143 void
2144 ehci_root_intr_done(usbd_xfer_handle xfer)
2145 {
2146 }
2147
2148 static usbd_status
2149 ehci_root_intr_transfer(usbd_xfer_handle xfer)
2150 {
2151         usbd_status err;
2152
2153         /* Insert last in queue. */
2154         err = usb_insert_transfer(xfer);
2155         if (err)
2156                 return (err);
2157
2158         /* Pipe isn't running, start first */
2159         return (ehci_root_intr_start(STAILQ_FIRST(&xfer->pipe->queue)));
2160 }
2161
2162 static usbd_status
2163 ehci_root_intr_start(usbd_xfer_handle xfer)
2164 {
2165         usbd_pipe_handle pipe = xfer->pipe;
2166         ehci_softc_t *sc = (ehci_softc_t *)pipe->device->bus;
2167
2168         if (sc->sc_dying)
2169                 return (USBD_IOERROR);
2170
2171         sc->sc_intrxfer = xfer;
2172
2173         return (USBD_IN_PROGRESS);
2174 }
2175
2176 /* Abort a root interrupt request. */
2177 static void
2178 ehci_root_intr_abort(usbd_xfer_handle xfer)
2179 {
2180         int s;
2181
2182         if (xfer->pipe->intrxfer == xfer) {
2183                 DPRINTF(("ehci_root_intr_abort: remove\n"));
2184                 xfer->pipe->intrxfer = NULL;
2185         }
2186         xfer->status = USBD_CANCELLED;
2187         s = splusb();
2188         usb_transfer_complete(xfer);
2189         splx(s);
2190 }
2191
2192 /* Close the root pipe. */
2193 static void
2194 ehci_root_intr_close(usbd_pipe_handle pipe)
2195 {
2196         ehci_softc_t *sc = (ehci_softc_t *)pipe->device->bus;
2197
2198         DPRINTF(("ehci_root_intr_close\n"));
2199
2200         sc->sc_intrxfer = NULL;
2201 }
2202
2203 void
2204 ehci_root_ctrl_done(usbd_xfer_handle xfer)
2205 {
2206 }
2207
2208 /************************/
2209
2210 ehci_soft_qh_t *
2211 ehci_alloc_sqh(ehci_softc_t *sc)
2212 {
2213         ehci_soft_qh_t *sqh;
2214         ehci_soft_qtd_t *sqtd;
2215         usbd_status err;
2216         int i, offs;
2217         usb_dma_t dma;
2218
2219         if (sc->sc_freeqhs == NULL) {
2220                 DPRINTFN(2, ("ehci_alloc_sqh: allocating chunk\n"));
2221                 err = usb_allocmem(&sc->sc_bus, EHCI_SQH_SIZE * EHCI_SQH_CHUNK,
2222                           EHCI_PAGE_SIZE, &dma);
2223 #ifdef EHCI_DEBUG
2224                 if (err)
2225                         printf("ehci_alloc_sqh: usb_allocmem()=%d\n", err);
2226 #endif
2227                 if (err)
2228                         return (NULL);
2229                 for(i = 0; i < EHCI_SQH_CHUNK; i++) {
2230                         offs = i * EHCI_SQH_SIZE;
2231                         sqh = KERNADDR(&dma, offs);
2232                         sqh->physaddr = DMAADDR(&dma, offs);
2233                         sqh->next = sc->sc_freeqhs;
2234                         sc->sc_freeqhs = sqh;
2235                 }
2236         }
2237         /* Allocate the initial inactive sqtd. */
2238         sqtd = ehci_alloc_sqtd(sc);
2239         if (sqtd == NULL)
2240                 return (NULL);
2241         sqtd->qtd.qtd_status = htole32(0);
2242         sqtd->qtd.qtd_next = EHCI_NULL;
2243         sqtd->qtd.qtd_altnext = EHCI_NULL;
2244
2245         sqh = sc->sc_freeqhs;
2246         sc->sc_freeqhs = sqh->next;
2247
2248         /* The overlay QTD should begin zeroed. */
2249         sqh->qh.qh_qtd.qtd_next = htole32(sqtd->physaddr);
2250         sqh->qh.qh_qtd.qtd_altnext = EHCI_NULL;
2251         sqh->qh.qh_qtd.qtd_status = 0;
2252         for (i = 0; i < EHCI_QTD_NBUFFERS; i++) {
2253                 sqh->qh.qh_qtd.qtd_buffer[i] = 0;
2254                 sqh->qh.qh_qtd.qtd_buffer_hi[i] = 0;
2255         }
2256         sqh->next = NULL;
2257         sqh->prev = NULL;
2258         sqh->sqtd = sqtd;
2259         sqh->inactivesqtd = sqtd;
2260         return (sqh);
2261 }
2262
2263 void
2264 ehci_free_sqh(ehci_softc_t *sc, ehci_soft_qh_t *sqh)
2265 {
2266         ehci_free_sqtd(sc, sqh->inactivesqtd);
2267         sqh->next = sc->sc_freeqhs;
2268         sc->sc_freeqhs = sqh;
2269 }
2270
2271 ehci_soft_qtd_t *
2272 ehci_alloc_sqtd(ehci_softc_t *sc)
2273 {
2274         ehci_soft_qtd_t *sqtd;
2275         usbd_status err;
2276         int i, offs;
2277         usb_dma_t dma;
2278         int s;
2279
2280         if (sc->sc_freeqtds == NULL) {
2281                 DPRINTFN(2, ("ehci_alloc_sqtd: allocating chunk\n"));
2282                 err = usb_allocmem(&sc->sc_bus, EHCI_SQTD_SIZE*EHCI_SQTD_CHUNK,
2283                           EHCI_PAGE_SIZE, &dma);
2284 #ifdef EHCI_DEBUG
2285                 if (err)
2286                         printf("ehci_alloc_sqtd: usb_allocmem()=%d\n", err);
2287 #endif
2288                 if (err)
2289                         return (NULL);
2290                 s = splusb();
2291                 for(i = 0; i < EHCI_SQTD_CHUNK; i++) {
2292                         offs = i * EHCI_SQTD_SIZE;
2293                         sqtd = KERNADDR(&dma, offs);
2294                         sqtd->physaddr = DMAADDR(&dma, offs);
2295                         sqtd->nextqtd = sc->sc_freeqtds;
2296                         sc->sc_freeqtds = sqtd;
2297                 }
2298                 splx(s);
2299         }
2300
2301         s = splusb();
2302         sqtd = sc->sc_freeqtds;
2303         sc->sc_freeqtds = sqtd->nextqtd;
2304         sqtd->qtd.qtd_next = EHCI_NULL;
2305         sqtd->qtd.qtd_altnext = EHCI_NULL;
2306         sqtd->qtd.qtd_status = 0;
2307         for (i = 0; i < EHCI_QTD_NBUFFERS; i++) {
2308                 sqtd->qtd.qtd_buffer[i] = 0;
2309                 sqtd->qtd.qtd_buffer_hi[i] = 0;
2310         }
2311         sqtd->nextqtd = NULL;
2312         sqtd->xfer = NULL;
2313         splx(s);
2314
2315         return (sqtd);
2316 }
2317
2318 void
2319 ehci_free_sqtd(ehci_softc_t *sc, ehci_soft_qtd_t *sqtd)
2320 {
2321         int s;
2322
2323         s = splusb();
2324         sqtd->nextqtd = sc->sc_freeqtds;
2325         sc->sc_freeqtds = sqtd;
2326         splx(s);
2327 }
2328
2329 usbd_status
2330 ehci_alloc_sqtd_chain(struct ehci_pipe *epipe, ehci_softc_t *sc,
2331      int alen, int rd, usbd_xfer_handle xfer, ehci_soft_qtd_t *start,
2332      ehci_soft_qtd_t *newinactive, ehci_soft_qtd_t **sp, ehci_soft_qtd_t **ep)
2333 {
2334         ehci_soft_qtd_t *next, *cur;
2335         ehci_physaddr_t dataphys, nextphys;
2336         u_int32_t qtdstatus;
2337         int adj, len, curlen, mps, offset, pagelen, seg, segoff;
2338         int i, iscontrol, forceshort;
2339         struct usb_dma_mapping *dma = &xfer->dmamap;
2340
2341         DPRINTFN(alen<4*4096,("ehci_alloc_sqtd_chain: start len=%d\n", alen));
2342
2343         offset = 0;
2344         len = alen;
2345         iscontrol = (epipe->pipe.endpoint->edesc->bmAttributes & UE_XFERTYPE) ==
2346             UE_CONTROL;
2347         qtdstatus = EHCI_QTD_ACTIVE |
2348             EHCI_QTD_SET_PID(rd ? EHCI_QTD_PID_IN : EHCI_QTD_PID_OUT) |
2349             EHCI_QTD_SET_CERR(3)
2350             /* IOC set below */
2351             /* BYTES set below */
2352             ;
2353         mps = UGETW(epipe->pipe.endpoint->edesc->wMaxPacketSize);
2354         forceshort = ((xfer->flags & USBD_FORCE_SHORT_XFER) || len == 0) &&
2355             len % mps == 0;
2356         /*
2357          * The control transfer data stage always starts with a toggle of 1.
2358          * For other transfers we let the hardware track the toggle state.
2359          */
2360         if (iscontrol)
2361                 qtdstatus |= EHCI_QTD_SET_TOGGLE(1);
2362
2363         if (start != NULL) {
2364                 /*
2365                  * If we are given a starting qTD, assume it is linked into
2366                  * an active QH so be careful not to mark it active.
2367                  */
2368                 cur = start;
2369                 *sp = cur;
2370                 qtdstatus &= ~EHCI_QTD_ACTIVE;
2371         } else {
2372                 cur = ehci_alloc_sqtd(sc);
2373                 *sp = cur;
2374                 if (cur == NULL)
2375                         goto nomem;
2376         }
2377         seg = 0;
2378         segoff = 0;
2379         for (;;) {
2380                 curlen = 0;
2381
2382                 /* The EHCI hardware can handle at most 5 pages. */
2383                 for (i = 0; i < EHCI_QTD_NBUFFERS && curlen < len; i++) {
2384                         KASSERT(seg < dma->nsegs,
2385                             ("ehci_alloc_sqtd_chain: overrun"));
2386                         dataphys = dma->segs[seg].ds_addr + segoff;
2387                         pagelen = dma->segs[seg].ds_len - segoff;
2388                         if (pagelen > len - curlen)
2389                                 pagelen = len - curlen;
2390                         if (pagelen > EHCI_PAGE_SIZE -
2391                             EHCI_PAGE_OFFSET(dataphys))
2392                                 pagelen = EHCI_PAGE_SIZE -
2393                                     EHCI_PAGE_OFFSET(dataphys);
2394                         segoff += pagelen;
2395                         if (segoff >= dma->segs[seg].ds_len) {
2396                                 KASSERT(segoff == dma->segs[seg].ds_len,
2397                                     ("ehci_alloc_sqtd_chain: overlap"));
2398                                 seg++;
2399                                 segoff = 0;
2400                         }
2401
2402                         cur->qtd.qtd_buffer[i] = htole32(dataphys);
2403                         cur->qtd.qtd_buffer_hi[i] = 0;
2404                         curlen += pagelen;
2405
2406                         /*
2407                          * Must stop if there is any gap before or after
2408                          * the page boundary.
2409                          */
2410                         if (EHCI_PAGE_OFFSET(dataphys + pagelen) != 0)
2411                                 break;
2412                         if (seg < dma->nsegs && EHCI_PAGE_OFFSET(segoff +
2413                             dma->segs[seg].ds_addr) != 0)
2414                                 break;
2415                 }
2416                 /* Adjust down to a multiple of mps if not at the end. */
2417                 if (curlen < len && curlen % mps != 0) {
2418                         adj = curlen % mps;
2419                         curlen -= adj;
2420                         KASSERT(curlen > 0,
2421                             ("ehci_alloc_sqtd_chain: need to copy"));
2422                         segoff -= adj;
2423                         if (segoff < 0) {
2424                                 seg--;
2425                                 segoff += dma->segs[seg].ds_len;
2426                         }
2427                         KASSERT(seg >= 0 && segoff >= 0,
2428                             ("ehci_alloc_sqtd_chain: adjust to mps"));
2429                 }
2430
2431                 len -= curlen;
2432
2433                 if (len != 0 || forceshort) {
2434                         next = ehci_alloc_sqtd(sc);
2435                         if (next == NULL)
2436                                 goto nomem;
2437                         nextphys = htole32(next->physaddr);
2438                 } else {
2439                         next = NULL;
2440                         nextphys = EHCI_NULL;
2441                 }
2442
2443                 cur->nextqtd = next;
2444                 cur->qtd.qtd_next = nextphys;
2445                 /* Make sure to stop after a short transfer. */
2446                 cur->qtd.qtd_altnext = htole32(newinactive->physaddr);
2447                 cur->qtd.qtd_status =
2448                     htole32(qtdstatus | EHCI_QTD_SET_BYTES(curlen));
2449                 cur->xfer = xfer;
2450                 cur->len = curlen;
2451                 DPRINTFN(10,("ehci_alloc_sqtd_chain: curlen=%d\n", curlen));
2452                 if (iscontrol) {
2453                         /*
2454                          * adjust the toggle based on the number of packets
2455                          * in this qtd
2456                          */
2457                         if ((((curlen + mps - 1) / mps) & 1) || curlen == 0)
2458                                 qtdstatus ^= EHCI_QTD_TOGGLE_MASK;
2459                 }
2460                 qtdstatus |= EHCI_QTD_ACTIVE;
2461                 if (len == 0) {
2462                         if (!forceshort)
2463                                 break;
2464                         forceshort = 0;
2465                 }
2466                 DPRINTFN(10,("ehci_alloc_sqtd_chain: extend chain\n"));
2467                 offset += curlen;
2468                 cur = next;
2469         }
2470         cur->qtd.qtd_status |= htole32(EHCI_QTD_IOC);
2471         *ep = cur;
2472
2473         DPRINTFN(10,("ehci_alloc_sqtd_chain: return sqtd=%p sqtdend=%p\n",
2474                      *sp, *ep));
2475
2476         return (USBD_NORMAL_COMPLETION);
2477
2478  nomem:
2479         /* XXX free chain */
2480         DPRINTFN(-1,("ehci_alloc_sqtd_chain: no memory\n"));
2481         return (USBD_NOMEM);
2482 }
2483
2484 /* Free the chain starting at sqtd and end at the qTD before sqtdend */
2485 static void
2486 ehci_free_sqtd_chain(ehci_softc_t *sc, ehci_soft_qh_t *sqh,
2487     ehci_soft_qtd_t *sqtd, ehci_soft_qtd_t *sqtdend)
2488 {
2489         ehci_soft_qtd_t *p, **prevp;
2490         int i;
2491
2492         DPRINTFN(10,("ehci_free_sqtd_chain: sqtd=%p sqtdend=%p\n",
2493                      sqtd, sqtdend));
2494
2495         /* First unlink the chain from the QH's software qTD list. */
2496         prevp = &sqh->sqtd;
2497         for (p = sqh->sqtd; p != NULL; p = p->nextqtd) {
2498                 if (p == sqtd) {
2499                         *prevp = sqtdend;
2500                         break;
2501                 }
2502                 prevp = &p->nextqtd;
2503         }
2504         KASSERT(p != NULL, ("ehci_free_sqtd_chain: chain not found"));
2505         for (i = 0; sqtd != sqtdend; sqtd = p, i++) {
2506                 p = sqtd->nextqtd;
2507                 ehci_free_sqtd(sc, sqtd);
2508         }
2509 }
2510
2511 /****************/
2512
2513 /*
2514  * Close a reqular pipe.
2515  * Assumes that there are no pending transactions.
2516  */
2517 void
2518 ehci_close_pipe(usbd_pipe_handle pipe, ehci_soft_qh_t *head)
2519 {
2520         struct ehci_pipe *epipe = (struct ehci_pipe *)pipe;
2521         ehci_softc_t *sc = (ehci_softc_t *)pipe->device->bus;
2522         ehci_soft_qh_t *sqh = epipe->sqh;
2523         int s;
2524
2525         s = splusb();
2526         ehci_rem_qh(sc, sqh, head);
2527         splx(s);
2528         pipe->endpoint->savedtoggle =
2529             EHCI_QTD_GET_TOGGLE(le32toh(sqh->qh.qh_qtd.qtd_status));
2530         ehci_free_sqh(sc, epipe->sqh);
2531 }
2532
2533 /*
2534  * Abort a device request.
2535  * If this routine is called at splusb() it guarantees that the request
2536  * will be removed from the hardware scheduling and that the callback
2537  * for it will be called with USBD_CANCELLED status.
2538  * It's impossible to guarantee that the requested transfer will not
2539  * have happened since the hardware runs concurrently.
2540  * If the transaction has already happened we rely on the ordinary
2541  * interrupt processing to process it.
2542  */
2543 void
2544 ehci_abort_xfer(usbd_xfer_handle xfer, usbd_status status)
2545 {
2546 #define exfer EXFER(xfer)
2547         struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
2548         ehci_softc_t *sc = (ehci_softc_t *)epipe->pipe.device->bus;
2549         ehci_soft_qh_t *sqh = epipe->sqh;
2550         ehci_soft_qtd_t *sqtd, *snext;
2551         ehci_physaddr_t cur, us, next;
2552         int s;
2553         int hit, i;
2554         /* int count = 0; */
2555         ehci_soft_qh_t *psqh;
2556
2557         DPRINTF(("ehci_abort_xfer: xfer=%p pipe=%p\n", xfer, epipe));
2558
2559         if (sc->sc_dying) {
2560                 /* If we're dying, just do the software part. */
2561                 s = splusb();
2562                 xfer->status = status;  /* make software ignore it */
2563                 callout_stop(&xfer->timeout_handle);
2564                 usb_rem_task(epipe->pipe.device, &exfer->abort_task);
2565                 usb_transfer_complete(xfer);
2566                 splx(s);
2567                 return;
2568         }
2569
2570         if (xfer->device->bus->intr_context || !curproc)
2571                 panic("ehci_abort_xfer: not in process context");
2572
2573         /*
2574          * If an abort is already in progress then just wait for it to
2575          * complete and return.
2576          */
2577         if (exfer->ehci_xfer_flags & EHCI_XFER_ABORTING) {
2578                 DPRINTFN(2, ("ehci_abort_xfer: already aborting\n"));
2579                 /* No need to wait if we're aborting from a timeout. */
2580                 if (status == USBD_TIMEOUT)
2581                         return;
2582                 /* Override the status which might be USBD_TIMEOUT. */
2583                 xfer->status = status;
2584                 DPRINTFN(2, ("ehci_abort_xfer: waiting for abort to finish\n"));
2585                 exfer->ehci_xfer_flags |= EHCI_XFER_ABORTWAIT;
2586                 while (exfer->ehci_xfer_flags & EHCI_XFER_ABORTING)
2587                         tsleep(&exfer->ehci_xfer_flags, PZERO, "ehciaw", 0);
2588                 return;
2589         }
2590
2591         /*
2592          * Step 1: Make interrupt routine and timeouts ignore xfer.
2593          */
2594         s = splusb();
2595         exfer->ehci_xfer_flags |= EHCI_XFER_ABORTING;
2596         xfer->status = status;  /* make software ignore it */
2597         callout_stop(&xfer->timeout_handle);
2598         usb_rem_task(epipe->pipe.device, &exfer->abort_task);
2599         splx(s);
2600
2601         /*
2602          * Step 2: Wait until we know hardware has finished any possible
2603          * use of the xfer. We do this by removing the entire
2604          * queue from the async schedule and waiting for the doorbell.
2605          * Nothing else should be touching the queue now.
2606          */
2607         psqh = sqh->prev;
2608         ehci_rem_qh(sc, sqh, psqh);
2609
2610         /*
2611          * Step 3:  make sure the soft interrupt routine
2612          * has run. This should remove any completed items off the queue.
2613          * The hardware has no reference to completed items (TDs).
2614          * It's safe to remove them at any time.
2615          */
2616         s = splusb();
2617 #ifdef USB_USE_SOFTINTR
2618         sc->sc_softwake = 1;
2619 #endif /* USB_USE_SOFTINTR */
2620         usb_schedsoftintr(&sc->sc_bus);
2621 #ifdef USB_USE_SOFTINTR
2622         tsleep(&sc->sc_softwake, PZERO, "ehciab", 0);
2623 #endif /* USB_USE_SOFTINTR */
2624
2625         /*
2626          * Step 4: Remove any vestiges of the xfer from the hardware.
2627          * The complication here is that the hardware may have executed
2628          * into or even beyond the xfer we're trying to abort.
2629          * So as we're scanning the TDs of this xfer we check if
2630          * the hardware points to any of them.
2631          *
2632          * first we need to see if there are any transfers
2633          * on this queue before the xfer we are aborting.. we need
2634          * to update any pointers that point to us to point past
2635          * the aborting xfer.  (If there is something past us).
2636          * Hardware and software.
2637          */
2638         cur = EHCI_LINK_ADDR(le32toh(sqh->qh.qh_curqtd));
2639         hit = 0;
2640
2641         /* If they initially point here. */
2642         us = exfer->sqtdstart->physaddr;
2643
2644         /* We will change them to point here */
2645         snext = exfer->sqtdend->nextqtd;
2646         next = htole32(snext->physaddr);
2647
2648         /*
2649          * Now loop through any qTDs before us and keep track of the pointer
2650          * that points to us for the end.
2651          */
2652         sqtd = sqh->sqtd;
2653         while (sqtd && sqtd != exfer->sqtdstart) {
2654                 hit |= (cur == sqtd->physaddr);
2655                 if (EHCI_LINK_ADDR(le32toh(sqtd->qtd.qtd_next)) == us)
2656                         sqtd->qtd.qtd_next = next;
2657                 if (EHCI_LINK_ADDR(le32toh(sqtd->qtd.qtd_altnext)) == us)
2658                         sqtd->qtd.qtd_altnext = next;
2659                 sqtd = sqtd->nextqtd;
2660         }
2661
2662         /*
2663          * If we already saw the active one then we are pretty much done.
2664          * We've done all the relinking we need to do.
2665          */
2666         if (!hit) {
2667
2668                 /*
2669                  * Now reinitialise the QH to point to the next qTD
2670                  * (if there is one). We only need to do this if
2671                  * it was previously pointing to us.
2672                  */
2673                 for (sqtd = exfer->sqtdstart; ; sqtd = sqtd->nextqtd) {
2674                         if (cur == sqtd->physaddr) {
2675                                 hit++;
2676                         }
2677                         if (sqtd == exfer->sqtdend)
2678                                 break;
2679                 }
2680                 sqtd = sqtd->nextqtd;
2681                 /*
2682                  * Only need to alter the QH if it was pointing at a qTD
2683                  * that we are removing.
2684                  */
2685                 if (hit) {
2686                         sqh->qh.qh_qtd.qtd_next = htole32(snext->physaddr);
2687                         sqh->qh.qh_qtd.qtd_altnext = EHCI_NULL;
2688                         sqh->qh.qh_qtd.qtd_status &=
2689                             htole32(EHCI_QTD_TOGGLE_MASK);
2690                         for (i = 0; i < EHCI_QTD_NBUFFERS; i++) {
2691                                 sqh->qh.qh_qtd.qtd_buffer[i] = 0;
2692                                 sqh->qh.qh_qtd.qtd_buffer_hi[i] = 0;
2693                         }
2694                 }
2695         }
2696         ehci_add_qh(sqh, psqh);
2697         /*
2698          * Step 5: Execute callback.
2699          */
2700 #ifdef DIAGNOSTIC
2701         exfer->isdone = 1;
2702 #endif
2703         /* Do the wakeup first to avoid touching the xfer after the callback. */
2704         exfer->ehci_xfer_flags &= ~EHCI_XFER_ABORTING;
2705         if (exfer->ehci_xfer_flags & EHCI_XFER_ABORTWAIT) {
2706                 exfer->ehci_xfer_flags &= ~EHCI_XFER_ABORTWAIT;
2707                 wakeup(&exfer->ehci_xfer_flags);
2708         }
2709         usb_transfer_complete(xfer);
2710
2711         /* printf("%s: %d TDs aborted\n", __func__, count); */
2712         splx(s);
2713 #undef exfer
2714 }
2715
2716 void
2717 ehci_timeout(void *addr)
2718 {
2719         struct ehci_xfer *exfer = addr;
2720         struct ehci_pipe *epipe = (struct ehci_pipe *)exfer->xfer.pipe;
2721         ehci_softc_t *sc = (ehci_softc_t *)epipe->pipe.device->bus;
2722
2723         DPRINTF(("ehci_timeout: exfer=%p\n", exfer));
2724 #ifdef USB_DEBUG
2725         if (ehcidebug > 1)
2726                 usbd_dump_pipe(exfer->xfer.pipe);
2727 #endif
2728
2729         if (sc->sc_dying) {
2730                 ehci_abort_xfer(&exfer->xfer, USBD_TIMEOUT);
2731                 return;
2732         }
2733
2734         /* Execute the abort in a process context. */
2735         usb_add_task(exfer->xfer.pipe->device, &exfer->abort_task,
2736             USB_TASKQ_HC);
2737 }
2738
2739 void
2740 ehci_timeout_task(void *addr)
2741 {
2742         usbd_xfer_handle xfer = addr;
2743         int s;
2744
2745         DPRINTF(("ehci_timeout_task: xfer=%p\n", xfer));
2746
2747         s = splusb();
2748         ehci_abort_xfer(xfer, USBD_TIMEOUT);
2749         splx(s);
2750 }
2751
2752 /*
2753  * Some EHCI chips from VIA / ATI seem to trigger interrupts before writing
2754  * back the qTD status, or miss signalling occasionally under heavy load.
2755  * If the host machine is too fast, we can miss transaction completion - when
2756  * we scan the active list the transaction still seems to be active. This
2757  * generally exhibits itself as a umass stall that never recovers.
2758  *
2759  * We work around this behaviour by setting up this callback after any softintr
2760  * that completes with transactions still pending, giving us another chance to
2761  * check for completion after the writeback has taken place.
2762  */
2763 void
2764 ehci_intrlist_timeout(void *arg)
2765 {
2766         ehci_softc_t *sc = arg;
2767         int s = splusb();
2768
2769         DPRINTFN(3, ("ehci_intrlist_timeout\n"));
2770         usb_schedsoftintr(&sc->sc_bus);
2771
2772         splx(s);
2773 }
2774
2775 /************************/
2776
2777 static usbd_status
2778 ehci_device_ctrl_transfer(usbd_xfer_handle xfer)
2779 {
2780         usbd_status err;
2781
2782         /* Insert last in queue. */
2783         err = usb_insert_transfer(xfer);
2784         if (err)
2785                 return (err);
2786
2787         /* Pipe isn't running, start first */
2788         return (ehci_device_ctrl_start(STAILQ_FIRST(&xfer->pipe->queue)));
2789 }
2790
2791 static usbd_status
2792 ehci_device_ctrl_start(usbd_xfer_handle xfer)
2793 {
2794         ehci_softc_t *sc = (ehci_softc_t *)xfer->pipe->device->bus;
2795         usbd_status err;
2796
2797         if (sc->sc_dying)
2798                 return (USBD_IOERROR);
2799
2800 #ifdef DIAGNOSTIC
2801         if (!(xfer->rqflags & URQ_REQUEST)) {
2802                 /* XXX panic */
2803                 printf("ehci_device_ctrl_transfer: not a request\n");
2804                 return (USBD_INVAL);
2805         }
2806 #endif
2807
2808         err = ehci_device_request(xfer);
2809         if (err)
2810                 return (err);
2811
2812         if (sc->sc_bus.use_polling)
2813                 ehci_waitintr(sc, xfer);
2814         return (USBD_IN_PROGRESS);
2815 }
2816
2817 void
2818 ehci_device_ctrl_done(usbd_xfer_handle xfer)
2819 {
2820         struct ehci_xfer *ex = EXFER(xfer);
2821         ehci_softc_t *sc = (ehci_softc_t *)xfer->pipe->device->bus;
2822         struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
2823
2824         DPRINTFN(10,("ehci_ctrl_done: xfer=%p\n", xfer));
2825
2826 #ifdef DIAGNOSTIC
2827         if (!(xfer->rqflags & URQ_REQUEST)) {
2828                 panic("ehci_ctrl_done: not a request");
2829         }
2830 #endif
2831
2832         if (xfer->status != USBD_NOMEM && ehci_active_intr_list(ex)) {
2833                 ehci_del_intr_list(ex); /* remove from active list */
2834                 ehci_free_sqtd_chain(sc, epipe->sqh, ex->sqtdstart,
2835                     ex->sqtdend->nextqtd);
2836         }
2837
2838         DPRINTFN(5, ("ehci_ctrl_done: length=%d\n", xfer->actlen));
2839 }
2840
2841 /* Abort a device control request. */
2842 static void
2843 ehci_device_ctrl_abort(usbd_xfer_handle xfer)
2844 {
2845         DPRINTF(("ehci_device_ctrl_abort: xfer=%p\n", xfer));
2846         ehci_abort_xfer(xfer, USBD_CANCELLED);
2847 }
2848
2849 /* Close a device control pipe. */
2850 static void
2851 ehci_device_ctrl_close(usbd_pipe_handle pipe)
2852 {
2853         ehci_softc_t *sc = (ehci_softc_t *)pipe->device->bus;
2854         /*struct ehci_pipe *epipe = (struct ehci_pipe *)pipe;*/
2855
2856         DPRINTF(("ehci_device_ctrl_close: pipe=%p\n", pipe));
2857         ehci_close_pipe(pipe, sc->sc_async_head);
2858 }
2859
2860 usbd_status
2861 ehci_device_request(usbd_xfer_handle xfer)
2862 {
2863 #define exfer EXFER(xfer)
2864         struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
2865         usb_device_request_t *req = &xfer->request;
2866         usbd_device_handle dev = epipe->pipe.device;
2867         ehci_softc_t *sc = (ehci_softc_t *)dev->bus;
2868         ehci_soft_qtd_t *newinactive, *setup, *stat, *next;
2869         ehci_soft_qh_t *sqh;
2870         int isread;
2871         int len;
2872         usbd_status err;
2873         int s;
2874
2875         isread = req->bmRequestType & UT_READ;
2876         len = UGETW(req->wLength);
2877
2878         DPRINTFN(3,("ehci_device_request: type=0x%02x, request=0x%02x, "
2879                     "wValue=0x%04x, wIndex=0x%04x len=%d, addr=%d, endpt=%d\n",
2880                     req->bmRequestType, req->bRequest, UGETW(req->wValue),
2881                     UGETW(req->wIndex), len, dev->address,
2882                     epipe->pipe.endpoint->edesc->bEndpointAddress));
2883
2884         newinactive = ehci_alloc_sqtd(sc);
2885         if (newinactive == NULL) {
2886                 err = USBD_NOMEM;
2887                 goto bad1;
2888         }
2889         newinactive->qtd.qtd_status = htole32(0);
2890         newinactive->qtd.qtd_next = EHCI_NULL;
2891         newinactive->qtd.qtd_altnext = EHCI_NULL;
2892         stat = ehci_alloc_sqtd(sc);
2893         if (stat == NULL) {
2894                 err = USBD_NOMEM;
2895                 goto bad2;
2896         }
2897
2898         sqh = epipe->sqh;
2899         setup = sqh->inactivesqtd;
2900         sqh->inactivesqtd = newinactive;
2901         epipe->u.ctl.length = len;
2902
2903         /* Set up data transaction */
2904         if (len != 0) {
2905                 ehci_soft_qtd_t *end;
2906
2907                 err = ehci_alloc_sqtd_chain(epipe, sc, len, isread, xfer,
2908                     NULL, newinactive, &next, &end);
2909                 if (err)
2910                         goto bad3;
2911                 end->qtd.qtd_status &= htole32(~EHCI_QTD_IOC);
2912                 end->nextqtd = stat;
2913                 end->qtd.qtd_next = htole32(stat->physaddr);
2914                 end->qtd.qtd_altnext = htole32(newinactive->physaddr);
2915         } else {
2916                 next = stat;
2917         }
2918
2919         memcpy(KERNADDR(&epipe->u.ctl.reqdma, 0), req, sizeof *req);
2920
2921         /* Clear toggle, and do not activate until complete */
2922         setup->qtd.qtd_status = htole32(
2923             EHCI_QTD_SET_PID(EHCI_QTD_PID_SETUP) |
2924             EHCI_QTD_SET_CERR(3) |
2925             EHCI_QTD_SET_TOGGLE(0) |
2926             EHCI_QTD_SET_BYTES(sizeof *req)
2927             );
2928         setup->qtd.qtd_buffer[0] = htole32(DMAADDR(&epipe->u.ctl.reqdma, 0));
2929         setup->qtd.qtd_buffer_hi[0] = 0;
2930         setup->nextqtd = next;
2931         setup->qtd.qtd_next = htole32(next->physaddr);
2932         setup->qtd.qtd_altnext = htole32(newinactive->physaddr);
2933         setup->xfer = xfer;
2934         setup->len = sizeof *req;
2935
2936         stat->qtd.qtd_status = htole32(
2937             EHCI_QTD_ACTIVE |
2938             EHCI_QTD_SET_PID(isread ? EHCI_QTD_PID_OUT : EHCI_QTD_PID_IN) |
2939             EHCI_QTD_SET_CERR(3) |
2940             EHCI_QTD_SET_TOGGLE(1) |
2941             EHCI_QTD_IOC
2942             );
2943         stat->qtd.qtd_buffer[0] = 0; /* XXX not needed? */
2944         stat->qtd.qtd_buffer_hi[0] = 0; /* XXX not needed? */
2945         stat->nextqtd = newinactive;
2946         stat->qtd.qtd_next = htole32(newinactive->physaddr);
2947         stat->qtd.qtd_altnext = htole32(newinactive->physaddr);
2948         stat->xfer = xfer;
2949         stat->len = 0;
2950
2951 #ifdef EHCI_DEBUG
2952         if (ehcidebug > 5) {
2953                 DPRINTF(("ehci_device_request:\n"));
2954                 ehci_dump_sqh(sqh);
2955                 ehci_dump_sqtds(setup);
2956         }
2957 #endif
2958
2959         exfer->sqtdstart = setup;
2960         exfer->sqtdend = stat;
2961 #ifdef DIAGNOSTIC
2962         if (!exfer->isdone) {
2963                 printf("ehci_device_request: not done, exfer=%p\n", exfer);
2964         }
2965         exfer->isdone = 0;
2966 #endif
2967
2968         /* Activate the new qTD in the QH list. */
2969         s = splusb();
2970         ehci_activate_qh(sqh, setup);
2971         if (xfer->timeout && !sc->sc_bus.use_polling) {
2972                 callout_reset(&xfer->timeout_handle, MS_TO_TICKS(xfer->timeout),
2973                     ehci_timeout, xfer);
2974         }
2975         ehci_add_intr_list(sc, exfer);
2976         xfer->status = USBD_IN_PROGRESS;
2977         splx(s);
2978
2979 #ifdef EHCI_DEBUG
2980         if (ehcidebug > 10) {
2981                 DPRINTF(("ehci_device_request: status=%x\n",
2982                          EOREAD4(sc, EHCI_USBSTS)));
2983                 delay(10000);
2984                 ehci_dump_regs(sc);
2985                 ehci_dump_sqh(sc->sc_async_head);
2986                 ehci_dump_sqh(sqh);
2987                 ehci_dump_sqtds(setup);
2988         }
2989 #endif
2990
2991         return (USBD_NORMAL_COMPLETION);
2992
2993  bad3:
2994         sqh->inactivesqtd = setup;
2995         ehci_free_sqtd(sc, stat);
2996  bad2:
2997         ehci_free_sqtd(sc, newinactive);
2998  bad1:
2999         DPRINTFN(-1,("ehci_device_request: no memory\n"));
3000         xfer->status = err;
3001         usb_transfer_complete(xfer);
3002         return (err);
3003 #undef exfer
3004 }
3005
3006 /************************/
3007
3008 static usbd_status
3009 ehci_device_bulk_transfer(usbd_xfer_handle xfer)
3010 {
3011         usbd_status err;
3012
3013         /* Insert last in queue. */
3014         err = usb_insert_transfer(xfer);
3015         if (err)
3016                 return (err);
3017
3018         /* Pipe isn't running, start first */
3019         return (ehci_device_bulk_start(STAILQ_FIRST(&xfer->pipe->queue)));
3020 }
3021
3022 usbd_status
3023 ehci_device_bulk_start(usbd_xfer_handle xfer)
3024 {
3025 #define exfer EXFER(xfer)
3026         struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
3027         usbd_device_handle dev = epipe->pipe.device;
3028         ehci_softc_t *sc = (ehci_softc_t *)dev->bus;
3029         ehci_soft_qtd_t *data, *dataend, *newinactive;
3030         ehci_soft_qh_t *sqh;
3031         usbd_status err;
3032         int len, isread, endpt;
3033         int s;
3034
3035         DPRINTFN(2, ("ehci_device_bulk_start: xfer=%p len=%d flags=%d\n",
3036                      xfer, xfer->length, xfer->flags));
3037
3038         if (sc->sc_dying)
3039                 return (USBD_IOERROR);
3040
3041 #ifdef DIAGNOSTIC
3042         if (xfer->rqflags & URQ_REQUEST)
3043                 panic("ehci_device_bulk_start: a request");
3044 #endif
3045
3046         len = xfer->length;
3047         endpt = epipe->pipe.endpoint->edesc->bEndpointAddress;
3048         isread = UE_GET_DIR(endpt) == UE_DIR_IN;
3049         sqh = epipe->sqh;
3050
3051         epipe->u.bulk.length = len;
3052
3053         newinactive = ehci_alloc_sqtd(sc);
3054         if (newinactive == NULL) {
3055                 DPRINTFN(-1,("ehci_device_bulk_start: no sqtd memory\n"));
3056                 err = USBD_NOMEM;
3057                 xfer->status = err;
3058                 usb_transfer_complete(xfer);
3059                 return (err);
3060         }
3061         newinactive->qtd.qtd_status = htole32(0);
3062         newinactive->qtd.qtd_next = EHCI_NULL;
3063         newinactive->qtd.qtd_altnext = EHCI_NULL;
3064         err = ehci_alloc_sqtd_chain(epipe, sc, len, isread, xfer,
3065             sqh->inactivesqtd, newinactive, &data, &dataend);
3066         if (err) {
3067                 DPRINTFN(-1,("ehci_device_bulk_start: no memory\n"));
3068                 ehci_free_sqtd(sc, newinactive);
3069                 xfer->status = err;
3070                 usb_transfer_complete(xfer);
3071                 return (err);
3072         }
3073         dataend->nextqtd = newinactive;
3074         dataend->qtd.qtd_next = htole32(newinactive->physaddr);
3075         dataend->qtd.qtd_altnext = htole32(newinactive->physaddr);
3076         sqh->inactivesqtd = newinactive;
3077
3078 #ifdef EHCI_DEBUG
3079         if (ehcidebug > 5) {
3080                 DPRINTF(("ehci_device_bulk_start: data(1)\n"));
3081                 ehci_dump_sqh(sqh);
3082                 ehci_dump_sqtds(data);
3083         }
3084 #endif
3085
3086         /* Set up interrupt info. */
3087         exfer->sqtdstart = data;
3088         exfer->sqtdend = dataend;
3089 #ifdef DIAGNOSTIC
3090         if (!exfer->isdone) {
3091                 printf("ehci_device_bulk_start: not done, ex=%p\n", exfer);
3092         }
3093         exfer->isdone = 0;
3094 #endif
3095
3096         s = splusb();
3097         ehci_activate_qh(sqh, data);
3098         if (xfer->timeout && !sc->sc_bus.use_polling) {
3099                 callout_reset(&xfer->timeout_handle, MS_TO_TICKS(xfer->timeout),
3100                     ehci_timeout, xfer);
3101         }
3102         ehci_add_intr_list(sc, exfer);
3103         xfer->status = USBD_IN_PROGRESS;
3104         splx(s);
3105
3106 #ifdef EHCI_DEBUG
3107         if (ehcidebug > 10) {
3108                 DPRINTF(("ehci_device_bulk_start: data(2)\n"));
3109                 delay(10000);
3110                 DPRINTF(("ehci_device_bulk_start: data(3)\n"));
3111                 ehci_dump_regs(sc);
3112 #if 0
3113                 printf("async_head:\n");
3114                 ehci_dump_sqh(sc->sc_async_head);
3115 #endif
3116                 printf("sqh:\n");
3117                 ehci_dump_sqh(sqh);
3118                 ehci_dump_sqtds(data);
3119         }
3120 #endif
3121
3122         if (sc->sc_bus.use_polling)
3123                 ehci_waitintr(sc, xfer);
3124
3125         return (USBD_IN_PROGRESS);
3126 #undef exfer
3127 }
3128
3129 static void
3130 ehci_device_bulk_abort(usbd_xfer_handle xfer)
3131 {
3132         DPRINTF(("ehci_device_bulk_abort: xfer=%p\n", xfer));
3133         ehci_abort_xfer(xfer, USBD_CANCELLED);
3134 }
3135
3136 /*
3137  * Close a device bulk pipe.
3138  */
3139 static void
3140 ehci_device_bulk_close(usbd_pipe_handle pipe)
3141 {
3142         ehci_softc_t *sc = (ehci_softc_t *)pipe->device->bus;
3143
3144         DPRINTF(("ehci_device_bulk_close: pipe=%p\n", pipe));
3145         ehci_close_pipe(pipe, sc->sc_async_head);
3146 }
3147
3148 void
3149 ehci_device_bulk_done(usbd_xfer_handle xfer)
3150 {
3151         struct ehci_xfer *ex = EXFER(xfer);
3152         ehci_softc_t *sc = (ehci_softc_t *)xfer->pipe->device->bus;
3153         struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
3154
3155         DPRINTFN(10,("ehci_bulk_done: xfer=%p, actlen=%d\n",
3156                      xfer, xfer->actlen));
3157
3158         if (xfer->status != USBD_NOMEM && ehci_active_intr_list(ex)) {
3159                 ehci_del_intr_list(ex); /* remove from active list */
3160                 ehci_free_sqtd_chain(sc, epipe->sqh, ex->sqtdstart,
3161                     ex->sqtdend->nextqtd);
3162         }
3163
3164         DPRINTFN(5, ("ehci_bulk_done: length=%d\n", xfer->actlen));
3165 }
3166
3167 /************************/
3168
3169 static usbd_status
3170 ehci_device_setintr(ehci_softc_t *sc, ehci_soft_qh_t *sqh, int ival)
3171 {
3172         struct ehci_soft_islot *isp;
3173         int islot, lev;
3174
3175         /* Find a poll rate that is large enough. */
3176         for (lev = EHCI_IPOLLRATES - 1; lev > 0; lev--)
3177                 if (EHCI_ILEV_IVAL(lev) <= ival)
3178                         break;
3179
3180         /* Pick an interrupt slot at the right level. */
3181         /* XXX could do better than picking at random. */
3182         islot = EHCI_IQHIDX(lev, arc4random());
3183
3184         sqh->islot = islot;
3185         isp = &sc->sc_islots[islot];
3186         ehci_add_qh(sqh, isp->sqh);
3187
3188         return (USBD_NORMAL_COMPLETION);
3189 }
3190
3191 static usbd_status
3192 ehci_device_intr_transfer(usbd_xfer_handle xfer)
3193 {
3194         usbd_status err;
3195
3196         /* Insert last in queue. */
3197         err = usb_insert_transfer(xfer);
3198         if (err)
3199                 return (err);
3200
3201         /*
3202          * Pipe isn't running (otherwise err would be USBD_INPROG),
3203          * so start it first.
3204          */
3205         return (ehci_device_intr_start(STAILQ_FIRST(&xfer->pipe->queue)));
3206 }
3207
3208 static usbd_status
3209 ehci_device_intr_start(usbd_xfer_handle xfer)
3210 {
3211 #define exfer EXFER(xfer)
3212         struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
3213         usbd_device_handle dev = xfer->pipe->device;
3214         ehci_softc_t *sc = (ehci_softc_t *)dev->bus;
3215         ehci_soft_qtd_t *data, *dataend, *newinactive;
3216         ehci_soft_qh_t *sqh;
3217         usbd_status err;
3218         int len, isread, endpt;
3219         int s;
3220
3221         DPRINTFN(2, ("ehci_device_intr_start: xfer=%p len=%d flags=%d\n",
3222             xfer, xfer->length, xfer->flags));
3223
3224         if (sc->sc_dying)
3225                 return (USBD_IOERROR);
3226
3227 #ifdef DIAGNOSTIC
3228         if (xfer->rqflags & URQ_REQUEST)
3229                 panic("ehci_device_intr_start: a request");
3230 #endif
3231
3232         len = xfer->length;
3233         endpt = epipe->pipe.endpoint->edesc->bEndpointAddress;
3234         isread = UE_GET_DIR(endpt) == UE_DIR_IN;
3235         sqh = epipe->sqh;
3236
3237         epipe->u.intr.length = len;
3238
3239         newinactive = ehci_alloc_sqtd(sc);
3240         if (newinactive == NULL) {
3241                 DPRINTFN(-1,("ehci_device_intr_start: no sqtd memory\n"));
3242                 err = USBD_NOMEM;
3243                 xfer->status = err;
3244                 usb_transfer_complete(xfer);
3245                 return (err);
3246         }
3247         newinactive->qtd.qtd_status = htole32(0);
3248         newinactive->qtd.qtd_next = EHCI_NULL;
3249         newinactive->qtd.qtd_altnext = EHCI_NULL;
3250         err = ehci_alloc_sqtd_chain(epipe, sc, len, isread, xfer,
3251             sqh->inactivesqtd, newinactive, &data, &dataend);
3252         if (err) {
3253                 DPRINTFN(-1, ("ehci_device_intr_start: no memory\n"));
3254                 xfer->status = err;
3255                 usb_transfer_complete(xfer);
3256                 return (err);
3257         }
3258         dataend->nextqtd = newinactive;
3259         dataend->qtd.qtd_next = htole32(newinactive->physaddr);
3260         dataend->qtd.qtd_altnext = htole32(newinactive->physaddr);
3261         sqh->inactivesqtd = newinactive;
3262
3263 #ifdef EHCI_DEBUG
3264         if (ehcidebug > 5) {
3265                 DPRINTF(("ehci_device_intr_start: data(1)\n"));
3266                 ehci_dump_sqh(sqh);
3267                 ehci_dump_sqtds(data);
3268         }
3269 #endif
3270
3271         /* Set up interrupt info. */
3272         exfer->sqtdstart = data;
3273         exfer->sqtdend = dataend;
3274 #ifdef DIAGNOSTIC
3275         if (!exfer->isdone) {
3276                 printf("ehci_device_intr_start: not done, ex=%p\n", exfer);
3277         }
3278         exfer->isdone = 0;
3279 #endif
3280
3281         s = splusb();
3282         ehci_activate_qh(sqh, data);
3283         if (xfer->timeout && !sc->sc_bus.use_polling) {
3284                 callout_reset(&xfer->timeout_handle, MS_TO_TICKS(xfer->timeout),
3285                     ehci_timeout, xfer);
3286         }
3287         ehci_add_intr_list(sc, exfer);
3288         xfer->status = USBD_IN_PROGRESS;
3289         splx(s);
3290
3291 #ifdef EHCI_DEBUG
3292         if (ehcidebug > 10) {
3293                 DPRINTF(("ehci_device_intr_start: data(2)\n"));
3294                 delay(10000);
3295                 DPRINTF(("ehci_device_intr_start: data(3)\n"));
3296                 ehci_dump_regs(sc);
3297                 printf("sqh:\n");
3298                 ehci_dump_sqh(sqh);
3299                 ehci_dump_sqtds(data);
3300         }
3301 #endif
3302
3303         if (sc->sc_bus.use_polling)
3304                 ehci_waitintr(sc, xfer);
3305
3306         return (USBD_IN_PROGRESS);
3307 #undef exfer
3308 }
3309
3310 static void
3311 ehci_device_intr_abort(usbd_xfer_handle xfer)
3312 {
3313         DPRINTFN(1, ("ehci_device_intr_abort: xfer=%p\n", xfer));
3314         if (xfer->pipe->intrxfer == xfer) {
3315                 DPRINTFN(1, ("ehci_device_intr_abort: remove\n"));
3316                 xfer->pipe->intrxfer = NULL;
3317         }
3318         ehci_abort_xfer(xfer, USBD_CANCELLED);
3319 }
3320
3321 static void
3322 ehci_device_intr_close(usbd_pipe_handle pipe)
3323 {
3324         ehci_softc_t *sc = (ehci_softc_t *)pipe->device->bus;
3325         struct ehci_pipe *epipe = (struct ehci_pipe *)pipe;
3326         struct ehci_soft_islot *isp;
3327
3328         isp = &sc->sc_islots[epipe->sqh->islot];
3329         ehci_close_pipe(pipe, isp->sqh);
3330 }
3331
3332 static void
3333 ehci_device_intr_done(usbd_xfer_handle xfer)
3334 {
3335 #define exfer EXFER(xfer)
3336         struct ehci_xfer *ex = EXFER(xfer);
3337         ehci_softc_t *sc = (ehci_softc_t *)xfer->pipe->device->bus;
3338         struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
3339         ehci_soft_qtd_t *data, *dataend, *newinactive;
3340         ehci_soft_qh_t *sqh;
3341         usbd_status err;
3342         int len, isread, endpt, s;
3343
3344         DPRINTFN(10, ("ehci_device_intr_done: xfer=%p, actlen=%d\n",
3345             xfer, xfer->actlen));
3346
3347         sqh = epipe->sqh;
3348         if (xfer->pipe->repeat) {
3349                 ehci_free_sqtd_chain(sc, sqh, ex->sqtdstart,
3350                     ex->sqtdend->nextqtd);
3351
3352                 len = epipe->u.intr.length;
3353                 xfer->length = len;
3354                 endpt = epipe->pipe.endpoint->edesc->bEndpointAddress;
3355                 isread = UE_GET_DIR(endpt) == UE_DIR_IN;
3356
3357                 newinactive = ehci_alloc_sqtd(sc);
3358                 if (newinactive == NULL) {
3359                         DPRINTFN(-1,
3360                             ("ehci_device_intr_done: no sqtd memory\n"));
3361                         err = USBD_NOMEM;
3362                         xfer->status = err;
3363                         return;
3364                 }
3365                 newinactive->qtd.qtd_status = htole32(0);
3366                 newinactive->qtd.qtd_next = EHCI_NULL;
3367                 newinactive->qtd.qtd_altnext = EHCI_NULL;
3368                 err = ehci_alloc_sqtd_chain(epipe, sc, len, isread, xfer,
3369                     sqh->inactivesqtd, newinactive, &data, &dataend);
3370                 if (err) {
3371                         DPRINTFN(-1, ("ehci_device_intr_done: no memory\n"));
3372                         xfer->status = err;
3373                         return;
3374                 }
3375                 dataend->nextqtd = newinactive;
3376                 dataend->qtd.qtd_next = htole32(newinactive->physaddr);
3377                 dataend->qtd.qtd_altnext = htole32(newinactive->physaddr);
3378                 sqh->inactivesqtd = newinactive;
3379
3380                 /* Set up interrupt info. */
3381                 exfer->sqtdstart = data;
3382                 exfer->sqtdend = dataend;
3383 #ifdef DIAGNOSTIC
3384                 if (!exfer->isdone) {
3385                         printf("ehci_device_intr_done: not done, ex=%p\n",
3386                             exfer);
3387                 }
3388                 exfer->isdone = 0;
3389 #endif
3390
3391                 s = splusb();
3392                 ehci_activate_qh(sqh, data);
3393                 if (xfer->timeout && !sc->sc_bus.use_polling) {
3394                         callout_reset(&xfer->timeout_handle,
3395                             MS_TO_TICKS(xfer->timeout), ehci_timeout, xfer);
3396                 }
3397                 splx(s);
3398
3399                 xfer->status = USBD_IN_PROGRESS;
3400         } else if (xfer->status != USBD_NOMEM && ehci_active_intr_list(ex)) {
3401                 ehci_del_intr_list(ex); /* remove from active list */
3402                 ehci_free_sqtd_chain(sc, sqh, ex->sqtdstart,
3403                     ex->sqtdend->nextqtd);
3404         }
3405 #undef exfer
3406 }
3407
3408 /************************/
3409
3410 static usbd_status      ehci_device_isoc_transfer(usbd_xfer_handle xfer) { return USBD_IOERROR; }
3411 static usbd_status      ehci_device_isoc_start(usbd_xfer_handle xfer) { return USBD_IOERROR; }
3412 static void             ehci_device_isoc_abort(usbd_xfer_handle xfer) { }
3413 static void             ehci_device_isoc_close(usbd_pipe_handle pipe) { }
3414 static void             ehci_device_isoc_done(usbd_xfer_handle xfer) { }