sys/dev/usb/ehci.c

   1 /*      $NetBSD: ehci.c,v 1.91 2005/02/27 00:27:51 perry Exp $ */
   2
   3 /*-
   4  * Copyright (c) 2004 The NetBSD Foundation, Inc.
   5  * All rights reserved.
   6  *
   7  * This code is derived from software contributed to The NetBSD Foundation
   8  * by Lennart Augustsson (lennart@augustsson.net) and by Charles M. Hannum.
   9  *
  10  * Redistribution and use in source and binary forms, with or without
  11  * modification, are permitted provided that the following conditions
  12  * are met:
  13  * 1. Redistributions of source code must retain the above copyright
  14  *    notice, this list of conditions and the following disclaimer.
  15  * 2. Redistributions in binary form must reproduce the above copyright
  16  *    notice, this list of conditions and the following disclaimer in the
  17  *    documentation and/or other materials provided with the distribution.
  18  * 3. All advertising materials mentioning features or use of this software
  19  *    must display the following acknowledgement:
  20  *        This product includes software developed by the NetBSD
  21  *        Foundation, Inc. and its contributors.
  22  * 4. Neither the name of The NetBSD Foundation nor the names of its
  23  *    contributors may be used to endorse or promote products derived
  24  *    from this software without specific prior written permission.
  25  *
  26  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
  27  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
  28  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  29  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
  30  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  31  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  32  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  33  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  34  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  35  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  36  * POSSIBILITY OF SUCH DAMAGE.
  37  */
  38
  39 /*
  40  * USB Enhanced Host Controller Driver, a.k.a. USB 2.0 controller.
  41  *
  42  * The EHCI 1.0 spec can be found at
  43  * http://developer.intel.com/technology/usb/download/ehci-r10.pdf
  44  * and the USB 2.0 spec at
  45  * http://www.usb.org/developers/docs/usb_20.zip
  46  *
  47  */
  48
  49 /*
  50  * TODO:
  51  * 1) The EHCI driver lacks support for isochronous transfers, so
  52  *    devices using them don't work.
  53  *
  54  * 2) Interrupt transfer scheduling does not manage the time available
  55  *    in each frame, so it is possible for the transfers to overrun
  56  *    the end of the frame.
  57  *
  58  * 3) Command failures are not recovered correctly.
  59  */
  60
  61 #include <sys/cdefs.h>
  62 __FBSDID("$FreeBSD$");
  63
  64 #include <sys/param.h>
  65 #include <sys/systm.h>
  66 #include <sys/malloc.h>
  67 #include <sys/kernel.h>
  68 #include <sys/endian.h>
  69 #include <sys/module.h>
  70 #include <sys/bus.h>
  71 #include <sys/lock.h>
  72 #include <sys/lockmgr.h>
  73 #if defined(DIAGNOSTIC) && defined(__i386__) && defined(__FreeBSD__)
  74 #include <machine/cpu.h>
  75 #endif
  76 #include <sys/proc.h>
  77 #include <sys/queue.h>
  78 #include <sys/sysctl.h>
  79
  80 #include <machine/bus.h>
  81 #include <machine/endian.h>
  82
  83 #include <dev/usb/usb.h>
  84 #include <dev/usb/usbdi.h>
  85 #include <dev/usb/usbdivar.h>
  86 #include <dev/usb/usb_mem.h>
  87 #include <dev/usb/usb_quirks.h>
  88
  89 #include <dev/usb/ehcireg.h>
  90 #include <dev/usb/ehcivar.h>
  91
  92 #define delay(d)                DELAY(d)
  93
  94 #ifdef USB_DEBUG
  95 #define EHCI_DEBUG USB_DEBUG
  96 #define DPRINTF(x)      do { if (ehcidebug) printf x; } while (0)
  97 #define DPRINTFN(n,x)   do { if (ehcidebug>(n)) printf x; } while (0)
  98 int ehcidebug = 0;
  99 SYSCTL_NODE(_hw_usb, OID_AUTO, ehci, CTLFLAG_RW, 0, "USB ehci");
 100 SYSCTL_INT(_hw_usb_ehci, OID_AUTO, debug, CTLFLAG_RW,
 101            &ehcidebug, 0, "ehci debug level");
 102 #define bitmask_snprintf(q,f,b,l) snprintf((b), (l), "%b", (q), (f))
 103 #else
 104 #define DPRINTF(x)
 105 #define DPRINTFN(n,x)
 106 #endif
 107
 108 struct ehci_pipe {
 109         struct usbd_pipe pipe;
 110
 111         ehci_soft_qh_t *sqh;
 112         union {
 113                 ehci_soft_qtd_t *qtd;
 114                 /* ehci_soft_itd_t *itd; */
 115         } tail;
 116         union {
 117                 /* Control pipe */
 118                 struct {
 119                         usb_dma_t reqdma;
 120                         u_int length;
 121                         /*ehci_soft_qtd_t *setup, *data, *stat;*/
 122                 } ctl;
 123                 /* Interrupt pipe */
 124                 struct {
 125                         u_int length;
 126                 } intr;
 127                 /* Bulk pipe */
 128                 struct {
 129                         u_int length;
 130                 } bulk;
 131                 /* Iso pipe */
 132                 /* XXX */
 133         } u;
 134 };
 135
 136 static usbd_status      ehci_open(usbd_pipe_handle);
 137 static void             ehci_poll(struct usbd_bus *);
 138 static void             ehci_softintr(void *);
 139 static int              ehci_intr1(ehci_softc_t *);
 140 static void             ehci_waitintr(ehci_softc_t *, usbd_xfer_handle);
 141 static void             ehci_check_intr(ehci_softc_t *, struct ehci_xfer *);
 142 static void             ehci_idone(struct ehci_xfer *);
 143 static void             ehci_timeout(void *);
 144 static void             ehci_timeout_task(void *);
 145 static void             ehci_intrlist_timeout(void *);
 146
 147 static usbd_status      ehci_allocm(struct usbd_bus *, usb_dma_t *, u_int32_t);
 148 static void             ehci_freem(struct usbd_bus *, usb_dma_t *);
 149
 150 static usbd_xfer_handle ehci_allocx(struct usbd_bus *);
 151 static void             ehci_freex(struct usbd_bus *, usbd_xfer_handle);
 152
 153 static usbd_status      ehci_root_ctrl_transfer(usbd_xfer_handle);
 154 static usbd_status      ehci_root_ctrl_start(usbd_xfer_handle);
 155 static void             ehci_root_ctrl_abort(usbd_xfer_handle);
 156 static void             ehci_root_ctrl_close(usbd_pipe_handle);
 157 static void             ehci_root_ctrl_done(usbd_xfer_handle);
 158
 159 static usbd_status      ehci_root_intr_transfer(usbd_xfer_handle);
 160 static usbd_status      ehci_root_intr_start(usbd_xfer_handle);
 161 static void             ehci_root_intr_abort(usbd_xfer_handle);
 162 static void             ehci_root_intr_close(usbd_pipe_handle);
 163 static void             ehci_root_intr_done(usbd_xfer_handle);
 164
 165 static usbd_status      ehci_device_ctrl_transfer(usbd_xfer_handle);
 166 static usbd_status      ehci_device_ctrl_start(usbd_xfer_handle);
 167 static void             ehci_device_ctrl_abort(usbd_xfer_handle);
 168 static void             ehci_device_ctrl_close(usbd_pipe_handle);
 169 static void             ehci_device_ctrl_done(usbd_xfer_handle);
 170
 171 static usbd_status      ehci_device_bulk_transfer(usbd_xfer_handle);
 172 static usbd_status      ehci_device_bulk_start(usbd_xfer_handle);
 173 static void             ehci_device_bulk_abort(usbd_xfer_handle);
 174 static void             ehci_device_bulk_close(usbd_pipe_handle);
 175 static void             ehci_device_bulk_done(usbd_xfer_handle);
 176
 177 static usbd_status      ehci_device_intr_transfer(usbd_xfer_handle);
 178 static usbd_status      ehci_device_intr_start(usbd_xfer_handle);
 179 static void             ehci_device_intr_abort(usbd_xfer_handle);
 180 static void             ehci_device_intr_close(usbd_pipe_handle);
 181 static void             ehci_device_intr_done(usbd_xfer_handle);
 182
 183 static usbd_status      ehci_device_isoc_transfer(usbd_xfer_handle);
 184 static usbd_status      ehci_device_isoc_start(usbd_xfer_handle);
 185 static void             ehci_device_isoc_abort(usbd_xfer_handle);
 186 static void             ehci_device_isoc_close(usbd_pipe_handle);
 187 static void             ehci_device_isoc_done(usbd_xfer_handle);
 188
 189 static void             ehci_device_clear_toggle(usbd_pipe_handle pipe);
 190 static void             ehci_noop(usbd_pipe_handle pipe);
 191
 192 static int              ehci_str(usb_string_descriptor_t *, int, char *);
 193 static void             ehci_pcd(ehci_softc_t *, usbd_xfer_handle);
 194 static void             ehci_pcd_able(ehci_softc_t *, int);
 195 static void             ehci_pcd_enable(void *);
 196 static void             ehci_disown(ehci_softc_t *, int, int);
 197
 198 static ehci_soft_qh_t  *ehci_alloc_sqh(ehci_softc_t *);
 199 static void             ehci_free_sqh(ehci_softc_t *, ehci_soft_qh_t *);
 200
 201 static ehci_soft_qtd_t  *ehci_alloc_sqtd(ehci_softc_t *);
 202 static void             ehci_free_sqtd(ehci_softc_t *, ehci_soft_qtd_t *);
 203 static usbd_status      ehci_alloc_sqtd_chain(struct ehci_pipe *,
 204                             ehci_softc_t *, int, int, usbd_xfer_handle,
 205                             ehci_soft_qtd_t *, ehci_soft_qtd_t *,
 206                             ehci_soft_qtd_t **, ehci_soft_qtd_t **);
 207 static void             ehci_free_sqtd_chain(ehci_softc_t *, ehci_soft_qh_t *,
 208                             ehci_soft_qtd_t *, ehci_soft_qtd_t *);
 209
 210 static usbd_status      ehci_device_request(usbd_xfer_handle xfer);
 211
 212 static usbd_status      ehci_device_setintr(ehci_softc_t *, ehci_soft_qh_t *,
 213                             int ival);
 214
 215 static void             ehci_add_qh(ehci_soft_qh_t *, ehci_soft_qh_t *);
 216 static void             ehci_rem_qh(ehci_softc_t *, ehci_soft_qh_t *,
 217                                     ehci_soft_qh_t *);
 218 static void             ehci_activate_qh(ehci_soft_qh_t *, ehci_soft_qtd_t *);
 219 static void             ehci_sync_hc(ehci_softc_t *);
 220
 221 static void             ehci_close_pipe(usbd_pipe_handle, ehci_soft_qh_t *);
 222 static void             ehci_abort_xfer(usbd_xfer_handle, usbd_status);
 223
 224 #ifdef EHCI_DEBUG
 225 static void             ehci_dump_regs(ehci_softc_t *);
 226 void                    ehci_dump(void);
 227 static ehci_softc_t     *theehci;
 228 static void             ehci_dump_link(ehci_link_t, int);
 229 static void             ehci_dump_sqtds(ehci_soft_qtd_t *);
 230 static void             ehci_dump_sqtd(ehci_soft_qtd_t *);
 231 static void             ehci_dump_qtd(ehci_qtd_t *);
 232 static void             ehci_dump_sqh(ehci_soft_qh_t *);
 233 #ifdef DIAGNOSTIC
 234 static void             ehci_dump_exfer(struct ehci_xfer *);
 235 #endif
 236 #endif
 237
 238 #define EHCI_NULL htole32(EHCI_LINK_TERMINATE)
 239
 240 #define EHCI_INTR_ENDPT 1
 241
 242 #define ehci_add_intr_list(sc, ex) \
 243         LIST_INSERT_HEAD(&(sc)->sc_intrhead, (ex), inext);
 244 #define ehci_del_intr_list(ex) \
 245         do { \
 246                 LIST_REMOVE((ex), inext); \
 247                 (ex)->inext.le_prev = NULL; \
 248         } while (0)
 249 #define ehci_active_intr_list(ex) ((ex)->inext.le_prev != NULL)
 250
 251 static struct usbd_bus_methods ehci_bus_methods = {
 252         ehci_open,
 253         ehci_softintr,
 254         ehci_poll,
 255         ehci_allocm,
 256         ehci_freem,
 257         ehci_allocx,
 258         ehci_freex,
 259 };
 260
 261 static struct usbd_pipe_methods ehci_root_ctrl_methods = {
 262         ehci_root_ctrl_transfer,
 263         ehci_root_ctrl_start,
 264         ehci_root_ctrl_abort,
 265         ehci_root_ctrl_close,
 266         ehci_noop,
 267         ehci_root_ctrl_done,
 268 };
 269
 270 static struct usbd_pipe_methods ehci_root_intr_methods = {
 271         ehci_root_intr_transfer,
 272         ehci_root_intr_start,
 273         ehci_root_intr_abort,
 274         ehci_root_intr_close,
 275         ehci_noop,
 276         ehci_root_intr_done,
 277 };
 278
 279 static struct usbd_pipe_methods ehci_device_ctrl_methods = {
 280         ehci_device_ctrl_transfer,
 281         ehci_device_ctrl_start,
 282         ehci_device_ctrl_abort,
 283         ehci_device_ctrl_close,
 284         ehci_noop,
 285         ehci_device_ctrl_done,
 286 };
 287
 288 static struct usbd_pipe_methods ehci_device_intr_methods = {
 289         ehci_device_intr_transfer,
 290         ehci_device_intr_start,
 291         ehci_device_intr_abort,
 292         ehci_device_intr_close,
 293         ehci_device_clear_toggle,
 294         ehci_device_intr_done,
 295 };
 296
 297 static struct usbd_pipe_methods ehci_device_bulk_methods = {
 298         ehci_device_bulk_transfer,
 299         ehci_device_bulk_start,
 300         ehci_device_bulk_abort,
 301         ehci_device_bulk_close,
 302         ehci_device_clear_toggle,
 303         ehci_device_bulk_done,
 304 };
 305
 306 static struct usbd_pipe_methods ehci_device_isoc_methods = {
 307         ehci_device_isoc_transfer,
 308         ehci_device_isoc_start,
 309         ehci_device_isoc_abort,
 310         ehci_device_isoc_close,
 311         ehci_noop,
 312         ehci_device_isoc_done,
 313 };
 314
 315 static usbd_status
 316 ehci_hcreset(ehci_softc_t *sc)
 317 {
 318         u_int32_t hcr;
 319         u_int i;
 320
 321         EOWRITE4(sc, EHCI_USBCMD, 0);   /* Halt controller */
 322         for (i = 0; i < 100; i++) {
 323                 usb_delay_ms(&sc->sc_bus, 1);
 324                 hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
 325                 if (hcr)
 326                         break;
 327         }
 328         if (!hcr)
 329                 /*
 330                  * Fall through and try reset anyway even though
 331                  * Table 2-9 in the EHCI spec says this will result
 332                  * in undefined behavior.
 333                  */
 334                 printf("%s: stop timeout\n",
 335                        device_get_nameunit(sc->sc_bus.bdev));
 336
 337         EOWRITE4(sc, EHCI_USBCMD, EHCI_CMD_HCRESET);
 338         for (i = 0; i < 100; i++) {
 339                 usb_delay_ms(&sc->sc_bus, 1);
 340                 hcr = EOREAD4(sc, EHCI_USBCMD) & EHCI_CMD_HCRESET;
 341                 if (!hcr)
 342                         return (USBD_NORMAL_COMPLETION);
 343         }
 344         printf("%s: reset timeout\n", device_get_nameunit(sc->sc_bus.bdev));
 345         return (USBD_IOERROR);
 346 }
 347
 348 usbd_status
 349 ehci_init(ehci_softc_t *sc)
 350 {
 351         u_int32_t version, sparams, cparams, hcr;
 352         u_int i;
 353         usbd_status err;
 354         ehci_soft_qh_t *sqh;
 355         u_int ncomp;
 356         int lev;
 357
 358         DPRINTF(("ehci_init: start\n"));
 359 #ifdef EHCI_DEBUG
 360         theehci = sc;
 361 #endif
 362
 363         sc->sc_offs = EREAD1(sc, EHCI_CAPLENGTH);
 364
 365         version = EREAD2(sc, EHCI_HCIVERSION);
 366         printf("%s: EHCI version %x.%x\n", device_get_nameunit(sc->sc_bus.bdev),
 367                version >> 8, version & 0xff);
 368
 369         sparams = EREAD4(sc, EHCI_HCSPARAMS);
 370         DPRINTF(("ehci_init: sparams=0x%x\n", sparams));
 371         sc->sc_npcomp = EHCI_HCS_N_PCC(sparams);
 372         ncomp = EHCI_HCS_N_CC(sparams);
 373         if (ncomp != sc->sc_ncomp) {
 374                 printf("%s: wrong number of companions (%d != %d)\n",
 375                        device_get_nameunit(sc->sc_bus.bdev),
 376                        ncomp, sc->sc_ncomp);
 377                 if (ncomp < sc->sc_ncomp)
 378                         sc->sc_ncomp = ncomp;
 379         }
 380         if (sc->sc_ncomp > 0) {
 381                 printf("%s: companion controller%s, %d port%s each:",
 382                     device_get_nameunit(sc->sc_bus.bdev), sc->sc_ncomp!=1 ? "s" : "",
 383                     EHCI_HCS_N_PCC(sparams),
 384                     EHCI_HCS_N_PCC(sparams)!=1 ? "s" : "");
 385                 for (i = 0; i < sc->sc_ncomp; i++)
 386                         printf(" %s", device_get_nameunit(sc->sc_comps[i]->bdev));
 387                 printf("\n");
 388         }
 389         sc->sc_noport = EHCI_HCS_N_PORTS(sparams);
 390         cparams = EREAD4(sc, EHCI_HCCPARAMS);
 391         DPRINTF(("ehci_init: cparams=0x%x\n", cparams));
 392
 393         if (EHCI_HCC_64BIT(cparams)) {
 394                 /* MUST clear segment register if 64 bit capable. */
 395                 EWRITE4(sc, EHCI_CTRLDSSEGMENT, 0);
 396         }
 397
 398         sc->sc_bus.usbrev = USBREV_2_0;
 399
 400         /* Reset the controller */
 401         DPRINTF(("%s: resetting\n", device_get_nameunit(sc->sc_bus.bdev)));
 402         err = ehci_hcreset(sc);
 403         if (err != USBD_NORMAL_COMPLETION)
 404                 return (err);
 405
 406         /* frame list size at default, read back what we got and use that */
 407         switch (EHCI_CMD_FLS(EOREAD4(sc, EHCI_USBCMD))) {
 408         case 0: sc->sc_flsize = 1024; break;
 409         case 1: sc->sc_flsize = 512; break;
 410         case 2: sc->sc_flsize = 256; break;
 411         case 3: return (USBD_IOERROR);
 412         }
 413         err = usb_allocmem(&sc->sc_bus, sc->sc_flsize * sizeof(ehci_link_t),
 414             EHCI_FLALIGN_ALIGN, &sc->sc_fldma);
 415         if (err)
 416                 return (err);
 417         DPRINTF(("%s: flsize=%d\n", device_get_nameunit(sc->sc_bus.bdev),sc->sc_flsize));
 418         sc->sc_flist = KERNADDR(&sc->sc_fldma, 0);
 419         EOWRITE4(sc, EHCI_PERIODICLISTBASE, DMAADDR(&sc->sc_fldma, 0));
 420
 421         /* Set up the bus struct. */
 422         sc->sc_bus.methods = &ehci_bus_methods;
 423         sc->sc_bus.pipe_size = sizeof(struct ehci_pipe);
 424
 425 #if defined(__NetBSD__) || defined(__OpenBSD__)
 426         sc->sc_powerhook = powerhook_establish(ehci_power, sc);
 427         sc->sc_shutdownhook = shutdownhook_establish(ehci_shutdown, sc);
 428 #endif
 429
 430         sc->sc_eintrs = EHCI_NORMAL_INTRS;
 431
 432         /*
 433          * Allocate the interrupt dummy QHs. These are arranged to give
 434          * poll intervals that are powers of 2 times 1ms.
 435          */
 436         for (i = 0; i < EHCI_INTRQHS; i++) {
 437                 sqh = ehci_alloc_sqh(sc);
 438                 if (sqh == NULL) {
 439                         err = USBD_NOMEM;
 440                         goto bad1;
 441                 }
 442                 sc->sc_islots[i].sqh = sqh;
 443         }
 444         lev = 0;
 445         for (i = 0; i < EHCI_INTRQHS; i++) {
 446                 if (i == EHCI_IQHIDX(lev + 1, 0))
 447                         lev++;
 448                 sqh = sc->sc_islots[i].sqh;
 449                 if (i == 0) {
 450                         /* The last (1ms) QH terminates. */
 451                         sqh->qh.qh_link = EHCI_NULL;
 452                         sqh->next = NULL;
 453                 } else {
 454                         /* Otherwise the next QH has half the poll interval */
 455                         sqh->next =
 456                             sc->sc_islots[EHCI_IQHIDX(lev - 1, i + 1)].sqh;
 457                         sqh->qh.qh_link = htole32(sqh->next->physaddr |
 458                             EHCI_LINK_QH);
 459                 }
 460                 sqh->qh.qh_endp = htole32(EHCI_QH_SET_EPS(EHCI_QH_SPEED_HIGH));
 461                 sqh->qh.qh_endphub = htole32(EHCI_QH_SET_MULT(1));
 462                 sqh->qh.qh_curqtd = EHCI_NULL;
 463                 sqh->qh.qh_qtd.qtd_next = EHCI_NULL;
 464                 sqh->qh.qh_qtd.qtd_altnext = EHCI_NULL;
 465                 sqh->qh.qh_qtd.qtd_status = htole32(EHCI_QTD_HALTED);
 466         }
 467         /* Point the frame list at the last level (128ms). */
 468         for (i = 0; i < sc->sc_flsize; i++) {
 469                 sc->sc_flist[i] = htole32(EHCI_LINK_QH |
 470                     sc->sc_islots[EHCI_IQHIDX(EHCI_IPOLLRATES - 1,
 471                     i)].sqh->physaddr);
 472         }
 473
 474         /* Allocate dummy QH that starts the async list. */
 475         sqh = ehci_alloc_sqh(sc);
 476         if (sqh == NULL) {
 477                 err = USBD_NOMEM;
 478                 goto bad1;
 479         }
 480         /* Fill the QH */
 481         sqh->qh.qh_endp =
 482             htole32(EHCI_QH_SET_EPS(EHCI_QH_SPEED_HIGH) | EHCI_QH_HRECL);
 483         sqh->qh.qh_link =
 484             htole32(sqh->physaddr | EHCI_LINK_QH);
 485         sqh->qh.qh_curqtd = EHCI_NULL;
 486         sqh->prev = sqh; /*It's a circular list.. */
 487         sqh->next = sqh;
 488         /* Fill the overlay qTD */
 489         sqh->qh.qh_qtd.qtd_next = EHCI_NULL;
 490         sqh->qh.qh_qtd.qtd_altnext = EHCI_NULL;
 491         sqh->qh.qh_qtd.qtd_status = htole32(0);
 492 #ifdef EHCI_DEBUG
 493         if (ehcidebug) {
 494                 ehci_dump_sqh(sqh);
 495         }
 496 #endif
 497
 498         /* Point to async list */
 499         sc->sc_async_head = sqh;
 500         EOWRITE4(sc, EHCI_ASYNCLISTADDR, sqh->physaddr | EHCI_LINK_QH);
 501
 502         callout_init(&sc->sc_tmo_pcd, 0);
 503         callout_init(&sc->sc_tmo_intrlist, 0);
 504
 505         lockinit(&sc->sc_doorbell_lock, PZERO, "ehcidb", 0, 0);
 506
 507         /* Enable interrupts */
 508         EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
 509
 510         /* Turn on controller */
 511         EOWRITE4(sc, EHCI_USBCMD,
 512                  EHCI_CMD_ITC_2 | /* 2 microframes interrupt delay */
 513                  (EOREAD4(sc, EHCI_USBCMD) & EHCI_CMD_FLS_M) |
 514                  EHCI_CMD_ASE |
 515                  EHCI_CMD_PSE |
 516                  EHCI_CMD_RS);
 517
 518         /* Take over port ownership */
 519         EOWRITE4(sc, EHCI_CONFIGFLAG, EHCI_CONF_CF);
 520
 521         for (i = 0; i < 100; i++) {
 522                 usb_delay_ms(&sc->sc_bus, 1);
 523                 hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
 524                 if (!hcr)
 525                         break;
 526         }
 527         if (hcr) {
 528                 printf("%s: run timeout\n", device_get_nameunit(sc->sc_bus.bdev));
 529                 return (USBD_IOERROR);
 530         }
 531
 532         return (USBD_NORMAL_COMPLETION);
 533
 534 #if 0
 535  bad2:
 536         ehci_free_sqh(sc, sc->sc_async_head);
 537 #endif
 538  bad1:
 539         usb_freemem(&sc->sc_bus, &sc->sc_fldma);
 540         return (err);
 541 }
 542
 543 int
 544 ehci_intr(void *v)
 545 {
 546         ehci_softc_t *sc = v;
 547
 548         if (sc == NULL || sc->sc_dying)
 549                 return (0);
 550
 551         /* If we get an interrupt while polling, then just ignore it. */
 552         if (sc->sc_bus.use_polling) {
 553                 u_int32_t intrs = EHCI_STS_INTRS(EOREAD4(sc, EHCI_USBSTS));
 554
 555                 if (intrs)
 556                         EOWRITE4(sc, EHCI_USBSTS, intrs); /* Acknowledge */
 557 #ifdef DIAGNOSTIC
 558                 DPRINTFN(16, ("ehci_intr: ignored interrupt while polling\n"));
 559 #endif
 560                 return (0);
 561         }
 562
 563         return (ehci_intr1(sc));
 564 }
 565
 566 static int
 567 ehci_intr1(ehci_softc_t *sc)
 568 {
 569         u_int32_t intrs, eintrs;
 570
 571         DPRINTFN(20,("ehci_intr1: enter\n"));
 572
 573         /* In case the interrupt occurs before initialization has completed. */
 574         if (sc == NULL) {
 575 #ifdef DIAGNOSTIC
 576                 printf("ehci_intr1: sc == NULL\n");
 577 #endif
 578                 return (0);
 579         }
 580
 581         intrs = EHCI_STS_INTRS(EOREAD4(sc, EHCI_USBSTS));
 582         if (!intrs)
 583                 return (0);
 584
 585         eintrs = intrs & sc->sc_eintrs;
 586         DPRINTFN(7, ("ehci_intr1: sc=%p intrs=0x%x(0x%x) eintrs=0x%x\n",
 587                      sc, (u_int)intrs, EOREAD4(sc, EHCI_USBSTS),
 588                      (u_int)eintrs));
 589         if (!eintrs)
 590                 return (0);
 591
 592         EOWRITE4(sc, EHCI_USBSTS, intrs); /* Acknowledge */
 593         sc->sc_bus.intr_context++;
 594         sc->sc_bus.no_intrs++;
 595         if (eintrs & EHCI_STS_IAA) {
 596                 DPRINTF(("ehci_intr1: door bell\n"));
 597                 wakeup(&sc->sc_async_head);
 598                 eintrs &= ~EHCI_STS_IAA;
 599         }
 600         if (eintrs & (EHCI_STS_INT | EHCI_STS_ERRINT)) {
 601                 DPRINTFN(5,("ehci_intr1: %s %s\n",
 602                             eintrs & EHCI_STS_INT ? "INT" : "",
 603                             eintrs & EHCI_STS_ERRINT ? "ERRINT" : ""));
 604                 usb_schedsoftintr(&sc->sc_bus);
 605                 eintrs &= ~(EHCI_STS_INT | EHCI_STS_ERRINT);
 606         }
 607         if (eintrs & EHCI_STS_HSE) {
 608                 printf("%s: unrecoverable error, controller halted\n",
 609                        device_get_nameunit(sc->sc_bus.bdev));
 610                 /* XXX what else */
 611         }
 612         if (eintrs & EHCI_STS_PCD) {
 613                 ehci_pcd(sc, sc->sc_intrxfer);
 614                 /*
 615                  * Disable PCD interrupt for now, because it will be
 616                  * on until the port has been reset.
 617                  */
 618                 ehci_pcd_able(sc, 0);
 619                 /* Do not allow RHSC interrupts > 1 per second */
 620                 callout_reset(&sc->sc_tmo_pcd, hz, ehci_pcd_enable, sc);
 621                 eintrs &= ~EHCI_STS_PCD;
 622         }
 623
 624         sc->sc_bus.intr_context--;
 625
 626         if (eintrs != 0) {
 627                 /* Block unprocessed interrupts. */
 628                 sc->sc_eintrs &= ~eintrs;
 629                 EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
 630                 printf("%s: blocking intrs 0x%x\n",
 631                        device_get_nameunit(sc->sc_bus.bdev), eintrs);
 632         }
 633
 634         return (1);
 635 }
 636
 637 void
 638 ehci_pcd_able(ehci_softc_t *sc, int on)
 639 {
 640         DPRINTFN(4, ("ehci_pcd_able: on=%d\n", on));
 641         if (on)
 642                 sc->sc_eintrs |= EHCI_STS_PCD;
 643         else
 644                 sc->sc_eintrs &= ~EHCI_STS_PCD;
 645         EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
 646 }
 647
 648 void
 649 ehci_pcd_enable(void *v_sc)
 650 {
 651         ehci_softc_t *sc = v_sc;
 652
 653         ehci_pcd_able(sc, 1);
 654 }
 655
 656 /*
 657  * XXX write back xfer data for architectures with a write-back
 658  *     data cache; this is a hack because usb is mis-architected
 659  *     in blindly mixing bus_dma w/ PIO.
 660  */
 661 static __inline void
 662 hacksync(usbd_xfer_handle xfer)
 663 {
 664         usbd_pipe_handle pipe = xfer->pipe;
 665         bus_dma_tag_t tag = pipe->device->bus->buffer_dmatag;
 666         struct usb_dma_mapping *dmap = &xfer->dmamap;
 667         bus_dmamap_sync(tag, dmap->map, BUS_DMASYNC_PREWRITE);
 668 }
 669
 670 void
 671 ehci_pcd(ehci_softc_t *sc, usbd_xfer_handle xfer)
 672 {
 673         usbd_pipe_handle pipe;
 674         u_char *p;
 675         int i, m;
 676
 677         if (xfer == NULL) {
 678                 /* Just ignore the change. */
 679                 return;
 680         }
 681
 682         pipe = xfer->pipe;
 683
 684         p = xfer->buffer;
 685         m = min(sc->sc_noport, xfer->length * 8 - 1);
 686         memset(p, 0, xfer->length);
 687         for (i = 1; i <= m; i++) {
 688                 /* Pick out CHANGE bits from the status reg. */
 689                 if (EOREAD4(sc, EHCI_PORTSC(i)) & EHCI_PS_CLEAR)
 690                         p[i/8] |= 1 << (i%8);
 691         }
 692         DPRINTF(("ehci_pcd: change=0x%02x\n", *p));
 693         xfer->actlen = xfer->length;
 694         xfer->status = USBD_NORMAL_COMPLETION;
 695
 696         hacksync(xfer); /* XXX to compensate for usb_transfer_complete */
 697         usb_transfer_complete(xfer);
 698 }
 699
 700 void
 701 ehci_softintr(void *v)
 702 {
 703         ehci_softc_t *sc = v;
 704         struct ehci_xfer *ex, *nextex;
 705
 706         DPRINTFN(10,("%s: ehci_softintr (%d)\n", device_get_nameunit(sc->sc_bus.bdev),
 707                      sc->sc_bus.intr_context));
 708
 709         sc->sc_bus.intr_context++;
 710
 711         /*
 712          * The only explanation I can think of for why EHCI is as brain dead
 713          * as UHCI interrupt-wise is that Intel was involved in both.
 714          * An interrupt just tells us that something is done, we have no
 715          * clue what, so we need to scan through all active transfers. :-(
 716          */
 717         for (ex = LIST_FIRST(&sc->sc_intrhead); ex; ex = nextex) {
 718                 nextex = LIST_NEXT(ex, inext);
 719                 ehci_check_intr(sc, ex);
 720         }
 721
 722         /* Schedule a callout to catch any dropped transactions. */
 723         if ((sc->sc_flags & EHCI_SCFLG_LOSTINTRBUG) &&
 724             !LIST_EMPTY(&sc->sc_intrhead))
 725                 callout_reset(&sc->sc_tmo_intrlist, hz / 5,
 726                     ehci_intrlist_timeout, sc);
 727
 728 #ifdef USB_USE_SOFTINTR
 729         if (sc->sc_softwake) {
 730                 sc->sc_softwake = 0;
 731                 wakeup(&sc->sc_softwake);
 732         }
 733 #endif /* USB_USE_SOFTINTR */
 734
 735         sc->sc_bus.intr_context--;
 736 }
 737
 738 /* Check for an interrupt. */
 739 void
 740 ehci_check_intr(ehci_softc_t *sc, struct ehci_xfer *ex)
 741 {
 742         ehci_soft_qtd_t *sqtd, *lsqtd;
 743         u_int32_t status;
 744
 745         DPRINTFN(/*15*/2, ("ehci_check_intr: ex=%p\n", ex));
 746
 747         if (ex->sqtdstart == NULL) {
 748                 printf("ehci_check_intr: sqtdstart=NULL\n");
 749                 return;
 750         }
 751         lsqtd = ex->sqtdend;
 752 #ifdef DIAGNOSTIC
 753         if (lsqtd == NULL) {
 754                 printf("ehci_check_intr: lsqtd==0\n");
 755                 return;
 756         }
 757 #endif
 758         /*
 759          * If the last TD is still active we need to check whether there
 760          * is a an error somewhere in the middle, or whether there was a
 761          * short packet (SPD and not ACTIVE).
 762          */
 763         if (le32toh(lsqtd->qtd.qtd_status) & EHCI_QTD_ACTIVE) {
 764                 DPRINTFN(12, ("ehci_check_intr: active ex=%p\n", ex));
 765                 for (sqtd = ex->sqtdstart; sqtd != lsqtd; sqtd=sqtd->nextqtd) {
 766                         status = le32toh(sqtd->qtd.qtd_status);
 767                         /* If there's an active QTD the xfer isn't done. */
 768                         if (status & EHCI_QTD_ACTIVE)
 769                                 break;
 770                         /* Any kind of error makes the xfer done. */
 771                         if (status & EHCI_QTD_HALTED)
 772                                 goto done;
 773                         /* We want short packets, and it is short: it's done */
 774                         if (EHCI_QTD_GET_BYTES(status) != 0)
 775                                 goto done;
 776                 }
 777                 DPRINTFN(12, ("ehci_check_intr: ex=%p std=%p still active\n",
 778                               ex, ex->sqtdstart));
 779                 return;
 780         }
 781  done:
 782         DPRINTFN(12, ("ehci_check_intr: ex=%p done\n", ex));
 783         callout_stop(&ex->xfer.timeout_handle);
 784         usb_rem_task(ex->xfer.pipe->device, &ex->abort_task);
 785         ehci_idone(ex);
 786 }
 787
 788 void
 789 ehci_idone(struct ehci_xfer *ex)
 790 {
 791         usbd_xfer_handle xfer = &ex->xfer;
 792         struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
 793         ehci_soft_qtd_t *sqtd, *lsqtd;
 794         u_int32_t status = 0, nstatus = 0;
 795         ehci_physaddr_t nextphys, altnextphys;
 796         int actlen, cerr;
 797
 798         DPRINTFN(/*12*/2, ("ehci_idone: ex=%p\n", ex));
 799 #ifdef DIAGNOSTIC
 800         {
 801                 int s = splhigh();
 802                 if (ex->isdone) {
 803                         splx(s);
 804 #ifdef EHCI_DEBUG
 805                         printf("ehci_idone: ex is done!\n   ");
 806                         ehci_dump_exfer(ex);
 807 #else
 808                         printf("ehci_idone: ex=%p is done!\n", ex);
 809 #endif
 810                         return;
 811                 }
 812                 ex->isdone = 1;
 813                 splx(s);
 814         }
 815 #endif
 816
 817         if (xfer->status == USBD_CANCELLED ||
 818             xfer->status == USBD_TIMEOUT) {
 819                 DPRINTF(("ehci_idone: aborted xfer=%p\n", xfer));
 820                 return;
 821         }
 822
 823 #ifdef EHCI_DEBUG
 824         DPRINTFN(/*10*/2, ("ehci_idone: xfer=%p, pipe=%p ready\n", xfer, epipe));
 825         if (ehcidebug > 10)
 826                 ehci_dump_sqtds(ex->sqtdstart);
 827 #endif
 828
 829         /*
 830          * Make sure that the QH overlay qTD does not reference any
 831          * of the qTDs we are about to free. This is probably only
 832          * necessary if the transfer is marked as HALTED.
 833          */
 834         nextphys = EHCI_LINK_ADDR(le32toh(epipe->sqh->qh.qh_qtd.qtd_next));
 835         altnextphys =
 836             EHCI_LINK_ADDR(le32toh(epipe->sqh->qh.qh_qtd.qtd_altnext));
 837         for (sqtd = ex->sqtdstart; sqtd != ex->sqtdend->nextqtd;
 838              sqtd = sqtd->nextqtd) {
 839                 if (sqtd->physaddr == nextphys) {
 840                         epipe->sqh->qh.qh_qtd.qtd_next =
 841                             htole32(ex->sqtdend->nextqtd->physaddr);
 842                         DPRINTFN(4, ("ehci_idone: updated overlay next ptr\n"));
 843
 844                 }
 845                 if (sqtd->physaddr == altnextphys) {
 846                         DPRINTFN(4,
 847                             ("ehci_idone: updated overlay altnext ptr\n"));
 848                         epipe->sqh->qh.qh_qtd.qtd_altnext =
 849                             htole32(ex->sqtdend->nextqtd->physaddr);
 850                 }
 851         }
 852
 853         /* The transfer is done, compute actual length and status. */
 854         lsqtd = ex->sqtdend;
 855         actlen = 0;
 856         for (sqtd = ex->sqtdstart; sqtd != lsqtd->nextqtd; sqtd=sqtd->nextqtd) {
 857                 nstatus = le32toh(sqtd->qtd.qtd_status);
 858                 if (nstatus & EHCI_QTD_ACTIVE)
 859                         break;
 860
 861                 status = nstatus;
 862                 /* halt is ok if descriptor is last, and complete */
 863                 if (sqtd == lsqtd && EHCI_QTD_GET_BYTES(status) == 0)
 864                         status &= ~EHCI_QTD_HALTED;
 865                 if (EHCI_QTD_GET_PID(status) != EHCI_QTD_PID_SETUP)
 866                         actlen += sqtd->len - EHCI_QTD_GET_BYTES(status);
 867         }
 868
 869         cerr = EHCI_QTD_GET_CERR(status);
 870         DPRINTFN(/*10*/2, ("ehci_idone: len=%d, actlen=%d, cerr=%d, "
 871             "status=0x%x\n", xfer->length, actlen, cerr, status));
 872         xfer->actlen = actlen;
 873         if ((status & EHCI_QTD_HALTED) != 0) {
 874 #ifdef EHCI_DEBUG
 875                 char sbuf[128];
 876
 877                 bitmask_snprintf((u_int32_t)status,
 878                     "\20\7HALTED\6BUFERR\5BABBLE\4XACTERR"
 879                     "\3MISSED\2SPLIT\1PING", sbuf, sizeof(sbuf));
 880
 881                 DPRINTFN(2,
 882                          ("ehci_idone: error, addr=%d, endpt=0x%02x, "
 883                           "status 0x%s\n",
 884                           xfer->pipe->device->address,
 885                           xfer->pipe->endpoint->edesc->bEndpointAddress,
 886                           sbuf));
 887                 if (ehcidebug > 2) {
 888                         ehci_dump_sqh(epipe->sqh);
 889                         ehci_dump_sqtds(ex->sqtdstart);
 890                 }
 891 #endif
 892                 if ((status & EHCI_QTD_BABBLE) == 0 && cerr > 0)
 893                         xfer->status = USBD_STALLED;
 894                 else
 895                         xfer->status = USBD_IOERROR; /* more info XXX */
 896         } else {
 897                 xfer->status = USBD_NORMAL_COMPLETION;
 898         }
 899
 900         usb_transfer_complete(xfer);
 901         DPRINTFN(/*12*/2, ("ehci_idone: ex=%p done\n", ex));
 902 }
 903
 904 /*
 905  * Wait here until controller claims to have an interrupt.
 906  * Then call ehci_intr and return.  Use timeout to avoid waiting
 907  * too long.
 908  */
 909 void
 910 ehci_waitintr(ehci_softc_t *sc, usbd_xfer_handle xfer)
 911 {
 912         int timo = xfer->timeout;
 913         int usecs;
 914         u_int32_t intrs;
 915
 916         xfer->status = USBD_IN_PROGRESS;
 917         for (usecs = timo * 1000000 / hz; usecs > 0; usecs -= 1000) {
 918                 usb_delay_ms(&sc->sc_bus, 1);
 919                 if (sc->sc_dying)
 920                         break;
 921                 intrs = EHCI_STS_INTRS(EOREAD4(sc, EHCI_USBSTS)) &
 922                         sc->sc_eintrs;
 923                 DPRINTFN(15,("ehci_waitintr: 0x%04x\n", intrs));
 924 #ifdef EHCI_DEBUG
 925                 if (ehcidebug > 15)
 926                         ehci_dump_regs(sc);
 927 #endif
 928                 if (intrs) {
 929                         ehci_intr1(sc);
 930                         if (xfer->status != USBD_IN_PROGRESS)
 931                                 return;
 932                 }
 933         }
 934
 935         /* Timeout */
 936         DPRINTF(("ehci_waitintr: timeout\n"));
 937         xfer->status = USBD_TIMEOUT;
 938         usb_transfer_complete(xfer);
 939         /* XXX should free TD */
 940 }
 941
 942 void
 943 ehci_poll(struct usbd_bus *bus)
 944 {
 945         ehci_softc_t *sc = (ehci_softc_t *)bus;
 946 #ifdef EHCI_DEBUG
 947         static int last;
 948         int new;
 949         new = EHCI_STS_INTRS(EOREAD4(sc, EHCI_USBSTS));
 950         if (new != last) {
 951                 DPRINTFN(10,("ehci_poll: intrs=0x%04x\n", new));
 952                 last = new;
 953         }
 954 #endif
 955
 956         if (EOREAD4(sc, EHCI_USBSTS) & sc->sc_eintrs)
 957                 ehci_intr1(sc);
 958 }
 959
 960 int
 961 ehci_detach(struct ehci_softc *sc, int flags)
 962 {
 963         int rv = 0;
 964
 965         sc->sc_dying = 1;
 966
 967         EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
 968         (void) ehci_hcreset(sc);
 969         callout_stop(&sc->sc_tmo_intrlist);
 970         callout_stop(&sc->sc_tmo_pcd);
 971
 972 #if defined(__NetBSD__) || defined(__OpenBSD__)
 973         if (sc->sc_powerhook != NULL)
 974                 powerhook_disestablish(sc->sc_powerhook);
 975         if (sc->sc_shutdownhook != NULL)
 976                 shutdownhook_disestablish(sc->sc_shutdownhook);
 977 #endif
 978         usb_delay_ms(&sc->sc_bus, 300); /* XXX let stray task complete */
 979
 980         usb_freemem(&sc->sc_bus, &sc->sc_fldma);
 981         /* XXX free other data structures XXX */
 982
 983         return (rv);
 984 }
 985
 986 /*
 987  * Handle suspend/resume.
 988  *
 989  * We need to switch to polling mode here, because this routine is
 990  * called from an interrupt context.  This is all right since we
 991  * are almost suspended anyway.
 992  */
 993 void
 994 ehci_power(int why, void *v)
 995 {
 996         ehci_softc_t *sc = v;
 997         u_int32_t cmd, hcr;
 998         int s, i;
 999
1000 #ifdef EHCI_DEBUG
1001         DPRINTF(("ehci_power: sc=%p, why=%d\n", sc, why));
1002         if (ehcidebug > 0)
1003                 ehci_dump_regs(sc);
1004 #endif
1005
1006         s = splhardusb();
1007         switch (why) {
1008         case PWR_SUSPEND:
1009         case PWR_STANDBY:
1010                 sc->sc_bus.use_polling++;
1011
1012                 for (i = 1; i <= sc->sc_noport; i++) {
1013                         cmd = EOREAD4(sc, EHCI_PORTSC(i));
1014                         if ((cmd & EHCI_PS_PO) == 0 &&
1015                             (cmd & EHCI_PS_PE) == EHCI_PS_PE)
1016                                 EOWRITE4(sc, EHCI_PORTSC(i),
1017                                     cmd | EHCI_PS_SUSP);
1018                 }
1019
1020                 sc->sc_cmd = EOREAD4(sc, EHCI_USBCMD);
1021
1022                 cmd = sc->sc_cmd & ~(EHCI_CMD_ASE | EHCI_CMD_PSE);
1023                 EOWRITE4(sc, EHCI_USBCMD, cmd);
1024
1025                 for (i = 0; i < 100; i++) {
1026                         hcr = EOREAD4(sc, EHCI_USBSTS) &
1027                             (EHCI_STS_ASS | EHCI_STS_PSS);
1028                         if (hcr == 0)
1029                                 break;
1030
1031                         usb_delay_ms(&sc->sc_bus, 1);
1032                 }
1033                 if (hcr != 0) {
1034                         printf("%s: reset timeout\n",
1035                             device_get_nameunit(sc->sc_bus.bdev));
1036                 }
1037
1038                 cmd &= ~EHCI_CMD_RS;
1039                 EOWRITE4(sc, EHCI_USBCMD, cmd);
1040
1041                 for (i = 0; i < 100; i++) {
1042                         hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
1043                         if (hcr == EHCI_STS_HCH)
1044                                 break;
1045
1046                         usb_delay_ms(&sc->sc_bus, 1);
1047                 }
1048                 if (hcr != EHCI_STS_HCH) {
1049                         printf("%s: config timeout\n",
1050                             device_get_nameunit(sc->sc_bus.bdev));
1051                 }
1052
1053                 sc->sc_bus.use_polling--;
1054                 break;
1055
1056         case PWR_RESUME:
1057                 sc->sc_bus.use_polling++;
1058
1059                 /* restore things in case the bios sucks */
1060                 EOWRITE4(sc, EHCI_CTRLDSSEGMENT, 0);
1061                 EOWRITE4(sc, EHCI_PERIODICLISTBASE, DMAADDR(&sc->sc_fldma, 0));
1062                 EOWRITE4(sc, EHCI_ASYNCLISTADDR,
1063                     sc->sc_async_head->physaddr | EHCI_LINK_QH);
1064                 EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
1065
1066                 hcr = 0;
1067                 for (i = 1; i <= sc->sc_noport; i++) {
1068                         cmd = EOREAD4(sc, EHCI_PORTSC(i));
1069                         if ((cmd & EHCI_PS_PO) == 0 &&
1070                             (cmd & EHCI_PS_SUSP) == EHCI_PS_SUSP) {
1071                                 EOWRITE4(sc, EHCI_PORTSC(i),
1072                                     cmd | EHCI_PS_FPR);
1073                                 hcr = 1;
1074                         }
1075                 }
1076
1077                 if (hcr) {
1078                         usb_delay_ms(&sc->sc_bus, USB_RESUME_WAIT);
1079
1080                         for (i = 1; i <= sc->sc_noport; i++) {
1081                                 cmd = EOREAD4(sc, EHCI_PORTSC(i));
1082                                 if ((cmd & EHCI_PS_PO) == 0 &&
1083                                     (cmd & EHCI_PS_SUSP) == EHCI_PS_SUSP)
1084                                         EOWRITE4(sc, EHCI_PORTSC(i),
1085                                             cmd & ~EHCI_PS_FPR);
1086                         }
1087                 }
1088
1089                 EOWRITE4(sc, EHCI_USBCMD, sc->sc_cmd);
1090
1091                 for (i = 0; i < 100; i++) {
1092                         hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
1093                         if (hcr != EHCI_STS_HCH)
1094                                 break;
1095
1096                         usb_delay_ms(&sc->sc_bus, 1);
1097                 }
1098                 if (hcr == EHCI_STS_HCH) {
1099                         printf("%s: config timeout\n",
1100                             device_get_nameunit(sc->sc_bus.bdev));
1101                 }
1102
1103                 usb_delay_ms(&sc->sc_bus, USB_RESUME_WAIT);
1104
1105                 sc->sc_bus.use_polling--;
1106                 break;
1107         case PWR_SOFTSUSPEND:
1108         case PWR_SOFTSTANDBY:
1109         case PWR_SOFTRESUME:
1110                 break;
1111         }
1112         splx(s);
1113
1114 #ifdef EHCI_DEBUG
1115         DPRINTF(("ehci_power: sc=%p\n", sc));
1116         if (ehcidebug > 0)
1117                 ehci_dump_regs(sc);
1118 #endif
1119 }
1120
1121 /*
1122  * Shut down the controller when the system is going down.
1123  */
1124 void
1125 ehci_shutdown(void *v)
1126 {
1127         ehci_softc_t *sc = v;
1128
1129         DPRINTF(("ehci_shutdown: stopping the HC\n"));
1130         (void) ehci_hcreset(sc);
1131 }
1132
1133 usbd_status
1134 ehci_allocm(struct usbd_bus *bus, usb_dma_t *dma, u_int32_t size)
1135 {
1136         usbd_status err;
1137
1138         err = usb_allocmem(bus, size, 0, dma);
1139 #ifdef EHCI_DEBUG
1140         if (err)
1141                 printf("ehci_allocm: usb_allocmem()=%d\n", err);
1142 #endif
1143         return (err);
1144 }
1145
1146 void
1147 ehci_freem(struct usbd_bus *bus, usb_dma_t *dma)
1148 {
1149         usb_freemem(bus, dma);
1150 }
1151
1152 usbd_xfer_handle
1153 ehci_allocx(struct usbd_bus *bus)
1154 {
1155         struct ehci_softc *sc = (struct ehci_softc *)bus;
1156         usbd_xfer_handle xfer;
1157
1158         xfer = STAILQ_FIRST(&sc->sc_free_xfers);
1159         if (xfer != NULL) {
1160                 STAILQ_REMOVE_HEAD(&sc->sc_free_xfers, next);
1161 #ifdef DIAGNOSTIC
1162                 if (xfer->busy_free != XFER_FREE) {
1163                         printf("ehci_allocx: xfer=%p not free, 0x%08x\n", xfer,
1164                                xfer->busy_free);
1165                 }
1166 #endif
1167         } else {
1168                 xfer = malloc(sizeof(struct ehci_xfer), M_USB, M_NOWAIT);
1169         }
1170         if (xfer != NULL) {
1171                 memset(xfer, 0, sizeof(struct ehci_xfer));
1172                 usb_init_task(&EXFER(xfer)->abort_task, ehci_timeout_task,
1173                     xfer);
1174                 EXFER(xfer)->ehci_xfer_flags = 0;
1175 #ifdef DIAGNOSTIC
1176                 EXFER(xfer)->isdone = 1;
1177                 xfer->busy_free = XFER_BUSY;
1178 #endif
1179         }
1180         return (xfer);
1181 }
1182
1183 void
1184 ehci_freex(struct usbd_bus *bus, usbd_xfer_handle xfer)
1185 {
1186         struct ehci_softc *sc = (struct ehci_softc *)bus;
1187
1188 #ifdef DIAGNOSTIC
1189         if (xfer->busy_free != XFER_BUSY) {
1190                 printf("ehci_freex: xfer=%p not busy, 0x%08x\n", xfer,
1191                        xfer->busy_free);
1192                 return;
1193         }
1194         xfer->busy_free = XFER_FREE;
1195         if (!EXFER(xfer)->isdone) {
1196                 printf("ehci_freex: !isdone\n");
1197                 return;
1198         }
1199 #endif
1200         STAILQ_INSERT_HEAD(&sc->sc_free_xfers, xfer, next);
1201 }
1202
1203 static void
1204 ehci_device_clear_toggle(usbd_pipe_handle pipe)
1205 {
1206         struct ehci_pipe *epipe = (struct ehci_pipe *)pipe;
1207
1208         DPRINTF(("ehci_device_clear_toggle: epipe=%p status=0x%x\n",
1209                  epipe, epipe->sqh->qh.qh_qtd.qtd_status));
1210 #ifdef USB_DEBUG
1211         if (ehcidebug)
1212                 usbd_dump_pipe(pipe);
1213 #endif
1214         KASSERT((epipe->sqh->qh.qh_qtd.qtd_status &
1215             htole32(EHCI_QTD_ACTIVE)) == 0,
1216             ("ehci_device_clear_toggle: queue active"));
1217         epipe->sqh->qh.qh_qtd.qtd_status &= htole32(~EHCI_QTD_TOGGLE_MASK);
1218 }
1219
1220 static void
1221 ehci_noop(usbd_pipe_handle pipe)
1222 {
1223 }
1224
1225 #ifdef EHCI_DEBUG
1226 void
1227 ehci_dump_regs(ehci_softc_t *sc)
1228 {
1229         int i;
1230         printf("cmd=0x%08x, sts=0x%08x, ien=0x%08x\n",
1231                EOREAD4(sc, EHCI_USBCMD),
1232                EOREAD4(sc, EHCI_USBSTS),
1233                EOREAD4(sc, EHCI_USBINTR));
1234         printf("frindex=0x%08x ctrdsegm=0x%08x periodic=0x%08x async=0x%08x\n",
1235                EOREAD4(sc, EHCI_FRINDEX),
1236                EOREAD4(sc, EHCI_CTRLDSSEGMENT),
1237                EOREAD4(sc, EHCI_PERIODICLISTBASE),
1238                EOREAD4(sc, EHCI_ASYNCLISTADDR));
1239         for (i = 1; i <= sc->sc_noport; i++)
1240                 printf("port %d status=0x%08x\n", i,
1241                        EOREAD4(sc, EHCI_PORTSC(i)));
1242 }
1243
1244 /*
1245  * Unused function - this is meant to be called from a kernel
1246  * debugger.
1247  */
1248 void
1249 ehci_dump()
1250 {
1251         ehci_dump_regs(theehci);
1252 }
1253
1254 void
1255 ehci_dump_link(ehci_link_t link, int type)
1256 {
1257         link = le32toh(link);
1258         printf("0x%08x", link);
1259         if (link & EHCI_LINK_TERMINATE)
1260                 printf("<T>");
1261         else {
1262                 printf("<");
1263                 if (type) {
1264                         switch (EHCI_LINK_TYPE(link)) {
1265                         case EHCI_LINK_ITD: printf("ITD"); break;
1266                         case EHCI_LINK_QH: printf("QH"); break;
1267                         case EHCI_LINK_SITD: printf("SITD"); break;
1268                         case EHCI_LINK_FSTN: printf("FSTN"); break;
1269                         }
1270                 }
1271                 printf(">");
1272         }
1273 }
1274
1275 void
1276 ehci_dump_sqtds(ehci_soft_qtd_t *sqtd)
1277 {
1278         int i;
1279         u_int32_t stop;
1280
1281         stop = 0;
1282         for (i = 0; sqtd && i < 20 && !stop; sqtd = sqtd->nextqtd, i++) {
1283                 ehci_dump_sqtd(sqtd);
1284                 stop = sqtd->qtd.qtd_next & htole32(EHCI_LINK_TERMINATE);
1285         }
1286         if (sqtd)
1287                 printf("dump aborted, too many TDs\n");
1288 }
1289
1290 void
1291 ehci_dump_sqtd(ehci_soft_qtd_t *sqtd)
1292 {
1293         printf("QTD(%p) at 0x%08x:\n", sqtd, sqtd->physaddr);
1294         ehci_dump_qtd(&sqtd->qtd);
1295 }
1296
1297 void
1298 ehci_dump_qtd(ehci_qtd_t *qtd)
1299 {
1300         u_int32_t s;
1301         char sbuf[128];
1302
1303         printf("  next="); ehci_dump_link(qtd->qtd_next, 0);
1304         printf(" altnext="); ehci_dump_link(qtd->qtd_altnext, 0);
1305         printf("\n");
1306         s = le32toh(qtd->qtd_status);
1307         bitmask_snprintf(EHCI_QTD_GET_STATUS(s),
1308                          "\20\10ACTIVE\7HALTED\6BUFERR\5BABBLE\4XACTERR"
1309                          "\3MISSED\2SPLIT\1PING", sbuf, sizeof(sbuf));
1310         printf("  status=0x%08x: toggle=%d bytes=0x%x ioc=%d c_page=0x%x\n",
1311                s, EHCI_QTD_GET_TOGGLE(s), EHCI_QTD_GET_BYTES(s),
1312                EHCI_QTD_GET_IOC(s), EHCI_QTD_GET_C_PAGE(s));
1313         printf("    cerr=%d pid=%d stat=0x%s\n", EHCI_QTD_GET_CERR(s),
1314                EHCI_QTD_GET_PID(s), sbuf);
1315         for (s = 0; s < 5; s++)
1316                 printf("  buffer[%d]=0x%08x\n", s, le32toh(qtd->qtd_buffer[s]));
1317 }
1318
1319 void
1320 ehci_dump_sqh(ehci_soft_qh_t *sqh)
1321 {
1322         ehci_qh_t *qh = &sqh->qh;
1323         u_int32_t endp, endphub;
1324
1325         printf("QH(%p) at 0x%08x:\n", sqh, sqh->physaddr);
1326         printf("  sqtd=%p inactivesqtd=%p\n", sqh->sqtd, sqh->inactivesqtd);
1327         printf("  link="); ehci_dump_link(qh->qh_link, 1); printf("\n");
1328         endp = le32toh(qh->qh_endp);
1329         printf("  endp=0x%08x\n", endp);
1330         printf("    addr=0x%02x inact=%d endpt=%d eps=%d dtc=%d hrecl=%d\n",
1331                EHCI_QH_GET_ADDR(endp), EHCI_QH_GET_INACT(endp),
1332                EHCI_QH_GET_ENDPT(endp),  EHCI_QH_GET_EPS(endp),
1333                EHCI_QH_GET_DTC(endp), EHCI_QH_GET_HRECL(endp));
1334         printf("    mpl=0x%x ctl=%d nrl=%d\n",
1335                EHCI_QH_GET_MPL(endp), EHCI_QH_GET_CTL(endp),
1336                EHCI_QH_GET_NRL(endp));
1337         endphub = le32toh(qh->qh_endphub);
1338         printf("  endphub=0x%08x\n", endphub);
1339         printf("    smask=0x%02x cmask=0x%02x huba=0x%02x port=%d mult=%d\n",
1340                EHCI_QH_GET_SMASK(endphub), EHCI_QH_GET_CMASK(endphub),
1341                EHCI_QH_GET_HUBA(endphub), EHCI_QH_GET_PORT(endphub),
1342                EHCI_QH_GET_MULT(endphub));
1343         printf("  curqtd="); ehci_dump_link(qh->qh_curqtd, 0); printf("\n");
1344         printf("Overlay qTD:\n");
1345         ehci_dump_qtd(&qh->qh_qtd);
1346 }
1347
1348 #ifdef DIAGNOSTIC
1349 static void
1350 ehci_dump_exfer(struct ehci_xfer *ex)
1351 {
1352         printf("ehci_dump_exfer: ex=%p\n", ex);
1353 }
1354 #endif
1355 #endif
1356
1357 usbd_status
1358 ehci_open(usbd_pipe_handle pipe)
1359 {
1360         usbd_device_handle dev = pipe->device;
1361         ehci_softc_t *sc = (ehci_softc_t *)dev->bus;
1362         usb_endpoint_descriptor_t *ed = pipe->endpoint->edesc;
1363         u_int8_t addr = dev->address;
1364         u_int8_t xfertype = ed->bmAttributes & UE_XFERTYPE;
1365         struct ehci_pipe *epipe = (struct ehci_pipe *)pipe;
1366         ehci_soft_qh_t *sqh;
1367         usbd_status err;
1368         int s;
1369         int ival, speed, naks;
1370         int hshubaddr, hshubport;
1371
1372         DPRINTFN(1, ("ehci_open: pipe=%p, addr=%d, endpt=%d (%d)\n",
1373                      pipe, addr, ed->bEndpointAddress, sc->sc_addr));
1374
1375         if (dev->myhsport) {
1376                 hshubaddr = dev->myhsport->parent->address;
1377                 hshubport = dev->myhsport->portno;
1378         } else {
1379                 hshubaddr = 0;
1380                 hshubport = 0;
1381         }
1382
1383         if (sc->sc_dying)
1384                 return (USBD_IOERROR);
1385
1386         if (addr == sc->sc_addr) {
1387                 switch (ed->bEndpointAddress) {
1388                 case USB_CONTROL_ENDPOINT:
1389                         pipe->methods = &ehci_root_ctrl_methods;
1390                         break;
1391                 case UE_DIR_IN | EHCI_INTR_ENDPT:
1392                         pipe->methods = &ehci_root_intr_methods;
1393                         break;
1394                 default:
1395                         return (USBD_INVAL);
1396                 }
1397                 return (USBD_NORMAL_COMPLETION);
1398         }
1399
1400         /* XXX All this stuff is only valid for async. */
1401         switch (dev->speed) {
1402         case USB_SPEED_LOW:  speed = EHCI_QH_SPEED_LOW;  break;
1403         case USB_SPEED_FULL: speed = EHCI_QH_SPEED_FULL; break;
1404         case USB_SPEED_HIGH: speed = EHCI_QH_SPEED_HIGH; break;
1405         default: panic("ehci_open: bad device speed %d", dev->speed);
1406         }
1407         if (speed != EHCI_QH_SPEED_HIGH && xfertype == UE_ISOCHRONOUS) {
1408                 printf("%s: *** WARNING: opening low/full speed device, this "
1409                        "does not work yet.\n",
1410                        device_get_nameunit(sc->sc_bus.bdev));
1411                 DPRINTFN(1,("ehci_open: hshubaddr=%d hshubport=%d\n",
1412                             hshubaddr, hshubport));
1413                 return USBD_INVAL;
1414         }
1415
1416         naks = 8;               /* XXX */
1417         sqh = ehci_alloc_sqh(sc);
1418         if (sqh == NULL)
1419                 goto bad0;
1420         /* qh_link filled when the QH is added */
1421         sqh->qh.qh_endp = htole32(
1422                 EHCI_QH_SET_ADDR(addr) |
1423                 EHCI_QH_SET_ENDPT(UE_GET_ADDR(ed->bEndpointAddress)) |
1424                 EHCI_QH_SET_EPS(speed) |
1425                 (xfertype == UE_CONTROL ? EHCI_QH_DTC : 0) |
1426                 EHCI_QH_SET_MPL(UGETW(ed->wMaxPacketSize)) |
1427                 (speed != EHCI_QH_SPEED_HIGH && xfertype == UE_CONTROL ?
1428                  EHCI_QH_CTL : 0) |
1429                 EHCI_QH_SET_NRL(naks)
1430                 );
1431         sqh->qh.qh_endphub = htole32(
1432                 EHCI_QH_SET_MULT(1) |
1433                 EHCI_QH_SET_HUBA(hshubaddr) |
1434                 EHCI_QH_SET_PORT(hshubport) |
1435                 EHCI_QH_SET_CMASK(0x1c) |
1436                 EHCI_QH_SET_SMASK(xfertype == UE_INTERRUPT ? 0x01 : 0)
1437                 );
1438         sqh->qh.qh_curqtd = EHCI_NULL;
1439         /* The overlay qTD was already set up by ehci_alloc_sqh(). */
1440         sqh->qh.qh_qtd.qtd_status =
1441             htole32(EHCI_QTD_SET_TOGGLE(pipe->endpoint->savedtoggle));
1442
1443         epipe->sqh = sqh;
1444
1445         switch (xfertype) {
1446         case UE_CONTROL:
1447                 err = usb_allocmem(&sc->sc_bus, sizeof(usb_device_request_t),
1448                                    0, &epipe->u.ctl.reqdma);
1449 #ifdef EHCI_DEBUG
1450                 if (err)
1451                         printf("ehci_open: usb_allocmem()=%d\n", err);
1452 #endif
1453                 if (err)
1454                         goto bad1;
1455                 pipe->methods = &ehci_device_ctrl_methods;
1456                 s = splusb();
1457                 ehci_add_qh(sqh, sc->sc_async_head);
1458                 splx(s);
1459                 break;
1460         case UE_BULK:
1461                 pipe->methods = &ehci_device_bulk_methods;
1462                 s = splusb();
1463                 ehci_add_qh(sqh, sc->sc_async_head);
1464                 splx(s);
1465                 break;
1466         case UE_INTERRUPT:
1467                 pipe->methods = &ehci_device_intr_methods;
1468                 ival = pipe->interval;
1469                 if (ival == USBD_DEFAULT_INTERVAL)
1470                         ival = ed->bInterval;
1471                 return (ehci_device_setintr(sc, sqh, ival));
1472         case UE_ISOCHRONOUS:
1473                 pipe->methods = &ehci_device_isoc_methods;
1474                 return (USBD_INVAL);
1475         default:
1476                 return (USBD_INVAL);
1477         }
1478         return (USBD_NORMAL_COMPLETION);
1479
1480  bad1:
1481         ehci_free_sqh(sc, sqh);
1482  bad0:
1483         return (USBD_NOMEM);
1484 }
1485
1486 /*
1487  * Add an ED to the schedule.  Called at splusb().
1488  * If in the async schedule, it will always have a next.
1489  * If in the intr schedule it may not.
1490  */
1491 void
1492 ehci_add_qh(ehci_soft_qh_t *sqh, ehci_soft_qh_t *head)
1493 {
1494         SPLUSBCHECK;
1495
1496         sqh->next = head->next;
1497         sqh->prev = head;
1498         sqh->qh.qh_link = head->qh.qh_link;
1499         head->next = sqh;
1500         if (sqh->next)
1501                 sqh->next->prev = sqh;
1502         head->qh.qh_link = htole32(sqh->physaddr | EHCI_LINK_QH);
1503
1504 #ifdef EHCI_DEBUG
1505         if (ehcidebug > 5) {
1506                 printf("ehci_add_qh:\n");
1507                 ehci_dump_sqh(sqh);
1508         }
1509 #endif
1510 }
1511
1512 /*
1513  * Remove an ED from the schedule.  Called at splusb().
1514  * Will always have a 'next' if it's in the async list as it's circular.
1515  */
1516 void
1517 ehci_rem_qh(ehci_softc_t *sc, ehci_soft_qh_t *sqh, ehci_soft_qh_t *head)
1518 {
1519         SPLUSBCHECK;
1520         /* XXX */
1521         sqh->prev->qh.qh_link = sqh->qh.qh_link;
1522         sqh->prev->next = sqh->next;
1523         if (sqh->next)
1524                 sqh->next->prev = sqh->prev;
1525         ehci_sync_hc(sc);
1526 }
1527
1528 /* Restart a QH following the addition of a qTD. */
1529 void
1530 ehci_activate_qh(ehci_soft_qh_t *sqh, ehci_soft_qtd_t *sqtd)
1531 {
1532         KASSERT((sqtd->qtd.qtd_status & htole32(EHCI_QTD_ACTIVE)) == 0,
1533             ("ehci_activate_qh: already active"));
1534
1535         /*
1536          * When a QH is idle, the overlay qTD should be marked as not
1537          * halted and not active. This causes the host controller to
1538          * retrieve the real qTD on each pass (rather than just examinig
1539          * the overlay), so it will notice when we activate the qTD.
1540          */
1541         if (sqtd == sqh->sqtd) {
1542                 /* Check that the hardware is in the state we expect. */
1543                 if (EHCI_LINK_ADDR(le32toh(sqh->qh.qh_qtd.qtd_next)) !=
1544                     sqtd->physaddr) {
1545 #ifdef EHCI_DEBUG
1546                         printf("ehci_activate_qh: unexpected next ptr\n");
1547                         ehci_dump_sqh(sqh);
1548                         ehci_dump_sqtds(sqh->sqtd);
1549 #endif
1550                         sqh->qh.qh_qtd.qtd_next = htole32(sqtd->physaddr);
1551                         sqh->qh.qh_qtd.qtd_altnext = EHCI_NULL;
1552                 }
1553                 /* Ensure the flags are correct. */
1554                 sqh->qh.qh_qtd.qtd_status &= htole32(EHCI_QTD_PINGSTATE |
1555                     EHCI_QTD_TOGGLE_MASK);
1556         }
1557
1558         /* Now activate the qTD. */
1559         sqtd->qtd.qtd_status |= htole32(EHCI_QTD_ACTIVE);
1560 }
1561
1562 /*
1563  * Ensure that the HC has released all references to the QH.  We do this
1564  * by asking for a Async Advance Doorbell interrupt and then we wait for
1565  * the interrupt.
1566  * To make this easier we first obtain exclusive use of the doorbell.
1567  */
1568 void
1569 ehci_sync_hc(ehci_softc_t *sc)
1570 {
1571         int s, error;
1572
1573         if (sc->sc_dying) {
1574                 DPRINTFN(2,("ehci_sync_hc: dying\n"));
1575                 return;
1576         }
1577         DPRINTFN(2,("ehci_sync_hc: enter\n"));
1578         /* get doorbell */
1579         lockmgr(&sc->sc_doorbell_lock, LK_EXCLUSIVE, NULL);
1580         s = splhardusb();
1581         /* ask for doorbell */
1582         EOWRITE4(sc, EHCI_USBCMD, EOREAD4(sc, EHCI_USBCMD) | EHCI_CMD_IAAD);
1583         DPRINTFN(1,("ehci_sync_hc: cmd=0x%08x sts=0x%08x\n",
1584                     EOREAD4(sc, EHCI_USBCMD), EOREAD4(sc, EHCI_USBSTS)));
1585         error = tsleep(&sc->sc_async_head, PZERO, "ehcidi", hz); /* bell wait */
1586         DPRINTFN(1,("ehci_sync_hc: cmd=0x%08x sts=0x%08x\n",
1587                     EOREAD4(sc, EHCI_USBCMD), EOREAD4(sc, EHCI_USBSTS)));
1588         splx(s);
1589         /* release doorbell */
1590         lockmgr(&sc->sc_doorbell_lock, LK_RELEASE, NULL);
1591 #ifdef DIAGNOSTIC
1592         if (error)
1593                 printf("ehci_sync_hc: tsleep() = %d\n", error);
1594 #endif
1595         DPRINTFN(2,("ehci_sync_hc: exit\n"));
1596 }
1597
1598 /***********/
1599
1600 /*
1601  * Data structures and routines to emulate the root hub.
1602  */
1603 static usb_device_descriptor_t ehci_devd = {
1604         USB_DEVICE_DESCRIPTOR_SIZE,
1605         UDESC_DEVICE,           /* type */
1606         {0x00, 0x02},           /* USB version */
1607         UDCLASS_HUB,            /* class */
1608         UDSUBCLASS_HUB,         /* subclass */
1609         UDPROTO_HSHUBSTT,       /* protocol */
1610         64,                     /* max packet */
1611         {0},{0},{0x00,0x01},    /* device id */
1612         1,2,0,                  /* string indicies */
1613         1                       /* # of configurations */
1614 };
1615
1616 static usb_device_qualifier_t ehci_odevd = {
1617         USB_DEVICE_DESCRIPTOR_SIZE,
1618         UDESC_DEVICE_QUALIFIER, /* type */
1619         {0x00, 0x02},           /* USB version */
1620         UDCLASS_HUB,            /* class */
1621         UDSUBCLASS_HUB,         /* subclass */
1622         UDPROTO_FSHUB,          /* protocol */
1623         64,                     /* max packet */
1624         1,                      /* # of configurations */
1625         0
1626 };
1627
1628 static usb_config_descriptor_t ehci_confd = {
1629         USB_CONFIG_DESCRIPTOR_SIZE,
1630         UDESC_CONFIG,
1631         {USB_CONFIG_DESCRIPTOR_SIZE +
1632          USB_INTERFACE_DESCRIPTOR_SIZE +
1633          USB_ENDPOINT_DESCRIPTOR_SIZE},
1634         1,
1635         1,
1636         0,
1637         UC_SELF_POWERED,
1638         0                       /* max power */
1639 };
1640
1641 static usb_interface_descriptor_t ehci_ifcd = {
1642         USB_INTERFACE_DESCRIPTOR_SIZE,
1643         UDESC_INTERFACE,
1644         0,
1645         0,
1646         1,
1647         UICLASS_HUB,
1648         UISUBCLASS_HUB,
1649         UIPROTO_HSHUBSTT,
1650         0
1651 };
1652
1653 static usb_endpoint_descriptor_t ehci_endpd = {
1654         USB_ENDPOINT_DESCRIPTOR_SIZE,
1655         UDESC_ENDPOINT,
1656         UE_DIR_IN | EHCI_INTR_ENDPT,
1657         UE_INTERRUPT,
1658         {8, 0},                 /* max packet */
1659         255
1660 };
1661
1662 static usb_hub_descriptor_t ehci_hubd = {
1663         USB_HUB_DESCRIPTOR_SIZE,
1664         UDESC_HUB,
1665         0,
1666         {0,0},
1667         0,
1668         0,
1669         {0},
1670 };
1671
1672 static int
1673 ehci_str(usb_string_descriptor_t *p, int l, char *s)
1674 {
1675         int i;
1676
1677         if (l == 0)
1678                 return (0);
1679         p->bLength = 2 * strlen(s) + 2;
1680         if (l == 1)
1681                 return (1);
1682         p->bDescriptorType = UDESC_STRING;
1683         l -= 2;
1684         for (i = 0; s[i] && l > 1; i++, l -= 2)
1685                 USETW2(p->bString[i], 0, s[i]);
1686         return (2*i+2);
1687 }
1688
1689 /*
1690  * Simulate a hardware hub by handling all the necessary requests.
1691  */
1692 static usbd_status
1693 ehci_root_ctrl_transfer(usbd_xfer_handle xfer)
1694 {
1695         usbd_status err;
1696
1697         /* Insert last in queue. */
1698         err = usb_insert_transfer(xfer);
1699         if (err)
1700                 return (err);
1701
1702         /* Pipe isn't running, start first */
1703         return (ehci_root_ctrl_start(STAILQ_FIRST(&xfer->pipe->queue)));
1704 }
1705
1706 static usbd_status
1707 ehci_root_ctrl_start(usbd_xfer_handle xfer)
1708 {
1709         ehci_softc_t *sc = (ehci_softc_t *)xfer->pipe->device->bus;
1710         usb_device_request_t *req;
1711         void *buf = NULL;
1712         int port, i;
1713         int s, len, value, index, l, totlen = 0;
1714         usb_port_status_t ps;
1715         usb_hub_descriptor_t hubd;
1716         usbd_status err;
1717         u_int32_t v;
1718
1719         if (sc->sc_dying)
1720                 return (USBD_IOERROR);
1721
1722 #ifdef DIAGNOSTIC
1723         if (!(xfer->rqflags & URQ_REQUEST))
1724                 /* XXX panic */
1725                 return (USBD_INVAL);
1726 #endif
1727         req = &xfer->request;
1728
1729         DPRINTFN(4,("ehci_root_ctrl_start: type=0x%02x request=%02x\n",
1730                     req->bmRequestType, req->bRequest));
1731
1732         len = UGETW(req->wLength);
1733         value = UGETW(req->wValue);
1734         index = UGETW(req->wIndex);
1735
1736         if (len != 0)
1737                 buf = xfer->buffer;
1738
1739 #define C(x,y) ((x) | ((y) << 8))
1740         switch(C(req->bRequest, req->bmRequestType)) {
1741         case C(UR_CLEAR_FEATURE, UT_WRITE_DEVICE):
1742         case C(UR_CLEAR_FEATURE, UT_WRITE_INTERFACE):
1743         case C(UR_CLEAR_FEATURE, UT_WRITE_ENDPOINT):
1744                 /*
1745                  * DEVICE_REMOTE_WAKEUP and ENDPOINT_HALT are no-ops
1746                  * for the integrated root hub.
1747                  */
1748                 break;
1749         case C(UR_GET_CONFIG, UT_READ_DEVICE):
1750                 if (len > 0) {
1751                         *(u_int8_t *)buf = sc->sc_conf;
1752                         totlen = 1;
1753                 }
1754                 break;
1755         case C(UR_GET_DESCRIPTOR, UT_READ_DEVICE):
1756                 DPRINTFN(8,("ehci_root_ctrl_start: wValue=0x%04x\n", value));
1757                 switch(value >> 8) {
1758                 case UDESC_DEVICE:
1759                         if ((value & 0xff) != 0) {
1760                                 err = USBD_IOERROR;
1761                                 goto ret;
1762                         }
1763                         totlen = l = min(len, USB_DEVICE_DESCRIPTOR_SIZE);
1764                         USETW(ehci_devd.idVendor, sc->sc_id_vendor);
1765                         memcpy(buf, &ehci_devd, l);
1766                         break;
1767                 /*
1768                  * We can't really operate at another speed, but the spec says
1769                  * we need this descriptor.
1770                  */
1771                 case UDESC_DEVICE_QUALIFIER:
1772                         if ((value & 0xff) != 0) {
1773                                 err = USBD_IOERROR;
1774                                 goto ret;
1775                         }
1776                         totlen = l = min(len, USB_DEVICE_DESCRIPTOR_SIZE);
1777                         memcpy(buf, &ehci_odevd, l);
1778                         break;
1779                 /*
1780                  * We can't really operate at another speed, but the spec says
1781                  * we need this descriptor.
1782                  */
1783                 case UDESC_OTHER_SPEED_CONFIGURATION:
1784                 case UDESC_CONFIG:
1785                         if ((value & 0xff) != 0) {
1786                                 err = USBD_IOERROR;
1787                                 goto ret;
1788                         }
1789                         totlen = l = min(len, USB_CONFIG_DESCRIPTOR_SIZE);
1790                         memcpy(buf, &ehci_confd, l);
1791                         ((usb_config_descriptor_t *)buf)->bDescriptorType =
1792                                 value >> 8;
1793                         buf = (char *)buf + l;
1794                         len -= l;
1795                         l = min(len, USB_INTERFACE_DESCRIPTOR_SIZE);
1796                         totlen += l;
1797                         memcpy(buf, &ehci_ifcd, l);
1798                         buf = (char *)buf + l;
1799                         len -= l;
1800                         l = min(len, USB_ENDPOINT_DESCRIPTOR_SIZE);
1801                         totlen += l;
1802                         memcpy(buf, &ehci_endpd, l);
1803                         break;
1804                 case UDESC_STRING:
1805                         if (len == 0)
1806                                 break;
1807                         *(u_int8_t *)buf = 0;
1808                         totlen = 1;
1809                         switch (value & 0xff) {
1810                         case 0: /* Language table */
1811                                 totlen = ehci_str(buf, len, "\001");
1812                                 break;
1813                         case 1: /* Vendor */
1814                                 totlen = ehci_str(buf, len, sc->sc_vendor);
1815                                 break;
1816                         case 2: /* Product */
1817                                 totlen = ehci_str(buf, len, "EHCI root hub");
1818                                 break;
1819                         }
1820                         break;
1821                 default:
1822                         err = USBD_IOERROR;
1823                         goto ret;
1824                 }
1825                 break;
1826         case C(UR_GET_INTERFACE, UT_READ_INTERFACE):
1827                 if (len > 0) {
1828                         *(u_int8_t *)buf = 0;
1829                         totlen = 1;
1830                 }
1831                 break;
1832         case C(UR_GET_STATUS, UT_READ_DEVICE):
1833                 if (len > 1) {
1834                         USETW(((usb_status_t *)buf)->wStatus,UDS_SELF_POWERED);
1835                         totlen = 2;
1836                 }
1837                 break;
1838         case C(UR_GET_STATUS, UT_READ_INTERFACE):
1839         case C(UR_GET_STATUS, UT_READ_ENDPOINT):
1840                 if (len > 1) {
1841                         USETW(((usb_status_t *)buf)->wStatus, 0);
1842                         totlen = 2;
1843                 }
1844                 break;
1845         case C(UR_SET_ADDRESS, UT_WRITE_DEVICE):
1846                 if (value >= USB_MAX_DEVICES) {
1847                         err = USBD_IOERROR;
1848                         goto ret;
1849                 }
1850                 sc->sc_addr = value;
1851                 break;
1852         case C(UR_SET_CONFIG, UT_WRITE_DEVICE):
1853                 if (value != 0 && value != 1) {
1854                         err = USBD_IOERROR;
1855                         goto ret;
1856                 }
1857                 sc->sc_conf = value;
1858                 break;
1859         case C(UR_SET_DESCRIPTOR, UT_WRITE_DEVICE):
1860                 break;
1861         case C(UR_SET_FEATURE, UT_WRITE_DEVICE):
1862         case C(UR_SET_FEATURE, UT_WRITE_INTERFACE):
1863         case C(UR_SET_FEATURE, UT_WRITE_ENDPOINT):
1864                 err = USBD_IOERROR;
1865                 goto ret;
1866         case C(UR_SET_INTERFACE, UT_WRITE_INTERFACE):
1867                 break;
1868         case C(UR_SYNCH_FRAME, UT_WRITE_ENDPOINT):
1869                 break;
1870         /* Hub requests */
1871         case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_DEVICE):
1872                 break;
1873         case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_OTHER):
1874                 DPRINTFN(8, ("ehci_root_ctrl_start: UR_CLEAR_PORT_FEATURE "
1875                              "port=%d feature=%d\n",
1876                              index, value));
1877                 if (index < 1 || index > sc->sc_noport) {
1878                         err = USBD_IOERROR;
1879                         goto ret;
1880                 }
1881                 port = EHCI_PORTSC(index);
1882                 v = EOREAD4(sc, port) &~ EHCI_PS_CLEAR;
1883                 switch(value) {
1884                 case UHF_PORT_ENABLE:
1885                         EOWRITE4(sc, port, v &~ EHCI_PS_PE);
1886                         break;
1887                 case UHF_PORT_SUSPEND:
1888                         EOWRITE4(sc, port, v &~ EHCI_PS_SUSP);
1889                         break;
1890                 case UHF_PORT_POWER:
1891                         EOWRITE4(sc, port, v &~ EHCI_PS_PP);
1892                         break;
1893                 case UHF_PORT_TEST:
1894                         DPRINTFN(2,("ehci_root_ctrl_start: clear port test "
1895                                     "%d\n", index));
1896                         break;
1897                 case UHF_PORT_INDICATOR:
1898                         DPRINTFN(2,("ehci_root_ctrl_start: clear port ind "
1899                                     "%d\n", index));
1900                         EOWRITE4(sc, port, v &~ EHCI_PS_PIC);
1901                         break;
1902                 case UHF_C_PORT_CONNECTION:
1903                         EOWRITE4(sc, port, v | EHCI_PS_CSC);
1904                         break;
1905                 case UHF_C_PORT_ENABLE:
1906                         EOWRITE4(sc, port, v | EHCI_PS_PEC);
1907                         break;
1908                 case UHF_C_PORT_SUSPEND:
1909                         /* how? */
1910                         break;
1911                 case UHF_C_PORT_OVER_CURRENT:
1912                         EOWRITE4(sc, port, v | EHCI_PS_OCC);
1913                         break;
1914                 case UHF_C_PORT_RESET:
1915                         sc->sc_isreset = 0;
1916                         break;
1917                 default:
1918                         err = USBD_IOERROR;
1919                         goto ret;
1920                 }
1921 #if 0
1922                 switch(value) {
1923                 case UHF_C_PORT_CONNECTION:
1924                 case UHF_C_PORT_ENABLE:
1925                 case UHF_C_PORT_SUSPEND:
1926                 case UHF_C_PORT_OVER_CURRENT:
1927                 case UHF_C_PORT_RESET:
1928                         /* Enable RHSC interrupt if condition is cleared. */
1929                         if ((OREAD4(sc, port) >> 16) == 0)
1930                                 ehci_pcd_able(sc, 1);
1931                         break;
1932                 default:
1933                         break;
1934                 }
1935 #endif
1936                 break;
1937         case C(UR_GET_DESCRIPTOR, UT_READ_CLASS_DEVICE):
1938                 if ((value & 0xff) != 0) {
1939                         err = USBD_IOERROR;
1940                         goto ret;
1941                 }
1942                 hubd = ehci_hubd;
1943                 hubd.bNbrPorts = sc->sc_noport;
1944                 v = EOREAD4(sc, EHCI_HCSPARAMS);
1945                 USETW(hubd.wHubCharacteristics,
1946                     EHCI_HCS_PPC(v) ? UHD_PWR_INDIVIDUAL : UHD_PWR_NO_SWITCH |
1947                     EHCI_HCS_P_INDICATOR(EREAD4(sc, EHCI_HCSPARAMS))
1948                         ? UHD_PORT_IND : 0);
1949                 hubd.bPwrOn2PwrGood = 200; /* XXX can't find out? */
1950                 for (i = 0, l = sc->sc_noport; l > 0; i++, l -= 8, v >>= 8)
1951                         hubd.DeviceRemovable[i++] = 0; /* XXX can't find out? */
1952                 hubd.bDescLength = USB_HUB_DESCRIPTOR_SIZE + i;
1953                 l = min(len, hubd.bDescLength);
1954                 totlen = l;
1955                 memcpy(buf, &hubd, l);
1956                 break;
1957         case C(UR_GET_STATUS, UT_READ_CLASS_DEVICE):
1958                 if (len != 4) {
1959                         err = USBD_IOERROR;
1960                         goto ret;
1961                 }
1962                 memset(buf, 0, len); /* ? XXX */
1963                 totlen = len;
1964                 break;
1965         case C(UR_GET_STATUS, UT_READ_CLASS_OTHER):
1966                 DPRINTFN(8,("ehci_root_ctrl_start: get port status i=%d\n",
1967                             index));
1968                 if (index < 1 || index > sc->sc_noport) {
1969                         err = USBD_IOERROR;
1970                         goto ret;
1971                 }
1972                 if (len != 4) {
1973                         err = USBD_IOERROR;
1974                         goto ret;
1975                 }
1976                 v = EOREAD4(sc, EHCI_PORTSC(index));
1977                 DPRINTFN(8,("ehci_root_ctrl_start: port status=0x%04x\n",
1978                             v));
1979                 i = UPS_HIGH_SPEED;
1980                 if (v & EHCI_PS_CS)     i |= UPS_CURRENT_CONNECT_STATUS;
1981                 if (v & EHCI_PS_PE)     i |= UPS_PORT_ENABLED;
1982                 if (v & EHCI_PS_SUSP)   i |= UPS_SUSPEND;
1983                 if (v & EHCI_PS_OCA)    i |= UPS_OVERCURRENT_INDICATOR;
1984                 if (v & EHCI_PS_PR)     i |= UPS_RESET;
1985                 if (v & EHCI_PS_PP)     i |= UPS_PORT_POWER;
1986                 USETW(ps.wPortStatus, i);
1987                 i = 0;
1988                 if (v & EHCI_PS_CSC)    i |= UPS_C_CONNECT_STATUS;
1989                 if (v & EHCI_PS_PEC)    i |= UPS_C_PORT_ENABLED;
1990                 if (v & EHCI_PS_OCC)    i |= UPS_C_OVERCURRENT_INDICATOR;
1991                 if (sc->sc_isreset)     i |= UPS_C_PORT_RESET;
1992                 USETW(ps.wPortChange, i);
1993                 l = min(len, sizeof ps);
1994                 memcpy(buf, &ps, l);
1995                 totlen = l;
1996                 break;
1997         case C(UR_SET_DESCRIPTOR, UT_WRITE_CLASS_DEVICE):
1998                 err = USBD_IOERROR;
1999                 goto ret;
2000         case C(UR_SET_FEATURE, UT_WRITE_CLASS_DEVICE):
2001                 break;
2002         case C(UR_SET_FEATURE, UT_WRITE_CLASS_OTHER):
2003                 if (index < 1 || index > sc->sc_noport) {
2004                         err = USBD_IOERROR;
2005                         goto ret;
2006                 }
2007                 port = EHCI_PORTSC(index);
2008                 v = EOREAD4(sc, port) &~ EHCI_PS_CLEAR;
2009                 switch(value) {
2010                 case UHF_PORT_ENABLE:
2011                         EOWRITE4(sc, port, v | EHCI_PS_PE);
2012                         break;
2013                 case UHF_PORT_SUSPEND:
2014                         EOWRITE4(sc, port, v | EHCI_PS_SUSP);
2015                         break;
2016                 case UHF_PORT_RESET:
2017                         DPRINTFN(5,("ehci_root_ctrl_start: reset port %d\n",
2018                                     index));
2019                         if (EHCI_PS_IS_LOWSPEED(v)) {
2020                                 /* Low speed device, give up ownership. */
2021                                 ehci_disown(sc, index, 1);
2022                                 break;
2023                         }
2024                         /* Start reset sequence. */
2025                         v &= ~ (EHCI_PS_PE | EHCI_PS_PR);
2026                         EOWRITE4(sc, port, v | EHCI_PS_PR);
2027                         /* Wait for reset to complete. */
2028                         usb_delay_ms(&sc->sc_bus, USB_PORT_ROOT_RESET_DELAY);
2029                         if (sc->sc_dying) {
2030                                 err = USBD_IOERROR;
2031                                 goto ret;
2032                         }
2033                         /* Terminate reset sequence. */
2034                         EOWRITE4(sc, port, v);
2035                         /* Wait for HC to complete reset. */
2036                         usb_delay_ms(&sc->sc_bus, EHCI_PORT_RESET_COMPLETE);
2037                         if (sc->sc_dying) {
2038                                 err = USBD_IOERROR;
2039                                 goto ret;
2040                         }
2041                         v = EOREAD4(sc, port);
2042                         DPRINTF(("ehci after reset, status=0x%08x\n", v));
2043                         if (v & EHCI_PS_PR) {
2044                                 printf("%s: port reset timeout\n",
2045                                        device_get_nameunit(sc->sc_bus.bdev));
2046                                 return (USBD_TIMEOUT);
2047                         }
2048                         if (!(v & EHCI_PS_PE)) {
2049                                 /* Not a high speed device, give up ownership.*/
2050                                 ehci_disown(sc, index, 0);
2051                                 break;
2052                         }
2053                         sc->sc_isreset = 1;
2054                         DPRINTF(("ehci port %d reset, status = 0x%08x\n",
2055                                  index, v));
2056                         break;
2057                 case UHF_PORT_POWER:
2058                         DPRINTFN(2,("ehci_root_ctrl_start: set port power "
2059                                     "%d\n", index));
2060                         EOWRITE4(sc, port, v | EHCI_PS_PP);
2061                         break;
2062                 case UHF_PORT_TEST:
2063                         DPRINTFN(2,("ehci_root_ctrl_start: set port test "
2064                                     "%d\n", index));
2065                         break;
2066                 case UHF_PORT_INDICATOR:
2067                         DPRINTFN(2,("ehci_root_ctrl_start: set port ind "
2068                                     "%d\n", index));
2069                         EOWRITE4(sc, port, v | EHCI_PS_PIC);
2070                         break;
2071                 default:
2072                         err = USBD_IOERROR;
2073                         goto ret;
2074                 }
2075                 break;
2076         case C(UR_CLEAR_TT_BUFFER, UT_WRITE_CLASS_OTHER):
2077         case C(UR_RESET_TT, UT_WRITE_CLASS_OTHER):
2078         case C(UR_GET_TT_STATE, UT_READ_CLASS_OTHER):
2079         case C(UR_STOP_TT, UT_WRITE_CLASS_OTHER):
2080                 break;
2081         default:
2082                 err = USBD_IOERROR;
2083                 goto ret;
2084         }
2085         xfer->actlen = totlen;
2086         err = USBD_NORMAL_COMPLETION;
2087  ret:
2088         xfer->status = err;
2089         s = splusb();
2090         hacksync(xfer); /* XXX to compensate for usb_transfer_complete */
2091         usb_transfer_complete(xfer);
2092         splx(s);
2093         return (USBD_IN_PROGRESS);
2094 }
2095
2096 void
2097 ehci_disown(ehci_softc_t *sc, int index, int lowspeed)
2098 {
2099         int port;
2100         u_int32_t v;
2101
2102         DPRINTF(("ehci_disown: index=%d lowspeed=%d\n", index, lowspeed));
2103 #ifdef DIAGNOSTIC
2104         if (sc->sc_npcomp != 0) {
2105                 int i = (index-1) / sc->sc_npcomp;
2106                 if (i >= sc->sc_ncomp)
2107                         printf("%s: strange port\n",
2108                                device_get_nameunit(sc->sc_bus.bdev));
2109                 else
2110                         printf("%s: handing over %s speed device on "
2111                                "port %d to %s\n",
2112                                device_get_nameunit(sc->sc_bus.bdev),
2113                                lowspeed ? "low" : "full",
2114                                index, device_get_nameunit(sc->sc_comps[i]->bdev));
2115         } else {
2116                 printf("%s: npcomp == 0\n", device_get_nameunit(sc->sc_bus.bdev));
2117         }
2118 #endif
2119         port = EHCI_PORTSC(index);
2120         v = EOREAD4(sc, port) &~ EHCI_PS_CLEAR;
2121         EOWRITE4(sc, port, v | EHCI_PS_PO);
2122 }
2123
2124 /* Abort a root control request. */
2125 static void
2126 ehci_root_ctrl_abort(usbd_xfer_handle xfer)
2127 {
2128         /* Nothing to do, all transfers are synchronous. */
2129 }
2130
2131 /* Close the root pipe. */
2132 static void
2133 ehci_root_ctrl_close(usbd_pipe_handle pipe)
2134 {
2135         DPRINTF(("ehci_root_ctrl_close\n"));
2136         /* Nothing to do. */
2137 }
2138
2139 void
2140 ehci_root_intr_done(usbd_xfer_handle xfer)
2141 {
2142 }
2143
2144 static usbd_status
2145 ehci_root_intr_transfer(usbd_xfer_handle xfer)
2146 {
2147         usbd_status err;
2148
2149         /* Insert last in queue. */
2150         err = usb_insert_transfer(xfer);
2151         if (err)
2152                 return (err);
2153
2154         /* Pipe isn't running, start first */
2155         return (ehci_root_intr_start(STAILQ_FIRST(&xfer->pipe->queue)));
2156 }
2157
2158 static usbd_status
2159 ehci_root_intr_start(usbd_xfer_handle xfer)
2160 {
2161         usbd_pipe_handle pipe = xfer->pipe;
2162         ehci_softc_t *sc = (ehci_softc_t *)pipe->device->bus;
2163
2164         if (sc->sc_dying)
2165                 return (USBD_IOERROR);
2166
2167         sc->sc_intrxfer = xfer;
2168
2169         return (USBD_IN_PROGRESS);
2170 }
2171
2172 /* Abort a root interrupt request. */
2173 static void
2174 ehci_root_intr_abort(usbd_xfer_handle xfer)
2175 {
2176         int s;
2177
2178         if (xfer->pipe->intrxfer == xfer) {
2179                 DPRINTF(("ehci_root_intr_abort: remove\n"));
2180                 xfer->pipe->intrxfer = NULL;
2181         }
2182         xfer->status = USBD_CANCELLED;
2183         s = splusb();
2184         usb_transfer_complete(xfer);
2185         splx(s);
2186 }
2187
2188 /* Close the root pipe. */
2189 static void
2190 ehci_root_intr_close(usbd_pipe_handle pipe)
2191 {
2192         ehci_softc_t *sc = (ehci_softc_t *)pipe->device->bus;
2193
2194         DPRINTF(("ehci_root_intr_close\n"));
2195
2196         sc->sc_intrxfer = NULL;
2197 }
2198
2199 void
2200 ehci_root_ctrl_done(usbd_xfer_handle xfer)
2201 {
2202 }
2203
2204 /************************/
2205
2206 ehci_soft_qh_t *
2207 ehci_alloc_sqh(ehci_softc_t *sc)
2208 {
2209         ehci_soft_qh_t *sqh;
2210         ehci_soft_qtd_t *sqtd;
2211         usbd_status err;
2212         int i, offs;
2213         usb_dma_t dma;
2214
2215         if (sc->sc_freeqhs == NULL) {
2216                 DPRINTFN(2, ("ehci_alloc_sqh: allocating chunk\n"));
2217                 err = usb_allocmem(&sc->sc_bus, EHCI_SQH_SIZE * EHCI_SQH_CHUNK,
2218                           EHCI_PAGE_SIZE, &dma);
2219 #ifdef EHCI_DEBUG
2220                 if (err)
2221                         printf("ehci_alloc_sqh: usb_allocmem()=%d\n", err);
2222 #endif
2223                 if (err)
2224                         return (NULL);
2225                 for(i = 0; i < EHCI_SQH_CHUNK; i++) {
2226                         offs = i * EHCI_SQH_SIZE;
2227                         sqh = KERNADDR(&dma, offs);
2228                         sqh->physaddr = DMAADDR(&dma, offs);
2229                         sqh->next = sc->sc_freeqhs;
2230                         sc->sc_freeqhs = sqh;
2231                 }
2232         }
2233         /* Allocate the initial inactive sqtd. */
2234         sqtd = ehci_alloc_sqtd(sc);
2235         if (sqtd == NULL)
2236                 return (NULL);
2237         sqtd->qtd.qtd_status = htole32(0);
2238         sqtd->qtd.qtd_next = EHCI_NULL;
2239         sqtd->qtd.qtd_altnext = EHCI_NULL;
2240
2241         sqh = sc->sc_freeqhs;
2242         sc->sc_freeqhs = sqh->next;
2243
2244         /* The overlay QTD should begin zeroed. */
2245         sqh->qh.qh_qtd.qtd_next = htole32(sqtd->physaddr);
2246         sqh->qh.qh_qtd.qtd_altnext = EHCI_NULL;
2247         sqh->qh.qh_qtd.qtd_status = 0;
2248         for (i = 0; i < EHCI_QTD_NBUFFERS; i++) {
2249                 sqh->qh.qh_qtd.qtd_buffer[i] = 0;
2250                 sqh->qh.qh_qtd.qtd_buffer_hi[i] = 0;
2251         }
2252         sqh->next = NULL;
2253         sqh->prev = NULL;
2254         sqh->sqtd = sqtd;
2255         sqh->inactivesqtd = sqtd;
2256         return (sqh);
2257 }
2258
2259 void
2260 ehci_free_sqh(ehci_softc_t *sc, ehci_soft_qh_t *sqh)
2261 {
2262         ehci_free_sqtd(sc, sqh->inactivesqtd);
2263         sqh->next = sc->sc_freeqhs;
2264         sc->sc_freeqhs = sqh;
2265 }
2266
2267 ehci_soft_qtd_t *
2268 ehci_alloc_sqtd(ehci_softc_t *sc)
2269 {
2270         ehci_soft_qtd_t *sqtd;
2271         usbd_status err;
2272         int i, offs;
2273         usb_dma_t dma;
2274         int s;
2275
2276         if (sc->sc_freeqtds == NULL) {
2277                 DPRINTFN(2, ("ehci_alloc_sqtd: allocating chunk\n"));
2278                 err = usb_allocmem(&sc->sc_bus, EHCI_SQTD_SIZE*EHCI_SQTD_CHUNK,
2279                           EHCI_PAGE_SIZE, &dma);
2280 #ifdef EHCI_DEBUG
2281                 if (err)
2282                         printf("ehci_alloc_sqtd: usb_allocmem()=%d\n", err);
2283 #endif
2284                 if (err)
2285                         return (NULL);
2286                 s = splusb();
2287                 for(i = 0; i < EHCI_SQTD_CHUNK; i++) {
2288                         offs = i * EHCI_SQTD_SIZE;
2289                         sqtd = KERNADDR(&dma, offs);
2290                         sqtd->physaddr = DMAADDR(&dma, offs);
2291                         sqtd->nextqtd = sc->sc_freeqtds;
2292                         sc->sc_freeqtds = sqtd;
2293                 }
2294                 splx(s);
2295         }
2296
2297         s = splusb();
2298         sqtd = sc->sc_freeqtds;
2299         sc->sc_freeqtds = sqtd->nextqtd;
2300         sqtd->qtd.qtd_next = EHCI_NULL;
2301         sqtd->qtd.qtd_altnext = EHCI_NULL;
2302         sqtd->qtd.qtd_status = 0;
2303         for (i = 0; i < EHCI_QTD_NBUFFERS; i++) {
2304                 sqtd->qtd.qtd_buffer[i] = 0;
2305                 sqtd->qtd.qtd_buffer_hi[i] = 0;
2306         }
2307         sqtd->nextqtd = NULL;
2308         sqtd->xfer = NULL;
2309         splx(s);
2310
2311         return (sqtd);
2312 }
2313
2314 void
2315 ehci_free_sqtd(ehci_softc_t *sc, ehci_soft_qtd_t *sqtd)
2316 {
2317         int s;
2318
2319         s = splusb();
2320         sqtd->nextqtd = sc->sc_freeqtds;
2321         sc->sc_freeqtds = sqtd;
2322         splx(s);
2323 }
2324
2325 usbd_status
2326 ehci_alloc_sqtd_chain(struct ehci_pipe *epipe, ehci_softc_t *sc,
2327      int alen, int rd, usbd_xfer_handle xfer, ehci_soft_qtd_t *start,
2328      ehci_soft_qtd_t *newinactive, ehci_soft_qtd_t **sp, ehci_soft_qtd_t **ep)
2329 {
2330         ehci_soft_qtd_t *next, *cur;
2331         ehci_physaddr_t dataphys, nextphys;
2332         u_int32_t qtdstatus;
2333         int adj, len, curlen, mps, offset, pagelen, seg, segoff;
2334         int i, iscontrol, forceshort;
2335         struct usb_dma_mapping *dma = &xfer->dmamap;
2336
2337         DPRINTFN(alen<4*4096,("ehci_alloc_sqtd_chain: start len=%d\n", alen));
2338
2339         offset = 0;
2340         len = alen;
2341         iscontrol = (epipe->pipe.endpoint->edesc->bmAttributes & UE_XFERTYPE) ==
2342             UE_CONTROL;
2343         qtdstatus = EHCI_QTD_ACTIVE |
2344             EHCI_QTD_SET_PID(rd ? EHCI_QTD_PID_IN : EHCI_QTD_PID_OUT) |
2345             EHCI_QTD_SET_CERR(3)
2346             /* IOC set below */
2347             /* BYTES set below */
2348             ;
2349         mps = UGETW(epipe->pipe.endpoint->edesc->wMaxPacketSize);
2350         forceshort = ((xfer->flags & USBD_FORCE_SHORT_XFER) || len == 0) &&
2351             len % mps == 0;
2352         /*
2353          * The control transfer data stage always starts with a toggle of 1.
2354          * For other transfers we let the hardware track the toggle state.
2355          */
2356         if (iscontrol)
2357                 qtdstatus |= EHCI_QTD_SET_TOGGLE(1);
2358
2359         if (start != NULL) {
2360                 /*
2361                  * If we are given a starting qTD, assume it is linked into
2362                  * an active QH so be careful not to mark it active.
2363                  */
2364                 cur = start;
2365                 *sp = cur;
2366                 qtdstatus &= ~EHCI_QTD_ACTIVE;
2367         } else {
2368                 cur = ehci_alloc_sqtd(sc);
2369                 *sp = cur;
2370                 if (cur == NULL)
2371                         goto nomem;
2372         }
2373         seg = 0;
2374         segoff = 0;
2375         for (;;) {
2376                 curlen = 0;
2377
2378                 /* The EHCI hardware can handle at most 5 pages. */
2379                 for (i = 0; i < EHCI_QTD_NBUFFERS && curlen < len; i++) {
2380                         KASSERT(seg < dma->nsegs,
2381                             ("ehci_alloc_sqtd_chain: overrun"));
2382                         dataphys = dma->segs[seg].ds_addr + segoff;
2383                         pagelen = dma->segs[seg].ds_len - segoff;
2384                         if (pagelen > len - curlen)
2385                                 pagelen = len - curlen;
2386                         if (pagelen > EHCI_PAGE_SIZE -
2387                             EHCI_PAGE_OFFSET(dataphys))
2388                                 pagelen = EHCI_PAGE_SIZE -
2389                                     EHCI_PAGE_OFFSET(dataphys);
2390                         segoff += pagelen;
2391                         if (segoff >= dma->segs[seg].ds_len) {
2392                                 KASSERT(segoff == dma->segs[seg].ds_len,
2393                                     ("ehci_alloc_sqtd_chain: overlap"));
2394                                 seg++;
2395                                 segoff = 0;
2396                         }
2397
2398                         cur->qtd.qtd_buffer[i] = htole32(dataphys);
2399                         cur->qtd.qtd_buffer_hi[i] = 0;
2400                         curlen += pagelen;
2401
2402                         /*
2403                          * Must stop if there is any gap before or after
2404                          * the page boundary.
2405                          */
2406                         if (EHCI_PAGE_OFFSET(dataphys + pagelen) != 0)
2407                                 break;
2408                         if (seg < dma->nsegs && EHCI_PAGE_OFFSET(segoff +
2409                             dma->segs[seg].ds_addr) != 0)
2410                                 break;
2411                 }
2412                 /* Adjust down to a multiple of mps if not at the end. */
2413                 if (curlen < len && curlen % mps != 0) {
2414                         adj = curlen % mps;
2415                         curlen -= adj;
2416                         KASSERT(curlen > 0,
2417                             ("ehci_alloc_sqtd_chain: need to copy"));
2418                         segoff -= adj;
2419                         if (segoff < 0) {
2420                                 seg--;
2421                                 segoff += dma->segs[seg].ds_len;
2422                         }
2423                         KASSERT(seg >= 0 && segoff >= 0,
2424                             ("ehci_alloc_sqtd_chain: adjust to mps"));
2425                 }
2426
2427                 len -= curlen;
2428
2429                 if (len != 0 || forceshort) {
2430                         next = ehci_alloc_sqtd(sc);
2431                         if (next == NULL)
2432                                 goto nomem;
2433                         nextphys = htole32(next->physaddr);
2434                 } else {
2435                         next = NULL;
2436                         nextphys = EHCI_NULL;
2437                 }
2438
2439                 cur->nextqtd = next;
2440                 cur->qtd.qtd_next = nextphys;
2441                 /* Make sure to stop after a short transfer. */
2442                 cur->qtd.qtd_altnext = htole32(newinactive->physaddr);
2443                 cur->qtd.qtd_status =
2444                     htole32(qtdstatus | EHCI_QTD_SET_BYTES(curlen));
2445                 cur->xfer = xfer;
2446                 cur->len = curlen;
2447                 DPRINTFN(10,("ehci_alloc_sqtd_chain: curlen=%d\n", curlen));
2448                 if (iscontrol) {
2449                         /*
2450                          * adjust the toggle based on the number of packets
2451                          * in this qtd
2452                          */
2453                         if ((((curlen + mps - 1) / mps) & 1) || curlen == 0)
2454                                 qtdstatus ^= EHCI_QTD_TOGGLE_MASK;
2455                 }
2456                 qtdstatus |= EHCI_QTD_ACTIVE;
2457                 if (len == 0) {
2458                         if (!forceshort)
2459                                 break;
2460                         forceshort = 0;
2461                 }
2462                 DPRINTFN(10,("ehci_alloc_sqtd_chain: extend chain\n"));
2463                 offset += curlen;
2464                 cur = next;
2465         }
2466         cur->qtd.qtd_status |= htole32(EHCI_QTD_IOC);
2467         *ep = cur;
2468
2469         DPRINTFN(10,("ehci_alloc_sqtd_chain: return sqtd=%p sqtdend=%p\n",
2470                      *sp, *ep));
2471
2472         return (USBD_NORMAL_COMPLETION);
2473
2474  nomem:
2475         /* XXX free chain */
2476         DPRINTFN(-1,("ehci_alloc_sqtd_chain: no memory\n"));
2477         return (USBD_NOMEM);
2478 }
2479
2480 /* Free the chain starting at sqtd and end at the qTD before sqtdend */
2481 static void
2482 ehci_free_sqtd_chain(ehci_softc_t *sc, ehci_soft_qh_t *sqh,
2483     ehci_soft_qtd_t *sqtd, ehci_soft_qtd_t *sqtdend)
2484 {
2485         ehci_soft_qtd_t *p, **prevp;
2486         int i;
2487
2488         DPRINTFN(10,("ehci_free_sqtd_chain: sqtd=%p sqtdend=%p\n",
2489                      sqtd, sqtdend));
2490
2491         /* First unlink the chain from the QH's software qTD list. */
2492         prevp = &sqh->sqtd;
2493         for (p = sqh->sqtd; p != NULL; p = p->nextqtd) {
2494                 if (p == sqtd) {
2495                         *prevp = sqtdend;
2496                         break;
2497                 }
2498                 prevp = &p->nextqtd;
2499         }
2500         KASSERT(p != NULL, ("ehci_free_sqtd_chain: chain not found"));
2501         for (i = 0; sqtd != sqtdend; sqtd = p, i++) {
2502                 p = sqtd->nextqtd;
2503                 ehci_free_sqtd(sc, sqtd);
2504         }
2505 }
2506
2507 /****************/
2508
2509 /*
2510  * Close a reqular pipe.
2511  * Assumes that there are no pending transactions.
2512  */
2513 void
2514 ehci_close_pipe(usbd_pipe_handle pipe, ehci_soft_qh_t *head)
2515 {
2516         struct ehci_pipe *epipe = (struct ehci_pipe *)pipe;
2517         ehci_softc_t *sc = (ehci_softc_t *)pipe->device->bus;
2518         ehci_soft_qh_t *sqh = epipe->sqh;
2519         int s;
2520
2521         s = splusb();
2522         ehci_rem_qh(sc, sqh, head);
2523         splx(s);
2524         pipe->endpoint->savedtoggle =
2525             EHCI_QTD_GET_TOGGLE(le32toh(sqh->qh.qh_qtd.qtd_status));
2526         ehci_free_sqh(sc, epipe->sqh);
2527 }
2528
2529 /*
2530  * Abort a device request.
2531  * If this routine is called at splusb() it guarantees that the request
2532  * will be removed from the hardware scheduling and that the callback
2533  * for it will be called with USBD_CANCELLED status.
2534  * It's impossible to guarantee that the requested transfer will not
2535  * have happened since the hardware runs concurrently.
2536  * If the transaction has already happened we rely on the ordinary
2537  * interrupt processing to process it.
2538  */
2539 void
2540 ehci_abort_xfer(usbd_xfer_handle xfer, usbd_status status)
2541 {
2542 #define exfer EXFER(xfer)
2543         struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
2544         ehci_softc_t *sc = (ehci_softc_t *)epipe->pipe.device->bus;
2545         ehci_soft_qh_t *sqh = epipe->sqh;
2546         ehci_soft_qtd_t *sqtd, *snext;
2547         ehci_physaddr_t cur, us, next;
2548         int s;
2549         int hit, i;
2550         /* int count = 0; */
2551         ehci_soft_qh_t *psqh;
2552
2553         DPRINTF(("ehci_abort_xfer: xfer=%p pipe=%p\n", xfer, epipe));
2554
2555         if (sc->sc_dying) {
2556                 /* If we're dying, just do the software part. */
2557                 s = splusb();
2558                 xfer->status = status;  /* make software ignore it */
2559                 callout_stop(&xfer->timeout_handle);
2560                 usb_rem_task(epipe->pipe.device, &exfer->abort_task);
2561                 usb_transfer_complete(xfer);
2562                 splx(s);
2563                 return;
2564         }
2565
2566         if (xfer->device->bus->intr_context || !curproc)
2567                 panic("ehci_abort_xfer: not in process context");
2568
2569         /*
2570          * If an abort is already in progress then just wait for it to
2571          * complete and return.
2572          */
2573         if (exfer->ehci_xfer_flags & EHCI_XFER_ABORTING) {
2574                 DPRINTFN(2, ("ehci_abort_xfer: already aborting\n"));
2575                 /* No need to wait if we're aborting from a timeout. */
2576                 if (status == USBD_TIMEOUT)
2577                         return;
2578                 /* Override the status which might be USBD_TIMEOUT. */
2579                 xfer->status = status;
2580                 DPRINTFN(2, ("ehci_abort_xfer: waiting for abort to finish\n"));
2581                 exfer->ehci_xfer_flags |= EHCI_XFER_ABORTWAIT;
2582                 while (exfer->ehci_xfer_flags & EHCI_XFER_ABORTING)
2583                         tsleep(&exfer->ehci_xfer_flags, PZERO, "ehciaw", 0);
2584                 return;
2585         }
2586
2587         /*
2588          * Step 1: Make interrupt routine and timeouts ignore xfer.
2589          */
2590         s = splusb();
2591         exfer->ehci_xfer_flags |= EHCI_XFER_ABORTING;
2592         xfer->status = status;  /* make software ignore it */
2593         callout_stop(&xfer->timeout_handle);
2594         usb_rem_task(epipe->pipe.device, &exfer->abort_task);
2595         splx(s);
2596
2597         /*
2598          * Step 2: Wait until we know hardware has finished any possible
2599          * use of the xfer. We do this by removing the entire
2600          * queue from the async schedule and waiting for the doorbell.
2601          * Nothing else should be touching the queue now.
2602          */
2603         psqh = sqh->prev;
2604         ehci_rem_qh(sc, sqh, psqh);
2605
2606         /*
2607          * Step 3:  make sure the soft interrupt routine
2608          * has run. This should remove any completed items off the queue.
2609          * The hardware has no reference to completed items (TDs).
2610          * It's safe to remove them at any time.
2611          */
2612         s = splusb();
2613 #ifdef USB_USE_SOFTINTR
2614         sc->sc_softwake = 1;
2615 #endif /* USB_USE_SOFTINTR */
2616         usb_schedsoftintr(&sc->sc_bus);
2617 #ifdef USB_USE_SOFTINTR
2618         tsleep(&sc->sc_softwake, PZERO, "ehciab", 0);
2619 #endif /* USB_USE_SOFTINTR */
2620
2621         /*
2622          * Step 4: Remove any vestiges of the xfer from the hardware.
2623          * The complication here is that the hardware may have executed
2624          * into or even beyond the xfer we're trying to abort.
2625          * So as we're scanning the TDs of this xfer we check if
2626          * the hardware points to any of them.
2627          *
2628          * first we need to see if there are any transfers
2629          * on this queue before the xfer we are aborting.. we need
2630          * to update any pointers that point to us to point past
2631          * the aborting xfer.  (If there is something past us).
2632          * Hardware and software.
2633          */
2634         cur = EHCI_LINK_ADDR(le32toh(sqh->qh.qh_curqtd));
2635         hit = 0;
2636
2637         /* If they initially point here. */
2638         us = exfer->sqtdstart->physaddr;
2639
2640         /* We will change them to point here */
2641         snext = exfer->sqtdend->nextqtd;
2642         next = htole32(snext->physaddr);
2643
2644         /*
2645          * Now loop through any qTDs before us and keep track of the pointer
2646          * that points to us for the end.
2647          */
2648         sqtd = sqh->sqtd;
2649         while (sqtd && sqtd != exfer->sqtdstart) {
2650                 hit |= (cur == sqtd->physaddr);
2651                 if (EHCI_LINK_ADDR(le32toh(sqtd->qtd.qtd_next)) == us)
2652                         sqtd->qtd.qtd_next = next;
2653                 if (EHCI_LINK_ADDR(le32toh(sqtd->qtd.qtd_altnext)) == us)
2654                         sqtd->qtd.qtd_altnext = next;
2655                 sqtd = sqtd->nextqtd;
2656         }
2657
2658         /*
2659          * If we already saw the active one then we are pretty much done.
2660          * We've done all the relinking we need to do.
2661          */
2662         if (!hit) {
2663
2664                 /*
2665                  * Now reinitialise the QH to point to the next qTD
2666                  * (if there is one). We only need to do this if
2667                  * it was previously pointing to us.
2668                  */
2669                 for (sqtd = exfer->sqtdstart; ; sqtd = sqtd->nextqtd) {
2670                         if (cur == sqtd->physaddr) {
2671                                 hit++;
2672                         }
2673                         if (sqtd == exfer->sqtdend)
2674                                 break;
2675                 }
2676                 sqtd = sqtd->nextqtd;
2677                 /*
2678                  * Only need to alter the QH if it was pointing at a qTD
2679                  * that we are removing.
2680                  */
2681                 if (hit) {
2682                         sqh->qh.qh_qtd.qtd_next = htole32(snext->physaddr);
2683                         sqh->qh.qh_qtd.qtd_altnext = EHCI_NULL;
2684                         sqh->qh.qh_qtd.qtd_status &=
2685                             htole32(EHCI_QTD_TOGGLE_MASK);
2686                         for (i = 0; i < EHCI_QTD_NBUFFERS; i++) {
2687                                 sqh->qh.qh_qtd.qtd_buffer[i] = 0;
2688                                 sqh->qh.qh_qtd.qtd_buffer_hi[i] = 0;
2689                         }
2690                 }
2691         }
2692         ehci_add_qh(sqh, psqh);
2693         /*
2694          * Step 5: Execute callback.
2695          */
2696 #ifdef DIAGNOSTIC
2697         exfer->isdone = 1;
2698 #endif
2699         /* Do the wakeup first to avoid touching the xfer after the callback. */
2700         exfer->ehci_xfer_flags &= ~EHCI_XFER_ABORTING;
2701         if (exfer->ehci_xfer_flags & EHCI_XFER_ABORTWAIT) {
2702                 exfer->ehci_xfer_flags &= ~EHCI_XFER_ABORTWAIT;
2703                 wakeup(&exfer->ehci_xfer_flags);
2704         }
2705         usb_transfer_complete(xfer);
2706
2707         /* printf("%s: %d TDs aborted\n", __func__, count); */
2708         splx(s);
2709 #undef exfer
2710 }
2711
2712 void
2713 ehci_timeout(void *addr)
2714 {
2715         struct ehci_xfer *exfer = addr;
2716         struct ehci_pipe *epipe = (struct ehci_pipe *)exfer->xfer.pipe;
2717         ehci_softc_t *sc = (ehci_softc_t *)epipe->pipe.device->bus;
2718
2719         DPRINTF(("ehci_timeout: exfer=%p\n", exfer));
2720 #ifdef USB_DEBUG
2721         if (ehcidebug > 1)
2722                 usbd_dump_pipe(exfer->xfer.pipe);
2723 #endif
2724
2725         if (sc->sc_dying) {
2726                 ehci_abort_xfer(&exfer->xfer, USBD_TIMEOUT);
2727                 return;
2728         }
2729
2730         /* Execute the abort in a process context. */
2731         usb_add_task(exfer->xfer.pipe->device, &exfer->abort_task,
2732             USB_TASKQ_HC);
2733 }
2734
2735 void
2736 ehci_timeout_task(void *addr)
2737 {
2738         usbd_xfer_handle xfer = addr;
2739         int s;
2740
2741         DPRINTF(("ehci_timeout_task: xfer=%p\n", xfer));
2742
2743         s = splusb();
2744         ehci_abort_xfer(xfer, USBD_TIMEOUT);
2745         splx(s);
2746 }
2747
2748 /*
2749  * Some EHCI chips from VIA / ATI seem to trigger interrupts before writing
2750  * back the qTD status, or miss signalling occasionally under heavy load.
2751  * If the host machine is too fast, we can miss transaction completion - when
2752  * we scan the active list the transaction still seems to be active. This
2753  * generally exhibits itself as a umass stall that never recovers.
2754  *
2755  * We work around this behaviour by setting up this callback after any softintr
2756  * that completes with transactions still pending, giving us another chance to
2757  * check for completion after the writeback has taken place.
2758  */
2759 void
2760 ehci_intrlist_timeout(void *arg)
2761 {
2762         ehci_softc_t *sc = arg;
2763         int s = splusb();
2764
2765         DPRINTFN(3, ("ehci_intrlist_timeout\n"));
2766         usb_schedsoftintr(&sc->sc_bus);
2767
2768         splx(s);
2769 }
2770
2771 /************************/
2772
2773 static usbd_status
2774 ehci_device_ctrl_transfer(usbd_xfer_handle xfer)
2775 {
2776         usbd_status err;
2777
2778         /* Insert last in queue. */
2779         err = usb_insert_transfer(xfer);
2780         if (err)
2781                 return (err);
2782
2783         /* Pipe isn't running, start first */
2784         return (ehci_device_ctrl_start(STAILQ_FIRST(&xfer->pipe->queue)));
2785 }
2786
2787 static usbd_status
2788 ehci_device_ctrl_start(usbd_xfer_handle xfer)
2789 {
2790         ehci_softc_t *sc = (ehci_softc_t *)xfer->pipe->device->bus;
2791         usbd_status err;
2792
2793         if (sc->sc_dying)
2794                 return (USBD_IOERROR);
2795
2796 #ifdef DIAGNOSTIC
2797         if (!(xfer->rqflags & URQ_REQUEST)) {
2798                 /* XXX panic */
2799                 printf("ehci_device_ctrl_transfer: not a request\n");
2800                 return (USBD_INVAL);
2801         }
2802 #endif
2803
2804         err = ehci_device_request(xfer);
2805         if (err)
2806                 return (err);
2807
2808         if (sc->sc_bus.use_polling)
2809                 ehci_waitintr(sc, xfer);
2810         return (USBD_IN_PROGRESS);
2811 }
2812
2813 void
2814 ehci_device_ctrl_done(usbd_xfer_handle xfer)
2815 {
2816         struct ehci_xfer *ex = EXFER(xfer);
2817         ehci_softc_t *sc = (ehci_softc_t *)xfer->pipe->device->bus;
2818         struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
2819
2820         DPRINTFN(10,("ehci_ctrl_done: xfer=%p\n", xfer));
2821
2822 #ifdef DIAGNOSTIC
2823         if (!(xfer->rqflags & URQ_REQUEST)) {
2824                 panic("ehci_ctrl_done: not a request");
2825         }
2826 #endif
2827
2828         if (xfer->status != USBD_NOMEM && ehci_active_intr_list(ex)) {
2829                 ehci_del_intr_list(ex); /* remove from active list */
2830                 ehci_free_sqtd_chain(sc, epipe->sqh, ex->sqtdstart,
2831                     ex->sqtdend->nextqtd);
2832         }
2833
2834         DPRINTFN(5, ("ehci_ctrl_done: length=%d\n", xfer->actlen));
2835 }
2836
2837 /* Abort a device control request. */
2838 static void
2839 ehci_device_ctrl_abort(usbd_xfer_handle xfer)
2840 {
2841         DPRINTF(("ehci_device_ctrl_abort: xfer=%p\n", xfer));
2842         ehci_abort_xfer(xfer, USBD_CANCELLED);
2843 }
2844
2845 /* Close a device control pipe. */
2846 static void
2847 ehci_device_ctrl_close(usbd_pipe_handle pipe)
2848 {
2849         ehci_softc_t *sc = (ehci_softc_t *)pipe->device->bus;
2850         /*struct ehci_pipe *epipe = (struct ehci_pipe *)pipe;*/
2851
2852         DPRINTF(("ehci_device_ctrl_close: pipe=%p\n", pipe));
2853         ehci_close_pipe(pipe, sc->sc_async_head);
2854 }
2855
2856 usbd_status
2857 ehci_device_request(usbd_xfer_handle xfer)
2858 {
2859 #define exfer EXFER(xfer)
2860         struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
2861         usb_device_request_t *req = &xfer->request;
2862         usbd_device_handle dev = epipe->pipe.device;
2863         ehci_softc_t *sc = (ehci_softc_t *)dev->bus;
2864         ehci_soft_qtd_t *newinactive, *setup, *stat, *next;
2865         ehci_soft_qh_t *sqh;
2866         int isread;
2867         int len;
2868         usbd_status err;
2869         int s;
2870
2871         isread = req->bmRequestType & UT_READ;
2872         len = UGETW(req->wLength);
2873
2874         DPRINTFN(3,("ehci_device_request: type=0x%02x, request=0x%02x, "
2875                     "wValue=0x%04x, wIndex=0x%04x len=%d, addr=%d, endpt=%d\n",
2876                     req->bmRequestType, req->bRequest, UGETW(req->wValue),
2877                     UGETW(req->wIndex), len, dev->address,
2878                     epipe->pipe.endpoint->edesc->bEndpointAddress));
2879
2880         newinactive = ehci_alloc_sqtd(sc);
2881         if (newinactive == NULL) {
2882                 err = USBD_NOMEM;
2883                 goto bad1;
2884         }
2885         newinactive->qtd.qtd_status = htole32(0);
2886         newinactive->qtd.qtd_next = EHCI_NULL;
2887         newinactive->qtd.qtd_altnext = EHCI_NULL;
2888         stat = ehci_alloc_sqtd(sc);
2889         if (stat == NULL) {
2890                 err = USBD_NOMEM;
2891                 goto bad2;
2892         }
2893
2894         sqh = epipe->sqh;
2895         setup = sqh->inactivesqtd;
2896         sqh->inactivesqtd = newinactive;
2897         epipe->u.ctl.length = len;
2898
2899         /* Set up data transaction */
2900         if (len != 0) {
2901                 ehci_soft_qtd_t *end;
2902
2903                 err = ehci_alloc_sqtd_chain(epipe, sc, len, isread, xfer,
2904                     NULL, newinactive, &next, &end);
2905                 if (err)
2906                         goto bad3;
2907                 end->qtd.qtd_status &= htole32(~EHCI_QTD_IOC);
2908                 end->nextqtd = stat;
2909                 end->qtd.qtd_next = htole32(stat->physaddr);
2910                 end->qtd.qtd_altnext = htole32(newinactive->physaddr);
2911         } else {
2912                 next = stat;
2913         }
2914
2915         memcpy(KERNADDR(&epipe->u.ctl.reqdma, 0), req, sizeof *req);
2916
2917         /* Clear toggle, and do not activate until complete */
2918         setup->qtd.qtd_status = htole32(
2919             EHCI_QTD_SET_PID(EHCI_QTD_PID_SETUP) |
2920             EHCI_QTD_SET_CERR(3) |
2921             EHCI_QTD_SET_TOGGLE(0) |
2922             EHCI_QTD_SET_BYTES(sizeof *req)
2923             );
2924         setup->qtd.qtd_buffer[0] = htole32(DMAADDR(&epipe->u.ctl.reqdma, 0));
2925         setup->qtd.qtd_buffer_hi[0] = 0;
2926         setup->nextqtd = next;
2927         setup->qtd.qtd_next = htole32(next->physaddr);
2928         setup->qtd.qtd_altnext = htole32(newinactive->physaddr);
2929         setup->xfer = xfer;
2930         setup->len = sizeof *req;
2931
2932         stat->qtd.qtd_status = htole32(
2933             EHCI_QTD_ACTIVE |
2934             EHCI_QTD_SET_PID(isread ? EHCI_QTD_PID_OUT : EHCI_QTD_PID_IN) |
2935             EHCI_QTD_SET_CERR(3) |
2936             EHCI_QTD_SET_TOGGLE(1) |
2937             EHCI_QTD_IOC
2938             );
2939         stat->qtd.qtd_buffer[0] = 0; /* XXX not needed? */
2940         stat->qtd.qtd_buffer_hi[0] = 0; /* XXX not needed? */
2941         stat->nextqtd = newinactive;
2942         stat->qtd.qtd_next = htole32(newinactive->physaddr);
2943         stat->qtd.qtd_altnext = htole32(newinactive->physaddr);
2944         stat->xfer = xfer;
2945         stat->len = 0;
2946
2947 #ifdef EHCI_DEBUG
2948         if (ehcidebug > 5) {
2949                 DPRINTF(("ehci_device_request:\n"));
2950                 ehci_dump_sqh(sqh);
2951                 ehci_dump_sqtds(setup);
2952         }
2953 #endif
2954
2955         exfer->sqtdstart = setup;
2956         exfer->sqtdend = stat;
2957 #ifdef DIAGNOSTIC
2958         if (!exfer->isdone) {
2959                 printf("ehci_device_request: not done, exfer=%p\n", exfer);
2960         }
2961         exfer->isdone = 0;
2962 #endif
2963
2964         /* Activate the new qTD in the QH list. */
2965         s = splusb();
2966         ehci_activate_qh(sqh, setup);
2967         if (xfer->timeout && !sc->sc_bus.use_polling) {
2968                 callout_reset(&xfer->timeout_handle, MS_TO_TICKS(xfer->timeout),
2969                     ehci_timeout, xfer);
2970         }
2971         ehci_add_intr_list(sc, exfer);
2972         xfer->status = USBD_IN_PROGRESS;
2973         splx(s);
2974
2975 #ifdef EHCI_DEBUG
2976         if (ehcidebug > 10) {
2977                 DPRINTF(("ehci_device_request: status=%x\n",
2978                          EOREAD4(sc, EHCI_USBSTS)));
2979                 delay(10000);
2980                 ehci_dump_regs(sc);
2981                 ehci_dump_sqh(sc->sc_async_head);
2982                 ehci_dump_sqh(sqh);
2983                 ehci_dump_sqtds(setup);
2984         }
2985 #endif
2986
2987         return (USBD_NORMAL_COMPLETION);
2988
2989  bad3:
2990         sqh->inactivesqtd = setup;
2991         ehci_free_sqtd(sc, stat);
2992  bad2:
2993         ehci_free_sqtd(sc, newinactive);
2994  bad1:
2995         DPRINTFN(-1,("ehci_device_request: no memory\n"));
2996         xfer->status = err;
2997         usb_transfer_complete(xfer);
2998         return (err);
2999 #undef exfer
3000 }
3001
3002 /************************/
3003
3004 static usbd_status
3005 ehci_device_bulk_transfer(usbd_xfer_handle xfer)
3006 {
3007         usbd_status err;
3008
3009         /* Insert last in queue. */
3010         err = usb_insert_transfer(xfer);
3011         if (err)
3012                 return (err);
3013
3014         /* Pipe isn't running, start first */
3015         return (ehci_device_bulk_start(STAILQ_FIRST(&xfer->pipe->queue)));
3016 }
3017
3018 usbd_status
3019 ehci_device_bulk_start(usbd_xfer_handle xfer)
3020 {
3021 #define exfer EXFER(xfer)
3022         struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
3023         usbd_device_handle dev = epipe->pipe.device;
3024         ehci_softc_t *sc = (ehci_softc_t *)dev->bus;
3025         ehci_soft_qtd_t *data, *dataend, *newinactive;
3026         ehci_soft_qh_t *sqh;
3027         usbd_status err;
3028         int len, isread, endpt;
3029         int s;
3030
3031         DPRINTFN(2, ("ehci_device_bulk_start: xfer=%p len=%d flags=%d\n",
3032                      xfer, xfer->length, xfer->flags));
3033
3034         if (sc->sc_dying)
3035                 return (USBD_IOERROR);
3036
3037 #ifdef DIAGNOSTIC
3038         if (xfer->rqflags & URQ_REQUEST)
3039                 panic("ehci_device_bulk_start: a request");
3040 #endif
3041
3042         len = xfer->length;
3043         endpt = epipe->pipe.endpoint->edesc->bEndpointAddress;
3044         isread = UE_GET_DIR(endpt) == UE_DIR_IN;
3045         sqh = epipe->sqh;
3046
3047         epipe->u.bulk.length = len;
3048
3049         newinactive = ehci_alloc_sqtd(sc);
3050         if (newinactive == NULL) {
3051                 DPRINTFN(-1,("ehci_device_bulk_start: no sqtd memory\n"));
3052                 err = USBD_NOMEM;
3053                 xfer->status = err;
3054                 usb_transfer_complete(xfer);
3055                 return (err);
3056         }
3057         newinactive->qtd.qtd_status = htole32(0);
3058         newinactive->qtd.qtd_next = EHCI_NULL;
3059         newinactive->qtd.qtd_altnext = EHCI_NULL;
3060         err = ehci_alloc_sqtd_chain(epipe, sc, len, isread, xfer,
3061             sqh->inactivesqtd, newinactive, &data, &dataend);
3062         if (err) {
3063                 DPRINTFN(-1,("ehci_device_bulk_start: no memory\n"));
3064                 ehci_free_sqtd(sc, newinactive);
3065                 xfer->status = err;
3066                 usb_transfer_complete(xfer);
3067                 return (err);
3068         }
3069         dataend->nextqtd = newinactive;
3070         dataend->qtd.qtd_next = htole32(newinactive->physaddr);
3071         dataend->qtd.qtd_altnext = htole32(newinactive->physaddr);
3072         sqh->inactivesqtd = newinactive;
3073
3074 #ifdef EHCI_DEBUG
3075         if (ehcidebug > 5) {
3076                 DPRINTF(("ehci_device_bulk_start: data(1)\n"));
3077                 ehci_dump_sqh(sqh);
3078                 ehci_dump_sqtds(data);
3079         }
3080 #endif
3081
3082         /* Set up interrupt info. */
3083         exfer->sqtdstart = data;
3084         exfer->sqtdend = dataend;
3085 #ifdef DIAGNOSTIC
3086         if (!exfer->isdone) {
3087                 printf("ehci_device_bulk_start: not done, ex=%p\n", exfer);
3088         }
3089         exfer->isdone = 0;
3090 #endif
3091
3092         s = splusb();
3093         ehci_activate_qh(sqh, data);
3094         if (xfer->timeout && !sc->sc_bus.use_polling) {
3095                 callout_reset(&xfer->timeout_handle, MS_TO_TICKS(xfer->timeout),
3096                     ehci_timeout, xfer);
3097         }
3098         ehci_add_intr_list(sc, exfer);
3099         xfer->status = USBD_IN_PROGRESS;
3100         splx(s);
3101
3102 #ifdef EHCI_DEBUG
3103         if (ehcidebug > 10) {
3104                 DPRINTF(("ehci_device_bulk_start: data(2)\n"));
3105                 delay(10000);
3106                 DPRINTF(("ehci_device_bulk_start: data(3)\n"));
3107                 ehci_dump_regs(sc);
3108 #if 0
3109                 printf("async_head:\n");
3110                 ehci_dump_sqh(sc->sc_async_head);
3111 #endif
3112                 printf("sqh:\n");
3113                 ehci_dump_sqh(sqh);
3114                 ehci_dump_sqtds(data);
3115         }
3116 #endif
3117
3118         if (sc->sc_bus.use_polling)
3119                 ehci_waitintr(sc, xfer);
3120
3121         return (USBD_IN_PROGRESS);
3122 #undef exfer
3123 }
3124
3125 static void
3126 ehci_device_bulk_abort(usbd_xfer_handle xfer)
3127 {
3128         DPRINTF(("ehci_device_bulk_abort: xfer=%p\n", xfer));
3129         ehci_abort_xfer(xfer, USBD_CANCELLED);
3130 }
3131
3132 /*
3133  * Close a device bulk pipe.
3134  */
3135 static void
3136 ehci_device_bulk_close(usbd_pipe_handle pipe)
3137 {
3138         ehci_softc_t *sc = (ehci_softc_t *)pipe->device->bus;
3139
3140         DPRINTF(("ehci_device_bulk_close: pipe=%p\n", pipe));
3141         ehci_close_pipe(pipe, sc->sc_async_head);
3142 }
3143
3144 void
3145 ehci_device_bulk_done(usbd_xfer_handle xfer)
3146 {
3147         struct ehci_xfer *ex = EXFER(xfer);
3148         ehci_softc_t *sc = (ehci_softc_t *)xfer->pipe->device->bus;
3149         struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
3150
3151         DPRINTFN(10,("ehci_bulk_done: xfer=%p, actlen=%d\n",
3152                      xfer, xfer->actlen));
3153
3154         if (xfer->status != USBD_NOMEM && ehci_active_intr_list(ex)) {
3155                 ehci_del_intr_list(ex); /* remove from active list */
3156                 ehci_free_sqtd_chain(sc, epipe->sqh, ex->sqtdstart,
3157                     ex->sqtdend->nextqtd);
3158         }
3159
3160         DPRINTFN(5, ("ehci_bulk_done: length=%d\n", xfer->actlen));
3161 }
3162
3163 /************************/
3164
3165 static usbd_status
3166 ehci_device_setintr(ehci_softc_t *sc, ehci_soft_qh_t *sqh, int ival)
3167 {
3168         struct ehci_soft_islot *isp;
3169         int islot, lev;
3170
3171         /* Find a poll rate that is large enough. */
3172         for (lev = EHCI_IPOLLRATES - 1; lev > 0; lev--)
3173                 if (EHCI_ILEV_IVAL(lev) <= ival)
3174                         break;
3175
3176         /* Pick an interrupt slot at the right level. */
3177         /* XXX could do better than picking at random. */
3178         islot = EHCI_IQHIDX(lev, arc4random());
3179
3180         sqh->islot = islot;
3181         isp = &sc->sc_islots[islot];
3182         ehci_add_qh(sqh, isp->sqh);
3183
3184         return (USBD_NORMAL_COMPLETION);
3185 }
3186
3187 static usbd_status
3188 ehci_device_intr_transfer(usbd_xfer_handle xfer)
3189 {
3190         usbd_status err;
3191
3192         /* Insert last in queue. */
3193         err = usb_insert_transfer(xfer);
3194         if (err)
3195                 return (err);
3196
3197         /*
3198          * Pipe isn't running (otherwise err would be USBD_INPROG),
3199          * so start it first.
3200          */
3201         return (ehci_device_intr_start(STAILQ_FIRST(&xfer->pipe->queue)));
3202 }
3203
3204 static usbd_status
3205 ehci_device_intr_start(usbd_xfer_handle xfer)
3206 {
3207 #define exfer EXFER(xfer)
3208         struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
3209         usbd_device_handle dev = xfer->pipe->device;
3210         ehci_softc_t *sc = (ehci_softc_t *)dev->bus;
3211         ehci_soft_qtd_t *data, *dataend, *newinactive;
3212         ehci_soft_qh_t *sqh;
3213         usbd_status err;
3214         int len, isread, endpt;
3215         int s;
3216
3217         DPRINTFN(2, ("ehci_device_intr_start: xfer=%p len=%d flags=%d\n",
3218             xfer, xfer->length, xfer->flags));
3219
3220         if (sc->sc_dying)
3221                 return (USBD_IOERROR);
3222
3223 #ifdef DIAGNOSTIC
3224         if (xfer->rqflags & URQ_REQUEST)
3225                 panic("ehci_device_intr_start: a request");
3226 #endif
3227
3228         len = xfer->length;
3229         endpt = epipe->pipe.endpoint->edesc->bEndpointAddress;
3230         isread = UE_GET_DIR(endpt) == UE_DIR_IN;
3231         sqh = epipe->sqh;
3232
3233         epipe->u.intr.length = len;
3234
3235         newinactive = ehci_alloc_sqtd(sc);
3236         if (newinactive == NULL) {
3237                 DPRINTFN(-1,("ehci_device_intr_start: no sqtd memory\n"));
3238                 err = USBD_NOMEM;
3239                 xfer->status = err;
3240                 usb_transfer_complete(xfer);
3241                 return (err);
3242         }
3243         newinactive->qtd.qtd_status = htole32(0);
3244         newinactive->qtd.qtd_next = EHCI_NULL;
3245         newinactive->qtd.qtd_altnext = EHCI_NULL;
3246         err = ehci_alloc_sqtd_chain(epipe, sc, len, isread, xfer,
3247             sqh->inactivesqtd, newinactive, &data, &dataend);
3248         if (err) {
3249                 DPRINTFN(-1, ("ehci_device_intr_start: no memory\n"));
3250                 xfer->status = err;
3251                 usb_transfer_complete(xfer);
3252                 return (err);
3253         }
3254         dataend->nextqtd = newinactive;
3255         dataend->qtd.qtd_next = htole32(newinactive->physaddr);
3256         dataend->qtd.qtd_altnext = htole32(newinactive->physaddr);
3257         sqh->inactivesqtd = newinactive;
3258
3259 #ifdef EHCI_DEBUG
3260         if (ehcidebug > 5) {
3261                 DPRINTF(("ehci_device_intr_start: data(1)\n"));
3262                 ehci_dump_sqh(sqh);
3263                 ehci_dump_sqtds(data);
3264         }
3265 #endif
3266
3267         /* Set up interrupt info. */
3268         exfer->sqtdstart = data;
3269         exfer->sqtdend = dataend;
3270 #ifdef DIAGNOSTIC
3271         if (!exfer->isdone) {
3272                 printf("ehci_device_intr_start: not done, ex=%p\n", exfer);
3273         }
3274         exfer->isdone = 0;
3275 #endif
3276
3277         s = splusb();
3278         ehci_activate_qh(sqh, data);
3279         if (xfer->timeout && !sc->sc_bus.use_polling) {
3280                 callout_reset(&xfer->timeout_handle, MS_TO_TICKS(xfer->timeout),
3281                     ehci_timeout, xfer);
3282         }
3283         ehci_add_intr_list(sc, exfer);
3284         xfer->status = USBD_IN_PROGRESS;
3285         splx(s);
3286
3287 #ifdef EHCI_DEBUG
3288         if (ehcidebug > 10) {
3289                 DPRINTF(("ehci_device_intr_start: data(2)\n"));
3290                 delay(10000);
3291                 DPRINTF(("ehci_device_intr_start: data(3)\n"));
3292                 ehci_dump_regs(sc);
3293                 printf("sqh:\n");
3294                 ehci_dump_sqh(sqh);
3295                 ehci_dump_sqtds(data);
3296         }
3297 #endif
3298
3299         if (sc->sc_bus.use_polling)
3300                 ehci_waitintr(sc, xfer);
3301
3302         return (USBD_IN_PROGRESS);
3303 #undef exfer
3304 }
3305
3306 static void
3307 ehci_device_intr_abort(usbd_xfer_handle xfer)
3308 {
3309         DPRINTFN(1, ("ehci_device_intr_abort: xfer=%p\n", xfer));
3310         if (xfer->pipe->intrxfer == xfer) {
3311                 DPRINTFN(1, ("ehci_device_intr_abort: remove\n"));
3312                 xfer->pipe->intrxfer = NULL;
3313         }
3314         ehci_abort_xfer(xfer, USBD_CANCELLED);
3315 }
3316
3317 static void
3318 ehci_device_intr_close(usbd_pipe_handle pipe)
3319 {
3320         ehci_softc_t *sc = (ehci_softc_t *)pipe->device->bus;
3321         struct ehci_pipe *epipe = (struct ehci_pipe *)pipe;
3322         struct ehci_soft_islot *isp;
3323
3324         isp = &sc->sc_islots[epipe->sqh->islot];
3325         ehci_close_pipe(pipe, isp->sqh);
3326 }
3327
3328 static void
3329 ehci_device_intr_done(usbd_xfer_handle xfer)
3330 {
3331 #define exfer EXFER(xfer)
3332         struct ehci_xfer *ex = EXFER(xfer);
3333         ehci_softc_t *sc = (ehci_softc_t *)xfer->pipe->device->bus;
3334         struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
3335         ehci_soft_qtd_t *data, *dataend, *newinactive;
3336         ehci_soft_qh_t *sqh;
3337         usbd_status err;
3338         int len, isread, endpt, s;
3339
3340         DPRINTFN(10, ("ehci_device_intr_done: xfer=%p, actlen=%d\n",
3341             xfer, xfer->actlen));
3342
3343         sqh = epipe->sqh;
3344         if (xfer->pipe->repeat) {
3345                 ehci_free_sqtd_chain(sc, sqh, ex->sqtdstart,
3346                     ex->sqtdend->nextqtd);
3347
3348                 len = epipe->u.intr.length;
3349                 xfer->length = len;
3350                 endpt = epipe->pipe.endpoint->edesc->bEndpointAddress;
3351                 isread = UE_GET_DIR(endpt) == UE_DIR_IN;
3352
3353                 newinactive = ehci_alloc_sqtd(sc);
3354                 if (newinactive == NULL) {
3355                         DPRINTFN(-1,
3356                             ("ehci_device_intr_done: no sqtd memory\n"));
3357                         err = USBD_NOMEM;
3358                         xfer->status = err;
3359                         return;
3360                 }
3361                 newinactive->qtd.qtd_status = htole32(0);
3362                 newinactive->qtd.qtd_next = EHCI_NULL;
3363                 newinactive->qtd.qtd_altnext = EHCI_NULL;
3364                 err = ehci_alloc_sqtd_chain(epipe, sc, len, isread, xfer,
3365                     sqh->inactivesqtd, newinactive, &data, &dataend);
3366                 if (err) {
3367                         DPRINTFN(-1, ("ehci_device_intr_done: no memory\n"));
3368                         xfer->status = err;
3369                         return;
3370                 }
3371                 dataend->nextqtd = newinactive;
3372                 dataend->qtd.qtd_next = htole32(newinactive->physaddr);
3373                 dataend->qtd.qtd_altnext = htole32(newinactive->physaddr);
3374                 sqh->inactivesqtd = newinactive;
3375
3376                 /* Set up interrupt info. */
3377                 exfer->sqtdstart = data;
3378                 exfer->sqtdend = dataend;
3379 #ifdef DIAGNOSTIC
3380                 if (!exfer->isdone) {
3381                         printf("ehci_device_intr_done: not done, ex=%p\n",
3382                             exfer);
3383                 }
3384                 exfer->isdone = 0;
3385 #endif
3386
3387                 s = splusb();
3388                 ehci_activate_qh(sqh, data);
3389                 if (xfer->timeout && !sc->sc_bus.use_polling) {
3390                         callout_reset(&xfer->timeout_handle,
3391                             MS_TO_TICKS(xfer->timeout), ehci_timeout, xfer);
3392                 }
3393                 splx(s);
3394
3395                 xfer->status = USBD_IN_PROGRESS;
3396         } else if (xfer->status != USBD_NOMEM && ehci_active_intr_list(ex)) {
3397                 ehci_del_intr_list(ex); /* remove from active list */
3398                 ehci_free_sqtd_chain(sc, sqh, ex->sqtdstart,
3399                     ex->sqtdend->nextqtd);
3400         }
3401 #undef exfer
3402 }
3403
3404 /************************/
3405
3406 static usbd_status      ehci_device_isoc_transfer(usbd_xfer_handle xfer) { return USBD_IOERROR; }
3407 static usbd_status      ehci_device_isoc_start(usbd_xfer_handle xfer) { return USBD_IOERROR; }
3408 static void             ehci_device_isoc_abort(usbd_xfer_handle xfer) { }
3409 static void             ehci_device_isoc_close(usbd_pipe_handle pipe) { }
3410 static void             ehci_device_isoc_done(usbd_xfer_handle xfer) { }