sys/dev/usb/ehci.c

   1 /*      $NetBSD: ehci.c,v 1.91 2005/02/27 00:27:51 perry Exp $ */
   2
   3 /*-
   4  * Copyright (c) 2004 The NetBSD Foundation, Inc.
   5  * All rights reserved.
   6  *
   7  * This code is derived from software contributed to The NetBSD Foundation
   8  * by Lennart Augustsson (lennart@augustsson.net) and by Charles M. Hannum.
   9  *
  10  * Redistribution and use in source and binary forms, with or without
  11  * modification, are permitted provided that the following conditions
  12  * are met:
  13  * 1. Redistributions of source code must retain the above copyright
  14  *    notice, this list of conditions and the following disclaimer.
  15  * 2. Redistributions in binary form must reproduce the above copyright
  16  *    notice, this list of conditions and the following disclaimer in the
  17  *    documentation and/or other materials provided with the distribution.
  18  * 3. All advertising materials mentioning features or use of this software
  19  *    must display the following acknowledgement:
  20  *        This product includes software developed by the NetBSD
  21  *        Foundation, Inc. and its contributors.
  22  * 4. Neither the name of The NetBSD Foundation nor the names of its
  23  *    contributors may be used to endorse or promote products derived
  24  *    from this software without specific prior written permission.
  25  *
  26  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
  27  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
  28  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  29  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
  30  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  31  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  32  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  33  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  34  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  35  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  36  * POSSIBILITY OF SUCH DAMAGE.
  37  */
  38
  39 /*
  40  * USB Enhanced Host Controller Driver, a.k.a. USB 2.0 controller.
  41  *
  42  * The EHCI 1.0 spec can be found at
  43  * http://developer.intel.com/technology/usb/download/ehci-r10.pdf
  44  * and the USB 2.0 spec at
  45  * http://www.usb.org/developers/docs/usb_20.zip
  46  *
  47  */
  48
  49 /*
  50  * TODO:
  51  * 1) The EHCI driver lacks support for isochronous transfers, so
  52  *    devices using them don't work.
  53  *
  54  * 2) Interrupt transfer scheduling does not manage the time available
  55  *    in each frame, so it is possible for the transfers to overrun
  56  *    the end of the frame.
  57  *
  58  * 3) Command failures are not recovered correctly.
  59  */
  60
  61 #include <sys/cdefs.h>
  62 __FBSDID("$FreeBSD$");
  63
  64 #include <sys/param.h>
  65 #include <sys/systm.h>
  66 #include <sys/malloc.h>
  67 #include <sys/kernel.h>
  68 #include <sys/endian.h>
  69 #include <sys/module.h>
  70 #include <sys/bus.h>
  71 #include <sys/lockmgr.h>
  72 #if defined(DIAGNOSTIC) && defined(__i386__) && defined(__FreeBSD__)
  73 #include <machine/cpu.h>
  74 #endif
  75 #include <sys/proc.h>
  76 #include <sys/queue.h>
  77 #include <sys/sysctl.h>
  78
  79 #include <machine/bus.h>
  80 #include <machine/endian.h>
  81
  82 #include <dev/usb/usb.h>
  83 #include <dev/usb/usbdi.h>
  84 #include <dev/usb/usbdivar.h>
  85 #include <dev/usb/usb_mem.h>
  86 #include <dev/usb/usb_quirks.h>
  87
  88 #include <dev/usb/ehcireg.h>
  89 #include <dev/usb/ehcivar.h>
  90
  91 #define delay(d)                DELAY(d)
  92
  93 #ifdef USB_DEBUG
  94 #define EHCI_DEBUG USB_DEBUG
  95 #define DPRINTF(x)      do { if (ehcidebug) printf x; } while (0)
  96 #define DPRINTFN(n,x)   do { if (ehcidebug>(n)) printf x; } while (0)
  97 int ehcidebug = 0;
  98 SYSCTL_NODE(_hw_usb, OID_AUTO, ehci, CTLFLAG_RW, 0, "USB ehci");
  99 SYSCTL_INT(_hw_usb_ehci, OID_AUTO, debug, CTLFLAG_RW,
 100            &ehcidebug, 0, "ehci debug level");
 101 #define bitmask_snprintf(q,f,b,l) snprintf((b), (l), "%b", (q), (f))
 102 #else
 103 #define DPRINTF(x)
 104 #define DPRINTFN(n,x)
 105 #endif
 106
 107 struct ehci_pipe {
 108         struct usbd_pipe pipe;
 109
 110         ehci_soft_qh_t *sqh;
 111         union {
 112                 ehci_soft_qtd_t *qtd;
 113                 /* ehci_soft_itd_t *itd; */
 114         } tail;
 115         union {
 116                 /* Control pipe */
 117                 struct {
 118                         usb_dma_t reqdma;
 119                         u_int length;
 120                         /*ehci_soft_qtd_t *setup, *data, *stat;*/
 121                 } ctl;
 122                 /* Interrupt pipe */
 123                 struct {
 124                         u_int length;
 125                 } intr;
 126                 /* Bulk pipe */
 127                 struct {
 128                         u_int length;
 129                 } bulk;
 130                 /* Iso pipe */
 131                 /* XXX */
 132         } u;
 133 };
 134
 135 static usbd_status      ehci_open(usbd_pipe_handle);
 136 static void             ehci_poll(struct usbd_bus *);
 137 static void             ehci_softintr(void *);
 138 static int              ehci_intr1(ehci_softc_t *);
 139 static void             ehci_waitintr(ehci_softc_t *, usbd_xfer_handle);
 140 static void             ehci_check_intr(ehci_softc_t *, struct ehci_xfer *);
 141 static void             ehci_idone(struct ehci_xfer *);
 142 static void             ehci_timeout(void *);
 143 static void             ehci_timeout_task(void *);
 144 static void             ehci_intrlist_timeout(void *);
 145
 146 static usbd_status      ehci_allocm(struct usbd_bus *, usb_dma_t *, u_int32_t);
 147 static void             ehci_freem(struct usbd_bus *, usb_dma_t *);
 148
 149 static usbd_xfer_handle ehci_allocx(struct usbd_bus *);
 150 static void             ehci_freex(struct usbd_bus *, usbd_xfer_handle);
 151
 152 static usbd_status      ehci_root_ctrl_transfer(usbd_xfer_handle);
 153 static usbd_status      ehci_root_ctrl_start(usbd_xfer_handle);
 154 static void             ehci_root_ctrl_abort(usbd_xfer_handle);
 155 static void             ehci_root_ctrl_close(usbd_pipe_handle);
 156 static void             ehci_root_ctrl_done(usbd_xfer_handle);
 157
 158 static usbd_status      ehci_root_intr_transfer(usbd_xfer_handle);
 159 static usbd_status      ehci_root_intr_start(usbd_xfer_handle);
 160 static void             ehci_root_intr_abort(usbd_xfer_handle);
 161 static void             ehci_root_intr_close(usbd_pipe_handle);
 162 static void             ehci_root_intr_done(usbd_xfer_handle);
 163
 164 static usbd_status      ehci_device_ctrl_transfer(usbd_xfer_handle);
 165 static usbd_status      ehci_device_ctrl_start(usbd_xfer_handle);
 166 static void             ehci_device_ctrl_abort(usbd_xfer_handle);
 167 static void             ehci_device_ctrl_close(usbd_pipe_handle);
 168 static void             ehci_device_ctrl_done(usbd_xfer_handle);
 169
 170 static usbd_status      ehci_device_bulk_transfer(usbd_xfer_handle);
 171 static usbd_status      ehci_device_bulk_start(usbd_xfer_handle);
 172 static void             ehci_device_bulk_abort(usbd_xfer_handle);
 173 static void             ehci_device_bulk_close(usbd_pipe_handle);
 174 static void             ehci_device_bulk_done(usbd_xfer_handle);
 175
 176 static usbd_status      ehci_device_intr_transfer(usbd_xfer_handle);
 177 static usbd_status      ehci_device_intr_start(usbd_xfer_handle);
 178 static void             ehci_device_intr_abort(usbd_xfer_handle);
 179 static void             ehci_device_intr_close(usbd_pipe_handle);
 180 static void             ehci_device_intr_done(usbd_xfer_handle);
 181
 182 static usbd_status      ehci_device_isoc_transfer(usbd_xfer_handle);
 183 static usbd_status      ehci_device_isoc_start(usbd_xfer_handle);
 184 static void             ehci_device_isoc_abort(usbd_xfer_handle);
 185 static void             ehci_device_isoc_close(usbd_pipe_handle);
 186 static void             ehci_device_isoc_done(usbd_xfer_handle);
 187
 188 static void             ehci_device_clear_toggle(usbd_pipe_handle pipe);
 189 static void             ehci_noop(usbd_pipe_handle pipe);
 190
 191 static int              ehci_str(usb_string_descriptor_t *, int, char *);
 192 static void             ehci_pcd(ehci_softc_t *, usbd_xfer_handle);
 193 static void             ehci_pcd_able(ehci_softc_t *, int);
 194 static void             ehci_pcd_enable(void *);
 195 static void             ehci_disown(ehci_softc_t *, int, int);
 196
 197 static ehci_soft_qh_t  *ehci_alloc_sqh(ehci_softc_t *);
 198 static void             ehci_free_sqh(ehci_softc_t *, ehci_soft_qh_t *);
 199
 200 static ehci_soft_qtd_t  *ehci_alloc_sqtd(ehci_softc_t *);
 201 static void             ehci_free_sqtd(ehci_softc_t *, ehci_soft_qtd_t *);
 202 static usbd_status      ehci_alloc_sqtd_chain(struct ehci_pipe *,
 203                             ehci_softc_t *, int, int, usbd_xfer_handle,
 204                             ehci_soft_qtd_t *, ehci_soft_qtd_t *,
 205                             ehci_soft_qtd_t **, ehci_soft_qtd_t **);
 206 static void             ehci_free_sqtd_chain(ehci_softc_t *, ehci_soft_qh_t *,
 207                             ehci_soft_qtd_t *, ehci_soft_qtd_t *);
 208
 209 static usbd_status      ehci_device_request(usbd_xfer_handle xfer);
 210
 211 static usbd_status      ehci_device_setintr(ehci_softc_t *, ehci_soft_qh_t *,
 212                             int ival);
 213
 214 static void             ehci_add_qh(ehci_soft_qh_t *, ehci_soft_qh_t *);
 215 static void             ehci_rem_qh(ehci_softc_t *, ehci_soft_qh_t *,
 216                                     ehci_soft_qh_t *);
 217 static void             ehci_activate_qh(ehci_soft_qh_t *, ehci_soft_qtd_t *);
 218 static void             ehci_sync_hc(ehci_softc_t *);
 219
 220 static void             ehci_close_pipe(usbd_pipe_handle, ehci_soft_qh_t *);
 221 static void             ehci_abort_xfer(usbd_xfer_handle, usbd_status);
 222
 223 #ifdef EHCI_DEBUG
 224 static void             ehci_dump_regs(ehci_softc_t *);
 225 void                    ehci_dump(void);
 226 static ehci_softc_t     *theehci;
 227 static void             ehci_dump_link(ehci_link_t, int);
 228 static void             ehci_dump_sqtds(ehci_soft_qtd_t *);
 229 static void             ehci_dump_sqtd(ehci_soft_qtd_t *);
 230 static void             ehci_dump_qtd(ehci_qtd_t *);
 231 static void             ehci_dump_sqh(ehci_soft_qh_t *);
 232 #ifdef DIAGNOSTIC
 233 static void             ehci_dump_exfer(struct ehci_xfer *);
 234 #endif
 235 #endif
 236
 237 #define EHCI_NULL htole32(EHCI_LINK_TERMINATE)
 238
 239 #define EHCI_INTR_ENDPT 1
 240
 241 #define ehci_add_intr_list(sc, ex) \
 242         LIST_INSERT_HEAD(&(sc)->sc_intrhead, (ex), inext);
 243 #define ehci_del_intr_list(ex) \
 244         do { \
 245                 LIST_REMOVE((ex), inext); \
 246                 (ex)->inext.le_prev = NULL; \
 247         } while (0)
 248 #define ehci_active_intr_list(ex) ((ex)->inext.le_prev != NULL)
 249
 250 static struct usbd_bus_methods ehci_bus_methods = {
 251         ehci_open,
 252         ehci_softintr,
 253         ehci_poll,
 254         ehci_allocm,
 255         ehci_freem,
 256         ehci_allocx,
 257         ehci_freex,
 258 };
 259
 260 static struct usbd_pipe_methods ehci_root_ctrl_methods = {
 261         ehci_root_ctrl_transfer,
 262         ehci_root_ctrl_start,
 263         ehci_root_ctrl_abort,
 264         ehci_root_ctrl_close,
 265         ehci_noop,
 266         ehci_root_ctrl_done,
 267 };
 268
 269 static struct usbd_pipe_methods ehci_root_intr_methods = {
 270         ehci_root_intr_transfer,
 271         ehci_root_intr_start,
 272         ehci_root_intr_abort,
 273         ehci_root_intr_close,
 274         ehci_noop,
 275         ehci_root_intr_done,
 276 };
 277
 278 static struct usbd_pipe_methods ehci_device_ctrl_methods = {
 279         ehci_device_ctrl_transfer,
 280         ehci_device_ctrl_start,
 281         ehci_device_ctrl_abort,
 282         ehci_device_ctrl_close,
 283         ehci_noop,
 284         ehci_device_ctrl_done,
 285 };
 286
 287 static struct usbd_pipe_methods ehci_device_intr_methods = {
 288         ehci_device_intr_transfer,
 289         ehci_device_intr_start,
 290         ehci_device_intr_abort,
 291         ehci_device_intr_close,
 292         ehci_device_clear_toggle,
 293         ehci_device_intr_done,
 294 };
 295
 296 static struct usbd_pipe_methods ehci_device_bulk_methods = {
 297         ehci_device_bulk_transfer,
 298         ehci_device_bulk_start,
 299         ehci_device_bulk_abort,
 300         ehci_device_bulk_close,
 301         ehci_device_clear_toggle,
 302         ehci_device_bulk_done,
 303 };
 304
 305 static struct usbd_pipe_methods ehci_device_isoc_methods = {
 306         ehci_device_isoc_transfer,
 307         ehci_device_isoc_start,
 308         ehci_device_isoc_abort,
 309         ehci_device_isoc_close,
 310         ehci_noop,
 311         ehci_device_isoc_done,
 312 };
 313
 314 usbd_status
 315 ehci_init(ehci_softc_t *sc)
 316 {
 317         u_int32_t version, sparams, cparams, hcr;
 318         u_int i;
 319         usbd_status err;
 320         ehci_soft_qh_t *sqh;
 321         u_int ncomp;
 322         int lev;
 323
 324         DPRINTF(("ehci_init: start\n"));
 325 #ifdef EHCI_DEBUG
 326         theehci = sc;
 327 #endif
 328
 329         sc->sc_offs = EREAD1(sc, EHCI_CAPLENGTH);
 330
 331         version = EREAD2(sc, EHCI_HCIVERSION);
 332         printf("%s: EHCI version %x.%x\n", device_get_nameunit(sc->sc_bus.bdev),
 333                version >> 8, version & 0xff);
 334
 335         sparams = EREAD4(sc, EHCI_HCSPARAMS);
 336         DPRINTF(("ehci_init: sparams=0x%x\n", sparams));
 337         sc->sc_npcomp = EHCI_HCS_N_PCC(sparams);
 338         ncomp = EHCI_HCS_N_CC(sparams);
 339         if (ncomp != sc->sc_ncomp) {
 340                 printf("%s: wrong number of companions (%d != %d)\n",
 341                        device_get_nameunit(sc->sc_bus.bdev),
 342                        ncomp, sc->sc_ncomp);
 343                 if (ncomp < sc->sc_ncomp)
 344                         sc->sc_ncomp = ncomp;
 345         }
 346         if (sc->sc_ncomp > 0) {
 347                 printf("%s: companion controller%s, %d port%s each:",
 348                     device_get_nameunit(sc->sc_bus.bdev), sc->sc_ncomp!=1 ? "s" : "",
 349                     EHCI_HCS_N_PCC(sparams),
 350                     EHCI_HCS_N_PCC(sparams)!=1 ? "s" : "");
 351                 for (i = 0; i < sc->sc_ncomp; i++)
 352                         printf(" %s", device_get_nameunit(sc->sc_comps[i]->bdev));
 353                 printf("\n");
 354         }
 355         sc->sc_noport = EHCI_HCS_N_PORTS(sparams);
 356         cparams = EREAD4(sc, EHCI_HCCPARAMS);
 357         DPRINTF(("ehci_init: cparams=0x%x\n", cparams));
 358
 359         if (EHCI_HCC_64BIT(cparams)) {
 360                 /* MUST clear segment register if 64 bit capable. */
 361                 EWRITE4(sc, EHCI_CTRLDSSEGMENT, 0);
 362         }
 363
 364         sc->sc_bus.usbrev = USBREV_2_0;
 365
 366         /* Reset the controller */
 367         DPRINTF(("%s: resetting\n", device_get_nameunit(sc->sc_bus.bdev)));
 368         EOWRITE4(sc, EHCI_USBCMD, 0);   /* Halt controller */
 369         usb_delay_ms(&sc->sc_bus, 1);
 370         EOWRITE4(sc, EHCI_USBCMD, EHCI_CMD_HCRESET);
 371         for (i = 0; i < 100; i++) {
 372                 usb_delay_ms(&sc->sc_bus, 1);
 373                 hcr = EOREAD4(sc, EHCI_USBCMD) & EHCI_CMD_HCRESET;
 374                 if (!hcr)
 375                         break;
 376         }
 377         if (hcr) {
 378                 printf("%s: reset timeout\n",
 379                     device_get_nameunit(sc->sc_bus.bdev));
 380                 return (USBD_IOERROR);
 381         }
 382
 383         /* frame list size at default, read back what we got and use that */
 384         switch (EHCI_CMD_FLS(EOREAD4(sc, EHCI_USBCMD))) {
 385         case 0: sc->sc_flsize = 1024; break;
 386         case 1: sc->sc_flsize = 512; break;
 387         case 2: sc->sc_flsize = 256; break;
 388         case 3: return (USBD_IOERROR);
 389         }
 390         err = usb_allocmem(&sc->sc_bus, sc->sc_flsize * sizeof(ehci_link_t),
 391             EHCI_FLALIGN_ALIGN, &sc->sc_fldma);
 392         if (err)
 393                 return (err);
 394         DPRINTF(("%s: flsize=%d\n", device_get_nameunit(sc->sc_bus.bdev),sc->sc_flsize));
 395         sc->sc_flist = KERNADDR(&sc->sc_fldma, 0);
 396         EOWRITE4(sc, EHCI_PERIODICLISTBASE, DMAADDR(&sc->sc_fldma, 0));
 397
 398         /* Set up the bus struct. */
 399         sc->sc_bus.methods = &ehci_bus_methods;
 400         sc->sc_bus.pipe_size = sizeof(struct ehci_pipe);
 401
 402 #if defined(__NetBSD__) || defined(__OpenBSD__)
 403         sc->sc_powerhook = powerhook_establish(ehci_power, sc);
 404         sc->sc_shutdownhook = shutdownhook_establish(ehci_shutdown, sc);
 405 #endif
 406
 407         sc->sc_eintrs = EHCI_NORMAL_INTRS;
 408
 409         /*
 410          * Allocate the interrupt dummy QHs. These are arranged to give
 411          * poll intervals that are powers of 2 times 1ms.
 412          */
 413         for (i = 0; i < EHCI_INTRQHS; i++) {
 414                 sqh = ehci_alloc_sqh(sc);
 415                 if (sqh == NULL) {
 416                         err = USBD_NOMEM;
 417                         goto bad1;
 418                 }
 419                 sc->sc_islots[i].sqh = sqh;
 420         }
 421         lev = 0;
 422         for (i = 0; i < EHCI_INTRQHS; i++) {
 423                 if (i == EHCI_IQHIDX(lev + 1, 0))
 424                         lev++;
 425                 sqh = sc->sc_islots[i].sqh;
 426                 if (i == 0) {
 427                         /* The last (1ms) QH terminates. */
 428                         sqh->qh.qh_link = EHCI_NULL;
 429                         sqh->next = NULL;
 430                 } else {
 431                         /* Otherwise the next QH has half the poll interval */
 432                         sqh->next =
 433                             sc->sc_islots[EHCI_IQHIDX(lev - 1, i + 1)].sqh;
 434                         sqh->qh.qh_link = htole32(sqh->next->physaddr |
 435                             EHCI_LINK_QH);
 436                 }
 437                 sqh->qh.qh_endp = htole32(EHCI_QH_SET_EPS(EHCI_QH_SPEED_HIGH));
 438                 sqh->qh.qh_endphub = htole32(EHCI_QH_SET_MULT(1));
 439                 sqh->qh.qh_curqtd = EHCI_NULL;
 440                 sqh->qh.qh_qtd.qtd_next = EHCI_NULL;
 441                 sqh->qh.qh_qtd.qtd_altnext = EHCI_NULL;
 442                 sqh->qh.qh_qtd.qtd_status = htole32(EHCI_QTD_HALTED);
 443         }
 444         /* Point the frame list at the last level (128ms). */
 445         for (i = 0; i < sc->sc_flsize; i++) {
 446                 sc->sc_flist[i] = htole32(EHCI_LINK_QH |
 447                     sc->sc_islots[EHCI_IQHIDX(EHCI_IPOLLRATES - 1,
 448                     i)].sqh->physaddr);
 449         }
 450
 451         /* Allocate dummy QH that starts the async list. */
 452         sqh = ehci_alloc_sqh(sc);
 453         if (sqh == NULL) {
 454                 err = USBD_NOMEM;
 455                 goto bad1;
 456         }
 457         /* Fill the QH */
 458         sqh->qh.qh_endp =
 459             htole32(EHCI_QH_SET_EPS(EHCI_QH_SPEED_HIGH) | EHCI_QH_HRECL);
 460         sqh->qh.qh_link =
 461             htole32(sqh->physaddr | EHCI_LINK_QH);
 462         sqh->qh.qh_curqtd = EHCI_NULL;
 463         sqh->prev = sqh; /*It's a circular list.. */
 464         sqh->next = sqh;
 465         /* Fill the overlay qTD */
 466         sqh->qh.qh_qtd.qtd_next = EHCI_NULL;
 467         sqh->qh.qh_qtd.qtd_altnext = EHCI_NULL;
 468         sqh->qh.qh_qtd.qtd_status = htole32(0);
 469 #ifdef EHCI_DEBUG
 470         if (ehcidebug) {
 471                 ehci_dump_sqh(sqh);
 472         }
 473 #endif
 474
 475         /* Point to async list */
 476         sc->sc_async_head = sqh;
 477         EOWRITE4(sc, EHCI_ASYNCLISTADDR, sqh->physaddr | EHCI_LINK_QH);
 478
 479         callout_init(&sc->sc_tmo_pcd, 0);
 480         callout_init(&sc->sc_tmo_intrlist, 0);
 481
 482         lockinit(&sc->sc_doorbell_lock, PZERO, "ehcidb", 0, 0);
 483
 484         /* Enable interrupts */
 485         EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
 486
 487         /* Turn on controller */
 488         EOWRITE4(sc, EHCI_USBCMD,
 489                  EHCI_CMD_ITC_2 | /* 2 microframes interrupt delay */
 490                  (EOREAD4(sc, EHCI_USBCMD) & EHCI_CMD_FLS_M) |
 491                  EHCI_CMD_ASE |
 492                  EHCI_CMD_PSE |
 493                  EHCI_CMD_RS);
 494
 495         /* Take over port ownership */
 496         EOWRITE4(sc, EHCI_CONFIGFLAG, EHCI_CONF_CF);
 497
 498         for (i = 0; i < 100; i++) {
 499                 usb_delay_ms(&sc->sc_bus, 1);
 500                 hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
 501                 if (!hcr)
 502                         break;
 503         }
 504         if (hcr) {
 505                 printf("%s: run timeout\n", device_get_nameunit(sc->sc_bus.bdev));
 506                 return (USBD_IOERROR);
 507         }
 508
 509         return (USBD_NORMAL_COMPLETION);
 510
 511 #if 0
 512  bad2:
 513         ehci_free_sqh(sc, sc->sc_async_head);
 514 #endif
 515  bad1:
 516         usb_freemem(&sc->sc_bus, &sc->sc_fldma);
 517         return (err);
 518 }
 519
 520 int
 521 ehci_intr(void *v)
 522 {
 523         ehci_softc_t *sc = v;
 524
 525         if (sc == NULL || sc->sc_dying)
 526                 return (0);
 527
 528         /* If we get an interrupt while polling, then just ignore it. */
 529         if (sc->sc_bus.use_polling) {
 530                 u_int32_t intrs = EHCI_STS_INTRS(EOREAD4(sc, EHCI_USBSTS));
 531
 532                 if (intrs)
 533                         EOWRITE4(sc, EHCI_USBSTS, intrs); /* Acknowledge */
 534 #ifdef DIAGNOSTIC
 535                 DPRINTFN(16, ("ehci_intr: ignored interrupt while polling\n"));
 536 #endif
 537                 return (0);
 538         }
 539
 540         return (ehci_intr1(sc));
 541 }
 542
 543 static int
 544 ehci_intr1(ehci_softc_t *sc)
 545 {
 546         u_int32_t intrs, eintrs;
 547
 548         DPRINTFN(20,("ehci_intr1: enter\n"));
 549
 550         /* In case the interrupt occurs before initialization has completed. */
 551         if (sc == NULL) {
 552 #ifdef DIAGNOSTIC
 553                 printf("ehci_intr1: sc == NULL\n");
 554 #endif
 555                 return (0);
 556         }
 557
 558         intrs = EHCI_STS_INTRS(EOREAD4(sc, EHCI_USBSTS));
 559         if (!intrs)
 560                 return (0);
 561
 562         eintrs = intrs & sc->sc_eintrs;
 563         DPRINTFN(7, ("ehci_intr1: sc=%p intrs=0x%x(0x%x) eintrs=0x%x\n",
 564                      sc, (u_int)intrs, EOREAD4(sc, EHCI_USBSTS),
 565                      (u_int)eintrs));
 566         if (!eintrs)
 567                 return (0);
 568
 569         EOWRITE4(sc, EHCI_USBSTS, intrs); /* Acknowledge */
 570         sc->sc_bus.intr_context++;
 571         sc->sc_bus.no_intrs++;
 572         if (eintrs & EHCI_STS_IAA) {
 573                 DPRINTF(("ehci_intr1: door bell\n"));
 574                 wakeup(&sc->sc_async_head);
 575                 eintrs &= ~EHCI_STS_IAA;
 576         }
 577         if (eintrs & (EHCI_STS_INT | EHCI_STS_ERRINT)) {
 578                 DPRINTFN(5,("ehci_intr1: %s %s\n",
 579                             eintrs & EHCI_STS_INT ? "INT" : "",
 580                             eintrs & EHCI_STS_ERRINT ? "ERRINT" : ""));
 581                 usb_schedsoftintr(&sc->sc_bus);
 582                 eintrs &= ~(EHCI_STS_INT | EHCI_STS_ERRINT);
 583         }
 584         if (eintrs & EHCI_STS_HSE) {
 585                 printf("%s: unrecoverable error, controller halted\n",
 586                        device_get_nameunit(sc->sc_bus.bdev));
 587                 /* XXX what else */
 588         }
 589         if (eintrs & EHCI_STS_PCD) {
 590                 ehci_pcd(sc, sc->sc_intrxfer);
 591                 /*
 592                  * Disable PCD interrupt for now, because it will be
 593                  * on until the port has been reset.
 594                  */
 595                 ehci_pcd_able(sc, 0);
 596                 /* Do not allow RHSC interrupts > 1 per second */
 597                 callout_reset(&sc->sc_tmo_pcd, hz, ehci_pcd_enable, sc);
 598                 eintrs &= ~EHCI_STS_PCD;
 599         }
 600
 601         sc->sc_bus.intr_context--;
 602
 603         if (eintrs != 0) {
 604                 /* Block unprocessed interrupts. */
 605                 sc->sc_eintrs &= ~eintrs;
 606                 EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
 607                 printf("%s: blocking intrs 0x%x\n",
 608                        device_get_nameunit(sc->sc_bus.bdev), eintrs);
 609         }
 610
 611         return (1);
 612 }
 613
 614 void
 615 ehci_pcd_able(ehci_softc_t *sc, int on)
 616 {
 617         DPRINTFN(4, ("ehci_pcd_able: on=%d\n", on));
 618         if (on)
 619                 sc->sc_eintrs |= EHCI_STS_PCD;
 620         else
 621                 sc->sc_eintrs &= ~EHCI_STS_PCD;
 622         EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
 623 }
 624
 625 void
 626 ehci_pcd_enable(void *v_sc)
 627 {
 628         ehci_softc_t *sc = v_sc;
 629
 630         ehci_pcd_able(sc, 1);
 631 }
 632
 633 void
 634 ehci_pcd(ehci_softc_t *sc, usbd_xfer_handle xfer)
 635 {
 636         usbd_pipe_handle pipe;
 637         u_char *p;
 638         int i, m;
 639
 640         if (xfer == NULL) {
 641                 /* Just ignore the change. */
 642                 return;
 643         }
 644
 645         pipe = xfer->pipe;
 646
 647         p = xfer->buffer;
 648         m = min(sc->sc_noport, xfer->length * 8 - 1);
 649         memset(p, 0, xfer->length);
 650         for (i = 1; i <= m; i++) {
 651                 /* Pick out CHANGE bits from the status reg. */
 652                 if (EOREAD4(sc, EHCI_PORTSC(i)) & EHCI_PS_CLEAR)
 653                         p[i/8] |= 1 << (i%8);
 654         }
 655         DPRINTF(("ehci_pcd: change=0x%02x\n", *p));
 656         xfer->actlen = xfer->length;
 657         xfer->status = USBD_NORMAL_COMPLETION;
 658
 659         usb_transfer_complete(xfer);
 660 }
 661
 662 void
 663 ehci_softintr(void *v)
 664 {
 665         ehci_softc_t *sc = v;
 666         struct ehci_xfer *ex, *nextex;
 667
 668         DPRINTFN(10,("%s: ehci_softintr (%d)\n", device_get_nameunit(sc->sc_bus.bdev),
 669                      sc->sc_bus.intr_context));
 670
 671         sc->sc_bus.intr_context++;
 672
 673         /*
 674          * The only explanation I can think of for why EHCI is as brain dead
 675          * as UHCI interrupt-wise is that Intel was involved in both.
 676          * An interrupt just tells us that something is done, we have no
 677          * clue what, so we need to scan through all active transfers. :-(
 678          */
 679         for (ex = LIST_FIRST(&sc->sc_intrhead); ex; ex = nextex) {
 680                 nextex = LIST_NEXT(ex, inext);
 681                 ehci_check_intr(sc, ex);
 682         }
 683
 684         /* Schedule a callout to catch any dropped transactions. */
 685         if ((sc->sc_flags & EHCI_SCFLG_LOSTINTRBUG) &&
 686             !LIST_EMPTY(&sc->sc_intrhead))
 687                 callout_reset(&sc->sc_tmo_intrlist, hz / 5,
 688                     ehci_intrlist_timeout, sc);
 689
 690 #ifdef USB_USE_SOFTINTR
 691         if (sc->sc_softwake) {
 692                 sc->sc_softwake = 0;
 693                 wakeup(&sc->sc_softwake);
 694         }
 695 #endif /* USB_USE_SOFTINTR */
 696
 697         sc->sc_bus.intr_context--;
 698 }
 699
 700 /* Check for an interrupt. */
 701 void
 702 ehci_check_intr(ehci_softc_t *sc, struct ehci_xfer *ex)
 703 {
 704         ehci_soft_qtd_t *sqtd, *lsqtd;
 705         u_int32_t status;
 706
 707         DPRINTFN(/*15*/2, ("ehci_check_intr: ex=%p\n", ex));
 708
 709         if (ex->sqtdstart == NULL) {
 710                 printf("ehci_check_intr: sqtdstart=NULL\n");
 711                 return;
 712         }
 713         lsqtd = ex->sqtdend;
 714 #ifdef DIAGNOSTIC
 715         if (lsqtd == NULL) {
 716                 printf("ehci_check_intr: lsqtd==0\n");
 717                 return;
 718         }
 719 #endif
 720         /*
 721          * If the last TD is still active we need to check whether there
 722          * is a an error somewhere in the middle, or whether there was a
 723          * short packet (SPD and not ACTIVE).
 724          */
 725         if (le32toh(lsqtd->qtd.qtd_status) & EHCI_QTD_ACTIVE) {
 726                 DPRINTFN(12, ("ehci_check_intr: active ex=%p\n", ex));
 727                 for (sqtd = ex->sqtdstart; sqtd != lsqtd; sqtd=sqtd->nextqtd) {
 728                         status = le32toh(sqtd->qtd.qtd_status);
 729                         /* If there's an active QTD the xfer isn't done. */
 730                         if (status & EHCI_QTD_ACTIVE)
 731                                 break;
 732                         /* Any kind of error makes the xfer done. */
 733                         if (status & EHCI_QTD_HALTED)
 734                                 goto done;
 735                         /* We want short packets, and it is short: it's done */
 736                         if (EHCI_QTD_GET_BYTES(status) != 0)
 737                                 goto done;
 738                 }
 739                 DPRINTFN(12, ("ehci_check_intr: ex=%p std=%p still active\n",
 740                               ex, ex->sqtdstart));
 741                 return;
 742         }
 743  done:
 744         DPRINTFN(12, ("ehci_check_intr: ex=%p done\n", ex));
 745         callout_stop(&ex->xfer.timeout_handle);
 746         usb_rem_task(ex->xfer.pipe->device, &ex->abort_task);
 747         ehci_idone(ex);
 748 }
 749
 750 void
 751 ehci_idone(struct ehci_xfer *ex)
 752 {
 753         usbd_xfer_handle xfer = &ex->xfer;
 754         struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
 755         ehci_soft_qtd_t *sqtd, *lsqtd;
 756         u_int32_t status = 0, nstatus = 0;
 757         ehci_physaddr_t nextphys, altnextphys;
 758         int actlen, cerr;
 759
 760         DPRINTFN(/*12*/2, ("ehci_idone: ex=%p\n", ex));
 761 #ifdef DIAGNOSTIC
 762         {
 763                 int s = splhigh();
 764                 if (ex->isdone) {
 765                         splx(s);
 766 #ifdef EHCI_DEBUG
 767                         printf("ehci_idone: ex is done!\n   ");
 768                         ehci_dump_exfer(ex);
 769 #else
 770                         printf("ehci_idone: ex=%p is done!\n", ex);
 771 #endif
 772                         return;
 773                 }
 774                 ex->isdone = 1;
 775                 splx(s);
 776         }
 777 #endif
 778
 779         if (xfer->status == USBD_CANCELLED ||
 780             xfer->status == USBD_TIMEOUT) {
 781                 DPRINTF(("ehci_idone: aborted xfer=%p\n", xfer));
 782                 return;
 783         }
 784
 785 #ifdef EHCI_DEBUG
 786         DPRINTFN(/*10*/2, ("ehci_idone: xfer=%p, pipe=%p ready\n", xfer, epipe));
 787         if (ehcidebug > 10)
 788                 ehci_dump_sqtds(ex->sqtdstart);
 789 #endif
 790
 791         /*
 792          * Make sure that the QH overlay qTD does not reference any
 793          * of the qTDs we are about to free. This is probably only
 794          * necessary if the transfer is marked as HALTED.
 795          */
 796         nextphys = EHCI_LINK_ADDR(le32toh(epipe->sqh->qh.qh_qtd.qtd_next));
 797         altnextphys =
 798             EHCI_LINK_ADDR(le32toh(epipe->sqh->qh.qh_qtd.qtd_altnext));
 799         for (sqtd = ex->sqtdstart; sqtd != ex->sqtdend->nextqtd;
 800              sqtd = sqtd->nextqtd) {
 801                 if (sqtd->physaddr == nextphys) {
 802                         epipe->sqh->qh.qh_qtd.qtd_next =
 803                             htole32(ex->sqtdend->nextqtd->physaddr);
 804                         DPRINTFN(4, ("ehci_idone: updated overlay next ptr\n"));
 805
 806                 }
 807                 if (sqtd->physaddr == altnextphys) {
 808                         DPRINTFN(4,
 809                             ("ehci_idone: updated overlay altnext ptr\n"));
 810                         epipe->sqh->qh.qh_qtd.qtd_altnext =
 811                             htole32(ex->sqtdend->nextqtd->physaddr);
 812                 }
 813         }
 814
 815         /* The transfer is done, compute actual length and status. */
 816         lsqtd = ex->sqtdend;
 817         actlen = 0;
 818         for (sqtd = ex->sqtdstart; sqtd != lsqtd->nextqtd; sqtd=sqtd->nextqtd) {
 819                 nstatus = le32toh(sqtd->qtd.qtd_status);
 820                 if (nstatus & EHCI_QTD_ACTIVE)
 821                         break;
 822
 823                 status = nstatus;
 824                 /* halt is ok if descriptor is last, and complete */
 825                 if (sqtd == lsqtd && EHCI_QTD_GET_BYTES(status) == 0)
 826                         status &= ~EHCI_QTD_HALTED;
 827                 if (EHCI_QTD_GET_PID(status) != EHCI_QTD_PID_SETUP)
 828                         actlen += sqtd->len - EHCI_QTD_GET_BYTES(status);
 829         }
 830
 831         cerr = EHCI_QTD_GET_CERR(status);
 832         DPRINTFN(/*10*/2, ("ehci_idone: len=%d, actlen=%d, cerr=%d, "
 833             "status=0x%x\n", xfer->length, actlen, cerr, status));
 834         xfer->actlen = actlen;
 835         if ((status & EHCI_QTD_HALTED) != 0) {
 836 #ifdef EHCI_DEBUG
 837                 char sbuf[128];
 838
 839                 bitmask_snprintf((u_int32_t)status,
 840                     "\20\7HALTED\6BUFERR\5BABBLE\4XACTERR"
 841                     "\3MISSED\2SPLIT\1PING", sbuf, sizeof(sbuf));
 842
 843                 DPRINTFN(2,
 844                          ("ehci_idone: error, addr=%d, endpt=0x%02x, "
 845                           "status 0x%s\n",
 846                           xfer->pipe->device->address,
 847                           xfer->pipe->endpoint->edesc->bEndpointAddress,
 848                           sbuf));
 849                 if (ehcidebug > 2) {
 850                         ehci_dump_sqh(epipe->sqh);
 851                         ehci_dump_sqtds(ex->sqtdstart);
 852                 }
 853 #endif
 854                 if ((status & EHCI_QTD_BABBLE) == 0 && cerr > 0)
 855                         xfer->status = USBD_STALLED;
 856                 else
 857                         xfer->status = USBD_IOERROR; /* more info XXX */
 858         } else {
 859                 xfer->status = USBD_NORMAL_COMPLETION;
 860         }
 861
 862         usb_transfer_complete(xfer);
 863         DPRINTFN(/*12*/2, ("ehci_idone: ex=%p done\n", ex));
 864 }
 865
 866 /*
 867  * Wait here until controller claims to have an interrupt.
 868  * Then call ehci_intr and return.  Use timeout to avoid waiting
 869  * too long.
 870  */
 871 void
 872 ehci_waitintr(ehci_softc_t *sc, usbd_xfer_handle xfer)
 873 {
 874         int timo = xfer->timeout;
 875         int usecs;
 876         u_int32_t intrs;
 877
 878         xfer->status = USBD_IN_PROGRESS;
 879         for (usecs = timo * 1000000 / hz; usecs > 0; usecs -= 1000) {
 880                 usb_delay_ms(&sc->sc_bus, 1);
 881                 if (sc->sc_dying)
 882                         break;
 883                 intrs = EHCI_STS_INTRS(EOREAD4(sc, EHCI_USBSTS)) &
 884                         sc->sc_eintrs;
 885                 DPRINTFN(15,("ehci_waitintr: 0x%04x\n", intrs));
 886 #ifdef EHCI_DEBUG
 887                 if (ehcidebug > 15)
 888                         ehci_dump_regs(sc);
 889 #endif
 890                 if (intrs) {
 891                         ehci_intr1(sc);
 892                         if (xfer->status != USBD_IN_PROGRESS)
 893                                 return;
 894                 }
 895         }
 896
 897         /* Timeout */
 898         DPRINTF(("ehci_waitintr: timeout\n"));
 899         xfer->status = USBD_TIMEOUT;
 900         usb_transfer_complete(xfer);
 901         /* XXX should free TD */
 902 }
 903
 904 void
 905 ehci_poll(struct usbd_bus *bus)
 906 {
 907         ehci_softc_t *sc = (ehci_softc_t *)bus;
 908 #ifdef EHCI_DEBUG
 909         static int last;
 910         int new;
 911         new = EHCI_STS_INTRS(EOREAD4(sc, EHCI_USBSTS));
 912         if (new != last) {
 913                 DPRINTFN(10,("ehci_poll: intrs=0x%04x\n", new));
 914                 last = new;
 915         }
 916 #endif
 917
 918         if (EOREAD4(sc, EHCI_USBSTS) & sc->sc_eintrs)
 919                 ehci_intr1(sc);
 920 }
 921
 922 int
 923 ehci_detach(struct ehci_softc *sc, int flags)
 924 {
 925         int rv = 0;
 926
 927         sc->sc_dying = 1;
 928
 929         EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
 930         EOWRITE4(sc, EHCI_USBCMD, 0);
 931         EOWRITE4(sc, EHCI_USBCMD, EHCI_CMD_HCRESET);
 932         callout_stop(&sc->sc_tmo_intrlist);
 933         callout_stop(&sc->sc_tmo_pcd);
 934
 935 #if defined(__NetBSD__) || defined(__OpenBSD__)
 936         if (sc->sc_powerhook != NULL)
 937                 powerhook_disestablish(sc->sc_powerhook);
 938         if (sc->sc_shutdownhook != NULL)
 939                 shutdownhook_disestablish(sc->sc_shutdownhook);
 940 #endif
 941         usb_delay_ms(&sc->sc_bus, 300); /* XXX let stray task complete */
 942
 943         usb_freemem(&sc->sc_bus, &sc->sc_fldma);
 944         /* XXX free other data structures XXX */
 945
 946         return (rv);
 947 }
 948
 949 /*
 950  * Handle suspend/resume.
 951  *
 952  * We need to switch to polling mode here, because this routine is
 953  * called from an interrupt context.  This is all right since we
 954  * are almost suspended anyway.
 955  */
 956 void
 957 ehci_power(int why, void *v)
 958 {
 959         ehci_softc_t *sc = v;
 960         u_int32_t cmd, hcr;
 961         int s, i;
 962
 963 #ifdef EHCI_DEBUG
 964         DPRINTF(("ehci_power: sc=%p, why=%d\n", sc, why));
 965         if (ehcidebug > 0)
 966                 ehci_dump_regs(sc);
 967 #endif
 968
 969         s = splhardusb();
 970         switch (why) {
 971         case PWR_SUSPEND:
 972         case PWR_STANDBY:
 973                 sc->sc_bus.use_polling++;
 974
 975                 for (i = 1; i <= sc->sc_noport; i++) {
 976                         cmd = EOREAD4(sc, EHCI_PORTSC(i));
 977                         if ((cmd & EHCI_PS_PO) == 0 &&
 978                             (cmd & EHCI_PS_PE) == EHCI_PS_PE)
 979                                 EOWRITE4(sc, EHCI_PORTSC(i),
 980                                     cmd | EHCI_PS_SUSP);
 981                 }
 982
 983                 sc->sc_cmd = EOREAD4(sc, EHCI_USBCMD);
 984
 985                 cmd = sc->sc_cmd & ~(EHCI_CMD_ASE | EHCI_CMD_PSE);
 986                 EOWRITE4(sc, EHCI_USBCMD, cmd);
 987
 988                 for (i = 0; i < 100; i++) {
 989                         hcr = EOREAD4(sc, EHCI_USBSTS) &
 990                             (EHCI_STS_ASS | EHCI_STS_PSS);
 991                         if (hcr == 0)
 992                                 break;
 993
 994                         usb_delay_ms(&sc->sc_bus, 1);
 995                 }
 996                 if (hcr != 0) {
 997                         printf("%s: reset timeout\n",
 998                             device_get_nameunit(sc->sc_bus.bdev));
 999                 }
1000
1001                 cmd &= ~EHCI_CMD_RS;
1002                 EOWRITE4(sc, EHCI_USBCMD, cmd);
1003
1004                 for (i = 0; i < 100; i++) {
1005                         hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
1006                         if (hcr == EHCI_STS_HCH)
1007                                 break;
1008
1009                         usb_delay_ms(&sc->sc_bus, 1);
1010                 }
1011                 if (hcr != EHCI_STS_HCH) {
1012                         printf("%s: config timeout\n",
1013                             device_get_nameunit(sc->sc_bus.bdev));
1014                 }
1015
1016                 sc->sc_bus.use_polling--;
1017                 break;
1018
1019         case PWR_RESUME:
1020                 sc->sc_bus.use_polling++;
1021
1022                 /* restore things in case the bios sucks */
1023                 EOWRITE4(sc, EHCI_CTRLDSSEGMENT, 0);
1024                 EOWRITE4(sc, EHCI_PERIODICLISTBASE, DMAADDR(&sc->sc_fldma, 0));
1025                 EOWRITE4(sc, EHCI_ASYNCLISTADDR,
1026                     sc->sc_async_head->physaddr | EHCI_LINK_QH);
1027                 EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
1028
1029                 hcr = 0;
1030                 for (i = 1; i <= sc->sc_noport; i++) {
1031                         cmd = EOREAD4(sc, EHCI_PORTSC(i));
1032                         if ((cmd & EHCI_PS_PO) == 0 &&
1033                             (cmd & EHCI_PS_SUSP) == EHCI_PS_SUSP) {
1034                                 EOWRITE4(sc, EHCI_PORTSC(i),
1035                                     cmd | EHCI_PS_FPR);
1036                                 hcr = 1;
1037                         }
1038                 }
1039
1040                 if (hcr) {
1041                         usb_delay_ms(&sc->sc_bus, USB_RESUME_WAIT);
1042
1043                         for (i = 1; i <= sc->sc_noport; i++) {
1044                                 cmd = EOREAD4(sc, EHCI_PORTSC(i));
1045                                 if ((cmd & EHCI_PS_PO) == 0 &&
1046                                     (cmd & EHCI_PS_SUSP) == EHCI_PS_SUSP)
1047                                         EOWRITE4(sc, EHCI_PORTSC(i),
1048                                             cmd & ~EHCI_PS_FPR);
1049                         }
1050                 }
1051
1052                 EOWRITE4(sc, EHCI_USBCMD, sc->sc_cmd);
1053
1054                 for (i = 0; i < 100; i++) {
1055                         hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
1056                         if (hcr != EHCI_STS_HCH)
1057                                 break;
1058
1059                         usb_delay_ms(&sc->sc_bus, 1);
1060                 }
1061                 if (hcr == EHCI_STS_HCH) {
1062                         printf("%s: config timeout\n",
1063                             device_get_nameunit(sc->sc_bus.bdev));
1064                 }
1065
1066                 usb_delay_ms(&sc->sc_bus, USB_RESUME_WAIT);
1067
1068                 sc->sc_bus.use_polling--;
1069                 break;
1070         case PWR_SOFTSUSPEND:
1071         case PWR_SOFTSTANDBY:
1072         case PWR_SOFTRESUME:
1073                 break;
1074         }
1075         splx(s);
1076
1077 #ifdef EHCI_DEBUG
1078         DPRINTF(("ehci_power: sc=%p\n", sc));
1079         if (ehcidebug > 0)
1080                 ehci_dump_regs(sc);
1081 #endif
1082 }
1083
1084 /*
1085  * Shut down the controller when the system is going down.
1086  */
1087 void
1088 ehci_shutdown(void *v)
1089 {
1090         ehci_softc_t *sc = v;
1091
1092         DPRINTF(("ehci_shutdown: stopping the HC\n"));
1093         EOWRITE4(sc, EHCI_USBCMD, 0);   /* Halt controller */
1094         EOWRITE4(sc, EHCI_USBCMD, EHCI_CMD_HCRESET);
1095 }
1096
1097 usbd_status
1098 ehci_allocm(struct usbd_bus *bus, usb_dma_t *dma, u_int32_t size)
1099 {
1100         usbd_status err;
1101
1102         err = usb_allocmem(bus, size, 0, dma);
1103 #ifdef EHCI_DEBUG
1104         if (err)
1105                 printf("ehci_allocm: usb_allocmem()=%d\n", err);
1106 #endif
1107         return (err);
1108 }
1109
1110 void
1111 ehci_freem(struct usbd_bus *bus, usb_dma_t *dma)
1112 {
1113         usb_freemem(bus, dma);
1114 }
1115
1116 usbd_xfer_handle
1117 ehci_allocx(struct usbd_bus *bus)
1118 {
1119         struct ehci_softc *sc = (struct ehci_softc *)bus;
1120         usbd_xfer_handle xfer;
1121
1122         xfer = STAILQ_FIRST(&sc->sc_free_xfers);
1123         if (xfer != NULL) {
1124                 STAILQ_REMOVE_HEAD(&sc->sc_free_xfers, next);
1125 #ifdef DIAGNOSTIC
1126                 if (xfer->busy_free != XFER_FREE) {
1127                         printf("ehci_allocx: xfer=%p not free, 0x%08x\n", xfer,
1128                                xfer->busy_free);
1129                 }
1130 #endif
1131         } else {
1132                 xfer = malloc(sizeof(struct ehci_xfer), M_USB, M_NOWAIT);
1133         }
1134         if (xfer != NULL) {
1135                 memset(xfer, 0, sizeof(struct ehci_xfer));
1136                 usb_init_task(&EXFER(xfer)->abort_task, ehci_timeout_task,
1137                     xfer);
1138                 EXFER(xfer)->ehci_xfer_flags = 0;
1139 #ifdef DIAGNOSTIC
1140                 EXFER(xfer)->isdone = 1;
1141                 xfer->busy_free = XFER_BUSY;
1142 #endif
1143         }
1144         return (xfer);
1145 }
1146
1147 void
1148 ehci_freex(struct usbd_bus *bus, usbd_xfer_handle xfer)
1149 {
1150         struct ehci_softc *sc = (struct ehci_softc *)bus;
1151
1152 #ifdef DIAGNOSTIC
1153         if (xfer->busy_free != XFER_BUSY) {
1154                 printf("ehci_freex: xfer=%p not busy, 0x%08x\n", xfer,
1155                        xfer->busy_free);
1156                 return;
1157         }
1158         xfer->busy_free = XFER_FREE;
1159         if (!EXFER(xfer)->isdone) {
1160                 printf("ehci_freex: !isdone\n");
1161                 return;
1162         }
1163 #endif
1164         STAILQ_INSERT_HEAD(&sc->sc_free_xfers, xfer, next);
1165 }
1166
1167 static void
1168 ehci_device_clear_toggle(usbd_pipe_handle pipe)
1169 {
1170         struct ehci_pipe *epipe = (struct ehci_pipe *)pipe;
1171
1172         DPRINTF(("ehci_device_clear_toggle: epipe=%p status=0x%x\n",
1173                  epipe, epipe->sqh->qh.qh_qtd.qtd_status));
1174 #ifdef USB_DEBUG
1175         if (ehcidebug)
1176                 usbd_dump_pipe(pipe);
1177 #endif
1178         KASSERT((epipe->sqh->qh.qh_qtd.qtd_status &
1179             htole32(EHCI_QTD_ACTIVE)) == 0,
1180             ("ehci_device_clear_toggle: queue active"));
1181         epipe->sqh->qh.qh_qtd.qtd_status &= htole32(~EHCI_QTD_TOGGLE_MASK);
1182 }
1183
1184 static void
1185 ehci_noop(usbd_pipe_handle pipe)
1186 {
1187 }
1188
1189 #ifdef EHCI_DEBUG
1190 void
1191 ehci_dump_regs(ehci_softc_t *sc)
1192 {
1193         int i;
1194         printf("cmd=0x%08x, sts=0x%08x, ien=0x%08x\n",
1195                EOREAD4(sc, EHCI_USBCMD),
1196                EOREAD4(sc, EHCI_USBSTS),
1197                EOREAD4(sc, EHCI_USBINTR));
1198         printf("frindex=0x%08x ctrdsegm=0x%08x periodic=0x%08x async=0x%08x\n",
1199                EOREAD4(sc, EHCI_FRINDEX),
1200                EOREAD4(sc, EHCI_CTRLDSSEGMENT),
1201                EOREAD4(sc, EHCI_PERIODICLISTBASE),
1202                EOREAD4(sc, EHCI_ASYNCLISTADDR));
1203         for (i = 1; i <= sc->sc_noport; i++)
1204                 printf("port %d status=0x%08x\n", i,
1205                        EOREAD4(sc, EHCI_PORTSC(i)));
1206 }
1207
1208 /*
1209  * Unused function - this is meant to be called from a kernel
1210  * debugger.
1211  */
1212 void
1213 ehci_dump()
1214 {
1215         ehci_dump_regs(theehci);
1216 }
1217
1218 void
1219 ehci_dump_link(ehci_link_t link, int type)
1220 {
1221         link = le32toh(link);
1222         printf("0x%08x", link);
1223         if (link & EHCI_LINK_TERMINATE)
1224                 printf("<T>");
1225         else {
1226                 printf("<");
1227                 if (type) {
1228                         switch (EHCI_LINK_TYPE(link)) {
1229                         case EHCI_LINK_ITD: printf("ITD"); break;
1230                         case EHCI_LINK_QH: printf("QH"); break;
1231                         case EHCI_LINK_SITD: printf("SITD"); break;
1232                         case EHCI_LINK_FSTN: printf("FSTN"); break;
1233                         }
1234                 }
1235                 printf(">");
1236         }
1237 }
1238
1239 void
1240 ehci_dump_sqtds(ehci_soft_qtd_t *sqtd)
1241 {
1242         int i;
1243         u_int32_t stop;
1244
1245         stop = 0;
1246         for (i = 0; sqtd && i < 20 && !stop; sqtd = sqtd->nextqtd, i++) {
1247                 ehci_dump_sqtd(sqtd);
1248                 stop = sqtd->qtd.qtd_next & htole32(EHCI_LINK_TERMINATE);
1249         }
1250         if (sqtd)
1251                 printf("dump aborted, too many TDs\n");
1252 }
1253
1254 void
1255 ehci_dump_sqtd(ehci_soft_qtd_t *sqtd)
1256 {
1257         printf("QTD(%p) at 0x%08x:\n", sqtd, sqtd->physaddr);
1258         ehci_dump_qtd(&sqtd->qtd);
1259 }
1260
1261 void
1262 ehci_dump_qtd(ehci_qtd_t *qtd)
1263 {
1264         u_int32_t s;
1265         char sbuf[128];
1266
1267         printf("  next="); ehci_dump_link(qtd->qtd_next, 0);
1268         printf(" altnext="); ehci_dump_link(qtd->qtd_altnext, 0);
1269         printf("\n");
1270         s = le32toh(qtd->qtd_status);
1271         bitmask_snprintf(EHCI_QTD_GET_STATUS(s),
1272                          "\20\10ACTIVE\7HALTED\6BUFERR\5BABBLE\4XACTERR"
1273                          "\3MISSED\2SPLIT\1PING", sbuf, sizeof(sbuf));
1274         printf("  status=0x%08x: toggle=%d bytes=0x%x ioc=%d c_page=0x%x\n",
1275                s, EHCI_QTD_GET_TOGGLE(s), EHCI_QTD_GET_BYTES(s),
1276                EHCI_QTD_GET_IOC(s), EHCI_QTD_GET_C_PAGE(s));
1277         printf("    cerr=%d pid=%d stat=0x%s\n", EHCI_QTD_GET_CERR(s),
1278                EHCI_QTD_GET_PID(s), sbuf);
1279         for (s = 0; s < 5; s++)
1280                 printf("  buffer[%d]=0x%08x\n", s, le32toh(qtd->qtd_buffer[s]));
1281 }
1282
1283 void
1284 ehci_dump_sqh(ehci_soft_qh_t *sqh)
1285 {
1286         ehci_qh_t *qh = &sqh->qh;
1287         u_int32_t endp, endphub;
1288
1289         printf("QH(%p) at 0x%08x:\n", sqh, sqh->physaddr);
1290         printf("  sqtd=%p inactivesqtd=%p\n", sqh->sqtd, sqh->inactivesqtd);
1291         printf("  link="); ehci_dump_link(qh->qh_link, 1); printf("\n");
1292         endp = le32toh(qh->qh_endp);
1293         printf("  endp=0x%08x\n", endp);
1294         printf("    addr=0x%02x inact=%d endpt=%d eps=%d dtc=%d hrecl=%d\n",
1295                EHCI_QH_GET_ADDR(endp), EHCI_QH_GET_INACT(endp),
1296                EHCI_QH_GET_ENDPT(endp),  EHCI_QH_GET_EPS(endp),
1297                EHCI_QH_GET_DTC(endp), EHCI_QH_GET_HRECL(endp));
1298         printf("    mpl=0x%x ctl=%d nrl=%d\n",
1299                EHCI_QH_GET_MPL(endp), EHCI_QH_GET_CTL(endp),
1300                EHCI_QH_GET_NRL(endp));
1301         endphub = le32toh(qh->qh_endphub);
1302         printf("  endphub=0x%08x\n", endphub);
1303         printf("    smask=0x%02x cmask=0x%02x huba=0x%02x port=%d mult=%d\n",
1304                EHCI_QH_GET_SMASK(endphub), EHCI_QH_GET_CMASK(endphub),
1305                EHCI_QH_GET_HUBA(endphub), EHCI_QH_GET_PORT(endphub),
1306                EHCI_QH_GET_MULT(endphub));
1307         printf("  curqtd="); ehci_dump_link(qh->qh_curqtd, 0); printf("\n");
1308         printf("Overlay qTD:\n");
1309         ehci_dump_qtd(&qh->qh_qtd);
1310 }
1311
1312 #ifdef DIAGNOSTIC
1313 static void
1314 ehci_dump_exfer(struct ehci_xfer *ex)
1315 {
1316         printf("ehci_dump_exfer: ex=%p\n", ex);
1317 }
1318 #endif
1319 #endif
1320
1321 usbd_status
1322 ehci_open(usbd_pipe_handle pipe)
1323 {
1324         usbd_device_handle dev = pipe->device;
1325         ehci_softc_t *sc = (ehci_softc_t *)dev->bus;
1326         usb_endpoint_descriptor_t *ed = pipe->endpoint->edesc;
1327         u_int8_t addr = dev->address;
1328         u_int8_t xfertype = ed->bmAttributes & UE_XFERTYPE;
1329         struct ehci_pipe *epipe = (struct ehci_pipe *)pipe;
1330         ehci_soft_qh_t *sqh;
1331         usbd_status err;
1332         int s;
1333         int ival, speed, naks;
1334         int hshubaddr, hshubport;
1335
1336         DPRINTFN(1, ("ehci_open: pipe=%p, addr=%d, endpt=%d (%d)\n",
1337                      pipe, addr, ed->bEndpointAddress, sc->sc_addr));
1338
1339         if (dev->myhsport) {
1340                 hshubaddr = dev->myhsport->parent->address;
1341                 hshubport = dev->myhsport->portno;
1342         } else {
1343                 hshubaddr = 0;
1344                 hshubport = 0;
1345         }
1346
1347         if (sc->sc_dying)
1348                 return (USBD_IOERROR);
1349
1350         if (addr == sc->sc_addr) {
1351                 switch (ed->bEndpointAddress) {
1352                 case USB_CONTROL_ENDPOINT:
1353                         pipe->methods = &ehci_root_ctrl_methods;
1354                         break;
1355                 case UE_DIR_IN | EHCI_INTR_ENDPT:
1356                         pipe->methods = &ehci_root_intr_methods;
1357                         break;
1358                 default:
1359                         return (USBD_INVAL);
1360                 }
1361                 return (USBD_NORMAL_COMPLETION);
1362         }
1363
1364         /* XXX All this stuff is only valid for async. */
1365         switch (dev->speed) {
1366         case USB_SPEED_LOW:  speed = EHCI_QH_SPEED_LOW;  break;
1367         case USB_SPEED_FULL: speed = EHCI_QH_SPEED_FULL; break;
1368         case USB_SPEED_HIGH: speed = EHCI_QH_SPEED_HIGH; break;
1369         default: panic("ehci_open: bad device speed %d", dev->speed);
1370         }
1371         if (speed != EHCI_QH_SPEED_HIGH && xfertype == UE_ISOCHRONOUS) {
1372                 printf("%s: *** WARNING: opening low/full speed device, this "
1373                        "does not work yet.\n",
1374                        device_get_nameunit(sc->sc_bus.bdev));
1375                 DPRINTFN(1,("ehci_open: hshubaddr=%d hshubport=%d\n",
1376                             hshubaddr, hshubport));
1377                 return USBD_INVAL;
1378         }
1379
1380         naks = 8;               /* XXX */
1381         sqh = ehci_alloc_sqh(sc);
1382         if (sqh == NULL)
1383                 goto bad0;
1384         /* qh_link filled when the QH is added */
1385         sqh->qh.qh_endp = htole32(
1386                 EHCI_QH_SET_ADDR(addr) |
1387                 EHCI_QH_SET_ENDPT(UE_GET_ADDR(ed->bEndpointAddress)) |
1388                 EHCI_QH_SET_EPS(speed) |
1389                 (xfertype == UE_CONTROL ? EHCI_QH_DTC : 0) |
1390                 EHCI_QH_SET_MPL(UGETW(ed->wMaxPacketSize)) |
1391                 (speed != EHCI_QH_SPEED_HIGH && xfertype == UE_CONTROL ?
1392                  EHCI_QH_CTL : 0) |
1393                 EHCI_QH_SET_NRL(naks)
1394                 );
1395         sqh->qh.qh_endphub = htole32(
1396                 EHCI_QH_SET_MULT(1) |
1397                 EHCI_QH_SET_HUBA(hshubaddr) |
1398                 EHCI_QH_SET_PORT(hshubport) |
1399                 EHCI_QH_SET_CMASK(0x1c) |
1400                 EHCI_QH_SET_SMASK(xfertype == UE_INTERRUPT ? 0x01 : 0)
1401                 );
1402         sqh->qh.qh_curqtd = EHCI_NULL;
1403         /* The overlay qTD was already set up by ehci_alloc_sqh(). */
1404         sqh->qh.qh_qtd.qtd_status =
1405             htole32(EHCI_QTD_SET_TOGGLE(pipe->endpoint->savedtoggle));
1406
1407         epipe->sqh = sqh;
1408
1409         switch (xfertype) {
1410         case UE_CONTROL:
1411                 err = usb_allocmem(&sc->sc_bus, sizeof(usb_device_request_t),
1412                                    0, &epipe->u.ctl.reqdma);
1413 #ifdef EHCI_DEBUG
1414                 if (err)
1415                         printf("ehci_open: usb_allocmem()=%d\n", err);
1416 #endif
1417                 if (err)
1418                         goto bad1;
1419                 pipe->methods = &ehci_device_ctrl_methods;
1420                 s = splusb();
1421                 ehci_add_qh(sqh, sc->sc_async_head);
1422                 splx(s);
1423                 break;
1424         case UE_BULK:
1425                 pipe->methods = &ehci_device_bulk_methods;
1426                 s = splusb();
1427                 ehci_add_qh(sqh, sc->sc_async_head);
1428                 splx(s);
1429                 break;
1430         case UE_INTERRUPT:
1431                 pipe->methods = &ehci_device_intr_methods;
1432                 ival = pipe->interval;
1433                 if (ival == USBD_DEFAULT_INTERVAL)
1434                         ival = ed->bInterval;
1435                 return (ehci_device_setintr(sc, sqh, ival));
1436         case UE_ISOCHRONOUS:
1437                 pipe->methods = &ehci_device_isoc_methods;
1438                 return (USBD_INVAL);
1439         default:
1440                 return (USBD_INVAL);
1441         }
1442         return (USBD_NORMAL_COMPLETION);
1443
1444  bad1:
1445         ehci_free_sqh(sc, sqh);
1446  bad0:
1447         return (USBD_NOMEM);
1448 }
1449
1450 /*
1451  * Add an ED to the schedule.  Called at splusb().
1452  * If in the async schedule, it will always have a next.
1453  * If in the intr schedule it may not.
1454  */
1455 void
1456 ehci_add_qh(ehci_soft_qh_t *sqh, ehci_soft_qh_t *head)
1457 {
1458         SPLUSBCHECK;
1459
1460         sqh->next = head->next;
1461         sqh->prev = head;
1462         sqh->qh.qh_link = head->qh.qh_link;
1463         head->next = sqh;
1464         if (sqh->next)
1465                 sqh->next->prev = sqh;
1466         head->qh.qh_link = htole32(sqh->physaddr | EHCI_LINK_QH);
1467
1468 #ifdef EHCI_DEBUG
1469         if (ehcidebug > 5) {
1470                 printf("ehci_add_qh:\n");
1471                 ehci_dump_sqh(sqh);
1472         }
1473 #endif
1474 }
1475
1476 /*
1477  * Remove an ED from the schedule.  Called at splusb().
1478  * Will always have a 'next' if it's in the async list as it's circular.
1479  */
1480 void
1481 ehci_rem_qh(ehci_softc_t *sc, ehci_soft_qh_t *sqh, ehci_soft_qh_t *head)
1482 {
1483         SPLUSBCHECK;
1484         /* XXX */
1485         sqh->prev->qh.qh_link = sqh->qh.qh_link;
1486         sqh->prev->next = sqh->next;
1487         if (sqh->next)
1488                 sqh->next->prev = sqh->prev;
1489         ehci_sync_hc(sc);
1490 }
1491
1492 /* Restart a QH following the addition of a qTD. */
1493 void
1494 ehci_activate_qh(ehci_soft_qh_t *sqh, ehci_soft_qtd_t *sqtd)
1495 {
1496         KASSERT((sqtd->qtd.qtd_status & htole32(EHCI_QTD_ACTIVE)) == 0,
1497             ("ehci_activate_qh: already active"));
1498
1499         /*
1500          * When a QH is idle, the overlay qTD should be marked as not
1501          * halted and not active. This causes the host controller to
1502          * retrieve the real qTD on each pass (rather than just examinig
1503          * the overlay), so it will notice when we activate the qTD.
1504          */
1505         if (sqtd == sqh->sqtd) {
1506                 /* Check that the hardware is in the state we expect. */
1507                 if (EHCI_LINK_ADDR(le32toh(sqh->qh.qh_qtd.qtd_next)) !=
1508                     sqtd->physaddr) {
1509 #ifdef EHCI_DEBUG
1510                         printf("ehci_activate_qh: unexpected next ptr\n");
1511                         ehci_dump_sqh(sqh);
1512                         ehci_dump_sqtds(sqh->sqtd);
1513 #endif
1514                         sqh->qh.qh_qtd.qtd_next = htole32(sqtd->physaddr);
1515                         sqh->qh.qh_qtd.qtd_altnext = EHCI_NULL;
1516                 }
1517                 /* Ensure the flags are correct. */
1518                 sqh->qh.qh_qtd.qtd_status &= htole32(EHCI_QTD_PINGSTATE |
1519                     EHCI_QTD_TOGGLE_MASK);
1520         }
1521
1522         /* Now activate the qTD. */
1523         sqtd->qtd.qtd_status |= htole32(EHCI_QTD_ACTIVE);
1524 }
1525
1526 /*
1527  * Ensure that the HC has released all references to the QH.  We do this
1528  * by asking for a Async Advance Doorbell interrupt and then we wait for
1529  * the interrupt.
1530  * To make this easier we first obtain exclusive use of the doorbell.
1531  */
1532 void
1533 ehci_sync_hc(ehci_softc_t *sc)
1534 {
1535         int s, error;
1536
1537         if (sc->sc_dying) {
1538                 DPRINTFN(2,("ehci_sync_hc: dying\n"));
1539                 return;
1540         }
1541         DPRINTFN(2,("ehci_sync_hc: enter\n"));
1542         /* get doorbell */
1543         lockmgr(&sc->sc_doorbell_lock, LK_EXCLUSIVE, NULL, NULL);
1544         s = splhardusb();
1545         /* ask for doorbell */
1546         EOWRITE4(sc, EHCI_USBCMD, EOREAD4(sc, EHCI_USBCMD) | EHCI_CMD_IAAD);
1547         DPRINTFN(1,("ehci_sync_hc: cmd=0x%08x sts=0x%08x\n",
1548                     EOREAD4(sc, EHCI_USBCMD), EOREAD4(sc, EHCI_USBSTS)));
1549         error = tsleep(&sc->sc_async_head, PZERO, "ehcidi", hz); /* bell wait */
1550         DPRINTFN(1,("ehci_sync_hc: cmd=0x%08x sts=0x%08x\n",
1551                     EOREAD4(sc, EHCI_USBCMD), EOREAD4(sc, EHCI_USBSTS)));
1552         splx(s);
1553         /* release doorbell */
1554         lockmgr(&sc->sc_doorbell_lock, LK_RELEASE, NULL, NULL);
1555 #ifdef DIAGNOSTIC
1556         if (error)
1557                 printf("ehci_sync_hc: tsleep() = %d\n", error);
1558 #endif
1559         DPRINTFN(2,("ehci_sync_hc: exit\n"));
1560 }
1561
1562 /***********/
1563
1564 /*
1565  * Data structures and routines to emulate the root hub.
1566  */
1567 static usb_device_descriptor_t ehci_devd = {
1568         USB_DEVICE_DESCRIPTOR_SIZE,
1569         UDESC_DEVICE,           /* type */
1570         {0x00, 0x02},           /* USB version */
1571         UDCLASS_HUB,            /* class */
1572         UDSUBCLASS_HUB,         /* subclass */
1573         UDPROTO_HSHUBSTT,       /* protocol */
1574         64,                     /* max packet */
1575         {0},{0},{0x00,0x01},    /* device id */
1576         1,2,0,                  /* string indicies */
1577         1                       /* # of configurations */
1578 };
1579
1580 static usb_device_qualifier_t ehci_odevd = {
1581         USB_DEVICE_DESCRIPTOR_SIZE,
1582         UDESC_DEVICE_QUALIFIER, /* type */
1583         {0x00, 0x02},           /* USB version */
1584         UDCLASS_HUB,            /* class */
1585         UDSUBCLASS_HUB,         /* subclass */
1586         UDPROTO_FSHUB,          /* protocol */
1587         64,                     /* max packet */
1588         1,                      /* # of configurations */
1589         0
1590 };
1591
1592 static usb_config_descriptor_t ehci_confd = {
1593         USB_CONFIG_DESCRIPTOR_SIZE,
1594         UDESC_CONFIG,
1595         {USB_CONFIG_DESCRIPTOR_SIZE +
1596          USB_INTERFACE_DESCRIPTOR_SIZE +
1597          USB_ENDPOINT_DESCRIPTOR_SIZE},
1598         1,
1599         1,
1600         0,
1601         UC_SELF_POWERED,
1602         0                       /* max power */
1603 };
1604
1605 static usb_interface_descriptor_t ehci_ifcd = {
1606         USB_INTERFACE_DESCRIPTOR_SIZE,
1607         UDESC_INTERFACE,
1608         0,
1609         0,
1610         1,
1611         UICLASS_HUB,
1612         UISUBCLASS_HUB,
1613         UIPROTO_HSHUBSTT,
1614         0
1615 };
1616
1617 static usb_endpoint_descriptor_t ehci_endpd = {
1618         USB_ENDPOINT_DESCRIPTOR_SIZE,
1619         UDESC_ENDPOINT,
1620         UE_DIR_IN | EHCI_INTR_ENDPT,
1621         UE_INTERRUPT,
1622         {8, 0},                 /* max packet */
1623         255
1624 };
1625
1626 static usb_hub_descriptor_t ehci_hubd = {
1627         USB_HUB_DESCRIPTOR_SIZE,
1628         UDESC_HUB,
1629         0,
1630         {0,0},
1631         0,
1632         0,
1633         {0},
1634 };
1635
1636 static int
1637 ehci_str(usb_string_descriptor_t *p, int l, char *s)
1638 {
1639         int i;
1640
1641         if (l == 0)
1642                 return (0);
1643         p->bLength = 2 * strlen(s) + 2;
1644         if (l == 1)
1645                 return (1);
1646         p->bDescriptorType = UDESC_STRING;
1647         l -= 2;
1648         for (i = 0; s[i] && l > 1; i++, l -= 2)
1649                 USETW2(p->bString[i], 0, s[i]);
1650         return (2*i+2);
1651 }
1652
1653 /*
1654  * Simulate a hardware hub by handling all the necessary requests.
1655  */
1656 static usbd_status
1657 ehci_root_ctrl_transfer(usbd_xfer_handle xfer)
1658 {
1659         usbd_status err;
1660
1661         /* Insert last in queue. */
1662         err = usb_insert_transfer(xfer);
1663         if (err)
1664                 return (err);
1665
1666         /* Pipe isn't running, start first */
1667         return (ehci_root_ctrl_start(STAILQ_FIRST(&xfer->pipe->queue)));
1668 }
1669
1670 static usbd_status
1671 ehci_root_ctrl_start(usbd_xfer_handle xfer)
1672 {
1673         ehci_softc_t *sc = (ehci_softc_t *)xfer->pipe->device->bus;
1674         usb_device_request_t *req;
1675         void *buf = NULL;
1676         int port, i;
1677         int s, len, value, index, l, totlen = 0;
1678         usb_port_status_t ps;
1679         usb_hub_descriptor_t hubd;
1680         usbd_status err;
1681         u_int32_t v;
1682
1683         if (sc->sc_dying)
1684                 return (USBD_IOERROR);
1685
1686 #ifdef DIAGNOSTIC
1687         if (!(xfer->rqflags & URQ_REQUEST))
1688                 /* XXX panic */
1689                 return (USBD_INVAL);
1690 #endif
1691         req = &xfer->request;
1692
1693         DPRINTFN(4,("ehci_root_ctrl_start: type=0x%02x request=%02x\n",
1694                     req->bmRequestType, req->bRequest));
1695
1696         len = UGETW(req->wLength);
1697         value = UGETW(req->wValue);
1698         index = UGETW(req->wIndex);
1699
1700         if (len != 0)
1701                 buf = xfer->buffer;
1702
1703 #define C(x,y) ((x) | ((y) << 8))
1704         switch(C(req->bRequest, req->bmRequestType)) {
1705         case C(UR_CLEAR_FEATURE, UT_WRITE_DEVICE):
1706         case C(UR_CLEAR_FEATURE, UT_WRITE_INTERFACE):
1707         case C(UR_CLEAR_FEATURE, UT_WRITE_ENDPOINT):
1708                 /*
1709                  * DEVICE_REMOTE_WAKEUP and ENDPOINT_HALT are no-ops
1710                  * for the integrated root hub.
1711                  */
1712                 break;
1713         case C(UR_GET_CONFIG, UT_READ_DEVICE):
1714                 if (len > 0) {
1715                         *(u_int8_t *)buf = sc->sc_conf;
1716                         totlen = 1;
1717                 }
1718                 break;
1719         case C(UR_GET_DESCRIPTOR, UT_READ_DEVICE):
1720                 DPRINTFN(8,("ehci_root_ctrl_start: wValue=0x%04x\n", value));
1721                 switch(value >> 8) {
1722                 case UDESC_DEVICE:
1723                         if ((value & 0xff) != 0) {
1724                                 err = USBD_IOERROR;
1725                                 goto ret;
1726                         }
1727                         totlen = l = min(len, USB_DEVICE_DESCRIPTOR_SIZE);
1728                         USETW(ehci_devd.idVendor, sc->sc_id_vendor);
1729                         memcpy(buf, &ehci_devd, l);
1730                         break;
1731                 /*
1732                  * We can't really operate at another speed, but the spec says
1733                  * we need this descriptor.
1734                  */
1735                 case UDESC_DEVICE_QUALIFIER:
1736                         if ((value & 0xff) != 0) {
1737                                 err = USBD_IOERROR;
1738                                 goto ret;
1739                         }
1740                         totlen = l = min(len, USB_DEVICE_DESCRIPTOR_SIZE);
1741                         memcpy(buf, &ehci_odevd, l);
1742                         break;
1743                 /*
1744                  * We can't really operate at another speed, but the spec says
1745                  * we need this descriptor.
1746                  */
1747                 case UDESC_OTHER_SPEED_CONFIGURATION:
1748                 case UDESC_CONFIG:
1749                         if ((value & 0xff) != 0) {
1750                                 err = USBD_IOERROR;
1751                                 goto ret;
1752                         }
1753                         totlen = l = min(len, USB_CONFIG_DESCRIPTOR_SIZE);
1754                         memcpy(buf, &ehci_confd, l);
1755                         ((usb_config_descriptor_t *)buf)->bDescriptorType =
1756                                 value >> 8;
1757                         buf = (char *)buf + l;
1758                         len -= l;
1759                         l = min(len, USB_INTERFACE_DESCRIPTOR_SIZE);
1760                         totlen += l;
1761                         memcpy(buf, &ehci_ifcd, l);
1762                         buf = (char *)buf + l;
1763                         len -= l;
1764                         l = min(len, USB_ENDPOINT_DESCRIPTOR_SIZE);
1765                         totlen += l;
1766                         memcpy(buf, &ehci_endpd, l);
1767                         break;
1768                 case UDESC_STRING:
1769                         if (len == 0)
1770                                 break;
1771                         *(u_int8_t *)buf = 0;
1772                         totlen = 1;
1773                         switch (value & 0xff) {
1774                         case 0: /* Language table */
1775                                 totlen = ehci_str(buf, len, "\001");
1776                                 break;
1777                         case 1: /* Vendor */
1778                                 totlen = ehci_str(buf, len, sc->sc_vendor);
1779                                 break;
1780                         case 2: /* Product */
1781                                 totlen = ehci_str(buf, len, "EHCI root hub");
1782                                 break;
1783                         }
1784                         break;
1785                 default:
1786                         err = USBD_IOERROR;
1787                         goto ret;
1788                 }
1789                 break;
1790         case C(UR_GET_INTERFACE, UT_READ_INTERFACE):
1791                 if (len > 0) {
1792                         *(u_int8_t *)buf = 0;
1793                         totlen = 1;
1794                 }
1795                 break;
1796         case C(UR_GET_STATUS, UT_READ_DEVICE):
1797                 if (len > 1) {
1798                         USETW(((usb_status_t *)buf)->wStatus,UDS_SELF_POWERED);
1799                         totlen = 2;
1800                 }
1801                 break;
1802         case C(UR_GET_STATUS, UT_READ_INTERFACE):
1803         case C(UR_GET_STATUS, UT_READ_ENDPOINT):
1804                 if (len > 1) {
1805                         USETW(((usb_status_t *)buf)->wStatus, 0);
1806                         totlen = 2;
1807                 }
1808                 break;
1809         case C(UR_SET_ADDRESS, UT_WRITE_DEVICE):
1810                 if (value >= USB_MAX_DEVICES) {
1811                         err = USBD_IOERROR;
1812                         goto ret;
1813                 }
1814                 sc->sc_addr = value;
1815                 break;
1816         case C(UR_SET_CONFIG, UT_WRITE_DEVICE):
1817                 if (value != 0 && value != 1) {
1818                         err = USBD_IOERROR;
1819                         goto ret;
1820                 }
1821                 sc->sc_conf = value;
1822                 break;
1823         case C(UR_SET_DESCRIPTOR, UT_WRITE_DEVICE):
1824                 break;
1825         case C(UR_SET_FEATURE, UT_WRITE_DEVICE):
1826         case C(UR_SET_FEATURE, UT_WRITE_INTERFACE):
1827         case C(UR_SET_FEATURE, UT_WRITE_ENDPOINT):
1828                 err = USBD_IOERROR;
1829                 goto ret;
1830         case C(UR_SET_INTERFACE, UT_WRITE_INTERFACE):
1831                 break;
1832         case C(UR_SYNCH_FRAME, UT_WRITE_ENDPOINT):
1833                 break;
1834         /* Hub requests */
1835         case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_DEVICE):
1836                 break;
1837         case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_OTHER):
1838                 DPRINTFN(8, ("ehci_root_ctrl_start: UR_CLEAR_PORT_FEATURE "
1839                              "port=%d feature=%d\n",
1840                              index, value));
1841                 if (index < 1 || index > sc->sc_noport) {
1842                         err = USBD_IOERROR;
1843                         goto ret;
1844                 }
1845                 port = EHCI_PORTSC(index);
1846                 v = EOREAD4(sc, port) &~ EHCI_PS_CLEAR;
1847                 switch(value) {
1848                 case UHF_PORT_ENABLE:
1849                         EOWRITE4(sc, port, v &~ EHCI_PS_PE);
1850                         break;
1851                 case UHF_PORT_SUSPEND:
1852                         EOWRITE4(sc, port, v &~ EHCI_PS_SUSP);
1853                         break;
1854                 case UHF_PORT_POWER:
1855                         EOWRITE4(sc, port, v &~ EHCI_PS_PP);
1856                         break;
1857                 case UHF_PORT_TEST:
1858                         DPRINTFN(2,("ehci_root_ctrl_start: clear port test "
1859                                     "%d\n", index));
1860                         break;
1861                 case UHF_PORT_INDICATOR:
1862                         DPRINTFN(2,("ehci_root_ctrl_start: clear port ind "
1863                                     "%d\n", index));
1864                         EOWRITE4(sc, port, v &~ EHCI_PS_PIC);
1865                         break;
1866                 case UHF_C_PORT_CONNECTION:
1867                         EOWRITE4(sc, port, v | EHCI_PS_CSC);
1868                         break;
1869                 case UHF_C_PORT_ENABLE:
1870                         EOWRITE4(sc, port, v | EHCI_PS_PEC);
1871                         break;
1872                 case UHF_C_PORT_SUSPEND:
1873                         /* how? */
1874                         break;
1875                 case UHF_C_PORT_OVER_CURRENT:
1876                         EOWRITE4(sc, port, v | EHCI_PS_OCC);
1877                         break;
1878                 case UHF_C_PORT_RESET:
1879                         sc->sc_isreset = 0;
1880                         break;
1881                 default:
1882                         err = USBD_IOERROR;
1883                         goto ret;
1884                 }
1885 #if 0
1886                 switch(value) {
1887                 case UHF_C_PORT_CONNECTION:
1888                 case UHF_C_PORT_ENABLE:
1889                 case UHF_C_PORT_SUSPEND:
1890                 case UHF_C_PORT_OVER_CURRENT:
1891                 case UHF_C_PORT_RESET:
1892                         /* Enable RHSC interrupt if condition is cleared. */
1893                         if ((OREAD4(sc, port) >> 16) == 0)
1894                                 ehci_pcd_able(sc, 1);
1895                         break;
1896                 default:
1897                         break;
1898                 }
1899 #endif
1900                 break;
1901         case C(UR_GET_DESCRIPTOR, UT_READ_CLASS_DEVICE):
1902                 if ((value & 0xff) != 0) {
1903                         err = USBD_IOERROR;
1904                         goto ret;
1905                 }
1906                 hubd = ehci_hubd;
1907                 hubd.bNbrPorts = sc->sc_noport;
1908                 v = EOREAD4(sc, EHCI_HCSPARAMS);
1909                 USETW(hubd.wHubCharacteristics,
1910                     EHCI_HCS_PPC(v) ? UHD_PWR_INDIVIDUAL : UHD_PWR_NO_SWITCH |
1911                     EHCI_HCS_P_INDICATOR(EREAD4(sc, EHCI_HCSPARAMS))
1912                         ? UHD_PORT_IND : 0);
1913                 hubd.bPwrOn2PwrGood = 200; /* XXX can't find out? */
1914                 for (i = 0, l = sc->sc_noport; l > 0; i++, l -= 8, v >>= 8)
1915                         hubd.DeviceRemovable[i++] = 0; /* XXX can't find out? */
1916                 hubd.bDescLength = USB_HUB_DESCRIPTOR_SIZE + i;
1917                 l = min(len, hubd.bDescLength);
1918                 totlen = l;
1919                 memcpy(buf, &hubd, l);
1920                 break;
1921         case C(UR_GET_STATUS, UT_READ_CLASS_DEVICE):
1922                 if (len != 4) {
1923                         err = USBD_IOERROR;
1924                         goto ret;
1925                 }
1926                 memset(buf, 0, len); /* ? XXX */
1927                 totlen = len;
1928                 break;
1929         case C(UR_GET_STATUS, UT_READ_CLASS_OTHER):
1930                 DPRINTFN(8,("ehci_root_ctrl_start: get port status i=%d\n",
1931                             index));
1932                 if (index < 1 || index > sc->sc_noport) {
1933                         err = USBD_IOERROR;
1934                         goto ret;
1935                 }
1936                 if (len != 4) {
1937                         err = USBD_IOERROR;
1938                         goto ret;
1939                 }
1940                 v = EOREAD4(sc, EHCI_PORTSC(index));
1941                 DPRINTFN(8,("ehci_root_ctrl_start: port status=0x%04x\n",
1942                             v));
1943                 i = UPS_HIGH_SPEED;
1944                 if (v & EHCI_PS_CS)     i |= UPS_CURRENT_CONNECT_STATUS;
1945                 if (v & EHCI_PS_PE)     i |= UPS_PORT_ENABLED;
1946                 if (v & EHCI_PS_SUSP)   i |= UPS_SUSPEND;
1947                 if (v & EHCI_PS_OCA)    i |= UPS_OVERCURRENT_INDICATOR;
1948                 if (v & EHCI_PS_PR)     i |= UPS_RESET;
1949                 if (v & EHCI_PS_PP)     i |= UPS_PORT_POWER;
1950                 USETW(ps.wPortStatus, i);
1951                 i = 0;
1952                 if (v & EHCI_PS_CSC)    i |= UPS_C_CONNECT_STATUS;
1953                 if (v & EHCI_PS_PEC)    i |= UPS_C_PORT_ENABLED;
1954                 if (v & EHCI_PS_OCC)    i |= UPS_C_OVERCURRENT_INDICATOR;
1955                 if (sc->sc_isreset)     i |= UPS_C_PORT_RESET;
1956                 USETW(ps.wPortChange, i);
1957                 l = min(len, sizeof ps);
1958                 memcpy(buf, &ps, l);
1959                 totlen = l;
1960                 break;
1961         case C(UR_SET_DESCRIPTOR, UT_WRITE_CLASS_DEVICE):
1962                 err = USBD_IOERROR;
1963                 goto ret;
1964         case C(UR_SET_FEATURE, UT_WRITE_CLASS_DEVICE):
1965                 break;
1966         case C(UR_SET_FEATURE, UT_WRITE_CLASS_OTHER):
1967                 if (index < 1 || index > sc->sc_noport) {
1968                         err = USBD_IOERROR;
1969                         goto ret;
1970                 }
1971                 port = EHCI_PORTSC(index);
1972                 v = EOREAD4(sc, port) &~ EHCI_PS_CLEAR;
1973                 switch(value) {
1974                 case UHF_PORT_ENABLE:
1975                         EOWRITE4(sc, port, v | EHCI_PS_PE);
1976                         break;
1977                 case UHF_PORT_SUSPEND:
1978                         EOWRITE4(sc, port, v | EHCI_PS_SUSP);
1979                         break;
1980                 case UHF_PORT_RESET:
1981                         DPRINTFN(5,("ehci_root_ctrl_start: reset port %d\n",
1982                                     index));
1983                         if (EHCI_PS_IS_LOWSPEED(v)) {
1984                                 /* Low speed device, give up ownership. */
1985                                 ehci_disown(sc, index, 1);
1986                                 break;
1987                         }
1988                         /* Start reset sequence. */
1989                         v &= ~ (EHCI_PS_PE | EHCI_PS_PR);
1990                         EOWRITE4(sc, port, v | EHCI_PS_PR);
1991                         /* Wait for reset to complete. */
1992                         usb_delay_ms(&sc->sc_bus, USB_PORT_ROOT_RESET_DELAY);
1993                         if (sc->sc_dying) {
1994                                 err = USBD_IOERROR;
1995                                 goto ret;
1996                         }
1997                         /* Terminate reset sequence. */
1998                         EOWRITE4(sc, port, v);
1999                         /* Wait for HC to complete reset. */
2000                         usb_delay_ms(&sc->sc_bus, EHCI_PORT_RESET_COMPLETE);
2001                         if (sc->sc_dying) {
2002                                 err = USBD_IOERROR;
2003                                 goto ret;
2004                         }
2005                         v = EOREAD4(sc, port);
2006                         DPRINTF(("ehci after reset, status=0x%08x\n", v));
2007                         if (v & EHCI_PS_PR) {
2008                                 printf("%s: port reset timeout\n",
2009                                        device_get_nameunit(sc->sc_bus.bdev));
2010                                 return (USBD_TIMEOUT);
2011                         }
2012                         if (!(v & EHCI_PS_PE)) {
2013                                 /* Not a high speed device, give up ownership.*/
2014                                 ehci_disown(sc, index, 0);
2015                                 break;
2016                         }
2017                         sc->sc_isreset = 1;
2018                         DPRINTF(("ehci port %d reset, status = 0x%08x\n",
2019                                  index, v));
2020                         break;
2021                 case UHF_PORT_POWER:
2022                         DPRINTFN(2,("ehci_root_ctrl_start: set port power "
2023                                     "%d\n", index));
2024                         EOWRITE4(sc, port, v | EHCI_PS_PP);
2025                         break;
2026                 case UHF_PORT_TEST:
2027                         DPRINTFN(2,("ehci_root_ctrl_start: set port test "
2028                                     "%d\n", index));
2029                         break;
2030                 case UHF_PORT_INDICATOR:
2031                         DPRINTFN(2,("ehci_root_ctrl_start: set port ind "
2032                                     "%d\n", index));
2033                         EOWRITE4(sc, port, v | EHCI_PS_PIC);
2034                         break;
2035                 default:
2036                         err = USBD_IOERROR;
2037                         goto ret;
2038                 }
2039                 break;
2040         case C(UR_CLEAR_TT_BUFFER, UT_WRITE_CLASS_OTHER):
2041         case C(UR_RESET_TT, UT_WRITE_CLASS_OTHER):
2042         case C(UR_GET_TT_STATE, UT_READ_CLASS_OTHER):
2043         case C(UR_STOP_TT, UT_WRITE_CLASS_OTHER):
2044                 break;
2045         default:
2046                 err = USBD_IOERROR;
2047                 goto ret;
2048         }
2049         xfer->actlen = totlen;
2050         err = USBD_NORMAL_COMPLETION;
2051  ret:
2052         xfer->status = err;
2053         s = splusb();
2054         usb_transfer_complete(xfer);
2055         splx(s);
2056         return (USBD_IN_PROGRESS);
2057 }
2058
2059 void
2060 ehci_disown(ehci_softc_t *sc, int index, int lowspeed)
2061 {
2062         int port;
2063         u_int32_t v;
2064
2065         DPRINTF(("ehci_disown: index=%d lowspeed=%d\n", index, lowspeed));
2066 #ifdef DIAGNOSTIC
2067         if (sc->sc_npcomp != 0) {
2068                 int i = (index-1) / sc->sc_npcomp;
2069                 if (i >= sc->sc_ncomp)
2070                         printf("%s: strange port\n",
2071                                device_get_nameunit(sc->sc_bus.bdev));
2072                 else
2073                         printf("%s: handing over %s speed device on "
2074                                "port %d to %s\n",
2075                                device_get_nameunit(sc->sc_bus.bdev),
2076                                lowspeed ? "low" : "full",
2077                                index, device_get_nameunit(sc->sc_comps[i]->bdev));
2078         } else {
2079                 printf("%s: npcomp == 0\n", device_get_nameunit(sc->sc_bus.bdev));
2080         }
2081 #endif
2082         port = EHCI_PORTSC(index);
2083         v = EOREAD4(sc, port) &~ EHCI_PS_CLEAR;
2084         EOWRITE4(sc, port, v | EHCI_PS_PO);
2085 }
2086
2087 /* Abort a root control request. */
2088 static void
2089 ehci_root_ctrl_abort(usbd_xfer_handle xfer)
2090 {
2091         /* Nothing to do, all transfers are synchronous. */
2092 }
2093
2094 /* Close the root pipe. */
2095 static void
2096 ehci_root_ctrl_close(usbd_pipe_handle pipe)
2097 {
2098         DPRINTF(("ehci_root_ctrl_close\n"));
2099         /* Nothing to do. */
2100 }
2101
2102 void
2103 ehci_root_intr_done(usbd_xfer_handle xfer)
2104 {
2105 }
2106
2107 static usbd_status
2108 ehci_root_intr_transfer(usbd_xfer_handle xfer)
2109 {
2110         usbd_status err;
2111
2112         /* Insert last in queue. */
2113         err = usb_insert_transfer(xfer);
2114         if (err)
2115                 return (err);
2116
2117         /* Pipe isn't running, start first */
2118         return (ehci_root_intr_start(STAILQ_FIRST(&xfer->pipe->queue)));
2119 }
2120
2121 static usbd_status
2122 ehci_root_intr_start(usbd_xfer_handle xfer)
2123 {
2124         usbd_pipe_handle pipe = xfer->pipe;
2125         ehci_softc_t *sc = (ehci_softc_t *)pipe->device->bus;
2126
2127         if (sc->sc_dying)
2128                 return (USBD_IOERROR);
2129
2130         sc->sc_intrxfer = xfer;
2131
2132         return (USBD_IN_PROGRESS);
2133 }
2134
2135 /* Abort a root interrupt request. */
2136 static void
2137 ehci_root_intr_abort(usbd_xfer_handle xfer)
2138 {
2139         int s;
2140
2141         if (xfer->pipe->intrxfer == xfer) {
2142                 DPRINTF(("ehci_root_intr_abort: remove\n"));
2143                 xfer->pipe->intrxfer = NULL;
2144         }
2145         xfer->status = USBD_CANCELLED;
2146         s = splusb();
2147         usb_transfer_complete(xfer);
2148         splx(s);
2149 }
2150
2151 /* Close the root pipe. */
2152 static void
2153 ehci_root_intr_close(usbd_pipe_handle pipe)
2154 {
2155         ehci_softc_t *sc = (ehci_softc_t *)pipe->device->bus;
2156
2157         DPRINTF(("ehci_root_intr_close\n"));
2158
2159         sc->sc_intrxfer = NULL;
2160 }
2161
2162 void
2163 ehci_root_ctrl_done(usbd_xfer_handle xfer)
2164 {
2165 }
2166
2167 /************************/
2168
2169 ehci_soft_qh_t *
2170 ehci_alloc_sqh(ehci_softc_t *sc)
2171 {
2172         ehci_soft_qh_t *sqh;
2173         ehci_soft_qtd_t *sqtd;
2174         usbd_status err;
2175         int i, offs;
2176         usb_dma_t dma;
2177
2178         if (sc->sc_freeqhs == NULL) {
2179                 DPRINTFN(2, ("ehci_alloc_sqh: allocating chunk\n"));
2180                 err = usb_allocmem(&sc->sc_bus, EHCI_SQH_SIZE * EHCI_SQH_CHUNK,
2181                           EHCI_PAGE_SIZE, &dma);
2182 #ifdef EHCI_DEBUG
2183                 if (err)
2184                         printf("ehci_alloc_sqh: usb_allocmem()=%d\n", err);
2185 #endif
2186                 if (err)
2187                         return (NULL);
2188                 for(i = 0; i < EHCI_SQH_CHUNK; i++) {
2189                         offs = i * EHCI_SQH_SIZE;
2190                         sqh = KERNADDR(&dma, offs);
2191                         sqh->physaddr = DMAADDR(&dma, offs);
2192                         sqh->next = sc->sc_freeqhs;
2193                         sc->sc_freeqhs = sqh;
2194                 }
2195         }
2196         /* Allocate the initial inactive sqtd. */
2197         sqtd = ehci_alloc_sqtd(sc);
2198         if (sqtd == NULL)
2199                 return (NULL);
2200         sqtd->qtd.qtd_status = htole32(0);
2201         sqtd->qtd.qtd_next = EHCI_NULL;
2202         sqtd->qtd.qtd_altnext = EHCI_NULL;
2203
2204         sqh = sc->sc_freeqhs;
2205         sc->sc_freeqhs = sqh->next;
2206
2207         /* The overlay QTD should begin zeroed. */
2208         sqh->qh.qh_qtd.qtd_next = htole32(sqtd->physaddr);
2209         sqh->qh.qh_qtd.qtd_altnext = EHCI_NULL;
2210         sqh->qh.qh_qtd.qtd_status = 0;
2211         for (i = 0; i < EHCI_QTD_NBUFFERS; i++) {
2212                 sqh->qh.qh_qtd.qtd_buffer[i] = 0;
2213                 sqh->qh.qh_qtd.qtd_buffer_hi[i] = 0;
2214         }
2215         sqh->next = NULL;
2216         sqh->prev = NULL;
2217         sqh->sqtd = sqtd;
2218         sqh->inactivesqtd = sqtd;
2219         return (sqh);
2220 }
2221
2222 void
2223 ehci_free_sqh(ehci_softc_t *sc, ehci_soft_qh_t *sqh)
2224 {
2225         ehci_free_sqtd(sc, sqh->inactivesqtd);
2226         sqh->next = sc->sc_freeqhs;
2227         sc->sc_freeqhs = sqh;
2228 }
2229
2230 ehci_soft_qtd_t *
2231 ehci_alloc_sqtd(ehci_softc_t *sc)
2232 {
2233         ehci_soft_qtd_t *sqtd;
2234         usbd_status err;
2235         int i, offs;
2236         usb_dma_t dma;
2237         int s;
2238
2239         if (sc->sc_freeqtds == NULL) {
2240                 DPRINTFN(2, ("ehci_alloc_sqtd: allocating chunk\n"));
2241                 err = usb_allocmem(&sc->sc_bus, EHCI_SQTD_SIZE*EHCI_SQTD_CHUNK,
2242                           EHCI_PAGE_SIZE, &dma);
2243 #ifdef EHCI_DEBUG
2244                 if (err)
2245                         printf("ehci_alloc_sqtd: usb_allocmem()=%d\n", err);
2246 #endif
2247                 if (err)
2248                         return (NULL);
2249                 s = splusb();
2250                 for(i = 0; i < EHCI_SQTD_CHUNK; i++) {
2251                         offs = i * EHCI_SQTD_SIZE;
2252                         sqtd = KERNADDR(&dma, offs);
2253                         sqtd->physaddr = DMAADDR(&dma, offs);
2254                         sqtd->nextqtd = sc->sc_freeqtds;
2255                         sc->sc_freeqtds = sqtd;
2256                 }
2257                 splx(s);
2258         }
2259
2260         s = splusb();
2261         sqtd = sc->sc_freeqtds;
2262         sc->sc_freeqtds = sqtd->nextqtd;
2263         sqtd->qtd.qtd_next = EHCI_NULL;
2264         sqtd->qtd.qtd_altnext = EHCI_NULL;
2265         sqtd->qtd.qtd_status = 0;
2266         for (i = 0; i < EHCI_QTD_NBUFFERS; i++) {
2267                 sqtd->qtd.qtd_buffer[i] = 0;
2268                 sqtd->qtd.qtd_buffer_hi[i] = 0;
2269         }
2270         sqtd->nextqtd = NULL;
2271         sqtd->xfer = NULL;
2272         splx(s);
2273
2274         return (sqtd);
2275 }
2276
2277 void
2278 ehci_free_sqtd(ehci_softc_t *sc, ehci_soft_qtd_t *sqtd)
2279 {
2280         int s;
2281
2282         s = splusb();
2283         sqtd->nextqtd = sc->sc_freeqtds;
2284         sc->sc_freeqtds = sqtd;
2285         splx(s);
2286 }
2287
2288 usbd_status
2289 ehci_alloc_sqtd_chain(struct ehci_pipe *epipe, ehci_softc_t *sc,
2290      int alen, int rd, usbd_xfer_handle xfer, ehci_soft_qtd_t *start,
2291      ehci_soft_qtd_t *newinactive, ehci_soft_qtd_t **sp, ehci_soft_qtd_t **ep)
2292 {
2293         ehci_soft_qtd_t *next, *cur;
2294         ehci_physaddr_t dataphys, nextphys;
2295         u_int32_t qtdstatus;
2296         int adj, len, curlen, mps, offset, pagelen, seg, segoff;
2297         int i, iscontrol, forceshort;
2298         struct usb_dma_mapping *dma = &xfer->dmamap;
2299
2300         DPRINTFN(alen<4*4096,("ehci_alloc_sqtd_chain: start len=%d\n", alen));
2301
2302         offset = 0;
2303         len = alen;
2304         iscontrol = (epipe->pipe.endpoint->edesc->bmAttributes & UE_XFERTYPE) ==
2305             UE_CONTROL;
2306         qtdstatus = EHCI_QTD_ACTIVE |
2307             EHCI_QTD_SET_PID(rd ? EHCI_QTD_PID_IN : EHCI_QTD_PID_OUT) |
2308             EHCI_QTD_SET_CERR(3)
2309             /* IOC set below */
2310             /* BYTES set below */
2311             ;
2312         mps = UGETW(epipe->pipe.endpoint->edesc->wMaxPacketSize);
2313         forceshort = ((xfer->flags & USBD_FORCE_SHORT_XFER) || len == 0) &&
2314             len % mps == 0;
2315         /*
2316          * The control transfer data stage always starts with a toggle of 1.
2317          * For other transfers we let the hardware track the toggle state.
2318          */
2319         if (iscontrol)
2320                 qtdstatus |= EHCI_QTD_SET_TOGGLE(1);
2321
2322         if (start != NULL) {
2323                 /*
2324                  * If we are given a starting qTD, assume it is linked into
2325                  * an active QH so be careful not to mark it active.
2326                  */
2327                 cur = start;
2328                 *sp = cur;
2329                 qtdstatus &= ~EHCI_QTD_ACTIVE;
2330         } else {
2331                 cur = ehci_alloc_sqtd(sc);
2332                 *sp = cur;
2333                 if (cur == NULL)
2334                         goto nomem;
2335         }
2336         seg = 0;
2337         segoff = 0;
2338         for (;;) {
2339                 curlen = 0;
2340
2341                 /* The EHCI hardware can handle at most 5 pages. */
2342                 for (i = 0; i < EHCI_QTD_NBUFFERS && curlen < len; i++) {
2343                         KASSERT(seg < dma->nsegs,
2344                             ("ehci_alloc_sqtd_chain: overrun"));
2345                         dataphys = dma->segs[seg].ds_addr + segoff;
2346                         pagelen = dma->segs[seg].ds_len - segoff;
2347                         if (pagelen > len - curlen)
2348                                 pagelen = len - curlen;
2349                         if (pagelen > EHCI_PAGE_SIZE -
2350                             EHCI_PAGE_OFFSET(dataphys))
2351                                 pagelen = EHCI_PAGE_SIZE -
2352                                     EHCI_PAGE_OFFSET(dataphys);
2353                         segoff += pagelen;
2354                         if (segoff >= dma->segs[seg].ds_len) {
2355                                 KASSERT(segoff == dma->segs[seg].ds_len,
2356                                     ("ehci_alloc_sqtd_chain: overlap"));
2357                                 seg++;
2358                                 segoff = 0;
2359                         }
2360
2361                         cur->qtd.qtd_buffer[i] = htole32(dataphys);
2362                         cur->qtd.qtd_buffer_hi[i] = 0;
2363                         curlen += pagelen;
2364
2365                         /*
2366                          * Must stop if there is any gap before or after
2367                          * the page boundary.
2368                          */
2369                         if (EHCI_PAGE_OFFSET(dataphys + pagelen) != 0)
2370                                 break;
2371                         if (seg < dma->nsegs && EHCI_PAGE_OFFSET(segoff +
2372                             dma->segs[seg].ds_addr) != 0)
2373                                 break;
2374                 }
2375                 /* Adjust down to a multiple of mps if not at the end. */
2376                 if (curlen < len && curlen % mps != 0) {
2377                         adj = curlen % mps;
2378                         curlen -= adj;
2379                         KASSERT(curlen > 0,
2380                             ("ehci_alloc_sqtd_chain: need to copy"));
2381                         segoff -= adj;
2382                         if (segoff < 0) {
2383                                 seg--;
2384                                 segoff += dma->segs[seg].ds_len;
2385                         }
2386                         KASSERT(seg >= 0 && segoff >= 0,
2387                             ("ehci_alloc_sqtd_chain: adjust to mps"));
2388                 }
2389
2390                 len -= curlen;
2391
2392                 if (len != 0 || forceshort) {
2393                         next = ehci_alloc_sqtd(sc);
2394                         if (next == NULL)
2395                                 goto nomem;
2396                         nextphys = htole32(next->physaddr);
2397                 } else {
2398                         next = NULL;
2399                         nextphys = EHCI_NULL;
2400                 }
2401
2402                 cur->nextqtd = next;
2403                 cur->qtd.qtd_next = nextphys;
2404                 /* Make sure to stop after a short transfer. */
2405                 cur->qtd.qtd_altnext = htole32(newinactive->physaddr);
2406                 cur->qtd.qtd_status =
2407                     htole32(qtdstatus | EHCI_QTD_SET_BYTES(curlen));
2408                 cur->xfer = xfer;
2409                 cur->len = curlen;
2410                 DPRINTFN(10,("ehci_alloc_sqtd_chain: curlen=%d\n", curlen));
2411                 if (iscontrol) {
2412                         /*
2413                          * adjust the toggle based on the number of packets
2414                          * in this qtd
2415                          */
2416                         if ((((curlen + mps - 1) / mps) & 1) || curlen == 0)
2417                                 qtdstatus ^= EHCI_QTD_TOGGLE_MASK;
2418                 }
2419                 qtdstatus |= EHCI_QTD_ACTIVE;
2420                 if (len == 0) {
2421                         if (!forceshort)
2422                                 break;
2423                         forceshort = 0;
2424                 }
2425                 DPRINTFN(10,("ehci_alloc_sqtd_chain: extend chain\n"));
2426                 offset += curlen;
2427                 cur = next;
2428         }
2429         cur->qtd.qtd_status |= htole32(EHCI_QTD_IOC);
2430         *ep = cur;
2431
2432         DPRINTFN(10,("ehci_alloc_sqtd_chain: return sqtd=%p sqtdend=%p\n",
2433                      *sp, *ep));
2434
2435         return (USBD_NORMAL_COMPLETION);
2436
2437  nomem:
2438         /* XXX free chain */
2439         DPRINTFN(-1,("ehci_alloc_sqtd_chain: no memory\n"));
2440         return (USBD_NOMEM);
2441 }
2442
2443 /* Free the chain starting at sqtd and end at the qTD before sqtdend */
2444 static void
2445 ehci_free_sqtd_chain(ehci_softc_t *sc, ehci_soft_qh_t *sqh,
2446     ehci_soft_qtd_t *sqtd, ehci_soft_qtd_t *sqtdend)
2447 {
2448         ehci_soft_qtd_t *p, **prevp;
2449         int i;
2450
2451         DPRINTFN(10,("ehci_free_sqtd_chain: sqtd=%p sqtdend=%p\n",
2452                      sqtd, sqtdend));
2453
2454         /* First unlink the chain from the QH's software qTD list. */
2455         prevp = &sqh->sqtd;
2456         for (p = sqh->sqtd; p != NULL; p = p->nextqtd) {
2457                 if (p == sqtd) {
2458                         *prevp = sqtdend;
2459                         break;
2460                 }
2461                 prevp = &p->nextqtd;
2462         }
2463         KASSERT(p != NULL, ("ehci_free_sqtd_chain: chain not found"));
2464         for (i = 0; sqtd != sqtdend; sqtd = p, i++) {
2465                 p = sqtd->nextqtd;
2466                 ehci_free_sqtd(sc, sqtd);
2467         }
2468 }
2469
2470 /****************/
2471
2472 /*
2473  * Close a reqular pipe.
2474  * Assumes that there are no pending transactions.
2475  */
2476 void
2477 ehci_close_pipe(usbd_pipe_handle pipe, ehci_soft_qh_t *head)
2478 {
2479         struct ehci_pipe *epipe = (struct ehci_pipe *)pipe;
2480         ehci_softc_t *sc = (ehci_softc_t *)pipe->device->bus;
2481         ehci_soft_qh_t *sqh = epipe->sqh;
2482         int s;
2483
2484         s = splusb();
2485         ehci_rem_qh(sc, sqh, head);
2486         splx(s);
2487         pipe->endpoint->savedtoggle =
2488             EHCI_QTD_GET_TOGGLE(le32toh(sqh->qh.qh_qtd.qtd_status));
2489         ehci_free_sqh(sc, epipe->sqh);
2490 }
2491
2492 /*
2493  * Abort a device request.
2494  * If this routine is called at splusb() it guarantees that the request
2495  * will be removed from the hardware scheduling and that the callback
2496  * for it will be called with USBD_CANCELLED status.
2497  * It's impossible to guarantee that the requested transfer will not
2498  * have happened since the hardware runs concurrently.
2499  * If the transaction has already happened we rely on the ordinary
2500  * interrupt processing to process it.
2501  */
2502 void
2503 ehci_abort_xfer(usbd_xfer_handle xfer, usbd_status status)
2504 {
2505 #define exfer EXFER(xfer)
2506         struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
2507         ehci_softc_t *sc = (ehci_softc_t *)epipe->pipe.device->bus;
2508         ehci_soft_qh_t *sqh = epipe->sqh;
2509         ehci_soft_qtd_t *sqtd, *snext;
2510         ehci_physaddr_t cur, us, next;
2511         int s;
2512         int hit, i;
2513         /* int count = 0; */
2514         ehci_soft_qh_t *psqh;
2515
2516         DPRINTF(("ehci_abort_xfer: xfer=%p pipe=%p\n", xfer, epipe));
2517
2518         if (sc->sc_dying) {
2519                 /* If we're dying, just do the software part. */
2520                 s = splusb();
2521                 xfer->status = status;  /* make software ignore it */
2522                 callout_stop(&xfer->timeout_handle);
2523                 usb_rem_task(epipe->pipe.device, &exfer->abort_task);
2524                 usb_transfer_complete(xfer);
2525                 splx(s);
2526                 return;
2527         }
2528
2529         if (xfer->device->bus->intr_context || !curproc)
2530                 panic("ehci_abort_xfer: not in process context");
2531
2532         /*
2533          * If an abort is already in progress then just wait for it to
2534          * complete and return.
2535          */
2536         if (exfer->ehci_xfer_flags & EHCI_XFER_ABORTING) {
2537                 DPRINTFN(2, ("ehci_abort_xfer: already aborting\n"));
2538                 /* No need to wait if we're aborting from a timeout. */
2539                 if (status == USBD_TIMEOUT)
2540                         return;
2541                 /* Override the status which might be USBD_TIMEOUT. */
2542                 xfer->status = status;
2543                 DPRINTFN(2, ("ehci_abort_xfer: waiting for abort to finish\n"));
2544                 exfer->ehci_xfer_flags |= EHCI_XFER_ABORTWAIT;
2545                 while (exfer->ehci_xfer_flags & EHCI_XFER_ABORTING)
2546                         tsleep(&exfer->ehci_xfer_flags, PZERO, "ehciaw", 0);
2547                 return;
2548         }
2549
2550         /*
2551          * Step 1: Make interrupt routine and timeouts ignore xfer.
2552          */
2553         s = splusb();
2554         exfer->ehci_xfer_flags |= EHCI_XFER_ABORTING;
2555         xfer->status = status;  /* make software ignore it */
2556         callout_stop(&xfer->timeout_handle);
2557         usb_rem_task(epipe->pipe.device, &exfer->abort_task);
2558         splx(s);
2559
2560         /*
2561          * Step 2: Wait until we know hardware has finished any possible
2562          * use of the xfer. We do this by removing the entire
2563          * queue from the async schedule and waiting for the doorbell.
2564          * Nothing else should be touching the queue now.
2565          */
2566         psqh = sqh->prev;
2567         ehci_rem_qh(sc, sqh, psqh);
2568
2569         /*
2570          * Step 3:  make sure the soft interrupt routine
2571          * has run. This should remove any completed items off the queue.
2572          * The hardware has no reference to completed items (TDs).
2573          * It's safe to remove them at any time.
2574          */
2575         s = splusb();
2576 #ifdef USB_USE_SOFTINTR
2577         sc->sc_softwake = 1;
2578 #endif /* USB_USE_SOFTINTR */
2579         usb_schedsoftintr(&sc->sc_bus);
2580 #ifdef USB_USE_SOFTINTR
2581         tsleep(&sc->sc_softwake, PZERO, "ehciab", 0);
2582 #endif /* USB_USE_SOFTINTR */
2583
2584         /*
2585          * Step 4: Remove any vestiges of the xfer from the hardware.
2586          * The complication here is that the hardware may have executed
2587          * into or even beyond the xfer we're trying to abort.
2588          * So as we're scanning the TDs of this xfer we check if
2589          * the hardware points to any of them.
2590          *
2591          * first we need to see if there are any transfers
2592          * on this queue before the xfer we are aborting.. we need
2593          * to update any pointers that point to us to point past
2594          * the aborting xfer.  (If there is something past us).
2595          * Hardware and software.
2596          */
2597         cur = EHCI_LINK_ADDR(le32toh(sqh->qh.qh_curqtd));
2598         hit = 0;
2599
2600         /* If they initially point here. */
2601         us = exfer->sqtdstart->physaddr;
2602
2603         /* We will change them to point here */
2604         snext = exfer->sqtdend->nextqtd;
2605         next = htole32(snext->physaddr);
2606
2607         /*
2608          * Now loop through any qTDs before us and keep track of the pointer
2609          * that points to us for the end.
2610          */
2611         sqtd = sqh->sqtd;
2612         while (sqtd && sqtd != exfer->sqtdstart) {
2613                 hit |= (cur == sqtd->physaddr);
2614                 if (EHCI_LINK_ADDR(le32toh(sqtd->qtd.qtd_next)) == us)
2615                         sqtd->qtd.qtd_next = next;
2616                 if (EHCI_LINK_ADDR(le32toh(sqtd->qtd.qtd_altnext)) == us)
2617                         sqtd->qtd.qtd_altnext = next;
2618                 sqtd = sqtd->nextqtd;
2619         }
2620
2621         /*
2622          * If we already saw the active one then we are pretty much done.
2623          * We've done all the relinking we need to do.
2624          */
2625         if (!hit) {
2626
2627                 /*
2628                  * Now reinitialise the QH to point to the next qTD
2629                  * (if there is one). We only need to do this if
2630                  * it was previously pointing to us.
2631                  */
2632                 for (sqtd = exfer->sqtdstart; ; sqtd = sqtd->nextqtd) {
2633                         if (cur == sqtd->physaddr) {
2634                                 hit++;
2635                         }
2636                         if (sqtd == exfer->sqtdend)
2637                                 break;
2638                 }
2639                 sqtd = sqtd->nextqtd;
2640                 /*
2641                  * Only need to alter the QH if it was pointing at a qTD
2642                  * that we are removing.
2643                  */
2644                 if (hit) {
2645                         sqh->qh.qh_qtd.qtd_next = htole32(snext->physaddr);
2646                         sqh->qh.qh_qtd.qtd_altnext = EHCI_NULL;
2647                         sqh->qh.qh_qtd.qtd_status &=
2648                             htole32(EHCI_QTD_TOGGLE_MASK);
2649                         for (i = 0; i < EHCI_QTD_NBUFFERS; i++) {
2650                                 sqh->qh.qh_qtd.qtd_buffer[i] = 0;
2651                                 sqh->qh.qh_qtd.qtd_buffer_hi[i] = 0;
2652                         }
2653                 }
2654         }
2655         ehci_add_qh(sqh, psqh);
2656         /*
2657          * Step 5: Execute callback.
2658          */
2659 #ifdef DIAGNOSTIC
2660         exfer->isdone = 1;
2661 #endif
2662         /* Do the wakeup first to avoid touching the xfer after the callback. */
2663         exfer->ehci_xfer_flags &= ~EHCI_XFER_ABORTING;
2664         if (exfer->ehci_xfer_flags & EHCI_XFER_ABORTWAIT) {
2665                 exfer->ehci_xfer_flags &= ~EHCI_XFER_ABORTWAIT;
2666                 wakeup(&exfer->ehci_xfer_flags);
2667         }
2668         usb_transfer_complete(xfer);
2669
2670         /* printf("%s: %d TDs aborted\n", __func__, count); */
2671         splx(s);
2672 #undef exfer
2673 }
2674
2675 void
2676 ehci_timeout(void *addr)
2677 {
2678         struct ehci_xfer *exfer = addr;
2679         struct ehci_pipe *epipe = (struct ehci_pipe *)exfer->xfer.pipe;
2680         ehci_softc_t *sc = (ehci_softc_t *)epipe->pipe.device->bus;
2681
2682         DPRINTF(("ehci_timeout: exfer=%p\n", exfer));
2683 #ifdef USB_DEBUG
2684         if (ehcidebug > 1)
2685                 usbd_dump_pipe(exfer->xfer.pipe);
2686 #endif
2687
2688         if (sc->sc_dying) {
2689                 ehci_abort_xfer(&exfer->xfer, USBD_TIMEOUT);
2690                 return;
2691         }
2692
2693         /* Execute the abort in a process context. */
2694         usb_add_task(exfer->xfer.pipe->device, &exfer->abort_task,
2695             USB_TASKQ_HC);
2696 }
2697
2698 void
2699 ehci_timeout_task(void *addr)
2700 {
2701         usbd_xfer_handle xfer = addr;
2702         int s;
2703
2704         DPRINTF(("ehci_timeout_task: xfer=%p\n", xfer));
2705
2706         s = splusb();
2707         ehci_abort_xfer(xfer, USBD_TIMEOUT);
2708         splx(s);
2709 }
2710
2711 /*
2712  * Some EHCI chips from VIA / ATI seem to trigger interrupts before writing
2713  * back the qTD status, or miss signalling occasionally under heavy load.
2714  * If the host machine is too fast, we can miss transaction completion - when
2715  * we scan the active list the transaction still seems to be active. This
2716  * generally exhibits itself as a umass stall that never recovers.
2717  *
2718  * We work around this behaviour by setting up this callback after any softintr
2719  * that completes with transactions still pending, giving us another chance to
2720  * check for completion after the writeback has taken place.
2721  */
2722 void
2723 ehci_intrlist_timeout(void *arg)
2724 {
2725         ehci_softc_t *sc = arg;
2726         int s = splusb();
2727
2728         DPRINTFN(3, ("ehci_intrlist_timeout\n"));
2729         usb_schedsoftintr(&sc->sc_bus);
2730
2731         splx(s);
2732 }
2733
2734 /************************/
2735
2736 static usbd_status
2737 ehci_device_ctrl_transfer(usbd_xfer_handle xfer)
2738 {
2739         usbd_status err;
2740
2741         /* Insert last in queue. */
2742         err = usb_insert_transfer(xfer);
2743         if (err)
2744                 return (err);
2745
2746         /* Pipe isn't running, start first */
2747         return (ehci_device_ctrl_start(STAILQ_FIRST(&xfer->pipe->queue)));
2748 }
2749
2750 static usbd_status
2751 ehci_device_ctrl_start(usbd_xfer_handle xfer)
2752 {
2753         ehci_softc_t *sc = (ehci_softc_t *)xfer->pipe->device->bus;
2754         usbd_status err;
2755
2756         if (sc->sc_dying)
2757                 return (USBD_IOERROR);
2758
2759 #ifdef DIAGNOSTIC
2760         if (!(xfer->rqflags & URQ_REQUEST)) {
2761                 /* XXX panic */
2762                 printf("ehci_device_ctrl_transfer: not a request\n");
2763                 return (USBD_INVAL);
2764         }
2765 #endif
2766
2767         err = ehci_device_request(xfer);
2768         if (err)
2769                 return (err);
2770
2771         if (sc->sc_bus.use_polling)
2772                 ehci_waitintr(sc, xfer);
2773         return (USBD_IN_PROGRESS);
2774 }
2775
2776 void
2777 ehci_device_ctrl_done(usbd_xfer_handle xfer)
2778 {
2779         struct ehci_xfer *ex = EXFER(xfer);
2780         ehci_softc_t *sc = (ehci_softc_t *)xfer->pipe->device->bus;
2781         struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
2782
2783         DPRINTFN(10,("ehci_ctrl_done: xfer=%p\n", xfer));
2784
2785 #ifdef DIAGNOSTIC
2786         if (!(xfer->rqflags & URQ_REQUEST)) {
2787                 panic("ehci_ctrl_done: not a request");
2788         }
2789 #endif
2790
2791         if (xfer->status != USBD_NOMEM && ehci_active_intr_list(ex)) {
2792                 ehci_del_intr_list(ex); /* remove from active list */
2793                 ehci_free_sqtd_chain(sc, epipe->sqh, ex->sqtdstart,
2794                     ex->sqtdend->nextqtd);
2795         }
2796
2797         DPRINTFN(5, ("ehci_ctrl_done: length=%d\n", xfer->actlen));
2798 }
2799
2800 /* Abort a device control request. */
2801 static void
2802 ehci_device_ctrl_abort(usbd_xfer_handle xfer)
2803 {
2804         DPRINTF(("ehci_device_ctrl_abort: xfer=%p\n", xfer));
2805         ehci_abort_xfer(xfer, USBD_CANCELLED);
2806 }
2807
2808 /* Close a device control pipe. */
2809 static void
2810 ehci_device_ctrl_close(usbd_pipe_handle pipe)
2811 {
2812         ehci_softc_t *sc = (ehci_softc_t *)pipe->device->bus;
2813         /*struct ehci_pipe *epipe = (struct ehci_pipe *)pipe;*/
2814
2815         DPRINTF(("ehci_device_ctrl_close: pipe=%p\n", pipe));
2816         ehci_close_pipe(pipe, sc->sc_async_head);
2817 }
2818
2819 usbd_status
2820 ehci_device_request(usbd_xfer_handle xfer)
2821 {
2822 #define exfer EXFER(xfer)
2823         struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
2824         usb_device_request_t *req = &xfer->request;
2825         usbd_device_handle dev = epipe->pipe.device;
2826         ehci_softc_t *sc = (ehci_softc_t *)dev->bus;
2827         ehci_soft_qtd_t *newinactive, *setup, *stat, *next;
2828         ehci_soft_qh_t *sqh;
2829         int isread;
2830         int len;
2831         usbd_status err;
2832         int s;
2833
2834         isread = req->bmRequestType & UT_READ;
2835         len = UGETW(req->wLength);
2836
2837         DPRINTFN(3,("ehci_device_request: type=0x%02x, request=0x%02x, "
2838                     "wValue=0x%04x, wIndex=0x%04x len=%d, addr=%d, endpt=%d\n",
2839                     req->bmRequestType, req->bRequest, UGETW(req->wValue),
2840                     UGETW(req->wIndex), len, dev->address,
2841                     epipe->pipe.endpoint->edesc->bEndpointAddress));
2842
2843         newinactive = ehci_alloc_sqtd(sc);
2844         if (newinactive == NULL) {
2845                 err = USBD_NOMEM;
2846                 goto bad1;
2847         }
2848         newinactive->qtd.qtd_status = htole32(0);
2849         newinactive->qtd.qtd_next = EHCI_NULL;
2850         newinactive->qtd.qtd_altnext = EHCI_NULL;
2851         stat = ehci_alloc_sqtd(sc);
2852         if (stat == NULL) {
2853                 err = USBD_NOMEM;
2854                 goto bad2;
2855         }
2856
2857         sqh = epipe->sqh;
2858         setup = sqh->inactivesqtd;
2859         sqh->inactivesqtd = newinactive;
2860         epipe->u.ctl.length = len;
2861
2862         /* Set up data transaction */
2863         if (len != 0) {
2864                 ehci_soft_qtd_t *end;
2865
2866                 err = ehci_alloc_sqtd_chain(epipe, sc, len, isread, xfer,
2867                     NULL, newinactive, &next, &end);
2868                 if (err)
2869                         goto bad3;
2870                 end->qtd.qtd_status &= htole32(~EHCI_QTD_IOC);
2871                 end->nextqtd = stat;
2872                 end->qtd.qtd_next = htole32(stat->physaddr);
2873                 end->qtd.qtd_altnext = htole32(newinactive->physaddr);
2874         } else {
2875                 next = stat;
2876         }
2877
2878         memcpy(KERNADDR(&epipe->u.ctl.reqdma, 0), req, sizeof *req);
2879
2880         /* Clear toggle, and do not activate until complete */
2881         setup->qtd.qtd_status = htole32(
2882             EHCI_QTD_SET_PID(EHCI_QTD_PID_SETUP) |
2883             EHCI_QTD_SET_CERR(3) |
2884             EHCI_QTD_SET_TOGGLE(0) |
2885             EHCI_QTD_SET_BYTES(sizeof *req)
2886             );
2887         setup->qtd.qtd_buffer[0] = htole32(DMAADDR(&epipe->u.ctl.reqdma, 0));
2888         setup->qtd.qtd_buffer_hi[0] = 0;
2889         setup->nextqtd = next;
2890         setup->qtd.qtd_next = htole32(next->physaddr);
2891         setup->qtd.qtd_altnext = htole32(newinactive->physaddr);
2892         setup->xfer = xfer;
2893         setup->len = sizeof *req;
2894
2895         stat->qtd.qtd_status = htole32(
2896             EHCI_QTD_ACTIVE |
2897             EHCI_QTD_SET_PID(isread ? EHCI_QTD_PID_OUT : EHCI_QTD_PID_IN) |
2898             EHCI_QTD_SET_CERR(3) |
2899             EHCI_QTD_SET_TOGGLE(1) |
2900             EHCI_QTD_IOC
2901             );
2902         stat->qtd.qtd_buffer[0] = 0; /* XXX not needed? */
2903         stat->qtd.qtd_buffer_hi[0] = 0; /* XXX not needed? */
2904         stat->nextqtd = newinactive;
2905         stat->qtd.qtd_next = htole32(newinactive->physaddr);
2906         stat->qtd.qtd_altnext = htole32(newinactive->physaddr);
2907         stat->xfer = xfer;
2908         stat->len = 0;
2909
2910 #ifdef EHCI_DEBUG
2911         if (ehcidebug > 5) {
2912                 DPRINTF(("ehci_device_request:\n"));
2913                 ehci_dump_sqh(sqh);
2914                 ehci_dump_sqtds(setup);
2915         }
2916 #endif
2917
2918         exfer->sqtdstart = setup;
2919         exfer->sqtdend = stat;
2920 #ifdef DIAGNOSTIC
2921         if (!exfer->isdone) {
2922                 printf("ehci_device_request: not done, exfer=%p\n", exfer);
2923         }
2924         exfer->isdone = 0;
2925 #endif
2926
2927         /* Activate the new qTD in the QH list. */
2928         s = splusb();
2929         ehci_activate_qh(sqh, setup);
2930         if (xfer->timeout && !sc->sc_bus.use_polling) {
2931                 callout_reset(&xfer->timeout_handle, MS_TO_TICKS(xfer->timeout),
2932                     ehci_timeout, xfer);
2933         }
2934         ehci_add_intr_list(sc, exfer);
2935         xfer->status = USBD_IN_PROGRESS;
2936         splx(s);
2937
2938 #ifdef EHCI_DEBUG
2939         if (ehcidebug > 10) {
2940                 DPRINTF(("ehci_device_request: status=%x\n",
2941                          EOREAD4(sc, EHCI_USBSTS)));
2942                 delay(10000);
2943                 ehci_dump_regs(sc);
2944                 ehci_dump_sqh(sc->sc_async_head);
2945                 ehci_dump_sqh(sqh);
2946                 ehci_dump_sqtds(setup);
2947         }
2948 #endif
2949
2950         return (USBD_NORMAL_COMPLETION);
2951
2952  bad3:
2953         sqh->inactivesqtd = setup;
2954         ehci_free_sqtd(sc, stat);
2955  bad2:
2956         ehci_free_sqtd(sc, newinactive);
2957  bad1:
2958         DPRINTFN(-1,("ehci_device_request: no memory\n"));
2959         xfer->status = err;
2960         usb_transfer_complete(xfer);
2961         return (err);
2962 #undef exfer
2963 }
2964
2965 /************************/
2966
2967 static usbd_status
2968 ehci_device_bulk_transfer(usbd_xfer_handle xfer)
2969 {
2970         usbd_status err;
2971
2972         /* Insert last in queue. */
2973         err = usb_insert_transfer(xfer);
2974         if (err)
2975                 return (err);
2976
2977         /* Pipe isn't running, start first */
2978         return (ehci_device_bulk_start(STAILQ_FIRST(&xfer->pipe->queue)));
2979 }
2980
2981 usbd_status
2982 ehci_device_bulk_start(usbd_xfer_handle xfer)
2983 {
2984 #define exfer EXFER(xfer)
2985         struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
2986         usbd_device_handle dev = epipe->pipe.device;
2987         ehci_softc_t *sc = (ehci_softc_t *)dev->bus;
2988         ehci_soft_qtd_t *data, *dataend, *newinactive;
2989         ehci_soft_qh_t *sqh;
2990         usbd_status err;
2991         int len, isread, endpt;
2992         int s;
2993
2994         DPRINTFN(2, ("ehci_device_bulk_start: xfer=%p len=%d flags=%d\n",
2995                      xfer, xfer->length, xfer->flags));
2996
2997         if (sc->sc_dying)
2998                 return (USBD_IOERROR);
2999
3000 #ifdef DIAGNOSTIC
3001         if (xfer->rqflags & URQ_REQUEST)
3002                 panic("ehci_device_bulk_start: a request");
3003 #endif
3004
3005         len = xfer->length;
3006         endpt = epipe->pipe.endpoint->edesc->bEndpointAddress;
3007         isread = UE_GET_DIR(endpt) == UE_DIR_IN;
3008         sqh = epipe->sqh;
3009
3010         epipe->u.bulk.length = len;
3011
3012         newinactive = ehci_alloc_sqtd(sc);
3013         if (newinactive == NULL) {
3014                 DPRINTFN(-1,("ehci_device_bulk_start: no sqtd memory\n"));
3015                 err = USBD_NOMEM;
3016                 xfer->status = err;
3017                 usb_transfer_complete(xfer);
3018                 return (err);
3019         }
3020         newinactive->qtd.qtd_status = htole32(0);
3021         newinactive->qtd.qtd_next = EHCI_NULL;
3022         newinactive->qtd.qtd_altnext = EHCI_NULL;
3023         err = ehci_alloc_sqtd_chain(epipe, sc, len, isread, xfer,
3024             sqh->inactivesqtd, newinactive, &data, &dataend);
3025         if (err) {
3026                 DPRINTFN(-1,("ehci_device_bulk_start: no memory\n"));
3027                 ehci_free_sqtd(sc, newinactive);
3028                 xfer->status = err;
3029                 usb_transfer_complete(xfer);
3030                 return (err);
3031         }
3032         dataend->nextqtd = newinactive;
3033         dataend->qtd.qtd_next = htole32(newinactive->physaddr);
3034         dataend->qtd.qtd_altnext = htole32(newinactive->physaddr);
3035         sqh->inactivesqtd = newinactive;
3036
3037 #ifdef EHCI_DEBUG
3038         if (ehcidebug > 5) {
3039                 DPRINTF(("ehci_device_bulk_start: data(1)\n"));
3040                 ehci_dump_sqh(sqh);
3041                 ehci_dump_sqtds(data);
3042         }
3043 #endif
3044
3045         /* Set up interrupt info. */
3046         exfer->sqtdstart = data;
3047         exfer->sqtdend = dataend;
3048 #ifdef DIAGNOSTIC
3049         if (!exfer->isdone) {
3050                 printf("ehci_device_bulk_start: not done, ex=%p\n", exfer);
3051         }
3052         exfer->isdone = 0;
3053 #endif
3054
3055         s = splusb();
3056         ehci_activate_qh(sqh, data);
3057         if (xfer->timeout && !sc->sc_bus.use_polling) {
3058                 callout_reset(&xfer->timeout_handle, MS_TO_TICKS(xfer->timeout),
3059                     ehci_timeout, xfer);
3060         }
3061         ehci_add_intr_list(sc, exfer);
3062         xfer->status = USBD_IN_PROGRESS;
3063         splx(s);
3064
3065 #ifdef EHCI_DEBUG
3066         if (ehcidebug > 10) {
3067                 DPRINTF(("ehci_device_bulk_start: data(2)\n"));
3068                 delay(10000);
3069                 DPRINTF(("ehci_device_bulk_start: data(3)\n"));
3070                 ehci_dump_regs(sc);
3071 #if 0
3072                 printf("async_head:\n");
3073                 ehci_dump_sqh(sc->sc_async_head);
3074 #endif
3075                 printf("sqh:\n");
3076                 ehci_dump_sqh(sqh);
3077                 ehci_dump_sqtds(data);
3078         }
3079 #endif
3080
3081         if (sc->sc_bus.use_polling)
3082                 ehci_waitintr(sc, xfer);
3083
3084         return (USBD_IN_PROGRESS);
3085 #undef exfer
3086 }
3087
3088 static void
3089 ehci_device_bulk_abort(usbd_xfer_handle xfer)
3090 {
3091         DPRINTF(("ehci_device_bulk_abort: xfer=%p\n", xfer));
3092         ehci_abort_xfer(xfer, USBD_CANCELLED);
3093 }
3094
3095 /*
3096  * Close a device bulk pipe.
3097  */
3098 static void
3099 ehci_device_bulk_close(usbd_pipe_handle pipe)
3100 {
3101         ehci_softc_t *sc = (ehci_softc_t *)pipe->device->bus;
3102
3103         DPRINTF(("ehci_device_bulk_close: pipe=%p\n", pipe));
3104         ehci_close_pipe(pipe, sc->sc_async_head);
3105 }
3106
3107 void
3108 ehci_device_bulk_done(usbd_xfer_handle xfer)
3109 {
3110         struct ehci_xfer *ex = EXFER(xfer);
3111         ehci_softc_t *sc = (ehci_softc_t *)xfer->pipe->device->bus;
3112         struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
3113
3114         DPRINTFN(10,("ehci_bulk_done: xfer=%p, actlen=%d\n",
3115                      xfer, xfer->actlen));
3116
3117         if (xfer->status != USBD_NOMEM && ehci_active_intr_list(ex)) {
3118                 ehci_del_intr_list(ex); /* remove from active list */
3119                 ehci_free_sqtd_chain(sc, epipe->sqh, ex->sqtdstart,
3120                     ex->sqtdend->nextqtd);
3121         }
3122
3123         DPRINTFN(5, ("ehci_bulk_done: length=%d\n", xfer->actlen));
3124 }
3125
3126 /************************/
3127
3128 static usbd_status
3129 ehci_device_setintr(ehci_softc_t *sc, ehci_soft_qh_t *sqh, int ival)
3130 {
3131         struct ehci_soft_islot *isp;
3132         int islot, lev;
3133
3134         /* Find a poll rate that is large enough. */
3135         for (lev = EHCI_IPOLLRATES - 1; lev > 0; lev--)
3136                 if (EHCI_ILEV_IVAL(lev) <= ival)
3137                         break;
3138
3139         /* Pick an interrupt slot at the right level. */
3140         /* XXX could do better than picking at random. */
3141         islot = EHCI_IQHIDX(lev, arc4random());
3142
3143         sqh->islot = islot;
3144         isp = &sc->sc_islots[islot];
3145         ehci_add_qh(sqh, isp->sqh);
3146
3147         return (USBD_NORMAL_COMPLETION);
3148 }
3149
3150 static usbd_status
3151 ehci_device_intr_transfer(usbd_xfer_handle xfer)
3152 {
3153         usbd_status err;
3154
3155         /* Insert last in queue. */
3156         err = usb_insert_transfer(xfer);
3157         if (err)
3158                 return (err);
3159
3160         /*
3161          * Pipe isn't running (otherwise err would be USBD_INPROG),
3162          * so start it first.
3163          */
3164         return (ehci_device_intr_start(STAILQ_FIRST(&xfer->pipe->queue)));
3165 }
3166
3167 static usbd_status
3168 ehci_device_intr_start(usbd_xfer_handle xfer)
3169 {
3170 #define exfer EXFER(xfer)
3171         struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
3172         usbd_device_handle dev = xfer->pipe->device;
3173         ehci_softc_t *sc = (ehci_softc_t *)dev->bus;
3174         ehci_soft_qtd_t *data, *dataend, *newinactive;
3175         ehci_soft_qh_t *sqh;
3176         usbd_status err;
3177         int len, isread, endpt;
3178         int s;
3179
3180         DPRINTFN(2, ("ehci_device_intr_start: xfer=%p len=%d flags=%d\n",
3181             xfer, xfer->length, xfer->flags));
3182
3183         if (sc->sc_dying)
3184                 return (USBD_IOERROR);
3185
3186 #ifdef DIAGNOSTIC
3187         if (xfer->rqflags & URQ_REQUEST)
3188                 panic("ehci_device_intr_start: a request");
3189 #endif
3190
3191         len = xfer->length;
3192         endpt = epipe->pipe.endpoint->edesc->bEndpointAddress;
3193         isread = UE_GET_DIR(endpt) == UE_DIR_IN;
3194         sqh = epipe->sqh;
3195
3196         epipe->u.intr.length = len;
3197
3198         newinactive = ehci_alloc_sqtd(sc);
3199         if (newinactive == NULL) {
3200                 DPRINTFN(-1,("ehci_device_intr_start: no sqtd memory\n"));
3201                 err = USBD_NOMEM;
3202                 xfer->status = err;
3203                 usb_transfer_complete(xfer);
3204                 return (err);
3205         }
3206         newinactive->qtd.qtd_status = htole32(0);
3207         newinactive->qtd.qtd_next = EHCI_NULL;
3208         newinactive->qtd.qtd_altnext = EHCI_NULL;
3209         err = ehci_alloc_sqtd_chain(epipe, sc, len, isread, xfer,
3210             sqh->inactivesqtd, newinactive, &data, &dataend);
3211         if (err) {
3212                 DPRINTFN(-1, ("ehci_device_intr_start: no memory\n"));
3213                 xfer->status = err;
3214                 usb_transfer_complete(xfer);
3215                 return (err);
3216         }
3217         dataend->nextqtd = newinactive;
3218         dataend->qtd.qtd_next = htole32(newinactive->physaddr);
3219         dataend->qtd.qtd_altnext = htole32(newinactive->physaddr);
3220         sqh->inactivesqtd = newinactive;
3221
3222 #ifdef EHCI_DEBUG
3223         if (ehcidebug > 5) {
3224                 DPRINTF(("ehci_device_intr_start: data(1)\n"));
3225                 ehci_dump_sqh(sqh);
3226                 ehci_dump_sqtds(data);
3227         }
3228 #endif
3229
3230         /* Set up interrupt info. */
3231         exfer->sqtdstart = data;
3232         exfer->sqtdend = dataend;
3233 #ifdef DIAGNOSTIC
3234         if (!exfer->isdone) {
3235                 printf("ehci_device_intr_start: not done, ex=%p\n", exfer);
3236         }
3237         exfer->isdone = 0;
3238 #endif
3239
3240         s = splusb();
3241         ehci_activate_qh(sqh, data);
3242         if (xfer->timeout && !sc->sc_bus.use_polling) {
3243                 callout_reset(&xfer->timeout_handle, MS_TO_TICKS(xfer->timeout),
3244                     ehci_timeout, xfer);
3245         }
3246         ehci_add_intr_list(sc, exfer);
3247         xfer->status = USBD_IN_PROGRESS;
3248         splx(s);
3249
3250 #ifdef EHCI_DEBUG
3251         if (ehcidebug > 10) {
3252                 DPRINTF(("ehci_device_intr_start: data(2)\n"));
3253                 delay(10000);
3254                 DPRINTF(("ehci_device_intr_start: data(3)\n"));
3255                 ehci_dump_regs(sc);
3256                 printf("sqh:\n");
3257                 ehci_dump_sqh(sqh);
3258                 ehci_dump_sqtds(data);
3259         }
3260 #endif
3261
3262         if (sc->sc_bus.use_polling)
3263                 ehci_waitintr(sc, xfer);
3264
3265         return (USBD_IN_PROGRESS);
3266 #undef exfer
3267 }
3268
3269 static void
3270 ehci_device_intr_abort(usbd_xfer_handle xfer)
3271 {
3272         DPRINTFN(1, ("ehci_device_intr_abort: xfer=%p\n", xfer));
3273         if (xfer->pipe->intrxfer == xfer) {
3274                 DPRINTFN(1, ("ehci_device_intr_abort: remove\n"));
3275                 xfer->pipe->intrxfer = NULL;
3276         }
3277         ehci_abort_xfer(xfer, USBD_CANCELLED);
3278 }
3279
3280 static void
3281 ehci_device_intr_close(usbd_pipe_handle pipe)
3282 {
3283         ehci_softc_t *sc = (ehci_softc_t *)pipe->device->bus;
3284         struct ehci_pipe *epipe = (struct ehci_pipe *)pipe;
3285         struct ehci_soft_islot *isp;
3286
3287         isp = &sc->sc_islots[epipe->sqh->islot];
3288         ehci_close_pipe(pipe, isp->sqh);
3289 }
3290
3291 static void
3292 ehci_device_intr_done(usbd_xfer_handle xfer)
3293 {
3294 #define exfer EXFER(xfer)
3295         struct ehci_xfer *ex = EXFER(xfer);
3296         ehci_softc_t *sc = (ehci_softc_t *)xfer->pipe->device->bus;
3297         struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
3298         ehci_soft_qtd_t *data, *dataend, *newinactive;
3299         ehci_soft_qh_t *sqh;
3300         usbd_status err;
3301         int len, isread, endpt, s;
3302
3303         DPRINTFN(10, ("ehci_device_intr_done: xfer=%p, actlen=%d\n",
3304             xfer, xfer->actlen));
3305
3306         sqh = epipe->sqh;
3307         if (xfer->pipe->repeat) {
3308                 ehci_free_sqtd_chain(sc, sqh, ex->sqtdstart,
3309                     ex->sqtdend->nextqtd);
3310
3311                 len = epipe->u.intr.length;
3312                 xfer->length = len;
3313                 endpt = epipe->pipe.endpoint->edesc->bEndpointAddress;
3314                 isread = UE_GET_DIR(endpt) == UE_DIR_IN;
3315
3316                 newinactive = ehci_alloc_sqtd(sc);
3317                 if (newinactive == NULL) {
3318                         DPRINTFN(-1,
3319                             ("ehci_device_intr_done: no sqtd memory\n"));
3320                         err = USBD_NOMEM;
3321                         xfer->status = err;
3322                         return;
3323                 }
3324                 newinactive->qtd.qtd_status = htole32(0);
3325                 newinactive->qtd.qtd_next = EHCI_NULL;
3326                 newinactive->qtd.qtd_altnext = EHCI_NULL;
3327                 err = ehci_alloc_sqtd_chain(epipe, sc, len, isread, xfer,
3328                     sqh->inactivesqtd, newinactive, &data, &dataend);
3329                 if (err) {
3330                         DPRINTFN(-1, ("ehci_device_intr_done: no memory\n"));
3331                         xfer->status = err;
3332                         return;
3333                 }
3334                 dataend->nextqtd = newinactive;
3335                 dataend->qtd.qtd_next = htole32(newinactive->physaddr);
3336                 dataend->qtd.qtd_altnext = htole32(newinactive->physaddr);
3337                 sqh->inactivesqtd = newinactive;
3338
3339                 /* Set up interrupt info. */
3340                 exfer->sqtdstart = data;
3341                 exfer->sqtdend = dataend;
3342 #ifdef DIAGNOSTIC
3343                 if (!exfer->isdone) {
3344                         printf("ehci_device_intr_done: not done, ex=%p\n",
3345                             exfer);
3346                 }
3347                 exfer->isdone = 0;
3348 #endif
3349
3350                 s = splusb();
3351                 ehci_activate_qh(sqh, data);
3352                 if (xfer->timeout && !sc->sc_bus.use_polling) {
3353                         callout_reset(&xfer->timeout_handle,
3354                             MS_TO_TICKS(xfer->timeout), ehci_timeout, xfer);
3355                 }
3356                 splx(s);
3357
3358                 xfer->status = USBD_IN_PROGRESS;
3359         } else if (xfer->status != USBD_NOMEM && ehci_active_intr_list(ex)) {
3360                 ehci_del_intr_list(ex); /* remove from active list */
3361                 ehci_free_sqtd_chain(sc, sqh, ex->sqtdstart,
3362                     ex->sqtdend->nextqtd);
3363         }
3364 #undef exfer
3365 }
3366
3367 /************************/
3368
3369 static usbd_status      ehci_device_isoc_transfer(usbd_xfer_handle xfer) { return USBD_IOERROR; }
3370 static usbd_status      ehci_device_isoc_start(usbd_xfer_handle xfer) { return USBD_IOERROR; }
3371 static void             ehci_device_isoc_abort(usbd_xfer_handle xfer) { }
3372 static void             ehci_device_isoc_close(usbd_pipe_handle pipe) { }
3373 static void             ehci_device_isoc_done(usbd_xfer_handle xfer) { }