]> CyberLeo.Net >> Repos - FreeBSD/stable/8.git/blob - sys/dev/usb/controller/ehci.c
MFC r199059
[FreeBSD/stable/8.git] / sys / dev / usb / controller / ehci.c
1 /*-
2  * Copyright (c) 2008 Hans Petter Selasky. All rights reserved.
3  * Copyright (c) 2004 The NetBSD Foundation, Inc. All rights reserved.
4  * Copyright (c) 2004 Lennart Augustsson. All rights reserved.
5  * Copyright (c) 2004 Charles M. Hannum. All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  *
16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26  * SUCH DAMAGE.
27  */
28
29 /*
30  * USB Enhanced Host Controller Driver, a.k.a. USB 2.0 controller.
31  *
32  * The EHCI 0.96 spec can be found at
33  * http://developer.intel.com/technology/usb/download/ehci-r096.pdf
34  * The EHCI 1.0 spec can be found at
35  * http://developer.intel.com/technology/usb/download/ehci-r10.pdf
36  * and the USB 2.0 spec at
37  * http://www.usb.org/developers/docs/usb_20.zip
38  *
39  */
40
41 /*
42  * TODO: 
43  * 1) command failures are not recovered correctly
44  */
45
46 #include <sys/cdefs.h>
47 __FBSDID("$FreeBSD$");
48
49 #include <sys/stdint.h>
50 #include <sys/stddef.h>
51 #include <sys/param.h>
52 #include <sys/queue.h>
53 #include <sys/types.h>
54 #include <sys/systm.h>
55 #include <sys/kernel.h>
56 #include <sys/bus.h>
57 #include <sys/linker_set.h>
58 #include <sys/module.h>
59 #include <sys/lock.h>
60 #include <sys/mutex.h>
61 #include <sys/condvar.h>
62 #include <sys/sysctl.h>
63 #include <sys/sx.h>
64 #include <sys/unistd.h>
65 #include <sys/callout.h>
66 #include <sys/malloc.h>
67 #include <sys/priv.h>
68
69 #include <dev/usb/usb.h>
70 #include <dev/usb/usbdi.h>
71
72 #define USB_DEBUG_VAR ehcidebug
73
74 #include <dev/usb/usb_core.h>
75 #include <dev/usb/usb_debug.h>
76 #include <dev/usb/usb_busdma.h>
77 #include <dev/usb/usb_process.h>
78 #include <dev/usb/usb_transfer.h>
79 #include <dev/usb/usb_device.h>
80 #include <dev/usb/usb_hub.h>
81 #include <dev/usb/usb_util.h>
82
83 #include <dev/usb/usb_controller.h>
84 #include <dev/usb/usb_bus.h>
85 #include <dev/usb/controller/ehci.h>
86 #include <dev/usb/controller/ehcireg.h>
87
88 #define EHCI_BUS2SC(bus) \
89    ((ehci_softc_t *)(((uint8_t *)(bus)) - \
90     ((uint8_t *)&(((ehci_softc_t *)0)->sc_bus))))
91
92 #if USB_DEBUG
93 static int ehcidebug = 0;
94 static int ehcinohighspeed = 0;
95
96 SYSCTL_NODE(_hw_usb, OID_AUTO, ehci, CTLFLAG_RW, 0, "USB ehci");
97 SYSCTL_INT(_hw_usb_ehci, OID_AUTO, debug, CTLFLAG_RW,
98     &ehcidebug, 0, "Debug level");
99 SYSCTL_INT(_hw_usb_ehci, OID_AUTO, no_hs, CTLFLAG_RW,
100     &ehcinohighspeed, 0, "Disable High Speed USB");
101
102 static void ehci_dump_regs(ehci_softc_t *sc);
103 static void ehci_dump_sqh(ehci_softc_t *sc, ehci_qh_t *sqh);
104
105 #endif
106
107 #define EHCI_INTR_ENDPT 1
108
109 extern struct usb_bus_methods ehci_bus_methods;
110 extern struct usb_pipe_methods ehci_device_bulk_methods;
111 extern struct usb_pipe_methods ehci_device_ctrl_methods;
112 extern struct usb_pipe_methods ehci_device_intr_methods;
113 extern struct usb_pipe_methods ehci_device_isoc_fs_methods;
114 extern struct usb_pipe_methods ehci_device_isoc_hs_methods;
115
116 static void ehci_do_poll(struct usb_bus *);
117 static void ehci_device_done(struct usb_xfer *, usb_error_t);
118 static uint8_t ehci_check_transfer(struct usb_xfer *);
119 static void ehci_timeout(void *);
120 static void ehci_poll_timeout(void *);
121
122 static void ehci_root_intr(ehci_softc_t *sc);
123
124 struct ehci_std_temp {
125         ehci_softc_t *sc;
126         struct usb_page_cache *pc;
127         ehci_qtd_t *td;
128         ehci_qtd_t *td_next;
129         uint32_t average;
130         uint32_t qtd_status;
131         uint32_t len;
132         uint16_t max_frame_size;
133         uint8_t shortpkt;
134         uint8_t auto_data_toggle;
135         uint8_t setup_alt_next;
136         uint8_t last_frame;
137         uint8_t can_use_next;
138 };
139
140 void
141 ehci_iterate_hw_softc(struct usb_bus *bus, usb_bus_mem_sub_cb_t *cb)
142 {
143         ehci_softc_t *sc = EHCI_BUS2SC(bus);
144         uint32_t i;
145
146         cb(bus, &sc->sc_hw.pframes_pc, &sc->sc_hw.pframes_pg,
147             sizeof(uint32_t) * EHCI_FRAMELIST_COUNT, EHCI_FRAMELIST_ALIGN);
148
149         cb(bus, &sc->sc_hw.async_start_pc, &sc->sc_hw.async_start_pg,
150             sizeof(ehci_qh_t), EHCI_QH_ALIGN);
151
152         for (i = 0; i != EHCI_VIRTUAL_FRAMELIST_COUNT; i++) {
153                 cb(bus, sc->sc_hw.intr_start_pc + i,
154                     sc->sc_hw.intr_start_pg + i,
155                     sizeof(ehci_qh_t), EHCI_QH_ALIGN);
156         }
157
158         for (i = 0; i != EHCI_VIRTUAL_FRAMELIST_COUNT; i++) {
159                 cb(bus, sc->sc_hw.isoc_hs_start_pc + i,
160                     sc->sc_hw.isoc_hs_start_pg + i,
161                     sizeof(ehci_itd_t), EHCI_ITD_ALIGN);
162         }
163
164         for (i = 0; i != EHCI_VIRTUAL_FRAMELIST_COUNT; i++) {
165                 cb(bus, sc->sc_hw.isoc_fs_start_pc + i,
166                     sc->sc_hw.isoc_fs_start_pg + i,
167                     sizeof(ehci_sitd_t), EHCI_SITD_ALIGN);
168         }
169 }
170
171 usb_error_t
172 ehci_reset(ehci_softc_t *sc)
173 {
174         uint32_t hcr;
175         int i;
176
177         EOWRITE4(sc, EHCI_USBCMD, EHCI_CMD_HCRESET);
178         for (i = 0; i < 100; i++) {
179                 usb_pause_mtx(NULL, hz / 1000);
180                 hcr = EOREAD4(sc, EHCI_USBCMD) & EHCI_CMD_HCRESET;
181                 if (!hcr) {
182                         if (sc->sc_flags & (EHCI_SCFLG_SETMODE | EHCI_SCFLG_BIGEMMIO)) {
183                                 /*
184                                  * Force USBMODE as requested.  Controllers
185                                  * may have multiple operating modes.
186                                  */
187                                 uint32_t usbmode = EOREAD4(sc, EHCI_USBMODE);
188                                 if (sc->sc_flags & EHCI_SCFLG_SETMODE) {
189                                         usbmode = (usbmode &~ EHCI_UM_CM) | EHCI_UM_CM_HOST;
190                                         device_printf(sc->sc_bus.bdev,
191                                             "set host controller mode\n");
192                                 }
193                                 if (sc->sc_flags & EHCI_SCFLG_BIGEMMIO) {
194                                         usbmode = (usbmode &~ EHCI_UM_ES) | EHCI_UM_ES_BE;
195                                         device_printf(sc->sc_bus.bdev,
196                                             "set big-endian mode\n");
197                                 }
198                                 EOWRITE4(sc,  EHCI_USBMODE, usbmode);
199                         }
200                         return (0);
201                 }
202         }
203         device_printf(sc->sc_bus.bdev, "reset timeout\n");
204         return (USB_ERR_IOERROR);
205 }
206
207 static usb_error_t
208 ehci_hcreset(ehci_softc_t *sc)
209 {
210         uint32_t hcr;
211         int i;
212
213         EOWRITE4(sc, EHCI_USBCMD, 0);   /* Halt controller */
214         for (i = 0; i < 100; i++) {
215                 usb_pause_mtx(NULL, hz / 1000);
216                 hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
217                 if (hcr)
218                         break;
219         }
220         if (!hcr)
221                 /*
222                  * Fall through and try reset anyway even though
223                  * Table 2-9 in the EHCI spec says this will result
224                  * in undefined behavior.
225                  */
226                 device_printf(sc->sc_bus.bdev, "stop timeout\n");
227
228         return ehci_reset(sc);
229 }
230
231 usb_error_t
232 ehci_init(ehci_softc_t *sc)
233 {
234         struct usb_page_search buf_res;
235         uint32_t version;
236         uint32_t sparams;
237         uint32_t cparams;
238         uint32_t hcr;
239         uint16_t i;
240         uint16_t x;
241         uint16_t y;
242         uint16_t bit;
243         usb_error_t err = 0;
244
245         DPRINTF("start\n");
246
247         usb_callout_init_mtx(&sc->sc_tmo_pcd, &sc->sc_bus.bus_mtx, 0);
248         usb_callout_init_mtx(&sc->sc_tmo_poll, &sc->sc_bus.bus_mtx, 0);
249
250 #if USB_DEBUG
251         if (ehcidebug > 2) {
252                 ehci_dump_regs(sc);
253         }
254 #endif
255
256         sc->sc_offs = EREAD1(sc, EHCI_CAPLENGTH);
257
258         version = EREAD2(sc, EHCI_HCIVERSION);
259         device_printf(sc->sc_bus.bdev, "EHCI version %x.%x\n",
260             version >> 8, version & 0xff);
261
262         sparams = EREAD4(sc, EHCI_HCSPARAMS);
263         DPRINTF("sparams=0x%x\n", sparams);
264
265         sc->sc_noport = EHCI_HCS_N_PORTS(sparams);
266         cparams = EREAD4(sc, EHCI_HCCPARAMS);
267         DPRINTF("cparams=0x%x\n", cparams);
268
269         if (EHCI_HCC_64BIT(cparams)) {
270                 DPRINTF("HCC uses 64-bit structures\n");
271
272                 /* MUST clear segment register if 64 bit capable */
273                 EWRITE4(sc, EHCI_CTRLDSSEGMENT, 0);
274         }
275         sc->sc_bus.usbrev = USB_REV_2_0;
276
277         /* Reset the controller */
278         DPRINTF("%s: resetting\n", device_get_nameunit(sc->sc_bus.bdev));
279
280         err = ehci_hcreset(sc);
281         if (err) {
282                 device_printf(sc->sc_bus.bdev, "reset timeout\n");
283                 return (err);
284         }
285         /*
286          * use current frame-list-size selection 0: 1024*4 bytes 1:  512*4
287          * bytes 2:  256*4 bytes 3:      unknown
288          */
289         if (EHCI_CMD_FLS(EOREAD4(sc, EHCI_USBCMD)) == 3) {
290                 device_printf(sc->sc_bus.bdev, "invalid frame-list-size\n");
291                 return (USB_ERR_IOERROR);
292         }
293         /* set up the bus struct */
294         sc->sc_bus.methods = &ehci_bus_methods;
295
296         sc->sc_eintrs = EHCI_NORMAL_INTRS;
297
298         for (i = 0; i < EHCI_VIRTUAL_FRAMELIST_COUNT; i++) {
299                 ehci_qh_t *qh;
300
301                 usbd_get_page(sc->sc_hw.intr_start_pc + i, 0, &buf_res);
302
303                 qh = buf_res.buffer;
304
305                 /* initialize page cache pointer */
306
307                 qh->page_cache = sc->sc_hw.intr_start_pc + i;
308
309                 /* store a pointer to queue head */
310
311                 sc->sc_intr_p_last[i] = qh;
312
313                 qh->qh_self =
314                     htohc32(sc, buf_res.physaddr) |
315                     htohc32(sc, EHCI_LINK_QH);
316
317                 qh->qh_endp =
318                     htohc32(sc, EHCI_QH_SET_EPS(EHCI_QH_SPEED_HIGH));
319                 qh->qh_endphub =
320                     htohc32(sc, EHCI_QH_SET_MULT(1));
321                 qh->qh_curqtd = 0;
322
323                 qh->qh_qtd.qtd_next =
324                     htohc32(sc, EHCI_LINK_TERMINATE);
325                 qh->qh_qtd.qtd_altnext =
326                     htohc32(sc, EHCI_LINK_TERMINATE);
327                 qh->qh_qtd.qtd_status =
328                     htohc32(sc, EHCI_QTD_HALTED);
329         }
330
331         /*
332          * the QHs are arranged to give poll intervals that are
333          * powers of 2 times 1ms
334          */
335         bit = EHCI_VIRTUAL_FRAMELIST_COUNT / 2;
336         while (bit) {
337                 x = bit;
338                 while (x & bit) {
339                         ehci_qh_t *qh_x;
340                         ehci_qh_t *qh_y;
341
342                         y = (x ^ bit) | (bit / 2);
343
344                         qh_x = sc->sc_intr_p_last[x];
345                         qh_y = sc->sc_intr_p_last[y];
346
347                         /*
348                          * the next QH has half the poll interval
349                          */
350                         qh_x->qh_link = qh_y->qh_self;
351
352                         x++;
353                 }
354                 bit >>= 1;
355         }
356
357         if (1) {
358                 ehci_qh_t *qh;
359
360                 qh = sc->sc_intr_p_last[0];
361
362                 /* the last (1ms) QH terminates */
363                 qh->qh_link = htohc32(sc, EHCI_LINK_TERMINATE);
364         }
365         for (i = 0; i < EHCI_VIRTUAL_FRAMELIST_COUNT; i++) {
366                 ehci_sitd_t *sitd;
367                 ehci_itd_t *itd;
368
369                 usbd_get_page(sc->sc_hw.isoc_fs_start_pc + i, 0, &buf_res);
370
371                 sitd = buf_res.buffer;
372
373                 /* initialize page cache pointer */
374
375                 sitd->page_cache = sc->sc_hw.isoc_fs_start_pc + i;
376
377                 /* store a pointer to the transfer descriptor */
378
379                 sc->sc_isoc_fs_p_last[i] = sitd;
380
381                 /* initialize full speed isochronous */
382
383                 sitd->sitd_self =
384                     htohc32(sc, buf_res.physaddr) |
385                     htohc32(sc, EHCI_LINK_SITD);
386
387                 sitd->sitd_back =
388                     htohc32(sc, EHCI_LINK_TERMINATE);
389
390                 sitd->sitd_next =
391                     sc->sc_intr_p_last[i | (EHCI_VIRTUAL_FRAMELIST_COUNT / 2)]->qh_self;
392
393
394                 usbd_get_page(sc->sc_hw.isoc_hs_start_pc + i, 0, &buf_res);
395
396                 itd = buf_res.buffer;
397
398                 /* initialize page cache pointer */
399
400                 itd->page_cache = sc->sc_hw.isoc_hs_start_pc + i;
401
402                 /* store a pointer to the transfer descriptor */
403
404                 sc->sc_isoc_hs_p_last[i] = itd;
405
406                 /* initialize high speed isochronous */
407
408                 itd->itd_self =
409                     htohc32(sc, buf_res.physaddr) |
410                     htohc32(sc, EHCI_LINK_ITD);
411
412                 itd->itd_next =
413                     sitd->sitd_self;
414         }
415
416         usbd_get_page(&sc->sc_hw.pframes_pc, 0, &buf_res);
417
418         if (1) {
419                 uint32_t *pframes;
420
421                 pframes = buf_res.buffer;
422
423                 /*
424                  * execution order:
425                  * pframes -> high speed isochronous ->
426                  *    full speed isochronous -> interrupt QH's
427                  */
428                 for (i = 0; i < EHCI_FRAMELIST_COUNT; i++) {
429                         pframes[i] = sc->sc_isoc_hs_p_last
430                             [i & (EHCI_VIRTUAL_FRAMELIST_COUNT - 1)]->itd_self;
431                 }
432         }
433         /* setup sync list pointer */
434         EOWRITE4(sc, EHCI_PERIODICLISTBASE, buf_res.physaddr);
435
436         usbd_get_page(&sc->sc_hw.async_start_pc, 0, &buf_res);
437
438         if (1) {
439
440                 ehci_qh_t *qh;
441
442                 qh = buf_res.buffer;
443
444                 /* initialize page cache pointer */
445
446                 qh->page_cache = &sc->sc_hw.async_start_pc;
447
448                 /* store a pointer to the queue head */
449
450                 sc->sc_async_p_last = qh;
451
452                 /* init dummy QH that starts the async list */
453
454                 qh->qh_self =
455                     htohc32(sc, buf_res.physaddr) |
456                     htohc32(sc, EHCI_LINK_QH);
457
458                 /* fill the QH */
459                 qh->qh_endp =
460                     htohc32(sc, EHCI_QH_SET_EPS(EHCI_QH_SPEED_HIGH) | EHCI_QH_HRECL);
461                 qh->qh_endphub = htohc32(sc, EHCI_QH_SET_MULT(1));
462                 qh->qh_link = qh->qh_self;
463                 qh->qh_curqtd = 0;
464
465                 /* fill the overlay qTD */
466                 qh->qh_qtd.qtd_next = htohc32(sc, EHCI_LINK_TERMINATE);
467                 qh->qh_qtd.qtd_altnext = htohc32(sc, EHCI_LINK_TERMINATE);
468                 qh->qh_qtd.qtd_status = htohc32(sc, EHCI_QTD_HALTED);
469         }
470         /* flush all cache into memory */
471
472         usb_bus_mem_flush_all(&sc->sc_bus, &ehci_iterate_hw_softc);
473
474 #if USB_DEBUG
475         if (ehcidebug) {
476                 ehci_dump_sqh(sc, sc->sc_async_p_last);
477         }
478 #endif
479
480         /* setup async list pointer */
481         EOWRITE4(sc, EHCI_ASYNCLISTADDR, buf_res.physaddr | EHCI_LINK_QH);
482
483
484         /* enable interrupts */
485         EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
486
487         /* turn on controller */
488         EOWRITE4(sc, EHCI_USBCMD,
489             EHCI_CMD_ITC_1 |            /* 1 microframes interrupt delay */
490             (EOREAD4(sc, EHCI_USBCMD) & EHCI_CMD_FLS_M) |
491             EHCI_CMD_ASE |
492             EHCI_CMD_PSE |
493             EHCI_CMD_RS);
494
495         /* Take over port ownership */
496         EOWRITE4(sc, EHCI_CONFIGFLAG, EHCI_CONF_CF);
497
498         for (i = 0; i < 100; i++) {
499                 usb_pause_mtx(NULL, hz / 1000);
500                 hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
501                 if (!hcr) {
502                         break;
503                 }
504         }
505         if (hcr) {
506                 device_printf(sc->sc_bus.bdev, "run timeout\n");
507                 return (USB_ERR_IOERROR);
508         }
509
510         if (!err) {
511                 /* catch any lost interrupts */
512                 ehci_do_poll(&sc->sc_bus);
513         }
514         return (err);
515 }
516
517 /*
518  * shut down the controller when the system is going down
519  */
520 void
521 ehci_detach(ehci_softc_t *sc)
522 {
523         USB_BUS_LOCK(&sc->sc_bus);
524
525         usb_callout_stop(&sc->sc_tmo_pcd);
526         usb_callout_stop(&sc->sc_tmo_poll);
527
528         EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
529         USB_BUS_UNLOCK(&sc->sc_bus);
530
531         if (ehci_hcreset(sc)) {
532                 DPRINTF("reset failed!\n");
533         }
534
535         /* XXX let stray task complete */
536         usb_pause_mtx(NULL, hz / 20);
537
538         usb_callout_drain(&sc->sc_tmo_pcd);
539         usb_callout_drain(&sc->sc_tmo_poll);
540 }
541
542 void
543 ehci_suspend(ehci_softc_t *sc)
544 {
545         uint32_t cmd;
546         uint32_t hcr;
547         uint8_t i;
548
549         USB_BUS_LOCK(&sc->sc_bus);
550
551         for (i = 1; i <= sc->sc_noport; i++) {
552                 cmd = EOREAD4(sc, EHCI_PORTSC(i));
553                 if (((cmd & EHCI_PS_PO) == 0) &&
554                     ((cmd & EHCI_PS_PE) == EHCI_PS_PE)) {
555                         EOWRITE4(sc, EHCI_PORTSC(i),
556                             cmd | EHCI_PS_SUSP);
557                 }
558         }
559
560         sc->sc_cmd = EOREAD4(sc, EHCI_USBCMD);
561
562         cmd = sc->sc_cmd & ~(EHCI_CMD_ASE | EHCI_CMD_PSE);
563         EOWRITE4(sc, EHCI_USBCMD, cmd);
564
565         for (i = 0; i < 100; i++) {
566                 hcr = EOREAD4(sc, EHCI_USBSTS) &
567                     (EHCI_STS_ASS | EHCI_STS_PSS);
568
569                 if (hcr == 0) {
570                         break;
571                 }
572                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 1000);
573         }
574
575         if (hcr != 0) {
576                 device_printf(sc->sc_bus.bdev, "reset timeout\n");
577         }
578         cmd &= ~EHCI_CMD_RS;
579         EOWRITE4(sc, EHCI_USBCMD, cmd);
580
581         for (i = 0; i < 100; i++) {
582                 hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
583                 if (hcr == EHCI_STS_HCH) {
584                         break;
585                 }
586                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 1000);
587         }
588
589         if (hcr != EHCI_STS_HCH) {
590                 device_printf(sc->sc_bus.bdev,
591                     "config timeout\n");
592         }
593         USB_BUS_UNLOCK(&sc->sc_bus);
594 }
595
596 void
597 ehci_resume(ehci_softc_t *sc)
598 {
599         struct usb_page_search buf_res;
600         uint32_t cmd;
601         uint32_t hcr;
602         uint8_t i;
603
604         USB_BUS_LOCK(&sc->sc_bus);
605
606         /* restore things in case the bios doesn't */
607         EOWRITE4(sc, EHCI_CTRLDSSEGMENT, 0);
608
609         usbd_get_page(&sc->sc_hw.pframes_pc, 0, &buf_res);
610         EOWRITE4(sc, EHCI_PERIODICLISTBASE, buf_res.physaddr);
611
612         usbd_get_page(&sc->sc_hw.async_start_pc, 0, &buf_res);
613         EOWRITE4(sc, EHCI_ASYNCLISTADDR, buf_res.physaddr | EHCI_LINK_QH);
614
615         EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
616
617         hcr = 0;
618         for (i = 1; i <= sc->sc_noport; i++) {
619                 cmd = EOREAD4(sc, EHCI_PORTSC(i));
620                 if (((cmd & EHCI_PS_PO) == 0) &&
621                     ((cmd & EHCI_PS_SUSP) == EHCI_PS_SUSP)) {
622                         EOWRITE4(sc, EHCI_PORTSC(i),
623                             cmd | EHCI_PS_FPR);
624                         hcr = 1;
625                 }
626         }
627
628         if (hcr) {
629                 usb_pause_mtx(&sc->sc_bus.bus_mtx,
630                     USB_MS_TO_TICKS(USB_RESUME_WAIT));
631
632                 for (i = 1; i <= sc->sc_noport; i++) {
633                         cmd = EOREAD4(sc, EHCI_PORTSC(i));
634                         if (((cmd & EHCI_PS_PO) == 0) &&
635                             ((cmd & EHCI_PS_SUSP) == EHCI_PS_SUSP)) {
636                                 EOWRITE4(sc, EHCI_PORTSC(i),
637                                     cmd & ~EHCI_PS_FPR);
638                         }
639                 }
640         }
641         EOWRITE4(sc, EHCI_USBCMD, sc->sc_cmd);
642
643         for (i = 0; i < 100; i++) {
644                 hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
645                 if (hcr != EHCI_STS_HCH) {
646                         break;
647                 }
648                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 1000);
649         }
650         if (hcr == EHCI_STS_HCH) {
651                 device_printf(sc->sc_bus.bdev, "config timeout\n");
652         }
653
654         USB_BUS_UNLOCK(&sc->sc_bus);
655
656         usb_pause_mtx(NULL,
657             USB_MS_TO_TICKS(USB_RESUME_WAIT));
658
659         /* catch any lost interrupts */
660         ehci_do_poll(&sc->sc_bus);
661 }
662
663 void
664 ehci_shutdown(ehci_softc_t *sc)
665 {
666         DPRINTF("stopping the HC\n");
667
668         if (ehci_hcreset(sc)) {
669                 DPRINTF("reset failed!\n");
670         }
671 }
672
673 #if USB_DEBUG
674 static void
675 ehci_dump_regs(ehci_softc_t *sc)
676 {
677         uint32_t i;
678
679         i = EOREAD4(sc, EHCI_USBCMD);
680         printf("cmd=0x%08x\n", i);
681
682         if (i & EHCI_CMD_ITC_1)
683                 printf(" EHCI_CMD_ITC_1\n");
684         if (i & EHCI_CMD_ITC_2)
685                 printf(" EHCI_CMD_ITC_2\n");
686         if (i & EHCI_CMD_ITC_4)
687                 printf(" EHCI_CMD_ITC_4\n");
688         if (i & EHCI_CMD_ITC_8)
689                 printf(" EHCI_CMD_ITC_8\n");
690         if (i & EHCI_CMD_ITC_16)
691                 printf(" EHCI_CMD_ITC_16\n");
692         if (i & EHCI_CMD_ITC_32)
693                 printf(" EHCI_CMD_ITC_32\n");
694         if (i & EHCI_CMD_ITC_64)
695                 printf(" EHCI_CMD_ITC_64\n");
696         if (i & EHCI_CMD_ASPME)
697                 printf(" EHCI_CMD_ASPME\n");
698         if (i & EHCI_CMD_ASPMC)
699                 printf(" EHCI_CMD_ASPMC\n");
700         if (i & EHCI_CMD_LHCR)
701                 printf(" EHCI_CMD_LHCR\n");
702         if (i & EHCI_CMD_IAAD)
703                 printf(" EHCI_CMD_IAAD\n");
704         if (i & EHCI_CMD_ASE)
705                 printf(" EHCI_CMD_ASE\n");
706         if (i & EHCI_CMD_PSE)
707                 printf(" EHCI_CMD_PSE\n");
708         if (i & EHCI_CMD_FLS_M)
709                 printf(" EHCI_CMD_FLS_M\n");
710         if (i & EHCI_CMD_HCRESET)
711                 printf(" EHCI_CMD_HCRESET\n");
712         if (i & EHCI_CMD_RS)
713                 printf(" EHCI_CMD_RS\n");
714
715         i = EOREAD4(sc, EHCI_USBSTS);
716
717         printf("sts=0x%08x\n", i);
718
719         if (i & EHCI_STS_ASS)
720                 printf(" EHCI_STS_ASS\n");
721         if (i & EHCI_STS_PSS)
722                 printf(" EHCI_STS_PSS\n");
723         if (i & EHCI_STS_REC)
724                 printf(" EHCI_STS_REC\n");
725         if (i & EHCI_STS_HCH)
726                 printf(" EHCI_STS_HCH\n");
727         if (i & EHCI_STS_IAA)
728                 printf(" EHCI_STS_IAA\n");
729         if (i & EHCI_STS_HSE)
730                 printf(" EHCI_STS_HSE\n");
731         if (i & EHCI_STS_FLR)
732                 printf(" EHCI_STS_FLR\n");
733         if (i & EHCI_STS_PCD)
734                 printf(" EHCI_STS_PCD\n");
735         if (i & EHCI_STS_ERRINT)
736                 printf(" EHCI_STS_ERRINT\n");
737         if (i & EHCI_STS_INT)
738                 printf(" EHCI_STS_INT\n");
739
740         printf("ien=0x%08x\n",
741             EOREAD4(sc, EHCI_USBINTR));
742         printf("frindex=0x%08x ctrdsegm=0x%08x periodic=0x%08x async=0x%08x\n",
743             EOREAD4(sc, EHCI_FRINDEX),
744             EOREAD4(sc, EHCI_CTRLDSSEGMENT),
745             EOREAD4(sc, EHCI_PERIODICLISTBASE),
746             EOREAD4(sc, EHCI_ASYNCLISTADDR));
747         for (i = 1; i <= sc->sc_noport; i++) {
748                 printf("port %d status=0x%08x\n", i,
749                     EOREAD4(sc, EHCI_PORTSC(i)));
750         }
751 }
752
753 static void
754 ehci_dump_link(ehci_softc_t *sc, uint32_t link, int type)
755 {
756         link = hc32toh(sc, link);
757         printf("0x%08x", link);
758         if (link & EHCI_LINK_TERMINATE)
759                 printf("<T>");
760         else {
761                 printf("<");
762                 if (type) {
763                         switch (EHCI_LINK_TYPE(link)) {
764                         case EHCI_LINK_ITD:
765                                 printf("ITD");
766                                 break;
767                         case EHCI_LINK_QH:
768                                 printf("QH");
769                                 break;
770                         case EHCI_LINK_SITD:
771                                 printf("SITD");
772                                 break;
773                         case EHCI_LINK_FSTN:
774                                 printf("FSTN");
775                                 break;
776                         }
777                 }
778                 printf(">");
779         }
780 }
781
782 static void
783 ehci_dump_qtd(ehci_softc_t *sc, ehci_qtd_t *qtd)
784 {
785         uint32_t s;
786
787         printf("  next=");
788         ehci_dump_link(sc, qtd->qtd_next, 0);
789         printf(" altnext=");
790         ehci_dump_link(sc, qtd->qtd_altnext, 0);
791         printf("\n");
792         s = hc32toh(sc, qtd->qtd_status);
793         printf("  status=0x%08x: toggle=%d bytes=0x%x ioc=%d c_page=0x%x\n",
794             s, EHCI_QTD_GET_TOGGLE(s), EHCI_QTD_GET_BYTES(s),
795             EHCI_QTD_GET_IOC(s), EHCI_QTD_GET_C_PAGE(s));
796         printf("    cerr=%d pid=%d stat=%s%s%s%s%s%s%s%s\n",
797             EHCI_QTD_GET_CERR(s), EHCI_QTD_GET_PID(s),
798             (s & EHCI_QTD_ACTIVE) ? "ACTIVE" : "NOT_ACTIVE",
799             (s & EHCI_QTD_HALTED) ? "-HALTED" : "",
800             (s & EHCI_QTD_BUFERR) ? "-BUFERR" : "",
801             (s & EHCI_QTD_BABBLE) ? "-BABBLE" : "",
802             (s & EHCI_QTD_XACTERR) ? "-XACTERR" : "",
803             (s & EHCI_QTD_MISSEDMICRO) ? "-MISSED" : "",
804             (s & EHCI_QTD_SPLITXSTATE) ? "-SPLIT" : "",
805             (s & EHCI_QTD_PINGSTATE) ? "-PING" : "");
806
807         for (s = 0; s < 5; s++) {
808                 printf("  buffer[%d]=0x%08x\n", s,
809                     hc32toh(sc, qtd->qtd_buffer[s]));
810         }
811         for (s = 0; s < 5; s++) {
812                 printf("  buffer_hi[%d]=0x%08x\n", s,
813                     hc32toh(sc, qtd->qtd_buffer_hi[s]));
814         }
815 }
816
817 static uint8_t
818 ehci_dump_sqtd(ehci_softc_t *sc, ehci_qtd_t *sqtd)
819 {
820         uint8_t temp;
821
822         usb_pc_cpu_invalidate(sqtd->page_cache);
823         printf("QTD(%p) at 0x%08x:\n", sqtd, hc32toh(sc, sqtd->qtd_self));
824         ehci_dump_qtd(sc, sqtd);
825         temp = (sqtd->qtd_next & htohc32(sc, EHCI_LINK_TERMINATE)) ? 1 : 0;
826         return (temp);
827 }
828
829 static void
830 ehci_dump_sqtds(ehci_softc_t *sc, ehci_qtd_t *sqtd)
831 {
832         uint16_t i;
833         uint8_t stop;
834
835         stop = 0;
836         for (i = 0; sqtd && (i < 20) && !stop; sqtd = sqtd->obj_next, i++) {
837                 stop = ehci_dump_sqtd(sc, sqtd);
838         }
839         if (sqtd) {
840                 printf("dump aborted, too many TDs\n");
841         }
842 }
843
844 static void
845 ehci_dump_sqh(ehci_softc_t *sc, ehci_qh_t *qh)
846 {
847         uint32_t endp;
848         uint32_t endphub;
849
850         usb_pc_cpu_invalidate(qh->page_cache);
851         printf("QH(%p) at 0x%08x:\n", qh, hc32toh(sc, qh->qh_self) & ~0x1F);
852         printf("  link=");
853         ehci_dump_link(sc, qh->qh_link, 1);
854         printf("\n");
855         endp = hc32toh(sc, qh->qh_endp);
856         printf("  endp=0x%08x\n", endp);
857         printf("    addr=0x%02x inact=%d endpt=%d eps=%d dtc=%d hrecl=%d\n",
858             EHCI_QH_GET_ADDR(endp), EHCI_QH_GET_INACT(endp),
859             EHCI_QH_GET_ENDPT(endp), EHCI_QH_GET_EPS(endp),
860             EHCI_QH_GET_DTC(endp), EHCI_QH_GET_HRECL(endp));
861         printf("    mpl=0x%x ctl=%d nrl=%d\n",
862             EHCI_QH_GET_MPL(endp), EHCI_QH_GET_CTL(endp),
863             EHCI_QH_GET_NRL(endp));
864         endphub = hc32toh(sc, qh->qh_endphub);
865         printf("  endphub=0x%08x\n", endphub);
866         printf("    smask=0x%02x cmask=0x%02x huba=0x%02x port=%d mult=%d\n",
867             EHCI_QH_GET_SMASK(endphub), EHCI_QH_GET_CMASK(endphub),
868             EHCI_QH_GET_HUBA(endphub), EHCI_QH_GET_PORT(endphub),
869             EHCI_QH_GET_MULT(endphub));
870         printf("  curqtd=");
871         ehci_dump_link(sc, qh->qh_curqtd, 0);
872         printf("\n");
873         printf("Overlay qTD:\n");
874         ehci_dump_qtd(sc, (void *)&qh->qh_qtd);
875 }
876
877 static void
878 ehci_dump_sitd(ehci_softc_t *sc, ehci_sitd_t *sitd)
879 {
880         usb_pc_cpu_invalidate(sitd->page_cache);
881         printf("SITD(%p) at 0x%08x\n", sitd, hc32toh(sc, sitd->sitd_self) & ~0x1F);
882         printf(" next=0x%08x\n", hc32toh(sc, sitd->sitd_next));
883         printf(" portaddr=0x%08x dir=%s addr=%d endpt=0x%x port=0x%x huba=0x%x\n",
884             hc32toh(sc, sitd->sitd_portaddr),
885             (sitd->sitd_portaddr & htohc32(sc, EHCI_SITD_SET_DIR_IN))
886             ? "in" : "out",
887             EHCI_SITD_GET_ADDR(hc32toh(sc, sitd->sitd_portaddr)),
888             EHCI_SITD_GET_ENDPT(hc32toh(sc, sitd->sitd_portaddr)),
889             EHCI_SITD_GET_PORT(hc32toh(sc, sitd->sitd_portaddr)),
890             EHCI_SITD_GET_HUBA(hc32toh(sc, sitd->sitd_portaddr)));
891         printf(" mask=0x%08x\n", hc32toh(sc, sitd->sitd_mask));
892         printf(" status=0x%08x <%s> len=0x%x\n", hc32toh(sc, sitd->sitd_status),
893             (sitd->sitd_status & htohc32(sc, EHCI_SITD_ACTIVE)) ? "ACTIVE" : "",
894             EHCI_SITD_GET_LEN(hc32toh(sc, sitd->sitd_status)));
895         printf(" back=0x%08x, bp=0x%08x,0x%08x,0x%08x,0x%08x\n",
896             hc32toh(sc, sitd->sitd_back),
897             hc32toh(sc, sitd->sitd_bp[0]),
898             hc32toh(sc, sitd->sitd_bp[1]),
899             hc32toh(sc, sitd->sitd_bp_hi[0]),
900             hc32toh(sc, sitd->sitd_bp_hi[1]));
901 }
902
903 static void
904 ehci_dump_itd(ehci_softc_t *sc, ehci_itd_t *itd)
905 {
906         usb_pc_cpu_invalidate(itd->page_cache);
907         printf("ITD(%p) at 0x%08x\n", itd, hc32toh(sc, itd->itd_self) & ~0x1F);
908         printf(" next=0x%08x\n", hc32toh(sc, itd->itd_next));
909         printf(" status[0]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[0]),
910             (itd->itd_status[0] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
911         printf(" status[1]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[1]),
912             (itd->itd_status[1] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
913         printf(" status[2]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[2]),
914             (itd->itd_status[2] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
915         printf(" status[3]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[3]),
916             (itd->itd_status[3] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
917         printf(" status[4]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[4]),
918             (itd->itd_status[4] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
919         printf(" status[5]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[5]),
920             (itd->itd_status[5] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
921         printf(" status[6]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[6]),
922             (itd->itd_status[6] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
923         printf(" status[7]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[7]),
924             (itd->itd_status[7] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
925         printf(" bp[0]=0x%08x\n", hc32toh(sc, itd->itd_bp[0]));
926         printf("  addr=0x%02x; endpt=0x%01x\n",
927             EHCI_ITD_GET_ADDR(hc32toh(sc, itd->itd_bp[0])),
928             EHCI_ITD_GET_ENDPT(hc32toh(sc, itd->itd_bp[0])));
929         printf(" bp[1]=0x%08x\n", hc32toh(sc, itd->itd_bp[1]));
930         printf(" dir=%s; mpl=0x%02x\n",
931             (hc32toh(sc, itd->itd_bp[1]) & EHCI_ITD_SET_DIR_IN) ? "in" : "out",
932             EHCI_ITD_GET_MPL(hc32toh(sc, itd->itd_bp[1])));
933         printf(" bp[2..6]=0x%08x,0x%08x,0x%08x,0x%08x,0x%08x\n",
934             hc32toh(sc, itd->itd_bp[2]),
935             hc32toh(sc, itd->itd_bp[3]),
936             hc32toh(sc, itd->itd_bp[4]),
937             hc32toh(sc, itd->itd_bp[5]),
938             hc32toh(sc, itd->itd_bp[6]));
939         printf(" bp_hi=0x%08x,0x%08x,0x%08x,0x%08x,\n"
940             "       0x%08x,0x%08x,0x%08x\n",
941             hc32toh(sc, itd->itd_bp_hi[0]),
942             hc32toh(sc, itd->itd_bp_hi[1]),
943             hc32toh(sc, itd->itd_bp_hi[2]),
944             hc32toh(sc, itd->itd_bp_hi[3]),
945             hc32toh(sc, itd->itd_bp_hi[4]),
946             hc32toh(sc, itd->itd_bp_hi[5]),
947             hc32toh(sc, itd->itd_bp_hi[6]));
948 }
949
950 static void
951 ehci_dump_isoc(ehci_softc_t *sc)
952 {
953         ehci_itd_t *itd;
954         ehci_sitd_t *sitd;
955         uint16_t max = 1000;
956         uint16_t pos;
957
958         pos = (EOREAD4(sc, EHCI_FRINDEX) / 8) &
959             (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
960
961         printf("%s: isochronous dump from frame 0x%03x:\n",
962             __FUNCTION__, pos);
963
964         itd = sc->sc_isoc_hs_p_last[pos];
965         sitd = sc->sc_isoc_fs_p_last[pos];
966
967         while (itd && max && max--) {
968                 ehci_dump_itd(sc, itd);
969                 itd = itd->prev;
970         }
971
972         while (sitd && max && max--) {
973                 ehci_dump_sitd(sc, sitd);
974                 sitd = sitd->prev;
975         }
976 }
977
978 #endif
979
980 static void
981 ehci_transfer_intr_enqueue(struct usb_xfer *xfer)
982 {
983         /* check for early completion */
984         if (ehci_check_transfer(xfer)) {
985                 return;
986         }
987         /* put transfer on interrupt queue */
988         usbd_transfer_enqueue(&xfer->xroot->bus->intr_q, xfer);
989
990         /* start timeout, if any */
991         if (xfer->timeout != 0) {
992                 usbd_transfer_timeout_ms(xfer, &ehci_timeout, xfer->timeout);
993         }
994 }
995
996 #define EHCI_APPEND_FS_TD(std,last) (last) = _ehci_append_fs_td(std,last)
997 static ehci_sitd_t *
998 _ehci_append_fs_td(ehci_sitd_t *std, ehci_sitd_t *last)
999 {
1000         DPRINTFN(11, "%p to %p\n", std, last);
1001
1002         /* (sc->sc_bus.mtx) must be locked */
1003
1004         std->next = last->next;
1005         std->sitd_next = last->sitd_next;
1006
1007         std->prev = last;
1008
1009         usb_pc_cpu_flush(std->page_cache);
1010
1011         /*
1012          * the last->next->prev is never followed: std->next->prev = std;
1013          */
1014         last->next = std;
1015         last->sitd_next = std->sitd_self;
1016
1017         usb_pc_cpu_flush(last->page_cache);
1018
1019         return (std);
1020 }
1021
1022 #define EHCI_APPEND_HS_TD(std,last) (last) = _ehci_append_hs_td(std,last)
1023 static ehci_itd_t *
1024 _ehci_append_hs_td(ehci_itd_t *std, ehci_itd_t *last)
1025 {
1026         DPRINTFN(11, "%p to %p\n", std, last);
1027
1028         /* (sc->sc_bus.mtx) must be locked */
1029
1030         std->next = last->next;
1031         std->itd_next = last->itd_next;
1032
1033         std->prev = last;
1034
1035         usb_pc_cpu_flush(std->page_cache);
1036
1037         /*
1038          * the last->next->prev is never followed: std->next->prev = std;
1039          */
1040         last->next = std;
1041         last->itd_next = std->itd_self;
1042
1043         usb_pc_cpu_flush(last->page_cache);
1044
1045         return (std);
1046 }
1047
1048 #define EHCI_APPEND_QH(sqh,last) (last) = _ehci_append_qh(sqh,last)
1049 static ehci_qh_t *
1050 _ehci_append_qh(ehci_qh_t *sqh, ehci_qh_t *last)
1051 {
1052         DPRINTFN(11, "%p to %p\n", sqh, last);
1053
1054         if (sqh->prev != NULL) {
1055                 /* should not happen */
1056                 DPRINTFN(0, "QH already linked!\n");
1057                 return (last);
1058         }
1059         /* (sc->sc_bus.mtx) must be locked */
1060
1061         sqh->next = last->next;
1062         sqh->qh_link = last->qh_link;
1063
1064         sqh->prev = last;
1065
1066         usb_pc_cpu_flush(sqh->page_cache);
1067
1068         /*
1069          * the last->next->prev is never followed: sqh->next->prev = sqh;
1070          */
1071
1072         last->next = sqh;
1073         last->qh_link = sqh->qh_self;
1074
1075         usb_pc_cpu_flush(last->page_cache);
1076
1077         return (sqh);
1078 }
1079
1080 #define EHCI_REMOVE_FS_TD(std,last) (last) = _ehci_remove_fs_td(std,last)
1081 static ehci_sitd_t *
1082 _ehci_remove_fs_td(ehci_sitd_t *std, ehci_sitd_t *last)
1083 {
1084         DPRINTFN(11, "%p from %p\n", std, last);
1085
1086         /* (sc->sc_bus.mtx) must be locked */
1087
1088         std->prev->next = std->next;
1089         std->prev->sitd_next = std->sitd_next;
1090
1091         usb_pc_cpu_flush(std->prev->page_cache);
1092
1093         if (std->next) {
1094                 std->next->prev = std->prev;
1095                 usb_pc_cpu_flush(std->next->page_cache);
1096         }
1097         return ((last == std) ? std->prev : last);
1098 }
1099
1100 #define EHCI_REMOVE_HS_TD(std,last) (last) = _ehci_remove_hs_td(std,last)
1101 static ehci_itd_t *
1102 _ehci_remove_hs_td(ehci_itd_t *std, ehci_itd_t *last)
1103 {
1104         DPRINTFN(11, "%p from %p\n", std, last);
1105
1106         /* (sc->sc_bus.mtx) must be locked */
1107
1108         std->prev->next = std->next;
1109         std->prev->itd_next = std->itd_next;
1110
1111         usb_pc_cpu_flush(std->prev->page_cache);
1112
1113         if (std->next) {
1114                 std->next->prev = std->prev;
1115                 usb_pc_cpu_flush(std->next->page_cache);
1116         }
1117         return ((last == std) ? std->prev : last);
1118 }
1119
1120 #define EHCI_REMOVE_QH(sqh,last) (last) = _ehci_remove_qh(sqh,last)
1121 static ehci_qh_t *
1122 _ehci_remove_qh(ehci_qh_t *sqh, ehci_qh_t *last)
1123 {
1124         DPRINTFN(11, "%p from %p\n", sqh, last);
1125
1126         /* (sc->sc_bus.mtx) must be locked */
1127
1128         /* only remove if not removed from a queue */
1129         if (sqh->prev) {
1130
1131                 sqh->prev->next = sqh->next;
1132                 sqh->prev->qh_link = sqh->qh_link;
1133
1134                 usb_pc_cpu_flush(sqh->prev->page_cache);
1135
1136                 if (sqh->next) {
1137                         sqh->next->prev = sqh->prev;
1138                         usb_pc_cpu_flush(sqh->next->page_cache);
1139                 }
1140                 last = ((last == sqh) ? sqh->prev : last);
1141
1142                 sqh->prev = 0;
1143
1144                 usb_pc_cpu_flush(sqh->page_cache);
1145         }
1146         return (last);
1147 }
1148
1149 static usb_error_t
1150 ehci_non_isoc_done_sub(struct usb_xfer *xfer)
1151 {
1152         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
1153         ehci_qtd_t *td;
1154         ehci_qtd_t *td_alt_next;
1155         uint32_t status;
1156         uint16_t len;
1157
1158         td = xfer->td_transfer_cache;
1159         td_alt_next = td->alt_next;
1160
1161         if (xfer->aframes != xfer->nframes) {
1162                 usbd_xfer_set_frame_len(xfer, xfer->aframes, 0);
1163         }
1164         while (1) {
1165
1166                 usb_pc_cpu_invalidate(td->page_cache);
1167                 status = hc32toh(sc, td->qtd_status);
1168
1169                 len = EHCI_QTD_GET_BYTES(status);
1170
1171                 /*
1172                  * Verify the status length and
1173                  * add the length to "frlengths[]":
1174                  */
1175                 if (len > td->len) {
1176                         /* should not happen */
1177                         DPRINTF("Invalid status length, "
1178                             "0x%04x/0x%04x bytes\n", len, td->len);
1179                         status |= EHCI_QTD_HALTED;
1180                 } else if (xfer->aframes != xfer->nframes) {
1181                         xfer->frlengths[xfer->aframes] += td->len - len;
1182                 }
1183                 /* Check for last transfer */
1184                 if (((void *)td) == xfer->td_transfer_last) {
1185                         td = NULL;
1186                         break;
1187                 }
1188                 /* Check for transfer error */
1189                 if (status & EHCI_QTD_HALTED) {
1190                         /* the transfer is finished */
1191                         td = NULL;
1192                         break;
1193                 }
1194                 /* Check for short transfer */
1195                 if (len > 0) {
1196                         if (xfer->flags_int.short_frames_ok) {
1197                                 /* follow alt next */
1198                                 td = td->alt_next;
1199                         } else {
1200                                 /* the transfer is finished */
1201                                 td = NULL;
1202                         }
1203                         break;
1204                 }
1205                 td = td->obj_next;
1206
1207                 if (td->alt_next != td_alt_next) {
1208                         /* this USB frame is complete */
1209                         break;
1210                 }
1211         }
1212
1213         /* update transfer cache */
1214
1215         xfer->td_transfer_cache = td;
1216
1217 #if USB_DEBUG
1218         if (status & EHCI_QTD_STATERRS) {
1219                 DPRINTFN(11, "error, addr=%d, endpt=0x%02x, frame=0x%02x"
1220                     "status=%s%s%s%s%s%s%s%s\n",
1221                     xfer->address, xfer->endpointno, xfer->aframes,
1222                     (status & EHCI_QTD_ACTIVE) ? "[ACTIVE]" : "[NOT_ACTIVE]",
1223                     (status & EHCI_QTD_HALTED) ? "[HALTED]" : "",
1224                     (status & EHCI_QTD_BUFERR) ? "[BUFERR]" : "",
1225                     (status & EHCI_QTD_BABBLE) ? "[BABBLE]" : "",
1226                     (status & EHCI_QTD_XACTERR) ? "[XACTERR]" : "",
1227                     (status & EHCI_QTD_MISSEDMICRO) ? "[MISSED]" : "",
1228                     (status & EHCI_QTD_SPLITXSTATE) ? "[SPLIT]" : "",
1229                     (status & EHCI_QTD_PINGSTATE) ? "[PING]" : "");
1230         }
1231 #endif
1232
1233         return ((status & EHCI_QTD_HALTED) ?
1234             USB_ERR_STALLED : USB_ERR_NORMAL_COMPLETION);
1235 }
1236
1237 static void
1238 ehci_non_isoc_done(struct usb_xfer *xfer)
1239 {
1240         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
1241         ehci_qh_t *qh;
1242         uint32_t status;
1243         usb_error_t err = 0;
1244
1245         DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
1246             xfer, xfer->endpoint);
1247
1248 #if USB_DEBUG
1249         if (ehcidebug > 10) {
1250                 ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
1251
1252                 ehci_dump_sqtds(sc, xfer->td_transfer_first);
1253         }
1254 #endif
1255
1256         /* extract data toggle directly from the QH's overlay area */
1257
1258         qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
1259
1260         usb_pc_cpu_invalidate(qh->page_cache);
1261
1262         status = hc32toh(sc, qh->qh_qtd.qtd_status);
1263
1264         xfer->endpoint->toggle_next =
1265             (status & EHCI_QTD_TOGGLE_MASK) ? 1 : 0;
1266
1267         /* reset scanner */
1268
1269         xfer->td_transfer_cache = xfer->td_transfer_first;
1270
1271         if (xfer->flags_int.control_xfr) {
1272
1273                 if (xfer->flags_int.control_hdr) {
1274
1275                         err = ehci_non_isoc_done_sub(xfer);
1276                 }
1277                 xfer->aframes = 1;
1278
1279                 if (xfer->td_transfer_cache == NULL) {
1280                         goto done;
1281                 }
1282         }
1283         while (xfer->aframes != xfer->nframes) {
1284
1285                 err = ehci_non_isoc_done_sub(xfer);
1286                 xfer->aframes++;
1287
1288                 if (xfer->td_transfer_cache == NULL) {
1289                         goto done;
1290                 }
1291         }
1292
1293         if (xfer->flags_int.control_xfr &&
1294             !xfer->flags_int.control_act) {
1295
1296                 err = ehci_non_isoc_done_sub(xfer);
1297         }
1298 done:
1299         ehci_device_done(xfer, err);
1300 }
1301
1302 /*------------------------------------------------------------------------*
1303  *      ehci_check_transfer
1304  *
1305  * Return values:
1306  *    0: USB transfer is not finished
1307  * Else: USB transfer is finished
1308  *------------------------------------------------------------------------*/
1309 static uint8_t
1310 ehci_check_transfer(struct usb_xfer *xfer)
1311 {
1312         struct usb_pipe_methods *methods = xfer->endpoint->methods;
1313         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
1314
1315         uint32_t status;
1316
1317         DPRINTFN(13, "xfer=%p checking transfer\n", xfer);
1318
1319         if (methods == &ehci_device_isoc_fs_methods) {
1320                 ehci_sitd_t *td;
1321
1322                 /* isochronous full speed transfer */
1323
1324                 td = xfer->td_transfer_last;
1325                 usb_pc_cpu_invalidate(td->page_cache);
1326                 status = hc32toh(sc, td->sitd_status);
1327
1328                 /* also check if first is complete */
1329
1330                 td = xfer->td_transfer_first;
1331                 usb_pc_cpu_invalidate(td->page_cache);
1332                 status |= hc32toh(sc, td->sitd_status);
1333
1334                 if (!(status & EHCI_SITD_ACTIVE)) {
1335                         ehci_device_done(xfer, USB_ERR_NORMAL_COMPLETION);
1336                         goto transferred;
1337                 }
1338         } else if (methods == &ehci_device_isoc_hs_methods) {
1339                 ehci_itd_t *td;
1340
1341                 /* isochronous high speed transfer */
1342
1343                 td = xfer->td_transfer_last;
1344                 usb_pc_cpu_invalidate(td->page_cache);
1345                 status =
1346                     td->itd_status[0] | td->itd_status[1] |
1347                     td->itd_status[2] | td->itd_status[3] |
1348                     td->itd_status[4] | td->itd_status[5] |
1349                     td->itd_status[6] | td->itd_status[7];
1350
1351                 /* also check first transfer */
1352                 td = xfer->td_transfer_first;
1353                 usb_pc_cpu_invalidate(td->page_cache);
1354                 status |=
1355                     td->itd_status[0] | td->itd_status[1] |
1356                     td->itd_status[2] | td->itd_status[3] |
1357                     td->itd_status[4] | td->itd_status[5] |
1358                     td->itd_status[6] | td->itd_status[7];
1359
1360                 /* if no transactions are active we continue */
1361                 if (!(status & htohc32(sc, EHCI_ITD_ACTIVE))) {
1362                         ehci_device_done(xfer, USB_ERR_NORMAL_COMPLETION);
1363                         goto transferred;
1364                 }
1365         } else {
1366                 ehci_qtd_t *td;
1367                 ehci_qh_t *qh;
1368
1369                 /* non-isochronous transfer */
1370
1371                 /*
1372                  * check whether there is an error somewhere in the middle,
1373                  * or whether there was a short packet (SPD and not ACTIVE)
1374                  */
1375                 td = xfer->td_transfer_cache;
1376
1377                 qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
1378
1379                 usb_pc_cpu_invalidate(qh->page_cache);
1380
1381                 status = hc32toh(sc, qh->qh_qtd.qtd_status);
1382                 if (status & EHCI_QTD_ACTIVE) {
1383                         /* transfer is pending */
1384                         goto done;
1385                 }
1386
1387                 while (1) {
1388                         usb_pc_cpu_invalidate(td->page_cache);
1389                         status = hc32toh(sc, td->qtd_status);
1390
1391                         /*
1392                          * Check if there is an active TD which
1393                          * indicates that the transfer isn't done.
1394                          */
1395                         if (status & EHCI_QTD_ACTIVE) {
1396                                 /* update cache */
1397                                 if (xfer->td_transfer_cache != td) {
1398                                         xfer->td_transfer_cache = td;
1399                                         if (qh->qh_qtd.qtd_next & 
1400                                             htohc32(sc, EHCI_LINK_TERMINATE)) {
1401                                                 /* XXX - manually advance to next frame */
1402                                                 qh->qh_qtd.qtd_next = td->qtd_self;
1403                                                 usb_pc_cpu_flush(td->page_cache);
1404                                         }
1405                                 }
1406                                 goto done;
1407                         }
1408                         /*
1409                          * last transfer descriptor makes the transfer done
1410                          */
1411                         if (((void *)td) == xfer->td_transfer_last) {
1412                                 break;
1413                         }
1414                         /*
1415                          * any kind of error makes the transfer done
1416                          */
1417                         if (status & EHCI_QTD_HALTED) {
1418                                 break;
1419                         }
1420                         /*
1421                          * if there is no alternate next transfer, a short
1422                          * packet also makes the transfer done
1423                          */
1424                         if (EHCI_QTD_GET_BYTES(status)) {
1425                                 if (xfer->flags_int.short_frames_ok) {
1426                                         /* follow alt next */
1427                                         if (td->alt_next) {
1428                                                 td = td->alt_next;
1429                                                 continue;
1430                                         }
1431                                 }
1432                                 /* transfer is done */
1433                                 break;
1434                         }
1435                         td = td->obj_next;
1436                 }
1437                 ehci_non_isoc_done(xfer);
1438                 goto transferred;
1439         }
1440
1441 done:
1442         DPRINTFN(13, "xfer=%p is still active\n", xfer);
1443         return (0);
1444
1445 transferred:
1446         return (1);
1447 }
1448
1449 static void
1450 ehci_pcd_enable(ehci_softc_t *sc)
1451 {
1452         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
1453
1454         sc->sc_eintrs |= EHCI_STS_PCD;
1455         EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
1456
1457         /* acknowledge any PCD interrupt */
1458         EOWRITE4(sc, EHCI_USBSTS, EHCI_STS_PCD);
1459
1460         ehci_root_intr(sc);
1461 }
1462
1463 static void
1464 ehci_interrupt_poll(ehci_softc_t *sc)
1465 {
1466         struct usb_xfer *xfer;
1467
1468 repeat:
1469         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
1470                 /*
1471                  * check if transfer is transferred
1472                  */
1473                 if (ehci_check_transfer(xfer)) {
1474                         /* queue has been modified */
1475                         goto repeat;
1476                 }
1477         }
1478 }
1479
1480 /*
1481  * Some EHCI chips from VIA / ATI seem to trigger interrupts before
1482  * writing back the qTD status, or miss signalling occasionally under
1483  * heavy load.  If the host machine is too fast, we can miss
1484  * transaction completion - when we scan the active list the
1485  * transaction still seems to be active. This generally exhibits
1486  * itself as a umass stall that never recovers.
1487  *
1488  * We work around this behaviour by setting up this callback after any
1489  * softintr that completes with transactions still pending, giving us
1490  * another chance to check for completion after the writeback has
1491  * taken place.
1492  */
1493 static void
1494 ehci_poll_timeout(void *arg)
1495 {
1496         ehci_softc_t *sc = arg;
1497
1498         DPRINTFN(3, "\n");
1499         ehci_interrupt_poll(sc);
1500 }
1501
1502 /*------------------------------------------------------------------------*
1503  *      ehci_interrupt - EHCI interrupt handler
1504  *
1505  * NOTE: Do not access "sc->sc_bus.bdev" inside the interrupt handler,
1506  * hence the interrupt handler will be setup before "sc->sc_bus.bdev"
1507  * is present !
1508  *------------------------------------------------------------------------*/
1509 void
1510 ehci_interrupt(ehci_softc_t *sc)
1511 {
1512         uint32_t status;
1513
1514         USB_BUS_LOCK(&sc->sc_bus);
1515
1516         DPRINTFN(16, "real interrupt\n");
1517
1518 #if USB_DEBUG
1519         if (ehcidebug > 15) {
1520                 ehci_dump_regs(sc);
1521         }
1522 #endif
1523
1524         status = EHCI_STS_INTRS(EOREAD4(sc, EHCI_USBSTS));
1525         if (status == 0) {
1526                 /* the interrupt was not for us */
1527                 goto done;
1528         }
1529         if (!(status & sc->sc_eintrs)) {
1530                 goto done;
1531         }
1532         EOWRITE4(sc, EHCI_USBSTS, status);      /* acknowledge */
1533
1534         status &= sc->sc_eintrs;
1535
1536         if (status & EHCI_STS_HSE) {
1537                 printf("%s: unrecoverable error, "
1538                     "controller halted\n", __FUNCTION__);
1539 #if USB_DEBUG
1540                 ehci_dump_regs(sc);
1541                 ehci_dump_isoc(sc);
1542 #endif
1543         }
1544         if (status & EHCI_STS_PCD) {
1545                 /*
1546                  * Disable PCD interrupt for now, because it will be
1547                  * on until the port has been reset.
1548                  */
1549                 sc->sc_eintrs &= ~EHCI_STS_PCD;
1550                 EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
1551
1552                 ehci_root_intr(sc);
1553
1554                 /* do not allow RHSC interrupts > 1 per second */
1555                 usb_callout_reset(&sc->sc_tmo_pcd, hz,
1556                     (void *)&ehci_pcd_enable, sc);
1557         }
1558         status &= ~(EHCI_STS_INT | EHCI_STS_ERRINT | EHCI_STS_PCD | EHCI_STS_IAA);
1559
1560         if (status != 0) {
1561                 /* block unprocessed interrupts */
1562                 sc->sc_eintrs &= ~status;
1563                 EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
1564                 printf("%s: blocking interrupts 0x%x\n", __FUNCTION__, status);
1565         }
1566         /* poll all the USB transfers */
1567         ehci_interrupt_poll(sc);
1568
1569         if (sc->sc_flags & EHCI_SCFLG_LOSTINTRBUG) {
1570                 usb_callout_reset(&sc->sc_tmo_poll, hz / 128,
1571                     (void *)&ehci_poll_timeout, sc);
1572         }
1573
1574 done:
1575         USB_BUS_UNLOCK(&sc->sc_bus);
1576 }
1577
1578 /*
1579  * called when a request does not complete
1580  */
1581 static void
1582 ehci_timeout(void *arg)
1583 {
1584         struct usb_xfer *xfer = arg;
1585
1586         DPRINTF("xfer=%p\n", xfer);
1587
1588         USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
1589
1590         /* transfer is transferred */
1591         ehci_device_done(xfer, USB_ERR_TIMEOUT);
1592 }
1593
1594 static void
1595 ehci_do_poll(struct usb_bus *bus)
1596 {
1597         ehci_softc_t *sc = EHCI_BUS2SC(bus);
1598
1599         USB_BUS_LOCK(&sc->sc_bus);
1600         ehci_interrupt_poll(sc);
1601         USB_BUS_UNLOCK(&sc->sc_bus);
1602 }
1603
1604 static void
1605 ehci_setup_standard_chain_sub(struct ehci_std_temp *temp)
1606 {
1607         struct usb_page_search buf_res;
1608         ehci_qtd_t *td;
1609         ehci_qtd_t *td_next;
1610         ehci_qtd_t *td_alt_next;
1611         uint32_t buf_offset;
1612         uint32_t average;
1613         uint32_t len_old;
1614         uint32_t terminate;
1615         uint8_t shortpkt_old;
1616         uint8_t precompute;
1617
1618         terminate = htohc32(temp->sc, EHCI_LINK_TERMINATE);
1619         td_alt_next = NULL;
1620         buf_offset = 0;
1621         shortpkt_old = temp->shortpkt;
1622         len_old = temp->len;
1623         precompute = 1;
1624
1625 restart:
1626
1627         td = temp->td;
1628         td_next = temp->td_next;
1629
1630         while (1) {
1631
1632                 if (temp->len == 0) {
1633
1634                         if (temp->shortpkt) {
1635                                 break;
1636                         }
1637                         /* send a Zero Length Packet, ZLP, last */
1638
1639                         temp->shortpkt = 1;
1640                         average = 0;
1641
1642                 } else {
1643
1644                         average = temp->average;
1645
1646                         if (temp->len < average) {
1647                                 if (temp->len % temp->max_frame_size) {
1648                                         temp->shortpkt = 1;
1649                                 }
1650                                 average = temp->len;
1651                         }
1652                 }
1653
1654                 if (td_next == NULL) {
1655                         panic("%s: out of EHCI transfer descriptors!", __FUNCTION__);
1656                 }
1657                 /* get next TD */
1658
1659                 td = td_next;
1660                 td_next = td->obj_next;
1661
1662                 /* check if we are pre-computing */
1663
1664                 if (precompute) {
1665
1666                         /* update remaining length */
1667
1668                         temp->len -= average;
1669
1670                         continue;
1671                 }
1672                 /* fill out current TD */
1673
1674                 td->qtd_status =
1675                     temp->qtd_status |
1676                     htohc32(temp->sc, EHCI_QTD_IOC |
1677                         EHCI_QTD_SET_BYTES(average));
1678
1679                 if (average == 0) {
1680
1681                         if (temp->auto_data_toggle == 0) {
1682
1683                                 /* update data toggle, ZLP case */
1684
1685                                 temp->qtd_status ^=
1686                                     htohc32(temp->sc, EHCI_QTD_TOGGLE_MASK);
1687                         }
1688                         td->len = 0;
1689
1690                         td->qtd_buffer[0] = 0;
1691                         td->qtd_buffer_hi[0] = 0;
1692
1693                         td->qtd_buffer[1] = 0;
1694                         td->qtd_buffer_hi[1] = 0;
1695
1696                 } else {
1697
1698                         uint8_t x;
1699
1700                         if (temp->auto_data_toggle == 0) {
1701
1702                                 /* update data toggle */
1703
1704                                 if (((average + temp->max_frame_size - 1) /
1705                                     temp->max_frame_size) & 1) {
1706                                         temp->qtd_status ^=
1707                                             htohc32(temp->sc, EHCI_QTD_TOGGLE_MASK);
1708                                 }
1709                         }
1710                         td->len = average;
1711
1712                         /* update remaining length */
1713
1714                         temp->len -= average;
1715
1716                         /* fill out buffer pointers */
1717
1718                         usbd_get_page(temp->pc, buf_offset, &buf_res);
1719                         td->qtd_buffer[0] =
1720                             htohc32(temp->sc, buf_res.physaddr);
1721                         td->qtd_buffer_hi[0] = 0;
1722
1723                         x = 1;
1724
1725                         while (average > EHCI_PAGE_SIZE) {
1726                                 average -= EHCI_PAGE_SIZE;
1727                                 buf_offset += EHCI_PAGE_SIZE;
1728                                 usbd_get_page(temp->pc, buf_offset, &buf_res);
1729                                 td->qtd_buffer[x] =
1730                                     htohc32(temp->sc,
1731                                     buf_res.physaddr & (~0xFFF));
1732                                 td->qtd_buffer_hi[x] = 0;
1733                                 x++;
1734                         }
1735
1736                         /*
1737                          * NOTE: The "average" variable is never zero after
1738                          * exiting the loop above !
1739                          *
1740                          * NOTE: We have to subtract one from the offset to
1741                          * ensure that we are computing the physical address
1742                          * of a valid page !
1743                          */
1744                         buf_offset += average;
1745                         usbd_get_page(temp->pc, buf_offset - 1, &buf_res);
1746                         td->qtd_buffer[x] =
1747                             htohc32(temp->sc,
1748                             buf_res.physaddr & (~0xFFF));
1749                         td->qtd_buffer_hi[x] = 0;
1750                 }
1751
1752                 if (temp->can_use_next) {
1753                         if (td_next) {
1754                                 /* link the current TD with the next one */
1755                                 td->qtd_next = td_next->qtd_self;
1756                         }
1757                 } else {
1758                         /*
1759                          * BUG WARNING: The EHCI HW can use the
1760                          * qtd_next field instead of qtd_altnext when
1761                          * a short packet is received! We work this
1762                          * around in software by not queueing more
1763                          * than one job/TD at a time!
1764                          */
1765                         td->qtd_next = terminate;
1766                 }
1767
1768                 td->qtd_altnext = terminate;
1769                 td->alt_next = td_alt_next;
1770
1771                 usb_pc_cpu_flush(td->page_cache);
1772         }
1773
1774         if (precompute) {
1775                 precompute = 0;
1776
1777                 /* setup alt next pointer, if any */
1778                 if (temp->last_frame) {
1779                         td_alt_next = NULL;
1780                 } else {
1781                         /* we use this field internally */
1782                         td_alt_next = td_next;
1783                 }
1784
1785                 /* restore */
1786                 temp->shortpkt = shortpkt_old;
1787                 temp->len = len_old;
1788                 goto restart;
1789         }
1790         temp->td = td;
1791         temp->td_next = td_next;
1792 }
1793
1794 static void
1795 ehci_setup_standard_chain(struct usb_xfer *xfer, ehci_qh_t **qh_last)
1796 {
1797         struct ehci_std_temp temp;
1798         struct usb_pipe_methods *methods;
1799         ehci_qh_t *qh;
1800         ehci_qtd_t *td;
1801         uint32_t qh_endp;
1802         uint32_t qh_endphub;
1803         uint32_t x;
1804
1805         DPRINTFN(9, "addr=%d endpt=%d sumlen=%d speed=%d\n",
1806             xfer->address, UE_GET_ADDR(xfer->endpointno),
1807             xfer->sumlen, usbd_get_speed(xfer->xroot->udev));
1808
1809         temp.average = xfer->max_hc_frame_size;
1810         temp.max_frame_size = xfer->max_frame_size;
1811         temp.sc = EHCI_BUS2SC(xfer->xroot->bus);
1812
1813         /* toggle the DMA set we are using */
1814         xfer->flags_int.curr_dma_set ^= 1;
1815
1816         /* get next DMA set */
1817         td = xfer->td_start[xfer->flags_int.curr_dma_set];
1818
1819         xfer->td_transfer_first = td;
1820         xfer->td_transfer_cache = td;
1821
1822         temp.td = NULL;
1823         temp.td_next = td;
1824         temp.qtd_status = 0;
1825         temp.last_frame = 0;
1826         temp.setup_alt_next = xfer->flags_int.short_frames_ok;
1827         temp.can_use_next = (xfer->flags_int.control_xfr ||
1828             (UE_GET_DIR(xfer->endpointno) == UE_DIR_OUT));
1829
1830         if (xfer->flags_int.control_xfr) {
1831                 if (xfer->endpoint->toggle_next) {
1832                         /* DATA1 is next */
1833                         temp.qtd_status |=
1834                             htohc32(temp.sc, EHCI_QTD_SET_TOGGLE(1));
1835                 }
1836                 temp.auto_data_toggle = 0;
1837         } else {
1838                 temp.auto_data_toggle = 1;
1839         }
1840
1841         if (usbd_get_speed(xfer->xroot->udev) != USB_SPEED_HIGH) {
1842                 /* max 3 retries */
1843                 temp.qtd_status |=
1844                     htohc32(temp.sc, EHCI_QTD_SET_CERR(3));
1845         }
1846         /* check if we should prepend a setup message */
1847
1848         if (xfer->flags_int.control_xfr) {
1849                 if (xfer->flags_int.control_hdr) {
1850
1851                         temp.qtd_status &=
1852                             htohc32(temp.sc, EHCI_QTD_SET_CERR(3));
1853                         temp.qtd_status |= htohc32(temp.sc,
1854                             EHCI_QTD_ACTIVE |
1855                             EHCI_QTD_SET_PID(EHCI_QTD_PID_SETUP) |
1856                             EHCI_QTD_SET_TOGGLE(0));
1857
1858                         temp.len = xfer->frlengths[0];
1859                         temp.pc = xfer->frbuffers + 0;
1860                         temp.shortpkt = temp.len ? 1 : 0;
1861                         /* check for last frame */
1862                         if (xfer->nframes == 1) {
1863                                 /* no STATUS stage yet, SETUP is last */
1864                                 if (xfer->flags_int.control_act) {
1865                                         temp.last_frame = 1;
1866                                         temp.setup_alt_next = 0;
1867                                 }
1868                         }
1869                         ehci_setup_standard_chain_sub(&temp);
1870                 }
1871                 x = 1;
1872         } else {
1873                 x = 0;
1874         }
1875
1876         while (x != xfer->nframes) {
1877
1878                 /* DATA0 / DATA1 message */
1879
1880                 temp.len = xfer->frlengths[x];
1881                 temp.pc = xfer->frbuffers + x;
1882
1883                 x++;
1884
1885                 if (x == xfer->nframes) {
1886                         if (xfer->flags_int.control_xfr) {
1887                                 /* no STATUS stage yet, DATA is last */
1888                                 if (xfer->flags_int.control_act) {
1889                                         temp.last_frame = 1;
1890                                         temp.setup_alt_next = 0;
1891                                 }
1892                         } else {
1893                                 temp.last_frame = 1;
1894                                 temp.setup_alt_next = 0;
1895                         }
1896                 }
1897                 /* keep previous data toggle and error count */
1898
1899                 temp.qtd_status &=
1900                     htohc32(temp.sc, EHCI_QTD_SET_CERR(3) |
1901                     EHCI_QTD_SET_TOGGLE(1));
1902
1903                 if (temp.len == 0) {
1904
1905                         /* make sure that we send an USB packet */
1906
1907                         temp.shortpkt = 0;
1908
1909                 } else {
1910
1911                         /* regular data transfer */
1912
1913                         temp.shortpkt = (xfer->flags.force_short_xfer) ? 0 : 1;
1914                 }
1915
1916                 /* set endpoint direction */
1917
1918                 temp.qtd_status |=
1919                     (UE_GET_DIR(xfer->endpointno) == UE_DIR_IN) ?
1920                     htohc32(temp.sc, EHCI_QTD_ACTIVE |
1921                     EHCI_QTD_SET_PID(EHCI_QTD_PID_IN)) :
1922                     htohc32(temp.sc, EHCI_QTD_ACTIVE |
1923                     EHCI_QTD_SET_PID(EHCI_QTD_PID_OUT));
1924
1925                 ehci_setup_standard_chain_sub(&temp);
1926         }
1927
1928         /* check if we should append a status stage */
1929
1930         if (xfer->flags_int.control_xfr &&
1931             !xfer->flags_int.control_act) {
1932
1933                 /*
1934                  * Send a DATA1 message and invert the current endpoint
1935                  * direction.
1936                  */
1937
1938                 temp.qtd_status &= htohc32(temp.sc, EHCI_QTD_SET_CERR(3) |
1939                     EHCI_QTD_SET_TOGGLE(1));
1940                 temp.qtd_status |=
1941                     (UE_GET_DIR(xfer->endpointno) == UE_DIR_OUT) ?
1942                     htohc32(temp.sc, EHCI_QTD_ACTIVE |
1943                     EHCI_QTD_SET_PID(EHCI_QTD_PID_IN) |
1944                     EHCI_QTD_SET_TOGGLE(1)) :
1945                     htohc32(temp.sc, EHCI_QTD_ACTIVE |
1946                     EHCI_QTD_SET_PID(EHCI_QTD_PID_OUT) |
1947                     EHCI_QTD_SET_TOGGLE(1));
1948
1949                 temp.len = 0;
1950                 temp.pc = NULL;
1951                 temp.shortpkt = 0;
1952                 temp.last_frame = 1;
1953                 temp.setup_alt_next = 0;
1954
1955                 ehci_setup_standard_chain_sub(&temp);
1956         }
1957         td = temp.td;
1958
1959         /* the last TD terminates the transfer: */
1960         td->qtd_next = htohc32(temp.sc, EHCI_LINK_TERMINATE);
1961         td->qtd_altnext = htohc32(temp.sc, EHCI_LINK_TERMINATE);
1962
1963         usb_pc_cpu_flush(td->page_cache);
1964
1965         /* must have at least one frame! */
1966
1967         xfer->td_transfer_last = td;
1968
1969 #if USB_DEBUG
1970         if (ehcidebug > 8) {
1971                 DPRINTF("nexttog=%d; data before transfer:\n",
1972                     xfer->endpoint->toggle_next);
1973                 ehci_dump_sqtds(temp.sc,
1974                     xfer->td_transfer_first);
1975         }
1976 #endif
1977
1978         methods = xfer->endpoint->methods;
1979
1980         qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
1981
1982         /* the "qh_link" field is filled when the QH is added */
1983
1984         qh_endp =
1985             (EHCI_QH_SET_ADDR(xfer->address) |
1986             EHCI_QH_SET_ENDPT(UE_GET_ADDR(xfer->endpointno)) |
1987             EHCI_QH_SET_MPL(xfer->max_packet_size));
1988
1989         if (usbd_get_speed(xfer->xroot->udev) == USB_SPEED_HIGH) {
1990                 qh_endp |= EHCI_QH_SET_EPS(EHCI_QH_SPEED_HIGH);
1991                 if (methods != &ehci_device_intr_methods)
1992                         qh_endp |= EHCI_QH_SET_NRL(8);
1993         } else {
1994
1995                 if (usbd_get_speed(xfer->xroot->udev) == USB_SPEED_FULL) {
1996                         qh_endp |= EHCI_QH_SET_EPS(EHCI_QH_SPEED_FULL);
1997                 } else {
1998                         qh_endp |= EHCI_QH_SET_EPS(EHCI_QH_SPEED_LOW);
1999                 }
2000
2001                 if (methods == &ehci_device_ctrl_methods) {
2002                         qh_endp |= EHCI_QH_CTL;
2003                 }
2004                 if (methods != &ehci_device_intr_methods) {
2005                         /* Only try one time per microframe! */
2006                         qh_endp |= EHCI_QH_SET_NRL(1);
2007                 }
2008         }
2009
2010         if (temp.auto_data_toggle == 0) {
2011                 /* software computes the data toggle */
2012                 qh_endp |= EHCI_QH_DTC;
2013         }
2014
2015         qh->qh_endp = htohc32(temp.sc, qh_endp);
2016
2017         qh_endphub =
2018             (EHCI_QH_SET_MULT(xfer->max_packet_count & 3) |
2019             EHCI_QH_SET_CMASK(xfer->usb_cmask) |
2020             EHCI_QH_SET_SMASK(xfer->usb_smask) |
2021             EHCI_QH_SET_HUBA(xfer->xroot->udev->hs_hub_addr) |
2022             EHCI_QH_SET_PORT(xfer->xroot->udev->hs_port_no));
2023
2024         qh->qh_endphub = htohc32(temp.sc, qh_endphub);
2025         qh->qh_curqtd = 0;
2026
2027         /* fill the overlay qTD */
2028
2029         if (temp.auto_data_toggle && xfer->endpoint->toggle_next) {
2030                 /* DATA1 is next */
2031                 qh->qh_qtd.qtd_status = htohc32(temp.sc, EHCI_QTD_SET_TOGGLE(1));
2032         } else {
2033                 qh->qh_qtd.qtd_status = 0;
2034         }
2035
2036         td = xfer->td_transfer_first;
2037
2038         qh->qh_qtd.qtd_next = td->qtd_self;
2039         qh->qh_qtd.qtd_altnext =
2040             htohc32(temp.sc, EHCI_LINK_TERMINATE);
2041
2042         usb_pc_cpu_flush(qh->page_cache);
2043
2044         if (xfer->xroot->udev->flags.self_suspended == 0) {
2045                 EHCI_APPEND_QH(qh, *qh_last);
2046         }
2047 }
2048
2049 static void
2050 ehci_root_intr(ehci_softc_t *sc)
2051 {
2052         uint16_t i;
2053         uint16_t m;
2054
2055         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
2056
2057         /* clear any old interrupt data */
2058         memset(sc->sc_hub_idata, 0, sizeof(sc->sc_hub_idata));
2059
2060         /* set bits */
2061         m = (sc->sc_noport + 1);
2062         if (m > (8 * sizeof(sc->sc_hub_idata))) {
2063                 m = (8 * sizeof(sc->sc_hub_idata));
2064         }
2065         for (i = 1; i < m; i++) {
2066                 /* pick out CHANGE bits from the status register */
2067                 if (EOREAD4(sc, EHCI_PORTSC(i)) & EHCI_PS_CLEAR) {
2068                         sc->sc_hub_idata[i / 8] |= 1 << (i % 8);
2069                         DPRINTF("port %d changed\n", i);
2070                 }
2071         }
2072         uhub_root_intr(&sc->sc_bus, sc->sc_hub_idata,
2073             sizeof(sc->sc_hub_idata));
2074 }
2075
2076 static void
2077 ehci_isoc_fs_done(ehci_softc_t *sc, struct usb_xfer *xfer)
2078 {
2079         uint32_t nframes = xfer->nframes;
2080         uint32_t status;
2081         uint32_t *plen = xfer->frlengths;
2082         uint16_t len = 0;
2083         ehci_sitd_t *td = xfer->td_transfer_first;
2084         ehci_sitd_t **pp_last = &sc->sc_isoc_fs_p_last[xfer->qh_pos];
2085
2086         DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
2087             xfer, xfer->endpoint);
2088
2089         while (nframes--) {
2090                 if (td == NULL) {
2091                         panic("%s:%d: out of TD's\n",
2092                             __FUNCTION__, __LINE__);
2093                 }
2094                 if (pp_last >= &sc->sc_isoc_fs_p_last[EHCI_VIRTUAL_FRAMELIST_COUNT]) {
2095                         pp_last = &sc->sc_isoc_fs_p_last[0];
2096                 }
2097 #if USB_DEBUG
2098                 if (ehcidebug > 15) {
2099                         DPRINTF("isoc FS-TD\n");
2100                         ehci_dump_sitd(sc, td);
2101                 }
2102 #endif
2103                 usb_pc_cpu_invalidate(td->page_cache);
2104                 status = hc32toh(sc, td->sitd_status);
2105
2106                 len = EHCI_SITD_GET_LEN(status);
2107
2108                 DPRINTFN(2, "status=0x%08x, rem=%u\n", status, len);
2109
2110                 if (*plen >= len) {
2111                         len = *plen - len;
2112                 } else {
2113                         len = 0;
2114                 }
2115
2116                 *plen = len;
2117
2118                 /* remove FS-TD from schedule */
2119                 EHCI_REMOVE_FS_TD(td, *pp_last);
2120
2121                 pp_last++;
2122                 plen++;
2123                 td = td->obj_next;
2124         }
2125
2126         xfer->aframes = xfer->nframes;
2127 }
2128
2129 static void
2130 ehci_isoc_hs_done(ehci_softc_t *sc, struct usb_xfer *xfer)
2131 {
2132         uint32_t nframes = xfer->nframes;
2133         uint32_t status;
2134         uint32_t *plen = xfer->frlengths;
2135         uint16_t len = 0;
2136         uint8_t td_no = 0;
2137         ehci_itd_t *td = xfer->td_transfer_first;
2138         ehci_itd_t **pp_last = &sc->sc_isoc_hs_p_last[xfer->qh_pos];
2139
2140         DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
2141             xfer, xfer->endpoint);
2142
2143         while (nframes) {
2144                 if (td == NULL) {
2145                         panic("%s:%d: out of TD's\n",
2146                             __FUNCTION__, __LINE__);
2147                 }
2148                 if (pp_last >= &sc->sc_isoc_hs_p_last[EHCI_VIRTUAL_FRAMELIST_COUNT]) {
2149                         pp_last = &sc->sc_isoc_hs_p_last[0];
2150                 }
2151 #if USB_DEBUG
2152                 if (ehcidebug > 15) {
2153                         DPRINTF("isoc HS-TD\n");
2154                         ehci_dump_itd(sc, td);
2155                 }
2156 #endif
2157
2158                 usb_pc_cpu_invalidate(td->page_cache);
2159                 status = hc32toh(sc, td->itd_status[td_no]);
2160
2161                 len = EHCI_ITD_GET_LEN(status);
2162
2163                 DPRINTFN(2, "status=0x%08x, len=%u\n", status, len);
2164
2165                 if (xfer->usb_smask & (1 << td_no)) {
2166
2167                         if (*plen >= len) {
2168                                 /*
2169                                  * The length is valid. NOTE: The
2170                                  * complete length is written back
2171                                  * into the status field, and not the
2172                                  * remainder like with other transfer
2173                                  * descriptor types.
2174                                  */
2175                         } else {
2176                                 /* Invalid length - truncate */
2177                                 len = 0;
2178                         }
2179
2180                         *plen = len;
2181                         plen++;
2182                         nframes--;
2183                 }
2184
2185                 td_no++;
2186
2187                 if ((td_no == 8) || (nframes == 0)) {
2188                         /* remove HS-TD from schedule */
2189                         EHCI_REMOVE_HS_TD(td, *pp_last);
2190                         pp_last++;
2191
2192                         td_no = 0;
2193                         td = td->obj_next;
2194                 }
2195         }
2196         xfer->aframes = xfer->nframes;
2197 }
2198
2199 /* NOTE: "done" can be run two times in a row,
2200  * from close and from interrupt
2201  */
2202 static void
2203 ehci_device_done(struct usb_xfer *xfer, usb_error_t error)
2204 {
2205         struct usb_pipe_methods *methods = xfer->endpoint->methods;
2206         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2207
2208         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
2209
2210         DPRINTFN(2, "xfer=%p, endpoint=%p, error=%d\n",
2211             xfer, xfer->endpoint, error);
2212
2213         if ((methods == &ehci_device_bulk_methods) ||
2214             (methods == &ehci_device_ctrl_methods)) {
2215 #if USB_DEBUG
2216                 if (ehcidebug > 8) {
2217                         DPRINTF("nexttog=%d; data after transfer:\n",
2218                             xfer->endpoint->toggle_next);
2219                         ehci_dump_sqtds(sc,
2220                             xfer->td_transfer_first);
2221                 }
2222 #endif
2223
2224                 EHCI_REMOVE_QH(xfer->qh_start[xfer->flags_int.curr_dma_set],
2225                     sc->sc_async_p_last);
2226         }
2227         if (methods == &ehci_device_intr_methods) {
2228                 EHCI_REMOVE_QH(xfer->qh_start[xfer->flags_int.curr_dma_set],
2229                     sc->sc_intr_p_last[xfer->qh_pos]);
2230         }
2231         /*
2232          * Only finish isochronous transfers once which will update
2233          * "xfer->frlengths".
2234          */
2235         if (xfer->td_transfer_first &&
2236             xfer->td_transfer_last) {
2237                 if (methods == &ehci_device_isoc_fs_methods) {
2238                         ehci_isoc_fs_done(sc, xfer);
2239                 }
2240                 if (methods == &ehci_device_isoc_hs_methods) {
2241                         ehci_isoc_hs_done(sc, xfer);
2242                 }
2243                 xfer->td_transfer_first = NULL;
2244                 xfer->td_transfer_last = NULL;
2245         }
2246         /* dequeue transfer and start next transfer */
2247         usbd_transfer_done(xfer, error);
2248 }
2249
2250 /*------------------------------------------------------------------------*
2251  * ehci bulk support
2252  *------------------------------------------------------------------------*/
2253 static void
2254 ehci_device_bulk_open(struct usb_xfer *xfer)
2255 {
2256         return;
2257 }
2258
2259 static void
2260 ehci_device_bulk_close(struct usb_xfer *xfer)
2261 {
2262         ehci_device_done(xfer, USB_ERR_CANCELLED);
2263 }
2264
2265 static void
2266 ehci_device_bulk_enter(struct usb_xfer *xfer)
2267 {
2268         return;
2269 }
2270
2271 static void
2272 ehci_device_bulk_start(struct usb_xfer *xfer)
2273 {
2274         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2275         uint32_t temp;
2276
2277         /* setup TD's and QH */
2278         ehci_setup_standard_chain(xfer, &sc->sc_async_p_last);
2279
2280         /* put transfer on interrupt queue */
2281         ehci_transfer_intr_enqueue(xfer);
2282
2283         /* XXX Performance quirk: Some Host Controllers have a too low
2284          * interrupt rate. Issue an IAAD to stimulate the Host
2285          * Controller after queueing the BULK transfer.
2286          */
2287         temp = EOREAD4(sc, EHCI_USBCMD);
2288         if (!(temp & EHCI_CMD_IAAD))
2289                 EOWRITE4(sc, EHCI_USBCMD, temp | EHCI_CMD_IAAD);
2290 }
2291
2292 struct usb_pipe_methods ehci_device_bulk_methods =
2293 {
2294         .open = ehci_device_bulk_open,
2295         .close = ehci_device_bulk_close,
2296         .enter = ehci_device_bulk_enter,
2297         .start = ehci_device_bulk_start,
2298 };
2299
2300 /*------------------------------------------------------------------------*
2301  * ehci control support
2302  *------------------------------------------------------------------------*/
2303 static void
2304 ehci_device_ctrl_open(struct usb_xfer *xfer)
2305 {
2306         return;
2307 }
2308
2309 static void
2310 ehci_device_ctrl_close(struct usb_xfer *xfer)
2311 {
2312         ehci_device_done(xfer, USB_ERR_CANCELLED);
2313 }
2314
2315 static void
2316 ehci_device_ctrl_enter(struct usb_xfer *xfer)
2317 {
2318         return;
2319 }
2320
2321 static void
2322 ehci_device_ctrl_start(struct usb_xfer *xfer)
2323 {
2324         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2325
2326         /* setup TD's and QH */
2327         ehci_setup_standard_chain(xfer, &sc->sc_async_p_last);
2328
2329         /* put transfer on interrupt queue */
2330         ehci_transfer_intr_enqueue(xfer);
2331 }
2332
2333 struct usb_pipe_methods ehci_device_ctrl_methods =
2334 {
2335         .open = ehci_device_ctrl_open,
2336         .close = ehci_device_ctrl_close,
2337         .enter = ehci_device_ctrl_enter,
2338         .start = ehci_device_ctrl_start,
2339 };
2340
2341 /*------------------------------------------------------------------------*
2342  * ehci interrupt support
2343  *------------------------------------------------------------------------*/
2344 static void
2345 ehci_device_intr_open(struct usb_xfer *xfer)
2346 {
2347         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2348         uint16_t best;
2349         uint16_t bit;
2350         uint16_t x;
2351         uint8_t slot;
2352
2353         /* Allocate a microframe slot first: */
2354
2355         slot = usb_intr_schedule_adjust
2356             (xfer->xroot->udev, xfer->max_frame_size, USB_HS_MICRO_FRAMES_MAX);
2357
2358         if (usbd_get_speed(xfer->xroot->udev) == USB_SPEED_HIGH) {
2359                 xfer->usb_uframe = slot;
2360                 xfer->usb_smask = (1 << slot) & 0xFF;
2361                 xfer->usb_cmask = 0;
2362         } else {
2363                 xfer->usb_uframe = slot;
2364                 xfer->usb_smask = (1 << slot) & 0x3F;
2365                 xfer->usb_cmask = (-(4 << slot)) & 0xFE;
2366         }
2367
2368         /*
2369          * Find the best QH position corresponding to the given interval:
2370          */
2371
2372         best = 0;
2373         bit = EHCI_VIRTUAL_FRAMELIST_COUNT / 2;
2374         while (bit) {
2375                 if (xfer->interval >= bit) {
2376                         x = bit;
2377                         best = bit;
2378                         while (x & bit) {
2379                                 if (sc->sc_intr_stat[x] <
2380                                     sc->sc_intr_stat[best]) {
2381                                         best = x;
2382                                 }
2383                                 x++;
2384                         }
2385                         break;
2386                 }
2387                 bit >>= 1;
2388         }
2389
2390         sc->sc_intr_stat[best]++;
2391         xfer->qh_pos = best;
2392
2393         DPRINTFN(3, "best=%d interval=%d\n",
2394             best, xfer->interval);
2395 }
2396
2397 static void
2398 ehci_device_intr_close(struct usb_xfer *xfer)
2399 {
2400         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2401
2402         usb_intr_schedule_adjust(xfer->xroot->udev,
2403             -(xfer->max_frame_size), xfer->usb_uframe);
2404
2405         sc->sc_intr_stat[xfer->qh_pos]--;
2406
2407         ehci_device_done(xfer, USB_ERR_CANCELLED);
2408 }
2409
2410 static void
2411 ehci_device_intr_enter(struct usb_xfer *xfer)
2412 {
2413         return;
2414 }
2415
2416 static void
2417 ehci_device_intr_start(struct usb_xfer *xfer)
2418 {
2419         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2420
2421         /* setup TD's and QH */
2422         ehci_setup_standard_chain(xfer, &sc->sc_intr_p_last[xfer->qh_pos]);
2423
2424         /* put transfer on interrupt queue */
2425         ehci_transfer_intr_enqueue(xfer);
2426 }
2427
2428 struct usb_pipe_methods ehci_device_intr_methods =
2429 {
2430         .open = ehci_device_intr_open,
2431         .close = ehci_device_intr_close,
2432         .enter = ehci_device_intr_enter,
2433         .start = ehci_device_intr_start,
2434 };
2435
2436 /*------------------------------------------------------------------------*
2437  * ehci full speed isochronous support
2438  *------------------------------------------------------------------------*/
2439 static void
2440 ehci_device_isoc_fs_open(struct usb_xfer *xfer)
2441 {
2442         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2443         ehci_sitd_t *td;
2444         uint32_t sitd_portaddr;
2445         uint8_t ds;
2446
2447         sitd_portaddr =
2448             EHCI_SITD_SET_ADDR(xfer->address) |
2449             EHCI_SITD_SET_ENDPT(UE_GET_ADDR(xfer->endpointno)) |
2450             EHCI_SITD_SET_HUBA(xfer->xroot->udev->hs_hub_addr) |
2451             EHCI_SITD_SET_PORT(xfer->xroot->udev->hs_port_no);
2452
2453         if (UE_GET_DIR(xfer->endpointno) == UE_DIR_IN) {
2454                 sitd_portaddr |= EHCI_SITD_SET_DIR_IN;
2455         }
2456         sitd_portaddr = htohc32(sc, sitd_portaddr);
2457
2458         /* initialize all TD's */
2459
2460         for (ds = 0; ds != 2; ds++) {
2461
2462                 for (td = xfer->td_start[ds]; td; td = td->obj_next) {
2463
2464                         td->sitd_portaddr = sitd_portaddr;
2465
2466                         /*
2467                          * TODO: make some kind of automatic
2468                          * SMASK/CMASK selection based on micro-frame
2469                          * usage
2470                          *
2471                          * micro-frame usage (8 microframes per 1ms)
2472                          */
2473                         td->sitd_back = htohc32(sc, EHCI_LINK_TERMINATE);
2474
2475                         usb_pc_cpu_flush(td->page_cache);
2476                 }
2477         }
2478 }
2479
2480 static void
2481 ehci_device_isoc_fs_close(struct usb_xfer *xfer)
2482 {
2483         ehci_device_done(xfer, USB_ERR_CANCELLED);
2484 }
2485
2486 static void
2487 ehci_device_isoc_fs_enter(struct usb_xfer *xfer)
2488 {
2489         struct usb_page_search buf_res;
2490         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2491         struct usb_fs_isoc_schedule *fss_start;
2492         struct usb_fs_isoc_schedule *fss_end;
2493         struct usb_fs_isoc_schedule *fss;
2494         ehci_sitd_t *td;
2495         ehci_sitd_t *td_last = NULL;
2496         ehci_sitd_t **pp_last;
2497         uint32_t *plen;
2498         uint32_t buf_offset;
2499         uint32_t nframes;
2500         uint32_t temp;
2501         uint32_t sitd_mask;
2502         uint16_t tlen;
2503         uint8_t sa;
2504         uint8_t sb;
2505         uint8_t error;
2506
2507 #if USB_DEBUG
2508         uint8_t once = 1;
2509
2510 #endif
2511
2512         DPRINTFN(6, "xfer=%p next=%d nframes=%d\n",
2513             xfer, xfer->endpoint->isoc_next, xfer->nframes);
2514
2515         /* get the current frame index */
2516
2517         nframes = EOREAD4(sc, EHCI_FRINDEX) / 8;
2518
2519         /*
2520          * check if the frame index is within the window where the frames
2521          * will be inserted
2522          */
2523         buf_offset = (nframes - xfer->endpoint->isoc_next) &
2524             (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2525
2526         if ((xfer->endpoint->is_synced == 0) ||
2527             (buf_offset < xfer->nframes)) {
2528                 /*
2529                  * If there is data underflow or the pipe queue is empty we
2530                  * schedule the transfer a few frames ahead of the current
2531                  * frame position. Else two isochronous transfers might
2532                  * overlap.
2533                  */
2534                 xfer->endpoint->isoc_next = (nframes + 3) &
2535                     (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2536                 xfer->endpoint->is_synced = 1;
2537                 DPRINTFN(3, "start next=%d\n", xfer->endpoint->isoc_next);
2538         }
2539         /*
2540          * compute how many milliseconds the insertion is ahead of the
2541          * current frame position:
2542          */
2543         buf_offset = (xfer->endpoint->isoc_next - nframes) &
2544             (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2545
2546         /*
2547          * pre-compute when the isochronous transfer will be finished:
2548          */
2549         xfer->isoc_time_complete =
2550             usbd_fs_isoc_schedule_isoc_time_expand
2551             (xfer->xroot->udev, &fss_start, &fss_end, nframes) + buf_offset +
2552             xfer->nframes;
2553
2554         /* get the real number of frames */
2555
2556         nframes = xfer->nframes;
2557
2558         buf_offset = 0;
2559
2560         plen = xfer->frlengths;
2561
2562         /* toggle the DMA set we are using */
2563         xfer->flags_int.curr_dma_set ^= 1;
2564
2565         /* get next DMA set */
2566         td = xfer->td_start[xfer->flags_int.curr_dma_set];
2567         xfer->td_transfer_first = td;
2568
2569         pp_last = &sc->sc_isoc_fs_p_last[xfer->endpoint->isoc_next];
2570
2571         /* store starting position */
2572
2573         xfer->qh_pos = xfer->endpoint->isoc_next;
2574
2575         fss = fss_start + (xfer->qh_pos % USB_ISOC_TIME_MAX);
2576
2577         while (nframes--) {
2578                 if (td == NULL) {
2579                         panic("%s:%d: out of TD's\n",
2580                             __FUNCTION__, __LINE__);
2581                 }
2582                 if (pp_last >= &sc->sc_isoc_fs_p_last[EHCI_VIRTUAL_FRAMELIST_COUNT]) {
2583                         pp_last = &sc->sc_isoc_fs_p_last[0];
2584                 }
2585                 if (fss >= fss_end) {
2586                         fss = fss_start;
2587                 }
2588                 /* reuse sitd_portaddr and sitd_back from last transfer */
2589
2590                 if (*plen > xfer->max_frame_size) {
2591 #if USB_DEBUG
2592                         if (once) {
2593                                 once = 0;
2594                                 printf("%s: frame length(%d) exceeds %d "
2595                                     "bytes (frame truncated)\n",
2596                                     __FUNCTION__, *plen,
2597                                     xfer->max_frame_size);
2598                         }
2599 #endif
2600                         *plen = xfer->max_frame_size;
2601                 }
2602                 /*
2603                  * We currently don't care if the ISOCHRONOUS schedule is
2604                  * full!
2605                  */
2606                 error = usbd_fs_isoc_schedule_alloc(fss, &sa, *plen);
2607                 if (error) {
2608                         /*
2609                          * The FULL speed schedule is FULL! Set length
2610                          * to zero.
2611                          */
2612                         *plen = 0;
2613                 }
2614                 if (*plen) {
2615                         /*
2616                          * only call "usbd_get_page()" when we have a
2617                          * non-zero length
2618                          */
2619                         usbd_get_page(xfer->frbuffers, buf_offset, &buf_res);
2620                         td->sitd_bp[0] = htohc32(sc, buf_res.physaddr);
2621                         buf_offset += *plen;
2622                         /*
2623                          * NOTE: We need to subtract one from the offset so
2624                          * that we are on a valid page!
2625                          */
2626                         usbd_get_page(xfer->frbuffers, buf_offset - 1,
2627                             &buf_res);
2628                         temp = buf_res.physaddr & ~0xFFF;
2629                 } else {
2630                         td->sitd_bp[0] = 0;
2631                         temp = 0;
2632                 }
2633
2634                 if (UE_GET_DIR(xfer->endpointno) == UE_DIR_OUT) {
2635                         tlen = *plen;
2636                         if (tlen <= 188) {
2637                                 temp |= 1;      /* T-count = 1, TP = ALL */
2638                                 tlen = 1;
2639                         } else {
2640                                 tlen += 187;
2641                                 tlen /= 188;
2642                                 temp |= tlen;   /* T-count = [1..6] */
2643                                 temp |= 8;      /* TP = Begin */
2644                         }
2645
2646                         tlen += sa;
2647
2648                         if (tlen >= 8) {
2649                                 sb = 0;
2650                         } else {
2651                                 sb = (1 << tlen);
2652                         }
2653
2654                         sa = (1 << sa);
2655                         sa = (sb - sa) & 0x3F;
2656                         sb = 0;
2657                 } else {
2658                         sb = (-(4 << sa)) & 0xFE;
2659                         sa = (1 << sa) & 0x3F;
2660                 }
2661
2662                 sitd_mask = (EHCI_SITD_SET_SMASK(sa) |
2663                     EHCI_SITD_SET_CMASK(sb));
2664
2665                 td->sitd_bp[1] = htohc32(sc, temp);
2666
2667                 td->sitd_mask = htohc32(sc, sitd_mask);
2668
2669                 if (nframes == 0) {
2670                         td->sitd_status = htohc32(sc,
2671                             EHCI_SITD_IOC |
2672                             EHCI_SITD_ACTIVE |
2673                             EHCI_SITD_SET_LEN(*plen));
2674                 } else {
2675                         td->sitd_status = htohc32(sc,
2676                             EHCI_SITD_ACTIVE |
2677                             EHCI_SITD_SET_LEN(*plen));
2678                 }
2679                 usb_pc_cpu_flush(td->page_cache);
2680
2681 #if USB_DEBUG
2682                 if (ehcidebug > 15) {
2683                         DPRINTF("FS-TD %d\n", nframes);
2684                         ehci_dump_sitd(sc, td);
2685                 }
2686 #endif
2687                 /* insert TD into schedule */
2688                 EHCI_APPEND_FS_TD(td, *pp_last);
2689                 pp_last++;
2690
2691                 plen++;
2692                 fss++;
2693                 td_last = td;
2694                 td = td->obj_next;
2695         }
2696
2697         xfer->td_transfer_last = td_last;
2698
2699         /* update isoc_next */
2700         xfer->endpoint->isoc_next = (pp_last - &sc->sc_isoc_fs_p_last[0]) &
2701             (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2702 }
2703
2704 static void
2705 ehci_device_isoc_fs_start(struct usb_xfer *xfer)
2706 {
2707         /* put transfer on interrupt queue */
2708         ehci_transfer_intr_enqueue(xfer);
2709 }
2710
2711 struct usb_pipe_methods ehci_device_isoc_fs_methods =
2712 {
2713         .open = ehci_device_isoc_fs_open,
2714         .close = ehci_device_isoc_fs_close,
2715         .enter = ehci_device_isoc_fs_enter,
2716         .start = ehci_device_isoc_fs_start,
2717 };
2718
2719 /*------------------------------------------------------------------------*
2720  * ehci high speed isochronous support
2721  *------------------------------------------------------------------------*/
2722 static void
2723 ehci_device_isoc_hs_open(struct usb_xfer *xfer)
2724 {
2725         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2726         ehci_itd_t *td;
2727         uint32_t temp;
2728         uint8_t ds;
2729         uint8_t slot;
2730
2731         slot = usb_intr_schedule_adjust(xfer->xroot->udev, xfer->max_frame_size,
2732             USB_HS_MICRO_FRAMES_MAX);
2733
2734         xfer->usb_uframe = slot;
2735         xfer->usb_cmask = 0;
2736
2737         switch (usbd_xfer_get_fps_shift(xfer)) {
2738         case 0:
2739                 xfer->usb_smask = 0xFF;
2740                 break;
2741         case 1:
2742                 xfer->usb_smask = 0x55 << (slot & 1);
2743                 break;
2744         case 2:
2745                 xfer->usb_smask = 0x11 << (slot & 3);
2746                 break;
2747         default:
2748                 xfer->usb_smask = 0x01 << (slot & 7);
2749                 break;
2750         }
2751
2752         /* initialize all TD's */
2753
2754         for (ds = 0; ds != 2; ds++) {
2755
2756                 for (td = xfer->td_start[ds]; td; td = td->obj_next) {
2757
2758                         /* set TD inactive */
2759                         td->itd_status[0] = 0;
2760                         td->itd_status[1] = 0;
2761                         td->itd_status[2] = 0;
2762                         td->itd_status[3] = 0;
2763                         td->itd_status[4] = 0;
2764                         td->itd_status[5] = 0;
2765                         td->itd_status[6] = 0;
2766                         td->itd_status[7] = 0;
2767
2768                         /* set endpoint and address */
2769                         td->itd_bp[0] = htohc32(sc,
2770                             EHCI_ITD_SET_ADDR(xfer->address) |
2771                             EHCI_ITD_SET_ENDPT(UE_GET_ADDR(xfer->endpointno)));
2772
2773                         temp =
2774                             EHCI_ITD_SET_MPL(xfer->max_packet_size & 0x7FF);
2775
2776                         /* set direction */
2777                         if (UE_GET_DIR(xfer->endpointno) == UE_DIR_IN) {
2778                                 temp |= EHCI_ITD_SET_DIR_IN;
2779                         }
2780                         /* set maximum packet size */
2781                         td->itd_bp[1] = htohc32(sc, temp);
2782
2783                         /* set transfer multiplier */
2784                         td->itd_bp[2] = htohc32(sc, xfer->max_packet_count & 3);
2785
2786                         usb_pc_cpu_flush(td->page_cache);
2787                 }
2788         }
2789 }
2790
2791 static void
2792 ehci_device_isoc_hs_close(struct usb_xfer *xfer)
2793 {
2794
2795         usb_intr_schedule_adjust(xfer->xroot->udev,
2796             -(xfer->max_frame_size), xfer->usb_uframe);
2797
2798         ehci_device_done(xfer, USB_ERR_CANCELLED);
2799 }
2800
2801 static void
2802 ehci_device_isoc_hs_enter(struct usb_xfer *xfer)
2803 {
2804         struct usb_page_search buf_res;
2805         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2806         ehci_itd_t *td;
2807         ehci_itd_t *td_last = NULL;
2808         ehci_itd_t **pp_last;
2809         bus_size_t page_addr;
2810         uint32_t *plen;
2811         uint32_t status;
2812         uint32_t buf_offset;
2813         uint32_t nframes;
2814         uint32_t itd_offset[8 + 1];
2815         uint8_t x;
2816         uint8_t td_no;
2817         uint8_t page_no;
2818
2819 #if USB_DEBUG
2820         uint8_t once = 1;
2821
2822 #endif
2823
2824         DPRINTFN(6, "xfer=%p next=%d nframes=%d\n",
2825             xfer, xfer->endpoint->isoc_next, xfer->nframes);
2826
2827         /* get the current frame index */
2828
2829         nframes = EOREAD4(sc, EHCI_FRINDEX) / 8;
2830
2831         /*
2832          * check if the frame index is within the window where the frames
2833          * will be inserted
2834          */
2835         buf_offset = (nframes - xfer->endpoint->isoc_next) &
2836             (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2837
2838         if ((xfer->endpoint->is_synced == 0) ||
2839             (buf_offset < ((xfer->nframes + 7) / 8))) {
2840                 /*
2841                  * If there is data underflow or the pipe queue is empty we
2842                  * schedule the transfer a few frames ahead of the current
2843                  * frame position. Else two isochronous transfers might
2844                  * overlap.
2845                  */
2846                 xfer->endpoint->isoc_next = (nframes + 3) &
2847                     (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2848                 xfer->endpoint->is_synced = 1;
2849                 DPRINTFN(3, "start next=%d\n", xfer->endpoint->isoc_next);
2850         }
2851         /*
2852          * compute how many milliseconds the insertion is ahead of the
2853          * current frame position:
2854          */
2855         buf_offset = (xfer->endpoint->isoc_next - nframes) &
2856             (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2857
2858         /*
2859          * pre-compute when the isochronous transfer will be finished:
2860          */
2861         xfer->isoc_time_complete =
2862             usb_isoc_time_expand(&sc->sc_bus, nframes) + buf_offset +
2863             ((xfer->nframes + 7) / 8);
2864
2865         /* get the real number of frames */
2866
2867         nframes = xfer->nframes;
2868
2869         buf_offset = 0;
2870         td_no = 0;
2871
2872         plen = xfer->frlengths;
2873
2874         /* toggle the DMA set we are using */
2875         xfer->flags_int.curr_dma_set ^= 1;
2876
2877         /* get next DMA set */
2878         td = xfer->td_start[xfer->flags_int.curr_dma_set];
2879         xfer->td_transfer_first = td;
2880
2881         pp_last = &sc->sc_isoc_hs_p_last[xfer->endpoint->isoc_next];
2882
2883         /* store starting position */
2884
2885         xfer->qh_pos = xfer->endpoint->isoc_next;
2886
2887         while (nframes) {
2888                 if (td == NULL) {
2889                         panic("%s:%d: out of TD's\n",
2890                             __FUNCTION__, __LINE__);
2891                 }
2892                 if (pp_last >= &sc->sc_isoc_hs_p_last[EHCI_VIRTUAL_FRAMELIST_COUNT]) {
2893                         pp_last = &sc->sc_isoc_hs_p_last[0];
2894                 }
2895                 /* range check */
2896                 if (*plen > xfer->max_frame_size) {
2897 #if USB_DEBUG
2898                         if (once) {
2899                                 once = 0;
2900                                 printf("%s: frame length(%d) exceeds %d bytes "
2901                                     "(frame truncated)\n",
2902                                     __FUNCTION__, *plen, xfer->max_frame_size);
2903                         }
2904 #endif
2905                         *plen = xfer->max_frame_size;
2906                 }
2907
2908                 if (xfer->usb_smask & (1 << td_no)) {
2909                         status = (EHCI_ITD_SET_LEN(*plen) |
2910                             EHCI_ITD_ACTIVE |
2911                             EHCI_ITD_SET_PG(0));
2912                         td->itd_status[td_no] = htohc32(sc, status);
2913                         itd_offset[td_no] = buf_offset;
2914                         buf_offset += *plen;
2915                         plen++;
2916                         nframes --;
2917                 } else {
2918                         td->itd_status[td_no] = 0;      /* not active */
2919                         itd_offset[td_no] = buf_offset;
2920                 }
2921
2922                 td_no++;
2923
2924                 if ((td_no == 8) || (nframes == 0)) {
2925
2926                         /* the rest of the transfers are not active, if any */
2927                         for (x = td_no; x != 8; x++) {
2928                                 td->itd_status[x] = 0;  /* not active */
2929                         }
2930
2931                         /* check if there is any data to be transferred */
2932                         if (itd_offset[0] != buf_offset) {
2933                                 page_no = 0;
2934                                 itd_offset[td_no] = buf_offset;
2935
2936                                 /* get first page offset */
2937                                 usbd_get_page(xfer->frbuffers, itd_offset[0], &buf_res);
2938                                 /* get page address */
2939                                 page_addr = buf_res.physaddr & ~0xFFF;
2940                                 /* update page address */
2941                                 td->itd_bp[0] &= htohc32(sc, 0xFFF);
2942                                 td->itd_bp[0] |= htohc32(sc, page_addr);
2943
2944                                 for (x = 0; x != td_no; x++) {
2945                                         /* set page number and page offset */
2946                                         status = (EHCI_ITD_SET_PG(page_no) |
2947                                             (buf_res.physaddr & 0xFFF));
2948                                         td->itd_status[x] |= htohc32(sc, status);
2949
2950                                         /* get next page offset */
2951                                         if (itd_offset[x + 1] == buf_offset) {
2952                                                 /*
2953                                                  * We subtract one so that
2954                                                  * we don't go off the last
2955                                                  * page!
2956                                                  */
2957                                                 usbd_get_page(xfer->frbuffers, buf_offset - 1, &buf_res);
2958                                         } else {
2959                                                 usbd_get_page(xfer->frbuffers, itd_offset[x + 1], &buf_res);
2960                                         }
2961
2962                                         /* check if we need a new page */
2963                                         if ((buf_res.physaddr ^ page_addr) & ~0xFFF) {
2964                                                 /* new page needed */
2965                                                 page_addr = buf_res.physaddr & ~0xFFF;
2966                                                 if (page_no == 6) {
2967                                                         panic("%s: too many pages\n", __FUNCTION__);
2968                                                 }
2969                                                 page_no++;
2970                                                 /* update page address */
2971                                                 td->itd_bp[page_no] &= htohc32(sc, 0xFFF);
2972                                                 td->itd_bp[page_no] |= htohc32(sc, page_addr);
2973                                         }
2974                                 }
2975                         }
2976                         /* set IOC bit if we are complete */
2977                         if (nframes == 0) {
2978                                 td->itd_status[td_no - 1] |= htohc32(sc, EHCI_ITD_IOC);
2979                         }
2980                         usb_pc_cpu_flush(td->page_cache);
2981 #if USB_DEBUG
2982                         if (ehcidebug > 15) {
2983                                 DPRINTF("HS-TD %d\n", nframes);
2984                                 ehci_dump_itd(sc, td);
2985                         }
2986 #endif
2987                         /* insert TD into schedule */
2988                         EHCI_APPEND_HS_TD(td, *pp_last);
2989                         pp_last++;
2990
2991                         td_no = 0;
2992                         td_last = td;
2993                         td = td->obj_next;
2994                 }
2995         }
2996
2997         xfer->td_transfer_last = td_last;
2998
2999         /* update isoc_next */
3000         xfer->endpoint->isoc_next = (pp_last - &sc->sc_isoc_hs_p_last[0]) &
3001             (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
3002 }
3003
3004 static void
3005 ehci_device_isoc_hs_start(struct usb_xfer *xfer)
3006 {
3007         /* put transfer on interrupt queue */
3008         ehci_transfer_intr_enqueue(xfer);
3009 }
3010
3011 struct usb_pipe_methods ehci_device_isoc_hs_methods =
3012 {
3013         .open = ehci_device_isoc_hs_open,
3014         .close = ehci_device_isoc_hs_close,
3015         .enter = ehci_device_isoc_hs_enter,
3016         .start = ehci_device_isoc_hs_start,
3017 };
3018
3019 /*------------------------------------------------------------------------*
3020  * ehci root control support
3021  *------------------------------------------------------------------------*
3022  * Simulate a hardware hub by handling all the necessary requests.
3023  *------------------------------------------------------------------------*/
3024
3025 static const
3026 struct usb_device_descriptor ehci_devd =
3027 {
3028         sizeof(struct usb_device_descriptor),
3029         UDESC_DEVICE,                   /* type */
3030         {0x00, 0x02},                   /* USB version */
3031         UDCLASS_HUB,                    /* class */
3032         UDSUBCLASS_HUB,                 /* subclass */
3033         UDPROTO_HSHUBSTT,               /* protocol */
3034         64,                             /* max packet */
3035         {0}, {0}, {0x00, 0x01},         /* device id */
3036         1, 2, 0,                        /* string indicies */
3037         1                               /* # of configurations */
3038 };
3039
3040 static const
3041 struct usb_device_qualifier ehci_odevd =
3042 {
3043         sizeof(struct usb_device_qualifier),
3044         UDESC_DEVICE_QUALIFIER,         /* type */
3045         {0x00, 0x02},                   /* USB version */
3046         UDCLASS_HUB,                    /* class */
3047         UDSUBCLASS_HUB,                 /* subclass */
3048         UDPROTO_FSHUB,                  /* protocol */
3049         0,                              /* max packet */
3050         0,                              /* # of configurations */
3051         0
3052 };
3053
3054 static const struct ehci_config_desc ehci_confd = {
3055         .confd = {
3056                 .bLength = sizeof(struct usb_config_descriptor),
3057                 .bDescriptorType = UDESC_CONFIG,
3058                 .wTotalLength[0] = sizeof(ehci_confd),
3059                 .bNumInterface = 1,
3060                 .bConfigurationValue = 1,
3061                 .iConfiguration = 0,
3062                 .bmAttributes = UC_SELF_POWERED,
3063                 .bMaxPower = 0          /* max power */
3064         },
3065         .ifcd = {
3066                 .bLength = sizeof(struct usb_interface_descriptor),
3067                 .bDescriptorType = UDESC_INTERFACE,
3068                 .bNumEndpoints = 1,
3069                 .bInterfaceClass = UICLASS_HUB,
3070                 .bInterfaceSubClass = UISUBCLASS_HUB,
3071                 .bInterfaceProtocol = UIPROTO_HSHUBSTT,
3072                 0
3073         },
3074         .endpd = {
3075                 .bLength = sizeof(struct usb_endpoint_descriptor),
3076                 .bDescriptorType = UDESC_ENDPOINT,
3077                 .bEndpointAddress = UE_DIR_IN | EHCI_INTR_ENDPT,
3078                 .bmAttributes = UE_INTERRUPT,
3079                 .wMaxPacketSize[0] = 8, /* max packet (63 ports) */
3080                 .bInterval = 255,
3081         },
3082 };
3083
3084 static const
3085 struct usb_hub_descriptor ehci_hubd =
3086 {
3087         0,                              /* dynamic length */
3088         UDESC_HUB,
3089         0,
3090         {0, 0},
3091         0,
3092         0,
3093         {0},
3094 };
3095
3096 static void
3097 ehci_disown(ehci_softc_t *sc, uint16_t index, uint8_t lowspeed)
3098 {
3099         uint32_t port;
3100         uint32_t v;
3101
3102         DPRINTF("index=%d lowspeed=%d\n", index, lowspeed);
3103
3104         port = EHCI_PORTSC(index);
3105         v = EOREAD4(sc, port) & ~EHCI_PS_CLEAR;
3106         EOWRITE4(sc, port, v | EHCI_PS_PO);
3107 }
3108
3109 static usb_error_t
3110 ehci_roothub_exec(struct usb_device *udev,
3111     struct usb_device_request *req, const void **pptr, uint16_t *plength)
3112 {
3113         ehci_softc_t *sc = EHCI_BUS2SC(udev->bus);
3114         const char *str_ptr;
3115         const void *ptr;
3116         uint32_t port;
3117         uint32_t v;
3118         uint16_t len;
3119         uint16_t i;
3120         uint16_t value;
3121         uint16_t index;
3122         uint8_t l;
3123         usb_error_t err;
3124
3125         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
3126
3127         /* buffer reset */
3128         ptr = (const void *)&sc->sc_hub_desc;
3129         len = 0;
3130         err = 0;
3131
3132         value = UGETW(req->wValue);
3133         index = UGETW(req->wIndex);
3134
3135         DPRINTFN(3, "type=0x%02x request=0x%02x wLen=0x%04x "
3136             "wValue=0x%04x wIndex=0x%04x\n",
3137             req->bmRequestType, req->bRequest,
3138             UGETW(req->wLength), value, index);
3139
3140 #define C(x,y) ((x) | ((y) << 8))
3141         switch (C(req->bRequest, req->bmRequestType)) {
3142         case C(UR_CLEAR_FEATURE, UT_WRITE_DEVICE):
3143         case C(UR_CLEAR_FEATURE, UT_WRITE_INTERFACE):
3144         case C(UR_CLEAR_FEATURE, UT_WRITE_ENDPOINT):
3145                 /*
3146                  * DEVICE_REMOTE_WAKEUP and ENDPOINT_HALT are no-ops
3147                  * for the integrated root hub.
3148                  */
3149                 break;
3150         case C(UR_GET_CONFIG, UT_READ_DEVICE):
3151                 len = 1;
3152                 sc->sc_hub_desc.temp[0] = sc->sc_conf;
3153                 break;
3154         case C(UR_GET_DESCRIPTOR, UT_READ_DEVICE):
3155                 switch (value >> 8) {
3156                 case UDESC_DEVICE:
3157                         if ((value & 0xff) != 0) {
3158                                 err = USB_ERR_IOERROR;
3159                                 goto done;
3160                         }
3161                         len = sizeof(ehci_devd);
3162                         ptr = (const void *)&ehci_devd;
3163                         break;
3164                         /*
3165                          * We can't really operate at another speed,
3166                          * but the specification says we need this
3167                          * descriptor:
3168                          */
3169                 case UDESC_DEVICE_QUALIFIER:
3170                         if ((value & 0xff) != 0) {
3171                                 err = USB_ERR_IOERROR;
3172                                 goto done;
3173                         }
3174                         len = sizeof(ehci_odevd);
3175                         ptr = (const void *)&ehci_odevd;
3176                         break;
3177
3178                 case UDESC_CONFIG:
3179                         if ((value & 0xff) != 0) {
3180                                 err = USB_ERR_IOERROR;
3181                                 goto done;
3182                         }
3183                         len = sizeof(ehci_confd);
3184                         ptr = (const void *)&ehci_confd;
3185                         break;
3186
3187                 case UDESC_STRING:
3188                         switch (value & 0xff) {
3189                         case 0: /* Language table */
3190                                 str_ptr = "\001";
3191                                 break;
3192
3193                         case 1: /* Vendor */
3194                                 str_ptr = sc->sc_vendor;
3195                                 break;
3196
3197                         case 2: /* Product */
3198                                 str_ptr = "EHCI root HUB";
3199                                 break;
3200
3201                         default:
3202                                 str_ptr = "";
3203                                 break;
3204                         }
3205
3206                         len = usb_make_str_desc(
3207                             sc->sc_hub_desc.temp,
3208                             sizeof(sc->sc_hub_desc.temp),
3209                             str_ptr);
3210                         break;
3211                 default:
3212                         err = USB_ERR_IOERROR;
3213                         goto done;
3214                 }
3215                 break;
3216         case C(UR_GET_INTERFACE, UT_READ_INTERFACE):
3217                 len = 1;
3218                 sc->sc_hub_desc.temp[0] = 0;
3219                 break;
3220         case C(UR_GET_STATUS, UT_READ_DEVICE):
3221                 len = 2;
3222                 USETW(sc->sc_hub_desc.stat.wStatus, UDS_SELF_POWERED);
3223                 break;
3224         case C(UR_GET_STATUS, UT_READ_INTERFACE):
3225         case C(UR_GET_STATUS, UT_READ_ENDPOINT):
3226                 len = 2;
3227                 USETW(sc->sc_hub_desc.stat.wStatus, 0);
3228                 break;
3229         case C(UR_SET_ADDRESS, UT_WRITE_DEVICE):
3230                 if (value >= EHCI_MAX_DEVICES) {
3231                         err = USB_ERR_IOERROR;
3232                         goto done;
3233                 }
3234                 sc->sc_addr = value;
3235                 break;
3236         case C(UR_SET_CONFIG, UT_WRITE_DEVICE):
3237                 if ((value != 0) && (value != 1)) {
3238                         err = USB_ERR_IOERROR;
3239                         goto done;
3240                 }
3241                 sc->sc_conf = value;
3242                 break;
3243         case C(UR_SET_DESCRIPTOR, UT_WRITE_DEVICE):
3244                 break;
3245         case C(UR_SET_FEATURE, UT_WRITE_DEVICE):
3246         case C(UR_SET_FEATURE, UT_WRITE_INTERFACE):
3247         case C(UR_SET_FEATURE, UT_WRITE_ENDPOINT):
3248                 err = USB_ERR_IOERROR;
3249                 goto done;
3250         case C(UR_SET_INTERFACE, UT_WRITE_INTERFACE):
3251                 break;
3252         case C(UR_SYNCH_FRAME, UT_WRITE_ENDPOINT):
3253                 break;
3254                 /* Hub requests */
3255         case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_DEVICE):
3256                 break;
3257         case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_OTHER):
3258                 DPRINTFN(9, "UR_CLEAR_PORT_FEATURE\n");
3259
3260                 if ((index < 1) ||
3261                     (index > sc->sc_noport)) {
3262                         err = USB_ERR_IOERROR;
3263                         goto done;
3264                 }
3265                 port = EHCI_PORTSC(index);
3266                 v = EOREAD4(sc, port) & ~EHCI_PS_CLEAR;
3267                 switch (value) {
3268                 case UHF_PORT_ENABLE:
3269                         EOWRITE4(sc, port, v & ~EHCI_PS_PE);
3270                         break;
3271                 case UHF_PORT_SUSPEND:
3272                         if ((v & EHCI_PS_SUSP) && (!(v & EHCI_PS_FPR))) {
3273
3274                                 /*
3275                                  * waking up a High Speed device is rather
3276                                  * complicated if
3277                                  */
3278                                 EOWRITE4(sc, port, v | EHCI_PS_FPR);
3279                         }
3280                         /* wait 20ms for resume sequence to complete */
3281                         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 50);
3282
3283                         EOWRITE4(sc, port, v & ~(EHCI_PS_SUSP |
3284                             EHCI_PS_FPR | (3 << 10) /* High Speed */ ));
3285
3286                         /* 4ms settle time */
3287                         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 250);
3288                         break;
3289                 case UHF_PORT_POWER:
3290                         EOWRITE4(sc, port, v & ~EHCI_PS_PP);
3291                         break;
3292                 case UHF_PORT_TEST:
3293                         DPRINTFN(3, "clear port test "
3294                             "%d\n", index);
3295                         break;
3296                 case UHF_PORT_INDICATOR:
3297                         DPRINTFN(3, "clear port ind "
3298                             "%d\n", index);
3299                         EOWRITE4(sc, port, v & ~EHCI_PS_PIC);
3300                         break;
3301                 case UHF_C_PORT_CONNECTION:
3302                         EOWRITE4(sc, port, v | EHCI_PS_CSC);
3303                         break;
3304                 case UHF_C_PORT_ENABLE:
3305                         EOWRITE4(sc, port, v | EHCI_PS_PEC);
3306                         break;
3307                 case UHF_C_PORT_SUSPEND:
3308                         EOWRITE4(sc, port, v | EHCI_PS_SUSP);
3309                         break;
3310                 case UHF_C_PORT_OVER_CURRENT:
3311                         EOWRITE4(sc, port, v | EHCI_PS_OCC);
3312                         break;
3313                 case UHF_C_PORT_RESET:
3314                         sc->sc_isreset = 0;
3315                         break;
3316                 default:
3317                         err = USB_ERR_IOERROR;
3318                         goto done;
3319                 }
3320                 break;
3321         case C(UR_GET_DESCRIPTOR, UT_READ_CLASS_DEVICE):
3322                 if ((value & 0xff) != 0) {
3323                         err = USB_ERR_IOERROR;
3324                         goto done;
3325                 }
3326                 v = EOREAD4(sc, EHCI_HCSPARAMS);
3327
3328                 sc->sc_hub_desc.hubd = ehci_hubd;
3329                 sc->sc_hub_desc.hubd.bNbrPorts = sc->sc_noport;
3330                 USETW(sc->sc_hub_desc.hubd.wHubCharacteristics,
3331                     (EHCI_HCS_PPC(v) ? UHD_PWR_INDIVIDUAL : UHD_PWR_NO_SWITCH) |
3332                     (EHCI_HCS_P_INDICATOR(EREAD4(sc, EHCI_HCSPARAMS)) ?
3333                     UHD_PORT_IND : 0));
3334                 /* XXX can't find out? */
3335                 sc->sc_hub_desc.hubd.bPwrOn2PwrGood = 200;
3336                 for (l = 0; l < sc->sc_noport; l++) {
3337                         /* XXX can't find out? */
3338                         sc->sc_hub_desc.hubd.DeviceRemovable[l / 8] &= ~(1 << (l % 8));
3339                 }
3340                 sc->sc_hub_desc.hubd.bDescLength =
3341                     8 + ((sc->sc_noport + 7) / 8);
3342                 len = sc->sc_hub_desc.hubd.bDescLength;
3343                 break;
3344         case C(UR_GET_STATUS, UT_READ_CLASS_DEVICE):
3345                 len = 16;
3346                 bzero(sc->sc_hub_desc.temp, 16);
3347                 break;
3348         case C(UR_GET_STATUS, UT_READ_CLASS_OTHER):
3349                 DPRINTFN(9, "get port status i=%d\n",
3350                     index);
3351                 if ((index < 1) ||
3352                     (index > sc->sc_noport)) {
3353                         err = USB_ERR_IOERROR;
3354                         goto done;
3355                 }
3356                 v = EOREAD4(sc, EHCI_PORTSC(index));
3357                 DPRINTFN(9, "port status=0x%04x\n", v);
3358                 if (sc->sc_flags & (EHCI_SCFLG_FORCESPEED | EHCI_SCFLG_TT)) {
3359                         if ((v & 0xc000000) == 0x8000000)
3360                                 i = UPS_HIGH_SPEED;
3361                         else if ((v & 0xc000000) == 0x4000000)
3362                                 i = UPS_LOW_SPEED;
3363                         else
3364                                 i = 0;
3365                 } else {
3366                         i = UPS_HIGH_SPEED;
3367                 }
3368                 if (v & EHCI_PS_CS)
3369                         i |= UPS_CURRENT_CONNECT_STATUS;
3370                 if (v & EHCI_PS_PE)
3371                         i |= UPS_PORT_ENABLED;
3372                 if ((v & EHCI_PS_SUSP) && !(v & EHCI_PS_FPR))
3373                         i |= UPS_SUSPEND;
3374                 if (v & EHCI_PS_OCA)
3375                         i |= UPS_OVERCURRENT_INDICATOR;
3376                 if (v & EHCI_PS_PR)
3377                         i |= UPS_RESET;
3378                 if (v & EHCI_PS_PP)
3379                         i |= UPS_PORT_POWER;
3380                 USETW(sc->sc_hub_desc.ps.wPortStatus, i);
3381                 i = 0;
3382                 if (v & EHCI_PS_CSC)
3383                         i |= UPS_C_CONNECT_STATUS;
3384                 if (v & EHCI_PS_PEC)
3385                         i |= UPS_C_PORT_ENABLED;
3386                 if (v & EHCI_PS_OCC)
3387                         i |= UPS_C_OVERCURRENT_INDICATOR;
3388                 if (v & EHCI_PS_FPR)
3389                         i |= UPS_C_SUSPEND;
3390                 if (sc->sc_isreset)
3391                         i |= UPS_C_PORT_RESET;
3392                 USETW(sc->sc_hub_desc.ps.wPortChange, i);
3393                 len = sizeof(sc->sc_hub_desc.ps);
3394                 break;
3395         case C(UR_SET_DESCRIPTOR, UT_WRITE_CLASS_DEVICE):
3396                 err = USB_ERR_IOERROR;
3397                 goto done;
3398         case C(UR_SET_FEATURE, UT_WRITE_CLASS_DEVICE):
3399                 break;
3400         case C(UR_SET_FEATURE, UT_WRITE_CLASS_OTHER):
3401                 if ((index < 1) ||
3402                     (index > sc->sc_noport)) {
3403                         err = USB_ERR_IOERROR;
3404                         goto done;
3405                 }
3406                 port = EHCI_PORTSC(index);
3407                 v = EOREAD4(sc, port) & ~EHCI_PS_CLEAR;
3408                 switch (value) {
3409                 case UHF_PORT_ENABLE:
3410                         EOWRITE4(sc, port, v | EHCI_PS_PE);
3411                         break;
3412                 case UHF_PORT_SUSPEND:
3413                         EOWRITE4(sc, port, v | EHCI_PS_SUSP);
3414                         break;
3415                 case UHF_PORT_RESET:
3416                         DPRINTFN(6, "reset port %d\n", index);
3417 #if USB_DEBUG
3418                         if (ehcinohighspeed) {
3419                                 /*
3420                                  * Connect USB device to companion
3421                                  * controller.
3422                                  */
3423                                 ehci_disown(sc, index, 1);
3424                                 break;
3425                         }
3426 #endif
3427                         if (EHCI_PS_IS_LOWSPEED(v) &&
3428                             (sc->sc_flags & EHCI_SCFLG_TT) == 0) {
3429                                 /* Low speed device, give up ownership. */
3430                                 ehci_disown(sc, index, 1);
3431                                 break;
3432                         }
3433                         /* Start reset sequence. */
3434                         v &= ~(EHCI_PS_PE | EHCI_PS_PR);
3435                         EOWRITE4(sc, port, v | EHCI_PS_PR);
3436
3437                         /* Wait for reset to complete. */
3438                         usb_pause_mtx(&sc->sc_bus.bus_mtx,
3439                             USB_MS_TO_TICKS(USB_PORT_ROOT_RESET_DELAY));
3440
3441                         /* Terminate reset sequence. */
3442                         if (!(sc->sc_flags & EHCI_SCFLG_NORESTERM))
3443                                 EOWRITE4(sc, port, v);
3444
3445                         /* Wait for HC to complete reset. */
3446                         usb_pause_mtx(&sc->sc_bus.bus_mtx,
3447                             USB_MS_TO_TICKS(EHCI_PORT_RESET_COMPLETE));
3448
3449                         v = EOREAD4(sc, port);
3450                         DPRINTF("ehci after reset, status=0x%08x\n", v);
3451                         if (v & EHCI_PS_PR) {
3452                                 device_printf(sc->sc_bus.bdev,
3453                                     "port reset timeout\n");
3454                                 err = USB_ERR_TIMEOUT;
3455                                 goto done;
3456                         }
3457                         if (!(v & EHCI_PS_PE) &&
3458                             (sc->sc_flags & EHCI_SCFLG_TT) == 0) {
3459                                 /* Not a high speed device, give up ownership.*/
3460                                 ehci_disown(sc, index, 0);
3461                                 break;
3462                         }
3463                         sc->sc_isreset = 1;
3464                         DPRINTF("ehci port %d reset, status = 0x%08x\n",
3465                             index, v);
3466                         break;
3467
3468                 case UHF_PORT_POWER:
3469                         DPRINTFN(3, "set port power %d\n", index);
3470                         EOWRITE4(sc, port, v | EHCI_PS_PP);
3471                         break;
3472
3473                 case UHF_PORT_TEST:
3474                         DPRINTFN(3, "set port test %d\n", index);
3475                         break;
3476
3477                 case UHF_PORT_INDICATOR:
3478                         DPRINTFN(3, "set port ind %d\n", index);
3479                         EOWRITE4(sc, port, v | EHCI_PS_PIC);
3480                         break;
3481
3482                 default:
3483                         err = USB_ERR_IOERROR;
3484                         goto done;
3485                 }
3486                 break;
3487         case C(UR_CLEAR_TT_BUFFER, UT_WRITE_CLASS_OTHER):
3488         case C(UR_RESET_TT, UT_WRITE_CLASS_OTHER):
3489         case C(UR_GET_TT_STATE, UT_READ_CLASS_OTHER):
3490         case C(UR_STOP_TT, UT_WRITE_CLASS_OTHER):
3491                 break;
3492         default:
3493                 err = USB_ERR_IOERROR;
3494                 goto done;
3495         }
3496 done:
3497         *plength = len;
3498         *pptr = ptr;
3499         return (err);
3500 }
3501
3502 static void
3503 ehci_xfer_setup(struct usb_setup_params *parm)
3504 {
3505         struct usb_page_search page_info;
3506         struct usb_page_cache *pc;
3507         ehci_softc_t *sc;
3508         struct usb_xfer *xfer;
3509         void *last_obj;
3510         uint32_t nqtd;
3511         uint32_t nqh;
3512         uint32_t nsitd;
3513         uint32_t nitd;
3514         uint32_t n;
3515
3516         sc = EHCI_BUS2SC(parm->udev->bus);
3517         xfer = parm->curr_xfer;
3518
3519         nqtd = 0;
3520         nqh = 0;
3521         nsitd = 0;
3522         nitd = 0;
3523
3524         /*
3525          * compute maximum number of some structures
3526          */
3527         if (parm->methods == &ehci_device_ctrl_methods) {
3528
3529                 /*
3530                  * The proof for the "nqtd" formula is illustrated like
3531                  * this:
3532                  *
3533                  * +------------------------------------+
3534                  * |                                    |
3535                  * |         |remainder ->              |
3536                  * |   +-----+---+                      |
3537                  * |   | xxx | x | frm 0                |
3538                  * |   +-----+---++                     |
3539                  * |   | xxx | xx | frm 1               |
3540                  * |   +-----+----+                     |
3541                  * |            ...                     |
3542                  * +------------------------------------+
3543                  *
3544                  * "xxx" means a completely full USB transfer descriptor
3545                  *
3546                  * "x" and "xx" means a short USB packet
3547                  *
3548                  * For the remainder of an USB transfer modulo
3549                  * "max_data_length" we need two USB transfer descriptors.
3550                  * One to transfer the remaining data and one to finalise
3551                  * with a zero length packet in case the "force_short_xfer"
3552                  * flag is set. We only need two USB transfer descriptors in
3553                  * the case where the transfer length of the first one is a
3554                  * factor of "max_frame_size". The rest of the needed USB
3555                  * transfer descriptors is given by the buffer size divided
3556                  * by the maximum data payload.
3557                  */
3558                 parm->hc_max_packet_size = 0x400;
3559                 parm->hc_max_packet_count = 1;
3560                 parm->hc_max_frame_size = EHCI_QTD_PAYLOAD_MAX;
3561                 xfer->flags_int.bdma_enable = 1;
3562
3563                 usbd_transfer_setup_sub(parm);
3564
3565                 nqh = 1;
3566                 nqtd = ((2 * xfer->nframes) + 1 /* STATUS */
3567                     + (xfer->max_data_length / xfer->max_hc_frame_size));
3568
3569         } else if (parm->methods == &ehci_device_bulk_methods) {
3570
3571                 parm->hc_max_packet_size = 0x400;
3572                 parm->hc_max_packet_count = 1;
3573                 parm->hc_max_frame_size = EHCI_QTD_PAYLOAD_MAX;
3574                 xfer->flags_int.bdma_enable = 1;
3575
3576                 usbd_transfer_setup_sub(parm);
3577
3578                 nqh = 1;
3579                 nqtd = ((2 * xfer->nframes)
3580                     + (xfer->max_data_length / xfer->max_hc_frame_size));
3581
3582         } else if (parm->methods == &ehci_device_intr_methods) {
3583
3584                 if (parm->speed == USB_SPEED_HIGH) {
3585                         parm->hc_max_packet_size = 0x400;
3586                         parm->hc_max_packet_count = 3;
3587                 } else if (parm->speed == USB_SPEED_FULL) {
3588                         parm->hc_max_packet_size = USB_FS_BYTES_PER_HS_UFRAME;
3589                         parm->hc_max_packet_count = 1;
3590                 } else {
3591                         parm->hc_max_packet_size = USB_FS_BYTES_PER_HS_UFRAME / 8;
3592                         parm->hc_max_packet_count = 1;
3593                 }
3594
3595                 parm->hc_max_frame_size = EHCI_QTD_PAYLOAD_MAX;
3596                 xfer->flags_int.bdma_enable = 1;
3597
3598                 usbd_transfer_setup_sub(parm);
3599
3600                 nqh = 1;
3601                 nqtd = ((2 * xfer->nframes)
3602                     + (xfer->max_data_length / xfer->max_hc_frame_size));
3603
3604         } else if (parm->methods == &ehci_device_isoc_fs_methods) {
3605
3606                 parm->hc_max_packet_size = 0x3FF;
3607                 parm->hc_max_packet_count = 1;
3608                 parm->hc_max_frame_size = 0x3FF;
3609                 xfer->flags_int.bdma_enable = 1;
3610
3611                 usbd_transfer_setup_sub(parm);
3612
3613                 nsitd = xfer->nframes;
3614
3615         } else if (parm->methods == &ehci_device_isoc_hs_methods) {
3616
3617                 parm->hc_max_packet_size = 0x400;
3618                 parm->hc_max_packet_count = 3;
3619                 parm->hc_max_frame_size = 0xC00;
3620                 xfer->flags_int.bdma_enable = 1;
3621
3622                 usbd_transfer_setup_sub(parm);
3623
3624                 nitd = ((xfer->nframes + 7) / 8) <<
3625                     usbd_xfer_get_fps_shift(xfer);
3626
3627         } else {
3628
3629                 parm->hc_max_packet_size = 0x400;
3630                 parm->hc_max_packet_count = 1;
3631                 parm->hc_max_frame_size = 0x400;
3632
3633                 usbd_transfer_setup_sub(parm);
3634         }
3635
3636 alloc_dma_set:
3637
3638         if (parm->err) {
3639                 return;
3640         }
3641         /*
3642          * Allocate queue heads and transfer descriptors
3643          */
3644         last_obj = NULL;
3645
3646         if (usbd_transfer_setup_sub_malloc(
3647             parm, &pc, sizeof(ehci_itd_t),
3648             EHCI_ITD_ALIGN, nitd)) {
3649                 parm->err = USB_ERR_NOMEM;
3650                 return;
3651         }
3652         if (parm->buf) {
3653                 for (n = 0; n != nitd; n++) {
3654                         ehci_itd_t *td;
3655
3656                         usbd_get_page(pc + n, 0, &page_info);
3657
3658                         td = page_info.buffer;
3659
3660                         /* init TD */
3661                         td->itd_self = htohc32(sc, page_info.physaddr | EHCI_LINK_ITD);
3662                         td->obj_next = last_obj;
3663                         td->page_cache = pc + n;
3664
3665                         last_obj = td;
3666
3667                         usb_pc_cpu_flush(pc + n);
3668                 }
3669         }
3670         if (usbd_transfer_setup_sub_malloc(
3671             parm, &pc, sizeof(ehci_sitd_t),
3672             EHCI_SITD_ALIGN, nsitd)) {
3673                 parm->err = USB_ERR_NOMEM;
3674                 return;
3675         }
3676         if (parm->buf) {
3677                 for (n = 0; n != nsitd; n++) {
3678                         ehci_sitd_t *td;
3679
3680                         usbd_get_page(pc + n, 0, &page_info);
3681
3682                         td = page_info.buffer;
3683
3684                         /* init TD */
3685                         td->sitd_self = htohc32(sc, page_info.physaddr | EHCI_LINK_SITD);
3686                         td->obj_next = last_obj;
3687                         td->page_cache = pc + n;
3688
3689                         last_obj = td;
3690
3691                         usb_pc_cpu_flush(pc + n);
3692                 }
3693         }
3694         if (usbd_transfer_setup_sub_malloc(
3695             parm, &pc, sizeof(ehci_qtd_t),
3696             EHCI_QTD_ALIGN, nqtd)) {
3697                 parm->err = USB_ERR_NOMEM;
3698                 return;
3699         }
3700         if (parm->buf) {
3701                 for (n = 0; n != nqtd; n++) {
3702                         ehci_qtd_t *qtd;
3703
3704                         usbd_get_page(pc + n, 0, &page_info);
3705
3706                         qtd = page_info.buffer;
3707
3708                         /* init TD */
3709                         qtd->qtd_self = htohc32(sc, page_info.physaddr);
3710                         qtd->obj_next = last_obj;
3711                         qtd->page_cache = pc + n;
3712
3713                         last_obj = qtd;
3714
3715                         usb_pc_cpu_flush(pc + n);
3716                 }
3717         }
3718         xfer->td_start[xfer->flags_int.curr_dma_set] = last_obj;
3719
3720         last_obj = NULL;
3721
3722         if (usbd_transfer_setup_sub_malloc(
3723             parm, &pc, sizeof(ehci_qh_t),
3724             EHCI_QH_ALIGN, nqh)) {
3725                 parm->err = USB_ERR_NOMEM;
3726                 return;
3727         }
3728         if (parm->buf) {
3729                 for (n = 0; n != nqh; n++) {
3730                         ehci_qh_t *qh;
3731
3732                         usbd_get_page(pc + n, 0, &page_info);
3733
3734                         qh = page_info.buffer;
3735
3736                         /* init QH */
3737                         qh->qh_self = htohc32(sc, page_info.physaddr | EHCI_LINK_QH);
3738                         qh->obj_next = last_obj;
3739                         qh->page_cache = pc + n;
3740
3741                         last_obj = qh;
3742
3743                         usb_pc_cpu_flush(pc + n);
3744                 }
3745         }
3746         xfer->qh_start[xfer->flags_int.curr_dma_set] = last_obj;
3747
3748         if (!xfer->flags_int.curr_dma_set) {
3749                 xfer->flags_int.curr_dma_set = 1;
3750                 goto alloc_dma_set;
3751         }
3752 }
3753
3754 static void
3755 ehci_xfer_unsetup(struct usb_xfer *xfer)
3756 {
3757         return;
3758 }
3759
3760 static void
3761 ehci_ep_init(struct usb_device *udev, struct usb_endpoint_descriptor *edesc,
3762     struct usb_endpoint *ep)
3763 {
3764         ehci_softc_t *sc = EHCI_BUS2SC(udev->bus);
3765
3766         DPRINTFN(2, "endpoint=%p, addr=%d, endpt=%d, mode=%d (%d)\n",
3767             ep, udev->address,
3768             edesc->bEndpointAddress, udev->flags.usb_mode,
3769             sc->sc_addr);
3770
3771         if (udev->flags.usb_mode != USB_MODE_HOST) {
3772                 /* not supported */
3773                 return;
3774         }
3775         if (udev->device_index != sc->sc_addr) {
3776
3777                 if ((udev->speed != USB_SPEED_HIGH) &&
3778                     ((udev->hs_hub_addr == 0) ||
3779                     (udev->hs_port_no == 0) ||
3780                     (udev->parent_hs_hub == NULL) ||
3781                     (udev->parent_hs_hub->hub == NULL))) {
3782                         /* We need a transaction translator */
3783                         goto done;
3784                 }
3785                 switch (edesc->bmAttributes & UE_XFERTYPE) {
3786                 case UE_CONTROL:
3787                         ep->methods = &ehci_device_ctrl_methods;
3788                         break;
3789                 case UE_INTERRUPT:
3790                         ep->methods = &ehci_device_intr_methods;
3791                         break;
3792                 case UE_ISOCHRONOUS:
3793                         if (udev->speed == USB_SPEED_HIGH) {
3794                                 ep->methods = &ehci_device_isoc_hs_methods;
3795                         } else if (udev->speed == USB_SPEED_FULL) {
3796                                 ep->methods = &ehci_device_isoc_fs_methods;
3797                         }
3798                         break;
3799                 case UE_BULK:
3800                         if (udev->speed != USB_SPEED_LOW) {
3801                                 ep->methods = &ehci_device_bulk_methods;
3802                         }
3803                         break;
3804                 default:
3805                         /* do nothing */
3806                         break;
3807                 }
3808         }
3809 done:
3810         return;
3811 }
3812
3813 static void
3814 ehci_get_dma_delay(struct usb_bus *bus, uint32_t *pus)
3815 {
3816         /*
3817          * Wait until the hardware has finished any possible use of
3818          * the transfer descriptor(s) and QH
3819          */
3820         *pus = (188);                   /* microseconds */
3821 }
3822
3823 static void
3824 ehci_device_resume(struct usb_device *udev)
3825 {
3826         ehci_softc_t *sc = EHCI_BUS2SC(udev->bus);
3827         struct usb_xfer *xfer;
3828         struct usb_pipe_methods *methods;
3829
3830         DPRINTF("\n");
3831
3832         USB_BUS_LOCK(udev->bus);
3833
3834         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
3835
3836                 if (xfer->xroot->udev == udev) {
3837
3838                         methods = xfer->endpoint->methods;
3839
3840                         if ((methods == &ehci_device_bulk_methods) ||
3841                             (methods == &ehci_device_ctrl_methods)) {
3842                                 EHCI_APPEND_QH(xfer->qh_start[xfer->flags_int.curr_dma_set],
3843                                     sc->sc_async_p_last);
3844                         }
3845                         if (methods == &ehci_device_intr_methods) {
3846                                 EHCI_APPEND_QH(xfer->qh_start[xfer->flags_int.curr_dma_set],
3847                                     sc->sc_intr_p_last[xfer->qh_pos]);
3848                         }
3849                 }
3850         }
3851
3852         USB_BUS_UNLOCK(udev->bus);
3853
3854         return;
3855 }
3856
3857 static void
3858 ehci_device_suspend(struct usb_device *udev)
3859 {
3860         ehci_softc_t *sc = EHCI_BUS2SC(udev->bus);
3861         struct usb_xfer *xfer;
3862         struct usb_pipe_methods *methods;
3863
3864         DPRINTF("\n");
3865
3866         USB_BUS_LOCK(udev->bus);
3867
3868         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
3869
3870                 if (xfer->xroot->udev == udev) {
3871
3872                         methods = xfer->endpoint->methods;
3873
3874                         if ((methods == &ehci_device_bulk_methods) ||
3875                             (methods == &ehci_device_ctrl_methods)) {
3876                                 EHCI_REMOVE_QH(xfer->qh_start[xfer->flags_int.curr_dma_set],
3877                                     sc->sc_async_p_last);
3878                         }
3879                         if (methods == &ehci_device_intr_methods) {
3880                                 EHCI_REMOVE_QH(xfer->qh_start[xfer->flags_int.curr_dma_set],
3881                                     sc->sc_intr_p_last[xfer->qh_pos]);
3882                         }
3883                 }
3884         }
3885
3886         USB_BUS_UNLOCK(udev->bus);
3887
3888         return;
3889 }
3890
3891 static void
3892 ehci_set_hw_power(struct usb_bus *bus)
3893 {
3894         ehci_softc_t *sc = EHCI_BUS2SC(bus);
3895         uint32_t temp;
3896         uint32_t flags;
3897
3898         DPRINTF("\n");
3899
3900         USB_BUS_LOCK(bus);
3901
3902         flags = bus->hw_power_state;
3903
3904         temp = EOREAD4(sc, EHCI_USBCMD);
3905
3906         temp &= ~(EHCI_CMD_ASE | EHCI_CMD_PSE);
3907
3908         if (flags & (USB_HW_POWER_CONTROL |
3909             USB_HW_POWER_BULK)) {
3910                 DPRINTF("Async is active\n");
3911                 temp |= EHCI_CMD_ASE;
3912         }
3913         if (flags & (USB_HW_POWER_INTERRUPT |
3914             USB_HW_POWER_ISOC)) {
3915                 DPRINTF("Periodic is active\n");
3916                 temp |= EHCI_CMD_PSE;
3917         }
3918         EOWRITE4(sc, EHCI_USBCMD, temp);
3919
3920         USB_BUS_UNLOCK(bus);
3921
3922         return;
3923 }
3924
3925 struct usb_bus_methods ehci_bus_methods =
3926 {
3927         .endpoint_init = ehci_ep_init,
3928         .xfer_setup = ehci_xfer_setup,
3929         .xfer_unsetup = ehci_xfer_unsetup,
3930         .get_dma_delay = ehci_get_dma_delay,
3931         .device_resume = ehci_device_resume,
3932         .device_suspend = ehci_device_suspend,
3933         .set_hw_power = ehci_set_hw_power,
3934         .roothub_exec = ehci_roothub_exec,
3935         .xfer_poll = ehci_do_poll,
3936 };