sys/dev/usb/controller/uhci.c

   1 /*-
   2  * Copyright (c) 2008 Hans Petter Selasky. All rights reserved.
   3  * Copyright (c) 1998 The NetBSD Foundation, Inc. All rights reserved.
   4  * Copyright (c) 1998 Lennart Augustsson. All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  */
  27
  28 #include <sys/cdefs.h>
  29 __FBSDID("$FreeBSD$");
  30
  31 /*
  32  * USB Universal Host Controller driver.
  33  * Handles e.g. PIIX3 and PIIX4.
  34  *
  35  * UHCI spec: http://developer.intel.com/design/USB/UHCI11D.htm
  36  * USB spec:  http://www.usb.org/developers/docs/usbspec.zip
  37  * PIIXn spec: ftp://download.intel.com/design/intarch/datashts/29055002.pdf
  38  *             ftp://download.intel.com/design/intarch/datashts/29056201.pdf
  39  */
  40
  41 #include <sys/stdint.h>
  42 #include <sys/stddef.h>
  43 #include <sys/param.h>
  44 #include <sys/queue.h>
  45 #include <sys/types.h>
  46 #include <sys/systm.h>
  47 #include <sys/kernel.h>
  48 #include <sys/bus.h>
  49 #include <sys/linker_set.h>
  50 #include <sys/module.h>
  51 #include <sys/lock.h>
  52 #include <sys/mutex.h>
  53 #include <sys/condvar.h>
  54 #include <sys/sysctl.h>
  55 #include <sys/sx.h>
  56 #include <sys/unistd.h>
  57 #include <sys/callout.h>
  58 #include <sys/malloc.h>
  59 #include <sys/priv.h>
  60
  61 #include <dev/usb/usb.h>
  62 #include <dev/usb/usbdi.h>
  63
  64 #define USB_DEBUG_VAR uhcidebug
  65
  66 #include <dev/usb/usb_core.h>
  67 #include <dev/usb/usb_debug.h>
  68 #include <dev/usb/usb_busdma.h>
  69 #include <dev/usb/usb_process.h>
  70 #include <dev/usb/usb_transfer.h>
  71 #include <dev/usb/usb_device.h>
  72 #include <dev/usb/usb_hub.h>
  73 #include <dev/usb/usb_util.h>
  74
  75 #include <dev/usb/usb_controller.h>
  76 #include <dev/usb/usb_bus.h>
  77 #include <dev/usb/controller/uhci.h>
  78 #include <dev/usb/controller/uhcireg.h>
  79
  80 #define alt_next next
  81 #define UHCI_BUS2SC(bus) \
  82    ((uhci_softc_t *)(((uint8_t *)(bus)) - \
  83     ((uint8_t *)&(((uhci_softc_t *)0)->sc_bus))))
  84
  85 #if USB_DEBUG
  86 static int uhcidebug = 0;
  87 static int uhcinoloop = 0;
  88
  89 SYSCTL_NODE(_hw_usb, OID_AUTO, uhci, CTLFLAG_RW, 0, "USB uhci");
  90 SYSCTL_INT(_hw_usb_uhci, OID_AUTO, debug, CTLFLAG_RW,
  91     &uhcidebug, 0, "uhci debug level");
  92 SYSCTL_INT(_hw_usb_uhci, OID_AUTO, loop, CTLFLAG_RW,
  93     &uhcinoloop, 0, "uhci noloop");
  94 static void uhci_dumpregs(uhci_softc_t *sc);
  95 static void uhci_dump_tds(uhci_td_t *td);
  96
  97 #endif
  98
  99 #define UBARR(sc) bus_space_barrier((sc)->sc_io_tag, (sc)->sc_io_hdl, 0, (sc)->sc_io_size, \
 100                         BUS_SPACE_BARRIER_READ|BUS_SPACE_BARRIER_WRITE)
 101 #define UWRITE1(sc, r, x) \
 102  do { UBARR(sc); bus_space_write_1((sc)->sc_io_tag, (sc)->sc_io_hdl, (r), (x)); \
 103  } while (/*CONSTCOND*/0)
 104 #define UWRITE2(sc, r, x) \
 105  do { UBARR(sc); bus_space_write_2((sc)->sc_io_tag, (sc)->sc_io_hdl, (r), (x)); \
 106  } while (/*CONSTCOND*/0)
 107 #define UWRITE4(sc, r, x) \
 108  do { UBARR(sc); bus_space_write_4((sc)->sc_io_tag, (sc)->sc_io_hdl, (r), (x)); \
 109  } while (/*CONSTCOND*/0)
 110 #define UREAD1(sc, r) (UBARR(sc), bus_space_read_1((sc)->sc_io_tag, (sc)->sc_io_hdl, (r)))
 111 #define UREAD2(sc, r) (UBARR(sc), bus_space_read_2((sc)->sc_io_tag, (sc)->sc_io_hdl, (r)))
 112 #define UREAD4(sc, r) (UBARR(sc), bus_space_read_4((sc)->sc_io_tag, (sc)->sc_io_hdl, (r)))
 113
 114 #define UHCICMD(sc, cmd) UWRITE2(sc, UHCI_CMD, cmd)
 115 #define UHCISTS(sc) UREAD2(sc, UHCI_STS)
 116
 117 #define UHCI_RESET_TIMEOUT 100          /* ms, reset timeout */
 118
 119 #define UHCI_INTR_ENDPT 1
 120
 121 struct uhci_mem_layout {
 122
 123         struct usb_page_search buf_res;
 124         struct usb_page_search fix_res;
 125
 126         struct usb_page_cache *buf_pc;
 127         struct usb_page_cache *fix_pc;
 128
 129         uint32_t buf_offset;
 130
 131         uint16_t max_frame_size;
 132 };
 133
 134 struct uhci_std_temp {
 135
 136         struct uhci_mem_layout ml;
 137         uhci_td_t *td;
 138         uhci_td_t *td_next;
 139         uint32_t average;
 140         uint32_t td_status;
 141         uint32_t td_token;
 142         uint32_t len;
 143         uint16_t max_frame_size;
 144         uint8_t shortpkt;
 145         uint8_t setup_alt_next;
 146         uint8_t last_frame;
 147 };
 148
 149 extern struct usb_bus_methods uhci_bus_methods;
 150 extern struct usb_pipe_methods uhci_device_bulk_methods;
 151 extern struct usb_pipe_methods uhci_device_ctrl_methods;
 152 extern struct usb_pipe_methods uhci_device_intr_methods;
 153 extern struct usb_pipe_methods uhci_device_isoc_methods;
 154
 155 static uint8_t  uhci_restart(uhci_softc_t *sc);
 156 static void     uhci_do_poll(struct usb_bus *);
 157 static void     uhci_device_done(struct usb_xfer *, usb_error_t);
 158 static void     uhci_transfer_intr_enqueue(struct usb_xfer *);
 159 static void     uhci_timeout(void *);
 160 static uint8_t  uhci_check_transfer(struct usb_xfer *);
 161 static void     uhci_root_intr(uhci_softc_t *sc);
 162
 163 void
 164 uhci_iterate_hw_softc(struct usb_bus *bus, usb_bus_mem_sub_cb_t *cb)
 165 {
 166         struct uhci_softc *sc = UHCI_BUS2SC(bus);
 167         uint32_t i;
 168
 169         cb(bus, &sc->sc_hw.pframes_pc, &sc->sc_hw.pframes_pg,
 170             sizeof(uint32_t) * UHCI_FRAMELIST_COUNT, UHCI_FRAMELIST_ALIGN);
 171
 172         cb(bus, &sc->sc_hw.ls_ctl_start_pc, &sc->sc_hw.ls_ctl_start_pg,
 173             sizeof(uhci_qh_t), UHCI_QH_ALIGN);
 174
 175         cb(bus, &sc->sc_hw.fs_ctl_start_pc, &sc->sc_hw.fs_ctl_start_pg,
 176             sizeof(uhci_qh_t), UHCI_QH_ALIGN);
 177
 178         cb(bus, &sc->sc_hw.bulk_start_pc, &sc->sc_hw.bulk_start_pg,
 179             sizeof(uhci_qh_t), UHCI_QH_ALIGN);
 180
 181         cb(bus, &sc->sc_hw.last_qh_pc, &sc->sc_hw.last_qh_pg,
 182             sizeof(uhci_qh_t), UHCI_QH_ALIGN);
 183
 184         cb(bus, &sc->sc_hw.last_td_pc, &sc->sc_hw.last_td_pg,
 185             sizeof(uhci_td_t), UHCI_TD_ALIGN);
 186
 187         for (i = 0; i != UHCI_VFRAMELIST_COUNT; i++) {
 188                 cb(bus, sc->sc_hw.isoc_start_pc + i,
 189                     sc->sc_hw.isoc_start_pg + i,
 190                     sizeof(uhci_td_t), UHCI_TD_ALIGN);
 191         }
 192
 193         for (i = 0; i != UHCI_IFRAMELIST_COUNT; i++) {
 194                 cb(bus, sc->sc_hw.intr_start_pc + i,
 195                     sc->sc_hw.intr_start_pg + i,
 196                     sizeof(uhci_qh_t), UHCI_QH_ALIGN);
 197         }
 198 }
 199
 200 static void
 201 uhci_mem_layout_init(struct uhci_mem_layout *ml, struct usb_xfer *xfer)
 202 {
 203         ml->buf_pc = xfer->frbuffers + 0;
 204         ml->fix_pc = xfer->buf_fixup;
 205
 206         ml->buf_offset = 0;
 207
 208         ml->max_frame_size = xfer->max_frame_size;
 209 }
 210
 211 static void
 212 uhci_mem_layout_fixup(struct uhci_mem_layout *ml, struct uhci_td *td)
 213 {
 214         usbd_get_page(ml->buf_pc, ml->buf_offset, &ml->buf_res);
 215
 216         if (ml->buf_res.length < td->len) {
 217
 218                 /* need to do a fixup */
 219
 220                 usbd_get_page(ml->fix_pc, 0, &ml->fix_res);
 221
 222                 td->td_buffer = htole32(ml->fix_res.physaddr);
 223
 224                 /*
 225                  * The UHCI driver cannot handle
 226                  * page crossings, so a fixup is
 227                  * needed:
 228                  *
 229                  *  +----+----+ - - -
 230                  *  | YYY|Y   |
 231                  *  +----+----+ - - -
 232                  *     \    \
 233                  *      \    \
 234                  *       +----+
 235                  *       |YYYY|  (fixup)
 236                  *       +----+
 237                  */
 238
 239                 if ((td->td_token & htole32(UHCI_TD_PID)) ==
 240                     htole32(UHCI_TD_PID_IN)) {
 241                         td->fix_pc = ml->fix_pc;
 242                         usb_pc_cpu_invalidate(ml->fix_pc);
 243
 244                 } else {
 245                         td->fix_pc = NULL;
 246
 247                         /* copy data to fixup location */
 248
 249                         usbd_copy_out(ml->buf_pc, ml->buf_offset,
 250                             ml->fix_res.buffer, td->len);
 251
 252                         usb_pc_cpu_flush(ml->fix_pc);
 253                 }
 254
 255                 /* prepare next fixup */
 256
 257                 ml->fix_pc++;
 258
 259         } else {
 260
 261                 td->td_buffer = htole32(ml->buf_res.physaddr);
 262                 td->fix_pc = NULL;
 263         }
 264
 265         /* prepare next data location */
 266
 267         ml->buf_offset += td->len;
 268 }
 269
 270 /*
 271  * Return values:
 272  * 0: Success
 273  * Else: Failure
 274  */
 275 static uint8_t
 276 uhci_restart(uhci_softc_t *sc)
 277 {
 278         struct usb_page_search buf_res;
 279
 280         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
 281
 282         if (UREAD2(sc, UHCI_CMD) & UHCI_CMD_RS) {
 283                 DPRINTFN(2, "Already started\n");
 284                 return (0);
 285         }
 286
 287         DPRINTFN(2, "Restarting\n");
 288
 289         usbd_get_page(&sc->sc_hw.pframes_pc, 0, &buf_res);
 290
 291         /* Reload fresh base address */
 292         UWRITE4(sc, UHCI_FLBASEADDR, buf_res.physaddr);
 293
 294         /*
 295          * Assume 64 byte packets at frame end and start HC controller:
 296          */
 297         UHCICMD(sc, (UHCI_CMD_MAXP | UHCI_CMD_RS));
 298
 299         /* wait 10 milliseconds */
 300
 301         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 100);
 302
 303         /* check that controller has started */
 304
 305         if (UREAD2(sc, UHCI_STS) & UHCI_STS_HCH) {
 306                 DPRINTFN(2, "Failed\n");
 307                 return (1);
 308         }
 309         return (0);
 310 }
 311
 312 void
 313 uhci_reset(uhci_softc_t *sc)
 314 {
 315         uint16_t n;
 316
 317         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
 318
 319         DPRINTF("resetting the HC\n");
 320
 321         /* disable interrupts */
 322
 323         UWRITE2(sc, UHCI_INTR, 0);
 324
 325         /* global reset */
 326
 327         UHCICMD(sc, UHCI_CMD_GRESET);
 328
 329         /* wait */
 330
 331         usb_pause_mtx(&sc->sc_bus.bus_mtx,
 332             USB_MS_TO_TICKS(USB_BUS_RESET_DELAY));
 333
 334         /* terminate all transfers */
 335
 336         UHCICMD(sc, UHCI_CMD_HCRESET);
 337
 338         /* the reset bit goes low when the controller is done */
 339
 340         n = UHCI_RESET_TIMEOUT;
 341         while (n--) {
 342                 /* wait one millisecond */
 343
 344                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 1000);
 345
 346                 if (!(UREAD2(sc, UHCI_CMD) & UHCI_CMD_HCRESET)) {
 347                         goto done_1;
 348                 }
 349         }
 350
 351         device_printf(sc->sc_bus.bdev,
 352             "controller did not reset\n");
 353
 354 done_1:
 355
 356         n = 10;
 357         while (n--) {
 358                 /* wait one millisecond */
 359
 360                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 1000);
 361
 362                 /* check if HC is stopped */
 363                 if (UREAD2(sc, UHCI_STS) & UHCI_STS_HCH) {
 364                         goto done_2;
 365                 }
 366         }
 367
 368         device_printf(sc->sc_bus.bdev,
 369             "controller did not stop\n");
 370
 371 done_2:
 372
 373         /* reload the configuration */
 374         UWRITE2(sc, UHCI_FRNUM, sc->sc_saved_frnum);
 375         UWRITE1(sc, UHCI_SOF, sc->sc_saved_sof);
 376
 377         USB_BUS_UNLOCK(&sc->sc_bus);
 378
 379         /* stop root interrupt */
 380         usb_callout_drain(&sc->sc_root_intr);
 381
 382         USB_BUS_LOCK(&sc->sc_bus);
 383 }
 384
 385 static void
 386 uhci_start(uhci_softc_t *sc)
 387 {
 388         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
 389
 390         DPRINTFN(2, "enabling\n");
 391
 392         /* enable interrupts */
 393
 394         UWRITE2(sc, UHCI_INTR,
 395             (UHCI_INTR_TOCRCIE |
 396             UHCI_INTR_RIE |
 397             UHCI_INTR_IOCE |
 398             UHCI_INTR_SPIE));
 399
 400         if (uhci_restart(sc)) {
 401                 device_printf(sc->sc_bus.bdev,
 402                     "cannot start HC controller\n");
 403         }
 404
 405         /* start root interrupt */
 406         uhci_root_intr(sc);
 407 }
 408
 409 static struct uhci_qh *
 410 uhci_init_qh(struct usb_page_cache *pc)
 411 {
 412         struct usb_page_search buf_res;
 413         struct uhci_qh *qh;
 414
 415         usbd_get_page(pc, 0, &buf_res);
 416
 417         qh = buf_res.buffer;
 418
 419         qh->qh_self =
 420             htole32(buf_res.physaddr) |
 421             htole32(UHCI_PTR_QH);
 422
 423         qh->page_cache = pc;
 424
 425         return (qh);
 426 }
 427
 428 static struct uhci_td *
 429 uhci_init_td(struct usb_page_cache *pc)
 430 {
 431         struct usb_page_search buf_res;
 432         struct uhci_td *td;
 433
 434         usbd_get_page(pc, 0, &buf_res);
 435
 436         td = buf_res.buffer;
 437
 438         td->td_self =
 439             htole32(buf_res.physaddr) |
 440             htole32(UHCI_PTR_TD);
 441
 442         td->page_cache = pc;
 443
 444         return (td);
 445 }
 446
 447 usb_error_t
 448 uhci_init(uhci_softc_t *sc)
 449 {
 450         uint16_t bit;
 451         uint16_t x;
 452         uint16_t y;
 453
 454         DPRINTF("start\n");
 455
 456         usb_callout_init_mtx(&sc->sc_root_intr, &sc->sc_bus.bus_mtx, 0);
 457
 458 #if USB_DEBUG
 459         if (uhcidebug > 2) {
 460                 uhci_dumpregs(sc);
 461         }
 462 #endif
 463         sc->sc_saved_sof = 0x40;        /* default value */
 464         sc->sc_saved_frnum = 0;         /* default frame number */
 465
 466         /*
 467          * Setup QH's
 468          */
 469         sc->sc_ls_ctl_p_last =
 470             uhci_init_qh(&sc->sc_hw.ls_ctl_start_pc);
 471
 472         sc->sc_fs_ctl_p_last =
 473             uhci_init_qh(&sc->sc_hw.fs_ctl_start_pc);
 474
 475         sc->sc_bulk_p_last =
 476             uhci_init_qh(&sc->sc_hw.bulk_start_pc);
 477 #if 0
 478         sc->sc_reclaim_qh_p =
 479             sc->sc_fs_ctl_p_last;
 480 #else
 481         /* setup reclaim looping point */
 482         sc->sc_reclaim_qh_p =
 483             sc->sc_bulk_p_last;
 484 #endif
 485
 486         sc->sc_last_qh_p =
 487             uhci_init_qh(&sc->sc_hw.last_qh_pc);
 488
 489         sc->sc_last_td_p =
 490             uhci_init_td(&sc->sc_hw.last_td_pc);
 491
 492         for (x = 0; x != UHCI_VFRAMELIST_COUNT; x++) {
 493                 sc->sc_isoc_p_last[x] =
 494                     uhci_init_td(sc->sc_hw.isoc_start_pc + x);
 495         }
 496
 497         for (x = 0; x != UHCI_IFRAMELIST_COUNT; x++) {
 498                 sc->sc_intr_p_last[x] =
 499                     uhci_init_qh(sc->sc_hw.intr_start_pc + x);
 500         }
 501
 502         /*
 503          * the QHs are arranged to give poll intervals that are
 504          * powers of 2 times 1ms
 505          */
 506         bit = UHCI_IFRAMELIST_COUNT / 2;
 507         while (bit) {
 508                 x = bit;
 509                 while (x & bit) {
 510                         uhci_qh_t *qh_x;
 511                         uhci_qh_t *qh_y;
 512
 513                         y = (x ^ bit) | (bit / 2);
 514
 515                         /*
 516                          * the next QH has half the poll interval
 517                          */
 518                         qh_x = sc->sc_intr_p_last[x];
 519                         qh_y = sc->sc_intr_p_last[y];
 520
 521                         qh_x->h_next = NULL;
 522                         qh_x->qh_h_next = qh_y->qh_self;
 523                         qh_x->e_next = NULL;
 524                         qh_x->qh_e_next = htole32(UHCI_PTR_T);
 525                         x++;
 526                 }
 527                 bit >>= 1;
 528         }
 529
 530         if (1) {
 531                 uhci_qh_t *qh_ls;
 532                 uhci_qh_t *qh_intr;
 533
 534                 qh_ls = sc->sc_ls_ctl_p_last;
 535                 qh_intr = sc->sc_intr_p_last[0];
 536
 537                 /* start QH for interrupt traffic */
 538                 qh_intr->h_next = qh_ls;
 539                 qh_intr->qh_h_next = qh_ls->qh_self;
 540                 qh_intr->e_next = 0;
 541                 qh_intr->qh_e_next = htole32(UHCI_PTR_T);
 542         }
 543         for (x = 0; x != UHCI_VFRAMELIST_COUNT; x++) {
 544
 545                 uhci_td_t *td_x;
 546                 uhci_qh_t *qh_intr;
 547
 548                 td_x = sc->sc_isoc_p_last[x];
 549                 qh_intr = sc->sc_intr_p_last[x | (UHCI_IFRAMELIST_COUNT / 2)];
 550
 551                 /* start TD for isochronous traffic */
 552                 td_x->next = NULL;
 553                 td_x->td_next = qh_intr->qh_self;
 554                 td_x->td_status = htole32(UHCI_TD_IOS);
 555                 td_x->td_token = htole32(0);
 556                 td_x->td_buffer = htole32(0);
 557         }
 558
 559         if (1) {
 560                 uhci_qh_t *qh_ls;
 561                 uhci_qh_t *qh_fs;
 562
 563                 qh_ls = sc->sc_ls_ctl_p_last;
 564                 qh_fs = sc->sc_fs_ctl_p_last;
 565
 566                 /* start QH where low speed control traffic will be queued */
 567                 qh_ls->h_next = qh_fs;
 568                 qh_ls->qh_h_next = qh_fs->qh_self;
 569                 qh_ls->e_next = 0;
 570                 qh_ls->qh_e_next = htole32(UHCI_PTR_T);
 571         }
 572         if (1) {
 573                 uhci_qh_t *qh_ctl;
 574                 uhci_qh_t *qh_blk;
 575                 uhci_qh_t *qh_lst;
 576                 uhci_td_t *td_lst;
 577
 578                 qh_ctl = sc->sc_fs_ctl_p_last;
 579                 qh_blk = sc->sc_bulk_p_last;
 580
 581                 /* start QH where full speed control traffic will be queued */
 582                 qh_ctl->h_next = qh_blk;
 583                 qh_ctl->qh_h_next = qh_blk->qh_self;
 584                 qh_ctl->e_next = 0;
 585                 qh_ctl->qh_e_next = htole32(UHCI_PTR_T);
 586
 587                 qh_lst = sc->sc_last_qh_p;
 588
 589                 /* start QH where bulk traffic will be queued */
 590                 qh_blk->h_next = qh_lst;
 591                 qh_blk->qh_h_next = qh_lst->qh_self;
 592                 qh_blk->e_next = 0;
 593                 qh_blk->qh_e_next = htole32(UHCI_PTR_T);
 594
 595                 td_lst = sc->sc_last_td_p;
 596
 597                 /* end QH which is used for looping the QHs */
 598                 qh_lst->h_next = 0;
 599                 qh_lst->qh_h_next = htole32(UHCI_PTR_T);        /* end of QH chain */
 600                 qh_lst->e_next = td_lst;
 601                 qh_lst->qh_e_next = td_lst->td_self;
 602
 603                 /*
 604                  * end TD which hangs from the last QH, to avoid a bug in the PIIX
 605                  * that makes it run berserk otherwise
 606                  */
 607                 td_lst->next = 0;
 608                 td_lst->td_next = htole32(UHCI_PTR_T);
 609                 td_lst->td_status = htole32(0); /* inactive */
 610                 td_lst->td_token = htole32(0);
 611                 td_lst->td_buffer = htole32(0);
 612         }
 613         if (1) {
 614                 struct usb_page_search buf_res;
 615                 uint32_t *pframes;
 616
 617                 usbd_get_page(&sc->sc_hw.pframes_pc, 0, &buf_res);
 618
 619                 pframes = buf_res.buffer;
 620
 621
 622                 /*
 623                  * Setup UHCI framelist
 624                  *
 625                  * Execution order:
 626                  *
 627                  * pframes -> full speed isochronous -> interrupt QH's -> low
 628                  * speed control -> full speed control -> bulk transfers
 629                  *
 630                  */
 631
 632                 for (x = 0; x != UHCI_FRAMELIST_COUNT; x++) {
 633                         pframes[x] =
 634                             sc->sc_isoc_p_last[x % UHCI_VFRAMELIST_COUNT]->td_self;
 635                 }
 636         }
 637         /* flush all cache into memory */
 638
 639         usb_bus_mem_flush_all(&sc->sc_bus, &uhci_iterate_hw_softc);
 640
 641         /* set up the bus struct */
 642         sc->sc_bus.methods = &uhci_bus_methods;
 643
 644         USB_BUS_LOCK(&sc->sc_bus);
 645         /* reset the controller */
 646         uhci_reset(sc);
 647
 648         /* start the controller */
 649         uhci_start(sc);
 650         USB_BUS_UNLOCK(&sc->sc_bus);
 651
 652         /* catch lost interrupts */
 653         uhci_do_poll(&sc->sc_bus);
 654
 655         return (0);
 656 }
 657
 658 /* NOTE: suspend/resume is called from
 659  * interrupt context and cannot sleep!
 660  */
 661
 662 void
 663 uhci_suspend(uhci_softc_t *sc)
 664 {
 665         USB_BUS_LOCK(&sc->sc_bus);
 666
 667 #if USB_DEBUG
 668         if (uhcidebug > 2) {
 669                 uhci_dumpregs(sc);
 670         }
 671 #endif
 672         /* save some state if BIOS doesn't */
 673
 674         sc->sc_saved_frnum = UREAD2(sc, UHCI_FRNUM);
 675         sc->sc_saved_sof = UREAD1(sc, UHCI_SOF);
 676
 677         /* stop the controller */
 678
 679         uhci_reset(sc);
 680
 681         /* enter global suspend */
 682
 683         UHCICMD(sc, UHCI_CMD_EGSM);
 684
 685         usb_pause_mtx(&sc->sc_bus.bus_mtx,
 686             USB_MS_TO_TICKS(USB_RESUME_WAIT));
 687
 688         USB_BUS_UNLOCK(&sc->sc_bus);
 689 }
 690
 691 void
 692 uhci_resume(uhci_softc_t *sc)
 693 {
 694         USB_BUS_LOCK(&sc->sc_bus);
 695
 696         /* reset the controller */
 697
 698         uhci_reset(sc);
 699
 700         /* force global resume */
 701
 702         UHCICMD(sc, UHCI_CMD_FGR);
 703
 704         usb_pause_mtx(&sc->sc_bus.bus_mtx,
 705             USB_MS_TO_TICKS(USB_RESUME_DELAY));
 706
 707         /* and start traffic again */
 708
 709         uhci_start(sc);
 710
 711 #if USB_DEBUG
 712         if (uhcidebug > 2) {
 713                 uhci_dumpregs(sc);
 714         }
 715 #endif
 716
 717         USB_BUS_UNLOCK(&sc->sc_bus);
 718
 719         /* catch lost interrupts */
 720         uhci_do_poll(&sc->sc_bus);
 721 }
 722
 723 #if USB_DEBUG
 724 static void
 725 uhci_dumpregs(uhci_softc_t *sc)
 726 {
 727         DPRINTFN(0, "%s regs: cmd=%04x, sts=%04x, intr=%04x, frnum=%04x, "
 728             "flbase=%08x, sof=%04x, portsc1=%04x, portsc2=%04x\n",
 729             device_get_nameunit(sc->sc_bus.bdev),
 730             UREAD2(sc, UHCI_CMD),
 731             UREAD2(sc, UHCI_STS),
 732             UREAD2(sc, UHCI_INTR),
 733             UREAD2(sc, UHCI_FRNUM),
 734             UREAD4(sc, UHCI_FLBASEADDR),
 735             UREAD1(sc, UHCI_SOF),
 736             UREAD2(sc, UHCI_PORTSC1),
 737             UREAD2(sc, UHCI_PORTSC2));
 738 }
 739
 740 static uint8_t
 741 uhci_dump_td(uhci_td_t *p)
 742 {
 743         uint32_t td_next;
 744         uint32_t td_status;
 745         uint32_t td_token;
 746         uint8_t temp;
 747
 748         usb_pc_cpu_invalidate(p->page_cache);
 749
 750         td_next = le32toh(p->td_next);
 751         td_status = le32toh(p->td_status);
 752         td_token = le32toh(p->td_token);
 753
 754         /*
 755          * Check whether the link pointer in this TD marks the link pointer
 756          * as end of queue:
 757          */
 758         temp = ((td_next & UHCI_PTR_T) || (td_next == 0));
 759
 760         printf("TD(%p) at 0x%08x = link=0x%08x status=0x%08x "
 761             "token=0x%08x buffer=0x%08x\n",
 762             p,
 763             le32toh(p->td_self),
 764             td_next,
 765             td_status,
 766             td_token,
 767             le32toh(p->td_buffer));
 768
 769         printf("TD(%p) td_next=%s%s%s td_status=%s%s%s%s%s%s%s%s%s%s%s, errcnt=%d, actlen=%d pid=%02x,"
 770             "addr=%d,endpt=%d,D=%d,maxlen=%d\n",
 771             p,
 772             (td_next & 1) ? "-T" : "",
 773             (td_next & 2) ? "-Q" : "",
 774             (td_next & 4) ? "-VF" : "",
 775             (td_status & UHCI_TD_BITSTUFF) ? "-BITSTUFF" : "",
 776             (td_status & UHCI_TD_CRCTO) ? "-CRCTO" : "",
 777             (td_status & UHCI_TD_NAK) ? "-NAK" : "",
 778             (td_status & UHCI_TD_BABBLE) ? "-BABBLE" : "",
 779             (td_status & UHCI_TD_DBUFFER) ? "-DBUFFER" : "",
 780             (td_status & UHCI_TD_STALLED) ? "-STALLED" : "",
 781             (td_status & UHCI_TD_ACTIVE) ? "-ACTIVE" : "",
 782             (td_status & UHCI_TD_IOC) ? "-IOC" : "",
 783             (td_status & UHCI_TD_IOS) ? "-IOS" : "",
 784             (td_status & UHCI_TD_LS) ? "-LS" : "",
 785             (td_status & UHCI_TD_SPD) ? "-SPD" : "",
 786             UHCI_TD_GET_ERRCNT(td_status),
 787             UHCI_TD_GET_ACTLEN(td_status),
 788             UHCI_TD_GET_PID(td_token),
 789             UHCI_TD_GET_DEVADDR(td_token),
 790             UHCI_TD_GET_ENDPT(td_token),
 791             UHCI_TD_GET_DT(td_token),
 792             UHCI_TD_GET_MAXLEN(td_token));
 793
 794         return (temp);
 795 }
 796
 797 static uint8_t
 798 uhci_dump_qh(uhci_qh_t *sqh)
 799 {
 800         uint8_t temp;
 801         uint32_t qh_h_next;
 802         uint32_t qh_e_next;
 803
 804         usb_pc_cpu_invalidate(sqh->page_cache);
 805
 806         qh_h_next = le32toh(sqh->qh_h_next);
 807         qh_e_next = le32toh(sqh->qh_e_next);
 808
 809         DPRINTFN(0, "QH(%p) at 0x%08x: h_next=0x%08x e_next=0x%08x\n", sqh,
 810             le32toh(sqh->qh_self), qh_h_next, qh_e_next);
 811
 812         temp = ((((sqh->h_next != NULL) && !(qh_h_next & UHCI_PTR_T)) ? 1 : 0) |
 813             (((sqh->e_next != NULL) && !(qh_e_next & UHCI_PTR_T)) ? 2 : 0));
 814
 815         return (temp);
 816 }
 817
 818 static void
 819 uhci_dump_all(uhci_softc_t *sc)
 820 {
 821         uhci_dumpregs(sc);
 822         uhci_dump_qh(sc->sc_ls_ctl_p_last);
 823         uhci_dump_qh(sc->sc_fs_ctl_p_last);
 824         uhci_dump_qh(sc->sc_bulk_p_last);
 825         uhci_dump_qh(sc->sc_last_qh_p);
 826 }
 827
 828 static void
 829 uhci_dump_qhs(uhci_qh_t *sqh)
 830 {
 831         uint8_t temp;
 832
 833         temp = uhci_dump_qh(sqh);
 834
 835         /*
 836          * uhci_dump_qhs displays all the QHs and TDs from the given QH
 837          * onwards Traverses sideways first, then down.
 838          *
 839          * QH1 QH2 No QH TD2.1 TD2.2 TD1.1 etc.
 840          *
 841          * TD2.x being the TDs queued at QH2 and QH1 being referenced from QH1.
 842          */
 843
 844         if (temp & 1)
 845                 uhci_dump_qhs(sqh->h_next);
 846         else
 847                 DPRINTF("No QH\n");
 848
 849         if (temp & 2)
 850                 uhci_dump_tds(sqh->e_next);
 851         else
 852                 DPRINTF("No TD\n");
 853 }
 854
 855 static void
 856 uhci_dump_tds(uhci_td_t *td)
 857 {
 858         for (;
 859             td != NULL;
 860             td = td->obj_next) {
 861                 if (uhci_dump_td(td)) {
 862                         break;
 863                 }
 864         }
 865 }
 866
 867 #endif
 868
 869 /*
 870  * Let the last QH loop back to the full speed control transfer QH.
 871  * This is what intel calls "bandwidth reclamation" and improves
 872  * USB performance a lot for some devices.
 873  * If we are already looping, just count it.
 874  */
 875 static void
 876 uhci_add_loop(uhci_softc_t *sc)
 877 {
 878         struct uhci_qh *qh_lst;
 879         struct uhci_qh *qh_rec;
 880
 881 #if USB_DEBUG
 882         if (uhcinoloop) {
 883                 return;
 884         }
 885 #endif
 886         if (++(sc->sc_loops) == 1) {
 887                 DPRINTFN(6, "add\n");
 888
 889                 qh_lst = sc->sc_last_qh_p;
 890                 qh_rec = sc->sc_reclaim_qh_p;
 891
 892                 /* NOTE: we don't loop back the soft pointer */
 893
 894                 qh_lst->qh_h_next = qh_rec->qh_self;
 895                 usb_pc_cpu_flush(qh_lst->page_cache);
 896         }
 897 }
 898
 899 static void
 900 uhci_rem_loop(uhci_softc_t *sc)
 901 {
 902         struct uhci_qh *qh_lst;
 903
 904 #if USB_DEBUG
 905         if (uhcinoloop) {
 906                 return;
 907         }
 908 #endif
 909         if (--(sc->sc_loops) == 0) {
 910                 DPRINTFN(6, "remove\n");
 911
 912                 qh_lst = sc->sc_last_qh_p;
 913                 qh_lst->qh_h_next = htole32(UHCI_PTR_T);
 914                 usb_pc_cpu_flush(qh_lst->page_cache);
 915         }
 916 }
 917
 918 static void
 919 uhci_transfer_intr_enqueue(struct usb_xfer *xfer)
 920 {
 921         /* check for early completion */
 922         if (uhci_check_transfer(xfer)) {
 923                 return;
 924         }
 925         /* put transfer on interrupt queue */
 926         usbd_transfer_enqueue(&xfer->xroot->bus->intr_q, xfer);
 927
 928         /* start timeout, if any */
 929         if (xfer->timeout != 0) {
 930                 usbd_transfer_timeout_ms(xfer, &uhci_timeout, xfer->timeout);
 931         }
 932 }
 933
 934 #define UHCI_APPEND_TD(std,last) (last) = _uhci_append_td(std,last)
 935 static uhci_td_t *
 936 _uhci_append_td(uhci_td_t *std, uhci_td_t *last)
 937 {
 938         DPRINTFN(11, "%p to %p\n", std, last);
 939
 940         /* (sc->sc_bus.mtx) must be locked */
 941
 942         std->next = last->next;
 943         std->td_next = last->td_next;
 944
 945         std->prev = last;
 946
 947         usb_pc_cpu_flush(std->page_cache);
 948
 949         /*
 950          * the last->next->prev is never followed: std->next->prev = std;
 951          */
 952         last->next = std;
 953         last->td_next = std->td_self;
 954
 955         usb_pc_cpu_flush(last->page_cache);
 956
 957         return (std);
 958 }
 959
 960 #define UHCI_APPEND_QH(sqh,last) (last) = _uhci_append_qh(sqh,last)
 961 static uhci_qh_t *
 962 _uhci_append_qh(uhci_qh_t *sqh, uhci_qh_t *last)
 963 {
 964         DPRINTFN(11, "%p to %p\n", sqh, last);
 965
 966         if (sqh->h_prev != NULL) {
 967                 /* should not happen */
 968                 DPRINTFN(0, "QH already linked!\n");
 969                 return (last);
 970         }
 971         /* (sc->sc_bus.mtx) must be locked */
 972
 973         sqh->h_next = last->h_next;
 974         sqh->qh_h_next = last->qh_h_next;
 975
 976         sqh->h_prev = last;
 977
 978         usb_pc_cpu_flush(sqh->page_cache);
 979
 980         /*
 981          * The "last->h_next->h_prev" is never followed:
 982          *
 983          * "sqh->h_next->h_prev" = sqh;
 984          */
 985
 986         last->h_next = sqh;
 987         last->qh_h_next = sqh->qh_self;
 988
 989         usb_pc_cpu_flush(last->page_cache);
 990
 991         return (sqh);
 992 }
 993
 994 /**/
 995
 996 #define UHCI_REMOVE_TD(std,last) (last) = _uhci_remove_td(std,last)
 997 static uhci_td_t *
 998 _uhci_remove_td(uhci_td_t *std, uhci_td_t *last)
 999 {
1000         DPRINTFN(11, "%p from %p\n", std, last);
1001
1002         /* (sc->sc_bus.mtx) must be locked */
1003
1004         std->prev->next = std->next;
1005         std->prev->td_next = std->td_next;
1006
1007         usb_pc_cpu_flush(std->prev->page_cache);
1008
1009         if (std->next) {
1010                 std->next->prev = std->prev;
1011                 usb_pc_cpu_flush(std->next->page_cache);
1012         }
1013         return ((last == std) ? std->prev : last);
1014 }
1015
1016 #define UHCI_REMOVE_QH(sqh,last) (last) = _uhci_remove_qh(sqh,last)
1017 static uhci_qh_t *
1018 _uhci_remove_qh(uhci_qh_t *sqh, uhci_qh_t *last)
1019 {
1020         DPRINTFN(11, "%p from %p\n", sqh, last);
1021
1022         /* (sc->sc_bus.mtx) must be locked */
1023
1024         /* only remove if not removed from a queue */
1025         if (sqh->h_prev) {
1026
1027                 sqh->h_prev->h_next = sqh->h_next;
1028                 sqh->h_prev->qh_h_next = sqh->qh_h_next;
1029
1030                 usb_pc_cpu_flush(sqh->h_prev->page_cache);
1031
1032                 if (sqh->h_next) {
1033                         sqh->h_next->h_prev = sqh->h_prev;
1034                         usb_pc_cpu_flush(sqh->h_next->page_cache);
1035                 }
1036                 last = ((last == sqh) ? sqh->h_prev : last);
1037
1038                 sqh->h_prev = 0;
1039
1040                 usb_pc_cpu_flush(sqh->page_cache);
1041         }
1042         return (last);
1043 }
1044
1045 static void
1046 uhci_isoc_done(uhci_softc_t *sc, struct usb_xfer *xfer)
1047 {
1048         struct usb_page_search res;
1049         uint32_t nframes = xfer->nframes;
1050         uint32_t status;
1051         uint32_t offset = 0;
1052         uint32_t *plen = xfer->frlengths;
1053         uint16_t len = 0;
1054         uhci_td_t *td = xfer->td_transfer_first;
1055         uhci_td_t **pp_last = &sc->sc_isoc_p_last[xfer->qh_pos];
1056
1057         DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
1058             xfer, xfer->endpoint);
1059
1060         /* sync any DMA memory before doing fixups */
1061
1062         usb_bdma_post_sync(xfer);
1063
1064         while (nframes--) {
1065                 if (td == NULL) {
1066                         panic("%s:%d: out of TD's\n",
1067                             __FUNCTION__, __LINE__);
1068                 }
1069                 if (pp_last >= &sc->sc_isoc_p_last[UHCI_VFRAMELIST_COUNT]) {
1070                         pp_last = &sc->sc_isoc_p_last[0];
1071                 }
1072 #if USB_DEBUG
1073                 if (uhcidebug > 5) {
1074                         DPRINTF("isoc TD\n");
1075                         uhci_dump_td(td);
1076                 }
1077 #endif
1078                 usb_pc_cpu_invalidate(td->page_cache);
1079                 status = le32toh(td->td_status);
1080
1081                 len = UHCI_TD_GET_ACTLEN(status);
1082
1083                 if (len > *plen) {
1084                         len = *plen;
1085                 }
1086                 if (td->fix_pc) {
1087
1088                         usbd_get_page(td->fix_pc, 0, &res);
1089
1090                         /* copy data from fixup location to real location */
1091
1092                         usb_pc_cpu_invalidate(td->fix_pc);
1093
1094                         usbd_copy_in(xfer->frbuffers, offset,
1095                             res.buffer, len);
1096                 }
1097                 offset += *plen;
1098
1099                 *plen = len;
1100
1101                 /* remove TD from schedule */
1102                 UHCI_REMOVE_TD(td, *pp_last);
1103
1104                 pp_last++;
1105                 plen++;
1106                 td = td->obj_next;
1107         }
1108
1109         xfer->aframes = xfer->nframes;
1110 }
1111
1112 static usb_error_t
1113 uhci_non_isoc_done_sub(struct usb_xfer *xfer)
1114 {
1115         struct usb_page_search res;
1116         uhci_td_t *td;
1117         uhci_td_t *td_alt_next;
1118         uint32_t status;
1119         uint32_t token;
1120         uint16_t len;
1121
1122         td = xfer->td_transfer_cache;
1123         td_alt_next = td->alt_next;
1124
1125         if (xfer->aframes != xfer->nframes) {
1126                 usbd_xfer_set_frame_len(xfer, xfer->aframes, 0);
1127         }
1128         while (1) {
1129
1130                 usb_pc_cpu_invalidate(td->page_cache);
1131                 status = le32toh(td->td_status);
1132                 token = le32toh(td->td_token);
1133
1134                 /*
1135                  * Verify the status and add
1136                  * up the actual length:
1137                  */
1138
1139                 len = UHCI_TD_GET_ACTLEN(status);
1140                 if (len > td->len) {
1141                         /* should not happen */
1142                         DPRINTF("Invalid status length, "
1143                             "0x%04x/0x%04x bytes\n", len, td->len);
1144                         status |= UHCI_TD_STALLED;
1145
1146                 } else if ((xfer->aframes != xfer->nframes) && (len > 0)) {
1147
1148                         if (td->fix_pc) {
1149
1150                                 usbd_get_page(td->fix_pc, 0, &res);
1151
1152                                 /*
1153                                  * copy data from fixup location to real
1154                                  * location
1155                                  */
1156
1157                                 usb_pc_cpu_invalidate(td->fix_pc);
1158
1159                                 usbd_copy_in(xfer->frbuffers + xfer->aframes,
1160                                     xfer->frlengths[xfer->aframes], res.buffer, len);
1161                         }
1162                         /* update actual length */
1163
1164                         xfer->frlengths[xfer->aframes] += len;
1165                 }
1166                 /* Check for last transfer */
1167                 if (((void *)td) == xfer->td_transfer_last) {
1168                         td = NULL;
1169                         break;
1170                 }
1171                 if (status & UHCI_TD_STALLED) {
1172                         /* the transfer is finished */
1173                         td = NULL;
1174                         break;
1175                 }
1176                 /* Check for short transfer */
1177                 if (len != td->len) {
1178                         if (xfer->flags_int.short_frames_ok) {
1179                                 /* follow alt next */
1180                                 td = td->alt_next;
1181                         } else {
1182                                 /* the transfer is finished */
1183                                 td = NULL;
1184                         }
1185                         break;
1186                 }
1187                 td = td->obj_next;
1188
1189                 if (td->alt_next != td_alt_next) {
1190                         /* this USB frame is complete */
1191                         break;
1192                 }
1193         }
1194
1195         /* update transfer cache */
1196
1197         xfer->td_transfer_cache = td;
1198
1199         /* update data toggle */
1200
1201         xfer->endpoint->toggle_next = (token & UHCI_TD_SET_DT(1)) ? 0 : 1;
1202
1203 #if USB_DEBUG
1204         if (status & UHCI_TD_ERROR) {
1205                 DPRINTFN(11, "error, addr=%d, endpt=0x%02x, frame=0x%02x "
1206                     "status=%s%s%s%s%s%s%s%s%s%s%s\n",
1207                     xfer->address, xfer->endpointno, xfer->aframes,
1208                     (status & UHCI_TD_BITSTUFF) ? "[BITSTUFF]" : "",
1209                     (status & UHCI_TD_CRCTO) ? "[CRCTO]" : "",
1210                     (status & UHCI_TD_NAK) ? "[NAK]" : "",
1211                     (status & UHCI_TD_BABBLE) ? "[BABBLE]" : "",
1212                     (status & UHCI_TD_DBUFFER) ? "[DBUFFER]" : "",
1213                     (status & UHCI_TD_STALLED) ? "[STALLED]" : "",
1214                     (status & UHCI_TD_ACTIVE) ? "[ACTIVE]" : "[NOT_ACTIVE]",
1215                     (status & UHCI_TD_IOC) ? "[IOC]" : "",
1216                     (status & UHCI_TD_IOS) ? "[IOS]" : "",
1217                     (status & UHCI_TD_LS) ? "[LS]" : "",
1218                     (status & UHCI_TD_SPD) ? "[SPD]" : "");
1219         }
1220 #endif
1221         return (status & UHCI_TD_STALLED) ?
1222             USB_ERR_STALLED : USB_ERR_NORMAL_COMPLETION;
1223 }
1224
1225 static void
1226 uhci_non_isoc_done(struct usb_xfer *xfer)
1227 {
1228         usb_error_t err = 0;
1229
1230         DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
1231             xfer, xfer->endpoint);
1232
1233 #if USB_DEBUG
1234         if (uhcidebug > 10) {
1235                 uhci_dump_tds(xfer->td_transfer_first);
1236         }
1237 #endif
1238
1239         /* sync any DMA memory before doing fixups */
1240
1241         usb_bdma_post_sync(xfer);
1242
1243         /* reset scanner */
1244
1245         xfer->td_transfer_cache = xfer->td_transfer_first;
1246
1247         if (xfer->flags_int.control_xfr) {
1248                 if (xfer->flags_int.control_hdr) {
1249
1250                         err = uhci_non_isoc_done_sub(xfer);
1251                 }
1252                 xfer->aframes = 1;
1253
1254                 if (xfer->td_transfer_cache == NULL) {
1255                         goto done;
1256                 }
1257         }
1258         while (xfer->aframes != xfer->nframes) {
1259
1260                 err = uhci_non_isoc_done_sub(xfer);
1261                 xfer->aframes++;
1262
1263                 if (xfer->td_transfer_cache == NULL) {
1264                         goto done;
1265                 }
1266         }
1267
1268         if (xfer->flags_int.control_xfr &&
1269             !xfer->flags_int.control_act) {
1270
1271                 err = uhci_non_isoc_done_sub(xfer);
1272         }
1273 done:
1274         uhci_device_done(xfer, err);
1275 }
1276
1277 /*------------------------------------------------------------------------*
1278  *      uhci_check_transfer_sub
1279  *
1280  * The main purpose of this function is to update the data-toggle
1281  * in case it is wrong.
1282  *------------------------------------------------------------------------*/
1283 static void
1284 uhci_check_transfer_sub(struct usb_xfer *xfer)
1285 {
1286         uhci_qh_t *qh;
1287         uhci_td_t *td;
1288         uhci_td_t *td_alt_next;
1289
1290         uint32_t td_token;
1291         uint32_t td_self;
1292
1293         td = xfer->td_transfer_cache;
1294         qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
1295
1296         td_token = td->obj_next->td_token;
1297         td = td->alt_next;
1298         xfer->td_transfer_cache = td;
1299         td_self = td->td_self;
1300         td_alt_next = td->alt_next;
1301
1302         if (xfer->flags_int.control_xfr)
1303                 goto skip;      /* don't touch the DT value! */
1304
1305         if (!((td->td_token ^ td_token) & htole32(UHCI_TD_SET_DT(1))))
1306                 goto skip;      /* data toggle has correct value */
1307
1308         /*
1309          * The data toggle is wrong and we need to toggle it !
1310          */
1311         while (1) {
1312
1313                 td->td_token ^= htole32(UHCI_TD_SET_DT(1));
1314                 usb_pc_cpu_flush(td->page_cache);
1315
1316                 if (td == xfer->td_transfer_last) {
1317                         /* last transfer */
1318                         break;
1319                 }
1320                 td = td->obj_next;
1321
1322                 if (td->alt_next != td_alt_next) {
1323                         /* next frame */
1324                         break;
1325                 }
1326         }
1327 skip:
1328
1329         /* update the QH */
1330         qh->qh_e_next = td_self;
1331         usb_pc_cpu_flush(qh->page_cache);
1332
1333         DPRINTFN(13, "xfer=%p following alt next\n", xfer);
1334 }
1335
1336 /*------------------------------------------------------------------------*
1337  *      uhci_check_transfer
1338  *
1339  * Return values:
1340  *    0: USB transfer is not finished
1341  * Else: USB transfer is finished
1342  *------------------------------------------------------------------------*/
1343 static uint8_t
1344 uhci_check_transfer(struct usb_xfer *xfer)
1345 {
1346         uint32_t status;
1347         uint32_t token;
1348         uhci_td_t *td;
1349
1350         DPRINTFN(16, "xfer=%p checking transfer\n", xfer);
1351
1352         if (xfer->endpoint->methods == &uhci_device_isoc_methods) {
1353                 /* isochronous transfer */
1354
1355                 td = xfer->td_transfer_last;
1356
1357                 usb_pc_cpu_invalidate(td->page_cache);
1358                 status = le32toh(td->td_status);
1359
1360                 /* check also if the first is complete */
1361
1362                 td = xfer->td_transfer_first;
1363
1364                 usb_pc_cpu_invalidate(td->page_cache);
1365                 status |= le32toh(td->td_status);
1366
1367                 if (!(status & UHCI_TD_ACTIVE)) {
1368                         uhci_device_done(xfer, USB_ERR_NORMAL_COMPLETION);
1369                         goto transferred;
1370                 }
1371         } else {
1372                 /* non-isochronous transfer */
1373
1374                 /*
1375                  * check whether there is an error somewhere
1376                  * in the middle, or whether there was a short
1377                  * packet (SPD and not ACTIVE)
1378                  */
1379                 td = xfer->td_transfer_cache;
1380
1381                 while (1) {
1382                         usb_pc_cpu_invalidate(td->page_cache);
1383                         status = le32toh(td->td_status);
1384                         token = le32toh(td->td_token);
1385
1386                         /*
1387                          * if there is an active TD the transfer isn't done
1388                          */
1389                         if (status & UHCI_TD_ACTIVE) {
1390                                 /* update cache */
1391                                 xfer->td_transfer_cache = td;
1392                                 goto done;
1393                         }
1394                         /*
1395                          * last transfer descriptor makes the transfer done
1396                          */
1397                         if (((void *)td) == xfer->td_transfer_last) {
1398                                 break;
1399                         }
1400                         /*
1401                          * any kind of error makes the transfer done
1402                          */
1403                         if (status & UHCI_TD_STALLED) {
1404                                 break;
1405                         }
1406                         /*
1407                          * check if we reached the last packet
1408                          * or if there is a short packet:
1409                          */
1410                         if ((td->td_next == htole32(UHCI_PTR_T)) ||
1411                             (UHCI_TD_GET_ACTLEN(status) < td->len)) {
1412
1413                                 if (xfer->flags_int.short_frames_ok) {
1414                                         /* follow alt next */
1415                                         if (td->alt_next) {
1416                                                 /* update cache */
1417                                                 xfer->td_transfer_cache = td;
1418                                                 uhci_check_transfer_sub(xfer);
1419                                                 goto done;
1420                                         }
1421                                 }
1422                                 /* transfer is done */
1423                                 break;
1424                         }
1425                         td = td->obj_next;
1426                 }
1427                 uhci_non_isoc_done(xfer);
1428                 goto transferred;
1429         }
1430
1431 done:
1432         DPRINTFN(13, "xfer=%p is still active\n", xfer);
1433         return (0);
1434
1435 transferred:
1436         return (1);
1437 }
1438
1439 static void
1440 uhci_interrupt_poll(uhci_softc_t *sc)
1441 {
1442         struct usb_xfer *xfer;
1443
1444 repeat:
1445         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
1446                 /*
1447                  * check if transfer is transferred
1448                  */
1449                 if (uhci_check_transfer(xfer)) {
1450                         /* queue has been modified */
1451                         goto repeat;
1452                 }
1453         }
1454 }
1455
1456 /*------------------------------------------------------------------------*
1457  *      uhci_interrupt - UHCI interrupt handler
1458  *
1459  * NOTE: Do not access "sc->sc_bus.bdev" inside the interrupt handler,
1460  * hence the interrupt handler will be setup before "sc->sc_bus.bdev"
1461  * is present !
1462  *------------------------------------------------------------------------*/
1463 void
1464 uhci_interrupt(uhci_softc_t *sc)
1465 {
1466         uint32_t status;
1467
1468         USB_BUS_LOCK(&sc->sc_bus);
1469
1470         DPRINTFN(16, "real interrupt\n");
1471
1472 #if USB_DEBUG
1473         if (uhcidebug > 15) {
1474                 uhci_dumpregs(sc);
1475         }
1476 #endif
1477         status = UREAD2(sc, UHCI_STS) & UHCI_STS_ALLINTRS;
1478         if (status == 0) {
1479                 /* the interrupt was not for us */
1480                 goto done;
1481         }
1482         if (status & (UHCI_STS_RD | UHCI_STS_HSE |
1483             UHCI_STS_HCPE | UHCI_STS_HCH)) {
1484
1485                 if (status & UHCI_STS_RD) {
1486 #if USB_DEBUG
1487                         printf("%s: resume detect\n",
1488                             __FUNCTION__);
1489 #endif
1490                 }
1491                 if (status & UHCI_STS_HSE) {
1492                         printf("%s: host system error\n",
1493                             __FUNCTION__);
1494                 }
1495                 if (status & UHCI_STS_HCPE) {
1496                         printf("%s: host controller process error\n",
1497                             __FUNCTION__);
1498                 }
1499                 if (status & UHCI_STS_HCH) {
1500                         /* no acknowledge needed */
1501                         DPRINTF("%s: host controller halted\n",
1502                             __FUNCTION__);
1503 #if USB_DEBUG
1504                         if (uhcidebug > 0) {
1505                                 uhci_dump_all(sc);
1506                         }
1507 #endif
1508                 }
1509         }
1510         /* get acknowledge bits */
1511         status &= (UHCI_STS_USBINT |
1512             UHCI_STS_USBEI |
1513             UHCI_STS_RD |
1514             UHCI_STS_HSE |
1515             UHCI_STS_HCPE);
1516
1517         if (status == 0) {
1518                 /* nothing to acknowledge */
1519                 goto done;
1520         }
1521         /* acknowledge interrupts */
1522         UWRITE2(sc, UHCI_STS, status);
1523
1524         /* poll all the USB transfers */
1525         uhci_interrupt_poll(sc);
1526
1527 done:
1528         USB_BUS_UNLOCK(&sc->sc_bus);
1529 }
1530
1531 /*
1532  * called when a request does not complete
1533  */
1534 static void
1535 uhci_timeout(void *arg)
1536 {
1537         struct usb_xfer *xfer = arg;
1538
1539         DPRINTF("xfer=%p\n", xfer);
1540
1541         USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
1542
1543         /* transfer is transferred */
1544         uhci_device_done(xfer, USB_ERR_TIMEOUT);
1545 }
1546
1547 static void
1548 uhci_do_poll(struct usb_bus *bus)
1549 {
1550         struct uhci_softc *sc = UHCI_BUS2SC(bus);
1551
1552         USB_BUS_LOCK(&sc->sc_bus);
1553         uhci_interrupt_poll(sc);
1554         USB_BUS_UNLOCK(&sc->sc_bus);
1555 }
1556
1557 static void
1558 uhci_setup_standard_chain_sub(struct uhci_std_temp *temp)
1559 {
1560         uhci_td_t *td;
1561         uhci_td_t *td_next;
1562         uhci_td_t *td_alt_next;
1563         uint32_t average;
1564         uint32_t len_old;
1565         uint8_t shortpkt_old;
1566         uint8_t precompute;
1567
1568         td_alt_next = NULL;
1569         shortpkt_old = temp->shortpkt;
1570         len_old = temp->len;
1571         precompute = 1;
1572
1573         /* software is used to detect short incoming transfers */
1574
1575         if ((temp->td_token & htole32(UHCI_TD_PID)) == htole32(UHCI_TD_PID_IN)) {
1576                 temp->td_status |= htole32(UHCI_TD_SPD);
1577         } else {
1578                 temp->td_status &= ~htole32(UHCI_TD_SPD);
1579         }
1580
1581         temp->ml.buf_offset = 0;
1582
1583 restart:
1584
1585         temp->td_token &= ~htole32(UHCI_TD_SET_MAXLEN(0));
1586         temp->td_token |= htole32(UHCI_TD_SET_MAXLEN(temp->average));
1587
1588         td = temp->td;
1589         td_next = temp->td_next;
1590
1591         while (1) {
1592
1593                 if (temp->len == 0) {
1594
1595                         if (temp->shortpkt) {
1596                                 break;
1597                         }
1598                         /* send a Zero Length Packet, ZLP, last */
1599
1600                         temp->shortpkt = 1;
1601                         temp->td_token |= htole32(UHCI_TD_SET_MAXLEN(0));
1602                         average = 0;
1603
1604                 } else {
1605
1606                         average = temp->average;
1607
1608                         if (temp->len < average) {
1609                                 temp->shortpkt = 1;
1610                                 temp->td_token &= ~htole32(UHCI_TD_SET_MAXLEN(0));
1611                                 temp->td_token |= htole32(UHCI_TD_SET_MAXLEN(temp->len));
1612                                 average = temp->len;
1613                         }
1614                 }
1615
1616                 if (td_next == NULL) {
1617                         panic("%s: out of UHCI transfer descriptors!", __FUNCTION__);
1618                 }
1619                 /* get next TD */
1620
1621                 td = td_next;
1622                 td_next = td->obj_next;
1623
1624                 /* check if we are pre-computing */
1625
1626                 if (precompute) {
1627
1628                         /* update remaining length */
1629
1630                         temp->len -= average;
1631
1632                         continue;
1633                 }
1634                 /* fill out current TD */
1635
1636                 td->td_status = temp->td_status;
1637                 td->td_token = temp->td_token;
1638
1639                 /* update data toggle */
1640
1641                 temp->td_token ^= htole32(UHCI_TD_SET_DT(1));
1642
1643                 if (average == 0) {
1644
1645                         td->len = 0;
1646                         td->td_buffer = 0;
1647                         td->fix_pc = NULL;
1648
1649                 } else {
1650
1651                         /* update remaining length */
1652
1653                         temp->len -= average;
1654
1655                         td->len = average;
1656
1657                         /* fill out buffer pointer and do fixup, if any */
1658
1659                         uhci_mem_layout_fixup(&temp->ml, td);
1660                 }
1661
1662                 td->alt_next = td_alt_next;
1663
1664                 if ((td_next == td_alt_next) && temp->setup_alt_next) {
1665                         /* we need to receive these frames one by one ! */
1666                         td->td_status |= htole32(UHCI_TD_IOC);
1667                         td->td_next = htole32(UHCI_PTR_T);
1668                 } else {
1669                         if (td_next) {
1670                                 /* link the current TD with the next one */
1671                                 td->td_next = td_next->td_self;
1672                         }
1673                 }
1674
1675                 usb_pc_cpu_flush(td->page_cache);
1676         }
1677
1678         if (precompute) {
1679                 precompute = 0;
1680
1681                 /* setup alt next pointer, if any */
1682                 if (temp->last_frame) {
1683                         td_alt_next = NULL;
1684                 } else {
1685                         /* we use this field internally */
1686                         td_alt_next = td_next;
1687                 }
1688
1689                 /* restore */
1690                 temp->shortpkt = shortpkt_old;
1691                 temp->len = len_old;
1692                 goto restart;
1693         }
1694         temp->td = td;
1695         temp->td_next = td_next;
1696 }
1697
1698 static uhci_td_t *
1699 uhci_setup_standard_chain(struct usb_xfer *xfer)
1700 {
1701         struct uhci_std_temp temp;
1702         uhci_td_t *td;
1703         uint32_t x;
1704
1705         DPRINTFN(9, "addr=%d endpt=%d sumlen=%d speed=%d\n",
1706             xfer->address, UE_GET_ADDR(xfer->endpointno),
1707             xfer->sumlen, usbd_get_speed(xfer->xroot->udev));
1708
1709         temp.average = xfer->max_frame_size;
1710         temp.max_frame_size = xfer->max_frame_size;
1711
1712         /* toggle the DMA set we are using */
1713         xfer->flags_int.curr_dma_set ^= 1;
1714
1715         /* get next DMA set */
1716         td = xfer->td_start[xfer->flags_int.curr_dma_set];
1717         xfer->td_transfer_first = td;
1718         xfer->td_transfer_cache = td;
1719
1720         temp.td = NULL;
1721         temp.td_next = td;
1722         temp.last_frame = 0;
1723         temp.setup_alt_next = xfer->flags_int.short_frames_ok;
1724
1725         uhci_mem_layout_init(&temp.ml, xfer);
1726
1727         temp.td_status =
1728             htole32(UHCI_TD_ZERO_ACTLEN(UHCI_TD_SET_ERRCNT(3) |
1729             UHCI_TD_ACTIVE));
1730
1731         if (xfer->xroot->udev->speed == USB_SPEED_LOW) {
1732                 temp.td_status |= htole32(UHCI_TD_LS);
1733         }
1734         temp.td_token =
1735             htole32(UHCI_TD_SET_ENDPT(xfer->endpointno) |
1736             UHCI_TD_SET_DEVADDR(xfer->address));
1737
1738         if (xfer->endpoint->toggle_next) {
1739                 /* DATA1 is next */
1740                 temp.td_token |= htole32(UHCI_TD_SET_DT(1));
1741         }
1742         /* check if we should prepend a setup message */
1743
1744         if (xfer->flags_int.control_xfr) {
1745
1746                 if (xfer->flags_int.control_hdr) {
1747
1748                         temp.td_token &= htole32(UHCI_TD_SET_DEVADDR(0x7F) |
1749                             UHCI_TD_SET_ENDPT(0xF));
1750                         temp.td_token |= htole32(UHCI_TD_PID_SETUP |
1751                             UHCI_TD_SET_DT(0));
1752
1753                         temp.len = xfer->frlengths[0];
1754                         temp.ml.buf_pc = xfer->frbuffers + 0;
1755                         temp.shortpkt = temp.len ? 1 : 0;
1756                         /* check for last frame */
1757                         if (xfer->nframes == 1) {
1758                                 /* no STATUS stage yet, SETUP is last */
1759                                 if (xfer->flags_int.control_act) {
1760                                         temp.last_frame = 1;
1761                                         temp.setup_alt_next = 0;
1762                                 }
1763                         }
1764                         uhci_setup_standard_chain_sub(&temp);
1765                 }
1766                 x = 1;
1767         } else {
1768                 x = 0;
1769         }
1770
1771         while (x != xfer->nframes) {
1772
1773                 /* DATA0 / DATA1 message */
1774
1775                 temp.len = xfer->frlengths[x];
1776                 temp.ml.buf_pc = xfer->frbuffers + x;
1777
1778                 x++;
1779
1780                 if (x == xfer->nframes) {
1781                         if (xfer->flags_int.control_xfr) {
1782                                 /* no STATUS stage yet, DATA is last */
1783                                 if (xfer->flags_int.control_act) {
1784                                         temp.last_frame = 1;
1785                                         temp.setup_alt_next = 0;
1786                                 }
1787                         } else {
1788                                 temp.last_frame = 1;
1789                                 temp.setup_alt_next = 0;
1790                         }
1791                 }
1792                 /*
1793                  * Keep previous data toggle,
1794                  * device address and endpoint number:
1795                  */
1796
1797                 temp.td_token &= htole32(UHCI_TD_SET_DEVADDR(0x7F) |
1798                     UHCI_TD_SET_ENDPT(0xF) |
1799                     UHCI_TD_SET_DT(1));
1800
1801                 if (temp.len == 0) {
1802
1803                         /* make sure that we send an USB packet */
1804
1805                         temp.shortpkt = 0;
1806
1807                 } else {
1808
1809                         /* regular data transfer */
1810
1811                         temp.shortpkt = (xfer->flags.force_short_xfer) ? 0 : 1;
1812                 }
1813
1814                 /* set endpoint direction */
1815
1816                 temp.td_token |=
1817                     (UE_GET_DIR(xfer->endpointno) == UE_DIR_IN) ?
1818                     htole32(UHCI_TD_PID_IN) :
1819                     htole32(UHCI_TD_PID_OUT);
1820
1821                 uhci_setup_standard_chain_sub(&temp);
1822         }
1823
1824         /* check if we should append a status stage */
1825
1826         if (xfer->flags_int.control_xfr &&
1827             !xfer->flags_int.control_act) {
1828
1829                 /*
1830                  * send a DATA1 message and reverse the current endpoint
1831                  * direction
1832                  */
1833
1834                 temp.td_token &= htole32(UHCI_TD_SET_DEVADDR(0x7F) |
1835                     UHCI_TD_SET_ENDPT(0xF) |
1836                     UHCI_TD_SET_DT(1));
1837                 temp.td_token |=
1838                     (UE_GET_DIR(xfer->endpointno) == UE_DIR_OUT) ?
1839                     htole32(UHCI_TD_PID_IN | UHCI_TD_SET_DT(1)) :
1840                     htole32(UHCI_TD_PID_OUT | UHCI_TD_SET_DT(1));
1841
1842                 temp.len = 0;
1843                 temp.ml.buf_pc = NULL;
1844                 temp.shortpkt = 0;
1845                 temp.last_frame = 1;
1846                 temp.setup_alt_next = 0;
1847
1848                 uhci_setup_standard_chain_sub(&temp);
1849         }
1850         td = temp.td;
1851
1852         /* Ensure that last TD is terminating: */
1853         td->td_next = htole32(UHCI_PTR_T);
1854
1855         /* set interrupt bit */
1856
1857         td->td_status |= htole32(UHCI_TD_IOC);
1858
1859         usb_pc_cpu_flush(td->page_cache);
1860
1861         /* must have at least one frame! */
1862
1863         xfer->td_transfer_last = td;
1864
1865 #if USB_DEBUG
1866         if (uhcidebug > 8) {
1867                 DPRINTF("nexttog=%d; data before transfer:\n",
1868                     xfer->endpoint->toggle_next);
1869                 uhci_dump_tds(xfer->td_transfer_first);
1870         }
1871 #endif
1872         return (xfer->td_transfer_first);
1873 }
1874
1875 /* NOTE: "done" can be run two times in a row,
1876  * from close and from interrupt
1877  */
1878
1879 static void
1880 uhci_device_done(struct usb_xfer *xfer, usb_error_t error)
1881 {
1882         struct usb_pipe_methods *methods = xfer->endpoint->methods;
1883         uhci_softc_t *sc = UHCI_BUS2SC(xfer->xroot->bus);
1884         uhci_qh_t *qh;
1885
1886         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
1887
1888         DPRINTFN(2, "xfer=%p, endpoint=%p, error=%d\n",
1889             xfer, xfer->endpoint, error);
1890
1891         qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
1892         if (qh) {
1893                 usb_pc_cpu_invalidate(qh->page_cache);
1894         }
1895         if (xfer->flags_int.bandwidth_reclaimed) {
1896                 xfer->flags_int.bandwidth_reclaimed = 0;
1897                 uhci_rem_loop(sc);
1898         }
1899         if (methods == &uhci_device_bulk_methods) {
1900                 UHCI_REMOVE_QH(qh, sc->sc_bulk_p_last);
1901         }
1902         if (methods == &uhci_device_ctrl_methods) {
1903                 if (xfer->xroot->udev->speed == USB_SPEED_LOW) {
1904                         UHCI_REMOVE_QH(qh, sc->sc_ls_ctl_p_last);
1905                 } else {
1906                         UHCI_REMOVE_QH(qh, sc->sc_fs_ctl_p_last);
1907                 }
1908         }
1909         if (methods == &uhci_device_intr_methods) {
1910                 UHCI_REMOVE_QH(qh, sc->sc_intr_p_last[xfer->qh_pos]);
1911         }
1912         /*
1913          * Only finish isochronous transfers once
1914          * which will update "xfer->frlengths".
1915          */
1916         if (xfer->td_transfer_first &&
1917             xfer->td_transfer_last) {
1918                 if (methods == &uhci_device_isoc_methods) {
1919                         uhci_isoc_done(sc, xfer);
1920                 }
1921                 xfer->td_transfer_first = NULL;
1922                 xfer->td_transfer_last = NULL;
1923         }
1924         /* dequeue transfer and start next transfer */
1925         usbd_transfer_done(xfer, error);
1926 }
1927
1928 /*------------------------------------------------------------------------*
1929  * uhci bulk support
1930  *------------------------------------------------------------------------*/
1931 static void
1932 uhci_device_bulk_open(struct usb_xfer *xfer)
1933 {
1934         return;
1935 }
1936
1937 static void
1938 uhci_device_bulk_close(struct usb_xfer *xfer)
1939 {
1940         uhci_device_done(xfer, USB_ERR_CANCELLED);
1941 }
1942
1943 static void
1944 uhci_device_bulk_enter(struct usb_xfer *xfer)
1945 {
1946         return;
1947 }
1948
1949 static void
1950 uhci_device_bulk_start(struct usb_xfer *xfer)
1951 {
1952         uhci_softc_t *sc = UHCI_BUS2SC(xfer->xroot->bus);
1953         uhci_td_t *td;
1954         uhci_qh_t *qh;
1955
1956         /* setup TD's */
1957         td = uhci_setup_standard_chain(xfer);
1958
1959         /* setup QH */
1960         qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
1961
1962         qh->e_next = td;
1963         qh->qh_e_next = td->td_self;
1964
1965         if (xfer->xroot->udev->flags.self_suspended == 0) {
1966                 UHCI_APPEND_QH(qh, sc->sc_bulk_p_last);
1967                 uhci_add_loop(sc);
1968                 xfer->flags_int.bandwidth_reclaimed = 1;
1969         } else {
1970                 usb_pc_cpu_flush(qh->page_cache);
1971         }
1972
1973         /* put transfer on interrupt queue */
1974         uhci_transfer_intr_enqueue(xfer);
1975 }
1976
1977 struct usb_pipe_methods uhci_device_bulk_methods =
1978 {
1979         .open = uhci_device_bulk_open,
1980         .close = uhci_device_bulk_close,
1981         .enter = uhci_device_bulk_enter,
1982         .start = uhci_device_bulk_start,
1983 };
1984
1985 /*------------------------------------------------------------------------*
1986  * uhci control support
1987  *------------------------------------------------------------------------*/
1988 static void
1989 uhci_device_ctrl_open(struct usb_xfer *xfer)
1990 {
1991         return;
1992 }
1993
1994 static void
1995 uhci_device_ctrl_close(struct usb_xfer *xfer)
1996 {
1997         uhci_device_done(xfer, USB_ERR_CANCELLED);
1998 }
1999
2000 static void
2001 uhci_device_ctrl_enter(struct usb_xfer *xfer)
2002 {
2003         return;
2004 }
2005
2006 static void
2007 uhci_device_ctrl_start(struct usb_xfer *xfer)
2008 {
2009         uhci_softc_t *sc = UHCI_BUS2SC(xfer->xroot->bus);
2010         uhci_qh_t *qh;
2011         uhci_td_t *td;
2012
2013         /* setup TD's */
2014         td = uhci_setup_standard_chain(xfer);
2015
2016         /* setup QH */
2017         qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
2018
2019         qh->e_next = td;
2020         qh->qh_e_next = td->td_self;
2021
2022         /*
2023          * NOTE: some devices choke on bandwidth- reclamation for control
2024          * transfers
2025          */
2026         if (xfer->xroot->udev->flags.self_suspended == 0) {
2027                 if (xfer->xroot->udev->speed == USB_SPEED_LOW) {
2028                         UHCI_APPEND_QH(qh, sc->sc_ls_ctl_p_last);
2029                 } else {
2030                         UHCI_APPEND_QH(qh, sc->sc_fs_ctl_p_last);
2031                 }
2032         } else {
2033                 usb_pc_cpu_flush(qh->page_cache);
2034         }
2035         /* put transfer on interrupt queue */
2036         uhci_transfer_intr_enqueue(xfer);
2037 }
2038
2039 struct usb_pipe_methods uhci_device_ctrl_methods =
2040 {
2041         .open = uhci_device_ctrl_open,
2042         .close = uhci_device_ctrl_close,
2043         .enter = uhci_device_ctrl_enter,
2044         .start = uhci_device_ctrl_start,
2045 };
2046
2047 /*------------------------------------------------------------------------*
2048  * uhci interrupt support
2049  *------------------------------------------------------------------------*/
2050 static void
2051 uhci_device_intr_open(struct usb_xfer *xfer)
2052 {
2053         uhci_softc_t *sc = UHCI_BUS2SC(xfer->xroot->bus);
2054         uint16_t best;
2055         uint16_t bit;
2056         uint16_t x;
2057
2058         best = 0;
2059         bit = UHCI_IFRAMELIST_COUNT / 2;
2060         while (bit) {
2061                 if (xfer->interval >= bit) {
2062                         x = bit;
2063                         best = bit;
2064                         while (x & bit) {
2065                                 if (sc->sc_intr_stat[x] <
2066                                     sc->sc_intr_stat[best]) {
2067                                         best = x;
2068                                 }
2069                                 x++;
2070                         }
2071                         break;
2072                 }
2073                 bit >>= 1;
2074         }
2075
2076         sc->sc_intr_stat[best]++;
2077         xfer->qh_pos = best;
2078
2079         DPRINTFN(3, "best=%d interval=%d\n",
2080             best, xfer->interval);
2081 }
2082
2083 static void
2084 uhci_device_intr_close(struct usb_xfer *xfer)
2085 {
2086         uhci_softc_t *sc = UHCI_BUS2SC(xfer->xroot->bus);
2087
2088         sc->sc_intr_stat[xfer->qh_pos]--;
2089
2090         uhci_device_done(xfer, USB_ERR_CANCELLED);
2091 }
2092
2093 static void
2094 uhci_device_intr_enter(struct usb_xfer *xfer)
2095 {
2096         return;
2097 }
2098
2099 static void
2100 uhci_device_intr_start(struct usb_xfer *xfer)
2101 {
2102         uhci_softc_t *sc = UHCI_BUS2SC(xfer->xroot->bus);
2103         uhci_qh_t *qh;
2104         uhci_td_t *td;
2105
2106         /* setup TD's */
2107         td = uhci_setup_standard_chain(xfer);
2108
2109         /* setup QH */
2110         qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
2111
2112         qh->e_next = td;
2113         qh->qh_e_next = td->td_self;
2114
2115         if (xfer->xroot->udev->flags.self_suspended == 0) {
2116                 /* enter QHs into the controller data structures */
2117                 UHCI_APPEND_QH(qh, sc->sc_intr_p_last[xfer->qh_pos]);
2118         } else {
2119                 usb_pc_cpu_flush(qh->page_cache);
2120         }
2121
2122         /* put transfer on interrupt queue */
2123         uhci_transfer_intr_enqueue(xfer);
2124 }
2125
2126 struct usb_pipe_methods uhci_device_intr_methods =
2127 {
2128         .open = uhci_device_intr_open,
2129         .close = uhci_device_intr_close,
2130         .enter = uhci_device_intr_enter,
2131         .start = uhci_device_intr_start,
2132 };
2133
2134 /*------------------------------------------------------------------------*
2135  * uhci isochronous support
2136  *------------------------------------------------------------------------*/
2137 static void
2138 uhci_device_isoc_open(struct usb_xfer *xfer)
2139 {
2140         uhci_td_t *td;
2141         uint32_t td_token;
2142         uint8_t ds;
2143
2144         td_token =
2145             (UE_GET_DIR(xfer->endpointno) == UE_DIR_IN) ?
2146             UHCI_TD_IN(0, xfer->endpointno, xfer->address, 0) :
2147             UHCI_TD_OUT(0, xfer->endpointno, xfer->address, 0);
2148
2149         td_token = htole32(td_token);
2150
2151         /* initialize all TD's */
2152
2153         for (ds = 0; ds != 2; ds++) {
2154
2155                 for (td = xfer->td_start[ds]; td; td = td->obj_next) {
2156
2157                         /* mark TD as inactive */
2158                         td->td_status = htole32(UHCI_TD_IOS);
2159                         td->td_token = td_token;
2160
2161                         usb_pc_cpu_flush(td->page_cache);
2162                 }
2163         }
2164 }
2165
2166 static void
2167 uhci_device_isoc_close(struct usb_xfer *xfer)
2168 {
2169         uhci_device_done(xfer, USB_ERR_CANCELLED);
2170 }
2171
2172 static void
2173 uhci_device_isoc_enter(struct usb_xfer *xfer)
2174 {
2175         struct uhci_mem_layout ml;
2176         uhci_softc_t *sc = UHCI_BUS2SC(xfer->xroot->bus);
2177         uint32_t nframes;
2178         uint32_t temp;
2179         uint32_t *plen;
2180
2181 #if USB_DEBUG
2182         uint8_t once = 1;
2183
2184 #endif
2185         uhci_td_t *td;
2186         uhci_td_t *td_last = NULL;
2187         uhci_td_t **pp_last;
2188
2189         DPRINTFN(6, "xfer=%p next=%d nframes=%d\n",
2190             xfer, xfer->endpoint->isoc_next, xfer->nframes);
2191
2192         nframes = UREAD2(sc, UHCI_FRNUM);
2193
2194         temp = (nframes - xfer->endpoint->isoc_next) &
2195             (UHCI_VFRAMELIST_COUNT - 1);
2196
2197         if ((xfer->endpoint->is_synced == 0) ||
2198             (temp < xfer->nframes)) {
2199                 /*
2200                  * If there is data underflow or the pipe queue is empty we
2201                  * schedule the transfer a few frames ahead of the current
2202                  * frame position. Else two isochronous transfers might
2203                  * overlap.
2204                  */
2205                 xfer->endpoint->isoc_next = (nframes + 3) & (UHCI_VFRAMELIST_COUNT - 1);
2206                 xfer->endpoint->is_synced = 1;
2207                 DPRINTFN(3, "start next=%d\n", xfer->endpoint->isoc_next);
2208         }
2209         /*
2210          * compute how many milliseconds the insertion is ahead of the
2211          * current frame position:
2212          */
2213         temp = (xfer->endpoint->isoc_next - nframes) &
2214             (UHCI_VFRAMELIST_COUNT - 1);
2215
2216         /*
2217          * pre-compute when the isochronous transfer will be finished:
2218          */
2219         xfer->isoc_time_complete =
2220             usb_isoc_time_expand(&sc->sc_bus, nframes) + temp +
2221             xfer->nframes;
2222
2223         /* get the real number of frames */
2224
2225         nframes = xfer->nframes;
2226
2227         uhci_mem_layout_init(&ml, xfer);
2228
2229         plen = xfer->frlengths;
2230
2231         /* toggle the DMA set we are using */
2232         xfer->flags_int.curr_dma_set ^= 1;
2233
2234         /* get next DMA set */
2235         td = xfer->td_start[xfer->flags_int.curr_dma_set];
2236         xfer->td_transfer_first = td;
2237
2238         pp_last = &sc->sc_isoc_p_last[xfer->endpoint->isoc_next];
2239
2240         /* store starting position */
2241
2242         xfer->qh_pos = xfer->endpoint->isoc_next;
2243
2244         while (nframes--) {
2245                 if (td == NULL) {
2246                         panic("%s:%d: out of TD's\n",
2247                             __FUNCTION__, __LINE__);
2248                 }
2249                 if (pp_last >= &sc->sc_isoc_p_last[UHCI_VFRAMELIST_COUNT]) {
2250                         pp_last = &sc->sc_isoc_p_last[0];
2251                 }
2252                 if (*plen > xfer->max_frame_size) {
2253 #if USB_DEBUG
2254                         if (once) {
2255                                 once = 0;
2256                                 printf("%s: frame length(%d) exceeds %d "
2257                                     "bytes (frame truncated)\n",
2258                                     __FUNCTION__, *plen,
2259                                     xfer->max_frame_size);
2260                         }
2261 #endif
2262                         *plen = xfer->max_frame_size;
2263                 }
2264                 /* reuse td_token from last transfer */
2265
2266                 td->td_token &= htole32(~UHCI_TD_MAXLEN_MASK);
2267                 td->td_token |= htole32(UHCI_TD_SET_MAXLEN(*plen));
2268
2269                 td->len = *plen;
2270
2271                 if (td->len == 0) {
2272                         /*
2273                          * Do not call "uhci_mem_layout_fixup()" when the
2274                          * length is zero!
2275                          */
2276                         td->td_buffer = 0;
2277                         td->fix_pc = NULL;
2278
2279                 } else {
2280
2281                         /* fill out buffer pointer and do fixup, if any */
2282
2283                         uhci_mem_layout_fixup(&ml, td);
2284
2285                 }
2286
2287                 /* update status */
2288                 if (nframes == 0) {
2289                         td->td_status = htole32
2290                             (UHCI_TD_ZERO_ACTLEN
2291                             (UHCI_TD_SET_ERRCNT(0) |
2292                             UHCI_TD_ACTIVE |
2293                             UHCI_TD_IOS |
2294                             UHCI_TD_IOC));
2295                 } else {
2296                         td->td_status = htole32
2297                             (UHCI_TD_ZERO_ACTLEN
2298                             (UHCI_TD_SET_ERRCNT(0) |
2299                             UHCI_TD_ACTIVE |
2300                             UHCI_TD_IOS));
2301                 }
2302
2303                 usb_pc_cpu_flush(td->page_cache);
2304
2305 #if USB_DEBUG
2306                 if (uhcidebug > 5) {
2307                         DPRINTF("TD %d\n", nframes);
2308                         uhci_dump_td(td);
2309                 }
2310 #endif
2311                 /* insert TD into schedule */
2312                 UHCI_APPEND_TD(td, *pp_last);
2313                 pp_last++;
2314
2315                 plen++;
2316                 td_last = td;
2317                 td = td->obj_next;
2318         }
2319
2320         xfer->td_transfer_last = td_last;
2321
2322         /* update isoc_next */
2323         xfer->endpoint->isoc_next = (pp_last - &sc->sc_isoc_p_last[0]) &
2324             (UHCI_VFRAMELIST_COUNT - 1);
2325 }
2326
2327 static void
2328 uhci_device_isoc_start(struct usb_xfer *xfer)
2329 {
2330         /* put transfer on interrupt queue */
2331         uhci_transfer_intr_enqueue(xfer);
2332 }
2333
2334 struct usb_pipe_methods uhci_device_isoc_methods =
2335 {
2336         .open = uhci_device_isoc_open,
2337         .close = uhci_device_isoc_close,
2338         .enter = uhci_device_isoc_enter,
2339         .start = uhci_device_isoc_start,
2340 };
2341
2342 /*------------------------------------------------------------------------*
2343  * uhci root control support
2344  *------------------------------------------------------------------------*
2345  * Simulate a hardware hub by handling all the necessary requests.
2346  *------------------------------------------------------------------------*/
2347
2348 static const
2349 struct usb_device_descriptor uhci_devd =
2350 {
2351         sizeof(struct usb_device_descriptor),
2352         UDESC_DEVICE,                   /* type */
2353         {0x00, 0x01},                   /* USB version */
2354         UDCLASS_HUB,                    /* class */
2355         UDSUBCLASS_HUB,                 /* subclass */
2356         UDPROTO_FSHUB,                  /* protocol */
2357         64,                             /* max packet */
2358         {0}, {0}, {0x00, 0x01},         /* device id */
2359         1, 2, 0,                        /* string indicies */
2360         1                               /* # of configurations */
2361 };
2362
2363 static const struct uhci_config_desc uhci_confd = {
2364         .confd = {
2365                 .bLength = sizeof(struct usb_config_descriptor),
2366                 .bDescriptorType = UDESC_CONFIG,
2367                 .wTotalLength[0] = sizeof(uhci_confd),
2368                 .bNumInterface = 1,
2369                 .bConfigurationValue = 1,
2370                 .iConfiguration = 0,
2371                 .bmAttributes = UC_SELF_POWERED,
2372                 .bMaxPower = 0          /* max power */
2373         },
2374         .ifcd = {
2375                 .bLength = sizeof(struct usb_interface_descriptor),
2376                 .bDescriptorType = UDESC_INTERFACE,
2377                 .bNumEndpoints = 1,
2378                 .bInterfaceClass = UICLASS_HUB,
2379                 .bInterfaceSubClass = UISUBCLASS_HUB,
2380                 .bInterfaceProtocol = UIPROTO_FSHUB,
2381         },
2382         .endpd = {
2383                 .bLength = sizeof(struct usb_endpoint_descriptor),
2384                 .bDescriptorType = UDESC_ENDPOINT,
2385                 .bEndpointAddress = UE_DIR_IN | UHCI_INTR_ENDPT,
2386                 .bmAttributes = UE_INTERRUPT,
2387                 .wMaxPacketSize[0] = 8, /* max packet (63 ports) */
2388                 .bInterval = 255,
2389         },
2390 };
2391
2392 static const
2393 struct usb_hub_descriptor_min uhci_hubd_piix =
2394 {
2395         sizeof(uhci_hubd_piix),
2396         UDESC_HUB,
2397         2,
2398         {UHD_PWR_NO_SWITCH | UHD_OC_INDIVIDUAL, 0},
2399         50,                             /* power on to power good */
2400         0,
2401         {0x00},                         /* both ports are removable */
2402 };
2403
2404 /*
2405  * The USB hub protocol requires that SET_FEATURE(PORT_RESET) also
2406  * enables the port, and also states that SET_FEATURE(PORT_ENABLE)
2407  * should not be used by the USB subsystem.  As we cannot issue a
2408  * SET_FEATURE(PORT_ENABLE) externally, we must ensure that the port
2409  * will be enabled as part of the reset.
2410  *
2411  * On the VT83C572, the port cannot be successfully enabled until the
2412  * outstanding "port enable change" and "connection status change"
2413  * events have been reset.
2414  */
2415 static usb_error_t
2416 uhci_portreset(uhci_softc_t *sc, uint16_t index)
2417 {
2418         uint16_t port;
2419         uint16_t x;
2420         uint8_t lim;
2421
2422         if (index == 1)
2423                 port = UHCI_PORTSC1;
2424         else if (index == 2)
2425                 port = UHCI_PORTSC2;
2426         else
2427                 return (USB_ERR_IOERROR);
2428
2429         /*
2430          * Before we do anything, turn on SOF messages on the USB
2431          * BUS. Some USB devices do not cope without them!
2432          */
2433         uhci_restart(sc);
2434
2435         x = URWMASK(UREAD2(sc, port));
2436         UWRITE2(sc, port, x | UHCI_PORTSC_PR);
2437
2438         usb_pause_mtx(&sc->sc_bus.bus_mtx,
2439             USB_MS_TO_TICKS(USB_PORT_ROOT_RESET_DELAY));
2440
2441         DPRINTFN(4, "uhci port %d reset, status0 = 0x%04x\n",
2442             index, UREAD2(sc, port));
2443
2444         x = URWMASK(UREAD2(sc, port));
2445         UWRITE2(sc, port, x & ~UHCI_PORTSC_PR);
2446
2447
2448         mtx_unlock(&sc->sc_bus.bus_mtx);
2449
2450         /*
2451          * This delay needs to be exactly 100us, else some USB devices
2452          * fail to attach!
2453          */
2454         DELAY(100);
2455
2456         mtx_lock(&sc->sc_bus.bus_mtx);
2457
2458         DPRINTFN(4, "uhci port %d reset, status1 = 0x%04x\n",
2459             index, UREAD2(sc, port));
2460
2461         x = URWMASK(UREAD2(sc, port));
2462         UWRITE2(sc, port, x | UHCI_PORTSC_PE);
2463
2464         for (lim = 0; lim < 12; lim++) {
2465
2466                 usb_pause_mtx(&sc->sc_bus.bus_mtx,
2467                     USB_MS_TO_TICKS(USB_PORT_RESET_DELAY));
2468
2469                 x = UREAD2(sc, port);
2470
2471                 DPRINTFN(4, "uhci port %d iteration %u, status = 0x%04x\n",
2472                     index, lim, x);
2473
2474                 if (!(x & UHCI_PORTSC_CCS)) {
2475                         /*
2476                          * No device is connected (or was disconnected
2477                          * during reset).  Consider the port reset.
2478                          * The delay must be long enough to ensure on
2479                          * the initial iteration that the device
2480                          * connection will have been registered.  50ms
2481                          * appears to be sufficient, but 20ms is not.
2482                          */
2483                         DPRINTFN(4, "uhci port %d loop %u, device detached\n",
2484                             index, lim);
2485                         goto done;
2486                 }
2487                 if (x & (UHCI_PORTSC_POEDC | UHCI_PORTSC_CSC)) {
2488                         /*
2489                          * Port enabled changed and/or connection
2490                          * status changed were set.  Reset either or
2491                          * both raised flags (by writing a 1 to that
2492                          * bit), and wait again for state to settle.
2493                          */
2494                         UWRITE2(sc, port, URWMASK(x) |
2495                             (x & (UHCI_PORTSC_POEDC | UHCI_PORTSC_CSC)));
2496                         continue;
2497                 }
2498                 if (x & UHCI_PORTSC_PE) {
2499                         /* port is enabled */
2500                         goto done;
2501                 }
2502                 UWRITE2(sc, port, URWMASK(x) | UHCI_PORTSC_PE);
2503         }
2504
2505         DPRINTFN(2, "uhci port %d reset timed out\n", index);
2506         return (USB_ERR_TIMEOUT);
2507
2508 done:
2509         DPRINTFN(4, "uhci port %d reset, status2 = 0x%04x\n",
2510             index, UREAD2(sc, port));
2511
2512         sc->sc_isreset = 1;
2513         return (USB_ERR_NORMAL_COMPLETION);
2514 }
2515
2516 static usb_error_t
2517 uhci_roothub_exec(struct usb_device *udev,
2518     struct usb_device_request *req, const void **pptr, uint16_t *plength)
2519 {
2520         uhci_softc_t *sc = UHCI_BUS2SC(udev->bus);
2521         const void *ptr;
2522         const char *str_ptr;
2523         uint16_t x;
2524         uint16_t port;
2525         uint16_t value;
2526         uint16_t index;
2527         uint16_t status;
2528         uint16_t change;
2529         uint16_t len;
2530         usb_error_t err;
2531
2532         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
2533
2534         /* buffer reset */
2535         ptr = (const void *)&sc->sc_hub_desc.temp;
2536         len = 0;
2537         err = 0;
2538
2539         value = UGETW(req->wValue);
2540         index = UGETW(req->wIndex);
2541
2542         DPRINTFN(3, "type=0x%02x request=0x%02x wLen=0x%04x "
2543             "wValue=0x%04x wIndex=0x%04x\n",
2544             req->bmRequestType, req->bRequest,
2545             UGETW(req->wLength), value, index);
2546
2547 #define C(x,y) ((x) | ((y) << 8))
2548         switch (C(req->bRequest, req->bmRequestType)) {
2549         case C(UR_CLEAR_FEATURE, UT_WRITE_DEVICE):
2550         case C(UR_CLEAR_FEATURE, UT_WRITE_INTERFACE):
2551         case C(UR_CLEAR_FEATURE, UT_WRITE_ENDPOINT):
2552                 /*
2553                  * DEVICE_REMOTE_WAKEUP and ENDPOINT_HALT are no-ops
2554                  * for the integrated root hub.
2555                  */
2556                 break;
2557         case C(UR_GET_CONFIG, UT_READ_DEVICE):
2558                 len = 1;
2559                 sc->sc_hub_desc.temp[0] = sc->sc_conf;
2560                 break;
2561         case C(UR_GET_DESCRIPTOR, UT_READ_DEVICE):
2562                 switch (value >> 8) {
2563                 case UDESC_DEVICE:
2564                         if ((value & 0xff) != 0) {
2565                                 err = USB_ERR_IOERROR;
2566                                 goto done;
2567                         }
2568                         len = sizeof(uhci_devd);
2569                         ptr = (const void *)&uhci_devd;
2570                         break;
2571
2572                 case UDESC_CONFIG:
2573                         if ((value & 0xff) != 0) {
2574                                 err = USB_ERR_IOERROR;
2575                                 goto done;
2576                         }
2577                         len = sizeof(uhci_confd);
2578                         ptr = (const void *)&uhci_confd;
2579                         break;
2580
2581                 case UDESC_STRING:
2582                         switch (value & 0xff) {
2583                         case 0: /* Language table */
2584                                 str_ptr = "\001";
2585                                 break;
2586
2587                         case 1: /* Vendor */
2588                                 str_ptr = sc->sc_vendor;
2589                                 break;
2590
2591                         case 2: /* Product */
2592                                 str_ptr = "UHCI root HUB";
2593                                 break;
2594
2595                         default:
2596                                 str_ptr = "";
2597                                 break;
2598                         }
2599
2600                         len = usb_make_str_desc
2601                             (sc->sc_hub_desc.temp,
2602                             sizeof(sc->sc_hub_desc.temp),
2603                             str_ptr);
2604                         break;
2605
2606                 default:
2607                         err = USB_ERR_IOERROR;
2608                         goto done;
2609                 }
2610                 break;
2611         case C(UR_GET_INTERFACE, UT_READ_INTERFACE):
2612                 len = 1;
2613                 sc->sc_hub_desc.temp[0] = 0;
2614                 break;
2615         case C(UR_GET_STATUS, UT_READ_DEVICE):
2616                 len = 2;
2617                 USETW(sc->sc_hub_desc.stat.wStatus, UDS_SELF_POWERED);
2618                 break;
2619         case C(UR_GET_STATUS, UT_READ_INTERFACE):
2620         case C(UR_GET_STATUS, UT_READ_ENDPOINT):
2621                 len = 2;
2622                 USETW(sc->sc_hub_desc.stat.wStatus, 0);
2623                 break;
2624         case C(UR_SET_ADDRESS, UT_WRITE_DEVICE):
2625                 if (value >= UHCI_MAX_DEVICES) {
2626                         err = USB_ERR_IOERROR;
2627                         goto done;
2628                 }
2629                 sc->sc_addr = value;
2630                 break;
2631         case C(UR_SET_CONFIG, UT_WRITE_DEVICE):
2632                 if ((value != 0) && (value != 1)) {
2633                         err = USB_ERR_IOERROR;
2634                         goto done;
2635                 }
2636                 sc->sc_conf = value;
2637                 break;
2638         case C(UR_SET_DESCRIPTOR, UT_WRITE_DEVICE):
2639                 break;
2640         case C(UR_SET_FEATURE, UT_WRITE_DEVICE):
2641         case C(UR_SET_FEATURE, UT_WRITE_INTERFACE):
2642         case C(UR_SET_FEATURE, UT_WRITE_ENDPOINT):
2643                 err = USB_ERR_IOERROR;
2644                 goto done;
2645         case C(UR_SET_INTERFACE, UT_WRITE_INTERFACE):
2646                 break;
2647         case C(UR_SYNCH_FRAME, UT_WRITE_ENDPOINT):
2648                 break;
2649                 /* Hub requests */
2650         case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_DEVICE):
2651                 break;
2652         case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_OTHER):
2653                 DPRINTFN(4, "UR_CLEAR_PORT_FEATURE "
2654                     "port=%d feature=%d\n",
2655                     index, value);
2656                 if (index == 1)
2657                         port = UHCI_PORTSC1;
2658                 else if (index == 2)
2659                         port = UHCI_PORTSC2;
2660                 else {
2661                         err = USB_ERR_IOERROR;
2662                         goto done;
2663                 }
2664                 switch (value) {
2665                 case UHF_PORT_ENABLE:
2666                         x = URWMASK(UREAD2(sc, port));
2667                         UWRITE2(sc, port, x & ~UHCI_PORTSC_PE);
2668                         break;
2669                 case UHF_PORT_SUSPEND:
2670                         x = URWMASK(UREAD2(sc, port));
2671                         UWRITE2(sc, port, x & ~(UHCI_PORTSC_SUSP));
2672                         break;
2673                 case UHF_PORT_RESET:
2674                         x = URWMASK(UREAD2(sc, port));
2675                         UWRITE2(sc, port, x & ~UHCI_PORTSC_PR);
2676                         break;
2677                 case UHF_C_PORT_CONNECTION:
2678                         x = URWMASK(UREAD2(sc, port));
2679                         UWRITE2(sc, port, x | UHCI_PORTSC_CSC);
2680                         break;
2681                 case UHF_C_PORT_ENABLE:
2682                         x = URWMASK(UREAD2(sc, port));
2683                         UWRITE2(sc, port, x | UHCI_PORTSC_POEDC);
2684                         break;
2685                 case UHF_C_PORT_OVER_CURRENT:
2686                         x = URWMASK(UREAD2(sc, port));
2687                         UWRITE2(sc, port, x | UHCI_PORTSC_OCIC);
2688                         break;
2689                 case UHF_C_PORT_RESET:
2690                         sc->sc_isreset = 0;
2691                         err = USB_ERR_NORMAL_COMPLETION;
2692                         goto done;
2693                 case UHF_C_PORT_SUSPEND:
2694                         sc->sc_isresumed &= ~(1 << index);
2695                         break;
2696                 case UHF_PORT_CONNECTION:
2697                 case UHF_PORT_OVER_CURRENT:
2698                 case UHF_PORT_POWER:
2699                 case UHF_PORT_LOW_SPEED:
2700                 default:
2701                         err = USB_ERR_IOERROR;
2702                         goto done;
2703                 }
2704                 break;
2705         case C(UR_GET_BUS_STATE, UT_READ_CLASS_OTHER):
2706                 if (index == 1)
2707                         port = UHCI_PORTSC1;
2708                 else if (index == 2)
2709                         port = UHCI_PORTSC2;
2710                 else {
2711                         err = USB_ERR_IOERROR;
2712                         goto done;
2713                 }
2714                 len = 1;
2715                 sc->sc_hub_desc.temp[0] =
2716                     ((UREAD2(sc, port) & UHCI_PORTSC_LS) >>
2717                     UHCI_PORTSC_LS_SHIFT);
2718                 break;
2719         case C(UR_GET_DESCRIPTOR, UT_READ_CLASS_DEVICE):
2720                 if ((value & 0xff) != 0) {
2721                         err = USB_ERR_IOERROR;
2722                         goto done;
2723                 }
2724                 len = sizeof(uhci_hubd_piix);
2725                 ptr = (const void *)&uhci_hubd_piix;
2726                 break;
2727         case C(UR_GET_STATUS, UT_READ_CLASS_DEVICE):
2728                 len = 16;
2729                 bzero(sc->sc_hub_desc.temp, 16);
2730                 break;
2731         case C(UR_GET_STATUS, UT_READ_CLASS_OTHER):
2732                 if (index == 1)
2733                         port = UHCI_PORTSC1;
2734                 else if (index == 2)
2735                         port = UHCI_PORTSC2;
2736                 else {
2737                         err = USB_ERR_IOERROR;
2738                         goto done;
2739                 }
2740                 x = UREAD2(sc, port);
2741                 status = change = 0;
2742                 if (x & UHCI_PORTSC_CCS)
2743                         status |= UPS_CURRENT_CONNECT_STATUS;
2744                 if (x & UHCI_PORTSC_CSC)
2745                         change |= UPS_C_CONNECT_STATUS;
2746                 if (x & UHCI_PORTSC_PE)
2747                         status |= UPS_PORT_ENABLED;
2748                 if (x & UHCI_PORTSC_POEDC)
2749                         change |= UPS_C_PORT_ENABLED;
2750                 if (x & UHCI_PORTSC_OCI)
2751                         status |= UPS_OVERCURRENT_INDICATOR;
2752                 if (x & UHCI_PORTSC_OCIC)
2753                         change |= UPS_C_OVERCURRENT_INDICATOR;
2754                 if (x & UHCI_PORTSC_LSDA)
2755                         status |= UPS_LOW_SPEED;
2756                 if ((x & UHCI_PORTSC_PE) && (x & UHCI_PORTSC_RD)) {
2757                         /* need to do a write back */
2758                         UWRITE2(sc, port, URWMASK(x));
2759
2760                         /* wait 20ms for resume sequence to complete */
2761                         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 50);
2762
2763                         /* clear suspend and resume detect */
2764                         UWRITE2(sc, port, URWMASK(x) & ~(UHCI_PORTSC_RD |
2765                             UHCI_PORTSC_SUSP));
2766
2767                         /* wait a little bit */
2768                         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 500);
2769
2770                         sc->sc_isresumed |= (1 << index);
2771
2772                 } else if (x & UHCI_PORTSC_SUSP) {
2773                         status |= UPS_SUSPEND;
2774                 }
2775                 status |= UPS_PORT_POWER;
2776                 if (sc->sc_isresumed & (1 << index))
2777                         change |= UPS_C_SUSPEND;
2778                 if (sc->sc_isreset)
2779                         change |= UPS_C_PORT_RESET;
2780                 USETW(sc->sc_hub_desc.ps.wPortStatus, status);
2781                 USETW(sc->sc_hub_desc.ps.wPortChange, change);
2782                 len = sizeof(sc->sc_hub_desc.ps);
2783                 break;
2784         case C(UR_SET_DESCRIPTOR, UT_WRITE_CLASS_DEVICE):
2785                 err = USB_ERR_IOERROR;
2786                 goto done;
2787         case C(UR_SET_FEATURE, UT_WRITE_CLASS_DEVICE):
2788                 break;
2789         case C(UR_SET_FEATURE, UT_WRITE_CLASS_OTHER):
2790                 if (index == 1)
2791                         port = UHCI_PORTSC1;
2792                 else if (index == 2)
2793                         port = UHCI_PORTSC2;
2794                 else {
2795                         err = USB_ERR_IOERROR;
2796                         goto done;
2797                 }
2798                 switch (value) {
2799                 case UHF_PORT_ENABLE:
2800                         x = URWMASK(UREAD2(sc, port));
2801                         UWRITE2(sc, port, x | UHCI_PORTSC_PE);
2802                         break;
2803                 case UHF_PORT_SUSPEND:
2804                         x = URWMASK(UREAD2(sc, port));
2805                         UWRITE2(sc, port, x | UHCI_PORTSC_SUSP);
2806                         break;
2807                 case UHF_PORT_RESET:
2808                         err = uhci_portreset(sc, index);
2809                         goto done;
2810                 case UHF_PORT_POWER:
2811                         /* pretend we turned on power */
2812                         err = USB_ERR_NORMAL_COMPLETION;
2813                         goto done;
2814                 case UHF_C_PORT_CONNECTION:
2815                 case UHF_C_PORT_ENABLE:
2816                 case UHF_C_PORT_OVER_CURRENT:
2817                 case UHF_PORT_CONNECTION:
2818                 case UHF_PORT_OVER_CURRENT:
2819                 case UHF_PORT_LOW_SPEED:
2820                 case UHF_C_PORT_SUSPEND:
2821                 case UHF_C_PORT_RESET:
2822                 default:
2823                         err = USB_ERR_IOERROR;
2824                         goto done;
2825                 }
2826                 break;
2827         default:
2828                 err = USB_ERR_IOERROR;
2829                 goto done;
2830         }
2831 done:
2832         *plength = len;
2833         *pptr = ptr;
2834         return (err);
2835 }
2836
2837 /*
2838  * This routine is executed periodically and simulates interrupts from
2839  * the root controller interrupt pipe for port status change:
2840  */
2841 static void
2842 uhci_root_intr(uhci_softc_t *sc)
2843 {
2844         DPRINTFN(21, "\n");
2845
2846         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
2847
2848         sc->sc_hub_idata[0] = 0;
2849
2850         if (UREAD2(sc, UHCI_PORTSC1) & (UHCI_PORTSC_CSC |
2851             UHCI_PORTSC_OCIC | UHCI_PORTSC_RD)) {
2852                 sc->sc_hub_idata[0] |= 1 << 1;
2853         }
2854         if (UREAD2(sc, UHCI_PORTSC2) & (UHCI_PORTSC_CSC |
2855             UHCI_PORTSC_OCIC | UHCI_PORTSC_RD)) {
2856                 sc->sc_hub_idata[0] |= 1 << 2;
2857         }
2858
2859         /* restart timer */
2860         usb_callout_reset(&sc->sc_root_intr, hz,
2861             (void *)&uhci_root_intr, sc);
2862
2863         if (sc->sc_hub_idata[0] != 0) {
2864                 uhub_root_intr(&sc->sc_bus, sc->sc_hub_idata,
2865                     sizeof(sc->sc_hub_idata));
2866         }
2867 }
2868
2869 static void
2870 uhci_xfer_setup(struct usb_setup_params *parm)
2871 {
2872         struct usb_page_search page_info;
2873         struct usb_page_cache *pc;
2874         uhci_softc_t *sc;
2875         struct usb_xfer *xfer;
2876         void *last_obj;
2877         uint32_t ntd;
2878         uint32_t nqh;
2879         uint32_t nfixup;
2880         uint32_t n;
2881         uint16_t align;
2882
2883         sc = UHCI_BUS2SC(parm->udev->bus);
2884         xfer = parm->curr_xfer;
2885
2886         parm->hc_max_packet_size = 0x500;
2887         parm->hc_max_packet_count = 1;
2888         parm->hc_max_frame_size = 0x500;
2889
2890         /*
2891          * compute ntd and nqh
2892          */
2893         if (parm->methods == &uhci_device_ctrl_methods) {
2894                 xfer->flags_int.bdma_enable = 1;
2895                 xfer->flags_int.bdma_no_post_sync = 1;
2896
2897                 usbd_transfer_setup_sub(parm);
2898
2899                 /* see EHCI HC driver for proof of "ntd" formula */
2900
2901                 nqh = 1;
2902                 ntd = ((2 * xfer->nframes) + 1  /* STATUS */
2903                     + (xfer->max_data_length / xfer->max_frame_size));
2904
2905         } else if (parm->methods == &uhci_device_bulk_methods) {
2906                 xfer->flags_int.bdma_enable = 1;
2907                 xfer->flags_int.bdma_no_post_sync = 1;
2908
2909                 usbd_transfer_setup_sub(parm);
2910
2911                 nqh = 1;
2912                 ntd = ((2 * xfer->nframes)
2913                     + (xfer->max_data_length / xfer->max_frame_size));
2914
2915         } else if (parm->methods == &uhci_device_intr_methods) {
2916                 xfer->flags_int.bdma_enable = 1;
2917                 xfer->flags_int.bdma_no_post_sync = 1;
2918
2919                 usbd_transfer_setup_sub(parm);
2920
2921                 nqh = 1;
2922                 ntd = ((2 * xfer->nframes)
2923                     + (xfer->max_data_length / xfer->max_frame_size));
2924
2925         } else if (parm->methods == &uhci_device_isoc_methods) {
2926                 xfer->flags_int.bdma_enable = 1;
2927                 xfer->flags_int.bdma_no_post_sync = 1;
2928
2929                 usbd_transfer_setup_sub(parm);
2930
2931                 nqh = 0;
2932                 ntd = xfer->nframes;
2933
2934         } else {
2935
2936                 usbd_transfer_setup_sub(parm);
2937
2938                 nqh = 0;
2939                 ntd = 0;
2940         }
2941
2942         if (parm->err) {
2943                 return;
2944         }
2945         /*
2946          * NOTE: the UHCI controller requires that
2947          * every packet must be contiguous on
2948          * the same USB memory page !
2949          */
2950         nfixup = (parm->bufsize / USB_PAGE_SIZE) + 1;
2951
2952         /*
2953          * Compute a suitable power of two alignment
2954          * for our "max_frame_size" fixup buffer(s):
2955          */
2956         align = xfer->max_frame_size;
2957         n = 0;
2958         while (align) {
2959                 align >>= 1;
2960                 n++;
2961         }
2962
2963         /* check for power of two */
2964         if (!(xfer->max_frame_size &
2965             (xfer->max_frame_size - 1))) {
2966                 n--;
2967         }
2968         /*
2969          * We don't allow alignments of
2970          * less than 8 bytes:
2971          *
2972          * NOTE: Allocating using an aligment
2973          * of 1 byte has special meaning!
2974          */
2975         if (n < 3) {
2976                 n = 3;
2977         }
2978         align = (1 << n);
2979
2980         if (usbd_transfer_setup_sub_malloc(
2981             parm, &pc, xfer->max_frame_size,
2982             align, nfixup)) {
2983                 parm->err = USB_ERR_NOMEM;
2984                 return;
2985         }
2986         xfer->buf_fixup = pc;
2987
2988 alloc_dma_set:
2989
2990         if (parm->err) {
2991                 return;
2992         }
2993         last_obj = NULL;
2994
2995         if (usbd_transfer_setup_sub_malloc(
2996             parm, &pc, sizeof(uhci_td_t),
2997             UHCI_TD_ALIGN, ntd)) {
2998                 parm->err = USB_ERR_NOMEM;
2999                 return;
3000         }
3001         if (parm->buf) {
3002                 for (n = 0; n != ntd; n++) {
3003                         uhci_td_t *td;
3004
3005                         usbd_get_page(pc + n, 0, &page_info);
3006
3007                         td = page_info.buffer;
3008
3009                         /* init TD */
3010                         if ((parm->methods == &uhci_device_bulk_methods) ||
3011                             (parm->methods == &uhci_device_ctrl_methods) ||
3012                             (parm->methods == &uhci_device_intr_methods)) {
3013                                 /* set depth first bit */
3014                                 td->td_self = htole32(page_info.physaddr |
3015                                     UHCI_PTR_TD | UHCI_PTR_VF);
3016                         } else {
3017                                 td->td_self = htole32(page_info.physaddr |
3018                                     UHCI_PTR_TD);
3019                         }
3020
3021                         td->obj_next = last_obj;
3022                         td->page_cache = pc + n;
3023
3024                         last_obj = td;
3025
3026                         usb_pc_cpu_flush(pc + n);
3027                 }
3028         }
3029         xfer->td_start[xfer->flags_int.curr_dma_set] = last_obj;
3030
3031         last_obj = NULL;
3032
3033         if (usbd_transfer_setup_sub_malloc(
3034             parm, &pc, sizeof(uhci_qh_t),
3035             UHCI_QH_ALIGN, nqh)) {
3036                 parm->err = USB_ERR_NOMEM;
3037                 return;
3038         }
3039         if (parm->buf) {
3040                 for (n = 0; n != nqh; n++) {
3041                         uhci_qh_t *qh;
3042
3043                         usbd_get_page(pc + n, 0, &page_info);
3044
3045                         qh = page_info.buffer;
3046
3047                         /* init QH */
3048                         qh->qh_self = htole32(page_info.physaddr | UHCI_PTR_QH);
3049                         qh->obj_next = last_obj;
3050                         qh->page_cache = pc + n;
3051
3052                         last_obj = qh;
3053
3054                         usb_pc_cpu_flush(pc + n);
3055                 }
3056         }
3057         xfer->qh_start[xfer->flags_int.curr_dma_set] = last_obj;
3058
3059         if (!xfer->flags_int.curr_dma_set) {
3060                 xfer->flags_int.curr_dma_set = 1;
3061                 goto alloc_dma_set;
3062         }
3063 }
3064
3065 static void
3066 uhci_ep_init(struct usb_device *udev, struct usb_endpoint_descriptor *edesc,
3067     struct usb_endpoint *ep)
3068 {
3069         uhci_softc_t *sc = UHCI_BUS2SC(udev->bus);
3070
3071         DPRINTFN(2, "endpoint=%p, addr=%d, endpt=%d, mode=%d (%d)\n",
3072             ep, udev->address,
3073             edesc->bEndpointAddress, udev->flags.usb_mode,
3074             sc->sc_addr);
3075
3076         if (udev->flags.usb_mode != USB_MODE_HOST) {
3077                 /* not supported */
3078                 return;
3079         }
3080         if (udev->device_index != sc->sc_addr) {
3081                 switch (edesc->bmAttributes & UE_XFERTYPE) {
3082                 case UE_CONTROL:
3083                         ep->methods = &uhci_device_ctrl_methods;
3084                         break;
3085                 case UE_INTERRUPT:
3086                         ep->methods = &uhci_device_intr_methods;
3087                         break;
3088                 case UE_ISOCHRONOUS:
3089                         if (udev->speed == USB_SPEED_FULL) {
3090                                 ep->methods = &uhci_device_isoc_methods;
3091                         }
3092                         break;
3093                 case UE_BULK:
3094                         if (udev->speed != USB_SPEED_LOW) {
3095                                 ep->methods = &uhci_device_bulk_methods;
3096                         }
3097                         break;
3098                 default:
3099                         /* do nothing */
3100                         break;
3101                 }
3102         }
3103 }
3104
3105 static void
3106 uhci_xfer_unsetup(struct usb_xfer *xfer)
3107 {
3108         return;
3109 }
3110
3111 static void
3112 uhci_get_dma_delay(struct usb_bus *bus, uint32_t *pus)
3113 {
3114         /*
3115          * Wait until hardware has finished any possible use of the
3116          * transfer descriptor(s) and QH
3117          */
3118         *pus = (1125);                  /* microseconds */
3119 }
3120
3121 static void
3122 uhci_device_resume(struct usb_device *udev)
3123 {
3124         struct uhci_softc *sc = UHCI_BUS2SC(udev->bus);
3125         struct usb_xfer *xfer;
3126         struct usb_pipe_methods *methods;
3127         uhci_qh_t *qh;
3128
3129         DPRINTF("\n");
3130
3131         USB_BUS_LOCK(udev->bus);
3132
3133         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
3134
3135                 if (xfer->xroot->udev == udev) {
3136
3137                         methods = xfer->endpoint->methods;
3138                         qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
3139
3140                         if (methods == &uhci_device_bulk_methods) {
3141                                 UHCI_APPEND_QH(qh, sc->sc_bulk_p_last);
3142                                 uhci_add_loop(sc);
3143                                 xfer->flags_int.bandwidth_reclaimed = 1;
3144                         }
3145                         if (methods == &uhci_device_ctrl_methods) {
3146                                 if (xfer->xroot->udev->speed == USB_SPEED_LOW) {
3147                                         UHCI_APPEND_QH(qh, sc->sc_ls_ctl_p_last);
3148                                 } else {
3149                                         UHCI_APPEND_QH(qh, sc->sc_fs_ctl_p_last);
3150                                 }
3151                         }
3152                         if (methods == &uhci_device_intr_methods) {
3153                                 UHCI_APPEND_QH(qh, sc->sc_intr_p_last[xfer->qh_pos]);
3154                         }
3155                 }
3156         }
3157
3158         USB_BUS_UNLOCK(udev->bus);
3159
3160         return;
3161 }
3162
3163 static void
3164 uhci_device_suspend(struct usb_device *udev)
3165 {
3166         struct uhci_softc *sc = UHCI_BUS2SC(udev->bus);
3167         struct usb_xfer *xfer;
3168         struct usb_pipe_methods *methods;
3169         uhci_qh_t *qh;
3170
3171         DPRINTF("\n");
3172
3173         USB_BUS_LOCK(udev->bus);
3174
3175         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
3176
3177                 if (xfer->xroot->udev == udev) {
3178
3179                         methods = xfer->endpoint->methods;
3180                         qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
3181
3182                         if (xfer->flags_int.bandwidth_reclaimed) {
3183                                 xfer->flags_int.bandwidth_reclaimed = 0;
3184                                 uhci_rem_loop(sc);
3185                         }
3186                         if (methods == &uhci_device_bulk_methods) {
3187                                 UHCI_REMOVE_QH(qh, sc->sc_bulk_p_last);
3188                         }
3189                         if (methods == &uhci_device_ctrl_methods) {
3190                                 if (xfer->xroot->udev->speed == USB_SPEED_LOW) {
3191                                         UHCI_REMOVE_QH(qh, sc->sc_ls_ctl_p_last);
3192                                 } else {
3193                                         UHCI_REMOVE_QH(qh, sc->sc_fs_ctl_p_last);
3194                                 }
3195                         }
3196                         if (methods == &uhci_device_intr_methods) {
3197                                 UHCI_REMOVE_QH(qh, sc->sc_intr_p_last[xfer->qh_pos]);
3198                         }
3199                 }
3200         }
3201
3202         USB_BUS_UNLOCK(udev->bus);
3203
3204         return;
3205 }
3206
3207 static void
3208 uhci_set_hw_power(struct usb_bus *bus)
3209 {
3210         struct uhci_softc *sc = UHCI_BUS2SC(bus);
3211         uint32_t flags;
3212
3213         DPRINTF("\n");
3214
3215         USB_BUS_LOCK(bus);
3216
3217         flags = bus->hw_power_state;
3218
3219         /*
3220          * WARNING: Some FULL speed USB devices require periodic SOF
3221          * messages! If any USB devices are connected through the
3222          * UHCI, power save will be disabled!
3223          */
3224         if (flags & (USB_HW_POWER_CONTROL |
3225             USB_HW_POWER_NON_ROOT_HUB |
3226             USB_HW_POWER_BULK |
3227             USB_HW_POWER_INTERRUPT |
3228             USB_HW_POWER_ISOC)) {
3229                 DPRINTF("Some USB transfer is "
3230                     "active on unit %u.\n",
3231                     device_get_unit(sc->sc_bus.bdev));
3232                 uhci_restart(sc);
3233         } else {
3234                 DPRINTF("Power save on unit %u.\n",
3235                     device_get_unit(sc->sc_bus.bdev));
3236                 UHCICMD(sc, UHCI_CMD_MAXP);
3237         }
3238
3239         USB_BUS_UNLOCK(bus);
3240
3241         return;
3242 }
3243
3244
3245 struct usb_bus_methods uhci_bus_methods =
3246 {
3247         .endpoint_init = uhci_ep_init,
3248         .xfer_setup = uhci_xfer_setup,
3249         .xfer_unsetup = uhci_xfer_unsetup,
3250         .get_dma_delay = uhci_get_dma_delay,
3251         .device_resume = uhci_device_resume,
3252         .device_suspend = uhci_device_suspend,
3253         .set_hw_power = uhci_set_hw_power,
3254         .roothub_exec = uhci_roothub_exec,
3255         .xfer_poll = uhci_do_poll,
3256 };