sys/dev/usb/controller/uhci.c

   1 /* $FreeBSD$ */
   2 /*-
   3  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   4  *
   5  * Copyright (c) 2008 Hans Petter Selasky. All rights reserved.
   6  * Copyright (c) 1998 The NetBSD Foundation, Inc. All rights reserved.
   7  * Copyright (c) 1998 Lennart Augustsson. All rights reserved.
   8  *
   9  * Redistribution and use in source and binary forms, with or without
  10  * modification, are permitted provided that the following conditions
  11  * are met:
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions and the following disclaimer.
  14  * 2. Redistributions in binary form must reproduce the above copyright
  15  *    notice, this list of conditions and the following disclaimer in the
  16  *    documentation and/or other materials provided with the distribution.
  17  *
  18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  21  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  28  * SUCH DAMAGE.
  29  */
  30
  31 /*
  32  * USB Universal Host Controller driver.
  33  * Handles e.g. PIIX3 and PIIX4.
  34  *
  35  * UHCI spec: http://developer.intel.com/design/USB/UHCI11D.htm
  36  * USB spec:  http://www.usb.org/developers/docs/usbspec.zip
  37  * PIIXn spec: ftp://download.intel.com/design/intarch/datashts/29055002.pdf
  38  *             ftp://download.intel.com/design/intarch/datashts/29056201.pdf
  39  */
  40
  41 #ifdef USB_GLOBAL_INCLUDE_FILE
  42 #include USB_GLOBAL_INCLUDE_FILE
  43 #else
  44 #include <sys/stdint.h>
  45 #include <sys/stddef.h>
  46 #include <sys/param.h>
  47 #include <sys/queue.h>
  48 #include <sys/types.h>
  49 #include <sys/systm.h>
  50 #include <sys/kernel.h>
  51 #include <sys/bus.h>
  52 #include <sys/module.h>
  53 #include <sys/lock.h>
  54 #include <sys/mutex.h>
  55 #include <sys/condvar.h>
  56 #include <sys/sysctl.h>
  57 #include <sys/sx.h>
  58 #include <sys/unistd.h>
  59 #include <sys/callout.h>
  60 #include <sys/malloc.h>
  61 #include <sys/priv.h>
  62
  63 #include <dev/usb/usb.h>
  64 #include <dev/usb/usbdi.h>
  65
  66 #define USB_DEBUG_VAR uhcidebug
  67
  68 #include <dev/usb/usb_core.h>
  69 #include <dev/usb/usb_debug.h>
  70 #include <dev/usb/usb_busdma.h>
  71 #include <dev/usb/usb_process.h>
  72 #include <dev/usb/usb_transfer.h>
  73 #include <dev/usb/usb_device.h>
  74 #include <dev/usb/usb_hub.h>
  75 #include <dev/usb/usb_util.h>
  76
  77 #include <dev/usb/usb_controller.h>
  78 #include <dev/usb/usb_bus.h>
  79 #endif                  /* USB_GLOBAL_INCLUDE_FILE */
  80
  81 #include <dev/usb/controller/uhci.h>
  82 #include <dev/usb/controller/uhcireg.h>
  83
  84 #define alt_next next
  85 #define UHCI_BUS2SC(bus) \
  86    ((uhci_softc_t *)(((uint8_t *)(bus)) - \
  87     ((uint8_t *)&(((uhci_softc_t *)0)->sc_bus))))
  88
  89 #ifdef USB_DEBUG
  90 static int uhcidebug = 0;
  91 static int uhcinoloop = 0;
  92
  93 static SYSCTL_NODE(_hw_usb, OID_AUTO, uhci, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
  94     "USB uhci");
  95 SYSCTL_INT(_hw_usb_uhci, OID_AUTO, debug, CTLFLAG_RWTUN,
  96     &uhcidebug, 0, "uhci debug level");
  97 SYSCTL_INT(_hw_usb_uhci, OID_AUTO, loop, CTLFLAG_RWTUN,
  98     &uhcinoloop, 0, "uhci noloop");
  99
 100 static void uhci_dumpregs(uhci_softc_t *sc);
 101 static void uhci_dump_tds(uhci_td_t *td);
 102
 103 #endif
 104
 105 #define UBARR(sc) bus_space_barrier((sc)->sc_io_tag, (sc)->sc_io_hdl, 0, (sc)->sc_io_size, \
 106                         BUS_SPACE_BARRIER_READ|BUS_SPACE_BARRIER_WRITE)
 107 #define UWRITE1(sc, r, x) \
 108  do { UBARR(sc); bus_space_write_1((sc)->sc_io_tag, (sc)->sc_io_hdl, (r), (x)); \
 109  } while (/*CONSTCOND*/0)
 110 #define UWRITE2(sc, r, x) \
 111  do { UBARR(sc); bus_space_write_2((sc)->sc_io_tag, (sc)->sc_io_hdl, (r), (x)); \
 112  } while (/*CONSTCOND*/0)
 113 #define UWRITE4(sc, r, x) \
 114  do { UBARR(sc); bus_space_write_4((sc)->sc_io_tag, (sc)->sc_io_hdl, (r), (x)); \
 115  } while (/*CONSTCOND*/0)
 116 #define UREAD1(sc, r) (UBARR(sc), bus_space_read_1((sc)->sc_io_tag, (sc)->sc_io_hdl, (r)))
 117 #define UREAD2(sc, r) (UBARR(sc), bus_space_read_2((sc)->sc_io_tag, (sc)->sc_io_hdl, (r)))
 118 #define UREAD4(sc, r) (UBARR(sc), bus_space_read_4((sc)->sc_io_tag, (sc)->sc_io_hdl, (r)))
 119
 120 #define UHCICMD(sc, cmd) UWRITE2(sc, UHCI_CMD, cmd)
 121 #define UHCISTS(sc) UREAD2(sc, UHCI_STS)
 122
 123 #define UHCI_RESET_TIMEOUT 100          /* ms, reset timeout */
 124
 125 #define UHCI_INTR_ENDPT 1
 126
 127 struct uhci_mem_layout {
 128
 129         struct usb_page_search buf_res;
 130         struct usb_page_search fix_res;
 131
 132         struct usb_page_cache *buf_pc;
 133         struct usb_page_cache *fix_pc;
 134
 135         uint32_t buf_offset;
 136
 137         uint16_t max_frame_size;
 138 };
 139
 140 struct uhci_std_temp {
 141
 142         struct uhci_mem_layout ml;
 143         uhci_td_t *td;
 144         uhci_td_t *td_next;
 145         uint32_t average;
 146         uint32_t td_status;
 147         uint32_t td_token;
 148         uint32_t len;
 149         uint16_t max_frame_size;
 150         uint8_t shortpkt;
 151         uint8_t setup_alt_next;
 152         uint8_t last_frame;
 153 };
 154
 155 static const struct usb_bus_methods uhci_bus_methods;
 156 static const struct usb_pipe_methods uhci_device_bulk_methods;
 157 static const struct usb_pipe_methods uhci_device_ctrl_methods;
 158 static const struct usb_pipe_methods uhci_device_intr_methods;
 159 static const struct usb_pipe_methods uhci_device_isoc_methods;
 160
 161 static uint8_t  uhci_restart(uhci_softc_t *sc);
 162 static void     uhci_do_poll(struct usb_bus *);
 163 static void     uhci_device_done(struct usb_xfer *, usb_error_t);
 164 static void     uhci_transfer_intr_enqueue(struct usb_xfer *);
 165 static void     uhci_timeout(void *);
 166 static uint8_t  uhci_check_transfer(struct usb_xfer *);
 167 static void     uhci_root_intr(uhci_softc_t *sc);
 168
 169 void
 170 uhci_iterate_hw_softc(struct usb_bus *bus, usb_bus_mem_sub_cb_t *cb)
 171 {
 172         struct uhci_softc *sc = UHCI_BUS2SC(bus);
 173         uint32_t i;
 174
 175         cb(bus, &sc->sc_hw.pframes_pc, &sc->sc_hw.pframes_pg,
 176             sizeof(uint32_t) * UHCI_FRAMELIST_COUNT, UHCI_FRAMELIST_ALIGN);
 177
 178         cb(bus, &sc->sc_hw.ls_ctl_start_pc, &sc->sc_hw.ls_ctl_start_pg,
 179             sizeof(uhci_qh_t), UHCI_QH_ALIGN);
 180
 181         cb(bus, &sc->sc_hw.fs_ctl_start_pc, &sc->sc_hw.fs_ctl_start_pg,
 182             sizeof(uhci_qh_t), UHCI_QH_ALIGN);
 183
 184         cb(bus, &sc->sc_hw.bulk_start_pc, &sc->sc_hw.bulk_start_pg,
 185             sizeof(uhci_qh_t), UHCI_QH_ALIGN);
 186
 187         cb(bus, &sc->sc_hw.last_qh_pc, &sc->sc_hw.last_qh_pg,
 188             sizeof(uhci_qh_t), UHCI_QH_ALIGN);
 189
 190         cb(bus, &sc->sc_hw.last_td_pc, &sc->sc_hw.last_td_pg,
 191             sizeof(uhci_td_t), UHCI_TD_ALIGN);
 192
 193         for (i = 0; i != UHCI_VFRAMELIST_COUNT; i++) {
 194                 cb(bus, sc->sc_hw.isoc_start_pc + i,
 195                     sc->sc_hw.isoc_start_pg + i,
 196                     sizeof(uhci_td_t), UHCI_TD_ALIGN);
 197         }
 198
 199         for (i = 0; i != UHCI_IFRAMELIST_COUNT; i++) {
 200                 cb(bus, sc->sc_hw.intr_start_pc + i,
 201                     sc->sc_hw.intr_start_pg + i,
 202                     sizeof(uhci_qh_t), UHCI_QH_ALIGN);
 203         }
 204 }
 205
 206 static void
 207 uhci_mem_layout_init(struct uhci_mem_layout *ml, struct usb_xfer *xfer)
 208 {
 209         ml->buf_pc = xfer->frbuffers + 0;
 210         ml->fix_pc = xfer->buf_fixup;
 211
 212         ml->buf_offset = 0;
 213
 214         ml->max_frame_size = xfer->max_frame_size;
 215 }
 216
 217 static void
 218 uhci_mem_layout_fixup(struct uhci_mem_layout *ml, struct uhci_td *td)
 219 {
 220         usbd_get_page(ml->buf_pc, ml->buf_offset, &ml->buf_res);
 221
 222         if (ml->buf_res.length < td->len) {
 223
 224                 /* need to do a fixup */
 225
 226                 usbd_get_page(ml->fix_pc, 0, &ml->fix_res);
 227
 228                 td->td_buffer = htole32(ml->fix_res.physaddr);
 229
 230                 /*
 231                  * The UHCI driver cannot handle
 232                  * page crossings, so a fixup is
 233                  * needed:
 234                  *
 235                  *  +----+----+ - - -
 236                  *  | YYY|Y   |
 237                  *  +----+----+ - - -
 238                  *     \    \
 239                  *      \    \
 240                  *       +----+
 241                  *       |YYYY|  (fixup)
 242                  *       +----+
 243                  */
 244
 245                 if ((td->td_token & htole32(UHCI_TD_PID)) ==
 246                     htole32(UHCI_TD_PID_IN)) {
 247                         td->fix_pc = ml->fix_pc;
 248                         usb_pc_cpu_invalidate(ml->fix_pc);
 249
 250                 } else {
 251                         td->fix_pc = NULL;
 252
 253                         /* copy data to fixup location */
 254
 255                         usbd_copy_out(ml->buf_pc, ml->buf_offset,
 256                             ml->fix_res.buffer, td->len);
 257
 258                         usb_pc_cpu_flush(ml->fix_pc);
 259                 }
 260
 261                 /* prepare next fixup */
 262
 263                 ml->fix_pc++;
 264
 265         } else {
 266
 267                 td->td_buffer = htole32(ml->buf_res.physaddr);
 268                 td->fix_pc = NULL;
 269         }
 270
 271         /* prepare next data location */
 272
 273         ml->buf_offset += td->len;
 274 }
 275
 276 /*
 277  * Return values:
 278  * 0: Success
 279  * Else: Failure
 280  */
 281 static uint8_t
 282 uhci_restart(uhci_softc_t *sc)
 283 {
 284         struct usb_page_search buf_res;
 285
 286         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
 287
 288         if (UREAD2(sc, UHCI_CMD) & UHCI_CMD_RS) {
 289                 DPRINTFN(2, "Already started\n");
 290                 return (0);
 291         }
 292
 293         DPRINTFN(2, "Restarting\n");
 294
 295         usbd_get_page(&sc->sc_hw.pframes_pc, 0, &buf_res);
 296
 297         /* Reload fresh base address */
 298         UWRITE4(sc, UHCI_FLBASEADDR, buf_res.physaddr);
 299
 300         /*
 301          * Assume 64 byte packets at frame end and start HC controller:
 302          */
 303         UHCICMD(sc, (UHCI_CMD_MAXP | UHCI_CMD_RS));
 304
 305         /* wait 10 milliseconds */
 306
 307         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 100);
 308
 309         /* check that controller has started */
 310
 311         if (UREAD2(sc, UHCI_STS) & UHCI_STS_HCH) {
 312                 DPRINTFN(2, "Failed\n");
 313                 return (1);
 314         }
 315         return (0);
 316 }
 317
 318 void
 319 uhci_reset(uhci_softc_t *sc)
 320 {
 321         uint16_t n;
 322
 323         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
 324
 325         DPRINTF("resetting the HC\n");
 326
 327         /* disable interrupts */
 328
 329         UWRITE2(sc, UHCI_INTR, 0);
 330
 331         /* global reset */
 332
 333         UHCICMD(sc, UHCI_CMD_GRESET);
 334
 335         /* wait */
 336
 337         usb_pause_mtx(&sc->sc_bus.bus_mtx,
 338             USB_MS_TO_TICKS(USB_BUS_RESET_DELAY));
 339
 340         /* terminate all transfers */
 341
 342         UHCICMD(sc, UHCI_CMD_HCRESET);
 343
 344         /* the reset bit goes low when the controller is done */
 345
 346         n = UHCI_RESET_TIMEOUT;
 347         while (n--) {
 348                 /* wait one millisecond */
 349
 350                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 1000);
 351
 352                 if (!(UREAD2(sc, UHCI_CMD) & UHCI_CMD_HCRESET)) {
 353                         goto done_1;
 354                 }
 355         }
 356
 357         device_printf(sc->sc_bus.bdev,
 358             "controller did not reset\n");
 359
 360 done_1:
 361
 362         n = 10;
 363         while (n--) {
 364                 /* wait one millisecond */
 365
 366                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 1000);
 367
 368                 /* check if HC is stopped */
 369                 if (UREAD2(sc, UHCI_STS) & UHCI_STS_HCH) {
 370                         goto done_2;
 371                 }
 372         }
 373
 374         device_printf(sc->sc_bus.bdev,
 375             "controller did not stop\n");
 376
 377 done_2:
 378
 379         /* reset frame number */
 380         UWRITE2(sc, UHCI_FRNUM, 0);
 381         /* set default SOF value */
 382         UWRITE1(sc, UHCI_SOF, 0x40);
 383
 384         USB_BUS_UNLOCK(&sc->sc_bus);
 385
 386         /* stop root interrupt */
 387         usb_callout_drain(&sc->sc_root_intr);
 388
 389         USB_BUS_LOCK(&sc->sc_bus);
 390 }
 391
 392 static void
 393 uhci_start(uhci_softc_t *sc)
 394 {
 395         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
 396
 397         DPRINTFN(2, "enabling\n");
 398
 399         /* enable interrupts */
 400
 401         UWRITE2(sc, UHCI_INTR,
 402             (UHCI_INTR_TOCRCIE |
 403             UHCI_INTR_RIE |
 404             UHCI_INTR_IOCE |
 405             UHCI_INTR_SPIE));
 406
 407         if (uhci_restart(sc)) {
 408                 device_printf(sc->sc_bus.bdev,
 409                     "cannot start HC controller\n");
 410         }
 411
 412         /* start root interrupt */
 413         uhci_root_intr(sc);
 414 }
 415
 416 static struct uhci_qh *
 417 uhci_init_qh(struct usb_page_cache *pc)
 418 {
 419         struct usb_page_search buf_res;
 420         struct uhci_qh *qh;
 421
 422         usbd_get_page(pc, 0, &buf_res);
 423
 424         qh = buf_res.buffer;
 425
 426         qh->qh_self =
 427             htole32(buf_res.physaddr) |
 428             htole32(UHCI_PTR_QH);
 429
 430         qh->page_cache = pc;
 431
 432         return (qh);
 433 }
 434
 435 static struct uhci_td *
 436 uhci_init_td(struct usb_page_cache *pc)
 437 {
 438         struct usb_page_search buf_res;
 439         struct uhci_td *td;
 440
 441         usbd_get_page(pc, 0, &buf_res);
 442
 443         td = buf_res.buffer;
 444
 445         td->td_self =
 446             htole32(buf_res.physaddr) |
 447             htole32(UHCI_PTR_TD);
 448
 449         td->page_cache = pc;
 450
 451         return (td);
 452 }
 453
 454 usb_error_t
 455 uhci_init(uhci_softc_t *sc)
 456 {
 457         uint16_t bit;
 458         uint16_t x;
 459         uint16_t y;
 460
 461         DPRINTF("start\n");
 462
 463         usb_callout_init_mtx(&sc->sc_root_intr, &sc->sc_bus.bus_mtx, 0);
 464
 465 #ifdef USB_DEBUG
 466         if (uhcidebug > 2) {
 467                 uhci_dumpregs(sc);
 468         }
 469 #endif
 470         /*
 471          * Setup QH's
 472          */
 473         sc->sc_ls_ctl_p_last =
 474             uhci_init_qh(&sc->sc_hw.ls_ctl_start_pc);
 475
 476         sc->sc_fs_ctl_p_last =
 477             uhci_init_qh(&sc->sc_hw.fs_ctl_start_pc);
 478
 479         sc->sc_bulk_p_last =
 480             uhci_init_qh(&sc->sc_hw.bulk_start_pc);
 481 #if 0
 482         sc->sc_reclaim_qh_p =
 483             sc->sc_fs_ctl_p_last;
 484 #else
 485         /* setup reclaim looping point */
 486         sc->sc_reclaim_qh_p =
 487             sc->sc_bulk_p_last;
 488 #endif
 489
 490         sc->sc_last_qh_p =
 491             uhci_init_qh(&sc->sc_hw.last_qh_pc);
 492
 493         sc->sc_last_td_p =
 494             uhci_init_td(&sc->sc_hw.last_td_pc);
 495
 496         for (x = 0; x != UHCI_VFRAMELIST_COUNT; x++) {
 497                 sc->sc_isoc_p_last[x] =
 498                     uhci_init_td(sc->sc_hw.isoc_start_pc + x);
 499         }
 500
 501         for (x = 0; x != UHCI_IFRAMELIST_COUNT; x++) {
 502                 sc->sc_intr_p_last[x] =
 503                     uhci_init_qh(sc->sc_hw.intr_start_pc + x);
 504         }
 505
 506         /*
 507          * the QHs are arranged to give poll intervals that are
 508          * powers of 2 times 1ms
 509          */
 510         bit = UHCI_IFRAMELIST_COUNT / 2;
 511         while (bit) {
 512                 x = bit;
 513                 while (x & bit) {
 514                         uhci_qh_t *qh_x;
 515                         uhci_qh_t *qh_y;
 516
 517                         y = (x ^ bit) | (bit / 2);
 518
 519                         /*
 520                          * the next QH has half the poll interval
 521                          */
 522                         qh_x = sc->sc_intr_p_last[x];
 523                         qh_y = sc->sc_intr_p_last[y];
 524
 525                         qh_x->h_next = NULL;
 526                         qh_x->qh_h_next = qh_y->qh_self;
 527                         qh_x->e_next = NULL;
 528                         qh_x->qh_e_next = htole32(UHCI_PTR_T);
 529                         x++;
 530                 }
 531                 bit >>= 1;
 532         }
 533
 534         if (1) {
 535                 uhci_qh_t *qh_ls;
 536                 uhci_qh_t *qh_intr;
 537
 538                 qh_ls = sc->sc_ls_ctl_p_last;
 539                 qh_intr = sc->sc_intr_p_last[0];
 540
 541                 /* start QH for interrupt traffic */
 542                 qh_intr->h_next = qh_ls;
 543                 qh_intr->qh_h_next = qh_ls->qh_self;
 544                 qh_intr->e_next = 0;
 545                 qh_intr->qh_e_next = htole32(UHCI_PTR_T);
 546         }
 547         for (x = 0; x != UHCI_VFRAMELIST_COUNT; x++) {
 548
 549                 uhci_td_t *td_x;
 550                 uhci_qh_t *qh_intr;
 551
 552                 td_x = sc->sc_isoc_p_last[x];
 553                 qh_intr = sc->sc_intr_p_last[x | (UHCI_IFRAMELIST_COUNT / 2)];
 554
 555                 /* start TD for isochronous traffic */
 556                 td_x->next = NULL;
 557                 td_x->td_next = qh_intr->qh_self;
 558                 td_x->td_status = htole32(UHCI_TD_IOS);
 559                 td_x->td_token = htole32(0);
 560                 td_x->td_buffer = htole32(0);
 561         }
 562
 563         if (1) {
 564                 uhci_qh_t *qh_ls;
 565                 uhci_qh_t *qh_fs;
 566
 567                 qh_ls = sc->sc_ls_ctl_p_last;
 568                 qh_fs = sc->sc_fs_ctl_p_last;
 569
 570                 /* start QH where low speed control traffic will be queued */
 571                 qh_ls->h_next = qh_fs;
 572                 qh_ls->qh_h_next = qh_fs->qh_self;
 573                 qh_ls->e_next = 0;
 574                 qh_ls->qh_e_next = htole32(UHCI_PTR_T);
 575         }
 576         if (1) {
 577                 uhci_qh_t *qh_ctl;
 578                 uhci_qh_t *qh_blk;
 579                 uhci_qh_t *qh_lst;
 580                 uhci_td_t *td_lst;
 581
 582                 qh_ctl = sc->sc_fs_ctl_p_last;
 583                 qh_blk = sc->sc_bulk_p_last;
 584
 585                 /* start QH where full speed control traffic will be queued */
 586                 qh_ctl->h_next = qh_blk;
 587                 qh_ctl->qh_h_next = qh_blk->qh_self;
 588                 qh_ctl->e_next = 0;
 589                 qh_ctl->qh_e_next = htole32(UHCI_PTR_T);
 590
 591                 qh_lst = sc->sc_last_qh_p;
 592
 593                 /* start QH where bulk traffic will be queued */
 594                 qh_blk->h_next = qh_lst;
 595                 qh_blk->qh_h_next = qh_lst->qh_self;
 596                 qh_blk->e_next = 0;
 597                 qh_blk->qh_e_next = htole32(UHCI_PTR_T);
 598
 599                 td_lst = sc->sc_last_td_p;
 600
 601                 /* end QH which is used for looping the QHs */
 602                 qh_lst->h_next = 0;
 603                 qh_lst->qh_h_next = htole32(UHCI_PTR_T);        /* end of QH chain */
 604                 qh_lst->e_next = td_lst;
 605                 qh_lst->qh_e_next = td_lst->td_self;
 606
 607                 /*
 608                  * end TD which hangs from the last QH, to avoid a bug in the PIIX
 609                  * that makes it run berserk otherwise
 610                  */
 611                 td_lst->next = 0;
 612                 td_lst->td_next = htole32(UHCI_PTR_T);
 613                 td_lst->td_status = htole32(0); /* inactive */
 614                 td_lst->td_token = htole32(0);
 615                 td_lst->td_buffer = htole32(0);
 616         }
 617         if (1) {
 618                 struct usb_page_search buf_res;
 619                 uint32_t *pframes;
 620
 621                 usbd_get_page(&sc->sc_hw.pframes_pc, 0, &buf_res);
 622
 623                 pframes = buf_res.buffer;
 624
 625
 626                 /*
 627                  * Setup UHCI framelist
 628                  *
 629                  * Execution order:
 630                  *
 631                  * pframes -> full speed isochronous -> interrupt QH's -> low
 632                  * speed control -> full speed control -> bulk transfers
 633                  *
 634                  */
 635
 636                 for (x = 0; x != UHCI_FRAMELIST_COUNT; x++) {
 637                         pframes[x] =
 638                             sc->sc_isoc_p_last[x % UHCI_VFRAMELIST_COUNT]->td_self;
 639                 }
 640         }
 641         /* flush all cache into memory */
 642
 643         usb_bus_mem_flush_all(&sc->sc_bus, &uhci_iterate_hw_softc);
 644
 645         /* set up the bus struct */
 646         sc->sc_bus.methods = &uhci_bus_methods;
 647
 648         USB_BUS_LOCK(&sc->sc_bus);
 649         /* reset the controller */
 650         uhci_reset(sc);
 651
 652         /* start the controller */
 653         uhci_start(sc);
 654         USB_BUS_UNLOCK(&sc->sc_bus);
 655
 656         /* catch lost interrupts */
 657         uhci_do_poll(&sc->sc_bus);
 658
 659         return (0);
 660 }
 661
 662 static void
 663 uhci_suspend(uhci_softc_t *sc)
 664 {
 665 #ifdef USB_DEBUG
 666         if (uhcidebug > 2) {
 667                 uhci_dumpregs(sc);
 668         }
 669 #endif
 670
 671         USB_BUS_LOCK(&sc->sc_bus);
 672
 673         /* stop the controller */
 674
 675         uhci_reset(sc);
 676
 677         /* enter global suspend */
 678
 679         UHCICMD(sc, UHCI_CMD_EGSM);
 680
 681         USB_BUS_UNLOCK(&sc->sc_bus);
 682 }
 683
 684 static void
 685 uhci_resume(uhci_softc_t *sc)
 686 {
 687         USB_BUS_LOCK(&sc->sc_bus);
 688
 689         /* reset the controller */
 690
 691         uhci_reset(sc);
 692
 693         /* force global resume */
 694
 695         UHCICMD(sc, UHCI_CMD_FGR);
 696
 697         /* and start traffic again */
 698
 699         uhci_start(sc);
 700
 701         USB_BUS_UNLOCK(&sc->sc_bus);
 702
 703 #ifdef USB_DEBUG
 704         if (uhcidebug > 2)
 705                 uhci_dumpregs(sc);
 706 #endif
 707
 708         /* catch lost interrupts */
 709         uhci_do_poll(&sc->sc_bus);
 710 }
 711
 712 #ifdef USB_DEBUG
 713 static void
 714 uhci_dumpregs(uhci_softc_t *sc)
 715 {
 716         DPRINTFN(0, "%s regs: cmd=%04x, sts=%04x, intr=%04x, frnum=%04x, "
 717             "flbase=%08x, sof=%04x, portsc1=%04x, portsc2=%04x\n",
 718             device_get_nameunit(sc->sc_bus.bdev),
 719             UREAD2(sc, UHCI_CMD),
 720             UREAD2(sc, UHCI_STS),
 721             UREAD2(sc, UHCI_INTR),
 722             UREAD2(sc, UHCI_FRNUM),
 723             UREAD4(sc, UHCI_FLBASEADDR),
 724             UREAD1(sc, UHCI_SOF),
 725             UREAD2(sc, UHCI_PORTSC1),
 726             UREAD2(sc, UHCI_PORTSC2));
 727 }
 728
 729 static uint8_t
 730 uhci_dump_td(uhci_td_t *p)
 731 {
 732         uint32_t td_next;
 733         uint32_t td_status;
 734         uint32_t td_token;
 735         uint8_t temp;
 736
 737         usb_pc_cpu_invalidate(p->page_cache);
 738
 739         td_next = le32toh(p->td_next);
 740         td_status = le32toh(p->td_status);
 741         td_token = le32toh(p->td_token);
 742
 743         /*
 744          * Check whether the link pointer in this TD marks the link pointer
 745          * as end of queue:
 746          */
 747         temp = ((td_next & UHCI_PTR_T) || (td_next == 0));
 748
 749         printf("TD(%p) at 0x%08x = link=0x%08x status=0x%08x "
 750             "token=0x%08x buffer=0x%08x\n",
 751             p,
 752             le32toh(p->td_self),
 753             td_next,
 754             td_status,
 755             td_token,
 756             le32toh(p->td_buffer));
 757
 758         printf("TD(%p) td_next=%s%s%s td_status=%s%s%s%s%s%s%s%s%s%s%s, errcnt=%d, actlen=%d pid=%02x,"
 759             "addr=%d,endpt=%d,D=%d,maxlen=%d\n",
 760             p,
 761             (td_next & 1) ? "-T" : "",
 762             (td_next & 2) ? "-Q" : "",
 763             (td_next & 4) ? "-VF" : "",
 764             (td_status & UHCI_TD_BITSTUFF) ? "-BITSTUFF" : "",
 765             (td_status & UHCI_TD_CRCTO) ? "-CRCTO" : "",
 766             (td_status & UHCI_TD_NAK) ? "-NAK" : "",
 767             (td_status & UHCI_TD_BABBLE) ? "-BABBLE" : "",
 768             (td_status & UHCI_TD_DBUFFER) ? "-DBUFFER" : "",
 769             (td_status & UHCI_TD_STALLED) ? "-STALLED" : "",
 770             (td_status & UHCI_TD_ACTIVE) ? "-ACTIVE" : "",
 771             (td_status & UHCI_TD_IOC) ? "-IOC" : "",
 772             (td_status & UHCI_TD_IOS) ? "-IOS" : "",
 773             (td_status & UHCI_TD_LS) ? "-LS" : "",
 774             (td_status & UHCI_TD_SPD) ? "-SPD" : "",
 775             UHCI_TD_GET_ERRCNT(td_status),
 776             UHCI_TD_GET_ACTLEN(td_status),
 777             UHCI_TD_GET_PID(td_token),
 778             UHCI_TD_GET_DEVADDR(td_token),
 779             UHCI_TD_GET_ENDPT(td_token),
 780             UHCI_TD_GET_DT(td_token),
 781             UHCI_TD_GET_MAXLEN(td_token));
 782
 783         return (temp);
 784 }
 785
 786 static uint8_t
 787 uhci_dump_qh(uhci_qh_t *sqh)
 788 {
 789         uint8_t temp;
 790         uint32_t qh_h_next;
 791         uint32_t qh_e_next;
 792
 793         usb_pc_cpu_invalidate(sqh->page_cache);
 794
 795         qh_h_next = le32toh(sqh->qh_h_next);
 796         qh_e_next = le32toh(sqh->qh_e_next);
 797
 798         DPRINTFN(0, "QH(%p) at 0x%08x: h_next=0x%08x e_next=0x%08x\n", sqh,
 799             le32toh(sqh->qh_self), qh_h_next, qh_e_next);
 800
 801         temp = ((((sqh->h_next != NULL) && !(qh_h_next & UHCI_PTR_T)) ? 1 : 0) |
 802             (((sqh->e_next != NULL) && !(qh_e_next & UHCI_PTR_T)) ? 2 : 0));
 803
 804         return (temp);
 805 }
 806
 807 static void
 808 uhci_dump_all(uhci_softc_t *sc)
 809 {
 810         uhci_dumpregs(sc);
 811         uhci_dump_qh(sc->sc_ls_ctl_p_last);
 812         uhci_dump_qh(sc->sc_fs_ctl_p_last);
 813         uhci_dump_qh(sc->sc_bulk_p_last);
 814         uhci_dump_qh(sc->sc_last_qh_p);
 815 }
 816
 817 static void
 818 uhci_dump_tds(uhci_td_t *td)
 819 {
 820         for (;
 821             td != NULL;
 822             td = td->obj_next) {
 823                 if (uhci_dump_td(td)) {
 824                         break;
 825                 }
 826         }
 827 }
 828
 829 #endif
 830
 831 /*
 832  * Let the last QH loop back to the full speed control transfer QH.
 833  * This is what intel calls "bandwidth reclamation" and improves
 834  * USB performance a lot for some devices.
 835  * If we are already looping, just count it.
 836  */
 837 static void
 838 uhci_add_loop(uhci_softc_t *sc)
 839 {
 840         struct uhci_qh *qh_lst;
 841         struct uhci_qh *qh_rec;
 842
 843 #ifdef USB_DEBUG
 844         if (uhcinoloop) {
 845                 return;
 846         }
 847 #endif
 848         if (++(sc->sc_loops) == 1) {
 849                 DPRINTFN(6, "add\n");
 850
 851                 qh_lst = sc->sc_last_qh_p;
 852                 qh_rec = sc->sc_reclaim_qh_p;
 853
 854                 /* NOTE: we don't loop back the soft pointer */
 855
 856                 qh_lst->qh_h_next = qh_rec->qh_self;
 857                 usb_pc_cpu_flush(qh_lst->page_cache);
 858         }
 859 }
 860
 861 static void
 862 uhci_rem_loop(uhci_softc_t *sc)
 863 {
 864         struct uhci_qh *qh_lst;
 865
 866 #ifdef USB_DEBUG
 867         if (uhcinoloop) {
 868                 return;
 869         }
 870 #endif
 871         if (--(sc->sc_loops) == 0) {
 872                 DPRINTFN(6, "remove\n");
 873
 874                 qh_lst = sc->sc_last_qh_p;
 875                 qh_lst->qh_h_next = htole32(UHCI_PTR_T);
 876                 usb_pc_cpu_flush(qh_lst->page_cache);
 877         }
 878 }
 879
 880 static void
 881 uhci_transfer_intr_enqueue(struct usb_xfer *xfer)
 882 {
 883         /* check for early completion */
 884         if (uhci_check_transfer(xfer)) {
 885                 return;
 886         }
 887         /* put transfer on interrupt queue */
 888         usbd_transfer_enqueue(&xfer->xroot->bus->intr_q, xfer);
 889
 890         /* start timeout, if any */
 891         if (xfer->timeout != 0) {
 892                 usbd_transfer_timeout_ms(xfer, &uhci_timeout, xfer->timeout);
 893         }
 894 }
 895
 896 #define UHCI_APPEND_TD(std,last) (last) = _uhci_append_td(std,last)
 897 static uhci_td_t *
 898 _uhci_append_td(uhci_td_t *std, uhci_td_t *last)
 899 {
 900         DPRINTFN(11, "%p to %p\n", std, last);
 901
 902         /* (sc->sc_bus.mtx) must be locked */
 903
 904         std->next = last->next;
 905         std->td_next = last->td_next;
 906
 907         std->prev = last;
 908
 909         usb_pc_cpu_flush(std->page_cache);
 910
 911         /*
 912          * the last->next->prev is never followed: std->next->prev = std;
 913          */
 914         last->next = std;
 915         last->td_next = std->td_self;
 916
 917         usb_pc_cpu_flush(last->page_cache);
 918
 919         return (std);
 920 }
 921
 922 #define UHCI_APPEND_QH(sqh,last) (last) = _uhci_append_qh(sqh,last)
 923 static uhci_qh_t *
 924 _uhci_append_qh(uhci_qh_t *sqh, uhci_qh_t *last)
 925 {
 926         DPRINTFN(11, "%p to %p\n", sqh, last);
 927
 928         if (sqh->h_prev != NULL) {
 929                 /* should not happen */
 930                 DPRINTFN(0, "QH already linked!\n");
 931                 return (last);
 932         }
 933         /* (sc->sc_bus.mtx) must be locked */
 934
 935         sqh->h_next = last->h_next;
 936         sqh->qh_h_next = last->qh_h_next;
 937
 938         sqh->h_prev = last;
 939
 940         usb_pc_cpu_flush(sqh->page_cache);
 941
 942         /*
 943          * The "last->h_next->h_prev" is never followed:
 944          *
 945          * "sqh->h_next->h_prev" = sqh;
 946          */
 947
 948         last->h_next = sqh;
 949         last->qh_h_next = sqh->qh_self;
 950
 951         usb_pc_cpu_flush(last->page_cache);
 952
 953         return (sqh);
 954 }
 955
 956 /**/
 957
 958 #define UHCI_REMOVE_TD(std,last) (last) = _uhci_remove_td(std,last)
 959 static uhci_td_t *
 960 _uhci_remove_td(uhci_td_t *std, uhci_td_t *last)
 961 {
 962         DPRINTFN(11, "%p from %p\n", std, last);
 963
 964         /* (sc->sc_bus.mtx) must be locked */
 965
 966         std->prev->next = std->next;
 967         std->prev->td_next = std->td_next;
 968
 969         usb_pc_cpu_flush(std->prev->page_cache);
 970
 971         if (std->next) {
 972                 std->next->prev = std->prev;
 973                 usb_pc_cpu_flush(std->next->page_cache);
 974         }
 975         return ((last == std) ? std->prev : last);
 976 }
 977
 978 #define UHCI_REMOVE_QH(sqh,last) (last) = _uhci_remove_qh(sqh,last)
 979 static uhci_qh_t *
 980 _uhci_remove_qh(uhci_qh_t *sqh, uhci_qh_t *last)
 981 {
 982         DPRINTFN(11, "%p from %p\n", sqh, last);
 983
 984         /* (sc->sc_bus.mtx) must be locked */
 985
 986         /* only remove if not removed from a queue */
 987         if (sqh->h_prev) {
 988
 989                 sqh->h_prev->h_next = sqh->h_next;
 990                 sqh->h_prev->qh_h_next = sqh->qh_h_next;
 991
 992                 usb_pc_cpu_flush(sqh->h_prev->page_cache);
 993
 994                 if (sqh->h_next) {
 995                         sqh->h_next->h_prev = sqh->h_prev;
 996                         usb_pc_cpu_flush(sqh->h_next->page_cache);
 997                 }
 998                 last = ((last == sqh) ? sqh->h_prev : last);
 999
1000                 sqh->h_prev = 0;
1001
1002                 usb_pc_cpu_flush(sqh->page_cache);
1003         }
1004         return (last);
1005 }
1006
1007 static void
1008 uhci_isoc_done(uhci_softc_t *sc, struct usb_xfer *xfer)
1009 {
1010         struct usb_page_search res;
1011         uint32_t nframes = xfer->nframes;
1012         uint32_t status;
1013         uint32_t offset = 0;
1014         uint32_t *plen = xfer->frlengths;
1015         uint16_t len = 0;
1016         uhci_td_t *td = xfer->td_transfer_first;
1017         uhci_td_t **pp_last = &sc->sc_isoc_p_last[xfer->qh_pos];
1018
1019         DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
1020             xfer, xfer->endpoint);
1021
1022         /* sync any DMA memory before doing fixups */
1023
1024         usb_bdma_post_sync(xfer);
1025
1026         while (nframes--) {
1027                 if (td == NULL) {
1028                         panic("%s:%d: out of TD's\n",
1029                             __FUNCTION__, __LINE__);
1030                 }
1031                 if (pp_last >= &sc->sc_isoc_p_last[UHCI_VFRAMELIST_COUNT]) {
1032                         pp_last = &sc->sc_isoc_p_last[0];
1033                 }
1034 #ifdef USB_DEBUG
1035                 if (uhcidebug > 5) {
1036                         DPRINTF("isoc TD\n");
1037                         uhci_dump_td(td);
1038                 }
1039 #endif
1040                 usb_pc_cpu_invalidate(td->page_cache);
1041                 status = le32toh(td->td_status);
1042
1043                 len = UHCI_TD_GET_ACTLEN(status);
1044
1045                 if (len > *plen) {
1046                         len = *plen;
1047                 }
1048                 if (td->fix_pc) {
1049
1050                         usbd_get_page(td->fix_pc, 0, &res);
1051
1052                         /* copy data from fixup location to real location */
1053
1054                         usb_pc_cpu_invalidate(td->fix_pc);
1055
1056                         usbd_copy_in(xfer->frbuffers, offset,
1057                             res.buffer, len);
1058                 }
1059                 offset += *plen;
1060
1061                 *plen = len;
1062
1063                 /* remove TD from schedule */
1064                 UHCI_REMOVE_TD(td, *pp_last);
1065
1066                 pp_last++;
1067                 plen++;
1068                 td = td->obj_next;
1069         }
1070
1071         xfer->aframes = xfer->nframes;
1072 }
1073
1074 static usb_error_t
1075 uhci_non_isoc_done_sub(struct usb_xfer *xfer)
1076 {
1077         struct usb_page_search res;
1078         uhci_td_t *td;
1079         uhci_td_t *td_alt_next;
1080         uint32_t status;
1081         uint32_t token;
1082         uint16_t len;
1083
1084         td = xfer->td_transfer_cache;
1085         td_alt_next = td->alt_next;
1086
1087         if (xfer->aframes != xfer->nframes) {
1088                 usbd_xfer_set_frame_len(xfer, xfer->aframes, 0);
1089         }
1090         while (1) {
1091
1092                 usb_pc_cpu_invalidate(td->page_cache);
1093                 status = le32toh(td->td_status);
1094                 token = le32toh(td->td_token);
1095
1096                 /*
1097                  * Verify the status and add
1098                  * up the actual length:
1099                  */
1100
1101                 len = UHCI_TD_GET_ACTLEN(status);
1102                 if (len > td->len) {
1103                         /* should not happen */
1104                         DPRINTF("Invalid status length, "
1105                             "0x%04x/0x%04x bytes\n", len, td->len);
1106                         status |= UHCI_TD_STALLED;
1107
1108                 } else if ((xfer->aframes != xfer->nframes) && (len > 0)) {
1109
1110                         if (td->fix_pc) {
1111
1112                                 usbd_get_page(td->fix_pc, 0, &res);
1113
1114                                 /*
1115                                  * copy data from fixup location to real
1116                                  * location
1117                                  */
1118
1119                                 usb_pc_cpu_invalidate(td->fix_pc);
1120
1121                                 usbd_copy_in(xfer->frbuffers + xfer->aframes,
1122                                     xfer->frlengths[xfer->aframes], res.buffer, len);
1123                         }
1124                         /* update actual length */
1125
1126                         xfer->frlengths[xfer->aframes] += len;
1127                 }
1128                 /* Check for last transfer */
1129                 if (((void *)td) == xfer->td_transfer_last) {
1130                         td = NULL;
1131                         break;
1132                 }
1133                 if (status & UHCI_TD_STALLED) {
1134                         /* the transfer is finished */
1135                         td = NULL;
1136                         break;
1137                 }
1138                 /* Check for short transfer */
1139                 if (len != td->len) {
1140                         if (xfer->flags_int.short_frames_ok) {
1141                                 /* follow alt next */
1142                                 td = td->alt_next;
1143                         } else {
1144                                 /* the transfer is finished */
1145                                 td = NULL;
1146                         }
1147                         break;
1148                 }
1149                 td = td->obj_next;
1150
1151                 if (td->alt_next != td_alt_next) {
1152                         /* this USB frame is complete */
1153                         break;
1154                 }
1155         }
1156
1157         /* update transfer cache */
1158
1159         xfer->td_transfer_cache = td;
1160
1161         /* update data toggle */
1162
1163         xfer->endpoint->toggle_next = (token & UHCI_TD_SET_DT(1)) ? 0 : 1;
1164
1165 #ifdef USB_DEBUG
1166         if (status & UHCI_TD_ERROR) {
1167                 DPRINTFN(11, "error, addr=%d, endpt=0x%02x, frame=0x%02x "
1168                     "status=%s%s%s%s%s%s%s%s%s%s%s\n",
1169                     xfer->address, xfer->endpointno, xfer->aframes,
1170                     (status & UHCI_TD_BITSTUFF) ? "[BITSTUFF]" : "",
1171                     (status & UHCI_TD_CRCTO) ? "[CRCTO]" : "",
1172                     (status & UHCI_TD_NAK) ? "[NAK]" : "",
1173                     (status & UHCI_TD_BABBLE) ? "[BABBLE]" : "",
1174                     (status & UHCI_TD_DBUFFER) ? "[DBUFFER]" : "",
1175                     (status & UHCI_TD_STALLED) ? "[STALLED]" : "",
1176                     (status & UHCI_TD_ACTIVE) ? "[ACTIVE]" : "[NOT_ACTIVE]",
1177                     (status & UHCI_TD_IOC) ? "[IOC]" : "",
1178                     (status & UHCI_TD_IOS) ? "[IOS]" : "",
1179                     (status & UHCI_TD_LS) ? "[LS]" : "",
1180                     (status & UHCI_TD_SPD) ? "[SPD]" : "");
1181         }
1182 #endif
1183         if (status & UHCI_TD_STALLED) {
1184                 /* try to separate I/O errors from STALL */
1185                 if (UHCI_TD_GET_ERRCNT(status) == 0)
1186                         return (USB_ERR_IOERROR);
1187                 return (USB_ERR_STALLED);
1188         }
1189         return (USB_ERR_NORMAL_COMPLETION);
1190 }
1191
1192 static void
1193 uhci_non_isoc_done(struct usb_xfer *xfer)
1194 {
1195         usb_error_t err = 0;
1196
1197         DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
1198             xfer, xfer->endpoint);
1199
1200 #ifdef USB_DEBUG
1201         if (uhcidebug > 10) {
1202                 uhci_dump_tds(xfer->td_transfer_first);
1203         }
1204 #endif
1205
1206         /* sync any DMA memory before doing fixups */
1207
1208         usb_bdma_post_sync(xfer);
1209
1210         /* reset scanner */
1211
1212         xfer->td_transfer_cache = xfer->td_transfer_first;
1213
1214         if (xfer->flags_int.control_xfr) {
1215                 if (xfer->flags_int.control_hdr) {
1216
1217                         err = uhci_non_isoc_done_sub(xfer);
1218                 }
1219                 xfer->aframes = 1;
1220
1221                 if (xfer->td_transfer_cache == NULL) {
1222                         goto done;
1223                 }
1224         }
1225         while (xfer->aframes != xfer->nframes) {
1226
1227                 err = uhci_non_isoc_done_sub(xfer);
1228                 xfer->aframes++;
1229
1230                 if (xfer->td_transfer_cache == NULL) {
1231                         goto done;
1232                 }
1233         }
1234
1235         if (xfer->flags_int.control_xfr &&
1236             !xfer->flags_int.control_act) {
1237
1238                 err = uhci_non_isoc_done_sub(xfer);
1239         }
1240 done:
1241         uhci_device_done(xfer, err);
1242 }
1243
1244 /*------------------------------------------------------------------------*
1245  *      uhci_check_transfer_sub
1246  *
1247  * The main purpose of this function is to update the data-toggle
1248  * in case it is wrong.
1249  *------------------------------------------------------------------------*/
1250 static void
1251 uhci_check_transfer_sub(struct usb_xfer *xfer)
1252 {
1253         uhci_qh_t *qh;
1254         uhci_td_t *td;
1255         uhci_td_t *td_alt_next;
1256
1257         uint32_t td_token;
1258         uint32_t td_self;
1259
1260         td = xfer->td_transfer_cache;
1261         qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
1262
1263         td_token = td->obj_next->td_token;
1264         td = td->alt_next;
1265         xfer->td_transfer_cache = td;
1266         td_self = td->td_self;
1267         td_alt_next = td->alt_next;
1268
1269         if (xfer->flags_int.control_xfr)
1270                 goto skip;      /* don't touch the DT value! */
1271
1272         if (!((td->td_token ^ td_token) & htole32(UHCI_TD_SET_DT(1))))
1273                 goto skip;      /* data toggle has correct value */
1274
1275         /*
1276          * The data toggle is wrong and we need to toggle it !
1277          */
1278         while (1) {
1279
1280                 td->td_token ^= htole32(UHCI_TD_SET_DT(1));
1281                 usb_pc_cpu_flush(td->page_cache);
1282
1283                 if (td == xfer->td_transfer_last) {
1284                         /* last transfer */
1285                         break;
1286                 }
1287                 td = td->obj_next;
1288
1289                 if (td->alt_next != td_alt_next) {
1290                         /* next frame */
1291                         break;
1292                 }
1293         }
1294 skip:
1295
1296         /* update the QH */
1297         qh->qh_e_next = td_self;
1298         usb_pc_cpu_flush(qh->page_cache);
1299
1300         DPRINTFN(13, "xfer=%p following alt next\n", xfer);
1301 }
1302
1303 /*------------------------------------------------------------------------*
1304  *      uhci_check_transfer
1305  *
1306  * Return values:
1307  *    0: USB transfer is not finished
1308  * Else: USB transfer is finished
1309  *------------------------------------------------------------------------*/
1310 static uint8_t
1311 uhci_check_transfer(struct usb_xfer *xfer)
1312 {
1313         uint32_t status;
1314         uint32_t token;
1315         uhci_td_t *td;
1316
1317         DPRINTFN(16, "xfer=%p checking transfer\n", xfer);
1318
1319         if (xfer->endpoint->methods == &uhci_device_isoc_methods) {
1320                 /* isochronous transfer */
1321
1322                 td = xfer->td_transfer_last;
1323
1324                 usb_pc_cpu_invalidate(td->page_cache);
1325                 status = le32toh(td->td_status);
1326
1327                 /* check also if the first is complete */
1328
1329                 td = xfer->td_transfer_first;
1330
1331                 usb_pc_cpu_invalidate(td->page_cache);
1332                 status |= le32toh(td->td_status);
1333
1334                 if (!(status & UHCI_TD_ACTIVE)) {
1335                         uhci_device_done(xfer, USB_ERR_NORMAL_COMPLETION);
1336                         goto transferred;
1337                 }
1338         } else {
1339                 /* non-isochronous transfer */
1340
1341                 /*
1342                  * check whether there is an error somewhere
1343                  * in the middle, or whether there was a short
1344                  * packet (SPD and not ACTIVE)
1345                  */
1346                 td = xfer->td_transfer_cache;
1347
1348                 while (1) {
1349                         usb_pc_cpu_invalidate(td->page_cache);
1350                         status = le32toh(td->td_status);
1351                         token = le32toh(td->td_token);
1352
1353                         /*
1354                          * if there is an active TD the transfer isn't done
1355                          */
1356                         if (status & UHCI_TD_ACTIVE) {
1357                                 /* update cache */
1358                                 xfer->td_transfer_cache = td;
1359                                 goto done;
1360                         }
1361                         /*
1362                          * last transfer descriptor makes the transfer done
1363                          */
1364                         if (((void *)td) == xfer->td_transfer_last) {
1365                                 break;
1366                         }
1367                         /*
1368                          * any kind of error makes the transfer done
1369                          */
1370                         if (status & UHCI_TD_STALLED) {
1371                                 break;
1372                         }
1373                         /*
1374                          * check if we reached the last packet
1375                          * or if there is a short packet:
1376                          */
1377                         if ((td->td_next == htole32(UHCI_PTR_T)) ||
1378                             (UHCI_TD_GET_ACTLEN(status) < td->len)) {
1379
1380                                 if (xfer->flags_int.short_frames_ok) {
1381                                         /* follow alt next */
1382                                         if (td->alt_next) {
1383                                                 /* update cache */
1384                                                 xfer->td_transfer_cache = td;
1385                                                 uhci_check_transfer_sub(xfer);
1386                                                 goto done;
1387                                         }
1388                                 }
1389                                 /* transfer is done */
1390                                 break;
1391                         }
1392                         td = td->obj_next;
1393                 }
1394                 uhci_non_isoc_done(xfer);
1395                 goto transferred;
1396         }
1397
1398 done:
1399         DPRINTFN(13, "xfer=%p is still active\n", xfer);
1400         return (0);
1401
1402 transferred:
1403         return (1);
1404 }
1405
1406 static void
1407 uhci_interrupt_poll(uhci_softc_t *sc)
1408 {
1409         struct usb_xfer *xfer;
1410
1411 repeat:
1412         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
1413                 /*
1414                  * check if transfer is transferred
1415                  */
1416                 if (uhci_check_transfer(xfer)) {
1417                         /* queue has been modified */
1418                         goto repeat;
1419                 }
1420         }
1421 }
1422
1423 /*------------------------------------------------------------------------*
1424  *      uhci_interrupt - UHCI interrupt handler
1425  *
1426  * NOTE: Do not access "sc->sc_bus.bdev" inside the interrupt handler,
1427  * hence the interrupt handler will be setup before "sc->sc_bus.bdev"
1428  * is present !
1429  *------------------------------------------------------------------------*/
1430 void
1431 uhci_interrupt(uhci_softc_t *sc)
1432 {
1433         uint32_t status;
1434
1435         USB_BUS_LOCK(&sc->sc_bus);
1436
1437         DPRINTFN(16, "real interrupt\n");
1438
1439 #ifdef USB_DEBUG
1440         if (uhcidebug > 15) {
1441                 uhci_dumpregs(sc);
1442         }
1443 #endif
1444         status = UREAD2(sc, UHCI_STS) & UHCI_STS_ALLINTRS;
1445         if (status == 0) {
1446                 /* the interrupt was not for us */
1447                 goto done;
1448         }
1449         if (status & (UHCI_STS_RD | UHCI_STS_HSE |
1450             UHCI_STS_HCPE | UHCI_STS_HCH)) {
1451
1452                 if (status & UHCI_STS_RD) {
1453 #ifdef USB_DEBUG
1454                         printf("%s: resume detect\n",
1455                             __FUNCTION__);
1456 #endif
1457                 }
1458                 if (status & UHCI_STS_HSE) {
1459                         printf("%s: host system error\n",
1460                             __FUNCTION__);
1461                 }
1462                 if (status & UHCI_STS_HCPE) {
1463                         printf("%s: host controller process error\n",
1464                             __FUNCTION__);
1465                 }
1466                 if (status & UHCI_STS_HCH) {
1467                         /* no acknowledge needed */
1468                         DPRINTF("%s: host controller halted\n",
1469                             __FUNCTION__);
1470 #ifdef USB_DEBUG
1471                         if (uhcidebug > 0) {
1472                                 uhci_dump_all(sc);
1473                         }
1474 #endif
1475                 }
1476         }
1477         /* get acknowledge bits */
1478         status &= (UHCI_STS_USBINT |
1479             UHCI_STS_USBEI |
1480             UHCI_STS_RD |
1481             UHCI_STS_HSE |
1482             UHCI_STS_HCPE |
1483             UHCI_STS_HCH);
1484
1485         if (status == 0) {
1486                 /* nothing to acknowledge */
1487                 goto done;
1488         }
1489         /* acknowledge interrupts */
1490         UWRITE2(sc, UHCI_STS, status);
1491
1492         /* poll all the USB transfers */
1493         uhci_interrupt_poll(sc);
1494
1495 done:
1496         USB_BUS_UNLOCK(&sc->sc_bus);
1497 }
1498
1499 /*
1500  * called when a request does not complete
1501  */
1502 static void
1503 uhci_timeout(void *arg)
1504 {
1505         struct usb_xfer *xfer = arg;
1506
1507         DPRINTF("xfer=%p\n", xfer);
1508
1509         USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
1510
1511         /* transfer is transferred */
1512         uhci_device_done(xfer, USB_ERR_TIMEOUT);
1513 }
1514
1515 static void
1516 uhci_do_poll(struct usb_bus *bus)
1517 {
1518         struct uhci_softc *sc = UHCI_BUS2SC(bus);
1519
1520         USB_BUS_LOCK(&sc->sc_bus);
1521         uhci_interrupt_poll(sc);
1522         USB_BUS_UNLOCK(&sc->sc_bus);
1523 }
1524
1525 static void
1526 uhci_setup_standard_chain_sub(struct uhci_std_temp *temp)
1527 {
1528         uhci_td_t *td;
1529         uhci_td_t *td_next;
1530         uhci_td_t *td_alt_next;
1531         uint32_t average;
1532         uint32_t len_old;
1533         uint8_t shortpkt_old;
1534         uint8_t precompute;
1535
1536         td_alt_next = NULL;
1537         shortpkt_old = temp->shortpkt;
1538         len_old = temp->len;
1539         precompute = 1;
1540
1541         /* software is used to detect short incoming transfers */
1542
1543         if ((temp->td_token & htole32(UHCI_TD_PID)) == htole32(UHCI_TD_PID_IN)) {
1544                 temp->td_status |= htole32(UHCI_TD_SPD);
1545         } else {
1546                 temp->td_status &= ~htole32(UHCI_TD_SPD);
1547         }
1548
1549         temp->ml.buf_offset = 0;
1550
1551 restart:
1552
1553         temp->td_token &= ~htole32(UHCI_TD_SET_MAXLEN(0));
1554         temp->td_token |= htole32(UHCI_TD_SET_MAXLEN(temp->average));
1555
1556         td = temp->td;
1557         td_next = temp->td_next;
1558
1559         while (1) {
1560
1561                 if (temp->len == 0) {
1562
1563                         if (temp->shortpkt) {
1564                                 break;
1565                         }
1566                         /* send a Zero Length Packet, ZLP, last */
1567
1568                         temp->shortpkt = 1;
1569                         temp->td_token |= htole32(UHCI_TD_SET_MAXLEN(0));
1570                         average = 0;
1571
1572                 } else {
1573
1574                         average = temp->average;
1575
1576                         if (temp->len < average) {
1577                                 temp->shortpkt = 1;
1578                                 temp->td_token &= ~htole32(UHCI_TD_SET_MAXLEN(0));
1579                                 temp->td_token |= htole32(UHCI_TD_SET_MAXLEN(temp->len));
1580                                 average = temp->len;
1581                         }
1582                 }
1583
1584                 if (td_next == NULL) {
1585                         panic("%s: out of UHCI transfer descriptors!", __FUNCTION__);
1586                 }
1587                 /* get next TD */
1588
1589                 td = td_next;
1590                 td_next = td->obj_next;
1591
1592                 /* check if we are pre-computing */
1593
1594                 if (precompute) {
1595
1596                         /* update remaining length */
1597
1598                         temp->len -= average;
1599
1600                         continue;
1601                 }
1602                 /* fill out current TD */
1603
1604                 td->td_status = temp->td_status;
1605                 td->td_token = temp->td_token;
1606
1607                 /* update data toggle */
1608
1609                 temp->td_token ^= htole32(UHCI_TD_SET_DT(1));
1610
1611                 if (average == 0) {
1612
1613                         td->len = 0;
1614                         td->td_buffer = 0;
1615                         td->fix_pc = NULL;
1616
1617                 } else {
1618
1619                         /* update remaining length */
1620
1621                         temp->len -= average;
1622
1623                         td->len = average;
1624
1625                         /* fill out buffer pointer and do fixup, if any */
1626
1627                         uhci_mem_layout_fixup(&temp->ml, td);
1628                 }
1629
1630                 td->alt_next = td_alt_next;
1631
1632                 if ((td_next == td_alt_next) && temp->setup_alt_next) {
1633                         /* we need to receive these frames one by one ! */
1634                         td->td_status |= htole32(UHCI_TD_IOC);
1635                         td->td_next = htole32(UHCI_PTR_T);
1636                 } else {
1637                         if (td_next) {
1638                                 /* link the current TD with the next one */
1639                                 td->td_next = td_next->td_self;
1640                         }
1641                 }
1642
1643                 usb_pc_cpu_flush(td->page_cache);
1644         }
1645
1646         if (precompute) {
1647                 precompute = 0;
1648
1649                 /* setup alt next pointer, if any */
1650                 if (temp->last_frame) {
1651                         td_alt_next = NULL;
1652                 } else {
1653                         /* we use this field internally */
1654                         td_alt_next = td_next;
1655                 }
1656
1657                 /* restore */
1658                 temp->shortpkt = shortpkt_old;
1659                 temp->len = len_old;
1660                 goto restart;
1661         }
1662         temp->td = td;
1663         temp->td_next = td_next;
1664 }
1665
1666 static uhci_td_t *
1667 uhci_setup_standard_chain(struct usb_xfer *xfer)
1668 {
1669         struct uhci_std_temp temp;
1670         uhci_td_t *td;
1671         uint32_t x;
1672
1673         DPRINTFN(9, "addr=%d endpt=%d sumlen=%d speed=%d\n",
1674             xfer->address, UE_GET_ADDR(xfer->endpointno),
1675             xfer->sumlen, usbd_get_speed(xfer->xroot->udev));
1676
1677         temp.average = xfer->max_frame_size;
1678         temp.max_frame_size = xfer->max_frame_size;
1679
1680         /* toggle the DMA set we are using */
1681         xfer->flags_int.curr_dma_set ^= 1;
1682
1683         /* get next DMA set */
1684         td = xfer->td_start[xfer->flags_int.curr_dma_set];
1685         xfer->td_transfer_first = td;
1686         xfer->td_transfer_cache = td;
1687
1688         temp.td = NULL;
1689         temp.td_next = td;
1690         temp.last_frame = 0;
1691         temp.setup_alt_next = xfer->flags_int.short_frames_ok;
1692
1693         uhci_mem_layout_init(&temp.ml, xfer);
1694
1695         temp.td_status =
1696             htole32(UHCI_TD_ZERO_ACTLEN(UHCI_TD_SET_ERRCNT(3) |
1697             UHCI_TD_ACTIVE));
1698
1699         if (xfer->xroot->udev->speed == USB_SPEED_LOW) {
1700                 temp.td_status |= htole32(UHCI_TD_LS);
1701         }
1702         temp.td_token =
1703             htole32(UHCI_TD_SET_ENDPT(xfer->endpointno) |
1704             UHCI_TD_SET_DEVADDR(xfer->address));
1705
1706         if (xfer->endpoint->toggle_next) {
1707                 /* DATA1 is next */
1708                 temp.td_token |= htole32(UHCI_TD_SET_DT(1));
1709         }
1710         /* check if we should prepend a setup message */
1711
1712         if (xfer->flags_int.control_xfr) {
1713
1714                 if (xfer->flags_int.control_hdr) {
1715
1716                         temp.td_token &= htole32(UHCI_TD_SET_DEVADDR(0x7F) |
1717                             UHCI_TD_SET_ENDPT(0xF));
1718                         temp.td_token |= htole32(UHCI_TD_PID_SETUP |
1719                             UHCI_TD_SET_DT(0));
1720
1721                         temp.len = xfer->frlengths[0];
1722                         temp.ml.buf_pc = xfer->frbuffers + 0;
1723                         temp.shortpkt = temp.len ? 1 : 0;
1724                         /* check for last frame */
1725                         if (xfer->nframes == 1) {
1726                                 /* no STATUS stage yet, SETUP is last */
1727                                 if (xfer->flags_int.control_act) {
1728                                         temp.last_frame = 1;
1729                                         temp.setup_alt_next = 0;
1730                                 }
1731                         }
1732                         uhci_setup_standard_chain_sub(&temp);
1733                 }
1734                 x = 1;
1735         } else {
1736                 x = 0;
1737         }
1738
1739         while (x != xfer->nframes) {
1740
1741                 /* DATA0 / DATA1 message */
1742
1743                 temp.len = xfer->frlengths[x];
1744                 temp.ml.buf_pc = xfer->frbuffers + x;
1745
1746                 x++;
1747
1748                 if (x == xfer->nframes) {
1749                         if (xfer->flags_int.control_xfr) {
1750                                 /* no STATUS stage yet, DATA is last */
1751                                 if (xfer->flags_int.control_act) {
1752                                         temp.last_frame = 1;
1753                                         temp.setup_alt_next = 0;
1754                                 }
1755                         } else {
1756                                 temp.last_frame = 1;
1757                                 temp.setup_alt_next = 0;
1758                         }
1759                 }
1760                 /*
1761                  * Keep previous data toggle,
1762                  * device address and endpoint number:
1763                  */
1764
1765                 temp.td_token &= htole32(UHCI_TD_SET_DEVADDR(0x7F) |
1766                     UHCI_TD_SET_ENDPT(0xF) |
1767                     UHCI_TD_SET_DT(1));
1768
1769                 if (temp.len == 0) {
1770
1771                         /* make sure that we send an USB packet */
1772
1773                         temp.shortpkt = 0;
1774
1775                 } else {
1776
1777                         /* regular data transfer */
1778
1779                         temp.shortpkt = (xfer->flags.force_short_xfer) ? 0 : 1;
1780                 }
1781
1782                 /* set endpoint direction */
1783
1784                 temp.td_token |=
1785                     (UE_GET_DIR(xfer->endpointno) == UE_DIR_IN) ?
1786                     htole32(UHCI_TD_PID_IN) :
1787                     htole32(UHCI_TD_PID_OUT);
1788
1789                 uhci_setup_standard_chain_sub(&temp);
1790         }
1791
1792         /* check if we should append a status stage */
1793
1794         if (xfer->flags_int.control_xfr &&
1795             !xfer->flags_int.control_act) {
1796
1797                 /*
1798                  * send a DATA1 message and reverse the current endpoint
1799                  * direction
1800                  */
1801
1802                 temp.td_token &= htole32(UHCI_TD_SET_DEVADDR(0x7F) |
1803                     UHCI_TD_SET_ENDPT(0xF) |
1804                     UHCI_TD_SET_DT(1));
1805                 temp.td_token |=
1806                     (UE_GET_DIR(xfer->endpointno) == UE_DIR_OUT) ?
1807                     htole32(UHCI_TD_PID_IN | UHCI_TD_SET_DT(1)) :
1808                     htole32(UHCI_TD_PID_OUT | UHCI_TD_SET_DT(1));
1809
1810                 temp.len = 0;
1811                 temp.ml.buf_pc = NULL;
1812                 temp.shortpkt = 0;
1813                 temp.last_frame = 1;
1814                 temp.setup_alt_next = 0;
1815
1816                 uhci_setup_standard_chain_sub(&temp);
1817         }
1818         td = temp.td;
1819
1820         /* Ensure that last TD is terminating: */
1821         td->td_next = htole32(UHCI_PTR_T);
1822
1823         /* set interrupt bit */
1824
1825         td->td_status |= htole32(UHCI_TD_IOC);
1826
1827         usb_pc_cpu_flush(td->page_cache);
1828
1829         /* must have at least one frame! */
1830
1831         xfer->td_transfer_last = td;
1832
1833 #ifdef USB_DEBUG
1834         if (uhcidebug > 8) {
1835                 DPRINTF("nexttog=%d; data before transfer:\n",
1836                     xfer->endpoint->toggle_next);
1837                 uhci_dump_tds(xfer->td_transfer_first);
1838         }
1839 #endif
1840         return (xfer->td_transfer_first);
1841 }
1842
1843 /* NOTE: "done" can be run two times in a row,
1844  * from close and from interrupt
1845  */
1846
1847 static void
1848 uhci_device_done(struct usb_xfer *xfer, usb_error_t error)
1849 {
1850         const struct usb_pipe_methods *methods = xfer->endpoint->methods;
1851         uhci_softc_t *sc = UHCI_BUS2SC(xfer->xroot->bus);
1852         uhci_qh_t *qh;
1853
1854         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
1855
1856         DPRINTFN(2, "xfer=%p, endpoint=%p, error=%d\n",
1857             xfer, xfer->endpoint, error);
1858
1859         qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
1860         if (qh) {
1861                 usb_pc_cpu_invalidate(qh->page_cache);
1862         }
1863         if (xfer->flags_int.bandwidth_reclaimed) {
1864                 xfer->flags_int.bandwidth_reclaimed = 0;
1865                 uhci_rem_loop(sc);
1866         }
1867         if (methods == &uhci_device_bulk_methods) {
1868                 UHCI_REMOVE_QH(qh, sc->sc_bulk_p_last);
1869         }
1870         if (methods == &uhci_device_ctrl_methods) {
1871                 if (xfer->xroot->udev->speed == USB_SPEED_LOW) {
1872                         UHCI_REMOVE_QH(qh, sc->sc_ls_ctl_p_last);
1873                 } else {
1874                         UHCI_REMOVE_QH(qh, sc->sc_fs_ctl_p_last);
1875                 }
1876         }
1877         if (methods == &uhci_device_intr_methods) {
1878                 UHCI_REMOVE_QH(qh, sc->sc_intr_p_last[xfer->qh_pos]);
1879         }
1880         /*
1881          * Only finish isochronous transfers once
1882          * which will update "xfer->frlengths".
1883          */
1884         if (xfer->td_transfer_first &&
1885             xfer->td_transfer_last) {
1886                 if (methods == &uhci_device_isoc_methods) {
1887                         uhci_isoc_done(sc, xfer);
1888                 }
1889                 xfer->td_transfer_first = NULL;
1890                 xfer->td_transfer_last = NULL;
1891         }
1892         /* dequeue transfer and start next transfer */
1893         usbd_transfer_done(xfer, error);
1894 }
1895
1896 /*------------------------------------------------------------------------*
1897  * uhci bulk support
1898  *------------------------------------------------------------------------*/
1899 static void
1900 uhci_device_bulk_open(struct usb_xfer *xfer)
1901 {
1902         return;
1903 }
1904
1905 static void
1906 uhci_device_bulk_close(struct usb_xfer *xfer)
1907 {
1908         uhci_device_done(xfer, USB_ERR_CANCELLED);
1909 }
1910
1911 static void
1912 uhci_device_bulk_enter(struct usb_xfer *xfer)
1913 {
1914         return;
1915 }
1916
1917 static void
1918 uhci_device_bulk_start(struct usb_xfer *xfer)
1919 {
1920         uhci_softc_t *sc = UHCI_BUS2SC(xfer->xroot->bus);
1921         uhci_td_t *td;
1922         uhci_qh_t *qh;
1923
1924         /* setup TD's */
1925         td = uhci_setup_standard_chain(xfer);
1926
1927         /* setup QH */
1928         qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
1929
1930         qh->e_next = td;
1931         qh->qh_e_next = td->td_self;
1932
1933         if (xfer->xroot->udev->flags.self_suspended == 0) {
1934                 UHCI_APPEND_QH(qh, sc->sc_bulk_p_last);
1935                 uhci_add_loop(sc);
1936                 xfer->flags_int.bandwidth_reclaimed = 1;
1937         } else {
1938                 usb_pc_cpu_flush(qh->page_cache);
1939         }
1940
1941         /* put transfer on interrupt queue */
1942         uhci_transfer_intr_enqueue(xfer);
1943 }
1944
1945 static const struct usb_pipe_methods uhci_device_bulk_methods =
1946 {
1947         .open = uhci_device_bulk_open,
1948         .close = uhci_device_bulk_close,
1949         .enter = uhci_device_bulk_enter,
1950         .start = uhci_device_bulk_start,
1951 };
1952
1953 /*------------------------------------------------------------------------*
1954  * uhci control support
1955  *------------------------------------------------------------------------*/
1956 static void
1957 uhci_device_ctrl_open(struct usb_xfer *xfer)
1958 {
1959         return;
1960 }
1961
1962 static void
1963 uhci_device_ctrl_close(struct usb_xfer *xfer)
1964 {
1965         uhci_device_done(xfer, USB_ERR_CANCELLED);
1966 }
1967
1968 static void
1969 uhci_device_ctrl_enter(struct usb_xfer *xfer)
1970 {
1971         return;
1972 }
1973
1974 static void
1975 uhci_device_ctrl_start(struct usb_xfer *xfer)
1976 {
1977         uhci_softc_t *sc = UHCI_BUS2SC(xfer->xroot->bus);
1978         uhci_qh_t *qh;
1979         uhci_td_t *td;
1980
1981         /* setup TD's */
1982         td = uhci_setup_standard_chain(xfer);
1983
1984         /* setup QH */
1985         qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
1986
1987         qh->e_next = td;
1988         qh->qh_e_next = td->td_self;
1989
1990         /*
1991          * NOTE: some devices choke on bandwidth- reclamation for control
1992          * transfers
1993          */
1994         if (xfer->xroot->udev->flags.self_suspended == 0) {
1995                 if (xfer->xroot->udev->speed == USB_SPEED_LOW) {
1996                         UHCI_APPEND_QH(qh, sc->sc_ls_ctl_p_last);
1997                 } else {
1998                         UHCI_APPEND_QH(qh, sc->sc_fs_ctl_p_last);
1999                 }
2000         } else {
2001                 usb_pc_cpu_flush(qh->page_cache);
2002         }
2003         /* put transfer on interrupt queue */
2004         uhci_transfer_intr_enqueue(xfer);
2005 }
2006
2007 static const struct usb_pipe_methods uhci_device_ctrl_methods =
2008 {
2009         .open = uhci_device_ctrl_open,
2010         .close = uhci_device_ctrl_close,
2011         .enter = uhci_device_ctrl_enter,
2012         .start = uhci_device_ctrl_start,
2013 };
2014
2015 /*------------------------------------------------------------------------*
2016  * uhci interrupt support
2017  *------------------------------------------------------------------------*/
2018 static void
2019 uhci_device_intr_open(struct usb_xfer *xfer)
2020 {
2021         uhci_softc_t *sc = UHCI_BUS2SC(xfer->xroot->bus);
2022         uint16_t best;
2023         uint16_t bit;
2024         uint16_t x;
2025
2026         best = 0;
2027         bit = UHCI_IFRAMELIST_COUNT / 2;
2028         while (bit) {
2029                 if (xfer->interval >= bit) {
2030                         x = bit;
2031                         best = bit;
2032                         while (x & bit) {
2033                                 if (sc->sc_intr_stat[x] <
2034                                     sc->sc_intr_stat[best]) {
2035                                         best = x;
2036                                 }
2037                                 x++;
2038                         }
2039                         break;
2040                 }
2041                 bit >>= 1;
2042         }
2043
2044         sc->sc_intr_stat[best]++;
2045         xfer->qh_pos = best;
2046
2047         DPRINTFN(3, "best=%d interval=%d\n",
2048             best, xfer->interval);
2049 }
2050
2051 static void
2052 uhci_device_intr_close(struct usb_xfer *xfer)
2053 {
2054         uhci_softc_t *sc = UHCI_BUS2SC(xfer->xroot->bus);
2055
2056         sc->sc_intr_stat[xfer->qh_pos]--;
2057
2058         uhci_device_done(xfer, USB_ERR_CANCELLED);
2059 }
2060
2061 static void
2062 uhci_device_intr_enter(struct usb_xfer *xfer)
2063 {
2064         return;
2065 }
2066
2067 static void
2068 uhci_device_intr_start(struct usb_xfer *xfer)
2069 {
2070         uhci_softc_t *sc = UHCI_BUS2SC(xfer->xroot->bus);
2071         uhci_qh_t *qh;
2072         uhci_td_t *td;
2073
2074         /* setup TD's */
2075         td = uhci_setup_standard_chain(xfer);
2076
2077         /* setup QH */
2078         qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
2079
2080         qh->e_next = td;
2081         qh->qh_e_next = td->td_self;
2082
2083         if (xfer->xroot->udev->flags.self_suspended == 0) {
2084                 /* enter QHs into the controller data structures */
2085                 UHCI_APPEND_QH(qh, sc->sc_intr_p_last[xfer->qh_pos]);
2086         } else {
2087                 usb_pc_cpu_flush(qh->page_cache);
2088         }
2089
2090         /* put transfer on interrupt queue */
2091         uhci_transfer_intr_enqueue(xfer);
2092 }
2093
2094 static const struct usb_pipe_methods uhci_device_intr_methods =
2095 {
2096         .open = uhci_device_intr_open,
2097         .close = uhci_device_intr_close,
2098         .enter = uhci_device_intr_enter,
2099         .start = uhci_device_intr_start,
2100 };
2101
2102 /*------------------------------------------------------------------------*
2103  * uhci isochronous support
2104  *------------------------------------------------------------------------*/
2105 static void
2106 uhci_device_isoc_open(struct usb_xfer *xfer)
2107 {
2108         uhci_td_t *td;
2109         uint32_t td_token;
2110         uint8_t ds;
2111
2112         td_token =
2113             (UE_GET_DIR(xfer->endpointno) == UE_DIR_IN) ?
2114             UHCI_TD_IN(0, xfer->endpointno, xfer->address, 0) :
2115             UHCI_TD_OUT(0, xfer->endpointno, xfer->address, 0);
2116
2117         td_token = htole32(td_token);
2118
2119         /* initialize all TD's */
2120
2121         for (ds = 0; ds != 2; ds++) {
2122
2123                 for (td = xfer->td_start[ds]; td; td = td->obj_next) {
2124
2125                         /* mark TD as inactive */
2126                         td->td_status = htole32(UHCI_TD_IOS);
2127                         td->td_token = td_token;
2128
2129                         usb_pc_cpu_flush(td->page_cache);
2130                 }
2131         }
2132 }
2133
2134 static void
2135 uhci_device_isoc_close(struct usb_xfer *xfer)
2136 {
2137         uhci_device_done(xfer, USB_ERR_CANCELLED);
2138 }
2139
2140 static void
2141 uhci_device_isoc_enter(struct usb_xfer *xfer)
2142 {
2143         struct uhci_mem_layout ml;
2144         uhci_softc_t *sc = UHCI_BUS2SC(xfer->xroot->bus);
2145         uint32_t nframes;
2146         uint32_t temp;
2147         uint32_t *plen;
2148
2149 #ifdef USB_DEBUG
2150         uint8_t once = 1;
2151
2152 #endif
2153         uhci_td_t *td;
2154         uhci_td_t *td_last = NULL;
2155         uhci_td_t **pp_last;
2156
2157         DPRINTFN(6, "xfer=%p next=%d nframes=%d\n",
2158             xfer, xfer->endpoint->isoc_next, xfer->nframes);
2159
2160         nframes = UREAD2(sc, UHCI_FRNUM);
2161
2162         temp = (nframes - xfer->endpoint->isoc_next) &
2163             (UHCI_VFRAMELIST_COUNT - 1);
2164
2165         if ((xfer->endpoint->is_synced == 0) ||
2166             (temp < xfer->nframes)) {
2167                 /*
2168                  * If there is data underflow or the pipe queue is empty we
2169                  * schedule the transfer a few frames ahead of the current
2170                  * frame position. Else two isochronous transfers might
2171                  * overlap.
2172                  */
2173                 xfer->endpoint->isoc_next = (nframes + 3) & (UHCI_VFRAMELIST_COUNT - 1);
2174                 xfer->endpoint->is_synced = 1;
2175                 DPRINTFN(3, "start next=%d\n", xfer->endpoint->isoc_next);
2176         }
2177         /*
2178          * compute how many milliseconds the insertion is ahead of the
2179          * current frame position:
2180          */
2181         temp = (xfer->endpoint->isoc_next - nframes) &
2182             (UHCI_VFRAMELIST_COUNT - 1);
2183
2184         /*
2185          * pre-compute when the isochronous transfer will be finished:
2186          */
2187         xfer->isoc_time_complete =
2188             usb_isoc_time_expand(&sc->sc_bus, nframes) + temp +
2189             xfer->nframes;
2190
2191         /* get the real number of frames */
2192
2193         nframes = xfer->nframes;
2194
2195         uhci_mem_layout_init(&ml, xfer);
2196
2197         plen = xfer->frlengths;
2198
2199         /* toggle the DMA set we are using */
2200         xfer->flags_int.curr_dma_set ^= 1;
2201
2202         /* get next DMA set */
2203         td = xfer->td_start[xfer->flags_int.curr_dma_set];
2204         xfer->td_transfer_first = td;
2205
2206         pp_last = &sc->sc_isoc_p_last[xfer->endpoint->isoc_next];
2207
2208         /* store starting position */
2209
2210         xfer->qh_pos = xfer->endpoint->isoc_next;
2211
2212         while (nframes--) {
2213                 if (td == NULL) {
2214                         panic("%s:%d: out of TD's\n",
2215                             __FUNCTION__, __LINE__);
2216                 }
2217                 if (pp_last >= &sc->sc_isoc_p_last[UHCI_VFRAMELIST_COUNT]) {
2218                         pp_last = &sc->sc_isoc_p_last[0];
2219                 }
2220                 if (*plen > xfer->max_frame_size) {
2221 #ifdef USB_DEBUG
2222                         if (once) {
2223                                 once = 0;
2224                                 printf("%s: frame length(%d) exceeds %d "
2225                                     "bytes (frame truncated)\n",
2226                                     __FUNCTION__, *plen,
2227                                     xfer->max_frame_size);
2228                         }
2229 #endif
2230                         *plen = xfer->max_frame_size;
2231                 }
2232                 /* reuse td_token from last transfer */
2233
2234                 td->td_token &= htole32(~UHCI_TD_MAXLEN_MASK);
2235                 td->td_token |= htole32(UHCI_TD_SET_MAXLEN(*plen));
2236
2237                 td->len = *plen;
2238
2239                 if (td->len == 0) {
2240                         /*
2241                          * Do not call "uhci_mem_layout_fixup()" when the
2242                          * length is zero!
2243                          */
2244                         td->td_buffer = 0;
2245                         td->fix_pc = NULL;
2246
2247                 } else {
2248
2249                         /* fill out buffer pointer and do fixup, if any */
2250
2251                         uhci_mem_layout_fixup(&ml, td);
2252
2253                 }
2254
2255                 /* update status */
2256                 if (nframes == 0) {
2257                         td->td_status = htole32
2258                             (UHCI_TD_ZERO_ACTLEN
2259                             (UHCI_TD_SET_ERRCNT(0) |
2260                             UHCI_TD_ACTIVE |
2261                             UHCI_TD_IOS |
2262                             UHCI_TD_IOC));
2263                 } else {
2264                         td->td_status = htole32
2265                             (UHCI_TD_ZERO_ACTLEN
2266                             (UHCI_TD_SET_ERRCNT(0) |
2267                             UHCI_TD_ACTIVE |
2268                             UHCI_TD_IOS));
2269                 }
2270
2271                 usb_pc_cpu_flush(td->page_cache);
2272
2273 #ifdef USB_DEBUG
2274                 if (uhcidebug > 5) {
2275                         DPRINTF("TD %d\n", nframes);
2276                         uhci_dump_td(td);
2277                 }
2278 #endif
2279                 /* insert TD into schedule */
2280                 UHCI_APPEND_TD(td, *pp_last);
2281                 pp_last++;
2282
2283                 plen++;
2284                 td_last = td;
2285                 td = td->obj_next;
2286         }
2287
2288         xfer->td_transfer_last = td_last;
2289
2290         /* update isoc_next */
2291         xfer->endpoint->isoc_next = (pp_last - &sc->sc_isoc_p_last[0]) &
2292             (UHCI_VFRAMELIST_COUNT - 1);
2293 }
2294
2295 static void
2296 uhci_device_isoc_start(struct usb_xfer *xfer)
2297 {
2298         /* put transfer on interrupt queue */
2299         uhci_transfer_intr_enqueue(xfer);
2300 }
2301
2302 static const struct usb_pipe_methods uhci_device_isoc_methods =
2303 {
2304         .open = uhci_device_isoc_open,
2305         .close = uhci_device_isoc_close,
2306         .enter = uhci_device_isoc_enter,
2307         .start = uhci_device_isoc_start,
2308 };
2309
2310 /*------------------------------------------------------------------------*
2311  * uhci root control support
2312  *------------------------------------------------------------------------*
2313  * Simulate a hardware hub by handling all the necessary requests.
2314  *------------------------------------------------------------------------*/
2315
2316 static const
2317 struct usb_device_descriptor uhci_devd =
2318 {
2319         sizeof(struct usb_device_descriptor),
2320         UDESC_DEVICE,                   /* type */
2321         {0x00, 0x01},                   /* USB version */
2322         UDCLASS_HUB,                    /* class */
2323         UDSUBCLASS_HUB,                 /* subclass */
2324         UDPROTO_FSHUB,                  /* protocol */
2325         64,                             /* max packet */
2326         {0}, {0}, {0x00, 0x01},         /* device id */
2327         1, 2, 0,                        /* string indexes */
2328         1                               /* # of configurations */
2329 };
2330
2331 static const struct uhci_config_desc uhci_confd = {
2332         .confd = {
2333                 .bLength = sizeof(struct usb_config_descriptor),
2334                 .bDescriptorType = UDESC_CONFIG,
2335                 .wTotalLength[0] = sizeof(uhci_confd),
2336                 .bNumInterface = 1,
2337                 .bConfigurationValue = 1,
2338                 .iConfiguration = 0,
2339                 .bmAttributes = UC_SELF_POWERED,
2340                 .bMaxPower = 0          /* max power */
2341         },
2342         .ifcd = {
2343                 .bLength = sizeof(struct usb_interface_descriptor),
2344                 .bDescriptorType = UDESC_INTERFACE,
2345                 .bNumEndpoints = 1,
2346                 .bInterfaceClass = UICLASS_HUB,
2347                 .bInterfaceSubClass = UISUBCLASS_HUB,
2348                 .bInterfaceProtocol = UIPROTO_FSHUB,
2349         },
2350         .endpd = {
2351                 .bLength = sizeof(struct usb_endpoint_descriptor),
2352                 .bDescriptorType = UDESC_ENDPOINT,
2353                 .bEndpointAddress = UE_DIR_IN | UHCI_INTR_ENDPT,
2354                 .bmAttributes = UE_INTERRUPT,
2355                 .wMaxPacketSize[0] = 8, /* max packet (63 ports) */
2356                 .bInterval = 255,
2357         },
2358 };
2359
2360 static const
2361 struct usb_hub_descriptor_min uhci_hubd_piix =
2362 {
2363         .bDescLength = sizeof(uhci_hubd_piix),
2364         .bDescriptorType = UDESC_HUB,
2365         .bNbrPorts = 2,
2366         .wHubCharacteristics = {UHD_PWR_NO_SWITCH | UHD_OC_INDIVIDUAL, 0},
2367         .bPwrOn2PwrGood = 50,
2368 };
2369
2370 /*
2371  * The USB hub protocol requires that SET_FEATURE(PORT_RESET) also
2372  * enables the port, and also states that SET_FEATURE(PORT_ENABLE)
2373  * should not be used by the USB subsystem.  As we cannot issue a
2374  * SET_FEATURE(PORT_ENABLE) externally, we must ensure that the port
2375  * will be enabled as part of the reset.
2376  *
2377  * On the VT83C572, the port cannot be successfully enabled until the
2378  * outstanding "port enable change" and "connection status change"
2379  * events have been reset.
2380  */
2381 static usb_error_t
2382 uhci_portreset(uhci_softc_t *sc, uint16_t index)
2383 {
2384         uint16_t port;
2385         uint16_t x;
2386         uint8_t lim;
2387
2388         if (index == 1)
2389                 port = UHCI_PORTSC1;
2390         else if (index == 2)
2391                 port = UHCI_PORTSC2;
2392         else
2393                 return (USB_ERR_IOERROR);
2394
2395         /*
2396          * Before we do anything, turn on SOF messages on the USB
2397          * BUS. Some USB devices do not cope without them!
2398          */
2399         uhci_restart(sc);
2400
2401         x = URWMASK(UREAD2(sc, port));
2402         UWRITE2(sc, port, x | UHCI_PORTSC_PR);
2403
2404         usb_pause_mtx(&sc->sc_bus.bus_mtx,
2405             USB_MS_TO_TICKS(usb_port_root_reset_delay));
2406
2407         DPRINTFN(4, "uhci port %d reset, status0 = 0x%04x\n",
2408             index, UREAD2(sc, port));
2409
2410         x = URWMASK(UREAD2(sc, port));
2411         UWRITE2(sc, port, x & ~UHCI_PORTSC_PR);
2412
2413
2414         mtx_unlock(&sc->sc_bus.bus_mtx);
2415
2416         /*
2417          * This delay needs to be exactly 100us, else some USB devices
2418          * fail to attach!
2419          */
2420         DELAY(100);
2421
2422         mtx_lock(&sc->sc_bus.bus_mtx);
2423
2424         DPRINTFN(4, "uhci port %d reset, status1 = 0x%04x\n",
2425             index, UREAD2(sc, port));
2426
2427         x = URWMASK(UREAD2(sc, port));
2428         UWRITE2(sc, port, x | UHCI_PORTSC_PE);
2429
2430         for (lim = 0; lim < 12; lim++) {
2431
2432                 usb_pause_mtx(&sc->sc_bus.bus_mtx,
2433                     USB_MS_TO_TICKS(usb_port_reset_delay));
2434
2435                 x = UREAD2(sc, port);
2436
2437                 DPRINTFN(4, "uhci port %d iteration %u, status = 0x%04x\n",
2438                     index, lim, x);
2439
2440                 if (!(x & UHCI_PORTSC_CCS)) {
2441                         /*
2442                          * No device is connected (or was disconnected
2443                          * during reset).  Consider the port reset.
2444                          * The delay must be long enough to ensure on
2445                          * the initial iteration that the device
2446                          * connection will have been registered.  50ms
2447                          * appears to be sufficient, but 20ms is not.
2448                          */
2449                         DPRINTFN(4, "uhci port %d loop %u, device detached\n",
2450                             index, lim);
2451                         goto done;
2452                 }
2453                 if (x & (UHCI_PORTSC_POEDC | UHCI_PORTSC_CSC)) {
2454                         /*
2455                          * Port enabled changed and/or connection
2456                          * status changed were set.  Reset either or
2457                          * both raised flags (by writing a 1 to that
2458                          * bit), and wait again for state to settle.
2459                          */
2460                         UWRITE2(sc, port, URWMASK(x) |
2461                             (x & (UHCI_PORTSC_POEDC | UHCI_PORTSC_CSC)));
2462                         continue;
2463                 }
2464                 if (x & UHCI_PORTSC_PE) {
2465                         /* port is enabled */
2466                         goto done;
2467                 }
2468                 UWRITE2(sc, port, URWMASK(x) | UHCI_PORTSC_PE);
2469         }
2470
2471         DPRINTFN(2, "uhci port %d reset timed out\n", index);
2472         return (USB_ERR_TIMEOUT);
2473
2474 done:
2475         DPRINTFN(4, "uhci port %d reset, status2 = 0x%04x\n",
2476             index, UREAD2(sc, port));
2477
2478         sc->sc_isreset = 1;
2479         return (USB_ERR_NORMAL_COMPLETION);
2480 }
2481
2482 static usb_error_t
2483 uhci_roothub_exec(struct usb_device *udev,
2484     struct usb_device_request *req, const void **pptr, uint16_t *plength)
2485 {
2486         uhci_softc_t *sc = UHCI_BUS2SC(udev->bus);
2487         const void *ptr;
2488         const char *str_ptr;
2489         uint16_t x;
2490         uint16_t port;
2491         uint16_t value;
2492         uint16_t index;
2493         uint16_t status;
2494         uint16_t change;
2495         uint16_t len;
2496         usb_error_t err;
2497
2498         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
2499
2500         /* buffer reset */
2501         ptr = (const void *)&sc->sc_hub_desc.temp;
2502         len = 0;
2503         err = 0;
2504
2505         value = UGETW(req->wValue);
2506         index = UGETW(req->wIndex);
2507
2508         DPRINTFN(3, "type=0x%02x request=0x%02x wLen=0x%04x "
2509             "wValue=0x%04x wIndex=0x%04x\n",
2510             req->bmRequestType, req->bRequest,
2511             UGETW(req->wLength), value, index);
2512
2513 #define C(x,y) ((x) | ((y) << 8))
2514         switch (C(req->bRequest, req->bmRequestType)) {
2515         case C(UR_CLEAR_FEATURE, UT_WRITE_DEVICE):
2516         case C(UR_CLEAR_FEATURE, UT_WRITE_INTERFACE):
2517         case C(UR_CLEAR_FEATURE, UT_WRITE_ENDPOINT):
2518                 /*
2519                  * DEVICE_REMOTE_WAKEUP and ENDPOINT_HALT are no-ops
2520                  * for the integrated root hub.
2521                  */
2522                 break;
2523         case C(UR_GET_CONFIG, UT_READ_DEVICE):
2524                 len = 1;
2525                 sc->sc_hub_desc.temp[0] = sc->sc_conf;
2526                 break;
2527         case C(UR_GET_DESCRIPTOR, UT_READ_DEVICE):
2528                 switch (value >> 8) {
2529                 case UDESC_DEVICE:
2530                         if ((value & 0xff) != 0) {
2531                                 err = USB_ERR_IOERROR;
2532                                 goto done;
2533                         }
2534                         len = sizeof(uhci_devd);
2535                         ptr = (const void *)&uhci_devd;
2536                         break;
2537
2538                 case UDESC_CONFIG:
2539                         if ((value & 0xff) != 0) {
2540                                 err = USB_ERR_IOERROR;
2541                                 goto done;
2542                         }
2543                         len = sizeof(uhci_confd);
2544                         ptr = (const void *)&uhci_confd;
2545                         break;
2546
2547                 case UDESC_STRING:
2548                         switch (value & 0xff) {
2549                         case 0: /* Language table */
2550                                 str_ptr = "\001";
2551                                 break;
2552
2553                         case 1: /* Vendor */
2554                                 str_ptr = sc->sc_vendor;
2555                                 break;
2556
2557                         case 2: /* Product */
2558                                 str_ptr = "UHCI root HUB";
2559                                 break;
2560
2561                         default:
2562                                 str_ptr = "";
2563                                 break;
2564                         }
2565
2566                         len = usb_make_str_desc
2567                             (sc->sc_hub_desc.temp,
2568                             sizeof(sc->sc_hub_desc.temp),
2569                             str_ptr);
2570                         break;
2571
2572                 default:
2573                         err = USB_ERR_IOERROR;
2574                         goto done;
2575                 }
2576                 break;
2577         case C(UR_GET_INTERFACE, UT_READ_INTERFACE):
2578                 len = 1;
2579                 sc->sc_hub_desc.temp[0] = 0;
2580                 break;
2581         case C(UR_GET_STATUS, UT_READ_DEVICE):
2582                 len = 2;
2583                 USETW(sc->sc_hub_desc.stat.wStatus, UDS_SELF_POWERED);
2584                 break;
2585         case C(UR_GET_STATUS, UT_READ_INTERFACE):
2586         case C(UR_GET_STATUS, UT_READ_ENDPOINT):
2587                 len = 2;
2588                 USETW(sc->sc_hub_desc.stat.wStatus, 0);
2589                 break;
2590         case C(UR_SET_ADDRESS, UT_WRITE_DEVICE):
2591                 if (value >= UHCI_MAX_DEVICES) {
2592                         err = USB_ERR_IOERROR;
2593                         goto done;
2594                 }
2595                 sc->sc_addr = value;
2596                 break;
2597         case C(UR_SET_CONFIG, UT_WRITE_DEVICE):
2598                 if ((value != 0) && (value != 1)) {
2599                         err = USB_ERR_IOERROR;
2600                         goto done;
2601                 }
2602                 sc->sc_conf = value;
2603                 break;
2604         case C(UR_SET_DESCRIPTOR, UT_WRITE_DEVICE):
2605                 break;
2606         case C(UR_SET_FEATURE, UT_WRITE_DEVICE):
2607         case C(UR_SET_FEATURE, UT_WRITE_INTERFACE):
2608         case C(UR_SET_FEATURE, UT_WRITE_ENDPOINT):
2609                 err = USB_ERR_IOERROR;
2610                 goto done;
2611         case C(UR_SET_INTERFACE, UT_WRITE_INTERFACE):
2612                 break;
2613         case C(UR_SYNCH_FRAME, UT_WRITE_ENDPOINT):
2614                 break;
2615                 /* Hub requests */
2616         case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_DEVICE):
2617                 break;
2618         case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_OTHER):
2619                 DPRINTFN(4, "UR_CLEAR_PORT_FEATURE "
2620                     "port=%d feature=%d\n",
2621                     index, value);
2622                 if (index == 1)
2623                         port = UHCI_PORTSC1;
2624                 else if (index == 2)
2625                         port = UHCI_PORTSC2;
2626                 else {
2627                         err = USB_ERR_IOERROR;
2628                         goto done;
2629                 }
2630                 switch (value) {
2631                 case UHF_PORT_ENABLE:
2632                         x = URWMASK(UREAD2(sc, port));
2633                         UWRITE2(sc, port, x & ~UHCI_PORTSC_PE);
2634                         break;
2635                 case UHF_PORT_SUSPEND:
2636                         x = URWMASK(UREAD2(sc, port));
2637                         UWRITE2(sc, port, x & ~(UHCI_PORTSC_SUSP));
2638                         break;
2639                 case UHF_PORT_RESET:
2640                         x = URWMASK(UREAD2(sc, port));
2641                         UWRITE2(sc, port, x & ~UHCI_PORTSC_PR);
2642                         break;
2643                 case UHF_C_PORT_CONNECTION:
2644                         x = URWMASK(UREAD2(sc, port));
2645                         UWRITE2(sc, port, x | UHCI_PORTSC_CSC);
2646                         break;
2647                 case UHF_C_PORT_ENABLE:
2648                         x = URWMASK(UREAD2(sc, port));
2649                         UWRITE2(sc, port, x | UHCI_PORTSC_POEDC);
2650                         break;
2651                 case UHF_C_PORT_OVER_CURRENT:
2652                         x = URWMASK(UREAD2(sc, port));
2653                         UWRITE2(sc, port, x | UHCI_PORTSC_OCIC);
2654                         break;
2655                 case UHF_C_PORT_RESET:
2656                         sc->sc_isreset = 0;
2657                         err = USB_ERR_NORMAL_COMPLETION;
2658                         goto done;
2659                 case UHF_C_PORT_SUSPEND:
2660                         sc->sc_isresumed &= ~(1 << index);
2661                         break;
2662                 case UHF_PORT_CONNECTION:
2663                 case UHF_PORT_OVER_CURRENT:
2664                 case UHF_PORT_POWER:
2665                 case UHF_PORT_LOW_SPEED:
2666                 default:
2667                         err = USB_ERR_IOERROR;
2668                         goto done;
2669                 }
2670                 break;
2671         case C(UR_GET_BUS_STATE, UT_READ_CLASS_OTHER):
2672                 if (index == 1)
2673                         port = UHCI_PORTSC1;
2674                 else if (index == 2)
2675                         port = UHCI_PORTSC2;
2676                 else {
2677                         err = USB_ERR_IOERROR;
2678                         goto done;
2679                 }
2680                 len = 1;
2681                 sc->sc_hub_desc.temp[0] =
2682                     ((UREAD2(sc, port) & UHCI_PORTSC_LS) >>
2683                     UHCI_PORTSC_LS_SHIFT);
2684                 break;
2685         case C(UR_GET_DESCRIPTOR, UT_READ_CLASS_DEVICE):
2686                 if ((value & 0xff) != 0) {
2687                         err = USB_ERR_IOERROR;
2688                         goto done;
2689                 }
2690                 len = sizeof(uhci_hubd_piix);
2691                 ptr = (const void *)&uhci_hubd_piix;
2692                 break;
2693         case C(UR_GET_STATUS, UT_READ_CLASS_DEVICE):
2694                 len = 16;
2695                 memset(sc->sc_hub_desc.temp, 0, 16);
2696                 break;
2697         case C(UR_GET_STATUS, UT_READ_CLASS_OTHER):
2698                 if (index == 1)
2699                         port = UHCI_PORTSC1;
2700                 else if (index == 2)
2701                         port = UHCI_PORTSC2;
2702                 else {
2703                         err = USB_ERR_IOERROR;
2704                         goto done;
2705                 }
2706                 x = UREAD2(sc, port);
2707                 status = change = 0;
2708                 if (x & UHCI_PORTSC_CCS)
2709                         status |= UPS_CURRENT_CONNECT_STATUS;
2710                 if (x & UHCI_PORTSC_CSC)
2711                         change |= UPS_C_CONNECT_STATUS;
2712                 if (x & UHCI_PORTSC_PE)
2713                         status |= UPS_PORT_ENABLED;
2714                 if (x & UHCI_PORTSC_POEDC)
2715                         change |= UPS_C_PORT_ENABLED;
2716                 if (x & UHCI_PORTSC_OCI)
2717                         status |= UPS_OVERCURRENT_INDICATOR;
2718                 if (x & UHCI_PORTSC_OCIC)
2719                         change |= UPS_C_OVERCURRENT_INDICATOR;
2720                 if (x & UHCI_PORTSC_LSDA)
2721                         status |= UPS_LOW_SPEED;
2722                 if ((x & UHCI_PORTSC_PE) && (x & UHCI_PORTSC_RD)) {
2723                         /* need to do a write back */
2724                         UWRITE2(sc, port, URWMASK(x));
2725
2726                         /* wait 20ms for resume sequence to complete */
2727                         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 50);
2728
2729                         /* clear suspend and resume detect */
2730                         UWRITE2(sc, port, URWMASK(x) & ~(UHCI_PORTSC_RD |
2731                             UHCI_PORTSC_SUSP));
2732
2733                         /* wait a little bit */
2734                         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 500);
2735
2736                         sc->sc_isresumed |= (1 << index);
2737
2738                 } else if (x & UHCI_PORTSC_SUSP) {
2739                         status |= UPS_SUSPEND;
2740                 }
2741                 status |= UPS_PORT_POWER;
2742                 if (sc->sc_isresumed & (1 << index))
2743                         change |= UPS_C_SUSPEND;
2744                 if (sc->sc_isreset)
2745                         change |= UPS_C_PORT_RESET;
2746                 USETW(sc->sc_hub_desc.ps.wPortStatus, status);
2747                 USETW(sc->sc_hub_desc.ps.wPortChange, change);
2748                 len = sizeof(sc->sc_hub_desc.ps);
2749                 break;
2750         case C(UR_SET_DESCRIPTOR, UT_WRITE_CLASS_DEVICE):
2751                 err = USB_ERR_IOERROR;
2752                 goto done;
2753         case C(UR_SET_FEATURE, UT_WRITE_CLASS_DEVICE):
2754                 break;
2755         case C(UR_SET_FEATURE, UT_WRITE_CLASS_OTHER):
2756                 if (index == 1)
2757                         port = UHCI_PORTSC1;
2758                 else if (index == 2)
2759                         port = UHCI_PORTSC2;
2760                 else {
2761                         err = USB_ERR_IOERROR;
2762                         goto done;
2763                 }
2764                 switch (value) {
2765                 case UHF_PORT_ENABLE:
2766                         x = URWMASK(UREAD2(sc, port));
2767                         UWRITE2(sc, port, x | UHCI_PORTSC_PE);
2768                         break;
2769                 case UHF_PORT_SUSPEND:
2770                         x = URWMASK(UREAD2(sc, port));
2771                         UWRITE2(sc, port, x | UHCI_PORTSC_SUSP);
2772                         break;
2773                 case UHF_PORT_RESET:
2774                         err = uhci_portreset(sc, index);
2775                         goto done;
2776                 case UHF_PORT_POWER:
2777                         /* pretend we turned on power */
2778                         err = USB_ERR_NORMAL_COMPLETION;
2779                         goto done;
2780                 case UHF_C_PORT_CONNECTION:
2781                 case UHF_C_PORT_ENABLE:
2782                 case UHF_C_PORT_OVER_CURRENT:
2783                 case UHF_PORT_CONNECTION:
2784                 case UHF_PORT_OVER_CURRENT:
2785                 case UHF_PORT_LOW_SPEED:
2786                 case UHF_C_PORT_SUSPEND:
2787                 case UHF_C_PORT_RESET:
2788                 default:
2789                         err = USB_ERR_IOERROR;
2790                         goto done;
2791                 }
2792                 break;
2793         default:
2794                 err = USB_ERR_IOERROR;
2795                 goto done;
2796         }
2797 done:
2798         *plength = len;
2799         *pptr = ptr;
2800         return (err);
2801 }
2802
2803 /*
2804  * This routine is executed periodically and simulates interrupts from
2805  * the root controller interrupt pipe for port status change:
2806  */
2807 static void
2808 uhci_root_intr(uhci_softc_t *sc)
2809 {
2810         DPRINTFN(21, "\n");
2811
2812         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
2813
2814         sc->sc_hub_idata[0] = 0;
2815
2816         if (UREAD2(sc, UHCI_PORTSC1) & (UHCI_PORTSC_CSC |
2817             UHCI_PORTSC_OCIC | UHCI_PORTSC_RD)) {
2818                 sc->sc_hub_idata[0] |= 1 << 1;
2819         }
2820         if (UREAD2(sc, UHCI_PORTSC2) & (UHCI_PORTSC_CSC |
2821             UHCI_PORTSC_OCIC | UHCI_PORTSC_RD)) {
2822                 sc->sc_hub_idata[0] |= 1 << 2;
2823         }
2824
2825         /* restart timer */
2826         usb_callout_reset(&sc->sc_root_intr, hz,
2827             (void *)&uhci_root_intr, sc);
2828
2829         if (sc->sc_hub_idata[0] != 0) {
2830                 uhub_root_intr(&sc->sc_bus, sc->sc_hub_idata,
2831                     sizeof(sc->sc_hub_idata));
2832         }
2833 }
2834
2835 static void
2836 uhci_xfer_setup(struct usb_setup_params *parm)
2837 {
2838         struct usb_page_search page_info;
2839         struct usb_page_cache *pc;
2840         uhci_softc_t *sc;
2841         struct usb_xfer *xfer;
2842         void *last_obj;
2843         uint32_t ntd;
2844         uint32_t nqh;
2845         uint32_t nfixup;
2846         uint32_t n;
2847         uint16_t align;
2848
2849         sc = UHCI_BUS2SC(parm->udev->bus);
2850         xfer = parm->curr_xfer;
2851
2852         parm->hc_max_packet_size = 0x500;
2853         parm->hc_max_packet_count = 1;
2854         parm->hc_max_frame_size = 0x500;
2855
2856         /*
2857          * compute ntd and nqh
2858          */
2859         if (parm->methods == &uhci_device_ctrl_methods) {
2860                 xfer->flags_int.bdma_enable = 1;
2861                 xfer->flags_int.bdma_no_post_sync = 1;
2862
2863                 usbd_transfer_setup_sub(parm);
2864
2865                 /* see EHCI HC driver for proof of "ntd" formula */
2866
2867                 nqh = 1;
2868                 ntd = ((2 * xfer->nframes) + 1  /* STATUS */
2869                     + (xfer->max_data_length / xfer->max_frame_size));
2870
2871         } else if (parm->methods == &uhci_device_bulk_methods) {
2872                 xfer->flags_int.bdma_enable = 1;
2873                 xfer->flags_int.bdma_no_post_sync = 1;
2874
2875                 usbd_transfer_setup_sub(parm);
2876
2877                 nqh = 1;
2878                 ntd = ((2 * xfer->nframes)
2879                     + (xfer->max_data_length / xfer->max_frame_size));
2880
2881         } else if (parm->methods == &uhci_device_intr_methods) {
2882                 xfer->flags_int.bdma_enable = 1;
2883                 xfer->flags_int.bdma_no_post_sync = 1;
2884
2885                 usbd_transfer_setup_sub(parm);
2886
2887                 nqh = 1;
2888                 ntd = ((2 * xfer->nframes)
2889                     + (xfer->max_data_length / xfer->max_frame_size));
2890
2891         } else if (parm->methods == &uhci_device_isoc_methods) {
2892                 xfer->flags_int.bdma_enable = 1;
2893                 xfer->flags_int.bdma_no_post_sync = 1;
2894
2895                 usbd_transfer_setup_sub(parm);
2896
2897                 nqh = 0;
2898                 ntd = xfer->nframes;
2899
2900         } else {
2901
2902                 usbd_transfer_setup_sub(parm);
2903
2904                 nqh = 0;
2905                 ntd = 0;
2906         }
2907
2908         if (parm->err) {
2909                 return;
2910         }
2911         /*
2912          * NOTE: the UHCI controller requires that
2913          * every packet must be contiguous on
2914          * the same USB memory page !
2915          */
2916         nfixup = (parm->bufsize / USB_PAGE_SIZE) + 1;
2917
2918         /*
2919          * Compute a suitable power of two alignment
2920          * for our "max_frame_size" fixup buffer(s):
2921          */
2922         align = xfer->max_frame_size;
2923         n = 0;
2924         while (align) {
2925                 align >>= 1;
2926                 n++;
2927         }
2928
2929         /* check for power of two */
2930         if (!(xfer->max_frame_size &
2931             (xfer->max_frame_size - 1))) {
2932                 n--;
2933         }
2934         /*
2935          * We don't allow alignments of
2936          * less than 8 bytes:
2937          *
2938          * NOTE: Allocating using an aligment
2939          * of 1 byte has special meaning!
2940          */
2941         if (n < 3) {
2942                 n = 3;
2943         }
2944         align = (1 << n);
2945
2946         if (usbd_transfer_setup_sub_malloc(
2947             parm, &pc, xfer->max_frame_size,
2948             align, nfixup)) {
2949                 parm->err = USB_ERR_NOMEM;
2950                 return;
2951         }
2952         xfer->buf_fixup = pc;
2953
2954 alloc_dma_set:
2955
2956         if (parm->err) {
2957                 return;
2958         }
2959         last_obj = NULL;
2960
2961         if (usbd_transfer_setup_sub_malloc(
2962             parm, &pc, sizeof(uhci_td_t),
2963             UHCI_TD_ALIGN, ntd)) {
2964                 parm->err = USB_ERR_NOMEM;
2965                 return;
2966         }
2967         if (parm->buf) {
2968                 for (n = 0; n != ntd; n++) {
2969                         uhci_td_t *td;
2970
2971                         usbd_get_page(pc + n, 0, &page_info);
2972
2973                         td = page_info.buffer;
2974
2975                         /* init TD */
2976                         if ((parm->methods == &uhci_device_bulk_methods) ||
2977                             (parm->methods == &uhci_device_ctrl_methods) ||
2978                             (parm->methods == &uhci_device_intr_methods)) {
2979                                 /* set depth first bit */
2980                                 td->td_self = htole32(page_info.physaddr |
2981                                     UHCI_PTR_TD | UHCI_PTR_VF);
2982                         } else {
2983                                 td->td_self = htole32(page_info.physaddr |
2984                                     UHCI_PTR_TD);
2985                         }
2986
2987                         td->obj_next = last_obj;
2988                         td->page_cache = pc + n;
2989
2990                         last_obj = td;
2991
2992                         usb_pc_cpu_flush(pc + n);
2993                 }
2994         }
2995         xfer->td_start[xfer->flags_int.curr_dma_set] = last_obj;
2996
2997         last_obj = NULL;
2998
2999         if (usbd_transfer_setup_sub_malloc(
3000             parm, &pc, sizeof(uhci_qh_t),
3001             UHCI_QH_ALIGN, nqh)) {
3002                 parm->err = USB_ERR_NOMEM;
3003                 return;
3004         }
3005         if (parm->buf) {
3006                 for (n = 0; n != nqh; n++) {
3007                         uhci_qh_t *qh;
3008
3009                         usbd_get_page(pc + n, 0, &page_info);
3010
3011                         qh = page_info.buffer;
3012
3013                         /* init QH */
3014                         qh->qh_self = htole32(page_info.physaddr | UHCI_PTR_QH);
3015                         qh->obj_next = last_obj;
3016                         qh->page_cache = pc + n;
3017
3018                         last_obj = qh;
3019
3020                         usb_pc_cpu_flush(pc + n);
3021                 }
3022         }
3023         xfer->qh_start[xfer->flags_int.curr_dma_set] = last_obj;
3024
3025         if (!xfer->flags_int.curr_dma_set) {
3026                 xfer->flags_int.curr_dma_set = 1;
3027                 goto alloc_dma_set;
3028         }
3029 }
3030
3031 static void
3032 uhci_ep_init(struct usb_device *udev, struct usb_endpoint_descriptor *edesc,
3033     struct usb_endpoint *ep)
3034 {
3035         uhci_softc_t *sc = UHCI_BUS2SC(udev->bus);
3036
3037         DPRINTFN(2, "endpoint=%p, addr=%d, endpt=%d, mode=%d (%d)\n",
3038             ep, udev->address,
3039             edesc->bEndpointAddress, udev->flags.usb_mode,
3040             sc->sc_addr);
3041
3042         if (udev->device_index != sc->sc_addr) {
3043                 switch (edesc->bmAttributes & UE_XFERTYPE) {
3044                 case UE_CONTROL:
3045                         ep->methods = &uhci_device_ctrl_methods;
3046                         break;
3047                 case UE_INTERRUPT:
3048                         ep->methods = &uhci_device_intr_methods;
3049                         break;
3050                 case UE_ISOCHRONOUS:
3051                         if (udev->speed == USB_SPEED_FULL) {
3052                                 ep->methods = &uhci_device_isoc_methods;
3053                         }
3054                         break;
3055                 case UE_BULK:
3056                         ep->methods = &uhci_device_bulk_methods;
3057                         break;
3058                 default:
3059                         /* do nothing */
3060                         break;
3061                 }
3062         }
3063 }
3064
3065 static void
3066 uhci_xfer_unsetup(struct usb_xfer *xfer)
3067 {
3068         return;
3069 }
3070
3071 static void
3072 uhci_get_dma_delay(struct usb_device *udev, uint32_t *pus)
3073 {
3074         /*
3075          * Wait until hardware has finished any possible use of the
3076          * transfer descriptor(s) and QH
3077          */
3078         *pus = (1125);                  /* microseconds */
3079 }
3080
3081 static void
3082 uhci_device_resume(struct usb_device *udev)
3083 {
3084         struct uhci_softc *sc = UHCI_BUS2SC(udev->bus);
3085         struct usb_xfer *xfer;
3086         const struct usb_pipe_methods *methods;
3087         uhci_qh_t *qh;
3088
3089         DPRINTF("\n");
3090
3091         USB_BUS_LOCK(udev->bus);
3092
3093         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
3094
3095                 if (xfer->xroot->udev == udev) {
3096
3097                         methods = xfer->endpoint->methods;
3098                         qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
3099
3100                         if (methods == &uhci_device_bulk_methods) {
3101                                 UHCI_APPEND_QH(qh, sc->sc_bulk_p_last);
3102                                 uhci_add_loop(sc);
3103                                 xfer->flags_int.bandwidth_reclaimed = 1;
3104                         }
3105                         if (methods == &uhci_device_ctrl_methods) {
3106                                 if (xfer->xroot->udev->speed == USB_SPEED_LOW) {
3107                                         UHCI_APPEND_QH(qh, sc->sc_ls_ctl_p_last);
3108                                 } else {
3109                                         UHCI_APPEND_QH(qh, sc->sc_fs_ctl_p_last);
3110                                 }
3111                         }
3112                         if (methods == &uhci_device_intr_methods) {
3113                                 UHCI_APPEND_QH(qh, sc->sc_intr_p_last[xfer->qh_pos]);
3114                         }
3115                 }
3116         }
3117
3118         USB_BUS_UNLOCK(udev->bus);
3119
3120         return;
3121 }
3122
3123 static void
3124 uhci_device_suspend(struct usb_device *udev)
3125 {
3126         struct uhci_softc *sc = UHCI_BUS2SC(udev->bus);
3127         struct usb_xfer *xfer;
3128         const struct usb_pipe_methods *methods;
3129         uhci_qh_t *qh;
3130
3131         DPRINTF("\n");
3132
3133         USB_BUS_LOCK(udev->bus);
3134
3135         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
3136
3137                 if (xfer->xroot->udev == udev) {
3138
3139                         methods = xfer->endpoint->methods;
3140                         qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
3141
3142                         if (xfer->flags_int.bandwidth_reclaimed) {
3143                                 xfer->flags_int.bandwidth_reclaimed = 0;
3144                                 uhci_rem_loop(sc);
3145                         }
3146                         if (methods == &uhci_device_bulk_methods) {
3147                                 UHCI_REMOVE_QH(qh, sc->sc_bulk_p_last);
3148                         }
3149                         if (methods == &uhci_device_ctrl_methods) {
3150                                 if (xfer->xroot->udev->speed == USB_SPEED_LOW) {
3151                                         UHCI_REMOVE_QH(qh, sc->sc_ls_ctl_p_last);
3152                                 } else {
3153                                         UHCI_REMOVE_QH(qh, sc->sc_fs_ctl_p_last);
3154                                 }
3155                         }
3156                         if (methods == &uhci_device_intr_methods) {
3157                                 UHCI_REMOVE_QH(qh, sc->sc_intr_p_last[xfer->qh_pos]);
3158                         }
3159                 }
3160         }
3161
3162         USB_BUS_UNLOCK(udev->bus);
3163
3164         return;
3165 }
3166
3167 static void
3168 uhci_set_hw_power_sleep(struct usb_bus *bus, uint32_t state)
3169 {
3170         struct uhci_softc *sc = UHCI_BUS2SC(bus);
3171
3172         switch (state) {
3173         case USB_HW_POWER_SUSPEND:
3174         case USB_HW_POWER_SHUTDOWN:
3175                 uhci_suspend(sc);
3176                 break;
3177         case USB_HW_POWER_RESUME:
3178                 uhci_resume(sc);
3179                 break;
3180         default:
3181                 break;
3182         }
3183 }
3184
3185 static void
3186 uhci_set_hw_power(struct usb_bus *bus)
3187 {
3188         struct uhci_softc *sc = UHCI_BUS2SC(bus);
3189         uint32_t flags;
3190
3191         DPRINTF("\n");
3192
3193         USB_BUS_LOCK(bus);
3194
3195         flags = bus->hw_power_state;
3196
3197         /*
3198          * WARNING: Some FULL speed USB devices require periodic SOF
3199          * messages! If any USB devices are connected through the
3200          * UHCI, power save will be disabled!
3201          */
3202         if (flags & (USB_HW_POWER_CONTROL |
3203             USB_HW_POWER_NON_ROOT_HUB |
3204             USB_HW_POWER_BULK |
3205             USB_HW_POWER_INTERRUPT |
3206             USB_HW_POWER_ISOC)) {
3207                 DPRINTF("Some USB transfer is "
3208                     "active on unit %u.\n",
3209                     device_get_unit(sc->sc_bus.bdev));
3210                 uhci_restart(sc);
3211         } else {
3212                 DPRINTF("Power save on unit %u.\n",
3213                     device_get_unit(sc->sc_bus.bdev));
3214                 UHCICMD(sc, UHCI_CMD_MAXP);
3215         }
3216
3217         USB_BUS_UNLOCK(bus);
3218
3219         return;
3220 }
3221
3222
3223 static const struct usb_bus_methods uhci_bus_methods =
3224 {
3225         .endpoint_init = uhci_ep_init,
3226         .xfer_setup = uhci_xfer_setup,
3227         .xfer_unsetup = uhci_xfer_unsetup,
3228         .get_dma_delay = uhci_get_dma_delay,
3229         .device_resume = uhci_device_resume,
3230         .device_suspend = uhci_device_suspend,
3231         .set_hw_power = uhci_set_hw_power,
3232         .set_hw_power_sleep = uhci_set_hw_power_sleep,
3233         .roothub_exec = uhci_roothub_exec,
3234         .xfer_poll = uhci_do_poll,
3235 };