sys/dev/usb/controller/uhci.c

   1 /* $FreeBSD$ */
   2 /*-
   3  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   4  *
   5  * Copyright (c) 2008 Hans Petter Selasky. All rights reserved.
   6  * Copyright (c) 1998 The NetBSD Foundation, Inc. All rights reserved.
   7  * Copyright (c) 1998 Lennart Augustsson. All rights reserved.
   8  *
   9  * Redistribution and use in source and binary forms, with or without
  10  * modification, are permitted provided that the following conditions
  11  * are met:
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions and the following disclaimer.
  14  * 2. Redistributions in binary form must reproduce the above copyright
  15  *    notice, this list of conditions and the following disclaimer in the
  16  *    documentation and/or other materials provided with the distribution.
  17  *
  18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  21  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  28  * SUCH DAMAGE.
  29  */
  30
  31 /*
  32  * USB Universal Host Controller driver.
  33  * Handles e.g. PIIX3 and PIIX4.
  34  *
  35  * UHCI spec: http://developer.intel.com/design/USB/UHCI11D.htm
  36  * USB spec:  http://www.usb.org/developers/docs/usbspec.zip
  37  * PIIXn spec: ftp://download.intel.com/design/intarch/datashts/29055002.pdf
  38  *             ftp://download.intel.com/design/intarch/datashts/29056201.pdf
  39  */
  40
  41 #ifdef USB_GLOBAL_INCLUDE_FILE
  42 #include USB_GLOBAL_INCLUDE_FILE
  43 #else
  44 #include <sys/stdint.h>
  45 #include <sys/stddef.h>
  46 #include <sys/param.h>
  47 #include <sys/queue.h>
  48 #include <sys/types.h>
  49 #include <sys/systm.h>
  50 #include <sys/kernel.h>
  51 #include <sys/bus.h>
  52 #include <sys/module.h>
  53 #include <sys/lock.h>
  54 #include <sys/mutex.h>
  55 #include <sys/condvar.h>
  56 #include <sys/sysctl.h>
  57 #include <sys/sx.h>
  58 #include <sys/unistd.h>
  59 #include <sys/callout.h>
  60 #include <sys/malloc.h>
  61 #include <sys/priv.h>
  62
  63 #include <dev/usb/usb.h>
  64 #include <dev/usb/usbdi.h>
  65
  66 #define USB_DEBUG_VAR uhcidebug
  67
  68 #include <dev/usb/usb_core.h>
  69 #include <dev/usb/usb_debug.h>
  70 #include <dev/usb/usb_busdma.h>
  71 #include <dev/usb/usb_process.h>
  72 #include <dev/usb/usb_transfer.h>
  73 #include <dev/usb/usb_device.h>
  74 #include <dev/usb/usb_hub.h>
  75 #include <dev/usb/usb_util.h>
  76
  77 #include <dev/usb/usb_controller.h>
  78 #include <dev/usb/usb_bus.h>
  79 #endif                  /* USB_GLOBAL_INCLUDE_FILE */
  80
  81 #include <dev/usb/controller/uhci.h>
  82 #include <dev/usb/controller/uhcireg.h>
  83
  84 #define alt_next next
  85 #define UHCI_BUS2SC(bus) \
  86         __containerof(bus, uhci_softc_t, sc_bus)
  87
  88 #ifdef USB_DEBUG
  89 static int uhcidebug = 0;
  90 static int uhcinoloop = 0;
  91
  92 static SYSCTL_NODE(_hw_usb, OID_AUTO, uhci, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
  93     "USB uhci");
  94 SYSCTL_INT(_hw_usb_uhci, OID_AUTO, debug, CTLFLAG_RWTUN,
  95     &uhcidebug, 0, "uhci debug level");
  96 SYSCTL_INT(_hw_usb_uhci, OID_AUTO, loop, CTLFLAG_RWTUN,
  97     &uhcinoloop, 0, "uhci noloop");
  98
  99 static void uhci_dumpregs(uhci_softc_t *sc);
 100 static void uhci_dump_tds(uhci_td_t *td);
 101
 102 #endif
 103
 104 #define UBARR(sc) bus_space_barrier((sc)->sc_io_tag, (sc)->sc_io_hdl, 0, (sc)->sc_io_size, \
 105                         BUS_SPACE_BARRIER_READ|BUS_SPACE_BARRIER_WRITE)
 106 #define UWRITE1(sc, r, x) \
 107  do { UBARR(sc); bus_space_write_1((sc)->sc_io_tag, (sc)->sc_io_hdl, (r), (x)); \
 108  } while (/*CONSTCOND*/0)
 109 #define UWRITE2(sc, r, x) \
 110  do { UBARR(sc); bus_space_write_2((sc)->sc_io_tag, (sc)->sc_io_hdl, (r), (x)); \
 111  } while (/*CONSTCOND*/0)
 112 #define UWRITE4(sc, r, x) \
 113  do { UBARR(sc); bus_space_write_4((sc)->sc_io_tag, (sc)->sc_io_hdl, (r), (x)); \
 114  } while (/*CONSTCOND*/0)
 115 #define UREAD1(sc, r) (UBARR(sc), bus_space_read_1((sc)->sc_io_tag, (sc)->sc_io_hdl, (r)))
 116 #define UREAD2(sc, r) (UBARR(sc), bus_space_read_2((sc)->sc_io_tag, (sc)->sc_io_hdl, (r)))
 117 #define UREAD4(sc, r) (UBARR(sc), bus_space_read_4((sc)->sc_io_tag, (sc)->sc_io_hdl, (r)))
 118
 119 #define UHCICMD(sc, cmd) UWRITE2(sc, UHCI_CMD, cmd)
 120 #define UHCISTS(sc) UREAD2(sc, UHCI_STS)
 121
 122 #define UHCI_RESET_TIMEOUT 100          /* ms, reset timeout */
 123
 124 #define UHCI_INTR_ENDPT 1
 125
 126 struct uhci_mem_layout {
 127         struct usb_page_search buf_res;
 128         struct usb_page_search fix_res;
 129
 130         struct usb_page_cache *buf_pc;
 131         struct usb_page_cache *fix_pc;
 132
 133         uint32_t buf_offset;
 134
 135         uint16_t max_frame_size;
 136 };
 137
 138 struct uhci_std_temp {
 139         struct uhci_mem_layout ml;
 140         uhci_td_t *td;
 141         uhci_td_t *td_next;
 142         uint32_t average;
 143         uint32_t td_status;
 144         uint32_t td_token;
 145         uint32_t len;
 146         uint16_t max_frame_size;
 147         uint8_t shortpkt;
 148         uint8_t setup_alt_next;
 149         uint8_t last_frame;
 150 };
 151
 152 static const struct usb_bus_methods uhci_bus_methods;
 153 static const struct usb_pipe_methods uhci_device_bulk_methods;
 154 static const struct usb_pipe_methods uhci_device_ctrl_methods;
 155 static const struct usb_pipe_methods uhci_device_intr_methods;
 156 static const struct usb_pipe_methods uhci_device_isoc_methods;
 157
 158 static uint8_t  uhci_restart(uhci_softc_t *sc);
 159 static void     uhci_do_poll(struct usb_bus *);
 160 static void     uhci_device_done(struct usb_xfer *, usb_error_t);
 161 static void     uhci_transfer_intr_enqueue(struct usb_xfer *);
 162 static void     uhci_timeout(void *);
 163 static uint8_t  uhci_check_transfer(struct usb_xfer *);
 164 static void     uhci_root_intr(uhci_softc_t *sc);
 165
 166 void
 167 uhci_iterate_hw_softc(struct usb_bus *bus, usb_bus_mem_sub_cb_t *cb)
 168 {
 169         struct uhci_softc *sc = UHCI_BUS2SC(bus);
 170         uint32_t i;
 171
 172         cb(bus, &sc->sc_hw.pframes_pc, &sc->sc_hw.pframes_pg,
 173             sizeof(uint32_t) * UHCI_FRAMELIST_COUNT, UHCI_FRAMELIST_ALIGN);
 174
 175         cb(bus, &sc->sc_hw.ls_ctl_start_pc, &sc->sc_hw.ls_ctl_start_pg,
 176             sizeof(uhci_qh_t), UHCI_QH_ALIGN);
 177
 178         cb(bus, &sc->sc_hw.fs_ctl_start_pc, &sc->sc_hw.fs_ctl_start_pg,
 179             sizeof(uhci_qh_t), UHCI_QH_ALIGN);
 180
 181         cb(bus, &sc->sc_hw.bulk_start_pc, &sc->sc_hw.bulk_start_pg,
 182             sizeof(uhci_qh_t), UHCI_QH_ALIGN);
 183
 184         cb(bus, &sc->sc_hw.last_qh_pc, &sc->sc_hw.last_qh_pg,
 185             sizeof(uhci_qh_t), UHCI_QH_ALIGN);
 186
 187         cb(bus, &sc->sc_hw.last_td_pc, &sc->sc_hw.last_td_pg,
 188             sizeof(uhci_td_t), UHCI_TD_ALIGN);
 189
 190         for (i = 0; i != UHCI_VFRAMELIST_COUNT; i++) {
 191                 cb(bus, sc->sc_hw.isoc_start_pc + i,
 192                     sc->sc_hw.isoc_start_pg + i,
 193                     sizeof(uhci_td_t), UHCI_TD_ALIGN);
 194         }
 195
 196         for (i = 0; i != UHCI_IFRAMELIST_COUNT; i++) {
 197                 cb(bus, sc->sc_hw.intr_start_pc + i,
 198                     sc->sc_hw.intr_start_pg + i,
 199                     sizeof(uhci_qh_t), UHCI_QH_ALIGN);
 200         }
 201 }
 202
 203 static void
 204 uhci_mem_layout_init(struct uhci_mem_layout *ml, struct usb_xfer *xfer)
 205 {
 206         ml->buf_pc = xfer->frbuffers + 0;
 207         ml->fix_pc = xfer->buf_fixup;
 208
 209         ml->buf_offset = 0;
 210
 211         ml->max_frame_size = xfer->max_frame_size;
 212 }
 213
 214 static void
 215 uhci_mem_layout_fixup(struct uhci_mem_layout *ml, struct uhci_td *td)
 216 {
 217         usbd_get_page(ml->buf_pc, ml->buf_offset, &ml->buf_res);
 218
 219         if (ml->buf_res.length < td->len) {
 220                 /* need to do a fixup */
 221
 222                 usbd_get_page(ml->fix_pc, 0, &ml->fix_res);
 223
 224                 td->td_buffer = htole32(ml->fix_res.physaddr);
 225
 226                 /*
 227                  * The UHCI driver cannot handle
 228                  * page crossings, so a fixup is
 229                  * needed:
 230                  *
 231                  *  +----+----+ - - -
 232                  *  | YYY|Y   |
 233                  *  +----+----+ - - -
 234                  *     \    \
 235                  *      \    \
 236                  *       +----+
 237                  *       |YYYY|  (fixup)
 238                  *       +----+
 239                  */
 240
 241                 if ((td->td_token & htole32(UHCI_TD_PID)) ==
 242                     htole32(UHCI_TD_PID_IN)) {
 243                         td->fix_pc = ml->fix_pc;
 244                         usb_pc_cpu_invalidate(ml->fix_pc);
 245
 246                 } else {
 247                         td->fix_pc = NULL;
 248
 249                         /* copy data to fixup location */
 250
 251                         usbd_copy_out(ml->buf_pc, ml->buf_offset,
 252                             ml->fix_res.buffer, td->len);
 253
 254                         usb_pc_cpu_flush(ml->fix_pc);
 255                 }
 256
 257                 /* prepare next fixup */
 258
 259                 ml->fix_pc++;
 260
 261         } else {
 262                 td->td_buffer = htole32(ml->buf_res.physaddr);
 263                 td->fix_pc = NULL;
 264         }
 265
 266         /* prepare next data location */
 267
 268         ml->buf_offset += td->len;
 269 }
 270
 271 /*
 272  * Return values:
 273  * 0: Success
 274  * Else: Failure
 275  */
 276 static uint8_t
 277 uhci_restart(uhci_softc_t *sc)
 278 {
 279         struct usb_page_search buf_res;
 280
 281         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
 282
 283         if (UREAD2(sc, UHCI_CMD) & UHCI_CMD_RS) {
 284                 DPRINTFN(2, "Already started\n");
 285                 return (0);
 286         }
 287
 288         DPRINTFN(2, "Restarting\n");
 289
 290         usbd_get_page(&sc->sc_hw.pframes_pc, 0, &buf_res);
 291
 292         /* Reload fresh base address */
 293         UWRITE4(sc, UHCI_FLBASEADDR, buf_res.physaddr);
 294
 295         /*
 296          * Assume 64 byte packets at frame end and start HC controller:
 297          */
 298         UHCICMD(sc, (UHCI_CMD_MAXP | UHCI_CMD_RS));
 299
 300         /* wait 10 milliseconds */
 301
 302         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 100);
 303
 304         /* check that controller has started */
 305
 306         if (UREAD2(sc, UHCI_STS) & UHCI_STS_HCH) {
 307                 DPRINTFN(2, "Failed\n");
 308                 return (1);
 309         }
 310         return (0);
 311 }
 312
 313 void
 314 uhci_reset(uhci_softc_t *sc)
 315 {
 316         uint16_t n;
 317
 318         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
 319
 320         DPRINTF("resetting the HC\n");
 321
 322         /* disable interrupts */
 323
 324         UWRITE2(sc, UHCI_INTR, 0);
 325
 326         /* global reset */
 327
 328         UHCICMD(sc, UHCI_CMD_GRESET);
 329
 330         /* wait */
 331
 332         usb_pause_mtx(&sc->sc_bus.bus_mtx,
 333             USB_MS_TO_TICKS(USB_BUS_RESET_DELAY));
 334
 335         /* terminate all transfers */
 336
 337         UHCICMD(sc, UHCI_CMD_HCRESET);
 338
 339         /* the reset bit goes low when the controller is done */
 340
 341         n = UHCI_RESET_TIMEOUT;
 342         while (n--) {
 343                 /* wait one millisecond */
 344
 345                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 1000);
 346
 347                 if (!(UREAD2(sc, UHCI_CMD) & UHCI_CMD_HCRESET)) {
 348                         goto done_1;
 349                 }
 350         }
 351
 352         device_printf(sc->sc_bus.bdev,
 353             "controller did not reset\n");
 354
 355 done_1:
 356
 357         n = 10;
 358         while (n--) {
 359                 /* wait one millisecond */
 360
 361                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 1000);
 362
 363                 /* check if HC is stopped */
 364                 if (UREAD2(sc, UHCI_STS) & UHCI_STS_HCH) {
 365                         goto done_2;
 366                 }
 367         }
 368
 369         device_printf(sc->sc_bus.bdev,
 370             "controller did not stop\n");
 371
 372 done_2:
 373
 374         /* reset frame number */
 375         UWRITE2(sc, UHCI_FRNUM, 0);
 376         /* set default SOF value */
 377         UWRITE1(sc, UHCI_SOF, 0x40);
 378
 379         USB_BUS_UNLOCK(&sc->sc_bus);
 380
 381         /* stop root interrupt */
 382         usb_callout_drain(&sc->sc_root_intr);
 383
 384         USB_BUS_LOCK(&sc->sc_bus);
 385 }
 386
 387 static void
 388 uhci_start(uhci_softc_t *sc)
 389 {
 390         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
 391
 392         DPRINTFN(2, "enabling\n");
 393
 394         /* enable interrupts */
 395
 396         UWRITE2(sc, UHCI_INTR,
 397             (UHCI_INTR_TOCRCIE |
 398             UHCI_INTR_RIE |
 399             UHCI_INTR_IOCE |
 400             UHCI_INTR_SPIE));
 401
 402         if (uhci_restart(sc)) {
 403                 device_printf(sc->sc_bus.bdev,
 404                     "cannot start HC controller\n");
 405         }
 406
 407         /* start root interrupt */
 408         uhci_root_intr(sc);
 409 }
 410
 411 static struct uhci_qh *
 412 uhci_init_qh(struct usb_page_cache *pc)
 413 {
 414         struct usb_page_search buf_res;
 415         struct uhci_qh *qh;
 416
 417         usbd_get_page(pc, 0, &buf_res);
 418
 419         qh = buf_res.buffer;
 420
 421         qh->qh_self =
 422             htole32(buf_res.physaddr) |
 423             htole32(UHCI_PTR_QH);
 424
 425         qh->page_cache = pc;
 426
 427         return (qh);
 428 }
 429
 430 static struct uhci_td *
 431 uhci_init_td(struct usb_page_cache *pc)
 432 {
 433         struct usb_page_search buf_res;
 434         struct uhci_td *td;
 435
 436         usbd_get_page(pc, 0, &buf_res);
 437
 438         td = buf_res.buffer;
 439
 440         td->td_self =
 441             htole32(buf_res.physaddr) |
 442             htole32(UHCI_PTR_TD);
 443
 444         td->page_cache = pc;
 445
 446         return (td);
 447 }
 448
 449 usb_error_t
 450 uhci_init(uhci_softc_t *sc)
 451 {
 452         uint16_t bit;
 453         uint16_t x;
 454         uint16_t y;
 455
 456         DPRINTF("start\n");
 457
 458         usb_callout_init_mtx(&sc->sc_root_intr, &sc->sc_bus.bus_mtx, 0);
 459
 460 #ifdef USB_DEBUG
 461         if (uhcidebug > 2) {
 462                 uhci_dumpregs(sc);
 463         }
 464 #endif
 465         /*
 466          * Setup QH's
 467          */
 468         sc->sc_ls_ctl_p_last =
 469             uhci_init_qh(&sc->sc_hw.ls_ctl_start_pc);
 470
 471         sc->sc_fs_ctl_p_last =
 472             uhci_init_qh(&sc->sc_hw.fs_ctl_start_pc);
 473
 474         sc->sc_bulk_p_last =
 475             uhci_init_qh(&sc->sc_hw.bulk_start_pc);
 476 #if 0
 477         sc->sc_reclaim_qh_p =
 478             sc->sc_fs_ctl_p_last;
 479 #else
 480         /* setup reclaim looping point */
 481         sc->sc_reclaim_qh_p =
 482             sc->sc_bulk_p_last;
 483 #endif
 484
 485         sc->sc_last_qh_p =
 486             uhci_init_qh(&sc->sc_hw.last_qh_pc);
 487
 488         sc->sc_last_td_p =
 489             uhci_init_td(&sc->sc_hw.last_td_pc);
 490
 491         for (x = 0; x != UHCI_VFRAMELIST_COUNT; x++) {
 492                 sc->sc_isoc_p_last[x] =
 493                     uhci_init_td(sc->sc_hw.isoc_start_pc + x);
 494         }
 495
 496         for (x = 0; x != UHCI_IFRAMELIST_COUNT; x++) {
 497                 sc->sc_intr_p_last[x] =
 498                     uhci_init_qh(sc->sc_hw.intr_start_pc + x);
 499         }
 500
 501         /*
 502          * the QHs are arranged to give poll intervals that are
 503          * powers of 2 times 1ms
 504          */
 505         bit = UHCI_IFRAMELIST_COUNT / 2;
 506         while (bit) {
 507                 x = bit;
 508                 while (x & bit) {
 509                         uhci_qh_t *qh_x;
 510                         uhci_qh_t *qh_y;
 511
 512                         y = (x ^ bit) | (bit / 2);
 513
 514                         /*
 515                          * the next QH has half the poll interval
 516                          */
 517                         qh_x = sc->sc_intr_p_last[x];
 518                         qh_y = sc->sc_intr_p_last[y];
 519
 520                         qh_x->h_next = NULL;
 521                         qh_x->qh_h_next = qh_y->qh_self;
 522                         qh_x->e_next = NULL;
 523                         qh_x->qh_e_next = htole32(UHCI_PTR_T);
 524                         x++;
 525                 }
 526                 bit >>= 1;
 527         }
 528
 529         if (1) {
 530                 uhci_qh_t *qh_ls;
 531                 uhci_qh_t *qh_intr;
 532
 533                 qh_ls = sc->sc_ls_ctl_p_last;
 534                 qh_intr = sc->sc_intr_p_last[0];
 535
 536                 /* start QH for interrupt traffic */
 537                 qh_intr->h_next = qh_ls;
 538                 qh_intr->qh_h_next = qh_ls->qh_self;
 539                 qh_intr->e_next = 0;
 540                 qh_intr->qh_e_next = htole32(UHCI_PTR_T);
 541         }
 542         for (x = 0; x != UHCI_VFRAMELIST_COUNT; x++) {
 543                 uhci_td_t *td_x;
 544                 uhci_qh_t *qh_intr;
 545
 546                 td_x = sc->sc_isoc_p_last[x];
 547                 qh_intr = sc->sc_intr_p_last[x | (UHCI_IFRAMELIST_COUNT / 2)];
 548
 549                 /* start TD for isochronous traffic */
 550                 td_x->next = NULL;
 551                 td_x->td_next = qh_intr->qh_self;
 552                 td_x->td_status = htole32(UHCI_TD_IOS);
 553                 td_x->td_token = htole32(0);
 554                 td_x->td_buffer = htole32(0);
 555         }
 556
 557         if (1) {
 558                 uhci_qh_t *qh_ls;
 559                 uhci_qh_t *qh_fs;
 560
 561                 qh_ls = sc->sc_ls_ctl_p_last;
 562                 qh_fs = sc->sc_fs_ctl_p_last;
 563
 564                 /* start QH where low speed control traffic will be queued */
 565                 qh_ls->h_next = qh_fs;
 566                 qh_ls->qh_h_next = qh_fs->qh_self;
 567                 qh_ls->e_next = 0;
 568                 qh_ls->qh_e_next = htole32(UHCI_PTR_T);
 569         }
 570         if (1) {
 571                 uhci_qh_t *qh_ctl;
 572                 uhci_qh_t *qh_blk;
 573                 uhci_qh_t *qh_lst;
 574                 uhci_td_t *td_lst;
 575
 576                 qh_ctl = sc->sc_fs_ctl_p_last;
 577                 qh_blk = sc->sc_bulk_p_last;
 578
 579                 /* start QH where full speed control traffic will be queued */
 580                 qh_ctl->h_next = qh_blk;
 581                 qh_ctl->qh_h_next = qh_blk->qh_self;
 582                 qh_ctl->e_next = 0;
 583                 qh_ctl->qh_e_next = htole32(UHCI_PTR_T);
 584
 585                 qh_lst = sc->sc_last_qh_p;
 586
 587                 /* start QH where bulk traffic will be queued */
 588                 qh_blk->h_next = qh_lst;
 589                 qh_blk->qh_h_next = qh_lst->qh_self;
 590                 qh_blk->e_next = 0;
 591                 qh_blk->qh_e_next = htole32(UHCI_PTR_T);
 592
 593                 td_lst = sc->sc_last_td_p;
 594
 595                 /* end QH which is used for looping the QHs */
 596                 qh_lst->h_next = 0;
 597                 qh_lst->qh_h_next = htole32(UHCI_PTR_T);        /* end of QH chain */
 598                 qh_lst->e_next = td_lst;
 599                 qh_lst->qh_e_next = td_lst->td_self;
 600
 601                 /*
 602                  * end TD which hangs from the last QH, to avoid a bug in the PIIX
 603                  * that makes it run berserk otherwise
 604                  */
 605                 td_lst->next = 0;
 606                 td_lst->td_next = htole32(UHCI_PTR_T);
 607                 td_lst->td_status = htole32(0); /* inactive */
 608                 td_lst->td_token = htole32(0);
 609                 td_lst->td_buffer = htole32(0);
 610         }
 611         if (1) {
 612                 struct usb_page_search buf_res;
 613                 uint32_t *pframes;
 614
 615                 usbd_get_page(&sc->sc_hw.pframes_pc, 0, &buf_res);
 616
 617                 pframes = buf_res.buffer;
 618
 619                 /*
 620                  * Setup UHCI framelist
 621                  *
 622                  * Execution order:
 623                  *
 624                  * pframes -> full speed isochronous -> interrupt QH's -> low
 625                  * speed control -> full speed control -> bulk transfers
 626                  *
 627                  */
 628
 629                 for (x = 0; x != UHCI_FRAMELIST_COUNT; x++) {
 630                         pframes[x] =
 631                             sc->sc_isoc_p_last[x % UHCI_VFRAMELIST_COUNT]->td_self;
 632                 }
 633         }
 634         /* flush all cache into memory */
 635
 636         usb_bus_mem_flush_all(&sc->sc_bus, &uhci_iterate_hw_softc);
 637
 638         /* set up the bus struct */
 639         sc->sc_bus.methods = &uhci_bus_methods;
 640
 641         USB_BUS_LOCK(&sc->sc_bus);
 642         /* reset the controller */
 643         uhci_reset(sc);
 644
 645         /* start the controller */
 646         uhci_start(sc);
 647         USB_BUS_UNLOCK(&sc->sc_bus);
 648
 649         /* catch lost interrupts */
 650         uhci_do_poll(&sc->sc_bus);
 651
 652         return (0);
 653 }
 654
 655 static void
 656 uhci_suspend(uhci_softc_t *sc)
 657 {
 658 #ifdef USB_DEBUG
 659         if (uhcidebug > 2) {
 660                 uhci_dumpregs(sc);
 661         }
 662 #endif
 663
 664         USB_BUS_LOCK(&sc->sc_bus);
 665
 666         /* stop the controller */
 667
 668         uhci_reset(sc);
 669
 670         /* enter global suspend */
 671
 672         UHCICMD(sc, UHCI_CMD_EGSM);
 673
 674         USB_BUS_UNLOCK(&sc->sc_bus);
 675 }
 676
 677 static void
 678 uhci_resume(uhci_softc_t *sc)
 679 {
 680         USB_BUS_LOCK(&sc->sc_bus);
 681
 682         /* reset the controller */
 683
 684         uhci_reset(sc);
 685
 686         /* force global resume */
 687
 688         UHCICMD(sc, UHCI_CMD_FGR);
 689
 690         /* and start traffic again */
 691
 692         uhci_start(sc);
 693
 694         USB_BUS_UNLOCK(&sc->sc_bus);
 695
 696 #ifdef USB_DEBUG
 697         if (uhcidebug > 2)
 698                 uhci_dumpregs(sc);
 699 #endif
 700
 701         /* catch lost interrupts */
 702         uhci_do_poll(&sc->sc_bus);
 703 }
 704
 705 #ifdef USB_DEBUG
 706 static void
 707 uhci_dumpregs(uhci_softc_t *sc)
 708 {
 709         DPRINTFN(0, "%s regs: cmd=%04x, sts=%04x, intr=%04x, frnum=%04x, "
 710             "flbase=%08x, sof=%04x, portsc1=%04x, portsc2=%04x\n",
 711             device_get_nameunit(sc->sc_bus.bdev),
 712             UREAD2(sc, UHCI_CMD),
 713             UREAD2(sc, UHCI_STS),
 714             UREAD2(sc, UHCI_INTR),
 715             UREAD2(sc, UHCI_FRNUM),
 716             UREAD4(sc, UHCI_FLBASEADDR),
 717             UREAD1(sc, UHCI_SOF),
 718             UREAD2(sc, UHCI_PORTSC1),
 719             UREAD2(sc, UHCI_PORTSC2));
 720 }
 721
 722 static uint8_t
 723 uhci_dump_td(uhci_td_t *p)
 724 {
 725         uint32_t td_next;
 726         uint32_t td_status;
 727         uint32_t td_token;
 728         uint8_t temp;
 729
 730         usb_pc_cpu_invalidate(p->page_cache);
 731
 732         td_next = le32toh(p->td_next);
 733         td_status = le32toh(p->td_status);
 734         td_token = le32toh(p->td_token);
 735
 736         /*
 737          * Check whether the link pointer in this TD marks the link pointer
 738          * as end of queue:
 739          */
 740         temp = ((td_next & UHCI_PTR_T) || (td_next == 0));
 741
 742         printf("TD(%p) at 0x%08x = link=0x%08x status=0x%08x "
 743             "token=0x%08x buffer=0x%08x\n",
 744             p,
 745             le32toh(p->td_self),
 746             td_next,
 747             td_status,
 748             td_token,
 749             le32toh(p->td_buffer));
 750
 751         printf("TD(%p) td_next=%s%s%s td_status=%s%s%s%s%s%s%s%s%s%s%s, errcnt=%d, actlen=%d pid=%02x,"
 752             "addr=%d,endpt=%d,D=%d,maxlen=%d\n",
 753             p,
 754             (td_next & 1) ? "-T" : "",
 755             (td_next & 2) ? "-Q" : "",
 756             (td_next & 4) ? "-VF" : "",
 757             (td_status & UHCI_TD_BITSTUFF) ? "-BITSTUFF" : "",
 758             (td_status & UHCI_TD_CRCTO) ? "-CRCTO" : "",
 759             (td_status & UHCI_TD_NAK) ? "-NAK" : "",
 760             (td_status & UHCI_TD_BABBLE) ? "-BABBLE" : "",
 761             (td_status & UHCI_TD_DBUFFER) ? "-DBUFFER" : "",
 762             (td_status & UHCI_TD_STALLED) ? "-STALLED" : "",
 763             (td_status & UHCI_TD_ACTIVE) ? "-ACTIVE" : "",
 764             (td_status & UHCI_TD_IOC) ? "-IOC" : "",
 765             (td_status & UHCI_TD_IOS) ? "-IOS" : "",
 766             (td_status & UHCI_TD_LS) ? "-LS" : "",
 767             (td_status & UHCI_TD_SPD) ? "-SPD" : "",
 768             UHCI_TD_GET_ERRCNT(td_status),
 769             UHCI_TD_GET_ACTLEN(td_status),
 770             UHCI_TD_GET_PID(td_token),
 771             UHCI_TD_GET_DEVADDR(td_token),
 772             UHCI_TD_GET_ENDPT(td_token),
 773             UHCI_TD_GET_DT(td_token),
 774             UHCI_TD_GET_MAXLEN(td_token));
 775
 776         return (temp);
 777 }
 778
 779 static uint8_t
 780 uhci_dump_qh(uhci_qh_t *sqh)
 781 {
 782         uint8_t temp;
 783         uint32_t qh_h_next;
 784         uint32_t qh_e_next;
 785
 786         usb_pc_cpu_invalidate(sqh->page_cache);
 787
 788         qh_h_next = le32toh(sqh->qh_h_next);
 789         qh_e_next = le32toh(sqh->qh_e_next);
 790
 791         DPRINTFN(0, "QH(%p) at 0x%08x: h_next=0x%08x e_next=0x%08x\n", sqh,
 792             le32toh(sqh->qh_self), qh_h_next, qh_e_next);
 793
 794         temp = ((((sqh->h_next != NULL) && !(qh_h_next & UHCI_PTR_T)) ? 1 : 0) |
 795             (((sqh->e_next != NULL) && !(qh_e_next & UHCI_PTR_T)) ? 2 : 0));
 796
 797         return (temp);
 798 }
 799
 800 static void
 801 uhci_dump_all(uhci_softc_t *sc)
 802 {
 803         uhci_dumpregs(sc);
 804         uhci_dump_qh(sc->sc_ls_ctl_p_last);
 805         uhci_dump_qh(sc->sc_fs_ctl_p_last);
 806         uhci_dump_qh(sc->sc_bulk_p_last);
 807         uhci_dump_qh(sc->sc_last_qh_p);
 808 }
 809
 810 static void
 811 uhci_dump_tds(uhci_td_t *td)
 812 {
 813         for (;
 814             td != NULL;
 815             td = td->obj_next) {
 816                 if (uhci_dump_td(td)) {
 817                         break;
 818                 }
 819         }
 820 }
 821
 822 #endif
 823
 824 /*
 825  * Let the last QH loop back to the full speed control transfer QH.
 826  * This is what intel calls "bandwidth reclamation" and improves
 827  * USB performance a lot for some devices.
 828  * If we are already looping, just count it.
 829  */
 830 static void
 831 uhci_add_loop(uhci_softc_t *sc)
 832 {
 833         struct uhci_qh *qh_lst;
 834         struct uhci_qh *qh_rec;
 835
 836 #ifdef USB_DEBUG
 837         if (uhcinoloop) {
 838                 return;
 839         }
 840 #endif
 841         if (++(sc->sc_loops) == 1) {
 842                 DPRINTFN(6, "add\n");
 843
 844                 qh_lst = sc->sc_last_qh_p;
 845                 qh_rec = sc->sc_reclaim_qh_p;
 846
 847                 /* NOTE: we don't loop back the soft pointer */
 848
 849                 qh_lst->qh_h_next = qh_rec->qh_self;
 850                 usb_pc_cpu_flush(qh_lst->page_cache);
 851         }
 852 }
 853
 854 static void
 855 uhci_rem_loop(uhci_softc_t *sc)
 856 {
 857         struct uhci_qh *qh_lst;
 858
 859 #ifdef USB_DEBUG
 860         if (uhcinoloop) {
 861                 return;
 862         }
 863 #endif
 864         if (--(sc->sc_loops) == 0) {
 865                 DPRINTFN(6, "remove\n");
 866
 867                 qh_lst = sc->sc_last_qh_p;
 868                 qh_lst->qh_h_next = htole32(UHCI_PTR_T);
 869                 usb_pc_cpu_flush(qh_lst->page_cache);
 870         }
 871 }
 872
 873 static void
 874 uhci_transfer_intr_enqueue(struct usb_xfer *xfer)
 875 {
 876         /* check for early completion */
 877         if (uhci_check_transfer(xfer)) {
 878                 return;
 879         }
 880         /* put transfer on interrupt queue */
 881         usbd_transfer_enqueue(&xfer->xroot->bus->intr_q, xfer);
 882
 883         /* start timeout, if any */
 884         if (xfer->timeout != 0) {
 885                 usbd_transfer_timeout_ms(xfer, &uhci_timeout, xfer->timeout);
 886         }
 887 }
 888
 889 #define UHCI_APPEND_TD(std,last) (last) = _uhci_append_td(std,last)
 890 static uhci_td_t *
 891 _uhci_append_td(uhci_td_t *std, uhci_td_t *last)
 892 {
 893         DPRINTFN(11, "%p to %p\n", std, last);
 894
 895         /* (sc->sc_bus.mtx) must be locked */
 896
 897         std->next = last->next;
 898         std->td_next = last->td_next;
 899
 900         std->prev = last;
 901
 902         usb_pc_cpu_flush(std->page_cache);
 903
 904         /*
 905          * the last->next->prev is never followed: std->next->prev = std;
 906          */
 907         last->next = std;
 908         last->td_next = std->td_self;
 909
 910         usb_pc_cpu_flush(last->page_cache);
 911
 912         return (std);
 913 }
 914
 915 #define UHCI_APPEND_QH(sqh,last) (last) = _uhci_append_qh(sqh,last)
 916 static uhci_qh_t *
 917 _uhci_append_qh(uhci_qh_t *sqh, uhci_qh_t *last)
 918 {
 919         DPRINTFN(11, "%p to %p\n", sqh, last);
 920
 921         if (sqh->h_prev != NULL) {
 922                 /* should not happen */
 923                 DPRINTFN(0, "QH already linked!\n");
 924                 return (last);
 925         }
 926         /* (sc->sc_bus.mtx) must be locked */
 927
 928         sqh->h_next = last->h_next;
 929         sqh->qh_h_next = last->qh_h_next;
 930
 931         sqh->h_prev = last;
 932
 933         usb_pc_cpu_flush(sqh->page_cache);
 934
 935         /*
 936          * The "last->h_next->h_prev" is never followed:
 937          *
 938          * "sqh->h_next->h_prev" = sqh;
 939          */
 940
 941         last->h_next = sqh;
 942         last->qh_h_next = sqh->qh_self;
 943
 944         usb_pc_cpu_flush(last->page_cache);
 945
 946         return (sqh);
 947 }
 948
 949 /**/
 950
 951 #define UHCI_REMOVE_TD(std,last) (last) = _uhci_remove_td(std,last)
 952 static uhci_td_t *
 953 _uhci_remove_td(uhci_td_t *std, uhci_td_t *last)
 954 {
 955         DPRINTFN(11, "%p from %p\n", std, last);
 956
 957         /* (sc->sc_bus.mtx) must be locked */
 958
 959         std->prev->next = std->next;
 960         std->prev->td_next = std->td_next;
 961
 962         usb_pc_cpu_flush(std->prev->page_cache);
 963
 964         if (std->next) {
 965                 std->next->prev = std->prev;
 966                 usb_pc_cpu_flush(std->next->page_cache);
 967         }
 968         return ((last == std) ? std->prev : last);
 969 }
 970
 971 #define UHCI_REMOVE_QH(sqh,last) (last) = _uhci_remove_qh(sqh,last)
 972 static uhci_qh_t *
 973 _uhci_remove_qh(uhci_qh_t *sqh, uhci_qh_t *last)
 974 {
 975         DPRINTFN(11, "%p from %p\n", sqh, last);
 976
 977         /* (sc->sc_bus.mtx) must be locked */
 978
 979         /* only remove if not removed from a queue */
 980         if (sqh->h_prev) {
 981                 sqh->h_prev->h_next = sqh->h_next;
 982                 sqh->h_prev->qh_h_next = sqh->qh_h_next;
 983
 984                 usb_pc_cpu_flush(sqh->h_prev->page_cache);
 985
 986                 if (sqh->h_next) {
 987                         sqh->h_next->h_prev = sqh->h_prev;
 988                         usb_pc_cpu_flush(sqh->h_next->page_cache);
 989                 }
 990                 last = ((last == sqh) ? sqh->h_prev : last);
 991
 992                 sqh->h_prev = 0;
 993
 994                 usb_pc_cpu_flush(sqh->page_cache);
 995         }
 996         return (last);
 997 }
 998
 999 static void
1000 uhci_isoc_done(uhci_softc_t *sc, struct usb_xfer *xfer)
1001 {
1002         struct usb_page_search res;
1003         uint32_t nframes = xfer->nframes;
1004         uint32_t status;
1005         uint32_t offset = 0;
1006         uint32_t *plen = xfer->frlengths;
1007         uint16_t len = 0;
1008         uhci_td_t *td = xfer->td_transfer_first;
1009         uhci_td_t **pp_last = &sc->sc_isoc_p_last[xfer->qh_pos];
1010
1011         DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
1012             xfer, xfer->endpoint);
1013
1014         /* sync any DMA memory before doing fixups */
1015
1016         usb_bdma_post_sync(xfer);
1017
1018         while (nframes--) {
1019                 if (td == NULL) {
1020                         panic("%s:%d: out of TD's\n",
1021                             __FUNCTION__, __LINE__);
1022                 }
1023                 if (pp_last >= &sc->sc_isoc_p_last[UHCI_VFRAMELIST_COUNT]) {
1024                         pp_last = &sc->sc_isoc_p_last[0];
1025                 }
1026 #ifdef USB_DEBUG
1027                 if (uhcidebug > 5) {
1028                         DPRINTF("isoc TD\n");
1029                         uhci_dump_td(td);
1030                 }
1031 #endif
1032                 usb_pc_cpu_invalidate(td->page_cache);
1033                 status = le32toh(td->td_status);
1034
1035                 len = UHCI_TD_GET_ACTLEN(status);
1036
1037                 if (len > *plen) {
1038                         len = *plen;
1039                 }
1040                 if (td->fix_pc) {
1041                         usbd_get_page(td->fix_pc, 0, &res);
1042
1043                         /* copy data from fixup location to real location */
1044
1045                         usb_pc_cpu_invalidate(td->fix_pc);
1046
1047                         usbd_copy_in(xfer->frbuffers, offset,
1048                             res.buffer, len);
1049                 }
1050                 offset += *plen;
1051
1052                 *plen = len;
1053
1054                 /* remove TD from schedule */
1055                 UHCI_REMOVE_TD(td, *pp_last);
1056
1057                 pp_last++;
1058                 plen++;
1059                 td = td->obj_next;
1060         }
1061
1062         xfer->aframes = xfer->nframes;
1063 }
1064
1065 static usb_error_t
1066 uhci_non_isoc_done_sub(struct usb_xfer *xfer)
1067 {
1068         struct usb_page_search res;
1069         uhci_td_t *td;
1070         uhci_td_t *td_alt_next;
1071         uint32_t status;
1072         uint32_t token;
1073         uint16_t len;
1074
1075         td = xfer->td_transfer_cache;
1076         td_alt_next = td->alt_next;
1077
1078         if (xfer->aframes != xfer->nframes) {
1079                 usbd_xfer_set_frame_len(xfer, xfer->aframes, 0);
1080         }
1081         while (1) {
1082                 usb_pc_cpu_invalidate(td->page_cache);
1083                 status = le32toh(td->td_status);
1084                 token = le32toh(td->td_token);
1085
1086                 /*
1087                  * Verify the status and add
1088                  * up the actual length:
1089                  */
1090
1091                 len = UHCI_TD_GET_ACTLEN(status);
1092                 if (len > td->len) {
1093                         /* should not happen */
1094                         DPRINTF("Invalid status length, "
1095                             "0x%04x/0x%04x bytes\n", len, td->len);
1096                         status |= UHCI_TD_STALLED;
1097
1098                 } else if ((xfer->aframes != xfer->nframes) && (len > 0)) {
1099                         if (td->fix_pc) {
1100                                 usbd_get_page(td->fix_pc, 0, &res);
1101
1102                                 /*
1103                                  * copy data from fixup location to real
1104                                  * location
1105                                  */
1106
1107                                 usb_pc_cpu_invalidate(td->fix_pc);
1108
1109                                 usbd_copy_in(xfer->frbuffers + xfer->aframes,
1110                                     xfer->frlengths[xfer->aframes], res.buffer, len);
1111                         }
1112                         /* update actual length */
1113
1114                         xfer->frlengths[xfer->aframes] += len;
1115                 }
1116                 /* Check for last transfer */
1117                 if (((void *)td) == xfer->td_transfer_last) {
1118                         td = NULL;
1119                         break;
1120                 }
1121                 if (status & UHCI_TD_STALLED) {
1122                         /* the transfer is finished */
1123                         td = NULL;
1124                         break;
1125                 }
1126                 /* Check for short transfer */
1127                 if (len != td->len) {
1128                         if (xfer->flags_int.short_frames_ok) {
1129                                 /* follow alt next */
1130                                 td = td->alt_next;
1131                         } else {
1132                                 /* the transfer is finished */
1133                                 td = NULL;
1134                         }
1135                         break;
1136                 }
1137                 td = td->obj_next;
1138
1139                 if (td->alt_next != td_alt_next) {
1140                         /* this USB frame is complete */
1141                         break;
1142                 }
1143         }
1144
1145         /* update transfer cache */
1146
1147         xfer->td_transfer_cache = td;
1148
1149         /* update data toggle */
1150
1151         xfer->endpoint->toggle_next = (token & UHCI_TD_SET_DT(1)) ? 0 : 1;
1152
1153 #ifdef USB_DEBUG
1154         if (status & UHCI_TD_ERROR) {
1155                 DPRINTFN(11, "error, addr=%d, endpt=0x%02x, frame=0x%02x "
1156                     "status=%s%s%s%s%s%s%s%s%s%s%s\n",
1157                     xfer->address, xfer->endpointno, xfer->aframes,
1158                     (status & UHCI_TD_BITSTUFF) ? "[BITSTUFF]" : "",
1159                     (status & UHCI_TD_CRCTO) ? "[CRCTO]" : "",
1160                     (status & UHCI_TD_NAK) ? "[NAK]" : "",
1161                     (status & UHCI_TD_BABBLE) ? "[BABBLE]" : "",
1162                     (status & UHCI_TD_DBUFFER) ? "[DBUFFER]" : "",
1163                     (status & UHCI_TD_STALLED) ? "[STALLED]" : "",
1164                     (status & UHCI_TD_ACTIVE) ? "[ACTIVE]" : "[NOT_ACTIVE]",
1165                     (status & UHCI_TD_IOC) ? "[IOC]" : "",
1166                     (status & UHCI_TD_IOS) ? "[IOS]" : "",
1167                     (status & UHCI_TD_LS) ? "[LS]" : "",
1168                     (status & UHCI_TD_SPD) ? "[SPD]" : "");
1169         }
1170 #endif
1171         if (status & UHCI_TD_STALLED) {
1172                 /* try to separate I/O errors from STALL */
1173                 if (UHCI_TD_GET_ERRCNT(status) == 0)
1174                         return (USB_ERR_IOERROR);
1175                 return (USB_ERR_STALLED);
1176         }
1177         return (USB_ERR_NORMAL_COMPLETION);
1178 }
1179
1180 static void
1181 uhci_non_isoc_done(struct usb_xfer *xfer)
1182 {
1183         usb_error_t err = 0;
1184
1185         DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
1186             xfer, xfer->endpoint);
1187
1188 #ifdef USB_DEBUG
1189         if (uhcidebug > 10) {
1190                 uhci_dump_tds(xfer->td_transfer_first);
1191         }
1192 #endif
1193
1194         /* sync any DMA memory before doing fixups */
1195
1196         usb_bdma_post_sync(xfer);
1197
1198         /* reset scanner */
1199
1200         xfer->td_transfer_cache = xfer->td_transfer_first;
1201
1202         if (xfer->flags_int.control_xfr) {
1203                 if (xfer->flags_int.control_hdr) {
1204                         err = uhci_non_isoc_done_sub(xfer);
1205                 }
1206                 xfer->aframes = 1;
1207
1208                 if (xfer->td_transfer_cache == NULL) {
1209                         goto done;
1210                 }
1211         }
1212         while (xfer->aframes != xfer->nframes) {
1213                 err = uhci_non_isoc_done_sub(xfer);
1214                 xfer->aframes++;
1215
1216                 if (xfer->td_transfer_cache == NULL) {
1217                         goto done;
1218                 }
1219         }
1220
1221         if (xfer->flags_int.control_xfr &&
1222             !xfer->flags_int.control_act) {
1223                 err = uhci_non_isoc_done_sub(xfer);
1224         }
1225 done:
1226         uhci_device_done(xfer, err);
1227 }
1228
1229 /*------------------------------------------------------------------------*
1230  *      uhci_check_transfer_sub
1231  *
1232  * The main purpose of this function is to update the data-toggle
1233  * in case it is wrong.
1234  *------------------------------------------------------------------------*/
1235 static void
1236 uhci_check_transfer_sub(struct usb_xfer *xfer)
1237 {
1238         uhci_qh_t *qh;
1239         uhci_td_t *td;
1240         uhci_td_t *td_alt_next;
1241
1242         uint32_t td_token;
1243         uint32_t td_self;
1244
1245         td = xfer->td_transfer_cache;
1246         qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
1247
1248         td_token = td->obj_next->td_token;
1249         td = td->alt_next;
1250         xfer->td_transfer_cache = td;
1251         td_self = td->td_self;
1252         td_alt_next = td->alt_next;
1253
1254         if (xfer->flags_int.control_xfr)
1255                 goto skip;      /* don't touch the DT value! */
1256
1257         if (!((td->td_token ^ td_token) & htole32(UHCI_TD_SET_DT(1))))
1258                 goto skip;      /* data toggle has correct value */
1259
1260         /*
1261          * The data toggle is wrong and we need to toggle it !
1262          */
1263         while (1) {
1264                 td->td_token ^= htole32(UHCI_TD_SET_DT(1));
1265                 usb_pc_cpu_flush(td->page_cache);
1266
1267                 if (td == xfer->td_transfer_last) {
1268                         /* last transfer */
1269                         break;
1270                 }
1271                 td = td->obj_next;
1272
1273                 if (td->alt_next != td_alt_next) {
1274                         /* next frame */
1275                         break;
1276                 }
1277         }
1278 skip:
1279
1280         /* update the QH */
1281         qh->qh_e_next = td_self;
1282         usb_pc_cpu_flush(qh->page_cache);
1283
1284         DPRINTFN(13, "xfer=%p following alt next\n", xfer);
1285 }
1286
1287 /*------------------------------------------------------------------------*
1288  *      uhci_check_transfer
1289  *
1290  * Return values:
1291  *    0: USB transfer is not finished
1292  * Else: USB transfer is finished
1293  *------------------------------------------------------------------------*/
1294 static uint8_t
1295 uhci_check_transfer(struct usb_xfer *xfer)
1296 {
1297         uint32_t status;
1298         uint32_t token;
1299         uhci_td_t *td;
1300
1301         DPRINTFN(16, "xfer=%p checking transfer\n", xfer);
1302
1303         if (xfer->endpoint->methods == &uhci_device_isoc_methods) {
1304                 /* isochronous transfer */
1305
1306                 td = xfer->td_transfer_last;
1307
1308                 usb_pc_cpu_invalidate(td->page_cache);
1309                 status = le32toh(td->td_status);
1310
1311                 /* check also if the first is complete */
1312
1313                 td = xfer->td_transfer_first;
1314
1315                 usb_pc_cpu_invalidate(td->page_cache);
1316                 status |= le32toh(td->td_status);
1317
1318                 if (!(status & UHCI_TD_ACTIVE)) {
1319                         uhci_device_done(xfer, USB_ERR_NORMAL_COMPLETION);
1320                         goto transferred;
1321                 }
1322         } else {
1323                 /* non-isochronous transfer */
1324
1325                 /*
1326                  * check whether there is an error somewhere
1327                  * in the middle, or whether there was a short
1328                  * packet (SPD and not ACTIVE)
1329                  */
1330                 td = xfer->td_transfer_cache;
1331
1332                 while (1) {
1333                         usb_pc_cpu_invalidate(td->page_cache);
1334                         status = le32toh(td->td_status);
1335                         token = le32toh(td->td_token);
1336
1337                         /*
1338                          * if there is an active TD the transfer isn't done
1339                          */
1340                         if (status & UHCI_TD_ACTIVE) {
1341                                 /* update cache */
1342                                 xfer->td_transfer_cache = td;
1343                                 goto done;
1344                         }
1345                         /*
1346                          * last transfer descriptor makes the transfer done
1347                          */
1348                         if (((void *)td) == xfer->td_transfer_last) {
1349                                 break;
1350                         }
1351                         /*
1352                          * any kind of error makes the transfer done
1353                          */
1354                         if (status & UHCI_TD_STALLED) {
1355                                 break;
1356                         }
1357                         /*
1358                          * check if we reached the last packet
1359                          * or if there is a short packet:
1360                          */
1361                         if ((td->td_next == htole32(UHCI_PTR_T)) ||
1362                             (UHCI_TD_GET_ACTLEN(status) < td->len)) {
1363                                 if (xfer->flags_int.short_frames_ok) {
1364                                         /* follow alt next */
1365                                         if (td->alt_next) {
1366                                                 /* update cache */
1367                                                 xfer->td_transfer_cache = td;
1368                                                 uhci_check_transfer_sub(xfer);
1369                                                 goto done;
1370                                         }
1371                                 }
1372                                 /* transfer is done */
1373                                 break;
1374                         }
1375                         td = td->obj_next;
1376                 }
1377                 uhci_non_isoc_done(xfer);
1378                 goto transferred;
1379         }
1380
1381 done:
1382         DPRINTFN(13, "xfer=%p is still active\n", xfer);
1383         return (0);
1384
1385 transferred:
1386         return (1);
1387 }
1388
1389 static void
1390 uhci_interrupt_poll(uhci_softc_t *sc)
1391 {
1392         struct usb_xfer *xfer;
1393
1394 repeat:
1395         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
1396                 /*
1397                  * check if transfer is transferred
1398                  */
1399                 if (uhci_check_transfer(xfer)) {
1400                         /* queue has been modified */
1401                         goto repeat;
1402                 }
1403         }
1404 }
1405
1406 /*------------------------------------------------------------------------*
1407  *      uhci_interrupt - UHCI interrupt handler
1408  *
1409  * NOTE: Do not access "sc->sc_bus.bdev" inside the interrupt handler,
1410  * hence the interrupt handler will be setup before "sc->sc_bus.bdev"
1411  * is present !
1412  *------------------------------------------------------------------------*/
1413 void
1414 uhci_interrupt(uhci_softc_t *sc)
1415 {
1416         uint32_t status;
1417
1418         USB_BUS_LOCK(&sc->sc_bus);
1419
1420         DPRINTFN(16, "real interrupt\n");
1421
1422 #ifdef USB_DEBUG
1423         if (uhcidebug > 15) {
1424                 uhci_dumpregs(sc);
1425         }
1426 #endif
1427         status = UREAD2(sc, UHCI_STS) & UHCI_STS_ALLINTRS;
1428         if (status == 0) {
1429                 /* the interrupt was not for us */
1430                 goto done;
1431         }
1432         if (status & (UHCI_STS_RD | UHCI_STS_HSE |
1433             UHCI_STS_HCPE | UHCI_STS_HCH)) {
1434                 if (status & UHCI_STS_RD) {
1435 #ifdef USB_DEBUG
1436                         printf("%s: resume detect\n",
1437                             __FUNCTION__);
1438 #endif
1439                 }
1440                 if (status & UHCI_STS_HSE) {
1441                         printf("%s: host system error\n",
1442                             __FUNCTION__);
1443                 }
1444                 if (status & UHCI_STS_HCPE) {
1445                         printf("%s: host controller process error\n",
1446                             __FUNCTION__);
1447                 }
1448                 if (status & UHCI_STS_HCH) {
1449                         /* no acknowledge needed */
1450                         DPRINTF("%s: host controller halted\n",
1451                             __FUNCTION__);
1452 #ifdef USB_DEBUG
1453                         if (uhcidebug > 0) {
1454                                 uhci_dump_all(sc);
1455                         }
1456 #endif
1457                 }
1458         }
1459         /* get acknowledge bits */
1460         status &= (UHCI_STS_USBINT |
1461             UHCI_STS_USBEI |
1462             UHCI_STS_RD |
1463             UHCI_STS_HSE |
1464             UHCI_STS_HCPE |
1465             UHCI_STS_HCH);
1466
1467         if (status == 0) {
1468                 /* nothing to acknowledge */
1469                 goto done;
1470         }
1471         /* acknowledge interrupts */
1472         UWRITE2(sc, UHCI_STS, status);
1473
1474         /* poll all the USB transfers */
1475         uhci_interrupt_poll(sc);
1476
1477 done:
1478         USB_BUS_UNLOCK(&sc->sc_bus);
1479 }
1480
1481 /*
1482  * called when a request does not complete
1483  */
1484 static void
1485 uhci_timeout(void *arg)
1486 {
1487         struct usb_xfer *xfer = arg;
1488
1489         DPRINTF("xfer=%p\n", xfer);
1490
1491         USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
1492
1493         /* transfer is transferred */
1494         uhci_device_done(xfer, USB_ERR_TIMEOUT);
1495 }
1496
1497 static void
1498 uhci_do_poll(struct usb_bus *bus)
1499 {
1500         struct uhci_softc *sc = UHCI_BUS2SC(bus);
1501
1502         USB_BUS_LOCK(&sc->sc_bus);
1503         uhci_interrupt_poll(sc);
1504         USB_BUS_UNLOCK(&sc->sc_bus);
1505 }
1506
1507 static void
1508 uhci_setup_standard_chain_sub(struct uhci_std_temp *temp)
1509 {
1510         uhci_td_t *td;
1511         uhci_td_t *td_next;
1512         uhci_td_t *td_alt_next;
1513         uint32_t average;
1514         uint32_t len_old;
1515         uint8_t shortpkt_old;
1516         uint8_t precompute;
1517
1518         td_alt_next = NULL;
1519         shortpkt_old = temp->shortpkt;
1520         len_old = temp->len;
1521         precompute = 1;
1522
1523         /* software is used to detect short incoming transfers */
1524
1525         if ((temp->td_token & htole32(UHCI_TD_PID)) == htole32(UHCI_TD_PID_IN)) {
1526                 temp->td_status |= htole32(UHCI_TD_SPD);
1527         } else {
1528                 temp->td_status &= ~htole32(UHCI_TD_SPD);
1529         }
1530
1531         temp->ml.buf_offset = 0;
1532
1533 restart:
1534
1535         temp->td_token &= ~htole32(UHCI_TD_SET_MAXLEN(0));
1536         temp->td_token |= htole32(UHCI_TD_SET_MAXLEN(temp->average));
1537
1538         td = temp->td;
1539         td_next = temp->td_next;
1540
1541         while (1) {
1542                 if (temp->len == 0) {
1543                         if (temp->shortpkt) {
1544                                 break;
1545                         }
1546                         /* send a Zero Length Packet, ZLP, last */
1547
1548                         temp->shortpkt = 1;
1549                         temp->td_token |= htole32(UHCI_TD_SET_MAXLEN(0));
1550                         average = 0;
1551
1552                 } else {
1553                         average = temp->average;
1554
1555                         if (temp->len < average) {
1556                                 temp->shortpkt = 1;
1557                                 temp->td_token &= ~htole32(UHCI_TD_SET_MAXLEN(0));
1558                                 temp->td_token |= htole32(UHCI_TD_SET_MAXLEN(temp->len));
1559                                 average = temp->len;
1560                         }
1561                 }
1562
1563                 if (td_next == NULL) {
1564                         panic("%s: out of UHCI transfer descriptors!", __FUNCTION__);
1565                 }
1566                 /* get next TD */
1567
1568                 td = td_next;
1569                 td_next = td->obj_next;
1570
1571                 /* check if we are pre-computing */
1572
1573                 if (precompute) {
1574                         /* update remaining length */
1575
1576                         temp->len -= average;
1577
1578                         continue;
1579                 }
1580                 /* fill out current TD */
1581
1582                 td->td_status = temp->td_status;
1583                 td->td_token = temp->td_token;
1584
1585                 /* update data toggle */
1586
1587                 temp->td_token ^= htole32(UHCI_TD_SET_DT(1));
1588
1589                 if (average == 0) {
1590                         td->len = 0;
1591                         td->td_buffer = 0;
1592                         td->fix_pc = NULL;
1593
1594                 } else {
1595                         /* update remaining length */
1596
1597                         temp->len -= average;
1598
1599                         td->len = average;
1600
1601                         /* fill out buffer pointer and do fixup, if any */
1602
1603                         uhci_mem_layout_fixup(&temp->ml, td);
1604                 }
1605
1606                 td->alt_next = td_alt_next;
1607
1608                 if ((td_next == td_alt_next) && temp->setup_alt_next) {
1609                         /* we need to receive these frames one by one ! */
1610                         td->td_status |= htole32(UHCI_TD_IOC);
1611                         td->td_next = htole32(UHCI_PTR_T);
1612                 } else {
1613                         if (td_next) {
1614                                 /* link the current TD with the next one */
1615                                 td->td_next = td_next->td_self;
1616                         }
1617                 }
1618
1619                 usb_pc_cpu_flush(td->page_cache);
1620         }
1621
1622         if (precompute) {
1623                 precompute = 0;
1624
1625                 /* setup alt next pointer, if any */
1626                 if (temp->last_frame) {
1627                         td_alt_next = NULL;
1628                 } else {
1629                         /* we use this field internally */
1630                         td_alt_next = td_next;
1631                 }
1632
1633                 /* restore */
1634                 temp->shortpkt = shortpkt_old;
1635                 temp->len = len_old;
1636                 goto restart;
1637         }
1638         temp->td = td;
1639         temp->td_next = td_next;
1640 }
1641
1642 static uhci_td_t *
1643 uhci_setup_standard_chain(struct usb_xfer *xfer)
1644 {
1645         struct uhci_std_temp temp;
1646         uhci_td_t *td;
1647         uint32_t x;
1648
1649         DPRINTFN(9, "addr=%d endpt=%d sumlen=%d speed=%d\n",
1650             xfer->address, UE_GET_ADDR(xfer->endpointno),
1651             xfer->sumlen, usbd_get_speed(xfer->xroot->udev));
1652
1653         temp.average = xfer->max_frame_size;
1654         temp.max_frame_size = xfer->max_frame_size;
1655
1656         /* toggle the DMA set we are using */
1657         xfer->flags_int.curr_dma_set ^= 1;
1658
1659         /* get next DMA set */
1660         td = xfer->td_start[xfer->flags_int.curr_dma_set];
1661         xfer->td_transfer_first = td;
1662         xfer->td_transfer_cache = td;
1663
1664         temp.td = NULL;
1665         temp.td_next = td;
1666         temp.last_frame = 0;
1667         temp.setup_alt_next = xfer->flags_int.short_frames_ok;
1668
1669         uhci_mem_layout_init(&temp.ml, xfer);
1670
1671         temp.td_status =
1672             htole32(UHCI_TD_ZERO_ACTLEN(UHCI_TD_SET_ERRCNT(3) |
1673             UHCI_TD_ACTIVE));
1674
1675         if (xfer->xroot->udev->speed == USB_SPEED_LOW) {
1676                 temp.td_status |= htole32(UHCI_TD_LS);
1677         }
1678         temp.td_token =
1679             htole32(UHCI_TD_SET_ENDPT(xfer->endpointno) |
1680             UHCI_TD_SET_DEVADDR(xfer->address));
1681
1682         if (xfer->endpoint->toggle_next) {
1683                 /* DATA1 is next */
1684                 temp.td_token |= htole32(UHCI_TD_SET_DT(1));
1685         }
1686         /* check if we should prepend a setup message */
1687
1688         if (xfer->flags_int.control_xfr) {
1689                 if (xfer->flags_int.control_hdr) {
1690                         temp.td_token &= htole32(UHCI_TD_SET_DEVADDR(0x7F) |
1691                             UHCI_TD_SET_ENDPT(0xF));
1692                         temp.td_token |= htole32(UHCI_TD_PID_SETUP |
1693                             UHCI_TD_SET_DT(0));
1694
1695                         temp.len = xfer->frlengths[0];
1696                         temp.ml.buf_pc = xfer->frbuffers + 0;
1697                         temp.shortpkt = temp.len ? 1 : 0;
1698                         /* check for last frame */
1699                         if (xfer->nframes == 1) {
1700                                 /* no STATUS stage yet, SETUP is last */
1701                                 if (xfer->flags_int.control_act) {
1702                                         temp.last_frame = 1;
1703                                         temp.setup_alt_next = 0;
1704                                 }
1705                         }
1706                         uhci_setup_standard_chain_sub(&temp);
1707                 }
1708                 x = 1;
1709         } else {
1710                 x = 0;
1711         }
1712
1713         while (x != xfer->nframes) {
1714                 /* DATA0 / DATA1 message */
1715
1716                 temp.len = xfer->frlengths[x];
1717                 temp.ml.buf_pc = xfer->frbuffers + x;
1718
1719                 x++;
1720
1721                 if (x == xfer->nframes) {
1722                         if (xfer->flags_int.control_xfr) {
1723                                 /* no STATUS stage yet, DATA is last */
1724                                 if (xfer->flags_int.control_act) {
1725                                         temp.last_frame = 1;
1726                                         temp.setup_alt_next = 0;
1727                                 }
1728                         } else {
1729                                 temp.last_frame = 1;
1730                                 temp.setup_alt_next = 0;
1731                         }
1732                 }
1733                 /*
1734                  * Keep previous data toggle,
1735                  * device address and endpoint number:
1736                  */
1737
1738                 temp.td_token &= htole32(UHCI_TD_SET_DEVADDR(0x7F) |
1739                     UHCI_TD_SET_ENDPT(0xF) |
1740                     UHCI_TD_SET_DT(1));
1741
1742                 if (temp.len == 0) {
1743                         /* make sure that we send an USB packet */
1744
1745                         temp.shortpkt = 0;
1746
1747                 } else {
1748                         /* regular data transfer */
1749
1750                         temp.shortpkt = (xfer->flags.force_short_xfer) ? 0 : 1;
1751                 }
1752
1753                 /* set endpoint direction */
1754
1755                 temp.td_token |=
1756                     (UE_GET_DIR(xfer->endpointno) == UE_DIR_IN) ?
1757                     htole32(UHCI_TD_PID_IN) :
1758                     htole32(UHCI_TD_PID_OUT);
1759
1760                 uhci_setup_standard_chain_sub(&temp);
1761         }
1762
1763         /* check if we should append a status stage */
1764
1765         if (xfer->flags_int.control_xfr &&
1766             !xfer->flags_int.control_act) {
1767                 /*
1768                  * send a DATA1 message and reverse the current endpoint
1769                  * direction
1770                  */
1771
1772                 temp.td_token &= htole32(UHCI_TD_SET_DEVADDR(0x7F) |
1773                     UHCI_TD_SET_ENDPT(0xF) |
1774                     UHCI_TD_SET_DT(1));
1775                 temp.td_token |=
1776                     (UE_GET_DIR(xfer->endpointno) == UE_DIR_OUT) ?
1777                     htole32(UHCI_TD_PID_IN | UHCI_TD_SET_DT(1)) :
1778                     htole32(UHCI_TD_PID_OUT | UHCI_TD_SET_DT(1));
1779
1780                 temp.len = 0;
1781                 temp.ml.buf_pc = NULL;
1782                 temp.shortpkt = 0;
1783                 temp.last_frame = 1;
1784                 temp.setup_alt_next = 0;
1785
1786                 uhci_setup_standard_chain_sub(&temp);
1787         }
1788         td = temp.td;
1789
1790         /* Ensure that last TD is terminating: */
1791         td->td_next = htole32(UHCI_PTR_T);
1792
1793         /* set interrupt bit */
1794
1795         td->td_status |= htole32(UHCI_TD_IOC);
1796
1797         usb_pc_cpu_flush(td->page_cache);
1798
1799         /* must have at least one frame! */
1800
1801         xfer->td_transfer_last = td;
1802
1803 #ifdef USB_DEBUG
1804         if (uhcidebug > 8) {
1805                 DPRINTF("nexttog=%d; data before transfer:\n",
1806                     xfer->endpoint->toggle_next);
1807                 uhci_dump_tds(xfer->td_transfer_first);
1808         }
1809 #endif
1810         return (xfer->td_transfer_first);
1811 }
1812
1813 /* NOTE: "done" can be run two times in a row,
1814  * from close and from interrupt
1815  */
1816
1817 static void
1818 uhci_device_done(struct usb_xfer *xfer, usb_error_t error)
1819 {
1820         const struct usb_pipe_methods *methods = xfer->endpoint->methods;
1821         uhci_softc_t *sc = UHCI_BUS2SC(xfer->xroot->bus);
1822         uhci_qh_t *qh;
1823
1824         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
1825
1826         DPRINTFN(2, "xfer=%p, endpoint=%p, error=%d\n",
1827             xfer, xfer->endpoint, error);
1828
1829         qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
1830         if (qh) {
1831                 usb_pc_cpu_invalidate(qh->page_cache);
1832         }
1833         if (xfer->flags_int.bandwidth_reclaimed) {
1834                 xfer->flags_int.bandwidth_reclaimed = 0;
1835                 uhci_rem_loop(sc);
1836         }
1837         if (methods == &uhci_device_bulk_methods) {
1838                 UHCI_REMOVE_QH(qh, sc->sc_bulk_p_last);
1839         }
1840         if (methods == &uhci_device_ctrl_methods) {
1841                 if (xfer->xroot->udev->speed == USB_SPEED_LOW) {
1842                         UHCI_REMOVE_QH(qh, sc->sc_ls_ctl_p_last);
1843                 } else {
1844                         UHCI_REMOVE_QH(qh, sc->sc_fs_ctl_p_last);
1845                 }
1846         }
1847         if (methods == &uhci_device_intr_methods) {
1848                 UHCI_REMOVE_QH(qh, sc->sc_intr_p_last[xfer->qh_pos]);
1849         }
1850         /*
1851          * Only finish isochronous transfers once
1852          * which will update "xfer->frlengths".
1853          */
1854         if (xfer->td_transfer_first &&
1855             xfer->td_transfer_last) {
1856                 if (methods == &uhci_device_isoc_methods) {
1857                         uhci_isoc_done(sc, xfer);
1858                 }
1859                 xfer->td_transfer_first = NULL;
1860                 xfer->td_transfer_last = NULL;
1861         }
1862         /* dequeue transfer and start next transfer */
1863         usbd_transfer_done(xfer, error);
1864 }
1865
1866 /*------------------------------------------------------------------------*
1867  * uhci bulk support
1868  *------------------------------------------------------------------------*/
1869 static void
1870 uhci_device_bulk_open(struct usb_xfer *xfer)
1871 {
1872         return;
1873 }
1874
1875 static void
1876 uhci_device_bulk_close(struct usb_xfer *xfer)
1877 {
1878         uhci_device_done(xfer, USB_ERR_CANCELLED);
1879 }
1880
1881 static void
1882 uhci_device_bulk_enter(struct usb_xfer *xfer)
1883 {
1884         return;
1885 }
1886
1887 static void
1888 uhci_device_bulk_start(struct usb_xfer *xfer)
1889 {
1890         uhci_softc_t *sc = UHCI_BUS2SC(xfer->xroot->bus);
1891         uhci_td_t *td;
1892         uhci_qh_t *qh;
1893
1894         /* setup TD's */
1895         td = uhci_setup_standard_chain(xfer);
1896
1897         /* setup QH */
1898         qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
1899
1900         qh->e_next = td;
1901         qh->qh_e_next = td->td_self;
1902
1903         if (xfer->xroot->udev->flags.self_suspended == 0) {
1904                 UHCI_APPEND_QH(qh, sc->sc_bulk_p_last);
1905                 uhci_add_loop(sc);
1906                 xfer->flags_int.bandwidth_reclaimed = 1;
1907         } else {
1908                 usb_pc_cpu_flush(qh->page_cache);
1909         }
1910
1911         /* put transfer on interrupt queue */
1912         uhci_transfer_intr_enqueue(xfer);
1913 }
1914
1915 static const struct usb_pipe_methods uhci_device_bulk_methods =
1916 {
1917         .open = uhci_device_bulk_open,
1918         .close = uhci_device_bulk_close,
1919         .enter = uhci_device_bulk_enter,
1920         .start = uhci_device_bulk_start,
1921 };
1922
1923 /*------------------------------------------------------------------------*
1924  * uhci control support
1925  *------------------------------------------------------------------------*/
1926 static void
1927 uhci_device_ctrl_open(struct usb_xfer *xfer)
1928 {
1929         return;
1930 }
1931
1932 static void
1933 uhci_device_ctrl_close(struct usb_xfer *xfer)
1934 {
1935         uhci_device_done(xfer, USB_ERR_CANCELLED);
1936 }
1937
1938 static void
1939 uhci_device_ctrl_enter(struct usb_xfer *xfer)
1940 {
1941         return;
1942 }
1943
1944 static void
1945 uhci_device_ctrl_start(struct usb_xfer *xfer)
1946 {
1947         uhci_softc_t *sc = UHCI_BUS2SC(xfer->xroot->bus);
1948         uhci_qh_t *qh;
1949         uhci_td_t *td;
1950
1951         /* setup TD's */
1952         td = uhci_setup_standard_chain(xfer);
1953
1954         /* setup QH */
1955         qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
1956
1957         qh->e_next = td;
1958         qh->qh_e_next = td->td_self;
1959
1960         /*
1961          * NOTE: some devices choke on bandwidth- reclamation for control
1962          * transfers
1963          */
1964         if (xfer->xroot->udev->flags.self_suspended == 0) {
1965                 if (xfer->xroot->udev->speed == USB_SPEED_LOW) {
1966                         UHCI_APPEND_QH(qh, sc->sc_ls_ctl_p_last);
1967                 } else {
1968                         UHCI_APPEND_QH(qh, sc->sc_fs_ctl_p_last);
1969                 }
1970         } else {
1971                 usb_pc_cpu_flush(qh->page_cache);
1972         }
1973         /* put transfer on interrupt queue */
1974         uhci_transfer_intr_enqueue(xfer);
1975 }
1976
1977 static const struct usb_pipe_methods uhci_device_ctrl_methods =
1978 {
1979         .open = uhci_device_ctrl_open,
1980         .close = uhci_device_ctrl_close,
1981         .enter = uhci_device_ctrl_enter,
1982         .start = uhci_device_ctrl_start,
1983 };
1984
1985 /*------------------------------------------------------------------------*
1986  * uhci interrupt support
1987  *------------------------------------------------------------------------*/
1988 static void
1989 uhci_device_intr_open(struct usb_xfer *xfer)
1990 {
1991         uhci_softc_t *sc = UHCI_BUS2SC(xfer->xroot->bus);
1992         uint16_t best;
1993         uint16_t bit;
1994         uint16_t x;
1995
1996         best = 0;
1997         bit = UHCI_IFRAMELIST_COUNT / 2;
1998         while (bit) {
1999                 if (xfer->interval >= bit) {
2000                         x = bit;
2001                         best = bit;
2002                         while (x & bit) {
2003                                 if (sc->sc_intr_stat[x] <
2004                                     sc->sc_intr_stat[best]) {
2005                                         best = x;
2006                                 }
2007                                 x++;
2008                         }
2009                         break;
2010                 }
2011                 bit >>= 1;
2012         }
2013
2014         sc->sc_intr_stat[best]++;
2015         xfer->qh_pos = best;
2016
2017         DPRINTFN(3, "best=%d interval=%d\n",
2018             best, xfer->interval);
2019 }
2020
2021 static void
2022 uhci_device_intr_close(struct usb_xfer *xfer)
2023 {
2024         uhci_softc_t *sc = UHCI_BUS2SC(xfer->xroot->bus);
2025
2026         sc->sc_intr_stat[xfer->qh_pos]--;
2027
2028         uhci_device_done(xfer, USB_ERR_CANCELLED);
2029 }
2030
2031 static void
2032 uhci_device_intr_enter(struct usb_xfer *xfer)
2033 {
2034         return;
2035 }
2036
2037 static void
2038 uhci_device_intr_start(struct usb_xfer *xfer)
2039 {
2040         uhci_softc_t *sc = UHCI_BUS2SC(xfer->xroot->bus);
2041         uhci_qh_t *qh;
2042         uhci_td_t *td;
2043
2044         /* setup TD's */
2045         td = uhci_setup_standard_chain(xfer);
2046
2047         /* setup QH */
2048         qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
2049
2050         qh->e_next = td;
2051         qh->qh_e_next = td->td_self;
2052
2053         if (xfer->xroot->udev->flags.self_suspended == 0) {
2054                 /* enter QHs into the controller data structures */
2055                 UHCI_APPEND_QH(qh, sc->sc_intr_p_last[xfer->qh_pos]);
2056         } else {
2057                 usb_pc_cpu_flush(qh->page_cache);
2058         }
2059
2060         /* put transfer on interrupt queue */
2061         uhci_transfer_intr_enqueue(xfer);
2062 }
2063
2064 static const struct usb_pipe_methods uhci_device_intr_methods =
2065 {
2066         .open = uhci_device_intr_open,
2067         .close = uhci_device_intr_close,
2068         .enter = uhci_device_intr_enter,
2069         .start = uhci_device_intr_start,
2070 };
2071
2072 /*------------------------------------------------------------------------*
2073  * uhci isochronous support
2074  *------------------------------------------------------------------------*/
2075 static void
2076 uhci_device_isoc_open(struct usb_xfer *xfer)
2077 {
2078         uhci_td_t *td;
2079         uint32_t td_token;
2080         uint8_t ds;
2081
2082         td_token =
2083             (UE_GET_DIR(xfer->endpointno) == UE_DIR_IN) ?
2084             UHCI_TD_IN(0, xfer->endpointno, xfer->address, 0) :
2085             UHCI_TD_OUT(0, xfer->endpointno, xfer->address, 0);
2086
2087         td_token = htole32(td_token);
2088
2089         /* initialize all TD's */
2090
2091         for (ds = 0; ds != 2; ds++) {
2092                 for (td = xfer->td_start[ds]; td; td = td->obj_next) {
2093                         /* mark TD as inactive */
2094                         td->td_status = htole32(UHCI_TD_IOS);
2095                         td->td_token = td_token;
2096
2097                         usb_pc_cpu_flush(td->page_cache);
2098                 }
2099         }
2100 }
2101
2102 static void
2103 uhci_device_isoc_close(struct usb_xfer *xfer)
2104 {
2105         uhci_device_done(xfer, USB_ERR_CANCELLED);
2106 }
2107
2108 static void
2109 uhci_device_isoc_enter(struct usb_xfer *xfer)
2110 {
2111         struct uhci_mem_layout ml;
2112         uhci_softc_t *sc = UHCI_BUS2SC(xfer->xroot->bus);
2113         uint32_t nframes;
2114         uint32_t temp;
2115         uint32_t *plen;
2116
2117 #ifdef USB_DEBUG
2118         uint8_t once = 1;
2119
2120 #endif
2121         uhci_td_t *td;
2122         uhci_td_t *td_last = NULL;
2123         uhci_td_t **pp_last;
2124
2125         DPRINTFN(6, "xfer=%p next=%d nframes=%d\n",
2126             xfer, xfer->endpoint->isoc_next, xfer->nframes);
2127
2128         nframes = UREAD2(sc, UHCI_FRNUM);
2129
2130         temp = (nframes - xfer->endpoint->isoc_next) &
2131             (UHCI_VFRAMELIST_COUNT - 1);
2132
2133         if ((xfer->endpoint->is_synced == 0) ||
2134             (temp < xfer->nframes)) {
2135                 /*
2136                  * If there is data underflow or the pipe queue is empty we
2137                  * schedule the transfer a few frames ahead of the current
2138                  * frame position. Else two isochronous transfers might
2139                  * overlap.
2140                  */
2141                 xfer->endpoint->isoc_next = (nframes + 3) & (UHCI_VFRAMELIST_COUNT - 1);
2142                 xfer->endpoint->is_synced = 1;
2143                 DPRINTFN(3, "start next=%d\n", xfer->endpoint->isoc_next);
2144         }
2145         /*
2146          * compute how many milliseconds the insertion is ahead of the
2147          * current frame position:
2148          */
2149         temp = (xfer->endpoint->isoc_next - nframes) &
2150             (UHCI_VFRAMELIST_COUNT - 1);
2151
2152         /*
2153          * pre-compute when the isochronous transfer will be finished:
2154          */
2155         xfer->isoc_time_complete =
2156             usb_isoc_time_expand(&sc->sc_bus, nframes) + temp +
2157             xfer->nframes;
2158
2159         /* get the real number of frames */
2160
2161         nframes = xfer->nframes;
2162
2163         uhci_mem_layout_init(&ml, xfer);
2164
2165         plen = xfer->frlengths;
2166
2167         /* toggle the DMA set we are using */
2168         xfer->flags_int.curr_dma_set ^= 1;
2169
2170         /* get next DMA set */
2171         td = xfer->td_start[xfer->flags_int.curr_dma_set];
2172         xfer->td_transfer_first = td;
2173
2174         pp_last = &sc->sc_isoc_p_last[xfer->endpoint->isoc_next];
2175
2176         /* store starting position */
2177
2178         xfer->qh_pos = xfer->endpoint->isoc_next;
2179
2180         while (nframes--) {
2181                 if (td == NULL) {
2182                         panic("%s:%d: out of TD's\n",
2183                             __FUNCTION__, __LINE__);
2184                 }
2185                 if (pp_last >= &sc->sc_isoc_p_last[UHCI_VFRAMELIST_COUNT]) {
2186                         pp_last = &sc->sc_isoc_p_last[0];
2187                 }
2188                 if (*plen > xfer->max_frame_size) {
2189 #ifdef USB_DEBUG
2190                         if (once) {
2191                                 once = 0;
2192                                 printf("%s: frame length(%d) exceeds %d "
2193                                     "bytes (frame truncated)\n",
2194                                     __FUNCTION__, *plen,
2195                                     xfer->max_frame_size);
2196                         }
2197 #endif
2198                         *plen = xfer->max_frame_size;
2199                 }
2200                 /* reuse td_token from last transfer */
2201
2202                 td->td_token &= htole32(~UHCI_TD_MAXLEN_MASK);
2203                 td->td_token |= htole32(UHCI_TD_SET_MAXLEN(*plen));
2204
2205                 td->len = *plen;
2206
2207                 if (td->len == 0) {
2208                         /*
2209                          * Do not call "uhci_mem_layout_fixup()" when the
2210                          * length is zero!
2211                          */
2212                         td->td_buffer = 0;
2213                         td->fix_pc = NULL;
2214
2215                 } else {
2216                         /* fill out buffer pointer and do fixup, if any */
2217
2218                         uhci_mem_layout_fixup(&ml, td);
2219                 }
2220
2221                 /* update status */
2222                 if (nframes == 0) {
2223                         td->td_status = htole32
2224                             (UHCI_TD_ZERO_ACTLEN
2225                             (UHCI_TD_SET_ERRCNT(0) |
2226                             UHCI_TD_ACTIVE |
2227                             UHCI_TD_IOS |
2228                             UHCI_TD_IOC));
2229                 } else {
2230                         td->td_status = htole32
2231                             (UHCI_TD_ZERO_ACTLEN
2232                             (UHCI_TD_SET_ERRCNT(0) |
2233                             UHCI_TD_ACTIVE |
2234                             UHCI_TD_IOS));
2235                 }
2236
2237                 usb_pc_cpu_flush(td->page_cache);
2238
2239 #ifdef USB_DEBUG
2240                 if (uhcidebug > 5) {
2241                         DPRINTF("TD %d\n", nframes);
2242                         uhci_dump_td(td);
2243                 }
2244 #endif
2245                 /* insert TD into schedule */
2246                 UHCI_APPEND_TD(td, *pp_last);
2247                 pp_last++;
2248
2249                 plen++;
2250                 td_last = td;
2251                 td = td->obj_next;
2252         }
2253
2254         xfer->td_transfer_last = td_last;
2255
2256         /* update isoc_next */
2257         xfer->endpoint->isoc_next = (pp_last - &sc->sc_isoc_p_last[0]) &
2258             (UHCI_VFRAMELIST_COUNT - 1);
2259 }
2260
2261 static void
2262 uhci_device_isoc_start(struct usb_xfer *xfer)
2263 {
2264         /* put transfer on interrupt queue */
2265         uhci_transfer_intr_enqueue(xfer);
2266 }
2267
2268 static const struct usb_pipe_methods uhci_device_isoc_methods =
2269 {
2270         .open = uhci_device_isoc_open,
2271         .close = uhci_device_isoc_close,
2272         .enter = uhci_device_isoc_enter,
2273         .start = uhci_device_isoc_start,
2274 };
2275
2276 /*------------------------------------------------------------------------*
2277  * uhci root control support
2278  *------------------------------------------------------------------------*
2279  * Simulate a hardware hub by handling all the necessary requests.
2280  *------------------------------------------------------------------------*/
2281
2282 static const
2283 struct usb_device_descriptor uhci_devd =
2284 {
2285         sizeof(struct usb_device_descriptor),
2286         UDESC_DEVICE,                   /* type */
2287         {0x00, 0x01},                   /* USB version */
2288         UDCLASS_HUB,                    /* class */
2289         UDSUBCLASS_HUB,                 /* subclass */
2290         UDPROTO_FSHUB,                  /* protocol */
2291         64,                             /* max packet */
2292         {0}, {0}, {0x00, 0x01},         /* device id */
2293         1, 2, 0,                        /* string indexes */
2294         1                               /* # of configurations */
2295 };
2296
2297 static const struct uhci_config_desc uhci_confd = {
2298         .confd = {
2299                 .bLength = sizeof(struct usb_config_descriptor),
2300                 .bDescriptorType = UDESC_CONFIG,
2301                 .wTotalLength[0] = sizeof(uhci_confd),
2302                 .bNumInterface = 1,
2303                 .bConfigurationValue = 1,
2304                 .iConfiguration = 0,
2305                 .bmAttributes = UC_SELF_POWERED,
2306                 .bMaxPower = 0          /* max power */
2307         },
2308         .ifcd = {
2309                 .bLength = sizeof(struct usb_interface_descriptor),
2310                 .bDescriptorType = UDESC_INTERFACE,
2311                 .bNumEndpoints = 1,
2312                 .bInterfaceClass = UICLASS_HUB,
2313                 .bInterfaceSubClass = UISUBCLASS_HUB,
2314                 .bInterfaceProtocol = UIPROTO_FSHUB,
2315         },
2316         .endpd = {
2317                 .bLength = sizeof(struct usb_endpoint_descriptor),
2318                 .bDescriptorType = UDESC_ENDPOINT,
2319                 .bEndpointAddress = UE_DIR_IN | UHCI_INTR_ENDPT,
2320                 .bmAttributes = UE_INTERRUPT,
2321                 .wMaxPacketSize[0] = 8, /* max packet (63 ports) */
2322                 .bInterval = 255,
2323         },
2324 };
2325
2326 static const
2327 struct usb_hub_descriptor_min uhci_hubd_piix =
2328 {
2329         .bDescLength = sizeof(uhci_hubd_piix),
2330         .bDescriptorType = UDESC_HUB,
2331         .bNbrPorts = 2,
2332         .wHubCharacteristics = {UHD_PWR_NO_SWITCH | UHD_OC_INDIVIDUAL, 0},
2333         .bPwrOn2PwrGood = 50,
2334 };
2335
2336 /*
2337  * The USB hub protocol requires that SET_FEATURE(PORT_RESET) also
2338  * enables the port, and also states that SET_FEATURE(PORT_ENABLE)
2339  * should not be used by the USB subsystem.  As we cannot issue a
2340  * SET_FEATURE(PORT_ENABLE) externally, we must ensure that the port
2341  * will be enabled as part of the reset.
2342  *
2343  * On the VT83C572, the port cannot be successfully enabled until the
2344  * outstanding "port enable change" and "connection status change"
2345  * events have been reset.
2346  */
2347 static usb_error_t
2348 uhci_portreset(uhci_softc_t *sc, uint16_t index)
2349 {
2350         uint16_t port;
2351         uint16_t x;
2352         uint8_t lim;
2353
2354         if (index == 1)
2355                 port = UHCI_PORTSC1;
2356         else if (index == 2)
2357                 port = UHCI_PORTSC2;
2358         else
2359                 return (USB_ERR_IOERROR);
2360
2361         /*
2362          * Before we do anything, turn on SOF messages on the USB
2363          * BUS. Some USB devices do not cope without them!
2364          */
2365         uhci_restart(sc);
2366
2367         x = URWMASK(UREAD2(sc, port));
2368         UWRITE2(sc, port, x | UHCI_PORTSC_PR);
2369
2370         usb_pause_mtx(&sc->sc_bus.bus_mtx,
2371             USB_MS_TO_TICKS(usb_port_root_reset_delay));
2372
2373         DPRINTFN(4, "uhci port %d reset, status0 = 0x%04x\n",
2374             index, UREAD2(sc, port));
2375
2376         x = URWMASK(UREAD2(sc, port));
2377         UWRITE2(sc, port, x & ~UHCI_PORTSC_PR);
2378
2379         mtx_unlock(&sc->sc_bus.bus_mtx);
2380
2381         /*
2382          * This delay needs to be exactly 100us, else some USB devices
2383          * fail to attach!
2384          */
2385         DELAY(100);
2386
2387         mtx_lock(&sc->sc_bus.bus_mtx);
2388
2389         DPRINTFN(4, "uhci port %d reset, status1 = 0x%04x\n",
2390             index, UREAD2(sc, port));
2391
2392         x = URWMASK(UREAD2(sc, port));
2393         UWRITE2(sc, port, x | UHCI_PORTSC_PE);
2394
2395         for (lim = 0; lim < 12; lim++) {
2396                 usb_pause_mtx(&sc->sc_bus.bus_mtx,
2397                     USB_MS_TO_TICKS(usb_port_reset_delay));
2398
2399                 x = UREAD2(sc, port);
2400
2401                 DPRINTFN(4, "uhci port %d iteration %u, status = 0x%04x\n",
2402                     index, lim, x);
2403
2404                 if (!(x & UHCI_PORTSC_CCS)) {
2405                         /*
2406                          * No device is connected (or was disconnected
2407                          * during reset).  Consider the port reset.
2408                          * The delay must be long enough to ensure on
2409                          * the initial iteration that the device
2410                          * connection will have been registered.  50ms
2411                          * appears to be sufficient, but 20ms is not.
2412                          */
2413                         DPRINTFN(4, "uhci port %d loop %u, device detached\n",
2414                             index, lim);
2415                         goto done;
2416                 }
2417                 if (x & (UHCI_PORTSC_POEDC | UHCI_PORTSC_CSC)) {
2418                         /*
2419                          * Port enabled changed and/or connection
2420                          * status changed were set.  Reset either or
2421                          * both raised flags (by writing a 1 to that
2422                          * bit), and wait again for state to settle.
2423                          */
2424                         UWRITE2(sc, port, URWMASK(x) |
2425                             (x & (UHCI_PORTSC_POEDC | UHCI_PORTSC_CSC)));
2426                         continue;
2427                 }
2428                 if (x & UHCI_PORTSC_PE) {
2429                         /* port is enabled */
2430                         goto done;
2431                 }
2432                 UWRITE2(sc, port, URWMASK(x) | UHCI_PORTSC_PE);
2433         }
2434
2435         DPRINTFN(2, "uhci port %d reset timed out\n", index);
2436         return (USB_ERR_TIMEOUT);
2437
2438 done:
2439         DPRINTFN(4, "uhci port %d reset, status2 = 0x%04x\n",
2440             index, UREAD2(sc, port));
2441
2442         sc->sc_isreset = 1;
2443         return (USB_ERR_NORMAL_COMPLETION);
2444 }
2445
2446 static usb_error_t
2447 uhci_roothub_exec(struct usb_device *udev,
2448     struct usb_device_request *req, const void **pptr, uint16_t *plength)
2449 {
2450         uhci_softc_t *sc = UHCI_BUS2SC(udev->bus);
2451         const void *ptr;
2452         const char *str_ptr;
2453         uint16_t x;
2454         uint16_t port;
2455         uint16_t value;
2456         uint16_t index;
2457         uint16_t status;
2458         uint16_t change;
2459         uint16_t len;
2460         usb_error_t err;
2461
2462         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
2463
2464         /* buffer reset */
2465         ptr = (const void *)&sc->sc_hub_desc.temp;
2466         len = 0;
2467         err = 0;
2468
2469         value = UGETW(req->wValue);
2470         index = UGETW(req->wIndex);
2471
2472         DPRINTFN(3, "type=0x%02x request=0x%02x wLen=0x%04x "
2473             "wValue=0x%04x wIndex=0x%04x\n",
2474             req->bmRequestType, req->bRequest,
2475             UGETW(req->wLength), value, index);
2476
2477 #define C(x,y) ((x) | ((y) << 8))
2478         switch (C(req->bRequest, req->bmRequestType)) {
2479         case C(UR_CLEAR_FEATURE, UT_WRITE_DEVICE):
2480         case C(UR_CLEAR_FEATURE, UT_WRITE_INTERFACE):
2481         case C(UR_CLEAR_FEATURE, UT_WRITE_ENDPOINT):
2482                 /*
2483                  * DEVICE_REMOTE_WAKEUP and ENDPOINT_HALT are no-ops
2484                  * for the integrated root hub.
2485                  */
2486                 break;
2487         case C(UR_GET_CONFIG, UT_READ_DEVICE):
2488                 len = 1;
2489                 sc->sc_hub_desc.temp[0] = sc->sc_conf;
2490                 break;
2491         case C(UR_GET_DESCRIPTOR, UT_READ_DEVICE):
2492                 switch (value >> 8) {
2493                 case UDESC_DEVICE:
2494                         if ((value & 0xff) != 0) {
2495                                 err = USB_ERR_IOERROR;
2496                                 goto done;
2497                         }
2498                         len = sizeof(uhci_devd);
2499                         ptr = (const void *)&uhci_devd;
2500                         break;
2501
2502                 case UDESC_CONFIG:
2503                         if ((value & 0xff) != 0) {
2504                                 err = USB_ERR_IOERROR;
2505                                 goto done;
2506                         }
2507                         len = sizeof(uhci_confd);
2508                         ptr = (const void *)&uhci_confd;
2509                         break;
2510
2511                 case UDESC_STRING:
2512                         switch (value & 0xff) {
2513                         case 0: /* Language table */
2514                                 str_ptr = "\001";
2515                                 break;
2516
2517                         case 1: /* Vendor */
2518                                 str_ptr = sc->sc_vendor;
2519                                 break;
2520
2521                         case 2: /* Product */
2522                                 str_ptr = "UHCI root HUB";
2523                                 break;
2524
2525                         default:
2526                                 str_ptr = "";
2527                                 break;
2528                         }
2529
2530                         len = usb_make_str_desc
2531                             (sc->sc_hub_desc.temp,
2532                             sizeof(sc->sc_hub_desc.temp),
2533                             str_ptr);
2534                         break;
2535
2536                 default:
2537                         err = USB_ERR_IOERROR;
2538                         goto done;
2539                 }
2540                 break;
2541         case C(UR_GET_INTERFACE, UT_READ_INTERFACE):
2542                 len = 1;
2543                 sc->sc_hub_desc.temp[0] = 0;
2544                 break;
2545         case C(UR_GET_STATUS, UT_READ_DEVICE):
2546                 len = 2;
2547                 USETW(sc->sc_hub_desc.stat.wStatus, UDS_SELF_POWERED);
2548                 break;
2549         case C(UR_GET_STATUS, UT_READ_INTERFACE):
2550         case C(UR_GET_STATUS, UT_READ_ENDPOINT):
2551                 len = 2;
2552                 USETW(sc->sc_hub_desc.stat.wStatus, 0);
2553                 break;
2554         case C(UR_SET_ADDRESS, UT_WRITE_DEVICE):
2555                 if (value >= UHCI_MAX_DEVICES) {
2556                         err = USB_ERR_IOERROR;
2557                         goto done;
2558                 }
2559                 sc->sc_addr = value;
2560                 break;
2561         case C(UR_SET_CONFIG, UT_WRITE_DEVICE):
2562                 if ((value != 0) && (value != 1)) {
2563                         err = USB_ERR_IOERROR;
2564                         goto done;
2565                 }
2566                 sc->sc_conf = value;
2567                 break;
2568         case C(UR_SET_DESCRIPTOR, UT_WRITE_DEVICE):
2569                 break;
2570         case C(UR_SET_FEATURE, UT_WRITE_DEVICE):
2571         case C(UR_SET_FEATURE, UT_WRITE_INTERFACE):
2572         case C(UR_SET_FEATURE, UT_WRITE_ENDPOINT):
2573                 err = USB_ERR_IOERROR;
2574                 goto done;
2575         case C(UR_SET_INTERFACE, UT_WRITE_INTERFACE):
2576                 break;
2577         case C(UR_SYNCH_FRAME, UT_WRITE_ENDPOINT):
2578                 break;
2579                 /* Hub requests */
2580         case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_DEVICE):
2581                 break;
2582         case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_OTHER):
2583                 DPRINTFN(4, "UR_CLEAR_PORT_FEATURE "
2584                     "port=%d feature=%d\n",
2585                     index, value);
2586                 if (index == 1)
2587                         port = UHCI_PORTSC1;
2588                 else if (index == 2)
2589                         port = UHCI_PORTSC2;
2590                 else {
2591                         err = USB_ERR_IOERROR;
2592                         goto done;
2593                 }
2594                 switch (value) {
2595                 case UHF_PORT_ENABLE:
2596                         x = URWMASK(UREAD2(sc, port));
2597                         UWRITE2(sc, port, x & ~UHCI_PORTSC_PE);
2598                         break;
2599                 case UHF_PORT_SUSPEND:
2600                         x = URWMASK(UREAD2(sc, port));
2601                         UWRITE2(sc, port, x & ~(UHCI_PORTSC_SUSP));
2602                         break;
2603                 case UHF_PORT_RESET:
2604                         x = URWMASK(UREAD2(sc, port));
2605                         UWRITE2(sc, port, x & ~UHCI_PORTSC_PR);
2606                         break;
2607                 case UHF_C_PORT_CONNECTION:
2608                         x = URWMASK(UREAD2(sc, port));
2609                         UWRITE2(sc, port, x | UHCI_PORTSC_CSC);
2610                         break;
2611                 case UHF_C_PORT_ENABLE:
2612                         x = URWMASK(UREAD2(sc, port));
2613                         UWRITE2(sc, port, x | UHCI_PORTSC_POEDC);
2614                         break;
2615                 case UHF_C_PORT_OVER_CURRENT:
2616                         x = URWMASK(UREAD2(sc, port));
2617                         UWRITE2(sc, port, x | UHCI_PORTSC_OCIC);
2618                         break;
2619                 case UHF_C_PORT_RESET:
2620                         sc->sc_isreset = 0;
2621                         err = USB_ERR_NORMAL_COMPLETION;
2622                         goto done;
2623                 case UHF_C_PORT_SUSPEND:
2624                         sc->sc_isresumed &= ~(1 << index);
2625                         break;
2626                 case UHF_PORT_CONNECTION:
2627                 case UHF_PORT_OVER_CURRENT:
2628                 case UHF_PORT_POWER:
2629                 case UHF_PORT_LOW_SPEED:
2630                 default:
2631                         err = USB_ERR_IOERROR;
2632                         goto done;
2633                 }
2634                 break;
2635         case C(UR_GET_BUS_STATE, UT_READ_CLASS_OTHER):
2636                 if (index == 1)
2637                         port = UHCI_PORTSC1;
2638                 else if (index == 2)
2639                         port = UHCI_PORTSC2;
2640                 else {
2641                         err = USB_ERR_IOERROR;
2642                         goto done;
2643                 }
2644                 len = 1;
2645                 sc->sc_hub_desc.temp[0] =
2646                     ((UREAD2(sc, port) & UHCI_PORTSC_LS) >>
2647                     UHCI_PORTSC_LS_SHIFT);
2648                 break;
2649         case C(UR_GET_DESCRIPTOR, UT_READ_CLASS_DEVICE):
2650                 if ((value & 0xff) != 0) {
2651                         err = USB_ERR_IOERROR;
2652                         goto done;
2653                 }
2654                 len = sizeof(uhci_hubd_piix);
2655                 ptr = (const void *)&uhci_hubd_piix;
2656                 break;
2657         case C(UR_GET_STATUS, UT_READ_CLASS_DEVICE):
2658                 len = 16;
2659                 memset(sc->sc_hub_desc.temp, 0, 16);
2660                 break;
2661         case C(UR_GET_STATUS, UT_READ_CLASS_OTHER):
2662                 if (index == 1)
2663                         port = UHCI_PORTSC1;
2664                 else if (index == 2)
2665                         port = UHCI_PORTSC2;
2666                 else {
2667                         err = USB_ERR_IOERROR;
2668                         goto done;
2669                 }
2670                 x = UREAD2(sc, port);
2671                 status = change = 0;
2672                 if (x & UHCI_PORTSC_CCS)
2673                         status |= UPS_CURRENT_CONNECT_STATUS;
2674                 if (x & UHCI_PORTSC_CSC)
2675                         change |= UPS_C_CONNECT_STATUS;
2676                 if (x & UHCI_PORTSC_PE)
2677                         status |= UPS_PORT_ENABLED;
2678                 if (x & UHCI_PORTSC_POEDC)
2679                         change |= UPS_C_PORT_ENABLED;
2680                 if (x & UHCI_PORTSC_OCI)
2681                         status |= UPS_OVERCURRENT_INDICATOR;
2682                 if (x & UHCI_PORTSC_OCIC)
2683                         change |= UPS_C_OVERCURRENT_INDICATOR;
2684                 if (x & UHCI_PORTSC_LSDA)
2685                         status |= UPS_LOW_SPEED;
2686                 if ((x & UHCI_PORTSC_PE) && (x & UHCI_PORTSC_RD)) {
2687                         /* need to do a write back */
2688                         UWRITE2(sc, port, URWMASK(x));
2689
2690                         /* wait 20ms for resume sequence to complete */
2691                         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 50);
2692
2693                         /* clear suspend and resume detect */
2694                         UWRITE2(sc, port, URWMASK(x) & ~(UHCI_PORTSC_RD |
2695                             UHCI_PORTSC_SUSP));
2696
2697                         /* wait a little bit */
2698                         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 500);
2699
2700                         sc->sc_isresumed |= (1 << index);
2701
2702                 } else if (x & UHCI_PORTSC_SUSP) {
2703                         status |= UPS_SUSPEND;
2704                 }
2705                 status |= UPS_PORT_POWER;
2706                 if (sc->sc_isresumed & (1 << index))
2707                         change |= UPS_C_SUSPEND;
2708                 if (sc->sc_isreset)
2709                         change |= UPS_C_PORT_RESET;
2710                 USETW(sc->sc_hub_desc.ps.wPortStatus, status);
2711                 USETW(sc->sc_hub_desc.ps.wPortChange, change);
2712                 len = sizeof(sc->sc_hub_desc.ps);
2713                 break;
2714         case C(UR_SET_DESCRIPTOR, UT_WRITE_CLASS_DEVICE):
2715                 err = USB_ERR_IOERROR;
2716                 goto done;
2717         case C(UR_SET_FEATURE, UT_WRITE_CLASS_DEVICE):
2718                 break;
2719         case C(UR_SET_FEATURE, UT_WRITE_CLASS_OTHER):
2720                 if (index == 1)
2721                         port = UHCI_PORTSC1;
2722                 else if (index == 2)
2723                         port = UHCI_PORTSC2;
2724                 else {
2725                         err = USB_ERR_IOERROR;
2726                         goto done;
2727                 }
2728                 switch (value) {
2729                 case UHF_PORT_ENABLE:
2730                         x = URWMASK(UREAD2(sc, port));
2731                         UWRITE2(sc, port, x | UHCI_PORTSC_PE);
2732                         break;
2733                 case UHF_PORT_SUSPEND:
2734                         x = URWMASK(UREAD2(sc, port));
2735                         UWRITE2(sc, port, x | UHCI_PORTSC_SUSP);
2736                         break;
2737                 case UHF_PORT_RESET:
2738                         err = uhci_portreset(sc, index);
2739                         goto done;
2740                 case UHF_PORT_POWER:
2741                         /* pretend we turned on power */
2742                         err = USB_ERR_NORMAL_COMPLETION;
2743                         goto done;
2744                 case UHF_C_PORT_CONNECTION:
2745                 case UHF_C_PORT_ENABLE:
2746                 case UHF_C_PORT_OVER_CURRENT:
2747                 case UHF_PORT_CONNECTION:
2748                 case UHF_PORT_OVER_CURRENT:
2749                 case UHF_PORT_LOW_SPEED:
2750                 case UHF_C_PORT_SUSPEND:
2751                 case UHF_C_PORT_RESET:
2752                 default:
2753                         err = USB_ERR_IOERROR;
2754                         goto done;
2755                 }
2756                 break;
2757         default:
2758                 err = USB_ERR_IOERROR;
2759                 goto done;
2760         }
2761 done:
2762         *plength = len;
2763         *pptr = ptr;
2764         return (err);
2765 }
2766
2767 /*
2768  * This routine is executed periodically and simulates interrupts from
2769  * the root controller interrupt pipe for port status change:
2770  */
2771 static void
2772 uhci_root_intr(uhci_softc_t *sc)
2773 {
2774         DPRINTFN(21, "\n");
2775
2776         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
2777
2778         sc->sc_hub_idata[0] = 0;
2779
2780         if (UREAD2(sc, UHCI_PORTSC1) & (UHCI_PORTSC_CSC |
2781             UHCI_PORTSC_OCIC | UHCI_PORTSC_RD)) {
2782                 sc->sc_hub_idata[0] |= 1 << 1;
2783         }
2784         if (UREAD2(sc, UHCI_PORTSC2) & (UHCI_PORTSC_CSC |
2785             UHCI_PORTSC_OCIC | UHCI_PORTSC_RD)) {
2786                 sc->sc_hub_idata[0] |= 1 << 2;
2787         }
2788
2789         /* restart timer */
2790         usb_callout_reset(&sc->sc_root_intr, hz,
2791             (void *)&uhci_root_intr, sc);
2792
2793         if (sc->sc_hub_idata[0] != 0) {
2794                 uhub_root_intr(&sc->sc_bus, sc->sc_hub_idata,
2795                     sizeof(sc->sc_hub_idata));
2796         }
2797 }
2798
2799 static void
2800 uhci_xfer_setup(struct usb_setup_params *parm)
2801 {
2802         struct usb_page_search page_info;
2803         struct usb_page_cache *pc;
2804         uhci_softc_t *sc;
2805         struct usb_xfer *xfer;
2806         void *last_obj;
2807         uint32_t ntd;
2808         uint32_t nqh;
2809         uint32_t nfixup;
2810         uint32_t n;
2811         uint16_t align;
2812
2813         sc = UHCI_BUS2SC(parm->udev->bus);
2814         xfer = parm->curr_xfer;
2815
2816         parm->hc_max_packet_size = 0x500;
2817         parm->hc_max_packet_count = 1;
2818         parm->hc_max_frame_size = 0x500;
2819
2820         /*
2821          * compute ntd and nqh
2822          */
2823         if (parm->methods == &uhci_device_ctrl_methods) {
2824                 xfer->flags_int.bdma_enable = 1;
2825                 xfer->flags_int.bdma_no_post_sync = 1;
2826
2827                 usbd_transfer_setup_sub(parm);
2828
2829                 /* see EHCI HC driver for proof of "ntd" formula */
2830
2831                 nqh = 1;
2832                 ntd = ((2 * xfer->nframes) + 1  /* STATUS */
2833                     + (xfer->max_data_length / xfer->max_frame_size));
2834
2835         } else if (parm->methods == &uhci_device_bulk_methods) {
2836                 xfer->flags_int.bdma_enable = 1;
2837                 xfer->flags_int.bdma_no_post_sync = 1;
2838
2839                 usbd_transfer_setup_sub(parm);
2840
2841                 nqh = 1;
2842                 ntd = ((2 * xfer->nframes)
2843                     + (xfer->max_data_length / xfer->max_frame_size));
2844
2845         } else if (parm->methods == &uhci_device_intr_methods) {
2846                 xfer->flags_int.bdma_enable = 1;
2847                 xfer->flags_int.bdma_no_post_sync = 1;
2848
2849                 usbd_transfer_setup_sub(parm);
2850
2851                 nqh = 1;
2852                 ntd = ((2 * xfer->nframes)
2853                     + (xfer->max_data_length / xfer->max_frame_size));
2854
2855         } else if (parm->methods == &uhci_device_isoc_methods) {
2856                 xfer->flags_int.bdma_enable = 1;
2857                 xfer->flags_int.bdma_no_post_sync = 1;
2858
2859                 usbd_transfer_setup_sub(parm);
2860
2861                 nqh = 0;
2862                 ntd = xfer->nframes;
2863
2864         } else {
2865                 usbd_transfer_setup_sub(parm);
2866
2867                 nqh = 0;
2868                 ntd = 0;
2869         }
2870
2871         if (parm->err) {
2872                 return;
2873         }
2874         /*
2875          * NOTE: the UHCI controller requires that
2876          * every packet must be contiguous on
2877          * the same USB memory page !
2878          */
2879         nfixup = (parm->bufsize / USB_PAGE_SIZE) + 1;
2880
2881         /*
2882          * Compute a suitable power of two alignment
2883          * for our "max_frame_size" fixup buffer(s):
2884          */
2885         align = xfer->max_frame_size;
2886         n = 0;
2887         while (align) {
2888                 align >>= 1;
2889                 n++;
2890         }
2891
2892         /* check for power of two */
2893         if (!(xfer->max_frame_size &
2894             (xfer->max_frame_size - 1))) {
2895                 n--;
2896         }
2897         /*
2898          * We don't allow alignments of
2899          * less than 8 bytes:
2900          *
2901          * NOTE: Allocating using an aligment
2902          * of 1 byte has special meaning!
2903          */
2904         if (n < 3) {
2905                 n = 3;
2906         }
2907         align = (1 << n);
2908
2909         if (usbd_transfer_setup_sub_malloc(
2910             parm, &pc, xfer->max_frame_size,
2911             align, nfixup)) {
2912                 parm->err = USB_ERR_NOMEM;
2913                 return;
2914         }
2915         xfer->buf_fixup = pc;
2916
2917 alloc_dma_set:
2918
2919         if (parm->err) {
2920                 return;
2921         }
2922         last_obj = NULL;
2923
2924         if (usbd_transfer_setup_sub_malloc(
2925             parm, &pc, sizeof(uhci_td_t),
2926             UHCI_TD_ALIGN, ntd)) {
2927                 parm->err = USB_ERR_NOMEM;
2928                 return;
2929         }
2930         if (parm->buf) {
2931                 for (n = 0; n != ntd; n++) {
2932                         uhci_td_t *td;
2933
2934                         usbd_get_page(pc + n, 0, &page_info);
2935
2936                         td = page_info.buffer;
2937
2938                         /* init TD */
2939                         if ((parm->methods == &uhci_device_bulk_methods) ||
2940                             (parm->methods == &uhci_device_ctrl_methods) ||
2941                             (parm->methods == &uhci_device_intr_methods)) {
2942                                 /* set depth first bit */
2943                                 td->td_self = htole32(page_info.physaddr |
2944                                     UHCI_PTR_TD | UHCI_PTR_VF);
2945                         } else {
2946                                 td->td_self = htole32(page_info.physaddr |
2947                                     UHCI_PTR_TD);
2948                         }
2949
2950                         td->obj_next = last_obj;
2951                         td->page_cache = pc + n;
2952
2953                         last_obj = td;
2954
2955                         usb_pc_cpu_flush(pc + n);
2956                 }
2957         }
2958         xfer->td_start[xfer->flags_int.curr_dma_set] = last_obj;
2959
2960         last_obj = NULL;
2961
2962         if (usbd_transfer_setup_sub_malloc(
2963             parm, &pc, sizeof(uhci_qh_t),
2964             UHCI_QH_ALIGN, nqh)) {
2965                 parm->err = USB_ERR_NOMEM;
2966                 return;
2967         }
2968         if (parm->buf) {
2969                 for (n = 0; n != nqh; n++) {
2970                         uhci_qh_t *qh;
2971
2972                         usbd_get_page(pc + n, 0, &page_info);
2973
2974                         qh = page_info.buffer;
2975
2976                         /* init QH */
2977                         qh->qh_self = htole32(page_info.physaddr | UHCI_PTR_QH);
2978                         qh->obj_next = last_obj;
2979                         qh->page_cache = pc + n;
2980
2981                         last_obj = qh;
2982
2983                         usb_pc_cpu_flush(pc + n);
2984                 }
2985         }
2986         xfer->qh_start[xfer->flags_int.curr_dma_set] = last_obj;
2987
2988         if (!xfer->flags_int.curr_dma_set) {
2989                 xfer->flags_int.curr_dma_set = 1;
2990                 goto alloc_dma_set;
2991         }
2992 }
2993
2994 static void
2995 uhci_ep_init(struct usb_device *udev, struct usb_endpoint_descriptor *edesc,
2996     struct usb_endpoint *ep)
2997 {
2998         uhci_softc_t *sc = UHCI_BUS2SC(udev->bus);
2999
3000         DPRINTFN(2, "endpoint=%p, addr=%d, endpt=%d, mode=%d (%d)\n",
3001             ep, udev->address,
3002             edesc->bEndpointAddress, udev->flags.usb_mode,
3003             sc->sc_addr);
3004
3005         if (udev->device_index != sc->sc_addr) {
3006                 switch (edesc->bmAttributes & UE_XFERTYPE) {
3007                 case UE_CONTROL:
3008                         ep->methods = &uhci_device_ctrl_methods;
3009                         break;
3010                 case UE_INTERRUPT:
3011                         ep->methods = &uhci_device_intr_methods;
3012                         break;
3013                 case UE_ISOCHRONOUS:
3014                         if (udev->speed == USB_SPEED_FULL) {
3015                                 ep->methods = &uhci_device_isoc_methods;
3016                         }
3017                         break;
3018                 case UE_BULK:
3019                         ep->methods = &uhci_device_bulk_methods;
3020                         break;
3021                 default:
3022                         /* do nothing */
3023                         break;
3024                 }
3025         }
3026 }
3027
3028 static void
3029 uhci_xfer_unsetup(struct usb_xfer *xfer)
3030 {
3031         return;
3032 }
3033
3034 static void
3035 uhci_get_dma_delay(struct usb_device *udev, uint32_t *pus)
3036 {
3037         /*
3038          * Wait until hardware has finished any possible use of the
3039          * transfer descriptor(s) and QH
3040          */
3041         *pus = (1125);                  /* microseconds */
3042 }
3043
3044 static void
3045 uhci_device_resume(struct usb_device *udev)
3046 {
3047         struct uhci_softc *sc = UHCI_BUS2SC(udev->bus);
3048         struct usb_xfer *xfer;
3049         const struct usb_pipe_methods *methods;
3050         uhci_qh_t *qh;
3051
3052         DPRINTF("\n");
3053
3054         USB_BUS_LOCK(udev->bus);
3055
3056         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
3057                 if (xfer->xroot->udev == udev) {
3058                         methods = xfer->endpoint->methods;
3059                         qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
3060
3061                         if (methods == &uhci_device_bulk_methods) {
3062                                 UHCI_APPEND_QH(qh, sc->sc_bulk_p_last);
3063                                 uhci_add_loop(sc);
3064                                 xfer->flags_int.bandwidth_reclaimed = 1;
3065                         }
3066                         if (methods == &uhci_device_ctrl_methods) {
3067                                 if (xfer->xroot->udev->speed == USB_SPEED_LOW) {
3068                                         UHCI_APPEND_QH(qh, sc->sc_ls_ctl_p_last);
3069                                 } else {
3070                                         UHCI_APPEND_QH(qh, sc->sc_fs_ctl_p_last);
3071                                 }
3072                         }
3073                         if (methods == &uhci_device_intr_methods) {
3074                                 UHCI_APPEND_QH(qh, sc->sc_intr_p_last[xfer->qh_pos]);
3075                         }
3076                 }
3077         }
3078
3079         USB_BUS_UNLOCK(udev->bus);
3080
3081         return;
3082 }
3083
3084 static void
3085 uhci_device_suspend(struct usb_device *udev)
3086 {
3087         struct uhci_softc *sc = UHCI_BUS2SC(udev->bus);
3088         struct usb_xfer *xfer;
3089         const struct usb_pipe_methods *methods;
3090         uhci_qh_t *qh;
3091
3092         DPRINTF("\n");
3093
3094         USB_BUS_LOCK(udev->bus);
3095
3096         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
3097                 if (xfer->xroot->udev == udev) {
3098                         methods = xfer->endpoint->methods;
3099                         qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
3100
3101                         if (xfer->flags_int.bandwidth_reclaimed) {
3102                                 xfer->flags_int.bandwidth_reclaimed = 0;
3103                                 uhci_rem_loop(sc);
3104                         }
3105                         if (methods == &uhci_device_bulk_methods) {
3106                                 UHCI_REMOVE_QH(qh, sc->sc_bulk_p_last);
3107                         }
3108                         if (methods == &uhci_device_ctrl_methods) {
3109                                 if (xfer->xroot->udev->speed == USB_SPEED_LOW) {
3110                                         UHCI_REMOVE_QH(qh, sc->sc_ls_ctl_p_last);
3111                                 } else {
3112                                         UHCI_REMOVE_QH(qh, sc->sc_fs_ctl_p_last);
3113                                 }
3114                         }
3115                         if (methods == &uhci_device_intr_methods) {
3116                                 UHCI_REMOVE_QH(qh, sc->sc_intr_p_last[xfer->qh_pos]);
3117                         }
3118                 }
3119         }
3120
3121         USB_BUS_UNLOCK(udev->bus);
3122
3123         return;
3124 }
3125
3126 static void
3127 uhci_set_hw_power_sleep(struct usb_bus *bus, uint32_t state)
3128 {
3129         struct uhci_softc *sc = UHCI_BUS2SC(bus);
3130
3131         switch (state) {
3132         case USB_HW_POWER_SUSPEND:
3133         case USB_HW_POWER_SHUTDOWN:
3134                 uhci_suspend(sc);
3135                 break;
3136         case USB_HW_POWER_RESUME:
3137                 uhci_resume(sc);
3138                 break;
3139         default:
3140                 break;
3141         }
3142 }
3143
3144 static void
3145 uhci_set_hw_power(struct usb_bus *bus)
3146 {
3147         struct uhci_softc *sc = UHCI_BUS2SC(bus);
3148         uint32_t flags;
3149
3150         DPRINTF("\n");
3151
3152         USB_BUS_LOCK(bus);
3153
3154         flags = bus->hw_power_state;
3155
3156         /*
3157          * WARNING: Some FULL speed USB devices require periodic SOF
3158          * messages! If any USB devices are connected through the
3159          * UHCI, power save will be disabled!
3160          */
3161         if (flags & (USB_HW_POWER_CONTROL |
3162             USB_HW_POWER_NON_ROOT_HUB |
3163             USB_HW_POWER_BULK |
3164             USB_HW_POWER_INTERRUPT |
3165             USB_HW_POWER_ISOC)) {
3166                 DPRINTF("Some USB transfer is "
3167                     "active on unit %u.\n",
3168                     device_get_unit(sc->sc_bus.bdev));
3169                 uhci_restart(sc);
3170         } else {
3171                 DPRINTF("Power save on unit %u.\n",
3172                     device_get_unit(sc->sc_bus.bdev));
3173                 UHCICMD(sc, UHCI_CMD_MAXP);
3174         }
3175
3176         USB_BUS_UNLOCK(bus);
3177
3178         return;
3179 }
3180
3181 static const struct usb_bus_methods uhci_bus_methods =
3182 {
3183         .endpoint_init = uhci_ep_init,
3184         .xfer_setup = uhci_xfer_setup,
3185         .xfer_unsetup = uhci_xfer_unsetup,
3186         .get_dma_delay = uhci_get_dma_delay,
3187         .device_resume = uhci_device_resume,
3188         .device_suspend = uhci_device_suspend,
3189         .set_hw_power = uhci_set_hw_power,
3190         .set_hw_power_sleep = uhci_set_hw_power_sleep,
3191         .roothub_exec = uhci_roothub_exec,
3192         .xfer_poll = uhci_do_poll,
3193 };