sys/dev/usb/controller/uhci.c

   1 /*-
   2  * Copyright (c) 2008 Hans Petter Selasky. All rights reserved.
   3  * Copyright (c) 1998 The NetBSD Foundation, Inc. All rights reserved.
   4  * Copyright (c) 1998 Lennart Augustsson. All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  */
  27
  28 #include <sys/cdefs.h>
  29 __FBSDID("$FreeBSD$");
  30
  31 /*
  32  * USB Universal Host Controller driver.
  33  * Handles e.g. PIIX3 and PIIX4.
  34  *
  35  * UHCI spec: http://developer.intel.com/design/USB/UHCI11D.htm
  36  * USB spec:  http://www.usb.org/developers/docs/usbspec.zip
  37  * PIIXn spec: ftp://download.intel.com/design/intarch/datashts/29055002.pdf
  38  *             ftp://download.intel.com/design/intarch/datashts/29056201.pdf
  39  */
  40
  41 #include <sys/stdint.h>
  42 #include <sys/stddef.h>
  43 #include <sys/param.h>
  44 #include <sys/queue.h>
  45 #include <sys/types.h>
  46 #include <sys/systm.h>
  47 #include <sys/kernel.h>
  48 #include <sys/bus.h>
  49 #include <sys/linker_set.h>
  50 #include <sys/module.h>
  51 #include <sys/lock.h>
  52 #include <sys/mutex.h>
  53 #include <sys/condvar.h>
  54 #include <sys/sysctl.h>
  55 #include <sys/sx.h>
  56 #include <sys/unistd.h>
  57 #include <sys/callout.h>
  58 #include <sys/malloc.h>
  59 #include <sys/priv.h>
  60
  61 #include <dev/usb/usb.h>
  62 #include <dev/usb/usbdi.h>
  63
  64 #define USB_DEBUG_VAR uhcidebug
  65
  66 #include <dev/usb/usb_core.h>
  67 #include <dev/usb/usb_debug.h>
  68 #include <dev/usb/usb_busdma.h>
  69 #include <dev/usb/usb_process.h>
  70 #include <dev/usb/usb_transfer.h>
  71 #include <dev/usb/usb_device.h>
  72 #include <dev/usb/usb_hub.h>
  73 #include <dev/usb/usb_util.h>
  74
  75 #include <dev/usb/usb_controller.h>
  76 #include <dev/usb/usb_bus.h>
  77 #include <dev/usb/controller/uhci.h>
  78 #include <dev/usb/controller/uhcireg.h>
  79
  80 #define alt_next next
  81 #define UHCI_BUS2SC(bus) \
  82    ((uhci_softc_t *)(((uint8_t *)(bus)) - \
  83     ((uint8_t *)&(((uhci_softc_t *)0)->sc_bus))))
  84
  85 #ifdef USB_DEBUG
  86 static int uhcidebug = 0;
  87 static int uhcinoloop = 0;
  88
  89 SYSCTL_NODE(_hw_usb, OID_AUTO, uhci, CTLFLAG_RW, 0, "USB uhci");
  90 SYSCTL_INT(_hw_usb_uhci, OID_AUTO, debug, CTLFLAG_RW,
  91     &uhcidebug, 0, "uhci debug level");
  92 SYSCTL_INT(_hw_usb_uhci, OID_AUTO, loop, CTLFLAG_RW,
  93     &uhcinoloop, 0, "uhci noloop");
  94
  95 TUNABLE_INT("hw.usb.uhci.debug", &uhcidebug);
  96 TUNABLE_INT("hw.usb.uhci.loop", &uhcinoloop);
  97
  98 static void uhci_dumpregs(uhci_softc_t *sc);
  99 static void uhci_dump_tds(uhci_td_t *td);
 100
 101 #endif
 102
 103 #define UBARR(sc) bus_space_barrier((sc)->sc_io_tag, (sc)->sc_io_hdl, 0, (sc)->sc_io_size, \
 104                         BUS_SPACE_BARRIER_READ|BUS_SPACE_BARRIER_WRITE)
 105 #define UWRITE1(sc, r, x) \
 106  do { UBARR(sc); bus_space_write_1((sc)->sc_io_tag, (sc)->sc_io_hdl, (r), (x)); \
 107  } while (/*CONSTCOND*/0)
 108 #define UWRITE2(sc, r, x) \
 109  do { UBARR(sc); bus_space_write_2((sc)->sc_io_tag, (sc)->sc_io_hdl, (r), (x)); \
 110  } while (/*CONSTCOND*/0)
 111 #define UWRITE4(sc, r, x) \
 112  do { UBARR(sc); bus_space_write_4((sc)->sc_io_tag, (sc)->sc_io_hdl, (r), (x)); \
 113  } while (/*CONSTCOND*/0)
 114 #define UREAD1(sc, r) (UBARR(sc), bus_space_read_1((sc)->sc_io_tag, (sc)->sc_io_hdl, (r)))
 115 #define UREAD2(sc, r) (UBARR(sc), bus_space_read_2((sc)->sc_io_tag, (sc)->sc_io_hdl, (r)))
 116 #define UREAD4(sc, r) (UBARR(sc), bus_space_read_4((sc)->sc_io_tag, (sc)->sc_io_hdl, (r)))
 117
 118 #define UHCICMD(sc, cmd) UWRITE2(sc, UHCI_CMD, cmd)
 119 #define UHCISTS(sc) UREAD2(sc, UHCI_STS)
 120
 121 #define UHCI_RESET_TIMEOUT 100          /* ms, reset timeout */
 122
 123 #define UHCI_INTR_ENDPT 1
 124
 125 struct uhci_mem_layout {
 126
 127         struct usb_page_search buf_res;
 128         struct usb_page_search fix_res;
 129
 130         struct usb_page_cache *buf_pc;
 131         struct usb_page_cache *fix_pc;
 132
 133         uint32_t buf_offset;
 134
 135         uint16_t max_frame_size;
 136 };
 137
 138 struct uhci_std_temp {
 139
 140         struct uhci_mem_layout ml;
 141         uhci_td_t *td;
 142         uhci_td_t *td_next;
 143         uint32_t average;
 144         uint32_t td_status;
 145         uint32_t td_token;
 146         uint32_t len;
 147         uint16_t max_frame_size;
 148         uint8_t shortpkt;
 149         uint8_t setup_alt_next;
 150         uint8_t last_frame;
 151 };
 152
 153 extern struct usb_bus_methods uhci_bus_methods;
 154 extern struct usb_pipe_methods uhci_device_bulk_methods;
 155 extern struct usb_pipe_methods uhci_device_ctrl_methods;
 156 extern struct usb_pipe_methods uhci_device_intr_methods;
 157 extern struct usb_pipe_methods uhci_device_isoc_methods;
 158
 159 static uint8_t  uhci_restart(uhci_softc_t *sc);
 160 static void     uhci_do_poll(struct usb_bus *);
 161 static void     uhci_device_done(struct usb_xfer *, usb_error_t);
 162 static void     uhci_transfer_intr_enqueue(struct usb_xfer *);
 163 static void     uhci_timeout(void *);
 164 static uint8_t  uhci_check_transfer(struct usb_xfer *);
 165 static void     uhci_root_intr(uhci_softc_t *sc);
 166
 167 void
 168 uhci_iterate_hw_softc(struct usb_bus *bus, usb_bus_mem_sub_cb_t *cb)
 169 {
 170         struct uhci_softc *sc = UHCI_BUS2SC(bus);
 171         uint32_t i;
 172
 173         cb(bus, &sc->sc_hw.pframes_pc, &sc->sc_hw.pframes_pg,
 174             sizeof(uint32_t) * UHCI_FRAMELIST_COUNT, UHCI_FRAMELIST_ALIGN);
 175
 176         cb(bus, &sc->sc_hw.ls_ctl_start_pc, &sc->sc_hw.ls_ctl_start_pg,
 177             sizeof(uhci_qh_t), UHCI_QH_ALIGN);
 178
 179         cb(bus, &sc->sc_hw.fs_ctl_start_pc, &sc->sc_hw.fs_ctl_start_pg,
 180             sizeof(uhci_qh_t), UHCI_QH_ALIGN);
 181
 182         cb(bus, &sc->sc_hw.bulk_start_pc, &sc->sc_hw.bulk_start_pg,
 183             sizeof(uhci_qh_t), UHCI_QH_ALIGN);
 184
 185         cb(bus, &sc->sc_hw.last_qh_pc, &sc->sc_hw.last_qh_pg,
 186             sizeof(uhci_qh_t), UHCI_QH_ALIGN);
 187
 188         cb(bus, &sc->sc_hw.last_td_pc, &sc->sc_hw.last_td_pg,
 189             sizeof(uhci_td_t), UHCI_TD_ALIGN);
 190
 191         for (i = 0; i != UHCI_VFRAMELIST_COUNT; i++) {
 192                 cb(bus, sc->sc_hw.isoc_start_pc + i,
 193                     sc->sc_hw.isoc_start_pg + i,
 194                     sizeof(uhci_td_t), UHCI_TD_ALIGN);
 195         }
 196
 197         for (i = 0; i != UHCI_IFRAMELIST_COUNT; i++) {
 198                 cb(bus, sc->sc_hw.intr_start_pc + i,
 199                     sc->sc_hw.intr_start_pg + i,
 200                     sizeof(uhci_qh_t), UHCI_QH_ALIGN);
 201         }
 202 }
 203
 204 static void
 205 uhci_mem_layout_init(struct uhci_mem_layout *ml, struct usb_xfer *xfer)
 206 {
 207         ml->buf_pc = xfer->frbuffers + 0;
 208         ml->fix_pc = xfer->buf_fixup;
 209
 210         ml->buf_offset = 0;
 211
 212         ml->max_frame_size = xfer->max_frame_size;
 213 }
 214
 215 static void
 216 uhci_mem_layout_fixup(struct uhci_mem_layout *ml, struct uhci_td *td)
 217 {
 218         usbd_get_page(ml->buf_pc, ml->buf_offset, &ml->buf_res);
 219
 220         if (ml->buf_res.length < td->len) {
 221
 222                 /* need to do a fixup */
 223
 224                 usbd_get_page(ml->fix_pc, 0, &ml->fix_res);
 225
 226                 td->td_buffer = htole32(ml->fix_res.physaddr);
 227
 228                 /*
 229                  * The UHCI driver cannot handle
 230                  * page crossings, so a fixup is
 231                  * needed:
 232                  *
 233                  *  +----+----+ - - -
 234                  *  | YYY|Y   |
 235                  *  +----+----+ - - -
 236                  *     \    \
 237                  *      \    \
 238                  *       +----+
 239                  *       |YYYY|  (fixup)
 240                  *       +----+
 241                  */
 242
 243                 if ((td->td_token & htole32(UHCI_TD_PID)) ==
 244                     htole32(UHCI_TD_PID_IN)) {
 245                         td->fix_pc = ml->fix_pc;
 246                         usb_pc_cpu_invalidate(ml->fix_pc);
 247
 248                 } else {
 249                         td->fix_pc = NULL;
 250
 251                         /* copy data to fixup location */
 252
 253                         usbd_copy_out(ml->buf_pc, ml->buf_offset,
 254                             ml->fix_res.buffer, td->len);
 255
 256                         usb_pc_cpu_flush(ml->fix_pc);
 257                 }
 258
 259                 /* prepare next fixup */
 260
 261                 ml->fix_pc++;
 262
 263         } else {
 264
 265                 td->td_buffer = htole32(ml->buf_res.physaddr);
 266                 td->fix_pc = NULL;
 267         }
 268
 269         /* prepare next data location */
 270
 271         ml->buf_offset += td->len;
 272 }
 273
 274 /*
 275  * Return values:
 276  * 0: Success
 277  * Else: Failure
 278  */
 279 static uint8_t
 280 uhci_restart(uhci_softc_t *sc)
 281 {
 282         struct usb_page_search buf_res;
 283
 284         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
 285
 286         if (UREAD2(sc, UHCI_CMD) & UHCI_CMD_RS) {
 287                 DPRINTFN(2, "Already started\n");
 288                 return (0);
 289         }
 290
 291         DPRINTFN(2, "Restarting\n");
 292
 293         usbd_get_page(&sc->sc_hw.pframes_pc, 0, &buf_res);
 294
 295         /* Reload fresh base address */
 296         UWRITE4(sc, UHCI_FLBASEADDR, buf_res.physaddr);
 297
 298         /*
 299          * Assume 64 byte packets at frame end and start HC controller:
 300          */
 301         UHCICMD(sc, (UHCI_CMD_MAXP | UHCI_CMD_RS));
 302
 303         /* wait 10 milliseconds */
 304
 305         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 100);
 306
 307         /* check that controller has started */
 308
 309         if (UREAD2(sc, UHCI_STS) & UHCI_STS_HCH) {
 310                 DPRINTFN(2, "Failed\n");
 311                 return (1);
 312         }
 313         return (0);
 314 }
 315
 316 void
 317 uhci_reset(uhci_softc_t *sc)
 318 {
 319         uint16_t n;
 320
 321         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
 322
 323         DPRINTF("resetting the HC\n");
 324
 325         /* disable interrupts */
 326
 327         UWRITE2(sc, UHCI_INTR, 0);
 328
 329         /* global reset */
 330
 331         UHCICMD(sc, UHCI_CMD_GRESET);
 332
 333         /* wait */
 334
 335         usb_pause_mtx(&sc->sc_bus.bus_mtx,
 336             USB_MS_TO_TICKS(USB_BUS_RESET_DELAY));
 337
 338         /* terminate all transfers */
 339
 340         UHCICMD(sc, UHCI_CMD_HCRESET);
 341
 342         /* the reset bit goes low when the controller is done */
 343
 344         n = UHCI_RESET_TIMEOUT;
 345         while (n--) {
 346                 /* wait one millisecond */
 347
 348                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 1000);
 349
 350                 if (!(UREAD2(sc, UHCI_CMD) & UHCI_CMD_HCRESET)) {
 351                         goto done_1;
 352                 }
 353         }
 354
 355         device_printf(sc->sc_bus.bdev,
 356             "controller did not reset\n");
 357
 358 done_1:
 359
 360         n = 10;
 361         while (n--) {
 362                 /* wait one millisecond */
 363
 364                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 1000);
 365
 366                 /* check if HC is stopped */
 367                 if (UREAD2(sc, UHCI_STS) & UHCI_STS_HCH) {
 368                         goto done_2;
 369                 }
 370         }
 371
 372         device_printf(sc->sc_bus.bdev,
 373             "controller did not stop\n");
 374
 375 done_2:
 376
 377         /* reload the configuration */
 378         UWRITE2(sc, UHCI_FRNUM, sc->sc_saved_frnum);
 379         UWRITE1(sc, UHCI_SOF, sc->sc_saved_sof);
 380
 381         USB_BUS_UNLOCK(&sc->sc_bus);
 382
 383         /* stop root interrupt */
 384         usb_callout_drain(&sc->sc_root_intr);
 385
 386         USB_BUS_LOCK(&sc->sc_bus);
 387 }
 388
 389 static void
 390 uhci_start(uhci_softc_t *sc)
 391 {
 392         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
 393
 394         DPRINTFN(2, "enabling\n");
 395
 396         /* enable interrupts */
 397
 398         UWRITE2(sc, UHCI_INTR,
 399             (UHCI_INTR_TOCRCIE |
 400             UHCI_INTR_RIE |
 401             UHCI_INTR_IOCE |
 402             UHCI_INTR_SPIE));
 403
 404         if (uhci_restart(sc)) {
 405                 device_printf(sc->sc_bus.bdev,
 406                     "cannot start HC controller\n");
 407         }
 408
 409         /* start root interrupt */
 410         uhci_root_intr(sc);
 411 }
 412
 413 static struct uhci_qh *
 414 uhci_init_qh(struct usb_page_cache *pc)
 415 {
 416         struct usb_page_search buf_res;
 417         struct uhci_qh *qh;
 418
 419         usbd_get_page(pc, 0, &buf_res);
 420
 421         qh = buf_res.buffer;
 422
 423         qh->qh_self =
 424             htole32(buf_res.physaddr) |
 425             htole32(UHCI_PTR_QH);
 426
 427         qh->page_cache = pc;
 428
 429         return (qh);
 430 }
 431
 432 static struct uhci_td *
 433 uhci_init_td(struct usb_page_cache *pc)
 434 {
 435         struct usb_page_search buf_res;
 436         struct uhci_td *td;
 437
 438         usbd_get_page(pc, 0, &buf_res);
 439
 440         td = buf_res.buffer;
 441
 442         td->td_self =
 443             htole32(buf_res.physaddr) |
 444             htole32(UHCI_PTR_TD);
 445
 446         td->page_cache = pc;
 447
 448         return (td);
 449 }
 450
 451 usb_error_t
 452 uhci_init(uhci_softc_t *sc)
 453 {
 454         uint16_t bit;
 455         uint16_t x;
 456         uint16_t y;
 457
 458         DPRINTF("start\n");
 459
 460         usb_callout_init_mtx(&sc->sc_root_intr, &sc->sc_bus.bus_mtx, 0);
 461
 462 #ifdef USB_DEBUG
 463         if (uhcidebug > 2) {
 464                 uhci_dumpregs(sc);
 465         }
 466 #endif
 467         sc->sc_saved_sof = 0x40;        /* default value */
 468         sc->sc_saved_frnum = 0;         /* default frame number */
 469
 470         /*
 471          * Setup QH's
 472          */
 473         sc->sc_ls_ctl_p_last =
 474             uhci_init_qh(&sc->sc_hw.ls_ctl_start_pc);
 475
 476         sc->sc_fs_ctl_p_last =
 477             uhci_init_qh(&sc->sc_hw.fs_ctl_start_pc);
 478
 479         sc->sc_bulk_p_last =
 480             uhci_init_qh(&sc->sc_hw.bulk_start_pc);
 481 #if 0
 482         sc->sc_reclaim_qh_p =
 483             sc->sc_fs_ctl_p_last;
 484 #else
 485         /* setup reclaim looping point */
 486         sc->sc_reclaim_qh_p =
 487             sc->sc_bulk_p_last;
 488 #endif
 489
 490         sc->sc_last_qh_p =
 491             uhci_init_qh(&sc->sc_hw.last_qh_pc);
 492
 493         sc->sc_last_td_p =
 494             uhci_init_td(&sc->sc_hw.last_td_pc);
 495
 496         for (x = 0; x != UHCI_VFRAMELIST_COUNT; x++) {
 497                 sc->sc_isoc_p_last[x] =
 498                     uhci_init_td(sc->sc_hw.isoc_start_pc + x);
 499         }
 500
 501         for (x = 0; x != UHCI_IFRAMELIST_COUNT; x++) {
 502                 sc->sc_intr_p_last[x] =
 503                     uhci_init_qh(sc->sc_hw.intr_start_pc + x);
 504         }
 505
 506         /*
 507          * the QHs are arranged to give poll intervals that are
 508          * powers of 2 times 1ms
 509          */
 510         bit = UHCI_IFRAMELIST_COUNT / 2;
 511         while (bit) {
 512                 x = bit;
 513                 while (x & bit) {
 514                         uhci_qh_t *qh_x;
 515                         uhci_qh_t *qh_y;
 516
 517                         y = (x ^ bit) | (bit / 2);
 518
 519                         /*
 520                          * the next QH has half the poll interval
 521                          */
 522                         qh_x = sc->sc_intr_p_last[x];
 523                         qh_y = sc->sc_intr_p_last[y];
 524
 525                         qh_x->h_next = NULL;
 526                         qh_x->qh_h_next = qh_y->qh_self;
 527                         qh_x->e_next = NULL;
 528                         qh_x->qh_e_next = htole32(UHCI_PTR_T);
 529                         x++;
 530                 }
 531                 bit >>= 1;
 532         }
 533
 534         if (1) {
 535                 uhci_qh_t *qh_ls;
 536                 uhci_qh_t *qh_intr;
 537
 538                 qh_ls = sc->sc_ls_ctl_p_last;
 539                 qh_intr = sc->sc_intr_p_last[0];
 540
 541                 /* start QH for interrupt traffic */
 542                 qh_intr->h_next = qh_ls;
 543                 qh_intr->qh_h_next = qh_ls->qh_self;
 544                 qh_intr->e_next = 0;
 545                 qh_intr->qh_e_next = htole32(UHCI_PTR_T);
 546         }
 547         for (x = 0; x != UHCI_VFRAMELIST_COUNT; x++) {
 548
 549                 uhci_td_t *td_x;
 550                 uhci_qh_t *qh_intr;
 551
 552                 td_x = sc->sc_isoc_p_last[x];
 553                 qh_intr = sc->sc_intr_p_last[x | (UHCI_IFRAMELIST_COUNT / 2)];
 554
 555                 /* start TD for isochronous traffic */
 556                 td_x->next = NULL;
 557                 td_x->td_next = qh_intr->qh_self;
 558                 td_x->td_status = htole32(UHCI_TD_IOS);
 559                 td_x->td_token = htole32(0);
 560                 td_x->td_buffer = htole32(0);
 561         }
 562
 563         if (1) {
 564                 uhci_qh_t *qh_ls;
 565                 uhci_qh_t *qh_fs;
 566
 567                 qh_ls = sc->sc_ls_ctl_p_last;
 568                 qh_fs = sc->sc_fs_ctl_p_last;
 569
 570                 /* start QH where low speed control traffic will be queued */
 571                 qh_ls->h_next = qh_fs;
 572                 qh_ls->qh_h_next = qh_fs->qh_self;
 573                 qh_ls->e_next = 0;
 574                 qh_ls->qh_e_next = htole32(UHCI_PTR_T);
 575         }
 576         if (1) {
 577                 uhci_qh_t *qh_ctl;
 578                 uhci_qh_t *qh_blk;
 579                 uhci_qh_t *qh_lst;
 580                 uhci_td_t *td_lst;
 581
 582                 qh_ctl = sc->sc_fs_ctl_p_last;
 583                 qh_blk = sc->sc_bulk_p_last;
 584
 585                 /* start QH where full speed control traffic will be queued */
 586                 qh_ctl->h_next = qh_blk;
 587                 qh_ctl->qh_h_next = qh_blk->qh_self;
 588                 qh_ctl->e_next = 0;
 589                 qh_ctl->qh_e_next = htole32(UHCI_PTR_T);
 590
 591                 qh_lst = sc->sc_last_qh_p;
 592
 593                 /* start QH where bulk traffic will be queued */
 594                 qh_blk->h_next = qh_lst;
 595                 qh_blk->qh_h_next = qh_lst->qh_self;
 596                 qh_blk->e_next = 0;
 597                 qh_blk->qh_e_next = htole32(UHCI_PTR_T);
 598
 599                 td_lst = sc->sc_last_td_p;
 600
 601                 /* end QH which is used for looping the QHs */
 602                 qh_lst->h_next = 0;
 603                 qh_lst->qh_h_next = htole32(UHCI_PTR_T);        /* end of QH chain */
 604                 qh_lst->e_next = td_lst;
 605                 qh_lst->qh_e_next = td_lst->td_self;
 606
 607                 /*
 608                  * end TD which hangs from the last QH, to avoid a bug in the PIIX
 609                  * that makes it run berserk otherwise
 610                  */
 611                 td_lst->next = 0;
 612                 td_lst->td_next = htole32(UHCI_PTR_T);
 613                 td_lst->td_status = htole32(0); /* inactive */
 614                 td_lst->td_token = htole32(0);
 615                 td_lst->td_buffer = htole32(0);
 616         }
 617         if (1) {
 618                 struct usb_page_search buf_res;
 619                 uint32_t *pframes;
 620
 621                 usbd_get_page(&sc->sc_hw.pframes_pc, 0, &buf_res);
 622
 623                 pframes = buf_res.buffer;
 624
 625
 626                 /*
 627                  * Setup UHCI framelist
 628                  *
 629                  * Execution order:
 630                  *
 631                  * pframes -> full speed isochronous -> interrupt QH's -> low
 632                  * speed control -> full speed control -> bulk transfers
 633                  *
 634                  */
 635
 636                 for (x = 0; x != UHCI_FRAMELIST_COUNT; x++) {
 637                         pframes[x] =
 638                             sc->sc_isoc_p_last[x % UHCI_VFRAMELIST_COUNT]->td_self;
 639                 }
 640         }
 641         /* flush all cache into memory */
 642
 643         usb_bus_mem_flush_all(&sc->sc_bus, &uhci_iterate_hw_softc);
 644
 645         /* set up the bus struct */
 646         sc->sc_bus.methods = &uhci_bus_methods;
 647
 648         USB_BUS_LOCK(&sc->sc_bus);
 649         /* reset the controller */
 650         uhci_reset(sc);
 651
 652         /* start the controller */
 653         uhci_start(sc);
 654         USB_BUS_UNLOCK(&sc->sc_bus);
 655
 656         /* catch lost interrupts */
 657         uhci_do_poll(&sc->sc_bus);
 658
 659         return (0);
 660 }
 661
 662 /* NOTE: suspend/resume is called from
 663  * interrupt context and cannot sleep!
 664  */
 665
 666 void
 667 uhci_suspend(uhci_softc_t *sc)
 668 {
 669         USB_BUS_LOCK(&sc->sc_bus);
 670
 671 #ifdef USB_DEBUG
 672         if (uhcidebug > 2) {
 673                 uhci_dumpregs(sc);
 674         }
 675 #endif
 676         /* save some state if BIOS doesn't */
 677
 678         sc->sc_saved_frnum = UREAD2(sc, UHCI_FRNUM);
 679         sc->sc_saved_sof = UREAD1(sc, UHCI_SOF);
 680
 681         /* stop the controller */
 682
 683         uhci_reset(sc);
 684
 685         /* enter global suspend */
 686
 687         UHCICMD(sc, UHCI_CMD_EGSM);
 688
 689         usb_pause_mtx(&sc->sc_bus.bus_mtx,
 690             USB_MS_TO_TICKS(USB_RESUME_WAIT));
 691
 692         USB_BUS_UNLOCK(&sc->sc_bus);
 693 }
 694
 695 void
 696 uhci_resume(uhci_softc_t *sc)
 697 {
 698         USB_BUS_LOCK(&sc->sc_bus);
 699
 700         /* reset the controller */
 701
 702         uhci_reset(sc);
 703
 704         /* force global resume */
 705
 706         UHCICMD(sc, UHCI_CMD_FGR);
 707
 708         usb_pause_mtx(&sc->sc_bus.bus_mtx,
 709             USB_MS_TO_TICKS(USB_RESUME_DELAY));
 710
 711         /* and start traffic again */
 712
 713         uhci_start(sc);
 714
 715 #ifdef USB_DEBUG
 716         if (uhcidebug > 2) {
 717                 uhci_dumpregs(sc);
 718         }
 719 #endif
 720
 721         USB_BUS_UNLOCK(&sc->sc_bus);
 722
 723         /* catch lost interrupts */
 724         uhci_do_poll(&sc->sc_bus);
 725 }
 726
 727 #ifdef USB_DEBUG
 728 static void
 729 uhci_dumpregs(uhci_softc_t *sc)
 730 {
 731         DPRINTFN(0, "%s regs: cmd=%04x, sts=%04x, intr=%04x, frnum=%04x, "
 732             "flbase=%08x, sof=%04x, portsc1=%04x, portsc2=%04x\n",
 733             device_get_nameunit(sc->sc_bus.bdev),
 734             UREAD2(sc, UHCI_CMD),
 735             UREAD2(sc, UHCI_STS),
 736             UREAD2(sc, UHCI_INTR),
 737             UREAD2(sc, UHCI_FRNUM),
 738             UREAD4(sc, UHCI_FLBASEADDR),
 739             UREAD1(sc, UHCI_SOF),
 740             UREAD2(sc, UHCI_PORTSC1),
 741             UREAD2(sc, UHCI_PORTSC2));
 742 }
 743
 744 static uint8_t
 745 uhci_dump_td(uhci_td_t *p)
 746 {
 747         uint32_t td_next;
 748         uint32_t td_status;
 749         uint32_t td_token;
 750         uint8_t temp;
 751
 752         usb_pc_cpu_invalidate(p->page_cache);
 753
 754         td_next = le32toh(p->td_next);
 755         td_status = le32toh(p->td_status);
 756         td_token = le32toh(p->td_token);
 757
 758         /*
 759          * Check whether the link pointer in this TD marks the link pointer
 760          * as end of queue:
 761          */
 762         temp = ((td_next & UHCI_PTR_T) || (td_next == 0));
 763
 764         printf("TD(%p) at 0x%08x = link=0x%08x status=0x%08x "
 765             "token=0x%08x buffer=0x%08x\n",
 766             p,
 767             le32toh(p->td_self),
 768             td_next,
 769             td_status,
 770             td_token,
 771             le32toh(p->td_buffer));
 772
 773         printf("TD(%p) td_next=%s%s%s td_status=%s%s%s%s%s%s%s%s%s%s%s, errcnt=%d, actlen=%d pid=%02x,"
 774             "addr=%d,endpt=%d,D=%d,maxlen=%d\n",
 775             p,
 776             (td_next & 1) ? "-T" : "",
 777             (td_next & 2) ? "-Q" : "",
 778             (td_next & 4) ? "-VF" : "",
 779             (td_status & UHCI_TD_BITSTUFF) ? "-BITSTUFF" : "",
 780             (td_status & UHCI_TD_CRCTO) ? "-CRCTO" : "",
 781             (td_status & UHCI_TD_NAK) ? "-NAK" : "",
 782             (td_status & UHCI_TD_BABBLE) ? "-BABBLE" : "",
 783             (td_status & UHCI_TD_DBUFFER) ? "-DBUFFER" : "",
 784             (td_status & UHCI_TD_STALLED) ? "-STALLED" : "",
 785             (td_status & UHCI_TD_ACTIVE) ? "-ACTIVE" : "",
 786             (td_status & UHCI_TD_IOC) ? "-IOC" : "",
 787             (td_status & UHCI_TD_IOS) ? "-IOS" : "",
 788             (td_status & UHCI_TD_LS) ? "-LS" : "",
 789             (td_status & UHCI_TD_SPD) ? "-SPD" : "",
 790             UHCI_TD_GET_ERRCNT(td_status),
 791             UHCI_TD_GET_ACTLEN(td_status),
 792             UHCI_TD_GET_PID(td_token),
 793             UHCI_TD_GET_DEVADDR(td_token),
 794             UHCI_TD_GET_ENDPT(td_token),
 795             UHCI_TD_GET_DT(td_token),
 796             UHCI_TD_GET_MAXLEN(td_token));
 797
 798         return (temp);
 799 }
 800
 801 static uint8_t
 802 uhci_dump_qh(uhci_qh_t *sqh)
 803 {
 804         uint8_t temp;
 805         uint32_t qh_h_next;
 806         uint32_t qh_e_next;
 807
 808         usb_pc_cpu_invalidate(sqh->page_cache);
 809
 810         qh_h_next = le32toh(sqh->qh_h_next);
 811         qh_e_next = le32toh(sqh->qh_e_next);
 812
 813         DPRINTFN(0, "QH(%p) at 0x%08x: h_next=0x%08x e_next=0x%08x\n", sqh,
 814             le32toh(sqh->qh_self), qh_h_next, qh_e_next);
 815
 816         temp = ((((sqh->h_next != NULL) && !(qh_h_next & UHCI_PTR_T)) ? 1 : 0) |
 817             (((sqh->e_next != NULL) && !(qh_e_next & UHCI_PTR_T)) ? 2 : 0));
 818
 819         return (temp);
 820 }
 821
 822 static void
 823 uhci_dump_all(uhci_softc_t *sc)
 824 {
 825         uhci_dumpregs(sc);
 826         uhci_dump_qh(sc->sc_ls_ctl_p_last);
 827         uhci_dump_qh(sc->sc_fs_ctl_p_last);
 828         uhci_dump_qh(sc->sc_bulk_p_last);
 829         uhci_dump_qh(sc->sc_last_qh_p);
 830 }
 831
 832 static void
 833 uhci_dump_tds(uhci_td_t *td)
 834 {
 835         for (;
 836             td != NULL;
 837             td = td->obj_next) {
 838                 if (uhci_dump_td(td)) {
 839                         break;
 840                 }
 841         }
 842 }
 843
 844 #endif
 845
 846 /*
 847  * Let the last QH loop back to the full speed control transfer QH.
 848  * This is what intel calls "bandwidth reclamation" and improves
 849  * USB performance a lot for some devices.
 850  * If we are already looping, just count it.
 851  */
 852 static void
 853 uhci_add_loop(uhci_softc_t *sc)
 854 {
 855         struct uhci_qh *qh_lst;
 856         struct uhci_qh *qh_rec;
 857
 858 #ifdef USB_DEBUG
 859         if (uhcinoloop) {
 860                 return;
 861         }
 862 #endif
 863         if (++(sc->sc_loops) == 1) {
 864                 DPRINTFN(6, "add\n");
 865
 866                 qh_lst = sc->sc_last_qh_p;
 867                 qh_rec = sc->sc_reclaim_qh_p;
 868
 869                 /* NOTE: we don't loop back the soft pointer */
 870
 871                 qh_lst->qh_h_next = qh_rec->qh_self;
 872                 usb_pc_cpu_flush(qh_lst->page_cache);
 873         }
 874 }
 875
 876 static void
 877 uhci_rem_loop(uhci_softc_t *sc)
 878 {
 879         struct uhci_qh *qh_lst;
 880
 881 #ifdef USB_DEBUG
 882         if (uhcinoloop) {
 883                 return;
 884         }
 885 #endif
 886         if (--(sc->sc_loops) == 0) {
 887                 DPRINTFN(6, "remove\n");
 888
 889                 qh_lst = sc->sc_last_qh_p;
 890                 qh_lst->qh_h_next = htole32(UHCI_PTR_T);
 891                 usb_pc_cpu_flush(qh_lst->page_cache);
 892         }
 893 }
 894
 895 static void
 896 uhci_transfer_intr_enqueue(struct usb_xfer *xfer)
 897 {
 898         /* check for early completion */
 899         if (uhci_check_transfer(xfer)) {
 900                 return;
 901         }
 902         /* put transfer on interrupt queue */
 903         usbd_transfer_enqueue(&xfer->xroot->bus->intr_q, xfer);
 904
 905         /* start timeout, if any */
 906         if (xfer->timeout != 0) {
 907                 usbd_transfer_timeout_ms(xfer, &uhci_timeout, xfer->timeout);
 908         }
 909 }
 910
 911 #define UHCI_APPEND_TD(std,last) (last) = _uhci_append_td(std,last)
 912 static uhci_td_t *
 913 _uhci_append_td(uhci_td_t *std, uhci_td_t *last)
 914 {
 915         DPRINTFN(11, "%p to %p\n", std, last);
 916
 917         /* (sc->sc_bus.mtx) must be locked */
 918
 919         std->next = last->next;
 920         std->td_next = last->td_next;
 921
 922         std->prev = last;
 923
 924         usb_pc_cpu_flush(std->page_cache);
 925
 926         /*
 927          * the last->next->prev is never followed: std->next->prev = std;
 928          */
 929         last->next = std;
 930         last->td_next = std->td_self;
 931
 932         usb_pc_cpu_flush(last->page_cache);
 933
 934         return (std);
 935 }
 936
 937 #define UHCI_APPEND_QH(sqh,last) (last) = _uhci_append_qh(sqh,last)
 938 static uhci_qh_t *
 939 _uhci_append_qh(uhci_qh_t *sqh, uhci_qh_t *last)
 940 {
 941         DPRINTFN(11, "%p to %p\n", sqh, last);
 942
 943         if (sqh->h_prev != NULL) {
 944                 /* should not happen */
 945                 DPRINTFN(0, "QH already linked!\n");
 946                 return (last);
 947         }
 948         /* (sc->sc_bus.mtx) must be locked */
 949
 950         sqh->h_next = last->h_next;
 951         sqh->qh_h_next = last->qh_h_next;
 952
 953         sqh->h_prev = last;
 954
 955         usb_pc_cpu_flush(sqh->page_cache);
 956
 957         /*
 958          * The "last->h_next->h_prev" is never followed:
 959          *
 960          * "sqh->h_next->h_prev" = sqh;
 961          */
 962
 963         last->h_next = sqh;
 964         last->qh_h_next = sqh->qh_self;
 965
 966         usb_pc_cpu_flush(last->page_cache);
 967
 968         return (sqh);
 969 }
 970
 971 /**/
 972
 973 #define UHCI_REMOVE_TD(std,last) (last) = _uhci_remove_td(std,last)
 974 static uhci_td_t *
 975 _uhci_remove_td(uhci_td_t *std, uhci_td_t *last)
 976 {
 977         DPRINTFN(11, "%p from %p\n", std, last);
 978
 979         /* (sc->sc_bus.mtx) must be locked */
 980
 981         std->prev->next = std->next;
 982         std->prev->td_next = std->td_next;
 983
 984         usb_pc_cpu_flush(std->prev->page_cache);
 985
 986         if (std->next) {
 987                 std->next->prev = std->prev;
 988                 usb_pc_cpu_flush(std->next->page_cache);
 989         }
 990         return ((last == std) ? std->prev : last);
 991 }
 992
 993 #define UHCI_REMOVE_QH(sqh,last) (last) = _uhci_remove_qh(sqh,last)
 994 static uhci_qh_t *
 995 _uhci_remove_qh(uhci_qh_t *sqh, uhci_qh_t *last)
 996 {
 997         DPRINTFN(11, "%p from %p\n", sqh, last);
 998
 999         /* (sc->sc_bus.mtx) must be locked */
1000
1001         /* only remove if not removed from a queue */
1002         if (sqh->h_prev) {
1003
1004                 sqh->h_prev->h_next = sqh->h_next;
1005                 sqh->h_prev->qh_h_next = sqh->qh_h_next;
1006
1007                 usb_pc_cpu_flush(sqh->h_prev->page_cache);
1008
1009                 if (sqh->h_next) {
1010                         sqh->h_next->h_prev = sqh->h_prev;
1011                         usb_pc_cpu_flush(sqh->h_next->page_cache);
1012                 }
1013                 last = ((last == sqh) ? sqh->h_prev : last);
1014
1015                 sqh->h_prev = 0;
1016
1017                 usb_pc_cpu_flush(sqh->page_cache);
1018         }
1019         return (last);
1020 }
1021
1022 static void
1023 uhci_isoc_done(uhci_softc_t *sc, struct usb_xfer *xfer)
1024 {
1025         struct usb_page_search res;
1026         uint32_t nframes = xfer->nframes;
1027         uint32_t status;
1028         uint32_t offset = 0;
1029         uint32_t *plen = xfer->frlengths;
1030         uint16_t len = 0;
1031         uhci_td_t *td = xfer->td_transfer_first;
1032         uhci_td_t **pp_last = &sc->sc_isoc_p_last[xfer->qh_pos];
1033
1034         DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
1035             xfer, xfer->endpoint);
1036
1037         /* sync any DMA memory before doing fixups */
1038
1039         usb_bdma_post_sync(xfer);
1040
1041         while (nframes--) {
1042                 if (td == NULL) {
1043                         panic("%s:%d: out of TD's\n",
1044                             __FUNCTION__, __LINE__);
1045                 }
1046                 if (pp_last >= &sc->sc_isoc_p_last[UHCI_VFRAMELIST_COUNT]) {
1047                         pp_last = &sc->sc_isoc_p_last[0];
1048                 }
1049 #ifdef USB_DEBUG
1050                 if (uhcidebug > 5) {
1051                         DPRINTF("isoc TD\n");
1052                         uhci_dump_td(td);
1053                 }
1054 #endif
1055                 usb_pc_cpu_invalidate(td->page_cache);
1056                 status = le32toh(td->td_status);
1057
1058                 len = UHCI_TD_GET_ACTLEN(status);
1059
1060                 if (len > *plen) {
1061                         len = *plen;
1062                 }
1063                 if (td->fix_pc) {
1064
1065                         usbd_get_page(td->fix_pc, 0, &res);
1066
1067                         /* copy data from fixup location to real location */
1068
1069                         usb_pc_cpu_invalidate(td->fix_pc);
1070
1071                         usbd_copy_in(xfer->frbuffers, offset,
1072                             res.buffer, len);
1073                 }
1074                 offset += *plen;
1075
1076                 *plen = len;
1077
1078                 /* remove TD from schedule */
1079                 UHCI_REMOVE_TD(td, *pp_last);
1080
1081                 pp_last++;
1082                 plen++;
1083                 td = td->obj_next;
1084         }
1085
1086         xfer->aframes = xfer->nframes;
1087 }
1088
1089 static usb_error_t
1090 uhci_non_isoc_done_sub(struct usb_xfer *xfer)
1091 {
1092         struct usb_page_search res;
1093         uhci_td_t *td;
1094         uhci_td_t *td_alt_next;
1095         uint32_t status;
1096         uint32_t token;
1097         uint16_t len;
1098
1099         td = xfer->td_transfer_cache;
1100         td_alt_next = td->alt_next;
1101
1102         if (xfer->aframes != xfer->nframes) {
1103                 usbd_xfer_set_frame_len(xfer, xfer->aframes, 0);
1104         }
1105         while (1) {
1106
1107                 usb_pc_cpu_invalidate(td->page_cache);
1108                 status = le32toh(td->td_status);
1109                 token = le32toh(td->td_token);
1110
1111                 /*
1112                  * Verify the status and add
1113                  * up the actual length:
1114                  */
1115
1116                 len = UHCI_TD_GET_ACTLEN(status);
1117                 if (len > td->len) {
1118                         /* should not happen */
1119                         DPRINTF("Invalid status length, "
1120                             "0x%04x/0x%04x bytes\n", len, td->len);
1121                         status |= UHCI_TD_STALLED;
1122
1123                 } else if ((xfer->aframes != xfer->nframes) && (len > 0)) {
1124
1125                         if (td->fix_pc) {
1126
1127                                 usbd_get_page(td->fix_pc, 0, &res);
1128
1129                                 /*
1130                                  * copy data from fixup location to real
1131                                  * location
1132                                  */
1133
1134                                 usb_pc_cpu_invalidate(td->fix_pc);
1135
1136                                 usbd_copy_in(xfer->frbuffers + xfer->aframes,
1137                                     xfer->frlengths[xfer->aframes], res.buffer, len);
1138                         }
1139                         /* update actual length */
1140
1141                         xfer->frlengths[xfer->aframes] += len;
1142                 }
1143                 /* Check for last transfer */
1144                 if (((void *)td) == xfer->td_transfer_last) {
1145                         td = NULL;
1146                         break;
1147                 }
1148                 if (status & UHCI_TD_STALLED) {
1149                         /* the transfer is finished */
1150                         td = NULL;
1151                         break;
1152                 }
1153                 /* Check for short transfer */
1154                 if (len != td->len) {
1155                         if (xfer->flags_int.short_frames_ok) {
1156                                 /* follow alt next */
1157                                 td = td->alt_next;
1158                         } else {
1159                                 /* the transfer is finished */
1160                                 td = NULL;
1161                         }
1162                         break;
1163                 }
1164                 td = td->obj_next;
1165
1166                 if (td->alt_next != td_alt_next) {
1167                         /* this USB frame is complete */
1168                         break;
1169                 }
1170         }
1171
1172         /* update transfer cache */
1173
1174         xfer->td_transfer_cache = td;
1175
1176         /* update data toggle */
1177
1178         xfer->endpoint->toggle_next = (token & UHCI_TD_SET_DT(1)) ? 0 : 1;
1179
1180 #ifdef USB_DEBUG
1181         if (status & UHCI_TD_ERROR) {
1182                 DPRINTFN(11, "error, addr=%d, endpt=0x%02x, frame=0x%02x "
1183                     "status=%s%s%s%s%s%s%s%s%s%s%s\n",
1184                     xfer->address, xfer->endpointno, xfer->aframes,
1185                     (status & UHCI_TD_BITSTUFF) ? "[BITSTUFF]" : "",
1186                     (status & UHCI_TD_CRCTO) ? "[CRCTO]" : "",
1187                     (status & UHCI_TD_NAK) ? "[NAK]" : "",
1188                     (status & UHCI_TD_BABBLE) ? "[BABBLE]" : "",
1189                     (status & UHCI_TD_DBUFFER) ? "[DBUFFER]" : "",
1190                     (status & UHCI_TD_STALLED) ? "[STALLED]" : "",
1191                     (status & UHCI_TD_ACTIVE) ? "[ACTIVE]" : "[NOT_ACTIVE]",
1192                     (status & UHCI_TD_IOC) ? "[IOC]" : "",
1193                     (status & UHCI_TD_IOS) ? "[IOS]" : "",
1194                     (status & UHCI_TD_LS) ? "[LS]" : "",
1195                     (status & UHCI_TD_SPD) ? "[SPD]" : "");
1196         }
1197 #endif
1198         return (status & UHCI_TD_STALLED) ?
1199             USB_ERR_STALLED : USB_ERR_NORMAL_COMPLETION;
1200 }
1201
1202 static void
1203 uhci_non_isoc_done(struct usb_xfer *xfer)
1204 {
1205         usb_error_t err = 0;
1206
1207         DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
1208             xfer, xfer->endpoint);
1209
1210 #ifdef USB_DEBUG
1211         if (uhcidebug > 10) {
1212                 uhci_dump_tds(xfer->td_transfer_first);
1213         }
1214 #endif
1215
1216         /* sync any DMA memory before doing fixups */
1217
1218         usb_bdma_post_sync(xfer);
1219
1220         /* reset scanner */
1221
1222         xfer->td_transfer_cache = xfer->td_transfer_first;
1223
1224         if (xfer->flags_int.control_xfr) {
1225                 if (xfer->flags_int.control_hdr) {
1226
1227                         err = uhci_non_isoc_done_sub(xfer);
1228                 }
1229                 xfer->aframes = 1;
1230
1231                 if (xfer->td_transfer_cache == NULL) {
1232                         goto done;
1233                 }
1234         }
1235         while (xfer->aframes != xfer->nframes) {
1236
1237                 err = uhci_non_isoc_done_sub(xfer);
1238                 xfer->aframes++;
1239
1240                 if (xfer->td_transfer_cache == NULL) {
1241                         goto done;
1242                 }
1243         }
1244
1245         if (xfer->flags_int.control_xfr &&
1246             !xfer->flags_int.control_act) {
1247
1248                 err = uhci_non_isoc_done_sub(xfer);
1249         }
1250 done:
1251         uhci_device_done(xfer, err);
1252 }
1253
1254 /*------------------------------------------------------------------------*
1255  *      uhci_check_transfer_sub
1256  *
1257  * The main purpose of this function is to update the data-toggle
1258  * in case it is wrong.
1259  *------------------------------------------------------------------------*/
1260 static void
1261 uhci_check_transfer_sub(struct usb_xfer *xfer)
1262 {
1263         uhci_qh_t *qh;
1264         uhci_td_t *td;
1265         uhci_td_t *td_alt_next;
1266
1267         uint32_t td_token;
1268         uint32_t td_self;
1269
1270         td = xfer->td_transfer_cache;
1271         qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
1272
1273         td_token = td->obj_next->td_token;
1274         td = td->alt_next;
1275         xfer->td_transfer_cache = td;
1276         td_self = td->td_self;
1277         td_alt_next = td->alt_next;
1278
1279         if (xfer->flags_int.control_xfr)
1280                 goto skip;      /* don't touch the DT value! */
1281
1282         if (!((td->td_token ^ td_token) & htole32(UHCI_TD_SET_DT(1))))
1283                 goto skip;      /* data toggle has correct value */
1284
1285         /*
1286          * The data toggle is wrong and we need to toggle it !
1287          */
1288         while (1) {
1289
1290                 td->td_token ^= htole32(UHCI_TD_SET_DT(1));
1291                 usb_pc_cpu_flush(td->page_cache);
1292
1293                 if (td == xfer->td_transfer_last) {
1294                         /* last transfer */
1295                         break;
1296                 }
1297                 td = td->obj_next;
1298
1299                 if (td->alt_next != td_alt_next) {
1300                         /* next frame */
1301                         break;
1302                 }
1303         }
1304 skip:
1305
1306         /* update the QH */
1307         qh->qh_e_next = td_self;
1308         usb_pc_cpu_flush(qh->page_cache);
1309
1310         DPRINTFN(13, "xfer=%p following alt next\n", xfer);
1311 }
1312
1313 /*------------------------------------------------------------------------*
1314  *      uhci_check_transfer
1315  *
1316  * Return values:
1317  *    0: USB transfer is not finished
1318  * Else: USB transfer is finished
1319  *------------------------------------------------------------------------*/
1320 static uint8_t
1321 uhci_check_transfer(struct usb_xfer *xfer)
1322 {
1323         uint32_t status;
1324         uint32_t token;
1325         uhci_td_t *td;
1326
1327         DPRINTFN(16, "xfer=%p checking transfer\n", xfer);
1328
1329         if (xfer->endpoint->methods == &uhci_device_isoc_methods) {
1330                 /* isochronous transfer */
1331
1332                 td = xfer->td_transfer_last;
1333
1334                 usb_pc_cpu_invalidate(td->page_cache);
1335                 status = le32toh(td->td_status);
1336
1337                 /* check also if the first is complete */
1338
1339                 td = xfer->td_transfer_first;
1340
1341                 usb_pc_cpu_invalidate(td->page_cache);
1342                 status |= le32toh(td->td_status);
1343
1344                 if (!(status & UHCI_TD_ACTIVE)) {
1345                         uhci_device_done(xfer, USB_ERR_NORMAL_COMPLETION);
1346                         goto transferred;
1347                 }
1348         } else {
1349                 /* non-isochronous transfer */
1350
1351                 /*
1352                  * check whether there is an error somewhere
1353                  * in the middle, or whether there was a short
1354                  * packet (SPD and not ACTIVE)
1355                  */
1356                 td = xfer->td_transfer_cache;
1357
1358                 while (1) {
1359                         usb_pc_cpu_invalidate(td->page_cache);
1360                         status = le32toh(td->td_status);
1361                         token = le32toh(td->td_token);
1362
1363                         /*
1364                          * if there is an active TD the transfer isn't done
1365                          */
1366                         if (status & UHCI_TD_ACTIVE) {
1367                                 /* update cache */
1368                                 xfer->td_transfer_cache = td;
1369                                 goto done;
1370                         }
1371                         /*
1372                          * last transfer descriptor makes the transfer done
1373                          */
1374                         if (((void *)td) == xfer->td_transfer_last) {
1375                                 break;
1376                         }
1377                         /*
1378                          * any kind of error makes the transfer done
1379                          */
1380                         if (status & UHCI_TD_STALLED) {
1381                                 break;
1382                         }
1383                         /*
1384                          * check if we reached the last packet
1385                          * or if there is a short packet:
1386                          */
1387                         if ((td->td_next == htole32(UHCI_PTR_T)) ||
1388                             (UHCI_TD_GET_ACTLEN(status) < td->len)) {
1389
1390                                 if (xfer->flags_int.short_frames_ok) {
1391                                         /* follow alt next */
1392                                         if (td->alt_next) {
1393                                                 /* update cache */
1394                                                 xfer->td_transfer_cache = td;
1395                                                 uhci_check_transfer_sub(xfer);
1396                                                 goto done;
1397                                         }
1398                                 }
1399                                 /* transfer is done */
1400                                 break;
1401                         }
1402                         td = td->obj_next;
1403                 }
1404                 uhci_non_isoc_done(xfer);
1405                 goto transferred;
1406         }
1407
1408 done:
1409         DPRINTFN(13, "xfer=%p is still active\n", xfer);
1410         return (0);
1411
1412 transferred:
1413         return (1);
1414 }
1415
1416 static void
1417 uhci_interrupt_poll(uhci_softc_t *sc)
1418 {
1419         struct usb_xfer *xfer;
1420
1421 repeat:
1422         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
1423                 /*
1424                  * check if transfer is transferred
1425                  */
1426                 if (uhci_check_transfer(xfer)) {
1427                         /* queue has been modified */
1428                         goto repeat;
1429                 }
1430         }
1431 }
1432
1433 /*------------------------------------------------------------------------*
1434  *      uhci_interrupt - UHCI interrupt handler
1435  *
1436  * NOTE: Do not access "sc->sc_bus.bdev" inside the interrupt handler,
1437  * hence the interrupt handler will be setup before "sc->sc_bus.bdev"
1438  * is present !
1439  *------------------------------------------------------------------------*/
1440 void
1441 uhci_interrupt(uhci_softc_t *sc)
1442 {
1443         uint32_t status;
1444
1445         USB_BUS_LOCK(&sc->sc_bus);
1446
1447         DPRINTFN(16, "real interrupt\n");
1448
1449 #ifdef USB_DEBUG
1450         if (uhcidebug > 15) {
1451                 uhci_dumpregs(sc);
1452         }
1453 #endif
1454         status = UREAD2(sc, UHCI_STS) & UHCI_STS_ALLINTRS;
1455         if (status == 0) {
1456                 /* the interrupt was not for us */
1457                 goto done;
1458         }
1459         if (status & (UHCI_STS_RD | UHCI_STS_HSE |
1460             UHCI_STS_HCPE | UHCI_STS_HCH)) {
1461
1462                 if (status & UHCI_STS_RD) {
1463 #ifdef USB_DEBUG
1464                         printf("%s: resume detect\n",
1465                             __FUNCTION__);
1466 #endif
1467                 }
1468                 if (status & UHCI_STS_HSE) {
1469                         printf("%s: host system error\n",
1470                             __FUNCTION__);
1471                 }
1472                 if (status & UHCI_STS_HCPE) {
1473                         printf("%s: host controller process error\n",
1474                             __FUNCTION__);
1475                 }
1476                 if (status & UHCI_STS_HCH) {
1477                         /* no acknowledge needed */
1478                         DPRINTF("%s: host controller halted\n",
1479                             __FUNCTION__);
1480 #ifdef USB_DEBUG
1481                         if (uhcidebug > 0) {
1482                                 uhci_dump_all(sc);
1483                         }
1484 #endif
1485                 }
1486         }
1487         /* get acknowledge bits */
1488         status &= (UHCI_STS_USBINT |
1489             UHCI_STS_USBEI |
1490             UHCI_STS_RD |
1491             UHCI_STS_HSE |
1492             UHCI_STS_HCPE);
1493
1494         if (status == 0) {
1495                 /* nothing to acknowledge */
1496                 goto done;
1497         }
1498         /* acknowledge interrupts */
1499         UWRITE2(sc, UHCI_STS, status);
1500
1501         /* poll all the USB transfers */
1502         uhci_interrupt_poll(sc);
1503
1504 done:
1505         USB_BUS_UNLOCK(&sc->sc_bus);
1506 }
1507
1508 /*
1509  * called when a request does not complete
1510  */
1511 static void
1512 uhci_timeout(void *arg)
1513 {
1514         struct usb_xfer *xfer = arg;
1515
1516         DPRINTF("xfer=%p\n", xfer);
1517
1518         USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
1519
1520         /* transfer is transferred */
1521         uhci_device_done(xfer, USB_ERR_TIMEOUT);
1522 }
1523
1524 static void
1525 uhci_do_poll(struct usb_bus *bus)
1526 {
1527         struct uhci_softc *sc = UHCI_BUS2SC(bus);
1528
1529         USB_BUS_LOCK(&sc->sc_bus);
1530         uhci_interrupt_poll(sc);
1531         USB_BUS_UNLOCK(&sc->sc_bus);
1532 }
1533
1534 static void
1535 uhci_setup_standard_chain_sub(struct uhci_std_temp *temp)
1536 {
1537         uhci_td_t *td;
1538         uhci_td_t *td_next;
1539         uhci_td_t *td_alt_next;
1540         uint32_t average;
1541         uint32_t len_old;
1542         uint8_t shortpkt_old;
1543         uint8_t precompute;
1544
1545         td_alt_next = NULL;
1546         shortpkt_old = temp->shortpkt;
1547         len_old = temp->len;
1548         precompute = 1;
1549
1550         /* software is used to detect short incoming transfers */
1551
1552         if ((temp->td_token & htole32(UHCI_TD_PID)) == htole32(UHCI_TD_PID_IN)) {
1553                 temp->td_status |= htole32(UHCI_TD_SPD);
1554         } else {
1555                 temp->td_status &= ~htole32(UHCI_TD_SPD);
1556         }
1557
1558         temp->ml.buf_offset = 0;
1559
1560 restart:
1561
1562         temp->td_token &= ~htole32(UHCI_TD_SET_MAXLEN(0));
1563         temp->td_token |= htole32(UHCI_TD_SET_MAXLEN(temp->average));
1564
1565         td = temp->td;
1566         td_next = temp->td_next;
1567
1568         while (1) {
1569
1570                 if (temp->len == 0) {
1571
1572                         if (temp->shortpkt) {
1573                                 break;
1574                         }
1575                         /* send a Zero Length Packet, ZLP, last */
1576
1577                         temp->shortpkt = 1;
1578                         temp->td_token |= htole32(UHCI_TD_SET_MAXLEN(0));
1579                         average = 0;
1580
1581                 } else {
1582
1583                         average = temp->average;
1584
1585                         if (temp->len < average) {
1586                                 temp->shortpkt = 1;
1587                                 temp->td_token &= ~htole32(UHCI_TD_SET_MAXLEN(0));
1588                                 temp->td_token |= htole32(UHCI_TD_SET_MAXLEN(temp->len));
1589                                 average = temp->len;
1590                         }
1591                 }
1592
1593                 if (td_next == NULL) {
1594                         panic("%s: out of UHCI transfer descriptors!", __FUNCTION__);
1595                 }
1596                 /* get next TD */
1597
1598                 td = td_next;
1599                 td_next = td->obj_next;
1600
1601                 /* check if we are pre-computing */
1602
1603                 if (precompute) {
1604
1605                         /* update remaining length */
1606
1607                         temp->len -= average;
1608
1609                         continue;
1610                 }
1611                 /* fill out current TD */
1612
1613                 td->td_status = temp->td_status;
1614                 td->td_token = temp->td_token;
1615
1616                 /* update data toggle */
1617
1618                 temp->td_token ^= htole32(UHCI_TD_SET_DT(1));
1619
1620                 if (average == 0) {
1621
1622                         td->len = 0;
1623                         td->td_buffer = 0;
1624                         td->fix_pc = NULL;
1625
1626                 } else {
1627
1628                         /* update remaining length */
1629
1630                         temp->len -= average;
1631
1632                         td->len = average;
1633
1634                         /* fill out buffer pointer and do fixup, if any */
1635
1636                         uhci_mem_layout_fixup(&temp->ml, td);
1637                 }
1638
1639                 td->alt_next = td_alt_next;
1640
1641                 if ((td_next == td_alt_next) && temp->setup_alt_next) {
1642                         /* we need to receive these frames one by one ! */
1643                         td->td_status |= htole32(UHCI_TD_IOC);
1644                         td->td_next = htole32(UHCI_PTR_T);
1645                 } else {
1646                         if (td_next) {
1647                                 /* link the current TD with the next one */
1648                                 td->td_next = td_next->td_self;
1649                         }
1650                 }
1651
1652                 usb_pc_cpu_flush(td->page_cache);
1653         }
1654
1655         if (precompute) {
1656                 precompute = 0;
1657
1658                 /* setup alt next pointer, if any */
1659                 if (temp->last_frame) {
1660                         td_alt_next = NULL;
1661                 } else {
1662                         /* we use this field internally */
1663                         td_alt_next = td_next;
1664                 }
1665
1666                 /* restore */
1667                 temp->shortpkt = shortpkt_old;
1668                 temp->len = len_old;
1669                 goto restart;
1670         }
1671         temp->td = td;
1672         temp->td_next = td_next;
1673 }
1674
1675 static uhci_td_t *
1676 uhci_setup_standard_chain(struct usb_xfer *xfer)
1677 {
1678         struct uhci_std_temp temp;
1679         uhci_td_t *td;
1680         uint32_t x;
1681
1682         DPRINTFN(9, "addr=%d endpt=%d sumlen=%d speed=%d\n",
1683             xfer->address, UE_GET_ADDR(xfer->endpointno),
1684             xfer->sumlen, usbd_get_speed(xfer->xroot->udev));
1685
1686         temp.average = xfer->max_frame_size;
1687         temp.max_frame_size = xfer->max_frame_size;
1688
1689         /* toggle the DMA set we are using */
1690         xfer->flags_int.curr_dma_set ^= 1;
1691
1692         /* get next DMA set */
1693         td = xfer->td_start[xfer->flags_int.curr_dma_set];
1694         xfer->td_transfer_first = td;
1695         xfer->td_transfer_cache = td;
1696
1697         temp.td = NULL;
1698         temp.td_next = td;
1699         temp.last_frame = 0;
1700         temp.setup_alt_next = xfer->flags_int.short_frames_ok;
1701
1702         uhci_mem_layout_init(&temp.ml, xfer);
1703
1704         temp.td_status =
1705             htole32(UHCI_TD_ZERO_ACTLEN(UHCI_TD_SET_ERRCNT(3) |
1706             UHCI_TD_ACTIVE));
1707
1708         if (xfer->xroot->udev->speed == USB_SPEED_LOW) {
1709                 temp.td_status |= htole32(UHCI_TD_LS);
1710         }
1711         temp.td_token =
1712             htole32(UHCI_TD_SET_ENDPT(xfer->endpointno) |
1713             UHCI_TD_SET_DEVADDR(xfer->address));
1714
1715         if (xfer->endpoint->toggle_next) {
1716                 /* DATA1 is next */
1717                 temp.td_token |= htole32(UHCI_TD_SET_DT(1));
1718         }
1719         /* check if we should prepend a setup message */
1720
1721         if (xfer->flags_int.control_xfr) {
1722
1723                 if (xfer->flags_int.control_hdr) {
1724
1725                         temp.td_token &= htole32(UHCI_TD_SET_DEVADDR(0x7F) |
1726                             UHCI_TD_SET_ENDPT(0xF));
1727                         temp.td_token |= htole32(UHCI_TD_PID_SETUP |
1728                             UHCI_TD_SET_DT(0));
1729
1730                         temp.len = xfer->frlengths[0];
1731                         temp.ml.buf_pc = xfer->frbuffers + 0;
1732                         temp.shortpkt = temp.len ? 1 : 0;
1733                         /* check for last frame */
1734                         if (xfer->nframes == 1) {
1735                                 /* no STATUS stage yet, SETUP is last */
1736                                 if (xfer->flags_int.control_act) {
1737                                         temp.last_frame = 1;
1738                                         temp.setup_alt_next = 0;
1739                                 }
1740                         }
1741                         uhci_setup_standard_chain_sub(&temp);
1742                 }
1743                 x = 1;
1744         } else {
1745                 x = 0;
1746         }
1747
1748         while (x != xfer->nframes) {
1749
1750                 /* DATA0 / DATA1 message */
1751
1752                 temp.len = xfer->frlengths[x];
1753                 temp.ml.buf_pc = xfer->frbuffers + x;
1754
1755                 x++;
1756
1757                 if (x == xfer->nframes) {
1758                         if (xfer->flags_int.control_xfr) {
1759                                 /* no STATUS stage yet, DATA is last */
1760                                 if (xfer->flags_int.control_act) {
1761                                         temp.last_frame = 1;
1762                                         temp.setup_alt_next = 0;
1763                                 }
1764                         } else {
1765                                 temp.last_frame = 1;
1766                                 temp.setup_alt_next = 0;
1767                         }
1768                 }
1769                 /*
1770                  * Keep previous data toggle,
1771                  * device address and endpoint number:
1772                  */
1773
1774                 temp.td_token &= htole32(UHCI_TD_SET_DEVADDR(0x7F) |
1775                     UHCI_TD_SET_ENDPT(0xF) |
1776                     UHCI_TD_SET_DT(1));
1777
1778                 if (temp.len == 0) {
1779
1780                         /* make sure that we send an USB packet */
1781
1782                         temp.shortpkt = 0;
1783
1784                 } else {
1785
1786                         /* regular data transfer */
1787
1788                         temp.shortpkt = (xfer->flags.force_short_xfer) ? 0 : 1;
1789                 }
1790
1791                 /* set endpoint direction */
1792
1793                 temp.td_token |=
1794                     (UE_GET_DIR(xfer->endpointno) == UE_DIR_IN) ?
1795                     htole32(UHCI_TD_PID_IN) :
1796                     htole32(UHCI_TD_PID_OUT);
1797
1798                 uhci_setup_standard_chain_sub(&temp);
1799         }
1800
1801         /* check if we should append a status stage */
1802
1803         if (xfer->flags_int.control_xfr &&
1804             !xfer->flags_int.control_act) {
1805
1806                 /*
1807                  * send a DATA1 message and reverse the current endpoint
1808                  * direction
1809                  */
1810
1811                 temp.td_token &= htole32(UHCI_TD_SET_DEVADDR(0x7F) |
1812                     UHCI_TD_SET_ENDPT(0xF) |
1813                     UHCI_TD_SET_DT(1));
1814                 temp.td_token |=
1815                     (UE_GET_DIR(xfer->endpointno) == UE_DIR_OUT) ?
1816                     htole32(UHCI_TD_PID_IN | UHCI_TD_SET_DT(1)) :
1817                     htole32(UHCI_TD_PID_OUT | UHCI_TD_SET_DT(1));
1818
1819                 temp.len = 0;
1820                 temp.ml.buf_pc = NULL;
1821                 temp.shortpkt = 0;
1822                 temp.last_frame = 1;
1823                 temp.setup_alt_next = 0;
1824
1825                 uhci_setup_standard_chain_sub(&temp);
1826         }
1827         td = temp.td;
1828
1829         /* Ensure that last TD is terminating: */
1830         td->td_next = htole32(UHCI_PTR_T);
1831
1832         /* set interrupt bit */
1833
1834         td->td_status |= htole32(UHCI_TD_IOC);
1835
1836         usb_pc_cpu_flush(td->page_cache);
1837
1838         /* must have at least one frame! */
1839
1840         xfer->td_transfer_last = td;
1841
1842 #ifdef USB_DEBUG
1843         if (uhcidebug > 8) {
1844                 DPRINTF("nexttog=%d; data before transfer:\n",
1845                     xfer->endpoint->toggle_next);
1846                 uhci_dump_tds(xfer->td_transfer_first);
1847         }
1848 #endif
1849         return (xfer->td_transfer_first);
1850 }
1851
1852 /* NOTE: "done" can be run two times in a row,
1853  * from close and from interrupt
1854  */
1855
1856 static void
1857 uhci_device_done(struct usb_xfer *xfer, usb_error_t error)
1858 {
1859         struct usb_pipe_methods *methods = xfer->endpoint->methods;
1860         uhci_softc_t *sc = UHCI_BUS2SC(xfer->xroot->bus);
1861         uhci_qh_t *qh;
1862
1863         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
1864
1865         DPRINTFN(2, "xfer=%p, endpoint=%p, error=%d\n",
1866             xfer, xfer->endpoint, error);
1867
1868         qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
1869         if (qh) {
1870                 usb_pc_cpu_invalidate(qh->page_cache);
1871         }
1872         if (xfer->flags_int.bandwidth_reclaimed) {
1873                 xfer->flags_int.bandwidth_reclaimed = 0;
1874                 uhci_rem_loop(sc);
1875         }
1876         if (methods == &uhci_device_bulk_methods) {
1877                 UHCI_REMOVE_QH(qh, sc->sc_bulk_p_last);
1878         }
1879         if (methods == &uhci_device_ctrl_methods) {
1880                 if (xfer->xroot->udev->speed == USB_SPEED_LOW) {
1881                         UHCI_REMOVE_QH(qh, sc->sc_ls_ctl_p_last);
1882                 } else {
1883                         UHCI_REMOVE_QH(qh, sc->sc_fs_ctl_p_last);
1884                 }
1885         }
1886         if (methods == &uhci_device_intr_methods) {
1887                 UHCI_REMOVE_QH(qh, sc->sc_intr_p_last[xfer->qh_pos]);
1888         }
1889         /*
1890          * Only finish isochronous transfers once
1891          * which will update "xfer->frlengths".
1892          */
1893         if (xfer->td_transfer_first &&
1894             xfer->td_transfer_last) {
1895                 if (methods == &uhci_device_isoc_methods) {
1896                         uhci_isoc_done(sc, xfer);
1897                 }
1898                 xfer->td_transfer_first = NULL;
1899                 xfer->td_transfer_last = NULL;
1900         }
1901         /* dequeue transfer and start next transfer */
1902         usbd_transfer_done(xfer, error);
1903 }
1904
1905 /*------------------------------------------------------------------------*
1906  * uhci bulk support
1907  *------------------------------------------------------------------------*/
1908 static void
1909 uhci_device_bulk_open(struct usb_xfer *xfer)
1910 {
1911         return;
1912 }
1913
1914 static void
1915 uhci_device_bulk_close(struct usb_xfer *xfer)
1916 {
1917         uhci_device_done(xfer, USB_ERR_CANCELLED);
1918 }
1919
1920 static void
1921 uhci_device_bulk_enter(struct usb_xfer *xfer)
1922 {
1923         return;
1924 }
1925
1926 static void
1927 uhci_device_bulk_start(struct usb_xfer *xfer)
1928 {
1929         uhci_softc_t *sc = UHCI_BUS2SC(xfer->xroot->bus);
1930         uhci_td_t *td;
1931         uhci_qh_t *qh;
1932
1933         /* setup TD's */
1934         td = uhci_setup_standard_chain(xfer);
1935
1936         /* setup QH */
1937         qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
1938
1939         qh->e_next = td;
1940         qh->qh_e_next = td->td_self;
1941
1942         if (xfer->xroot->udev->flags.self_suspended == 0) {
1943                 UHCI_APPEND_QH(qh, sc->sc_bulk_p_last);
1944                 uhci_add_loop(sc);
1945                 xfer->flags_int.bandwidth_reclaimed = 1;
1946         } else {
1947                 usb_pc_cpu_flush(qh->page_cache);
1948         }
1949
1950         /* put transfer on interrupt queue */
1951         uhci_transfer_intr_enqueue(xfer);
1952 }
1953
1954 struct usb_pipe_methods uhci_device_bulk_methods =
1955 {
1956         .open = uhci_device_bulk_open,
1957         .close = uhci_device_bulk_close,
1958         .enter = uhci_device_bulk_enter,
1959         .start = uhci_device_bulk_start,
1960 };
1961
1962 /*------------------------------------------------------------------------*
1963  * uhci control support
1964  *------------------------------------------------------------------------*/
1965 static void
1966 uhci_device_ctrl_open(struct usb_xfer *xfer)
1967 {
1968         return;
1969 }
1970
1971 static void
1972 uhci_device_ctrl_close(struct usb_xfer *xfer)
1973 {
1974         uhci_device_done(xfer, USB_ERR_CANCELLED);
1975 }
1976
1977 static void
1978 uhci_device_ctrl_enter(struct usb_xfer *xfer)
1979 {
1980         return;
1981 }
1982
1983 static void
1984 uhci_device_ctrl_start(struct usb_xfer *xfer)
1985 {
1986         uhci_softc_t *sc = UHCI_BUS2SC(xfer->xroot->bus);
1987         uhci_qh_t *qh;
1988         uhci_td_t *td;
1989
1990         /* setup TD's */
1991         td = uhci_setup_standard_chain(xfer);
1992
1993         /* setup QH */
1994         qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
1995
1996         qh->e_next = td;
1997         qh->qh_e_next = td->td_self;
1998
1999         /*
2000          * NOTE: some devices choke on bandwidth- reclamation for control
2001          * transfers
2002          */
2003         if (xfer->xroot->udev->flags.self_suspended == 0) {
2004                 if (xfer->xroot->udev->speed == USB_SPEED_LOW) {
2005                         UHCI_APPEND_QH(qh, sc->sc_ls_ctl_p_last);
2006                 } else {
2007                         UHCI_APPEND_QH(qh, sc->sc_fs_ctl_p_last);
2008                 }
2009         } else {
2010                 usb_pc_cpu_flush(qh->page_cache);
2011         }
2012         /* put transfer on interrupt queue */
2013         uhci_transfer_intr_enqueue(xfer);
2014 }
2015
2016 struct usb_pipe_methods uhci_device_ctrl_methods =
2017 {
2018         .open = uhci_device_ctrl_open,
2019         .close = uhci_device_ctrl_close,
2020         .enter = uhci_device_ctrl_enter,
2021         .start = uhci_device_ctrl_start,
2022 };
2023
2024 /*------------------------------------------------------------------------*
2025  * uhci interrupt support
2026  *------------------------------------------------------------------------*/
2027 static void
2028 uhci_device_intr_open(struct usb_xfer *xfer)
2029 {
2030         uhci_softc_t *sc = UHCI_BUS2SC(xfer->xroot->bus);
2031         uint16_t best;
2032         uint16_t bit;
2033         uint16_t x;
2034
2035         best = 0;
2036         bit = UHCI_IFRAMELIST_COUNT / 2;
2037         while (bit) {
2038                 if (xfer->interval >= bit) {
2039                         x = bit;
2040                         best = bit;
2041                         while (x & bit) {
2042                                 if (sc->sc_intr_stat[x] <
2043                                     sc->sc_intr_stat[best]) {
2044                                         best = x;
2045                                 }
2046                                 x++;
2047                         }
2048                         break;
2049                 }
2050                 bit >>= 1;
2051         }
2052
2053         sc->sc_intr_stat[best]++;
2054         xfer->qh_pos = best;
2055
2056         DPRINTFN(3, "best=%d interval=%d\n",
2057             best, xfer->interval);
2058 }
2059
2060 static void
2061 uhci_device_intr_close(struct usb_xfer *xfer)
2062 {
2063         uhci_softc_t *sc = UHCI_BUS2SC(xfer->xroot->bus);
2064
2065         sc->sc_intr_stat[xfer->qh_pos]--;
2066
2067         uhci_device_done(xfer, USB_ERR_CANCELLED);
2068 }
2069
2070 static void
2071 uhci_device_intr_enter(struct usb_xfer *xfer)
2072 {
2073         return;
2074 }
2075
2076 static void
2077 uhci_device_intr_start(struct usb_xfer *xfer)
2078 {
2079         uhci_softc_t *sc = UHCI_BUS2SC(xfer->xroot->bus);
2080         uhci_qh_t *qh;
2081         uhci_td_t *td;
2082
2083         /* setup TD's */
2084         td = uhci_setup_standard_chain(xfer);
2085
2086         /* setup QH */
2087         qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
2088
2089         qh->e_next = td;
2090         qh->qh_e_next = td->td_self;
2091
2092         if (xfer->xroot->udev->flags.self_suspended == 0) {
2093                 /* enter QHs into the controller data structures */
2094                 UHCI_APPEND_QH(qh, sc->sc_intr_p_last[xfer->qh_pos]);
2095         } else {
2096                 usb_pc_cpu_flush(qh->page_cache);
2097         }
2098
2099         /* put transfer on interrupt queue */
2100         uhci_transfer_intr_enqueue(xfer);
2101 }
2102
2103 struct usb_pipe_methods uhci_device_intr_methods =
2104 {
2105         .open = uhci_device_intr_open,
2106         .close = uhci_device_intr_close,
2107         .enter = uhci_device_intr_enter,
2108         .start = uhci_device_intr_start,
2109 };
2110
2111 /*------------------------------------------------------------------------*
2112  * uhci isochronous support
2113  *------------------------------------------------------------------------*/
2114 static void
2115 uhci_device_isoc_open(struct usb_xfer *xfer)
2116 {
2117         uhci_td_t *td;
2118         uint32_t td_token;
2119         uint8_t ds;
2120
2121         td_token =
2122             (UE_GET_DIR(xfer->endpointno) == UE_DIR_IN) ?
2123             UHCI_TD_IN(0, xfer->endpointno, xfer->address, 0) :
2124             UHCI_TD_OUT(0, xfer->endpointno, xfer->address, 0);
2125
2126         td_token = htole32(td_token);
2127
2128         /* initialize all TD's */
2129
2130         for (ds = 0; ds != 2; ds++) {
2131
2132                 for (td = xfer->td_start[ds]; td; td = td->obj_next) {
2133
2134                         /* mark TD as inactive */
2135                         td->td_status = htole32(UHCI_TD_IOS);
2136                         td->td_token = td_token;
2137
2138                         usb_pc_cpu_flush(td->page_cache);
2139                 }
2140         }
2141 }
2142
2143 static void
2144 uhci_device_isoc_close(struct usb_xfer *xfer)
2145 {
2146         uhci_device_done(xfer, USB_ERR_CANCELLED);
2147 }
2148
2149 static void
2150 uhci_device_isoc_enter(struct usb_xfer *xfer)
2151 {
2152         struct uhci_mem_layout ml;
2153         uhci_softc_t *sc = UHCI_BUS2SC(xfer->xroot->bus);
2154         uint32_t nframes;
2155         uint32_t temp;
2156         uint32_t *plen;
2157
2158 #ifdef USB_DEBUG
2159         uint8_t once = 1;
2160
2161 #endif
2162         uhci_td_t *td;
2163         uhci_td_t *td_last = NULL;
2164         uhci_td_t **pp_last;
2165
2166         DPRINTFN(6, "xfer=%p next=%d nframes=%d\n",
2167             xfer, xfer->endpoint->isoc_next, xfer->nframes);
2168
2169         nframes = UREAD2(sc, UHCI_FRNUM);
2170
2171         temp = (nframes - xfer->endpoint->isoc_next) &
2172             (UHCI_VFRAMELIST_COUNT - 1);
2173
2174         if ((xfer->endpoint->is_synced == 0) ||
2175             (temp < xfer->nframes)) {
2176                 /*
2177                  * If there is data underflow or the pipe queue is empty we
2178                  * schedule the transfer a few frames ahead of the current
2179                  * frame position. Else two isochronous transfers might
2180                  * overlap.
2181                  */
2182                 xfer->endpoint->isoc_next = (nframes + 3) & (UHCI_VFRAMELIST_COUNT - 1);
2183                 xfer->endpoint->is_synced = 1;
2184                 DPRINTFN(3, "start next=%d\n", xfer->endpoint->isoc_next);
2185         }
2186         /*
2187          * compute how many milliseconds the insertion is ahead of the
2188          * current frame position:
2189          */
2190         temp = (xfer->endpoint->isoc_next - nframes) &
2191             (UHCI_VFRAMELIST_COUNT - 1);
2192
2193         /*
2194          * pre-compute when the isochronous transfer will be finished:
2195          */
2196         xfer->isoc_time_complete =
2197             usb_isoc_time_expand(&sc->sc_bus, nframes) + temp +
2198             xfer->nframes;
2199
2200         /* get the real number of frames */
2201
2202         nframes = xfer->nframes;
2203
2204         uhci_mem_layout_init(&ml, xfer);
2205
2206         plen = xfer->frlengths;
2207
2208         /* toggle the DMA set we are using */
2209         xfer->flags_int.curr_dma_set ^= 1;
2210
2211         /* get next DMA set */
2212         td = xfer->td_start[xfer->flags_int.curr_dma_set];
2213         xfer->td_transfer_first = td;
2214
2215         pp_last = &sc->sc_isoc_p_last[xfer->endpoint->isoc_next];
2216
2217         /* store starting position */
2218
2219         xfer->qh_pos = xfer->endpoint->isoc_next;
2220
2221         while (nframes--) {
2222                 if (td == NULL) {
2223                         panic("%s:%d: out of TD's\n",
2224                             __FUNCTION__, __LINE__);
2225                 }
2226                 if (pp_last >= &sc->sc_isoc_p_last[UHCI_VFRAMELIST_COUNT]) {
2227                         pp_last = &sc->sc_isoc_p_last[0];
2228                 }
2229                 if (*plen > xfer->max_frame_size) {
2230 #ifdef USB_DEBUG
2231                         if (once) {
2232                                 once = 0;
2233                                 printf("%s: frame length(%d) exceeds %d "
2234                                     "bytes (frame truncated)\n",
2235                                     __FUNCTION__, *plen,
2236                                     xfer->max_frame_size);
2237                         }
2238 #endif
2239                         *plen = xfer->max_frame_size;
2240                 }
2241                 /* reuse td_token from last transfer */
2242
2243                 td->td_token &= htole32(~UHCI_TD_MAXLEN_MASK);
2244                 td->td_token |= htole32(UHCI_TD_SET_MAXLEN(*plen));
2245
2246                 td->len = *plen;
2247
2248                 if (td->len == 0) {
2249                         /*
2250                          * Do not call "uhci_mem_layout_fixup()" when the
2251                          * length is zero!
2252                          */
2253                         td->td_buffer = 0;
2254                         td->fix_pc = NULL;
2255
2256                 } else {
2257
2258                         /* fill out buffer pointer and do fixup, if any */
2259
2260                         uhci_mem_layout_fixup(&ml, td);
2261
2262                 }
2263
2264                 /* update status */
2265                 if (nframes == 0) {
2266                         td->td_status = htole32
2267                             (UHCI_TD_ZERO_ACTLEN
2268                             (UHCI_TD_SET_ERRCNT(0) |
2269                             UHCI_TD_ACTIVE |
2270                             UHCI_TD_IOS |
2271                             UHCI_TD_IOC));
2272                 } else {
2273                         td->td_status = htole32
2274                             (UHCI_TD_ZERO_ACTLEN
2275                             (UHCI_TD_SET_ERRCNT(0) |
2276                             UHCI_TD_ACTIVE |
2277                             UHCI_TD_IOS));
2278                 }
2279
2280                 usb_pc_cpu_flush(td->page_cache);
2281
2282 #ifdef USB_DEBUG
2283                 if (uhcidebug > 5) {
2284                         DPRINTF("TD %d\n", nframes);
2285                         uhci_dump_td(td);
2286                 }
2287 #endif
2288                 /* insert TD into schedule */
2289                 UHCI_APPEND_TD(td, *pp_last);
2290                 pp_last++;
2291
2292                 plen++;
2293                 td_last = td;
2294                 td = td->obj_next;
2295         }
2296
2297         xfer->td_transfer_last = td_last;
2298
2299         /* update isoc_next */
2300         xfer->endpoint->isoc_next = (pp_last - &sc->sc_isoc_p_last[0]) &
2301             (UHCI_VFRAMELIST_COUNT - 1);
2302 }
2303
2304 static void
2305 uhci_device_isoc_start(struct usb_xfer *xfer)
2306 {
2307         /* put transfer on interrupt queue */
2308         uhci_transfer_intr_enqueue(xfer);
2309 }
2310
2311 struct usb_pipe_methods uhci_device_isoc_methods =
2312 {
2313         .open = uhci_device_isoc_open,
2314         .close = uhci_device_isoc_close,
2315         .enter = uhci_device_isoc_enter,
2316         .start = uhci_device_isoc_start,
2317 };
2318
2319 /*------------------------------------------------------------------------*
2320  * uhci root control support
2321  *------------------------------------------------------------------------*
2322  * Simulate a hardware hub by handling all the necessary requests.
2323  *------------------------------------------------------------------------*/
2324
2325 static const
2326 struct usb_device_descriptor uhci_devd =
2327 {
2328         sizeof(struct usb_device_descriptor),
2329         UDESC_DEVICE,                   /* type */
2330         {0x00, 0x01},                   /* USB version */
2331         UDCLASS_HUB,                    /* class */
2332         UDSUBCLASS_HUB,                 /* subclass */
2333         UDPROTO_FSHUB,                  /* protocol */
2334         64,                             /* max packet */
2335         {0}, {0}, {0x00, 0x01},         /* device id */
2336         1, 2, 0,                        /* string indicies */
2337         1                               /* # of configurations */
2338 };
2339
2340 static const struct uhci_config_desc uhci_confd = {
2341         .confd = {
2342                 .bLength = sizeof(struct usb_config_descriptor),
2343                 .bDescriptorType = UDESC_CONFIG,
2344                 .wTotalLength[0] = sizeof(uhci_confd),
2345                 .bNumInterface = 1,
2346                 .bConfigurationValue = 1,
2347                 .iConfiguration = 0,
2348                 .bmAttributes = UC_SELF_POWERED,
2349                 .bMaxPower = 0          /* max power */
2350         },
2351         .ifcd = {
2352                 .bLength = sizeof(struct usb_interface_descriptor),
2353                 .bDescriptorType = UDESC_INTERFACE,
2354                 .bNumEndpoints = 1,
2355                 .bInterfaceClass = UICLASS_HUB,
2356                 .bInterfaceSubClass = UISUBCLASS_HUB,
2357                 .bInterfaceProtocol = UIPROTO_FSHUB,
2358         },
2359         .endpd = {
2360                 .bLength = sizeof(struct usb_endpoint_descriptor),
2361                 .bDescriptorType = UDESC_ENDPOINT,
2362                 .bEndpointAddress = UE_DIR_IN | UHCI_INTR_ENDPT,
2363                 .bmAttributes = UE_INTERRUPT,
2364                 .wMaxPacketSize[0] = 8, /* max packet (63 ports) */
2365                 .bInterval = 255,
2366         },
2367 };
2368
2369 static const
2370 struct usb_hub_descriptor_min uhci_hubd_piix =
2371 {
2372         sizeof(uhci_hubd_piix),
2373         UDESC_HUB,
2374         2,
2375         {UHD_PWR_NO_SWITCH | UHD_OC_INDIVIDUAL, 0},
2376         50,                             /* power on to power good */
2377         0,
2378         {0x00},                         /* both ports are removable */
2379 };
2380
2381 /*
2382  * The USB hub protocol requires that SET_FEATURE(PORT_RESET) also
2383  * enables the port, and also states that SET_FEATURE(PORT_ENABLE)
2384  * should not be used by the USB subsystem.  As we cannot issue a
2385  * SET_FEATURE(PORT_ENABLE) externally, we must ensure that the port
2386  * will be enabled as part of the reset.
2387  *
2388  * On the VT83C572, the port cannot be successfully enabled until the
2389  * outstanding "port enable change" and "connection status change"
2390  * events have been reset.
2391  */
2392 static usb_error_t
2393 uhci_portreset(uhci_softc_t *sc, uint16_t index)
2394 {
2395         uint16_t port;
2396         uint16_t x;
2397         uint8_t lim;
2398
2399         if (index == 1)
2400                 port = UHCI_PORTSC1;
2401         else if (index == 2)
2402                 port = UHCI_PORTSC2;
2403         else
2404                 return (USB_ERR_IOERROR);
2405
2406         /*
2407          * Before we do anything, turn on SOF messages on the USB
2408          * BUS. Some USB devices do not cope without them!
2409          */
2410         uhci_restart(sc);
2411
2412         x = URWMASK(UREAD2(sc, port));
2413         UWRITE2(sc, port, x | UHCI_PORTSC_PR);
2414
2415         usb_pause_mtx(&sc->sc_bus.bus_mtx,
2416             USB_MS_TO_TICKS(USB_PORT_ROOT_RESET_DELAY));
2417
2418         DPRINTFN(4, "uhci port %d reset, status0 = 0x%04x\n",
2419             index, UREAD2(sc, port));
2420
2421         x = URWMASK(UREAD2(sc, port));
2422         UWRITE2(sc, port, x & ~UHCI_PORTSC_PR);
2423
2424
2425         mtx_unlock(&sc->sc_bus.bus_mtx);
2426
2427         /*
2428          * This delay needs to be exactly 100us, else some USB devices
2429          * fail to attach!
2430          */
2431         DELAY(100);
2432
2433         mtx_lock(&sc->sc_bus.bus_mtx);
2434
2435         DPRINTFN(4, "uhci port %d reset, status1 = 0x%04x\n",
2436             index, UREAD2(sc, port));
2437
2438         x = URWMASK(UREAD2(sc, port));
2439         UWRITE2(sc, port, x | UHCI_PORTSC_PE);
2440
2441         for (lim = 0; lim < 12; lim++) {
2442
2443                 usb_pause_mtx(&sc->sc_bus.bus_mtx,
2444                     USB_MS_TO_TICKS(USB_PORT_RESET_DELAY));
2445
2446                 x = UREAD2(sc, port);
2447
2448                 DPRINTFN(4, "uhci port %d iteration %u, status = 0x%04x\n",
2449                     index, lim, x);
2450
2451                 if (!(x & UHCI_PORTSC_CCS)) {
2452                         /*
2453                          * No device is connected (or was disconnected
2454                          * during reset).  Consider the port reset.
2455                          * The delay must be long enough to ensure on
2456                          * the initial iteration that the device
2457                          * connection will have been registered.  50ms
2458                          * appears to be sufficient, but 20ms is not.
2459                          */
2460                         DPRINTFN(4, "uhci port %d loop %u, device detached\n",
2461                             index, lim);
2462                         goto done;
2463                 }
2464                 if (x & (UHCI_PORTSC_POEDC | UHCI_PORTSC_CSC)) {
2465                         /*
2466                          * Port enabled changed and/or connection
2467                          * status changed were set.  Reset either or
2468                          * both raised flags (by writing a 1 to that
2469                          * bit), and wait again for state to settle.
2470                          */
2471                         UWRITE2(sc, port, URWMASK(x) |
2472                             (x & (UHCI_PORTSC_POEDC | UHCI_PORTSC_CSC)));
2473                         continue;
2474                 }
2475                 if (x & UHCI_PORTSC_PE) {
2476                         /* port is enabled */
2477                         goto done;
2478                 }
2479                 UWRITE2(sc, port, URWMASK(x) | UHCI_PORTSC_PE);
2480         }
2481
2482         DPRINTFN(2, "uhci port %d reset timed out\n", index);
2483         return (USB_ERR_TIMEOUT);
2484
2485 done:
2486         DPRINTFN(4, "uhci port %d reset, status2 = 0x%04x\n",
2487             index, UREAD2(sc, port));
2488
2489         sc->sc_isreset = 1;
2490         return (USB_ERR_NORMAL_COMPLETION);
2491 }
2492
2493 static usb_error_t
2494 uhci_roothub_exec(struct usb_device *udev,
2495     struct usb_device_request *req, const void **pptr, uint16_t *plength)
2496 {
2497         uhci_softc_t *sc = UHCI_BUS2SC(udev->bus);
2498         const void *ptr;
2499         const char *str_ptr;
2500         uint16_t x;
2501         uint16_t port;
2502         uint16_t value;
2503         uint16_t index;
2504         uint16_t status;
2505         uint16_t change;
2506         uint16_t len;
2507         usb_error_t err;
2508
2509         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
2510
2511         /* buffer reset */
2512         ptr = (const void *)&sc->sc_hub_desc.temp;
2513         len = 0;
2514         err = 0;
2515
2516         value = UGETW(req->wValue);
2517         index = UGETW(req->wIndex);
2518
2519         DPRINTFN(3, "type=0x%02x request=0x%02x wLen=0x%04x "
2520             "wValue=0x%04x wIndex=0x%04x\n",
2521             req->bmRequestType, req->bRequest,
2522             UGETW(req->wLength), value, index);
2523
2524 #define C(x,y) ((x) | ((y) << 8))
2525         switch (C(req->bRequest, req->bmRequestType)) {
2526         case C(UR_CLEAR_FEATURE, UT_WRITE_DEVICE):
2527         case C(UR_CLEAR_FEATURE, UT_WRITE_INTERFACE):
2528         case C(UR_CLEAR_FEATURE, UT_WRITE_ENDPOINT):
2529                 /*
2530                  * DEVICE_REMOTE_WAKEUP and ENDPOINT_HALT are no-ops
2531                  * for the integrated root hub.
2532                  */
2533                 break;
2534         case C(UR_GET_CONFIG, UT_READ_DEVICE):
2535                 len = 1;
2536                 sc->sc_hub_desc.temp[0] = sc->sc_conf;
2537                 break;
2538         case C(UR_GET_DESCRIPTOR, UT_READ_DEVICE):
2539                 switch (value >> 8) {
2540                 case UDESC_DEVICE:
2541                         if ((value & 0xff) != 0) {
2542                                 err = USB_ERR_IOERROR;
2543                                 goto done;
2544                         }
2545                         len = sizeof(uhci_devd);
2546                         ptr = (const void *)&uhci_devd;
2547                         break;
2548
2549                 case UDESC_CONFIG:
2550                         if ((value & 0xff) != 0) {
2551                                 err = USB_ERR_IOERROR;
2552                                 goto done;
2553                         }
2554                         len = sizeof(uhci_confd);
2555                         ptr = (const void *)&uhci_confd;
2556                         break;
2557
2558                 case UDESC_STRING:
2559                         switch (value & 0xff) {
2560                         case 0: /* Language table */
2561                                 str_ptr = "\001";
2562                                 break;
2563
2564                         case 1: /* Vendor */
2565                                 str_ptr = sc->sc_vendor;
2566                                 break;
2567
2568                         case 2: /* Product */
2569                                 str_ptr = "UHCI root HUB";
2570                                 break;
2571
2572                         default:
2573                                 str_ptr = "";
2574                                 break;
2575                         }
2576
2577                         len = usb_make_str_desc
2578                             (sc->sc_hub_desc.temp,
2579                             sizeof(sc->sc_hub_desc.temp),
2580                             str_ptr);
2581                         break;
2582
2583                 default:
2584                         err = USB_ERR_IOERROR;
2585                         goto done;
2586                 }
2587                 break;
2588         case C(UR_GET_INTERFACE, UT_READ_INTERFACE):
2589                 len = 1;
2590                 sc->sc_hub_desc.temp[0] = 0;
2591                 break;
2592         case C(UR_GET_STATUS, UT_READ_DEVICE):
2593                 len = 2;
2594                 USETW(sc->sc_hub_desc.stat.wStatus, UDS_SELF_POWERED);
2595                 break;
2596         case C(UR_GET_STATUS, UT_READ_INTERFACE):
2597         case C(UR_GET_STATUS, UT_READ_ENDPOINT):
2598                 len = 2;
2599                 USETW(sc->sc_hub_desc.stat.wStatus, 0);
2600                 break;
2601         case C(UR_SET_ADDRESS, UT_WRITE_DEVICE):
2602                 if (value >= UHCI_MAX_DEVICES) {
2603                         err = USB_ERR_IOERROR;
2604                         goto done;
2605                 }
2606                 sc->sc_addr = value;
2607                 break;
2608         case C(UR_SET_CONFIG, UT_WRITE_DEVICE):
2609                 if ((value != 0) && (value != 1)) {
2610                         err = USB_ERR_IOERROR;
2611                         goto done;
2612                 }
2613                 sc->sc_conf = value;
2614                 break;
2615         case C(UR_SET_DESCRIPTOR, UT_WRITE_DEVICE):
2616                 break;
2617         case C(UR_SET_FEATURE, UT_WRITE_DEVICE):
2618         case C(UR_SET_FEATURE, UT_WRITE_INTERFACE):
2619         case C(UR_SET_FEATURE, UT_WRITE_ENDPOINT):
2620                 err = USB_ERR_IOERROR;
2621                 goto done;
2622         case C(UR_SET_INTERFACE, UT_WRITE_INTERFACE):
2623                 break;
2624         case C(UR_SYNCH_FRAME, UT_WRITE_ENDPOINT):
2625                 break;
2626                 /* Hub requests */
2627         case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_DEVICE):
2628                 break;
2629         case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_OTHER):
2630                 DPRINTFN(4, "UR_CLEAR_PORT_FEATURE "
2631                     "port=%d feature=%d\n",
2632                     index, value);
2633                 if (index == 1)
2634                         port = UHCI_PORTSC1;
2635                 else if (index == 2)
2636                         port = UHCI_PORTSC2;
2637                 else {
2638                         err = USB_ERR_IOERROR;
2639                         goto done;
2640                 }
2641                 switch (value) {
2642                 case UHF_PORT_ENABLE:
2643                         x = URWMASK(UREAD2(sc, port));
2644                         UWRITE2(sc, port, x & ~UHCI_PORTSC_PE);
2645                         break;
2646                 case UHF_PORT_SUSPEND:
2647                         x = URWMASK(UREAD2(sc, port));
2648                         UWRITE2(sc, port, x & ~(UHCI_PORTSC_SUSP));
2649                         break;
2650                 case UHF_PORT_RESET:
2651                         x = URWMASK(UREAD2(sc, port));
2652                         UWRITE2(sc, port, x & ~UHCI_PORTSC_PR);
2653                         break;
2654                 case UHF_C_PORT_CONNECTION:
2655                         x = URWMASK(UREAD2(sc, port));
2656                         UWRITE2(sc, port, x | UHCI_PORTSC_CSC);
2657                         break;
2658                 case UHF_C_PORT_ENABLE:
2659                         x = URWMASK(UREAD2(sc, port));
2660                         UWRITE2(sc, port, x | UHCI_PORTSC_POEDC);
2661                         break;
2662                 case UHF_C_PORT_OVER_CURRENT:
2663                         x = URWMASK(UREAD2(sc, port));
2664                         UWRITE2(sc, port, x | UHCI_PORTSC_OCIC);
2665                         break;
2666                 case UHF_C_PORT_RESET:
2667                         sc->sc_isreset = 0;
2668                         err = USB_ERR_NORMAL_COMPLETION;
2669                         goto done;
2670                 case UHF_C_PORT_SUSPEND:
2671                         sc->sc_isresumed &= ~(1 << index);
2672                         break;
2673                 case UHF_PORT_CONNECTION:
2674                 case UHF_PORT_OVER_CURRENT:
2675                 case UHF_PORT_POWER:
2676                 case UHF_PORT_LOW_SPEED:
2677                 default:
2678                         err = USB_ERR_IOERROR;
2679                         goto done;
2680                 }
2681                 break;
2682         case C(UR_GET_BUS_STATE, UT_READ_CLASS_OTHER):
2683                 if (index == 1)
2684                         port = UHCI_PORTSC1;
2685                 else if (index == 2)
2686                         port = UHCI_PORTSC2;
2687                 else {
2688                         err = USB_ERR_IOERROR;
2689                         goto done;
2690                 }
2691                 len = 1;
2692                 sc->sc_hub_desc.temp[0] =
2693                     ((UREAD2(sc, port) & UHCI_PORTSC_LS) >>
2694                     UHCI_PORTSC_LS_SHIFT);
2695                 break;
2696         case C(UR_GET_DESCRIPTOR, UT_READ_CLASS_DEVICE):
2697                 if ((value & 0xff) != 0) {
2698                         err = USB_ERR_IOERROR;
2699                         goto done;
2700                 }
2701                 len = sizeof(uhci_hubd_piix);
2702                 ptr = (const void *)&uhci_hubd_piix;
2703                 break;
2704         case C(UR_GET_STATUS, UT_READ_CLASS_DEVICE):
2705                 len = 16;
2706                 bzero(sc->sc_hub_desc.temp, 16);
2707                 break;
2708         case C(UR_GET_STATUS, UT_READ_CLASS_OTHER):
2709                 if (index == 1)
2710                         port = UHCI_PORTSC1;
2711                 else if (index == 2)
2712                         port = UHCI_PORTSC2;
2713                 else {
2714                         err = USB_ERR_IOERROR;
2715                         goto done;
2716                 }
2717                 x = UREAD2(sc, port);
2718                 status = change = 0;
2719                 if (x & UHCI_PORTSC_CCS)
2720                         status |= UPS_CURRENT_CONNECT_STATUS;
2721                 if (x & UHCI_PORTSC_CSC)
2722                         change |= UPS_C_CONNECT_STATUS;
2723                 if (x & UHCI_PORTSC_PE)
2724                         status |= UPS_PORT_ENABLED;
2725                 if (x & UHCI_PORTSC_POEDC)
2726                         change |= UPS_C_PORT_ENABLED;
2727                 if (x & UHCI_PORTSC_OCI)
2728                         status |= UPS_OVERCURRENT_INDICATOR;
2729                 if (x & UHCI_PORTSC_OCIC)
2730                         change |= UPS_C_OVERCURRENT_INDICATOR;
2731                 if (x & UHCI_PORTSC_LSDA)
2732                         status |= UPS_LOW_SPEED;
2733                 if ((x & UHCI_PORTSC_PE) && (x & UHCI_PORTSC_RD)) {
2734                         /* need to do a write back */
2735                         UWRITE2(sc, port, URWMASK(x));
2736
2737                         /* wait 20ms for resume sequence to complete */
2738                         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 50);
2739
2740                         /* clear suspend and resume detect */
2741                         UWRITE2(sc, port, URWMASK(x) & ~(UHCI_PORTSC_RD |
2742                             UHCI_PORTSC_SUSP));
2743
2744                         /* wait a little bit */
2745                         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 500);
2746
2747                         sc->sc_isresumed |= (1 << index);
2748
2749                 } else if (x & UHCI_PORTSC_SUSP) {
2750                         status |= UPS_SUSPEND;
2751                 }
2752                 status |= UPS_PORT_POWER;
2753                 if (sc->sc_isresumed & (1 << index))
2754                         change |= UPS_C_SUSPEND;
2755                 if (sc->sc_isreset)
2756                         change |= UPS_C_PORT_RESET;
2757                 USETW(sc->sc_hub_desc.ps.wPortStatus, status);
2758                 USETW(sc->sc_hub_desc.ps.wPortChange, change);
2759                 len = sizeof(sc->sc_hub_desc.ps);
2760                 break;
2761         case C(UR_SET_DESCRIPTOR, UT_WRITE_CLASS_DEVICE):
2762                 err = USB_ERR_IOERROR;
2763                 goto done;
2764         case C(UR_SET_FEATURE, UT_WRITE_CLASS_DEVICE):
2765                 break;
2766         case C(UR_SET_FEATURE, UT_WRITE_CLASS_OTHER):
2767                 if (index == 1)
2768                         port = UHCI_PORTSC1;
2769                 else if (index == 2)
2770                         port = UHCI_PORTSC2;
2771                 else {
2772                         err = USB_ERR_IOERROR;
2773                         goto done;
2774                 }
2775                 switch (value) {
2776                 case UHF_PORT_ENABLE:
2777                         x = URWMASK(UREAD2(sc, port));
2778                         UWRITE2(sc, port, x | UHCI_PORTSC_PE);
2779                         break;
2780                 case UHF_PORT_SUSPEND:
2781                         x = URWMASK(UREAD2(sc, port));
2782                         UWRITE2(sc, port, x | UHCI_PORTSC_SUSP);
2783                         break;
2784                 case UHF_PORT_RESET:
2785                         err = uhci_portreset(sc, index);
2786                         goto done;
2787                 case UHF_PORT_POWER:
2788                         /* pretend we turned on power */
2789                         err = USB_ERR_NORMAL_COMPLETION;
2790                         goto done;
2791                 case UHF_C_PORT_CONNECTION:
2792                 case UHF_C_PORT_ENABLE:
2793                 case UHF_C_PORT_OVER_CURRENT:
2794                 case UHF_PORT_CONNECTION:
2795                 case UHF_PORT_OVER_CURRENT:
2796                 case UHF_PORT_LOW_SPEED:
2797                 case UHF_C_PORT_SUSPEND:
2798                 case UHF_C_PORT_RESET:
2799                 default:
2800                         err = USB_ERR_IOERROR;
2801                         goto done;
2802                 }
2803                 break;
2804         default:
2805                 err = USB_ERR_IOERROR;
2806                 goto done;
2807         }
2808 done:
2809         *plength = len;
2810         *pptr = ptr;
2811         return (err);
2812 }
2813
2814 /*
2815  * This routine is executed periodically and simulates interrupts from
2816  * the root controller interrupt pipe for port status change:
2817  */
2818 static void
2819 uhci_root_intr(uhci_softc_t *sc)
2820 {
2821         DPRINTFN(21, "\n");
2822
2823         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
2824
2825         sc->sc_hub_idata[0] = 0;
2826
2827         if (UREAD2(sc, UHCI_PORTSC1) & (UHCI_PORTSC_CSC |
2828             UHCI_PORTSC_OCIC | UHCI_PORTSC_RD)) {
2829                 sc->sc_hub_idata[0] |= 1 << 1;
2830         }
2831         if (UREAD2(sc, UHCI_PORTSC2) & (UHCI_PORTSC_CSC |
2832             UHCI_PORTSC_OCIC | UHCI_PORTSC_RD)) {
2833                 sc->sc_hub_idata[0] |= 1 << 2;
2834         }
2835
2836         /* restart timer */
2837         usb_callout_reset(&sc->sc_root_intr, hz,
2838             (void *)&uhci_root_intr, sc);
2839
2840         if (sc->sc_hub_idata[0] != 0) {
2841                 uhub_root_intr(&sc->sc_bus, sc->sc_hub_idata,
2842                     sizeof(sc->sc_hub_idata));
2843         }
2844 }
2845
2846 static void
2847 uhci_xfer_setup(struct usb_setup_params *parm)
2848 {
2849         struct usb_page_search page_info;
2850         struct usb_page_cache *pc;
2851         uhci_softc_t *sc;
2852         struct usb_xfer *xfer;
2853         void *last_obj;
2854         uint32_t ntd;
2855         uint32_t nqh;
2856         uint32_t nfixup;
2857         uint32_t n;
2858         uint16_t align;
2859
2860         sc = UHCI_BUS2SC(parm->udev->bus);
2861         xfer = parm->curr_xfer;
2862
2863         parm->hc_max_packet_size = 0x500;
2864         parm->hc_max_packet_count = 1;
2865         parm->hc_max_frame_size = 0x500;
2866
2867         /*
2868          * compute ntd and nqh
2869          */
2870         if (parm->methods == &uhci_device_ctrl_methods) {
2871                 xfer->flags_int.bdma_enable = 1;
2872                 xfer->flags_int.bdma_no_post_sync = 1;
2873
2874                 usbd_transfer_setup_sub(parm);
2875
2876                 /* see EHCI HC driver for proof of "ntd" formula */
2877
2878                 nqh = 1;
2879                 ntd = ((2 * xfer->nframes) + 1  /* STATUS */
2880                     + (xfer->max_data_length / xfer->max_frame_size));
2881
2882         } else if (parm->methods == &uhci_device_bulk_methods) {
2883                 xfer->flags_int.bdma_enable = 1;
2884                 xfer->flags_int.bdma_no_post_sync = 1;
2885
2886                 usbd_transfer_setup_sub(parm);
2887
2888                 nqh = 1;
2889                 ntd = ((2 * xfer->nframes)
2890                     + (xfer->max_data_length / xfer->max_frame_size));
2891
2892         } else if (parm->methods == &uhci_device_intr_methods) {
2893                 xfer->flags_int.bdma_enable = 1;
2894                 xfer->flags_int.bdma_no_post_sync = 1;
2895
2896                 usbd_transfer_setup_sub(parm);
2897
2898                 nqh = 1;
2899                 ntd = ((2 * xfer->nframes)
2900                     + (xfer->max_data_length / xfer->max_frame_size));
2901
2902         } else if (parm->methods == &uhci_device_isoc_methods) {
2903                 xfer->flags_int.bdma_enable = 1;
2904                 xfer->flags_int.bdma_no_post_sync = 1;
2905
2906                 usbd_transfer_setup_sub(parm);
2907
2908                 nqh = 0;
2909                 ntd = xfer->nframes;
2910
2911         } else {
2912
2913                 usbd_transfer_setup_sub(parm);
2914
2915                 nqh = 0;
2916                 ntd = 0;
2917         }
2918
2919         if (parm->err) {
2920                 return;
2921         }
2922         /*
2923          * NOTE: the UHCI controller requires that
2924          * every packet must be contiguous on
2925          * the same USB memory page !
2926          */
2927         nfixup = (parm->bufsize / USB_PAGE_SIZE) + 1;
2928
2929         /*
2930          * Compute a suitable power of two alignment
2931          * for our "max_frame_size" fixup buffer(s):
2932          */
2933         align = xfer->max_frame_size;
2934         n = 0;
2935         while (align) {
2936                 align >>= 1;
2937                 n++;
2938         }
2939
2940         /* check for power of two */
2941         if (!(xfer->max_frame_size &
2942             (xfer->max_frame_size - 1))) {
2943                 n--;
2944         }
2945         /*
2946          * We don't allow alignments of
2947          * less than 8 bytes:
2948          *
2949          * NOTE: Allocating using an aligment
2950          * of 1 byte has special meaning!
2951          */
2952         if (n < 3) {
2953                 n = 3;
2954         }
2955         align = (1 << n);
2956
2957         if (usbd_transfer_setup_sub_malloc(
2958             parm, &pc, xfer->max_frame_size,
2959             align, nfixup)) {
2960                 parm->err = USB_ERR_NOMEM;
2961                 return;
2962         }
2963         xfer->buf_fixup = pc;
2964
2965 alloc_dma_set:
2966
2967         if (parm->err) {
2968                 return;
2969         }
2970         last_obj = NULL;
2971
2972         if (usbd_transfer_setup_sub_malloc(
2973             parm, &pc, sizeof(uhci_td_t),
2974             UHCI_TD_ALIGN, ntd)) {
2975                 parm->err = USB_ERR_NOMEM;
2976                 return;
2977         }
2978         if (parm->buf) {
2979                 for (n = 0; n != ntd; n++) {
2980                         uhci_td_t *td;
2981
2982                         usbd_get_page(pc + n, 0, &page_info);
2983
2984                         td = page_info.buffer;
2985
2986                         /* init TD */
2987                         if ((parm->methods == &uhci_device_bulk_methods) ||
2988                             (parm->methods == &uhci_device_ctrl_methods) ||
2989                             (parm->methods == &uhci_device_intr_methods)) {
2990                                 /* set depth first bit */
2991                                 td->td_self = htole32(page_info.physaddr |
2992                                     UHCI_PTR_TD | UHCI_PTR_VF);
2993                         } else {
2994                                 td->td_self = htole32(page_info.physaddr |
2995                                     UHCI_PTR_TD);
2996                         }
2997
2998                         td->obj_next = last_obj;
2999                         td->page_cache = pc + n;
3000
3001                         last_obj = td;
3002
3003                         usb_pc_cpu_flush(pc + n);
3004                 }
3005         }
3006         xfer->td_start[xfer->flags_int.curr_dma_set] = last_obj;
3007
3008         last_obj = NULL;
3009
3010         if (usbd_transfer_setup_sub_malloc(
3011             parm, &pc, sizeof(uhci_qh_t),
3012             UHCI_QH_ALIGN, nqh)) {
3013                 parm->err = USB_ERR_NOMEM;
3014                 return;
3015         }
3016         if (parm->buf) {
3017                 for (n = 0; n != nqh; n++) {
3018                         uhci_qh_t *qh;
3019
3020                         usbd_get_page(pc + n, 0, &page_info);
3021
3022                         qh = page_info.buffer;
3023
3024                         /* init QH */
3025                         qh->qh_self = htole32(page_info.physaddr | UHCI_PTR_QH);
3026                         qh->obj_next = last_obj;
3027                         qh->page_cache = pc + n;
3028
3029                         last_obj = qh;
3030
3031                         usb_pc_cpu_flush(pc + n);
3032                 }
3033         }
3034         xfer->qh_start[xfer->flags_int.curr_dma_set] = last_obj;
3035
3036         if (!xfer->flags_int.curr_dma_set) {
3037                 xfer->flags_int.curr_dma_set = 1;
3038                 goto alloc_dma_set;
3039         }
3040 }
3041
3042 static void
3043 uhci_ep_init(struct usb_device *udev, struct usb_endpoint_descriptor *edesc,
3044     struct usb_endpoint *ep)
3045 {
3046         uhci_softc_t *sc = UHCI_BUS2SC(udev->bus);
3047
3048         DPRINTFN(2, "endpoint=%p, addr=%d, endpt=%d, mode=%d (%d)\n",
3049             ep, udev->address,
3050             edesc->bEndpointAddress, udev->flags.usb_mode,
3051             sc->sc_addr);
3052
3053         if (udev->flags.usb_mode != USB_MODE_HOST) {
3054                 /* not supported */
3055                 return;
3056         }
3057         if (udev->device_index != sc->sc_addr) {
3058                 switch (edesc->bmAttributes & UE_XFERTYPE) {
3059                 case UE_CONTROL:
3060                         ep->methods = &uhci_device_ctrl_methods;
3061                         break;
3062                 case UE_INTERRUPT:
3063                         ep->methods = &uhci_device_intr_methods;
3064                         break;
3065                 case UE_ISOCHRONOUS:
3066                         if (udev->speed == USB_SPEED_FULL) {
3067                                 ep->methods = &uhci_device_isoc_methods;
3068                         }
3069                         break;
3070                 case UE_BULK:
3071                         if (udev->speed != USB_SPEED_LOW) {
3072                                 ep->methods = &uhci_device_bulk_methods;
3073                         }
3074                         break;
3075                 default:
3076                         /* do nothing */
3077                         break;
3078                 }
3079         }
3080 }
3081
3082 static void
3083 uhci_xfer_unsetup(struct usb_xfer *xfer)
3084 {
3085         return;
3086 }
3087
3088 static void
3089 uhci_get_dma_delay(struct usb_bus *bus, uint32_t *pus)
3090 {
3091         /*
3092          * Wait until hardware has finished any possible use of the
3093          * transfer descriptor(s) and QH
3094          */
3095         *pus = (1125);                  /* microseconds */
3096 }
3097
3098 static void
3099 uhci_device_resume(struct usb_device *udev)
3100 {
3101         struct uhci_softc *sc = UHCI_BUS2SC(udev->bus);
3102         struct usb_xfer *xfer;
3103         struct usb_pipe_methods *methods;
3104         uhci_qh_t *qh;
3105
3106         DPRINTF("\n");
3107
3108         USB_BUS_LOCK(udev->bus);
3109
3110         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
3111
3112                 if (xfer->xroot->udev == udev) {
3113
3114                         methods = xfer->endpoint->methods;
3115                         qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
3116
3117                         if (methods == &uhci_device_bulk_methods) {
3118                                 UHCI_APPEND_QH(qh, sc->sc_bulk_p_last);
3119                                 uhci_add_loop(sc);
3120                                 xfer->flags_int.bandwidth_reclaimed = 1;
3121                         }
3122                         if (methods == &uhci_device_ctrl_methods) {
3123                                 if (xfer->xroot->udev->speed == USB_SPEED_LOW) {
3124                                         UHCI_APPEND_QH(qh, sc->sc_ls_ctl_p_last);
3125                                 } else {
3126                                         UHCI_APPEND_QH(qh, sc->sc_fs_ctl_p_last);
3127                                 }
3128                         }
3129                         if (methods == &uhci_device_intr_methods) {
3130                                 UHCI_APPEND_QH(qh, sc->sc_intr_p_last[xfer->qh_pos]);
3131                         }
3132                 }
3133         }
3134
3135         USB_BUS_UNLOCK(udev->bus);
3136
3137         return;
3138 }
3139
3140 static void
3141 uhci_device_suspend(struct usb_device *udev)
3142 {
3143         struct uhci_softc *sc = UHCI_BUS2SC(udev->bus);
3144         struct usb_xfer *xfer;
3145         struct usb_pipe_methods *methods;
3146         uhci_qh_t *qh;
3147
3148         DPRINTF("\n");
3149
3150         USB_BUS_LOCK(udev->bus);
3151
3152         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
3153
3154                 if (xfer->xroot->udev == udev) {
3155
3156                         methods = xfer->endpoint->methods;
3157                         qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
3158
3159                         if (xfer->flags_int.bandwidth_reclaimed) {
3160                                 xfer->flags_int.bandwidth_reclaimed = 0;
3161                                 uhci_rem_loop(sc);
3162                         }
3163                         if (methods == &uhci_device_bulk_methods) {
3164                                 UHCI_REMOVE_QH(qh, sc->sc_bulk_p_last);
3165                         }
3166                         if (methods == &uhci_device_ctrl_methods) {
3167                                 if (xfer->xroot->udev->speed == USB_SPEED_LOW) {
3168                                         UHCI_REMOVE_QH(qh, sc->sc_ls_ctl_p_last);
3169                                 } else {
3170                                         UHCI_REMOVE_QH(qh, sc->sc_fs_ctl_p_last);
3171                                 }
3172                         }
3173                         if (methods == &uhci_device_intr_methods) {
3174                                 UHCI_REMOVE_QH(qh, sc->sc_intr_p_last[xfer->qh_pos]);
3175                         }
3176                 }
3177         }
3178
3179         USB_BUS_UNLOCK(udev->bus);
3180
3181         return;
3182 }
3183
3184 static void
3185 uhci_set_hw_power(struct usb_bus *bus)
3186 {
3187         struct uhci_softc *sc = UHCI_BUS2SC(bus);
3188         uint32_t flags;
3189
3190         DPRINTF("\n");
3191
3192         USB_BUS_LOCK(bus);
3193
3194         flags = bus->hw_power_state;
3195
3196         /*
3197          * WARNING: Some FULL speed USB devices require periodic SOF
3198          * messages! If any USB devices are connected through the
3199          * UHCI, power save will be disabled!
3200          */
3201         if (flags & (USB_HW_POWER_CONTROL |
3202             USB_HW_POWER_NON_ROOT_HUB |
3203             USB_HW_POWER_BULK |
3204             USB_HW_POWER_INTERRUPT |
3205             USB_HW_POWER_ISOC)) {
3206                 DPRINTF("Some USB transfer is "
3207                     "active on unit %u.\n",
3208                     device_get_unit(sc->sc_bus.bdev));
3209                 uhci_restart(sc);
3210         } else {
3211                 DPRINTF("Power save on unit %u.\n",
3212                     device_get_unit(sc->sc_bus.bdev));
3213                 UHCICMD(sc, UHCI_CMD_MAXP);
3214         }
3215
3216         USB_BUS_UNLOCK(bus);
3217
3218         return;
3219 }
3220
3221
3222 struct usb_bus_methods uhci_bus_methods =
3223 {
3224         .endpoint_init = uhci_ep_init,
3225         .xfer_setup = uhci_xfer_setup,
3226         .xfer_unsetup = uhci_xfer_unsetup,
3227         .get_dma_delay = uhci_get_dma_delay,
3228         .device_resume = uhci_device_resume,
3229         .device_suspend = uhci_device_suspend,
3230         .set_hw_power = uhci_set_hw_power,
3231         .roothub_exec = uhci_roothub_exec,
3232         .xfer_poll = uhci_do_poll,
3233 };