sys/dev/usb/controller/uhci.c

   1 /*-
   2  * Copyright (c) 2008 Hans Petter Selasky. All rights reserved.
   3  * Copyright (c) 1998 The NetBSD Foundation, Inc. All rights reserved.
   4  * Copyright (c) 1998 Lennart Augustsson. All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  */
  27
  28 #include <sys/cdefs.h>
  29 __FBSDID("$FreeBSD$");
  30
  31 /*
  32  * USB Universal Host Controller driver.
  33  * Handles e.g. PIIX3 and PIIX4.
  34  *
  35  * UHCI spec: http://developer.intel.com/design/USB/UHCI11D.htm
  36  * USB spec:  http://www.usb.org/developers/docs/usbspec.zip
  37  * PIIXn spec: ftp://download.intel.com/design/intarch/datashts/29055002.pdf
  38  *             ftp://download.intel.com/design/intarch/datashts/29056201.pdf
  39  */
  40
  41 #include <sys/stdint.h>
  42 #include <sys/stddef.h>
  43 #include <sys/param.h>
  44 #include <sys/queue.h>
  45 #include <sys/types.h>
  46 #include <sys/systm.h>
  47 #include <sys/kernel.h>
  48 #include <sys/bus.h>
  49 #include <sys/module.h>
  50 #include <sys/lock.h>
  51 #include <sys/mutex.h>
  52 #include <sys/condvar.h>
  53 #include <sys/sysctl.h>
  54 #include <sys/sx.h>
  55 #include <sys/unistd.h>
  56 #include <sys/callout.h>
  57 #include <sys/malloc.h>
  58 #include <sys/priv.h>
  59
  60 #include <dev/usb/usb.h>
  61 #include <dev/usb/usbdi.h>
  62
  63 #define USB_DEBUG_VAR uhcidebug
  64
  65 #include <dev/usb/usb_core.h>
  66 #include <dev/usb/usb_debug.h>
  67 #include <dev/usb/usb_busdma.h>
  68 #include <dev/usb/usb_process.h>
  69 #include <dev/usb/usb_transfer.h>
  70 #include <dev/usb/usb_device.h>
  71 #include <dev/usb/usb_hub.h>
  72 #include <dev/usb/usb_util.h>
  73
  74 #include <dev/usb/usb_controller.h>
  75 #include <dev/usb/usb_bus.h>
  76 #include <dev/usb/controller/uhci.h>
  77 #include <dev/usb/controller/uhcireg.h>
  78
  79 #define alt_next next
  80 #define UHCI_BUS2SC(bus) \
  81    ((uhci_softc_t *)(((uint8_t *)(bus)) - \
  82     ((uint8_t *)&(((uhci_softc_t *)0)->sc_bus))))
  83
  84 #ifdef USB_DEBUG
  85 static int uhcidebug = 0;
  86 static int uhcinoloop = 0;
  87
  88 SYSCTL_NODE(_hw_usb, OID_AUTO, uhci, CTLFLAG_RW, 0, "USB uhci");
  89 SYSCTL_INT(_hw_usb_uhci, OID_AUTO, debug, CTLFLAG_RW,
  90     &uhcidebug, 0, "uhci debug level");
  91 SYSCTL_INT(_hw_usb_uhci, OID_AUTO, loop, CTLFLAG_RW,
  92     &uhcinoloop, 0, "uhci noloop");
  93
  94 TUNABLE_INT("hw.usb.uhci.debug", &uhcidebug);
  95 TUNABLE_INT("hw.usb.uhci.loop", &uhcinoloop);
  96
  97 static void uhci_dumpregs(uhci_softc_t *sc);
  98 static void uhci_dump_tds(uhci_td_t *td);
  99
 100 #endif
 101
 102 #define UBARR(sc) bus_space_barrier((sc)->sc_io_tag, (sc)->sc_io_hdl, 0, (sc)->sc_io_size, \
 103                         BUS_SPACE_BARRIER_READ|BUS_SPACE_BARRIER_WRITE)
 104 #define UWRITE1(sc, r, x) \
 105  do { UBARR(sc); bus_space_write_1((sc)->sc_io_tag, (sc)->sc_io_hdl, (r), (x)); \
 106  } while (/*CONSTCOND*/0)
 107 #define UWRITE2(sc, r, x) \
 108  do { UBARR(sc); bus_space_write_2((sc)->sc_io_tag, (sc)->sc_io_hdl, (r), (x)); \
 109  } while (/*CONSTCOND*/0)
 110 #define UWRITE4(sc, r, x) \
 111  do { UBARR(sc); bus_space_write_4((sc)->sc_io_tag, (sc)->sc_io_hdl, (r), (x)); \
 112  } while (/*CONSTCOND*/0)
 113 #define UREAD1(sc, r) (UBARR(sc), bus_space_read_1((sc)->sc_io_tag, (sc)->sc_io_hdl, (r)))
 114 #define UREAD2(sc, r) (UBARR(sc), bus_space_read_2((sc)->sc_io_tag, (sc)->sc_io_hdl, (r)))
 115 #define UREAD4(sc, r) (UBARR(sc), bus_space_read_4((sc)->sc_io_tag, (sc)->sc_io_hdl, (r)))
 116
 117 #define UHCICMD(sc, cmd) UWRITE2(sc, UHCI_CMD, cmd)
 118 #define UHCISTS(sc) UREAD2(sc, UHCI_STS)
 119
 120 #define UHCI_RESET_TIMEOUT 100          /* ms, reset timeout */
 121
 122 #define UHCI_INTR_ENDPT 1
 123
 124 struct uhci_mem_layout {
 125
 126         struct usb_page_search buf_res;
 127         struct usb_page_search fix_res;
 128
 129         struct usb_page_cache *buf_pc;
 130         struct usb_page_cache *fix_pc;
 131
 132         uint32_t buf_offset;
 133
 134         uint16_t max_frame_size;
 135 };
 136
 137 struct uhci_std_temp {
 138
 139         struct uhci_mem_layout ml;
 140         uhci_td_t *td;
 141         uhci_td_t *td_next;
 142         uint32_t average;
 143         uint32_t td_status;
 144         uint32_t td_token;
 145         uint32_t len;
 146         uint16_t max_frame_size;
 147         uint8_t shortpkt;
 148         uint8_t setup_alt_next;
 149         uint8_t last_frame;
 150 };
 151
 152 extern struct usb_bus_methods uhci_bus_methods;
 153 extern struct usb_pipe_methods uhci_device_bulk_methods;
 154 extern struct usb_pipe_methods uhci_device_ctrl_methods;
 155 extern struct usb_pipe_methods uhci_device_intr_methods;
 156 extern struct usb_pipe_methods uhci_device_isoc_methods;
 157
 158 static uint8_t  uhci_restart(uhci_softc_t *sc);
 159 static void     uhci_do_poll(struct usb_bus *);
 160 static void     uhci_device_done(struct usb_xfer *, usb_error_t);
 161 static void     uhci_transfer_intr_enqueue(struct usb_xfer *);
 162 static void     uhci_timeout(void *);
 163 static uint8_t  uhci_check_transfer(struct usb_xfer *);
 164 static void     uhci_root_intr(uhci_softc_t *sc);
 165
 166 void
 167 uhci_iterate_hw_softc(struct usb_bus *bus, usb_bus_mem_sub_cb_t *cb)
 168 {
 169         struct uhci_softc *sc = UHCI_BUS2SC(bus);
 170         uint32_t i;
 171
 172         cb(bus, &sc->sc_hw.pframes_pc, &sc->sc_hw.pframes_pg,
 173             sizeof(uint32_t) * UHCI_FRAMELIST_COUNT, UHCI_FRAMELIST_ALIGN);
 174
 175         cb(bus, &sc->sc_hw.ls_ctl_start_pc, &sc->sc_hw.ls_ctl_start_pg,
 176             sizeof(uhci_qh_t), UHCI_QH_ALIGN);
 177
 178         cb(bus, &sc->sc_hw.fs_ctl_start_pc, &sc->sc_hw.fs_ctl_start_pg,
 179             sizeof(uhci_qh_t), UHCI_QH_ALIGN);
 180
 181         cb(bus, &sc->sc_hw.bulk_start_pc, &sc->sc_hw.bulk_start_pg,
 182             sizeof(uhci_qh_t), UHCI_QH_ALIGN);
 183
 184         cb(bus, &sc->sc_hw.last_qh_pc, &sc->sc_hw.last_qh_pg,
 185             sizeof(uhci_qh_t), UHCI_QH_ALIGN);
 186
 187         cb(bus, &sc->sc_hw.last_td_pc, &sc->sc_hw.last_td_pg,
 188             sizeof(uhci_td_t), UHCI_TD_ALIGN);
 189
 190         for (i = 0; i != UHCI_VFRAMELIST_COUNT; i++) {
 191                 cb(bus, sc->sc_hw.isoc_start_pc + i,
 192                     sc->sc_hw.isoc_start_pg + i,
 193                     sizeof(uhci_td_t), UHCI_TD_ALIGN);
 194         }
 195
 196         for (i = 0; i != UHCI_IFRAMELIST_COUNT; i++) {
 197                 cb(bus, sc->sc_hw.intr_start_pc + i,
 198                     sc->sc_hw.intr_start_pg + i,
 199                     sizeof(uhci_qh_t), UHCI_QH_ALIGN);
 200         }
 201 }
 202
 203 static void
 204 uhci_mem_layout_init(struct uhci_mem_layout *ml, struct usb_xfer *xfer)
 205 {
 206         ml->buf_pc = xfer->frbuffers + 0;
 207         ml->fix_pc = xfer->buf_fixup;
 208
 209         ml->buf_offset = 0;
 210
 211         ml->max_frame_size = xfer->max_frame_size;
 212 }
 213
 214 static void
 215 uhci_mem_layout_fixup(struct uhci_mem_layout *ml, struct uhci_td *td)
 216 {
 217         usbd_get_page(ml->buf_pc, ml->buf_offset, &ml->buf_res);
 218
 219         if (ml->buf_res.length < td->len) {
 220
 221                 /* need to do a fixup */
 222
 223                 usbd_get_page(ml->fix_pc, 0, &ml->fix_res);
 224
 225                 td->td_buffer = htole32(ml->fix_res.physaddr);
 226
 227                 /*
 228                  * The UHCI driver cannot handle
 229                  * page crossings, so a fixup is
 230                  * needed:
 231                  *
 232                  *  +----+----+ - - -
 233                  *  | YYY|Y   |
 234                  *  +----+----+ - - -
 235                  *     \    \
 236                  *      \    \
 237                  *       +----+
 238                  *       |YYYY|  (fixup)
 239                  *       +----+
 240                  */
 241
 242                 if ((td->td_token & htole32(UHCI_TD_PID)) ==
 243                     htole32(UHCI_TD_PID_IN)) {
 244                         td->fix_pc = ml->fix_pc;
 245                         usb_pc_cpu_invalidate(ml->fix_pc);
 246
 247                 } else {
 248                         td->fix_pc = NULL;
 249
 250                         /* copy data to fixup location */
 251
 252                         usbd_copy_out(ml->buf_pc, ml->buf_offset,
 253                             ml->fix_res.buffer, td->len);
 254
 255                         usb_pc_cpu_flush(ml->fix_pc);
 256                 }
 257
 258                 /* prepare next fixup */
 259
 260                 ml->fix_pc++;
 261
 262         } else {
 263
 264                 td->td_buffer = htole32(ml->buf_res.physaddr);
 265                 td->fix_pc = NULL;
 266         }
 267
 268         /* prepare next data location */
 269
 270         ml->buf_offset += td->len;
 271 }
 272
 273 /*
 274  * Return values:
 275  * 0: Success
 276  * Else: Failure
 277  */
 278 static uint8_t
 279 uhci_restart(uhci_softc_t *sc)
 280 {
 281         struct usb_page_search buf_res;
 282
 283         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
 284
 285         if (UREAD2(sc, UHCI_CMD) & UHCI_CMD_RS) {
 286                 DPRINTFN(2, "Already started\n");
 287                 return (0);
 288         }
 289
 290         DPRINTFN(2, "Restarting\n");
 291
 292         usbd_get_page(&sc->sc_hw.pframes_pc, 0, &buf_res);
 293
 294         /* Reload fresh base address */
 295         UWRITE4(sc, UHCI_FLBASEADDR, buf_res.physaddr);
 296
 297         /*
 298          * Assume 64 byte packets at frame end and start HC controller:
 299          */
 300         UHCICMD(sc, (UHCI_CMD_MAXP | UHCI_CMD_RS));
 301
 302         /* wait 10 milliseconds */
 303
 304         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 100);
 305
 306         /* check that controller has started */
 307
 308         if (UREAD2(sc, UHCI_STS) & UHCI_STS_HCH) {
 309                 DPRINTFN(2, "Failed\n");
 310                 return (1);
 311         }
 312         return (0);
 313 }
 314
 315 void
 316 uhci_reset(uhci_softc_t *sc)
 317 {
 318         uint16_t n;
 319
 320         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
 321
 322         DPRINTF("resetting the HC\n");
 323
 324         /* disable interrupts */
 325
 326         UWRITE2(sc, UHCI_INTR, 0);
 327
 328         /* global reset */
 329
 330         UHCICMD(sc, UHCI_CMD_GRESET);
 331
 332         /* wait */
 333
 334         usb_pause_mtx(&sc->sc_bus.bus_mtx,
 335             USB_MS_TO_TICKS(USB_BUS_RESET_DELAY));
 336
 337         /* terminate all transfers */
 338
 339         UHCICMD(sc, UHCI_CMD_HCRESET);
 340
 341         /* the reset bit goes low when the controller is done */
 342
 343         n = UHCI_RESET_TIMEOUT;
 344         while (n--) {
 345                 /* wait one millisecond */
 346
 347                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 1000);
 348
 349                 if (!(UREAD2(sc, UHCI_CMD) & UHCI_CMD_HCRESET)) {
 350                         goto done_1;
 351                 }
 352         }
 353
 354         device_printf(sc->sc_bus.bdev,
 355             "controller did not reset\n");
 356
 357 done_1:
 358
 359         n = 10;
 360         while (n--) {
 361                 /* wait one millisecond */
 362
 363                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 1000);
 364
 365                 /* check if HC is stopped */
 366                 if (UREAD2(sc, UHCI_STS) & UHCI_STS_HCH) {
 367                         goto done_2;
 368                 }
 369         }
 370
 371         device_printf(sc->sc_bus.bdev,
 372             "controller did not stop\n");
 373
 374 done_2:
 375
 376         /* reload the configuration */
 377         UWRITE2(sc, UHCI_FRNUM, sc->sc_saved_frnum);
 378         UWRITE1(sc, UHCI_SOF, sc->sc_saved_sof);
 379
 380         USB_BUS_UNLOCK(&sc->sc_bus);
 381
 382         /* stop root interrupt */
 383         usb_callout_drain(&sc->sc_root_intr);
 384
 385         USB_BUS_LOCK(&sc->sc_bus);
 386 }
 387
 388 static void
 389 uhci_start(uhci_softc_t *sc)
 390 {
 391         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
 392
 393         DPRINTFN(2, "enabling\n");
 394
 395         /* enable interrupts */
 396
 397         UWRITE2(sc, UHCI_INTR,
 398             (UHCI_INTR_TOCRCIE |
 399             UHCI_INTR_RIE |
 400             UHCI_INTR_IOCE |
 401             UHCI_INTR_SPIE));
 402
 403         if (uhci_restart(sc)) {
 404                 device_printf(sc->sc_bus.bdev,
 405                     "cannot start HC controller\n");
 406         }
 407
 408         /* start root interrupt */
 409         uhci_root_intr(sc);
 410 }
 411
 412 static struct uhci_qh *
 413 uhci_init_qh(struct usb_page_cache *pc)
 414 {
 415         struct usb_page_search buf_res;
 416         struct uhci_qh *qh;
 417
 418         usbd_get_page(pc, 0, &buf_res);
 419
 420         qh = buf_res.buffer;
 421
 422         qh->qh_self =
 423             htole32(buf_res.physaddr) |
 424             htole32(UHCI_PTR_QH);
 425
 426         qh->page_cache = pc;
 427
 428         return (qh);
 429 }
 430
 431 static struct uhci_td *
 432 uhci_init_td(struct usb_page_cache *pc)
 433 {
 434         struct usb_page_search buf_res;
 435         struct uhci_td *td;
 436
 437         usbd_get_page(pc, 0, &buf_res);
 438
 439         td = buf_res.buffer;
 440
 441         td->td_self =
 442             htole32(buf_res.physaddr) |
 443             htole32(UHCI_PTR_TD);
 444
 445         td->page_cache = pc;
 446
 447         return (td);
 448 }
 449
 450 usb_error_t
 451 uhci_init(uhci_softc_t *sc)
 452 {
 453         uint16_t bit;
 454         uint16_t x;
 455         uint16_t y;
 456
 457         DPRINTF("start\n");
 458
 459         usb_callout_init_mtx(&sc->sc_root_intr, &sc->sc_bus.bus_mtx, 0);
 460
 461 #ifdef USB_DEBUG
 462         if (uhcidebug > 2) {
 463                 uhci_dumpregs(sc);
 464         }
 465 #endif
 466         sc->sc_saved_sof = 0x40;        /* default value */
 467         sc->sc_saved_frnum = 0;         /* default frame number */
 468
 469         /*
 470          * Setup QH's
 471          */
 472         sc->sc_ls_ctl_p_last =
 473             uhci_init_qh(&sc->sc_hw.ls_ctl_start_pc);
 474
 475         sc->sc_fs_ctl_p_last =
 476             uhci_init_qh(&sc->sc_hw.fs_ctl_start_pc);
 477
 478         sc->sc_bulk_p_last =
 479             uhci_init_qh(&sc->sc_hw.bulk_start_pc);
 480 #if 0
 481         sc->sc_reclaim_qh_p =
 482             sc->sc_fs_ctl_p_last;
 483 #else
 484         /* setup reclaim looping point */
 485         sc->sc_reclaim_qh_p =
 486             sc->sc_bulk_p_last;
 487 #endif
 488
 489         sc->sc_last_qh_p =
 490             uhci_init_qh(&sc->sc_hw.last_qh_pc);
 491
 492         sc->sc_last_td_p =
 493             uhci_init_td(&sc->sc_hw.last_td_pc);
 494
 495         for (x = 0; x != UHCI_VFRAMELIST_COUNT; x++) {
 496                 sc->sc_isoc_p_last[x] =
 497                     uhci_init_td(sc->sc_hw.isoc_start_pc + x);
 498         }
 499
 500         for (x = 0; x != UHCI_IFRAMELIST_COUNT; x++) {
 501                 sc->sc_intr_p_last[x] =
 502                     uhci_init_qh(sc->sc_hw.intr_start_pc + x);
 503         }
 504
 505         /*
 506          * the QHs are arranged to give poll intervals that are
 507          * powers of 2 times 1ms
 508          */
 509         bit = UHCI_IFRAMELIST_COUNT / 2;
 510         while (bit) {
 511                 x = bit;
 512                 while (x & bit) {
 513                         uhci_qh_t *qh_x;
 514                         uhci_qh_t *qh_y;
 515
 516                         y = (x ^ bit) | (bit / 2);
 517
 518                         /*
 519                          * the next QH has half the poll interval
 520                          */
 521                         qh_x = sc->sc_intr_p_last[x];
 522                         qh_y = sc->sc_intr_p_last[y];
 523
 524                         qh_x->h_next = NULL;
 525                         qh_x->qh_h_next = qh_y->qh_self;
 526                         qh_x->e_next = NULL;
 527                         qh_x->qh_e_next = htole32(UHCI_PTR_T);
 528                         x++;
 529                 }
 530                 bit >>= 1;
 531         }
 532
 533         if (1) {
 534                 uhci_qh_t *qh_ls;
 535                 uhci_qh_t *qh_intr;
 536
 537                 qh_ls = sc->sc_ls_ctl_p_last;
 538                 qh_intr = sc->sc_intr_p_last[0];
 539
 540                 /* start QH for interrupt traffic */
 541                 qh_intr->h_next = qh_ls;
 542                 qh_intr->qh_h_next = qh_ls->qh_self;
 543                 qh_intr->e_next = 0;
 544                 qh_intr->qh_e_next = htole32(UHCI_PTR_T);
 545         }
 546         for (x = 0; x != UHCI_VFRAMELIST_COUNT; x++) {
 547
 548                 uhci_td_t *td_x;
 549                 uhci_qh_t *qh_intr;
 550
 551                 td_x = sc->sc_isoc_p_last[x];
 552                 qh_intr = sc->sc_intr_p_last[x | (UHCI_IFRAMELIST_COUNT / 2)];
 553
 554                 /* start TD for isochronous traffic */
 555                 td_x->next = NULL;
 556                 td_x->td_next = qh_intr->qh_self;
 557                 td_x->td_status = htole32(UHCI_TD_IOS);
 558                 td_x->td_token = htole32(0);
 559                 td_x->td_buffer = htole32(0);
 560         }
 561
 562         if (1) {
 563                 uhci_qh_t *qh_ls;
 564                 uhci_qh_t *qh_fs;
 565
 566                 qh_ls = sc->sc_ls_ctl_p_last;
 567                 qh_fs = sc->sc_fs_ctl_p_last;
 568
 569                 /* start QH where low speed control traffic will be queued */
 570                 qh_ls->h_next = qh_fs;
 571                 qh_ls->qh_h_next = qh_fs->qh_self;
 572                 qh_ls->e_next = 0;
 573                 qh_ls->qh_e_next = htole32(UHCI_PTR_T);
 574         }
 575         if (1) {
 576                 uhci_qh_t *qh_ctl;
 577                 uhci_qh_t *qh_blk;
 578                 uhci_qh_t *qh_lst;
 579                 uhci_td_t *td_lst;
 580
 581                 qh_ctl = sc->sc_fs_ctl_p_last;
 582                 qh_blk = sc->sc_bulk_p_last;
 583
 584                 /* start QH where full speed control traffic will be queued */
 585                 qh_ctl->h_next = qh_blk;
 586                 qh_ctl->qh_h_next = qh_blk->qh_self;
 587                 qh_ctl->e_next = 0;
 588                 qh_ctl->qh_e_next = htole32(UHCI_PTR_T);
 589
 590                 qh_lst = sc->sc_last_qh_p;
 591
 592                 /* start QH where bulk traffic will be queued */
 593                 qh_blk->h_next = qh_lst;
 594                 qh_blk->qh_h_next = qh_lst->qh_self;
 595                 qh_blk->e_next = 0;
 596                 qh_blk->qh_e_next = htole32(UHCI_PTR_T);
 597
 598                 td_lst = sc->sc_last_td_p;
 599
 600                 /* end QH which is used for looping the QHs */
 601                 qh_lst->h_next = 0;
 602                 qh_lst->qh_h_next = htole32(UHCI_PTR_T);        /* end of QH chain */
 603                 qh_lst->e_next = td_lst;
 604                 qh_lst->qh_e_next = td_lst->td_self;
 605
 606                 /*
 607                  * end TD which hangs from the last QH, to avoid a bug in the PIIX
 608                  * that makes it run berserk otherwise
 609                  */
 610                 td_lst->next = 0;
 611                 td_lst->td_next = htole32(UHCI_PTR_T);
 612                 td_lst->td_status = htole32(0); /* inactive */
 613                 td_lst->td_token = htole32(0);
 614                 td_lst->td_buffer = htole32(0);
 615         }
 616         if (1) {
 617                 struct usb_page_search buf_res;
 618                 uint32_t *pframes;
 619
 620                 usbd_get_page(&sc->sc_hw.pframes_pc, 0, &buf_res);
 621
 622                 pframes = buf_res.buffer;
 623
 624
 625                 /*
 626                  * Setup UHCI framelist
 627                  *
 628                  * Execution order:
 629                  *
 630                  * pframes -> full speed isochronous -> interrupt QH's -> low
 631                  * speed control -> full speed control -> bulk transfers
 632                  *
 633                  */
 634
 635                 for (x = 0; x != UHCI_FRAMELIST_COUNT; x++) {
 636                         pframes[x] =
 637                             sc->sc_isoc_p_last[x % UHCI_VFRAMELIST_COUNT]->td_self;
 638                 }
 639         }
 640         /* flush all cache into memory */
 641
 642         usb_bus_mem_flush_all(&sc->sc_bus, &uhci_iterate_hw_softc);
 643
 644         /* set up the bus struct */
 645         sc->sc_bus.methods = &uhci_bus_methods;
 646
 647         USB_BUS_LOCK(&sc->sc_bus);
 648         /* reset the controller */
 649         uhci_reset(sc);
 650
 651         /* start the controller */
 652         uhci_start(sc);
 653         USB_BUS_UNLOCK(&sc->sc_bus);
 654
 655         /* catch lost interrupts */
 656         uhci_do_poll(&sc->sc_bus);
 657
 658         return (0);
 659 }
 660
 661 /* NOTE: suspend/resume is called from
 662  * interrupt context and cannot sleep!
 663  */
 664
 665 void
 666 uhci_suspend(uhci_softc_t *sc)
 667 {
 668         USB_BUS_LOCK(&sc->sc_bus);
 669
 670 #ifdef USB_DEBUG
 671         if (uhcidebug > 2) {
 672                 uhci_dumpregs(sc);
 673         }
 674 #endif
 675         /* save some state if BIOS doesn't */
 676
 677         sc->sc_saved_frnum = UREAD2(sc, UHCI_FRNUM);
 678         sc->sc_saved_sof = UREAD1(sc, UHCI_SOF);
 679
 680         /* stop the controller */
 681
 682         uhci_reset(sc);
 683
 684         /* enter global suspend */
 685
 686         UHCICMD(sc, UHCI_CMD_EGSM);
 687
 688         usb_pause_mtx(&sc->sc_bus.bus_mtx,
 689             USB_MS_TO_TICKS(USB_RESUME_WAIT));
 690
 691         USB_BUS_UNLOCK(&sc->sc_bus);
 692 }
 693
 694 void
 695 uhci_resume(uhci_softc_t *sc)
 696 {
 697         USB_BUS_LOCK(&sc->sc_bus);
 698
 699         /* reset the controller */
 700
 701         uhci_reset(sc);
 702
 703         /* force global resume */
 704
 705         UHCICMD(sc, UHCI_CMD_FGR);
 706
 707         usb_pause_mtx(&sc->sc_bus.bus_mtx,
 708             USB_MS_TO_TICKS(USB_RESUME_DELAY));
 709
 710         /* and start traffic again */
 711
 712         uhci_start(sc);
 713
 714 #ifdef USB_DEBUG
 715         if (uhcidebug > 2) {
 716                 uhci_dumpregs(sc);
 717         }
 718 #endif
 719
 720         USB_BUS_UNLOCK(&sc->sc_bus);
 721
 722         /* catch lost interrupts */
 723         uhci_do_poll(&sc->sc_bus);
 724 }
 725
 726 #ifdef USB_DEBUG
 727 static void
 728 uhci_dumpregs(uhci_softc_t *sc)
 729 {
 730         DPRINTFN(0, "%s regs: cmd=%04x, sts=%04x, intr=%04x, frnum=%04x, "
 731             "flbase=%08x, sof=%04x, portsc1=%04x, portsc2=%04x\n",
 732             device_get_nameunit(sc->sc_bus.bdev),
 733             UREAD2(sc, UHCI_CMD),
 734             UREAD2(sc, UHCI_STS),
 735             UREAD2(sc, UHCI_INTR),
 736             UREAD2(sc, UHCI_FRNUM),
 737             UREAD4(sc, UHCI_FLBASEADDR),
 738             UREAD1(sc, UHCI_SOF),
 739             UREAD2(sc, UHCI_PORTSC1),
 740             UREAD2(sc, UHCI_PORTSC2));
 741 }
 742
 743 static uint8_t
 744 uhci_dump_td(uhci_td_t *p)
 745 {
 746         uint32_t td_next;
 747         uint32_t td_status;
 748         uint32_t td_token;
 749         uint8_t temp;
 750
 751         usb_pc_cpu_invalidate(p->page_cache);
 752
 753         td_next = le32toh(p->td_next);
 754         td_status = le32toh(p->td_status);
 755         td_token = le32toh(p->td_token);
 756
 757         /*
 758          * Check whether the link pointer in this TD marks the link pointer
 759          * as end of queue:
 760          */
 761         temp = ((td_next & UHCI_PTR_T) || (td_next == 0));
 762
 763         printf("TD(%p) at 0x%08x = link=0x%08x status=0x%08x "
 764             "token=0x%08x buffer=0x%08x\n",
 765             p,
 766             le32toh(p->td_self),
 767             td_next,
 768             td_status,
 769             td_token,
 770             le32toh(p->td_buffer));
 771
 772         printf("TD(%p) td_next=%s%s%s td_status=%s%s%s%s%s%s%s%s%s%s%s, errcnt=%d, actlen=%d pid=%02x,"
 773             "addr=%d,endpt=%d,D=%d,maxlen=%d\n",
 774             p,
 775             (td_next & 1) ? "-T" : "",
 776             (td_next & 2) ? "-Q" : "",
 777             (td_next & 4) ? "-VF" : "",
 778             (td_status & UHCI_TD_BITSTUFF) ? "-BITSTUFF" : "",
 779             (td_status & UHCI_TD_CRCTO) ? "-CRCTO" : "",
 780             (td_status & UHCI_TD_NAK) ? "-NAK" : "",
 781             (td_status & UHCI_TD_BABBLE) ? "-BABBLE" : "",
 782             (td_status & UHCI_TD_DBUFFER) ? "-DBUFFER" : "",
 783             (td_status & UHCI_TD_STALLED) ? "-STALLED" : "",
 784             (td_status & UHCI_TD_ACTIVE) ? "-ACTIVE" : "",
 785             (td_status & UHCI_TD_IOC) ? "-IOC" : "",
 786             (td_status & UHCI_TD_IOS) ? "-IOS" : "",
 787             (td_status & UHCI_TD_LS) ? "-LS" : "",
 788             (td_status & UHCI_TD_SPD) ? "-SPD" : "",
 789             UHCI_TD_GET_ERRCNT(td_status),
 790             UHCI_TD_GET_ACTLEN(td_status),
 791             UHCI_TD_GET_PID(td_token),
 792             UHCI_TD_GET_DEVADDR(td_token),
 793             UHCI_TD_GET_ENDPT(td_token),
 794             UHCI_TD_GET_DT(td_token),
 795             UHCI_TD_GET_MAXLEN(td_token));
 796
 797         return (temp);
 798 }
 799
 800 static uint8_t
 801 uhci_dump_qh(uhci_qh_t *sqh)
 802 {
 803         uint8_t temp;
 804         uint32_t qh_h_next;
 805         uint32_t qh_e_next;
 806
 807         usb_pc_cpu_invalidate(sqh->page_cache);
 808
 809         qh_h_next = le32toh(sqh->qh_h_next);
 810         qh_e_next = le32toh(sqh->qh_e_next);
 811
 812         DPRINTFN(0, "QH(%p) at 0x%08x: h_next=0x%08x e_next=0x%08x\n", sqh,
 813             le32toh(sqh->qh_self), qh_h_next, qh_e_next);
 814
 815         temp = ((((sqh->h_next != NULL) && !(qh_h_next & UHCI_PTR_T)) ? 1 : 0) |
 816             (((sqh->e_next != NULL) && !(qh_e_next & UHCI_PTR_T)) ? 2 : 0));
 817
 818         return (temp);
 819 }
 820
 821 static void
 822 uhci_dump_all(uhci_softc_t *sc)
 823 {
 824         uhci_dumpregs(sc);
 825         uhci_dump_qh(sc->sc_ls_ctl_p_last);
 826         uhci_dump_qh(sc->sc_fs_ctl_p_last);
 827         uhci_dump_qh(sc->sc_bulk_p_last);
 828         uhci_dump_qh(sc->sc_last_qh_p);
 829 }
 830
 831 static void
 832 uhci_dump_tds(uhci_td_t *td)
 833 {
 834         for (;
 835             td != NULL;
 836             td = td->obj_next) {
 837                 if (uhci_dump_td(td)) {
 838                         break;
 839                 }
 840         }
 841 }
 842
 843 #endif
 844
 845 /*
 846  * Let the last QH loop back to the full speed control transfer QH.
 847  * This is what intel calls "bandwidth reclamation" and improves
 848  * USB performance a lot for some devices.
 849  * If we are already looping, just count it.
 850  */
 851 static void
 852 uhci_add_loop(uhci_softc_t *sc)
 853 {
 854         struct uhci_qh *qh_lst;
 855         struct uhci_qh *qh_rec;
 856
 857 #ifdef USB_DEBUG
 858         if (uhcinoloop) {
 859                 return;
 860         }
 861 #endif
 862         if (++(sc->sc_loops) == 1) {
 863                 DPRINTFN(6, "add\n");
 864
 865                 qh_lst = sc->sc_last_qh_p;
 866                 qh_rec = sc->sc_reclaim_qh_p;
 867
 868                 /* NOTE: we don't loop back the soft pointer */
 869
 870                 qh_lst->qh_h_next = qh_rec->qh_self;
 871                 usb_pc_cpu_flush(qh_lst->page_cache);
 872         }
 873 }
 874
 875 static void
 876 uhci_rem_loop(uhci_softc_t *sc)
 877 {
 878         struct uhci_qh *qh_lst;
 879
 880 #ifdef USB_DEBUG
 881         if (uhcinoloop) {
 882                 return;
 883         }
 884 #endif
 885         if (--(sc->sc_loops) == 0) {
 886                 DPRINTFN(6, "remove\n");
 887
 888                 qh_lst = sc->sc_last_qh_p;
 889                 qh_lst->qh_h_next = htole32(UHCI_PTR_T);
 890                 usb_pc_cpu_flush(qh_lst->page_cache);
 891         }
 892 }
 893
 894 static void
 895 uhci_transfer_intr_enqueue(struct usb_xfer *xfer)
 896 {
 897         /* check for early completion */
 898         if (uhci_check_transfer(xfer)) {
 899                 return;
 900         }
 901         /* put transfer on interrupt queue */
 902         usbd_transfer_enqueue(&xfer->xroot->bus->intr_q, xfer);
 903
 904         /* start timeout, if any */
 905         if (xfer->timeout != 0) {
 906                 usbd_transfer_timeout_ms(xfer, &uhci_timeout, xfer->timeout);
 907         }
 908 }
 909
 910 #define UHCI_APPEND_TD(std,last) (last) = _uhci_append_td(std,last)
 911 static uhci_td_t *
 912 _uhci_append_td(uhci_td_t *std, uhci_td_t *last)
 913 {
 914         DPRINTFN(11, "%p to %p\n", std, last);
 915
 916         /* (sc->sc_bus.mtx) must be locked */
 917
 918         std->next = last->next;
 919         std->td_next = last->td_next;
 920
 921         std->prev = last;
 922
 923         usb_pc_cpu_flush(std->page_cache);
 924
 925         /*
 926          * the last->next->prev is never followed: std->next->prev = std;
 927          */
 928         last->next = std;
 929         last->td_next = std->td_self;
 930
 931         usb_pc_cpu_flush(last->page_cache);
 932
 933         return (std);
 934 }
 935
 936 #define UHCI_APPEND_QH(sqh,last) (last) = _uhci_append_qh(sqh,last)
 937 static uhci_qh_t *
 938 _uhci_append_qh(uhci_qh_t *sqh, uhci_qh_t *last)
 939 {
 940         DPRINTFN(11, "%p to %p\n", sqh, last);
 941
 942         if (sqh->h_prev != NULL) {
 943                 /* should not happen */
 944                 DPRINTFN(0, "QH already linked!\n");
 945                 return (last);
 946         }
 947         /* (sc->sc_bus.mtx) must be locked */
 948
 949         sqh->h_next = last->h_next;
 950         sqh->qh_h_next = last->qh_h_next;
 951
 952         sqh->h_prev = last;
 953
 954         usb_pc_cpu_flush(sqh->page_cache);
 955
 956         /*
 957          * The "last->h_next->h_prev" is never followed:
 958          *
 959          * "sqh->h_next->h_prev" = sqh;
 960          */
 961
 962         last->h_next = sqh;
 963         last->qh_h_next = sqh->qh_self;
 964
 965         usb_pc_cpu_flush(last->page_cache);
 966
 967         return (sqh);
 968 }
 969
 970 /**/
 971
 972 #define UHCI_REMOVE_TD(std,last) (last) = _uhci_remove_td(std,last)
 973 static uhci_td_t *
 974 _uhci_remove_td(uhci_td_t *std, uhci_td_t *last)
 975 {
 976         DPRINTFN(11, "%p from %p\n", std, last);
 977
 978         /* (sc->sc_bus.mtx) must be locked */
 979
 980         std->prev->next = std->next;
 981         std->prev->td_next = std->td_next;
 982
 983         usb_pc_cpu_flush(std->prev->page_cache);
 984
 985         if (std->next) {
 986                 std->next->prev = std->prev;
 987                 usb_pc_cpu_flush(std->next->page_cache);
 988         }
 989         return ((last == std) ? std->prev : last);
 990 }
 991
 992 #define UHCI_REMOVE_QH(sqh,last) (last) = _uhci_remove_qh(sqh,last)
 993 static uhci_qh_t *
 994 _uhci_remove_qh(uhci_qh_t *sqh, uhci_qh_t *last)
 995 {
 996         DPRINTFN(11, "%p from %p\n", sqh, last);
 997
 998         /* (sc->sc_bus.mtx) must be locked */
 999
1000         /* only remove if not removed from a queue */
1001         if (sqh->h_prev) {
1002
1003                 sqh->h_prev->h_next = sqh->h_next;
1004                 sqh->h_prev->qh_h_next = sqh->qh_h_next;
1005
1006                 usb_pc_cpu_flush(sqh->h_prev->page_cache);
1007
1008                 if (sqh->h_next) {
1009                         sqh->h_next->h_prev = sqh->h_prev;
1010                         usb_pc_cpu_flush(sqh->h_next->page_cache);
1011                 }
1012                 last = ((last == sqh) ? sqh->h_prev : last);
1013
1014                 sqh->h_prev = 0;
1015
1016                 usb_pc_cpu_flush(sqh->page_cache);
1017         }
1018         return (last);
1019 }
1020
1021 static void
1022 uhci_isoc_done(uhci_softc_t *sc, struct usb_xfer *xfer)
1023 {
1024         struct usb_page_search res;
1025         uint32_t nframes = xfer->nframes;
1026         uint32_t status;
1027         uint32_t offset = 0;
1028         uint32_t *plen = xfer->frlengths;
1029         uint16_t len = 0;
1030         uhci_td_t *td = xfer->td_transfer_first;
1031         uhci_td_t **pp_last = &sc->sc_isoc_p_last[xfer->qh_pos];
1032
1033         DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
1034             xfer, xfer->endpoint);
1035
1036         /* sync any DMA memory before doing fixups */
1037
1038         usb_bdma_post_sync(xfer);
1039
1040         while (nframes--) {
1041                 if (td == NULL) {
1042                         panic("%s:%d: out of TD's\n",
1043                             __FUNCTION__, __LINE__);
1044                 }
1045                 if (pp_last >= &sc->sc_isoc_p_last[UHCI_VFRAMELIST_COUNT]) {
1046                         pp_last = &sc->sc_isoc_p_last[0];
1047                 }
1048 #ifdef USB_DEBUG
1049                 if (uhcidebug > 5) {
1050                         DPRINTF("isoc TD\n");
1051                         uhci_dump_td(td);
1052                 }
1053 #endif
1054                 usb_pc_cpu_invalidate(td->page_cache);
1055                 status = le32toh(td->td_status);
1056
1057                 len = UHCI_TD_GET_ACTLEN(status);
1058
1059                 if (len > *plen) {
1060                         len = *plen;
1061                 }
1062                 if (td->fix_pc) {
1063
1064                         usbd_get_page(td->fix_pc, 0, &res);
1065
1066                         /* copy data from fixup location to real location */
1067
1068                         usb_pc_cpu_invalidate(td->fix_pc);
1069
1070                         usbd_copy_in(xfer->frbuffers, offset,
1071                             res.buffer, len);
1072                 }
1073                 offset += *plen;
1074
1075                 *plen = len;
1076
1077                 /* remove TD from schedule */
1078                 UHCI_REMOVE_TD(td, *pp_last);
1079
1080                 pp_last++;
1081                 plen++;
1082                 td = td->obj_next;
1083         }
1084
1085         xfer->aframes = xfer->nframes;
1086 }
1087
1088 static usb_error_t
1089 uhci_non_isoc_done_sub(struct usb_xfer *xfer)
1090 {
1091         struct usb_page_search res;
1092         uhci_td_t *td;
1093         uhci_td_t *td_alt_next;
1094         uint32_t status;
1095         uint32_t token;
1096         uint16_t len;
1097
1098         td = xfer->td_transfer_cache;
1099         td_alt_next = td->alt_next;
1100
1101         if (xfer->aframes != xfer->nframes) {
1102                 usbd_xfer_set_frame_len(xfer, xfer->aframes, 0);
1103         }
1104         while (1) {
1105
1106                 usb_pc_cpu_invalidate(td->page_cache);
1107                 status = le32toh(td->td_status);
1108                 token = le32toh(td->td_token);
1109
1110                 /*
1111                  * Verify the status and add
1112                  * up the actual length:
1113                  */
1114
1115                 len = UHCI_TD_GET_ACTLEN(status);
1116                 if (len > td->len) {
1117                         /* should not happen */
1118                         DPRINTF("Invalid status length, "
1119                             "0x%04x/0x%04x bytes\n", len, td->len);
1120                         status |= UHCI_TD_STALLED;
1121
1122                 } else if ((xfer->aframes != xfer->nframes) && (len > 0)) {
1123
1124                         if (td->fix_pc) {
1125
1126                                 usbd_get_page(td->fix_pc, 0, &res);
1127
1128                                 /*
1129                                  * copy data from fixup location to real
1130                                  * location
1131                                  */
1132
1133                                 usb_pc_cpu_invalidate(td->fix_pc);
1134
1135                                 usbd_copy_in(xfer->frbuffers + xfer->aframes,
1136                                     xfer->frlengths[xfer->aframes], res.buffer, len);
1137                         }
1138                         /* update actual length */
1139
1140                         xfer->frlengths[xfer->aframes] += len;
1141                 }
1142                 /* Check for last transfer */
1143                 if (((void *)td) == xfer->td_transfer_last) {
1144                         td = NULL;
1145                         break;
1146                 }
1147                 if (status & UHCI_TD_STALLED) {
1148                         /* the transfer is finished */
1149                         td = NULL;
1150                         break;
1151                 }
1152                 /* Check for short transfer */
1153                 if (len != td->len) {
1154                         if (xfer->flags_int.short_frames_ok) {
1155                                 /* follow alt next */
1156                                 td = td->alt_next;
1157                         } else {
1158                                 /* the transfer is finished */
1159                                 td = NULL;
1160                         }
1161                         break;
1162                 }
1163                 td = td->obj_next;
1164
1165                 if (td->alt_next != td_alt_next) {
1166                         /* this USB frame is complete */
1167                         break;
1168                 }
1169         }
1170
1171         /* update transfer cache */
1172
1173         xfer->td_transfer_cache = td;
1174
1175         /* update data toggle */
1176
1177         xfer->endpoint->toggle_next = (token & UHCI_TD_SET_DT(1)) ? 0 : 1;
1178
1179 #ifdef USB_DEBUG
1180         if (status & UHCI_TD_ERROR) {
1181                 DPRINTFN(11, "error, addr=%d, endpt=0x%02x, frame=0x%02x "
1182                     "status=%s%s%s%s%s%s%s%s%s%s%s\n",
1183                     xfer->address, xfer->endpointno, xfer->aframes,
1184                     (status & UHCI_TD_BITSTUFF) ? "[BITSTUFF]" : "",
1185                     (status & UHCI_TD_CRCTO) ? "[CRCTO]" : "",
1186                     (status & UHCI_TD_NAK) ? "[NAK]" : "",
1187                     (status & UHCI_TD_BABBLE) ? "[BABBLE]" : "",
1188                     (status & UHCI_TD_DBUFFER) ? "[DBUFFER]" : "",
1189                     (status & UHCI_TD_STALLED) ? "[STALLED]" : "",
1190                     (status & UHCI_TD_ACTIVE) ? "[ACTIVE]" : "[NOT_ACTIVE]",
1191                     (status & UHCI_TD_IOC) ? "[IOC]" : "",
1192                     (status & UHCI_TD_IOS) ? "[IOS]" : "",
1193                     (status & UHCI_TD_LS) ? "[LS]" : "",
1194                     (status & UHCI_TD_SPD) ? "[SPD]" : "");
1195         }
1196 #endif
1197         return (status & UHCI_TD_STALLED) ?
1198             USB_ERR_STALLED : USB_ERR_NORMAL_COMPLETION;
1199 }
1200
1201 static void
1202 uhci_non_isoc_done(struct usb_xfer *xfer)
1203 {
1204         usb_error_t err = 0;
1205
1206         DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
1207             xfer, xfer->endpoint);
1208
1209 #ifdef USB_DEBUG
1210         if (uhcidebug > 10) {
1211                 uhci_dump_tds(xfer->td_transfer_first);
1212         }
1213 #endif
1214
1215         /* sync any DMA memory before doing fixups */
1216
1217         usb_bdma_post_sync(xfer);
1218
1219         /* reset scanner */
1220
1221         xfer->td_transfer_cache = xfer->td_transfer_first;
1222
1223         if (xfer->flags_int.control_xfr) {
1224                 if (xfer->flags_int.control_hdr) {
1225
1226                         err = uhci_non_isoc_done_sub(xfer);
1227                 }
1228                 xfer->aframes = 1;
1229
1230                 if (xfer->td_transfer_cache == NULL) {
1231                         goto done;
1232                 }
1233         }
1234         while (xfer->aframes != xfer->nframes) {
1235
1236                 err = uhci_non_isoc_done_sub(xfer);
1237                 xfer->aframes++;
1238
1239                 if (xfer->td_transfer_cache == NULL) {
1240                         goto done;
1241                 }
1242         }
1243
1244         if (xfer->flags_int.control_xfr &&
1245             !xfer->flags_int.control_act) {
1246
1247                 err = uhci_non_isoc_done_sub(xfer);
1248         }
1249 done:
1250         uhci_device_done(xfer, err);
1251 }
1252
1253 /*------------------------------------------------------------------------*
1254  *      uhci_check_transfer_sub
1255  *
1256  * The main purpose of this function is to update the data-toggle
1257  * in case it is wrong.
1258  *------------------------------------------------------------------------*/
1259 static void
1260 uhci_check_transfer_sub(struct usb_xfer *xfer)
1261 {
1262         uhci_qh_t *qh;
1263         uhci_td_t *td;
1264         uhci_td_t *td_alt_next;
1265
1266         uint32_t td_token;
1267         uint32_t td_self;
1268
1269         td = xfer->td_transfer_cache;
1270         qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
1271
1272         td_token = td->obj_next->td_token;
1273         td = td->alt_next;
1274         xfer->td_transfer_cache = td;
1275         td_self = td->td_self;
1276         td_alt_next = td->alt_next;
1277
1278         if (xfer->flags_int.control_xfr)
1279                 goto skip;      /* don't touch the DT value! */
1280
1281         if (!((td->td_token ^ td_token) & htole32(UHCI_TD_SET_DT(1))))
1282                 goto skip;      /* data toggle has correct value */
1283
1284         /*
1285          * The data toggle is wrong and we need to toggle it !
1286          */
1287         while (1) {
1288
1289                 td->td_token ^= htole32(UHCI_TD_SET_DT(1));
1290                 usb_pc_cpu_flush(td->page_cache);
1291
1292                 if (td == xfer->td_transfer_last) {
1293                         /* last transfer */
1294                         break;
1295                 }
1296                 td = td->obj_next;
1297
1298                 if (td->alt_next != td_alt_next) {
1299                         /* next frame */
1300                         break;
1301                 }
1302         }
1303 skip:
1304
1305         /* update the QH */
1306         qh->qh_e_next = td_self;
1307         usb_pc_cpu_flush(qh->page_cache);
1308
1309         DPRINTFN(13, "xfer=%p following alt next\n", xfer);
1310 }
1311
1312 /*------------------------------------------------------------------------*
1313  *      uhci_check_transfer
1314  *
1315  * Return values:
1316  *    0: USB transfer is not finished
1317  * Else: USB transfer is finished
1318  *------------------------------------------------------------------------*/
1319 static uint8_t
1320 uhci_check_transfer(struct usb_xfer *xfer)
1321 {
1322         uint32_t status;
1323         uint32_t token;
1324         uhci_td_t *td;
1325
1326         DPRINTFN(16, "xfer=%p checking transfer\n", xfer);
1327
1328         if (xfer->endpoint->methods == &uhci_device_isoc_methods) {
1329                 /* isochronous transfer */
1330
1331                 td = xfer->td_transfer_last;
1332
1333                 usb_pc_cpu_invalidate(td->page_cache);
1334                 status = le32toh(td->td_status);
1335
1336                 /* check also if the first is complete */
1337
1338                 td = xfer->td_transfer_first;
1339
1340                 usb_pc_cpu_invalidate(td->page_cache);
1341                 status |= le32toh(td->td_status);
1342
1343                 if (!(status & UHCI_TD_ACTIVE)) {
1344                         uhci_device_done(xfer, USB_ERR_NORMAL_COMPLETION);
1345                         goto transferred;
1346                 }
1347         } else {
1348                 /* non-isochronous transfer */
1349
1350                 /*
1351                  * check whether there is an error somewhere
1352                  * in the middle, or whether there was a short
1353                  * packet (SPD and not ACTIVE)
1354                  */
1355                 td = xfer->td_transfer_cache;
1356
1357                 while (1) {
1358                         usb_pc_cpu_invalidate(td->page_cache);
1359                         status = le32toh(td->td_status);
1360                         token = le32toh(td->td_token);
1361
1362                         /*
1363                          * if there is an active TD the transfer isn't done
1364                          */
1365                         if (status & UHCI_TD_ACTIVE) {
1366                                 /* update cache */
1367                                 xfer->td_transfer_cache = td;
1368                                 goto done;
1369                         }
1370                         /*
1371                          * last transfer descriptor makes the transfer done
1372                          */
1373                         if (((void *)td) == xfer->td_transfer_last) {
1374                                 break;
1375                         }
1376                         /*
1377                          * any kind of error makes the transfer done
1378                          */
1379                         if (status & UHCI_TD_STALLED) {
1380                                 break;
1381                         }
1382                         /*
1383                          * check if we reached the last packet
1384                          * or if there is a short packet:
1385                          */
1386                         if ((td->td_next == htole32(UHCI_PTR_T)) ||
1387                             (UHCI_TD_GET_ACTLEN(status) < td->len)) {
1388
1389                                 if (xfer->flags_int.short_frames_ok) {
1390                                         /* follow alt next */
1391                                         if (td->alt_next) {
1392                                                 /* update cache */
1393                                                 xfer->td_transfer_cache = td;
1394                                                 uhci_check_transfer_sub(xfer);
1395                                                 goto done;
1396                                         }
1397                                 }
1398                                 /* transfer is done */
1399                                 break;
1400                         }
1401                         td = td->obj_next;
1402                 }
1403                 uhci_non_isoc_done(xfer);
1404                 goto transferred;
1405         }
1406
1407 done:
1408         DPRINTFN(13, "xfer=%p is still active\n", xfer);
1409         return (0);
1410
1411 transferred:
1412         return (1);
1413 }
1414
1415 static void
1416 uhci_interrupt_poll(uhci_softc_t *sc)
1417 {
1418         struct usb_xfer *xfer;
1419
1420 repeat:
1421         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
1422                 /*
1423                  * check if transfer is transferred
1424                  */
1425                 if (uhci_check_transfer(xfer)) {
1426                         /* queue has been modified */
1427                         goto repeat;
1428                 }
1429         }
1430 }
1431
1432 /*------------------------------------------------------------------------*
1433  *      uhci_interrupt - UHCI interrupt handler
1434  *
1435  * NOTE: Do not access "sc->sc_bus.bdev" inside the interrupt handler,
1436  * hence the interrupt handler will be setup before "sc->sc_bus.bdev"
1437  * is present !
1438  *------------------------------------------------------------------------*/
1439 void
1440 uhci_interrupt(uhci_softc_t *sc)
1441 {
1442         uint32_t status;
1443
1444         USB_BUS_LOCK(&sc->sc_bus);
1445
1446         DPRINTFN(16, "real interrupt\n");
1447
1448 #ifdef USB_DEBUG
1449         if (uhcidebug > 15) {
1450                 uhci_dumpregs(sc);
1451         }
1452 #endif
1453         status = UREAD2(sc, UHCI_STS) & UHCI_STS_ALLINTRS;
1454         if (status == 0) {
1455                 /* the interrupt was not for us */
1456                 goto done;
1457         }
1458         if (status & (UHCI_STS_RD | UHCI_STS_HSE |
1459             UHCI_STS_HCPE | UHCI_STS_HCH)) {
1460
1461                 if (status & UHCI_STS_RD) {
1462 #ifdef USB_DEBUG
1463                         printf("%s: resume detect\n",
1464                             __FUNCTION__);
1465 #endif
1466                 }
1467                 if (status & UHCI_STS_HSE) {
1468                         printf("%s: host system error\n",
1469                             __FUNCTION__);
1470                 }
1471                 if (status & UHCI_STS_HCPE) {
1472                         printf("%s: host controller process error\n",
1473                             __FUNCTION__);
1474                 }
1475                 if (status & UHCI_STS_HCH) {
1476                         /* no acknowledge needed */
1477                         DPRINTF("%s: host controller halted\n",
1478                             __FUNCTION__);
1479 #ifdef USB_DEBUG
1480                         if (uhcidebug > 0) {
1481                                 uhci_dump_all(sc);
1482                         }
1483 #endif
1484                 }
1485         }
1486         /* get acknowledge bits */
1487         status &= (UHCI_STS_USBINT |
1488             UHCI_STS_USBEI |
1489             UHCI_STS_RD |
1490             UHCI_STS_HSE |
1491             UHCI_STS_HCPE);
1492
1493         if (status == 0) {
1494                 /* nothing to acknowledge */
1495                 goto done;
1496         }
1497         /* acknowledge interrupts */
1498         UWRITE2(sc, UHCI_STS, status);
1499
1500         /* poll all the USB transfers */
1501         uhci_interrupt_poll(sc);
1502
1503 done:
1504         USB_BUS_UNLOCK(&sc->sc_bus);
1505 }
1506
1507 /*
1508  * called when a request does not complete
1509  */
1510 static void
1511 uhci_timeout(void *arg)
1512 {
1513         struct usb_xfer *xfer = arg;
1514
1515         DPRINTF("xfer=%p\n", xfer);
1516
1517         USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
1518
1519         /* transfer is transferred */
1520         uhci_device_done(xfer, USB_ERR_TIMEOUT);
1521 }
1522
1523 static void
1524 uhci_do_poll(struct usb_bus *bus)
1525 {
1526         struct uhci_softc *sc = UHCI_BUS2SC(bus);
1527
1528         USB_BUS_LOCK(&sc->sc_bus);
1529         uhci_interrupt_poll(sc);
1530         USB_BUS_UNLOCK(&sc->sc_bus);
1531 }
1532
1533 static void
1534 uhci_setup_standard_chain_sub(struct uhci_std_temp *temp)
1535 {
1536         uhci_td_t *td;
1537         uhci_td_t *td_next;
1538         uhci_td_t *td_alt_next;
1539         uint32_t average;
1540         uint32_t len_old;
1541         uint8_t shortpkt_old;
1542         uint8_t precompute;
1543
1544         td_alt_next = NULL;
1545         shortpkt_old = temp->shortpkt;
1546         len_old = temp->len;
1547         precompute = 1;
1548
1549         /* software is used to detect short incoming transfers */
1550
1551         if ((temp->td_token & htole32(UHCI_TD_PID)) == htole32(UHCI_TD_PID_IN)) {
1552                 temp->td_status |= htole32(UHCI_TD_SPD);
1553         } else {
1554                 temp->td_status &= ~htole32(UHCI_TD_SPD);
1555         }
1556
1557         temp->ml.buf_offset = 0;
1558
1559 restart:
1560
1561         temp->td_token &= ~htole32(UHCI_TD_SET_MAXLEN(0));
1562         temp->td_token |= htole32(UHCI_TD_SET_MAXLEN(temp->average));
1563
1564         td = temp->td;
1565         td_next = temp->td_next;
1566
1567         while (1) {
1568
1569                 if (temp->len == 0) {
1570
1571                         if (temp->shortpkt) {
1572                                 break;
1573                         }
1574                         /* send a Zero Length Packet, ZLP, last */
1575
1576                         temp->shortpkt = 1;
1577                         temp->td_token |= htole32(UHCI_TD_SET_MAXLEN(0));
1578                         average = 0;
1579
1580                 } else {
1581
1582                         average = temp->average;
1583
1584                         if (temp->len < average) {
1585                                 temp->shortpkt = 1;
1586                                 temp->td_token &= ~htole32(UHCI_TD_SET_MAXLEN(0));
1587                                 temp->td_token |= htole32(UHCI_TD_SET_MAXLEN(temp->len));
1588                                 average = temp->len;
1589                         }
1590                 }
1591
1592                 if (td_next == NULL) {
1593                         panic("%s: out of UHCI transfer descriptors!", __FUNCTION__);
1594                 }
1595                 /* get next TD */
1596
1597                 td = td_next;
1598                 td_next = td->obj_next;
1599
1600                 /* check if we are pre-computing */
1601
1602                 if (precompute) {
1603
1604                         /* update remaining length */
1605
1606                         temp->len -= average;
1607
1608                         continue;
1609                 }
1610                 /* fill out current TD */
1611
1612                 td->td_status = temp->td_status;
1613                 td->td_token = temp->td_token;
1614
1615                 /* update data toggle */
1616
1617                 temp->td_token ^= htole32(UHCI_TD_SET_DT(1));
1618
1619                 if (average == 0) {
1620
1621                         td->len = 0;
1622                         td->td_buffer = 0;
1623                         td->fix_pc = NULL;
1624
1625                 } else {
1626
1627                         /* update remaining length */
1628
1629                         temp->len -= average;
1630
1631                         td->len = average;
1632
1633                         /* fill out buffer pointer and do fixup, if any */
1634
1635                         uhci_mem_layout_fixup(&temp->ml, td);
1636                 }
1637
1638                 td->alt_next = td_alt_next;
1639
1640                 if ((td_next == td_alt_next) && temp->setup_alt_next) {
1641                         /* we need to receive these frames one by one ! */
1642                         td->td_status |= htole32(UHCI_TD_IOC);
1643                         td->td_next = htole32(UHCI_PTR_T);
1644                 } else {
1645                         if (td_next) {
1646                                 /* link the current TD with the next one */
1647                                 td->td_next = td_next->td_self;
1648                         }
1649                 }
1650
1651                 usb_pc_cpu_flush(td->page_cache);
1652         }
1653
1654         if (precompute) {
1655                 precompute = 0;
1656
1657                 /* setup alt next pointer, if any */
1658                 if (temp->last_frame) {
1659                         td_alt_next = NULL;
1660                 } else {
1661                         /* we use this field internally */
1662                         td_alt_next = td_next;
1663                 }
1664
1665                 /* restore */
1666                 temp->shortpkt = shortpkt_old;
1667                 temp->len = len_old;
1668                 goto restart;
1669         }
1670         temp->td = td;
1671         temp->td_next = td_next;
1672 }
1673
1674 static uhci_td_t *
1675 uhci_setup_standard_chain(struct usb_xfer *xfer)
1676 {
1677         struct uhci_std_temp temp;
1678         uhci_td_t *td;
1679         uint32_t x;
1680
1681         DPRINTFN(9, "addr=%d endpt=%d sumlen=%d speed=%d\n",
1682             xfer->address, UE_GET_ADDR(xfer->endpointno),
1683             xfer->sumlen, usbd_get_speed(xfer->xroot->udev));
1684
1685         temp.average = xfer->max_frame_size;
1686         temp.max_frame_size = xfer->max_frame_size;
1687
1688         /* toggle the DMA set we are using */
1689         xfer->flags_int.curr_dma_set ^= 1;
1690
1691         /* get next DMA set */
1692         td = xfer->td_start[xfer->flags_int.curr_dma_set];
1693         xfer->td_transfer_first = td;
1694         xfer->td_transfer_cache = td;
1695
1696         temp.td = NULL;
1697         temp.td_next = td;
1698         temp.last_frame = 0;
1699         temp.setup_alt_next = xfer->flags_int.short_frames_ok;
1700
1701         uhci_mem_layout_init(&temp.ml, xfer);
1702
1703         temp.td_status =
1704             htole32(UHCI_TD_ZERO_ACTLEN(UHCI_TD_SET_ERRCNT(3) |
1705             UHCI_TD_ACTIVE));
1706
1707         if (xfer->xroot->udev->speed == USB_SPEED_LOW) {
1708                 temp.td_status |= htole32(UHCI_TD_LS);
1709         }
1710         temp.td_token =
1711             htole32(UHCI_TD_SET_ENDPT(xfer->endpointno) |
1712             UHCI_TD_SET_DEVADDR(xfer->address));
1713
1714         if (xfer->endpoint->toggle_next) {
1715                 /* DATA1 is next */
1716                 temp.td_token |= htole32(UHCI_TD_SET_DT(1));
1717         }
1718         /* check if we should prepend a setup message */
1719
1720         if (xfer->flags_int.control_xfr) {
1721
1722                 if (xfer->flags_int.control_hdr) {
1723
1724                         temp.td_token &= htole32(UHCI_TD_SET_DEVADDR(0x7F) |
1725                             UHCI_TD_SET_ENDPT(0xF));
1726                         temp.td_token |= htole32(UHCI_TD_PID_SETUP |
1727                             UHCI_TD_SET_DT(0));
1728
1729                         temp.len = xfer->frlengths[0];
1730                         temp.ml.buf_pc = xfer->frbuffers + 0;
1731                         temp.shortpkt = temp.len ? 1 : 0;
1732                         /* check for last frame */
1733                         if (xfer->nframes == 1) {
1734                                 /* no STATUS stage yet, SETUP is last */
1735                                 if (xfer->flags_int.control_act) {
1736                                         temp.last_frame = 1;
1737                                         temp.setup_alt_next = 0;
1738                                 }
1739                         }
1740                         uhci_setup_standard_chain_sub(&temp);
1741                 }
1742                 x = 1;
1743         } else {
1744                 x = 0;
1745         }
1746
1747         while (x != xfer->nframes) {
1748
1749                 /* DATA0 / DATA1 message */
1750
1751                 temp.len = xfer->frlengths[x];
1752                 temp.ml.buf_pc = xfer->frbuffers + x;
1753
1754                 x++;
1755
1756                 if (x == xfer->nframes) {
1757                         if (xfer->flags_int.control_xfr) {
1758                                 /* no STATUS stage yet, DATA is last */
1759                                 if (xfer->flags_int.control_act) {
1760                                         temp.last_frame = 1;
1761                                         temp.setup_alt_next = 0;
1762                                 }
1763                         } else {
1764                                 temp.last_frame = 1;
1765                                 temp.setup_alt_next = 0;
1766                         }
1767                 }
1768                 /*
1769                  * Keep previous data toggle,
1770                  * device address and endpoint number:
1771                  */
1772
1773                 temp.td_token &= htole32(UHCI_TD_SET_DEVADDR(0x7F) |
1774                     UHCI_TD_SET_ENDPT(0xF) |
1775                     UHCI_TD_SET_DT(1));
1776
1777                 if (temp.len == 0) {
1778
1779                         /* make sure that we send an USB packet */
1780
1781                         temp.shortpkt = 0;
1782
1783                 } else {
1784
1785                         /* regular data transfer */
1786
1787                         temp.shortpkt = (xfer->flags.force_short_xfer) ? 0 : 1;
1788                 }
1789
1790                 /* set endpoint direction */
1791
1792                 temp.td_token |=
1793                     (UE_GET_DIR(xfer->endpointno) == UE_DIR_IN) ?
1794                     htole32(UHCI_TD_PID_IN) :
1795                     htole32(UHCI_TD_PID_OUT);
1796
1797                 uhci_setup_standard_chain_sub(&temp);
1798         }
1799
1800         /* check if we should append a status stage */
1801
1802         if (xfer->flags_int.control_xfr &&
1803             !xfer->flags_int.control_act) {
1804
1805                 /*
1806                  * send a DATA1 message and reverse the current endpoint
1807                  * direction
1808                  */
1809
1810                 temp.td_token &= htole32(UHCI_TD_SET_DEVADDR(0x7F) |
1811                     UHCI_TD_SET_ENDPT(0xF) |
1812                     UHCI_TD_SET_DT(1));
1813                 temp.td_token |=
1814                     (UE_GET_DIR(xfer->endpointno) == UE_DIR_OUT) ?
1815                     htole32(UHCI_TD_PID_IN | UHCI_TD_SET_DT(1)) :
1816                     htole32(UHCI_TD_PID_OUT | UHCI_TD_SET_DT(1));
1817
1818                 temp.len = 0;
1819                 temp.ml.buf_pc = NULL;
1820                 temp.shortpkt = 0;
1821                 temp.last_frame = 1;
1822                 temp.setup_alt_next = 0;
1823
1824                 uhci_setup_standard_chain_sub(&temp);
1825         }
1826         td = temp.td;
1827
1828         /* Ensure that last TD is terminating: */
1829         td->td_next = htole32(UHCI_PTR_T);
1830
1831         /* set interrupt bit */
1832
1833         td->td_status |= htole32(UHCI_TD_IOC);
1834
1835         usb_pc_cpu_flush(td->page_cache);
1836
1837         /* must have at least one frame! */
1838
1839         xfer->td_transfer_last = td;
1840
1841 #ifdef USB_DEBUG
1842         if (uhcidebug > 8) {
1843                 DPRINTF("nexttog=%d; data before transfer:\n",
1844                     xfer->endpoint->toggle_next);
1845                 uhci_dump_tds(xfer->td_transfer_first);
1846         }
1847 #endif
1848         return (xfer->td_transfer_first);
1849 }
1850
1851 /* NOTE: "done" can be run two times in a row,
1852  * from close and from interrupt
1853  */
1854
1855 static void
1856 uhci_device_done(struct usb_xfer *xfer, usb_error_t error)
1857 {
1858         struct usb_pipe_methods *methods = xfer->endpoint->methods;
1859         uhci_softc_t *sc = UHCI_BUS2SC(xfer->xroot->bus);
1860         uhci_qh_t *qh;
1861
1862         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
1863
1864         DPRINTFN(2, "xfer=%p, endpoint=%p, error=%d\n",
1865             xfer, xfer->endpoint, error);
1866
1867         qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
1868         if (qh) {
1869                 usb_pc_cpu_invalidate(qh->page_cache);
1870         }
1871         if (xfer->flags_int.bandwidth_reclaimed) {
1872                 xfer->flags_int.bandwidth_reclaimed = 0;
1873                 uhci_rem_loop(sc);
1874         }
1875         if (methods == &uhci_device_bulk_methods) {
1876                 UHCI_REMOVE_QH(qh, sc->sc_bulk_p_last);
1877         }
1878         if (methods == &uhci_device_ctrl_methods) {
1879                 if (xfer->xroot->udev->speed == USB_SPEED_LOW) {
1880                         UHCI_REMOVE_QH(qh, sc->sc_ls_ctl_p_last);
1881                 } else {
1882                         UHCI_REMOVE_QH(qh, sc->sc_fs_ctl_p_last);
1883                 }
1884         }
1885         if (methods == &uhci_device_intr_methods) {
1886                 UHCI_REMOVE_QH(qh, sc->sc_intr_p_last[xfer->qh_pos]);
1887         }
1888         /*
1889          * Only finish isochronous transfers once
1890          * which will update "xfer->frlengths".
1891          */
1892         if (xfer->td_transfer_first &&
1893             xfer->td_transfer_last) {
1894                 if (methods == &uhci_device_isoc_methods) {
1895                         uhci_isoc_done(sc, xfer);
1896                 }
1897                 xfer->td_transfer_first = NULL;
1898                 xfer->td_transfer_last = NULL;
1899         }
1900         /* dequeue transfer and start next transfer */
1901         usbd_transfer_done(xfer, error);
1902 }
1903
1904 /*------------------------------------------------------------------------*
1905  * uhci bulk support
1906  *------------------------------------------------------------------------*/
1907 static void
1908 uhci_device_bulk_open(struct usb_xfer *xfer)
1909 {
1910         return;
1911 }
1912
1913 static void
1914 uhci_device_bulk_close(struct usb_xfer *xfer)
1915 {
1916         uhci_device_done(xfer, USB_ERR_CANCELLED);
1917 }
1918
1919 static void
1920 uhci_device_bulk_enter(struct usb_xfer *xfer)
1921 {
1922         return;
1923 }
1924
1925 static void
1926 uhci_device_bulk_start(struct usb_xfer *xfer)
1927 {
1928         uhci_softc_t *sc = UHCI_BUS2SC(xfer->xroot->bus);
1929         uhci_td_t *td;
1930         uhci_qh_t *qh;
1931
1932         /* setup TD's */
1933         td = uhci_setup_standard_chain(xfer);
1934
1935         /* setup QH */
1936         qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
1937
1938         qh->e_next = td;
1939         qh->qh_e_next = td->td_self;
1940
1941         if (xfer->xroot->udev->flags.self_suspended == 0) {
1942                 UHCI_APPEND_QH(qh, sc->sc_bulk_p_last);
1943                 uhci_add_loop(sc);
1944                 xfer->flags_int.bandwidth_reclaimed = 1;
1945         } else {
1946                 usb_pc_cpu_flush(qh->page_cache);
1947         }
1948
1949         /* put transfer on interrupt queue */
1950         uhci_transfer_intr_enqueue(xfer);
1951 }
1952
1953 struct usb_pipe_methods uhci_device_bulk_methods =
1954 {
1955         .open = uhci_device_bulk_open,
1956         .close = uhci_device_bulk_close,
1957         .enter = uhci_device_bulk_enter,
1958         .start = uhci_device_bulk_start,
1959 };
1960
1961 /*------------------------------------------------------------------------*
1962  * uhci control support
1963  *------------------------------------------------------------------------*/
1964 static void
1965 uhci_device_ctrl_open(struct usb_xfer *xfer)
1966 {
1967         return;
1968 }
1969
1970 static void
1971 uhci_device_ctrl_close(struct usb_xfer *xfer)
1972 {
1973         uhci_device_done(xfer, USB_ERR_CANCELLED);
1974 }
1975
1976 static void
1977 uhci_device_ctrl_enter(struct usb_xfer *xfer)
1978 {
1979         return;
1980 }
1981
1982 static void
1983 uhci_device_ctrl_start(struct usb_xfer *xfer)
1984 {
1985         uhci_softc_t *sc = UHCI_BUS2SC(xfer->xroot->bus);
1986         uhci_qh_t *qh;
1987         uhci_td_t *td;
1988
1989         /* setup TD's */
1990         td = uhci_setup_standard_chain(xfer);
1991
1992         /* setup QH */
1993         qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
1994
1995         qh->e_next = td;
1996         qh->qh_e_next = td->td_self;
1997
1998         /*
1999          * NOTE: some devices choke on bandwidth- reclamation for control
2000          * transfers
2001          */
2002         if (xfer->xroot->udev->flags.self_suspended == 0) {
2003                 if (xfer->xroot->udev->speed == USB_SPEED_LOW) {
2004                         UHCI_APPEND_QH(qh, sc->sc_ls_ctl_p_last);
2005                 } else {
2006                         UHCI_APPEND_QH(qh, sc->sc_fs_ctl_p_last);
2007                 }
2008         } else {
2009                 usb_pc_cpu_flush(qh->page_cache);
2010         }
2011         /* put transfer on interrupt queue */
2012         uhci_transfer_intr_enqueue(xfer);
2013 }
2014
2015 struct usb_pipe_methods uhci_device_ctrl_methods =
2016 {
2017         .open = uhci_device_ctrl_open,
2018         .close = uhci_device_ctrl_close,
2019         .enter = uhci_device_ctrl_enter,
2020         .start = uhci_device_ctrl_start,
2021 };
2022
2023 /*------------------------------------------------------------------------*
2024  * uhci interrupt support
2025  *------------------------------------------------------------------------*/
2026 static void
2027 uhci_device_intr_open(struct usb_xfer *xfer)
2028 {
2029         uhci_softc_t *sc = UHCI_BUS2SC(xfer->xroot->bus);
2030         uint16_t best;
2031         uint16_t bit;
2032         uint16_t x;
2033
2034         best = 0;
2035         bit = UHCI_IFRAMELIST_COUNT / 2;
2036         while (bit) {
2037                 if (xfer->interval >= bit) {
2038                         x = bit;
2039                         best = bit;
2040                         while (x & bit) {
2041                                 if (sc->sc_intr_stat[x] <
2042                                     sc->sc_intr_stat[best]) {
2043                                         best = x;
2044                                 }
2045                                 x++;
2046                         }
2047                         break;
2048                 }
2049                 bit >>= 1;
2050         }
2051
2052         sc->sc_intr_stat[best]++;
2053         xfer->qh_pos = best;
2054
2055         DPRINTFN(3, "best=%d interval=%d\n",
2056             best, xfer->interval);
2057 }
2058
2059 static void
2060 uhci_device_intr_close(struct usb_xfer *xfer)
2061 {
2062         uhci_softc_t *sc = UHCI_BUS2SC(xfer->xroot->bus);
2063
2064         sc->sc_intr_stat[xfer->qh_pos]--;
2065
2066         uhci_device_done(xfer, USB_ERR_CANCELLED);
2067 }
2068
2069 static void
2070 uhci_device_intr_enter(struct usb_xfer *xfer)
2071 {
2072         return;
2073 }
2074
2075 static void
2076 uhci_device_intr_start(struct usb_xfer *xfer)
2077 {
2078         uhci_softc_t *sc = UHCI_BUS2SC(xfer->xroot->bus);
2079         uhci_qh_t *qh;
2080         uhci_td_t *td;
2081
2082         /* setup TD's */
2083         td = uhci_setup_standard_chain(xfer);
2084
2085         /* setup QH */
2086         qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
2087
2088         qh->e_next = td;
2089         qh->qh_e_next = td->td_self;
2090
2091         if (xfer->xroot->udev->flags.self_suspended == 0) {
2092                 /* enter QHs into the controller data structures */
2093                 UHCI_APPEND_QH(qh, sc->sc_intr_p_last[xfer->qh_pos]);
2094         } else {
2095                 usb_pc_cpu_flush(qh->page_cache);
2096         }
2097
2098         /* put transfer on interrupt queue */
2099         uhci_transfer_intr_enqueue(xfer);
2100 }
2101
2102 struct usb_pipe_methods uhci_device_intr_methods =
2103 {
2104         .open = uhci_device_intr_open,
2105         .close = uhci_device_intr_close,
2106         .enter = uhci_device_intr_enter,
2107         .start = uhci_device_intr_start,
2108 };
2109
2110 /*------------------------------------------------------------------------*
2111  * uhci isochronous support
2112  *------------------------------------------------------------------------*/
2113 static void
2114 uhci_device_isoc_open(struct usb_xfer *xfer)
2115 {
2116         uhci_td_t *td;
2117         uint32_t td_token;
2118         uint8_t ds;
2119
2120         td_token =
2121             (UE_GET_DIR(xfer->endpointno) == UE_DIR_IN) ?
2122             UHCI_TD_IN(0, xfer->endpointno, xfer->address, 0) :
2123             UHCI_TD_OUT(0, xfer->endpointno, xfer->address, 0);
2124
2125         td_token = htole32(td_token);
2126
2127         /* initialize all TD's */
2128
2129         for (ds = 0; ds != 2; ds++) {
2130
2131                 for (td = xfer->td_start[ds]; td; td = td->obj_next) {
2132
2133                         /* mark TD as inactive */
2134                         td->td_status = htole32(UHCI_TD_IOS);
2135                         td->td_token = td_token;
2136
2137                         usb_pc_cpu_flush(td->page_cache);
2138                 }
2139         }
2140 }
2141
2142 static void
2143 uhci_device_isoc_close(struct usb_xfer *xfer)
2144 {
2145         uhci_device_done(xfer, USB_ERR_CANCELLED);
2146 }
2147
2148 static void
2149 uhci_device_isoc_enter(struct usb_xfer *xfer)
2150 {
2151         struct uhci_mem_layout ml;
2152         uhci_softc_t *sc = UHCI_BUS2SC(xfer->xroot->bus);
2153         uint32_t nframes;
2154         uint32_t temp;
2155         uint32_t *plen;
2156
2157 #ifdef USB_DEBUG
2158         uint8_t once = 1;
2159
2160 #endif
2161         uhci_td_t *td;
2162         uhci_td_t *td_last = NULL;
2163         uhci_td_t **pp_last;
2164
2165         DPRINTFN(6, "xfer=%p next=%d nframes=%d\n",
2166             xfer, xfer->endpoint->isoc_next, xfer->nframes);
2167
2168         nframes = UREAD2(sc, UHCI_FRNUM);
2169
2170         temp = (nframes - xfer->endpoint->isoc_next) &
2171             (UHCI_VFRAMELIST_COUNT - 1);
2172
2173         if ((xfer->endpoint->is_synced == 0) ||
2174             (temp < xfer->nframes)) {
2175                 /*
2176                  * If there is data underflow or the pipe queue is empty we
2177                  * schedule the transfer a few frames ahead of the current
2178                  * frame position. Else two isochronous transfers might
2179                  * overlap.
2180                  */
2181                 xfer->endpoint->isoc_next = (nframes + 3) & (UHCI_VFRAMELIST_COUNT - 1);
2182                 xfer->endpoint->is_synced = 1;
2183                 DPRINTFN(3, "start next=%d\n", xfer->endpoint->isoc_next);
2184         }
2185         /*
2186          * compute how many milliseconds the insertion is ahead of the
2187          * current frame position:
2188          */
2189         temp = (xfer->endpoint->isoc_next - nframes) &
2190             (UHCI_VFRAMELIST_COUNT - 1);
2191
2192         /*
2193          * pre-compute when the isochronous transfer will be finished:
2194          */
2195         xfer->isoc_time_complete =
2196             usb_isoc_time_expand(&sc->sc_bus, nframes) + temp +
2197             xfer->nframes;
2198
2199         /* get the real number of frames */
2200
2201         nframes = xfer->nframes;
2202
2203         uhci_mem_layout_init(&ml, xfer);
2204
2205         plen = xfer->frlengths;
2206
2207         /* toggle the DMA set we are using */
2208         xfer->flags_int.curr_dma_set ^= 1;
2209
2210         /* get next DMA set */
2211         td = xfer->td_start[xfer->flags_int.curr_dma_set];
2212         xfer->td_transfer_first = td;
2213
2214         pp_last = &sc->sc_isoc_p_last[xfer->endpoint->isoc_next];
2215
2216         /* store starting position */
2217
2218         xfer->qh_pos = xfer->endpoint->isoc_next;
2219
2220         while (nframes--) {
2221                 if (td == NULL) {
2222                         panic("%s:%d: out of TD's\n",
2223                             __FUNCTION__, __LINE__);
2224                 }
2225                 if (pp_last >= &sc->sc_isoc_p_last[UHCI_VFRAMELIST_COUNT]) {
2226                         pp_last = &sc->sc_isoc_p_last[0];
2227                 }
2228                 if (*plen > xfer->max_frame_size) {
2229 #ifdef USB_DEBUG
2230                         if (once) {
2231                                 once = 0;
2232                                 printf("%s: frame length(%d) exceeds %d "
2233                                     "bytes (frame truncated)\n",
2234                                     __FUNCTION__, *plen,
2235                                     xfer->max_frame_size);
2236                         }
2237 #endif
2238                         *plen = xfer->max_frame_size;
2239                 }
2240                 /* reuse td_token from last transfer */
2241
2242                 td->td_token &= htole32(~UHCI_TD_MAXLEN_MASK);
2243                 td->td_token |= htole32(UHCI_TD_SET_MAXLEN(*plen));
2244
2245                 td->len = *plen;
2246
2247                 if (td->len == 0) {
2248                         /*
2249                          * Do not call "uhci_mem_layout_fixup()" when the
2250                          * length is zero!
2251                          */
2252                         td->td_buffer = 0;
2253                         td->fix_pc = NULL;
2254
2255                 } else {
2256
2257                         /* fill out buffer pointer and do fixup, if any */
2258
2259                         uhci_mem_layout_fixup(&ml, td);
2260
2261                 }
2262
2263                 /* update status */
2264                 if (nframes == 0) {
2265                         td->td_status = htole32
2266                             (UHCI_TD_ZERO_ACTLEN
2267                             (UHCI_TD_SET_ERRCNT(0) |
2268                             UHCI_TD_ACTIVE |
2269                             UHCI_TD_IOS |
2270                             UHCI_TD_IOC));
2271                 } else {
2272                         td->td_status = htole32
2273                             (UHCI_TD_ZERO_ACTLEN
2274                             (UHCI_TD_SET_ERRCNT(0) |
2275                             UHCI_TD_ACTIVE |
2276                             UHCI_TD_IOS));
2277                 }
2278
2279                 usb_pc_cpu_flush(td->page_cache);
2280
2281 #ifdef USB_DEBUG
2282                 if (uhcidebug > 5) {
2283                         DPRINTF("TD %d\n", nframes);
2284                         uhci_dump_td(td);
2285                 }
2286 #endif
2287                 /* insert TD into schedule */
2288                 UHCI_APPEND_TD(td, *pp_last);
2289                 pp_last++;
2290
2291                 plen++;
2292                 td_last = td;
2293                 td = td->obj_next;
2294         }
2295
2296         xfer->td_transfer_last = td_last;
2297
2298         /* update isoc_next */
2299         xfer->endpoint->isoc_next = (pp_last - &sc->sc_isoc_p_last[0]) &
2300             (UHCI_VFRAMELIST_COUNT - 1);
2301 }
2302
2303 static void
2304 uhci_device_isoc_start(struct usb_xfer *xfer)
2305 {
2306         /* put transfer on interrupt queue */
2307         uhci_transfer_intr_enqueue(xfer);
2308 }
2309
2310 struct usb_pipe_methods uhci_device_isoc_methods =
2311 {
2312         .open = uhci_device_isoc_open,
2313         .close = uhci_device_isoc_close,
2314         .enter = uhci_device_isoc_enter,
2315         .start = uhci_device_isoc_start,
2316 };
2317
2318 /*------------------------------------------------------------------------*
2319  * uhci root control support
2320  *------------------------------------------------------------------------*
2321  * Simulate a hardware hub by handling all the necessary requests.
2322  *------------------------------------------------------------------------*/
2323
2324 static const
2325 struct usb_device_descriptor uhci_devd =
2326 {
2327         sizeof(struct usb_device_descriptor),
2328         UDESC_DEVICE,                   /* type */
2329         {0x00, 0x01},                   /* USB version */
2330         UDCLASS_HUB,                    /* class */
2331         UDSUBCLASS_HUB,                 /* subclass */
2332         UDPROTO_FSHUB,                  /* protocol */
2333         64,                             /* max packet */
2334         {0}, {0}, {0x00, 0x01},         /* device id */
2335         1, 2, 0,                        /* string indicies */
2336         1                               /* # of configurations */
2337 };
2338
2339 static const struct uhci_config_desc uhci_confd = {
2340         .confd = {
2341                 .bLength = sizeof(struct usb_config_descriptor),
2342                 .bDescriptorType = UDESC_CONFIG,
2343                 .wTotalLength[0] = sizeof(uhci_confd),
2344                 .bNumInterface = 1,
2345                 .bConfigurationValue = 1,
2346                 .iConfiguration = 0,
2347                 .bmAttributes = UC_SELF_POWERED,
2348                 .bMaxPower = 0          /* max power */
2349         },
2350         .ifcd = {
2351                 .bLength = sizeof(struct usb_interface_descriptor),
2352                 .bDescriptorType = UDESC_INTERFACE,
2353                 .bNumEndpoints = 1,
2354                 .bInterfaceClass = UICLASS_HUB,
2355                 .bInterfaceSubClass = UISUBCLASS_HUB,
2356                 .bInterfaceProtocol = UIPROTO_FSHUB,
2357         },
2358         .endpd = {
2359                 .bLength = sizeof(struct usb_endpoint_descriptor),
2360                 .bDescriptorType = UDESC_ENDPOINT,
2361                 .bEndpointAddress = UE_DIR_IN | UHCI_INTR_ENDPT,
2362                 .bmAttributes = UE_INTERRUPT,
2363                 .wMaxPacketSize[0] = 8, /* max packet (63 ports) */
2364                 .bInterval = 255,
2365         },
2366 };
2367
2368 static const
2369 struct usb_hub_descriptor_min uhci_hubd_piix =
2370 {
2371         sizeof(uhci_hubd_piix),
2372         UDESC_HUB,
2373         2,
2374         {UHD_PWR_NO_SWITCH | UHD_OC_INDIVIDUAL, 0},
2375         50,                             /* power on to power good */
2376         0,
2377         {0x00},                         /* both ports are removable */
2378 };
2379
2380 /*
2381  * The USB hub protocol requires that SET_FEATURE(PORT_RESET) also
2382  * enables the port, and also states that SET_FEATURE(PORT_ENABLE)
2383  * should not be used by the USB subsystem.  As we cannot issue a
2384  * SET_FEATURE(PORT_ENABLE) externally, we must ensure that the port
2385  * will be enabled as part of the reset.
2386  *
2387  * On the VT83C572, the port cannot be successfully enabled until the
2388  * outstanding "port enable change" and "connection status change"
2389  * events have been reset.
2390  */
2391 static usb_error_t
2392 uhci_portreset(uhci_softc_t *sc, uint16_t index)
2393 {
2394         uint16_t port;
2395         uint16_t x;
2396         uint8_t lim;
2397
2398         if (index == 1)
2399                 port = UHCI_PORTSC1;
2400         else if (index == 2)
2401                 port = UHCI_PORTSC2;
2402         else
2403                 return (USB_ERR_IOERROR);
2404
2405         /*
2406          * Before we do anything, turn on SOF messages on the USB
2407          * BUS. Some USB devices do not cope without them!
2408          */
2409         uhci_restart(sc);
2410
2411         x = URWMASK(UREAD2(sc, port));
2412         UWRITE2(sc, port, x | UHCI_PORTSC_PR);
2413
2414         usb_pause_mtx(&sc->sc_bus.bus_mtx,
2415             USB_MS_TO_TICKS(USB_PORT_ROOT_RESET_DELAY));
2416
2417         DPRINTFN(4, "uhci port %d reset, status0 = 0x%04x\n",
2418             index, UREAD2(sc, port));
2419
2420         x = URWMASK(UREAD2(sc, port));
2421         UWRITE2(sc, port, x & ~UHCI_PORTSC_PR);
2422
2423
2424         mtx_unlock(&sc->sc_bus.bus_mtx);
2425
2426         /*
2427          * This delay needs to be exactly 100us, else some USB devices
2428          * fail to attach!
2429          */
2430         DELAY(100);
2431
2432         mtx_lock(&sc->sc_bus.bus_mtx);
2433
2434         DPRINTFN(4, "uhci port %d reset, status1 = 0x%04x\n",
2435             index, UREAD2(sc, port));
2436
2437         x = URWMASK(UREAD2(sc, port));
2438         UWRITE2(sc, port, x | UHCI_PORTSC_PE);
2439
2440         for (lim = 0; lim < 12; lim++) {
2441
2442                 usb_pause_mtx(&sc->sc_bus.bus_mtx,
2443                     USB_MS_TO_TICKS(USB_PORT_RESET_DELAY));
2444
2445                 x = UREAD2(sc, port);
2446
2447                 DPRINTFN(4, "uhci port %d iteration %u, status = 0x%04x\n",
2448                     index, lim, x);
2449
2450                 if (!(x & UHCI_PORTSC_CCS)) {
2451                         /*
2452                          * No device is connected (or was disconnected
2453                          * during reset).  Consider the port reset.
2454                          * The delay must be long enough to ensure on
2455                          * the initial iteration that the device
2456                          * connection will have been registered.  50ms
2457                          * appears to be sufficient, but 20ms is not.
2458                          */
2459                         DPRINTFN(4, "uhci port %d loop %u, device detached\n",
2460                             index, lim);
2461                         goto done;
2462                 }
2463                 if (x & (UHCI_PORTSC_POEDC | UHCI_PORTSC_CSC)) {
2464                         /*
2465                          * Port enabled changed and/or connection
2466                          * status changed were set.  Reset either or
2467                          * both raised flags (by writing a 1 to that
2468                          * bit), and wait again for state to settle.
2469                          */
2470                         UWRITE2(sc, port, URWMASK(x) |
2471                             (x & (UHCI_PORTSC_POEDC | UHCI_PORTSC_CSC)));
2472                         continue;
2473                 }
2474                 if (x & UHCI_PORTSC_PE) {
2475                         /* port is enabled */
2476                         goto done;
2477                 }
2478                 UWRITE2(sc, port, URWMASK(x) | UHCI_PORTSC_PE);
2479         }
2480
2481         DPRINTFN(2, "uhci port %d reset timed out\n", index);
2482         return (USB_ERR_TIMEOUT);
2483
2484 done:
2485         DPRINTFN(4, "uhci port %d reset, status2 = 0x%04x\n",
2486             index, UREAD2(sc, port));
2487
2488         sc->sc_isreset = 1;
2489         return (USB_ERR_NORMAL_COMPLETION);
2490 }
2491
2492 static usb_error_t
2493 uhci_roothub_exec(struct usb_device *udev,
2494     struct usb_device_request *req, const void **pptr, uint16_t *plength)
2495 {
2496         uhci_softc_t *sc = UHCI_BUS2SC(udev->bus);
2497         const void *ptr;
2498         const char *str_ptr;
2499         uint16_t x;
2500         uint16_t port;
2501         uint16_t value;
2502         uint16_t index;
2503         uint16_t status;
2504         uint16_t change;
2505         uint16_t len;
2506         usb_error_t err;
2507
2508         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
2509
2510         /* buffer reset */
2511         ptr = (const void *)&sc->sc_hub_desc.temp;
2512         len = 0;
2513         err = 0;
2514
2515         value = UGETW(req->wValue);
2516         index = UGETW(req->wIndex);
2517
2518         DPRINTFN(3, "type=0x%02x request=0x%02x wLen=0x%04x "
2519             "wValue=0x%04x wIndex=0x%04x\n",
2520             req->bmRequestType, req->bRequest,
2521             UGETW(req->wLength), value, index);
2522
2523 #define C(x,y) ((x) | ((y) << 8))
2524         switch (C(req->bRequest, req->bmRequestType)) {
2525         case C(UR_CLEAR_FEATURE, UT_WRITE_DEVICE):
2526         case C(UR_CLEAR_FEATURE, UT_WRITE_INTERFACE):
2527         case C(UR_CLEAR_FEATURE, UT_WRITE_ENDPOINT):
2528                 /*
2529                  * DEVICE_REMOTE_WAKEUP and ENDPOINT_HALT are no-ops
2530                  * for the integrated root hub.
2531                  */
2532                 break;
2533         case C(UR_GET_CONFIG, UT_READ_DEVICE):
2534                 len = 1;
2535                 sc->sc_hub_desc.temp[0] = sc->sc_conf;
2536                 break;
2537         case C(UR_GET_DESCRIPTOR, UT_READ_DEVICE):
2538                 switch (value >> 8) {
2539                 case UDESC_DEVICE:
2540                         if ((value & 0xff) != 0) {
2541                                 err = USB_ERR_IOERROR;
2542                                 goto done;
2543                         }
2544                         len = sizeof(uhci_devd);
2545                         ptr = (const void *)&uhci_devd;
2546                         break;
2547
2548                 case UDESC_CONFIG:
2549                         if ((value & 0xff) != 0) {
2550                                 err = USB_ERR_IOERROR;
2551                                 goto done;
2552                         }
2553                         len = sizeof(uhci_confd);
2554                         ptr = (const void *)&uhci_confd;
2555                         break;
2556
2557                 case UDESC_STRING:
2558                         switch (value & 0xff) {
2559                         case 0: /* Language table */
2560                                 str_ptr = "\001";
2561                                 break;
2562
2563                         case 1: /* Vendor */
2564                                 str_ptr = sc->sc_vendor;
2565                                 break;
2566
2567                         case 2: /* Product */
2568                                 str_ptr = "UHCI root HUB";
2569                                 break;
2570
2571                         default:
2572                                 str_ptr = "";
2573                                 break;
2574                         }
2575
2576                         len = usb_make_str_desc
2577                             (sc->sc_hub_desc.temp,
2578                             sizeof(sc->sc_hub_desc.temp),
2579                             str_ptr);
2580                         break;
2581
2582                 default:
2583                         err = USB_ERR_IOERROR;
2584                         goto done;
2585                 }
2586                 break;
2587         case C(UR_GET_INTERFACE, UT_READ_INTERFACE):
2588                 len = 1;
2589                 sc->sc_hub_desc.temp[0] = 0;
2590                 break;
2591         case C(UR_GET_STATUS, UT_READ_DEVICE):
2592                 len = 2;
2593                 USETW(sc->sc_hub_desc.stat.wStatus, UDS_SELF_POWERED);
2594                 break;
2595         case C(UR_GET_STATUS, UT_READ_INTERFACE):
2596         case C(UR_GET_STATUS, UT_READ_ENDPOINT):
2597                 len = 2;
2598                 USETW(sc->sc_hub_desc.stat.wStatus, 0);
2599                 break;
2600         case C(UR_SET_ADDRESS, UT_WRITE_DEVICE):
2601                 if (value >= UHCI_MAX_DEVICES) {
2602                         err = USB_ERR_IOERROR;
2603                         goto done;
2604                 }
2605                 sc->sc_addr = value;
2606                 break;
2607         case C(UR_SET_CONFIG, UT_WRITE_DEVICE):
2608                 if ((value != 0) && (value != 1)) {
2609                         err = USB_ERR_IOERROR;
2610                         goto done;
2611                 }
2612                 sc->sc_conf = value;
2613                 break;
2614         case C(UR_SET_DESCRIPTOR, UT_WRITE_DEVICE):
2615                 break;
2616         case C(UR_SET_FEATURE, UT_WRITE_DEVICE):
2617         case C(UR_SET_FEATURE, UT_WRITE_INTERFACE):
2618         case C(UR_SET_FEATURE, UT_WRITE_ENDPOINT):
2619                 err = USB_ERR_IOERROR;
2620                 goto done;
2621         case C(UR_SET_INTERFACE, UT_WRITE_INTERFACE):
2622                 break;
2623         case C(UR_SYNCH_FRAME, UT_WRITE_ENDPOINT):
2624                 break;
2625                 /* Hub requests */
2626         case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_DEVICE):
2627                 break;
2628         case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_OTHER):
2629                 DPRINTFN(4, "UR_CLEAR_PORT_FEATURE "
2630                     "port=%d feature=%d\n",
2631                     index, value);
2632                 if (index == 1)
2633                         port = UHCI_PORTSC1;
2634                 else if (index == 2)
2635                         port = UHCI_PORTSC2;
2636                 else {
2637                         err = USB_ERR_IOERROR;
2638                         goto done;
2639                 }
2640                 switch (value) {
2641                 case UHF_PORT_ENABLE:
2642                         x = URWMASK(UREAD2(sc, port));
2643                         UWRITE2(sc, port, x & ~UHCI_PORTSC_PE);
2644                         break;
2645                 case UHF_PORT_SUSPEND:
2646                         x = URWMASK(UREAD2(sc, port));
2647                         UWRITE2(sc, port, x & ~(UHCI_PORTSC_SUSP));
2648                         break;
2649                 case UHF_PORT_RESET:
2650                         x = URWMASK(UREAD2(sc, port));
2651                         UWRITE2(sc, port, x & ~UHCI_PORTSC_PR);
2652                         break;
2653                 case UHF_C_PORT_CONNECTION:
2654                         x = URWMASK(UREAD2(sc, port));
2655                         UWRITE2(sc, port, x | UHCI_PORTSC_CSC);
2656                         break;
2657                 case UHF_C_PORT_ENABLE:
2658                         x = URWMASK(UREAD2(sc, port));
2659                         UWRITE2(sc, port, x | UHCI_PORTSC_POEDC);
2660                         break;
2661                 case UHF_C_PORT_OVER_CURRENT:
2662                         x = URWMASK(UREAD2(sc, port));
2663                         UWRITE2(sc, port, x | UHCI_PORTSC_OCIC);
2664                         break;
2665                 case UHF_C_PORT_RESET:
2666                         sc->sc_isreset = 0;
2667                         err = USB_ERR_NORMAL_COMPLETION;
2668                         goto done;
2669                 case UHF_C_PORT_SUSPEND:
2670                         sc->sc_isresumed &= ~(1 << index);
2671                         break;
2672                 case UHF_PORT_CONNECTION:
2673                 case UHF_PORT_OVER_CURRENT:
2674                 case UHF_PORT_POWER:
2675                 case UHF_PORT_LOW_SPEED:
2676                 default:
2677                         err = USB_ERR_IOERROR;
2678                         goto done;
2679                 }
2680                 break;
2681         case C(UR_GET_BUS_STATE, UT_READ_CLASS_OTHER):
2682                 if (index == 1)
2683                         port = UHCI_PORTSC1;
2684                 else if (index == 2)
2685                         port = UHCI_PORTSC2;
2686                 else {
2687                         err = USB_ERR_IOERROR;
2688                         goto done;
2689                 }
2690                 len = 1;
2691                 sc->sc_hub_desc.temp[0] =
2692                     ((UREAD2(sc, port) & UHCI_PORTSC_LS) >>
2693                     UHCI_PORTSC_LS_SHIFT);
2694                 break;
2695         case C(UR_GET_DESCRIPTOR, UT_READ_CLASS_DEVICE):
2696                 if ((value & 0xff) != 0) {
2697                         err = USB_ERR_IOERROR;
2698                         goto done;
2699                 }
2700                 len = sizeof(uhci_hubd_piix);
2701                 ptr = (const void *)&uhci_hubd_piix;
2702                 break;
2703         case C(UR_GET_STATUS, UT_READ_CLASS_DEVICE):
2704                 len = 16;
2705                 bzero(sc->sc_hub_desc.temp, 16);
2706                 break;
2707         case C(UR_GET_STATUS, UT_READ_CLASS_OTHER):
2708                 if (index == 1)
2709                         port = UHCI_PORTSC1;
2710                 else if (index == 2)
2711                         port = UHCI_PORTSC2;
2712                 else {
2713                         err = USB_ERR_IOERROR;
2714                         goto done;
2715                 }
2716                 x = UREAD2(sc, port);
2717                 status = change = 0;
2718                 if (x & UHCI_PORTSC_CCS)
2719                         status |= UPS_CURRENT_CONNECT_STATUS;
2720                 if (x & UHCI_PORTSC_CSC)
2721                         change |= UPS_C_CONNECT_STATUS;
2722                 if (x & UHCI_PORTSC_PE)
2723                         status |= UPS_PORT_ENABLED;
2724                 if (x & UHCI_PORTSC_POEDC)
2725                         change |= UPS_C_PORT_ENABLED;
2726                 if (x & UHCI_PORTSC_OCI)
2727                         status |= UPS_OVERCURRENT_INDICATOR;
2728                 if (x & UHCI_PORTSC_OCIC)
2729                         change |= UPS_C_OVERCURRENT_INDICATOR;
2730                 if (x & UHCI_PORTSC_LSDA)
2731                         status |= UPS_LOW_SPEED;
2732                 if ((x & UHCI_PORTSC_PE) && (x & UHCI_PORTSC_RD)) {
2733                         /* need to do a write back */
2734                         UWRITE2(sc, port, URWMASK(x));
2735
2736                         /* wait 20ms for resume sequence to complete */
2737                         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 50);
2738
2739                         /* clear suspend and resume detect */
2740                         UWRITE2(sc, port, URWMASK(x) & ~(UHCI_PORTSC_RD |
2741                             UHCI_PORTSC_SUSP));
2742
2743                         /* wait a little bit */
2744                         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 500);
2745
2746                         sc->sc_isresumed |= (1 << index);
2747
2748                 } else if (x & UHCI_PORTSC_SUSP) {
2749                         status |= UPS_SUSPEND;
2750                 }
2751                 status |= UPS_PORT_POWER;
2752                 if (sc->sc_isresumed & (1 << index))
2753                         change |= UPS_C_SUSPEND;
2754                 if (sc->sc_isreset)
2755                         change |= UPS_C_PORT_RESET;
2756                 USETW(sc->sc_hub_desc.ps.wPortStatus, status);
2757                 USETW(sc->sc_hub_desc.ps.wPortChange, change);
2758                 len = sizeof(sc->sc_hub_desc.ps);
2759                 break;
2760         case C(UR_SET_DESCRIPTOR, UT_WRITE_CLASS_DEVICE):
2761                 err = USB_ERR_IOERROR;
2762                 goto done;
2763         case C(UR_SET_FEATURE, UT_WRITE_CLASS_DEVICE):
2764                 break;
2765         case C(UR_SET_FEATURE, UT_WRITE_CLASS_OTHER):
2766                 if (index == 1)
2767                         port = UHCI_PORTSC1;
2768                 else if (index == 2)
2769                         port = UHCI_PORTSC2;
2770                 else {
2771                         err = USB_ERR_IOERROR;
2772                         goto done;
2773                 }
2774                 switch (value) {
2775                 case UHF_PORT_ENABLE:
2776                         x = URWMASK(UREAD2(sc, port));
2777                         UWRITE2(sc, port, x | UHCI_PORTSC_PE);
2778                         break;
2779                 case UHF_PORT_SUSPEND:
2780                         x = URWMASK(UREAD2(sc, port));
2781                         UWRITE2(sc, port, x | UHCI_PORTSC_SUSP);
2782                         break;
2783                 case UHF_PORT_RESET:
2784                         err = uhci_portreset(sc, index);
2785                         goto done;
2786                 case UHF_PORT_POWER:
2787                         /* pretend we turned on power */
2788                         err = USB_ERR_NORMAL_COMPLETION;
2789                         goto done;
2790                 case UHF_C_PORT_CONNECTION:
2791                 case UHF_C_PORT_ENABLE:
2792                 case UHF_C_PORT_OVER_CURRENT:
2793                 case UHF_PORT_CONNECTION:
2794                 case UHF_PORT_OVER_CURRENT:
2795                 case UHF_PORT_LOW_SPEED:
2796                 case UHF_C_PORT_SUSPEND:
2797                 case UHF_C_PORT_RESET:
2798                 default:
2799                         err = USB_ERR_IOERROR;
2800                         goto done;
2801                 }
2802                 break;
2803         default:
2804                 err = USB_ERR_IOERROR;
2805                 goto done;
2806         }
2807 done:
2808         *plength = len;
2809         *pptr = ptr;
2810         return (err);
2811 }
2812
2813 /*
2814  * This routine is executed periodically and simulates interrupts from
2815  * the root controller interrupt pipe for port status change:
2816  */
2817 static void
2818 uhci_root_intr(uhci_softc_t *sc)
2819 {
2820         DPRINTFN(21, "\n");
2821
2822         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
2823
2824         sc->sc_hub_idata[0] = 0;
2825
2826         if (UREAD2(sc, UHCI_PORTSC1) & (UHCI_PORTSC_CSC |
2827             UHCI_PORTSC_OCIC | UHCI_PORTSC_RD)) {
2828                 sc->sc_hub_idata[0] |= 1 << 1;
2829         }
2830         if (UREAD2(sc, UHCI_PORTSC2) & (UHCI_PORTSC_CSC |
2831             UHCI_PORTSC_OCIC | UHCI_PORTSC_RD)) {
2832                 sc->sc_hub_idata[0] |= 1 << 2;
2833         }
2834
2835         /* restart timer */
2836         usb_callout_reset(&sc->sc_root_intr, hz,
2837             (void *)&uhci_root_intr, sc);
2838
2839         if (sc->sc_hub_idata[0] != 0) {
2840                 uhub_root_intr(&sc->sc_bus, sc->sc_hub_idata,
2841                     sizeof(sc->sc_hub_idata));
2842         }
2843 }
2844
2845 static void
2846 uhci_xfer_setup(struct usb_setup_params *parm)
2847 {
2848         struct usb_page_search page_info;
2849         struct usb_page_cache *pc;
2850         uhci_softc_t *sc;
2851         struct usb_xfer *xfer;
2852         void *last_obj;
2853         uint32_t ntd;
2854         uint32_t nqh;
2855         uint32_t nfixup;
2856         uint32_t n;
2857         uint16_t align;
2858
2859         sc = UHCI_BUS2SC(parm->udev->bus);
2860         xfer = parm->curr_xfer;
2861
2862         parm->hc_max_packet_size = 0x500;
2863         parm->hc_max_packet_count = 1;
2864         parm->hc_max_frame_size = 0x500;
2865
2866         /*
2867          * compute ntd and nqh
2868          */
2869         if (parm->methods == &uhci_device_ctrl_methods) {
2870                 xfer->flags_int.bdma_enable = 1;
2871                 xfer->flags_int.bdma_no_post_sync = 1;
2872
2873                 usbd_transfer_setup_sub(parm);
2874
2875                 /* see EHCI HC driver for proof of "ntd" formula */
2876
2877                 nqh = 1;
2878                 ntd = ((2 * xfer->nframes) + 1  /* STATUS */
2879                     + (xfer->max_data_length / xfer->max_frame_size));
2880
2881         } else if (parm->methods == &uhci_device_bulk_methods) {
2882                 xfer->flags_int.bdma_enable = 1;
2883                 xfer->flags_int.bdma_no_post_sync = 1;
2884
2885                 usbd_transfer_setup_sub(parm);
2886
2887                 nqh = 1;
2888                 ntd = ((2 * xfer->nframes)
2889                     + (xfer->max_data_length / xfer->max_frame_size));
2890
2891         } else if (parm->methods == &uhci_device_intr_methods) {
2892                 xfer->flags_int.bdma_enable = 1;
2893                 xfer->flags_int.bdma_no_post_sync = 1;
2894
2895                 usbd_transfer_setup_sub(parm);
2896
2897                 nqh = 1;
2898                 ntd = ((2 * xfer->nframes)
2899                     + (xfer->max_data_length / xfer->max_frame_size));
2900
2901         } else if (parm->methods == &uhci_device_isoc_methods) {
2902                 xfer->flags_int.bdma_enable = 1;
2903                 xfer->flags_int.bdma_no_post_sync = 1;
2904
2905                 usbd_transfer_setup_sub(parm);
2906
2907                 nqh = 0;
2908                 ntd = xfer->nframes;
2909
2910         } else {
2911
2912                 usbd_transfer_setup_sub(parm);
2913
2914                 nqh = 0;
2915                 ntd = 0;
2916         }
2917
2918         if (parm->err) {
2919                 return;
2920         }
2921         /*
2922          * NOTE: the UHCI controller requires that
2923          * every packet must be contiguous on
2924          * the same USB memory page !
2925          */
2926         nfixup = (parm->bufsize / USB_PAGE_SIZE) + 1;
2927
2928         /*
2929          * Compute a suitable power of two alignment
2930          * for our "max_frame_size" fixup buffer(s):
2931          */
2932         align = xfer->max_frame_size;
2933         n = 0;
2934         while (align) {
2935                 align >>= 1;
2936                 n++;
2937         }
2938
2939         /* check for power of two */
2940         if (!(xfer->max_frame_size &
2941             (xfer->max_frame_size - 1))) {
2942                 n--;
2943         }
2944         /*
2945          * We don't allow alignments of
2946          * less than 8 bytes:
2947          *
2948          * NOTE: Allocating using an aligment
2949          * of 1 byte has special meaning!
2950          */
2951         if (n < 3) {
2952                 n = 3;
2953         }
2954         align = (1 << n);
2955
2956         if (usbd_transfer_setup_sub_malloc(
2957             parm, &pc, xfer->max_frame_size,
2958             align, nfixup)) {
2959                 parm->err = USB_ERR_NOMEM;
2960                 return;
2961         }
2962         xfer->buf_fixup = pc;
2963
2964 alloc_dma_set:
2965
2966         if (parm->err) {
2967                 return;
2968         }
2969         last_obj = NULL;
2970
2971         if (usbd_transfer_setup_sub_malloc(
2972             parm, &pc, sizeof(uhci_td_t),
2973             UHCI_TD_ALIGN, ntd)) {
2974                 parm->err = USB_ERR_NOMEM;
2975                 return;
2976         }
2977         if (parm->buf) {
2978                 for (n = 0; n != ntd; n++) {
2979                         uhci_td_t *td;
2980
2981                         usbd_get_page(pc + n, 0, &page_info);
2982
2983                         td = page_info.buffer;
2984
2985                         /* init TD */
2986                         if ((parm->methods == &uhci_device_bulk_methods) ||
2987                             (parm->methods == &uhci_device_ctrl_methods) ||
2988                             (parm->methods == &uhci_device_intr_methods)) {
2989                                 /* set depth first bit */
2990                                 td->td_self = htole32(page_info.physaddr |
2991                                     UHCI_PTR_TD | UHCI_PTR_VF);
2992                         } else {
2993                                 td->td_self = htole32(page_info.physaddr |
2994                                     UHCI_PTR_TD);
2995                         }
2996
2997                         td->obj_next = last_obj;
2998                         td->page_cache = pc + n;
2999
3000                         last_obj = td;
3001
3002                         usb_pc_cpu_flush(pc + n);
3003                 }
3004         }
3005         xfer->td_start[xfer->flags_int.curr_dma_set] = last_obj;
3006
3007         last_obj = NULL;
3008
3009         if (usbd_transfer_setup_sub_malloc(
3010             parm, &pc, sizeof(uhci_qh_t),
3011             UHCI_QH_ALIGN, nqh)) {
3012                 parm->err = USB_ERR_NOMEM;
3013                 return;
3014         }
3015         if (parm->buf) {
3016                 for (n = 0; n != nqh; n++) {
3017                         uhci_qh_t *qh;
3018
3019                         usbd_get_page(pc + n, 0, &page_info);
3020
3021                         qh = page_info.buffer;
3022
3023                         /* init QH */
3024                         qh->qh_self = htole32(page_info.physaddr | UHCI_PTR_QH);
3025                         qh->obj_next = last_obj;
3026                         qh->page_cache = pc + n;
3027
3028                         last_obj = qh;
3029
3030                         usb_pc_cpu_flush(pc + n);
3031                 }
3032         }
3033         xfer->qh_start[xfer->flags_int.curr_dma_set] = last_obj;
3034
3035         if (!xfer->flags_int.curr_dma_set) {
3036                 xfer->flags_int.curr_dma_set = 1;
3037                 goto alloc_dma_set;
3038         }
3039 }
3040
3041 static void
3042 uhci_ep_init(struct usb_device *udev, struct usb_endpoint_descriptor *edesc,
3043     struct usb_endpoint *ep)
3044 {
3045         uhci_softc_t *sc = UHCI_BUS2SC(udev->bus);
3046
3047         DPRINTFN(2, "endpoint=%p, addr=%d, endpt=%d, mode=%d (%d)\n",
3048             ep, udev->address,
3049             edesc->bEndpointAddress, udev->flags.usb_mode,
3050             sc->sc_addr);
3051
3052         if (udev->flags.usb_mode != USB_MODE_HOST) {
3053                 /* not supported */
3054                 return;
3055         }
3056         if (udev->device_index != sc->sc_addr) {
3057                 switch (edesc->bmAttributes & UE_XFERTYPE) {
3058                 case UE_CONTROL:
3059                         ep->methods = &uhci_device_ctrl_methods;
3060                         break;
3061                 case UE_INTERRUPT:
3062                         ep->methods = &uhci_device_intr_methods;
3063                         break;
3064                 case UE_ISOCHRONOUS:
3065                         if (udev->speed == USB_SPEED_FULL) {
3066                                 ep->methods = &uhci_device_isoc_methods;
3067                         }
3068                         break;
3069                 case UE_BULK:
3070                         ep->methods = &uhci_device_bulk_methods;
3071                         break;
3072                 default:
3073                         /* do nothing */
3074                         break;
3075                 }
3076         }
3077 }
3078
3079 static void
3080 uhci_xfer_unsetup(struct usb_xfer *xfer)
3081 {
3082         return;
3083 }
3084
3085 static void
3086 uhci_get_dma_delay(struct usb_device *udev, uint32_t *pus)
3087 {
3088         /*
3089          * Wait until hardware has finished any possible use of the
3090          * transfer descriptor(s) and QH
3091          */
3092         *pus = (1125);                  /* microseconds */
3093 }
3094
3095 static void
3096 uhci_device_resume(struct usb_device *udev)
3097 {
3098         struct uhci_softc *sc = UHCI_BUS2SC(udev->bus);
3099         struct usb_xfer *xfer;
3100         struct usb_pipe_methods *methods;
3101         uhci_qh_t *qh;
3102
3103         DPRINTF("\n");
3104
3105         USB_BUS_LOCK(udev->bus);
3106
3107         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
3108
3109                 if (xfer->xroot->udev == udev) {
3110
3111                         methods = xfer->endpoint->methods;
3112                         qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
3113
3114                         if (methods == &uhci_device_bulk_methods) {
3115                                 UHCI_APPEND_QH(qh, sc->sc_bulk_p_last);
3116                                 uhci_add_loop(sc);
3117                                 xfer->flags_int.bandwidth_reclaimed = 1;
3118                         }
3119                         if (methods == &uhci_device_ctrl_methods) {
3120                                 if (xfer->xroot->udev->speed == USB_SPEED_LOW) {
3121                                         UHCI_APPEND_QH(qh, sc->sc_ls_ctl_p_last);
3122                                 } else {
3123                                         UHCI_APPEND_QH(qh, sc->sc_fs_ctl_p_last);
3124                                 }
3125                         }
3126                         if (methods == &uhci_device_intr_methods) {
3127                                 UHCI_APPEND_QH(qh, sc->sc_intr_p_last[xfer->qh_pos]);
3128                         }
3129                 }
3130         }
3131
3132         USB_BUS_UNLOCK(udev->bus);
3133
3134         return;
3135 }
3136
3137 static void
3138 uhci_device_suspend(struct usb_device *udev)
3139 {
3140         struct uhci_softc *sc = UHCI_BUS2SC(udev->bus);
3141         struct usb_xfer *xfer;
3142         struct usb_pipe_methods *methods;
3143         uhci_qh_t *qh;
3144
3145         DPRINTF("\n");
3146
3147         USB_BUS_LOCK(udev->bus);
3148
3149         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
3150
3151                 if (xfer->xroot->udev == udev) {
3152
3153                         methods = xfer->endpoint->methods;
3154                         qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
3155
3156                         if (xfer->flags_int.bandwidth_reclaimed) {
3157                                 xfer->flags_int.bandwidth_reclaimed = 0;
3158                                 uhci_rem_loop(sc);
3159                         }
3160                         if (methods == &uhci_device_bulk_methods) {
3161                                 UHCI_REMOVE_QH(qh, sc->sc_bulk_p_last);
3162                         }
3163                         if (methods == &uhci_device_ctrl_methods) {
3164                                 if (xfer->xroot->udev->speed == USB_SPEED_LOW) {
3165                                         UHCI_REMOVE_QH(qh, sc->sc_ls_ctl_p_last);
3166                                 } else {
3167                                         UHCI_REMOVE_QH(qh, sc->sc_fs_ctl_p_last);
3168                                 }
3169                         }
3170                         if (methods == &uhci_device_intr_methods) {
3171                                 UHCI_REMOVE_QH(qh, sc->sc_intr_p_last[xfer->qh_pos]);
3172                         }
3173                 }
3174         }
3175
3176         USB_BUS_UNLOCK(udev->bus);
3177
3178         return;
3179 }
3180
3181 static void
3182 uhci_set_hw_power(struct usb_bus *bus)
3183 {
3184         struct uhci_softc *sc = UHCI_BUS2SC(bus);
3185         uint32_t flags;
3186
3187         DPRINTF("\n");
3188
3189         USB_BUS_LOCK(bus);
3190
3191         flags = bus->hw_power_state;
3192
3193         /*
3194          * WARNING: Some FULL speed USB devices require periodic SOF
3195          * messages! If any USB devices are connected through the
3196          * UHCI, power save will be disabled!
3197          */
3198         if (flags & (USB_HW_POWER_CONTROL |
3199             USB_HW_POWER_NON_ROOT_HUB |
3200             USB_HW_POWER_BULK |
3201             USB_HW_POWER_INTERRUPT |
3202             USB_HW_POWER_ISOC)) {
3203                 DPRINTF("Some USB transfer is "
3204                     "active on unit %u.\n",
3205                     device_get_unit(sc->sc_bus.bdev));
3206                 uhci_restart(sc);
3207         } else {
3208                 DPRINTF("Power save on unit %u.\n",
3209                     device_get_unit(sc->sc_bus.bdev));
3210                 UHCICMD(sc, UHCI_CMD_MAXP);
3211         }
3212
3213         USB_BUS_UNLOCK(bus);
3214
3215         return;
3216 }
3217
3218
3219 struct usb_bus_methods uhci_bus_methods =
3220 {
3221         .endpoint_init = uhci_ep_init,
3222         .xfer_setup = uhci_xfer_setup,
3223         .xfer_unsetup = uhci_xfer_unsetup,
3224         .get_dma_delay = uhci_get_dma_delay,
3225         .device_resume = uhci_device_resume,
3226         .device_suspend = uhci_device_suspend,
3227         .set_hw_power = uhci_set_hw_power,
3228         .roothub_exec = uhci_roothub_exec,
3229         .xfer_poll = uhci_do_poll,
3230 };