sys/dev/usb/controller/uhci.c

   1 /*-
   2  * Copyright (c) 2008 Hans Petter Selasky. All rights reserved.
   3  * Copyright (c) 1998 The NetBSD Foundation, Inc. All rights reserved.
   4  * Copyright (c) 1998 Lennart Augustsson. All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  */
  27
  28 #include <sys/cdefs.h>
  29 __FBSDID("$FreeBSD$");
  30
  31 /*
  32  * USB Universal Host Controller driver.
  33  * Handles e.g. PIIX3 and PIIX4.
  34  *
  35  * UHCI spec: http://developer.intel.com/design/USB/UHCI11D.htm
  36  * USB spec:  http://www.usb.org/developers/docs/usbspec.zip
  37  * PIIXn spec: ftp://download.intel.com/design/intarch/datashts/29055002.pdf
  38  *             ftp://download.intel.com/design/intarch/datashts/29056201.pdf
  39  */
  40
  41 #include <sys/stdint.h>
  42 #include <sys/stddef.h>
  43 #include <sys/param.h>
  44 #include <sys/queue.h>
  45 #include <sys/types.h>
  46 #include <sys/systm.h>
  47 #include <sys/kernel.h>
  48 #include <sys/bus.h>
  49 #include <sys/linker_set.h>
  50 #include <sys/module.h>
  51 #include <sys/lock.h>
  52 #include <sys/mutex.h>
  53 #include <sys/condvar.h>
  54 #include <sys/sysctl.h>
  55 #include <sys/sx.h>
  56 #include <sys/unistd.h>
  57 #include <sys/callout.h>
  58 #include <sys/malloc.h>
  59 #include <sys/priv.h>
  60
  61 #include <dev/usb/usb.h>
  62 #include <dev/usb/usbdi.h>
  63
  64 #define USB_DEBUG_VAR uhcidebug
  65
  66 #include <dev/usb/usb_core.h>
  67 #include <dev/usb/usb_debug.h>
  68 #include <dev/usb/usb_busdma.h>
  69 #include <dev/usb/usb_process.h>
  70 #include <dev/usb/usb_transfer.h>
  71 #include <dev/usb/usb_device.h>
  72 #include <dev/usb/usb_hub.h>
  73 #include <dev/usb/usb_util.h>
  74
  75 #include <dev/usb/usb_controller.h>
  76 #include <dev/usb/usb_bus.h>
  77 #include <dev/usb/controller/uhci.h>
  78 #include <dev/usb/controller/uhcireg.h>
  79
  80 #define alt_next next
  81 #define UHCI_BUS2SC(bus) \
  82    ((uhci_softc_t *)(((uint8_t *)(bus)) - \
  83     ((uint8_t *)&(((uhci_softc_t *)0)->sc_bus))))
  84
  85 #ifdef USB_DEBUG
  86 static int uhcidebug = 0;
  87 static int uhcinoloop = 0;
  88
  89 SYSCTL_NODE(_hw_usb, OID_AUTO, uhci, CTLFLAG_RW, 0, "USB uhci");
  90 SYSCTL_INT(_hw_usb_uhci, OID_AUTO, debug, CTLFLAG_RW,
  91     &uhcidebug, 0, "uhci debug level");
  92 SYSCTL_INT(_hw_usb_uhci, OID_AUTO, loop, CTLFLAG_RW,
  93     &uhcinoloop, 0, "uhci noloop");
  94
  95 TUNABLE_INT("hw.usb.uhci.debug", &uhcidebug);
  96 TUNABLE_INT("hw.usb.uhci.loop", &uhcinoloop);
  97
  98 static void uhci_dumpregs(uhci_softc_t *sc);
  99 static void uhci_dump_tds(uhci_td_t *td);
 100
 101 #endif
 102
 103 #define UBARR(sc) bus_space_barrier((sc)->sc_io_tag, (sc)->sc_io_hdl, 0, (sc)->sc_io_size, \
 104                         BUS_SPACE_BARRIER_READ|BUS_SPACE_BARRIER_WRITE)
 105 #define UWRITE1(sc, r, x) \
 106  do { UBARR(sc); bus_space_write_1((sc)->sc_io_tag, (sc)->sc_io_hdl, (r), (x)); \
 107  } while (/*CONSTCOND*/0)
 108 #define UWRITE2(sc, r, x) \
 109  do { UBARR(sc); bus_space_write_2((sc)->sc_io_tag, (sc)->sc_io_hdl, (r), (x)); \
 110  } while (/*CONSTCOND*/0)
 111 #define UWRITE4(sc, r, x) \
 112  do { UBARR(sc); bus_space_write_4((sc)->sc_io_tag, (sc)->sc_io_hdl, (r), (x)); \
 113  } while (/*CONSTCOND*/0)
 114 #define UREAD1(sc, r) (UBARR(sc), bus_space_read_1((sc)->sc_io_tag, (sc)->sc_io_hdl, (r)))
 115 #define UREAD2(sc, r) (UBARR(sc), bus_space_read_2((sc)->sc_io_tag, (sc)->sc_io_hdl, (r)))
 116 #define UREAD4(sc, r) (UBARR(sc), bus_space_read_4((sc)->sc_io_tag, (sc)->sc_io_hdl, (r)))
 117
 118 #define UHCICMD(sc, cmd) UWRITE2(sc, UHCI_CMD, cmd)
 119 #define UHCISTS(sc) UREAD2(sc, UHCI_STS)
 120
 121 #define UHCI_RESET_TIMEOUT 100          /* ms, reset timeout */
 122
 123 #define UHCI_INTR_ENDPT 1
 124
 125 struct uhci_mem_layout {
 126
 127         struct usb_page_search buf_res;
 128         struct usb_page_search fix_res;
 129
 130         struct usb_page_cache *buf_pc;
 131         struct usb_page_cache *fix_pc;
 132
 133         uint32_t buf_offset;
 134
 135         uint16_t max_frame_size;
 136 };
 137
 138 struct uhci_std_temp {
 139
 140         struct uhci_mem_layout ml;
 141         uhci_td_t *td;
 142         uhci_td_t *td_next;
 143         uint32_t average;
 144         uint32_t td_status;
 145         uint32_t td_token;
 146         uint32_t len;
 147         uint16_t max_frame_size;
 148         uint8_t shortpkt;
 149         uint8_t setup_alt_next;
 150         uint8_t last_frame;
 151 };
 152
 153 extern struct usb_bus_methods uhci_bus_methods;
 154 extern struct usb_pipe_methods uhci_device_bulk_methods;
 155 extern struct usb_pipe_methods uhci_device_ctrl_methods;
 156 extern struct usb_pipe_methods uhci_device_intr_methods;
 157 extern struct usb_pipe_methods uhci_device_isoc_methods;
 158
 159 static uint8_t  uhci_restart(uhci_softc_t *sc);
 160 static void     uhci_do_poll(struct usb_bus *);
 161 static void     uhci_device_done(struct usb_xfer *, usb_error_t);
 162 static void     uhci_transfer_intr_enqueue(struct usb_xfer *);
 163 static void     uhci_timeout(void *);
 164 static uint8_t  uhci_check_transfer(struct usb_xfer *);
 165 static void     uhci_root_intr(uhci_softc_t *sc);
 166
 167 void
 168 uhci_iterate_hw_softc(struct usb_bus *bus, usb_bus_mem_sub_cb_t *cb)
 169 {
 170         struct uhci_softc *sc = UHCI_BUS2SC(bus);
 171         uint32_t i;
 172
 173         cb(bus, &sc->sc_hw.pframes_pc, &sc->sc_hw.pframes_pg,
 174             sizeof(uint32_t) * UHCI_FRAMELIST_COUNT, UHCI_FRAMELIST_ALIGN);
 175
 176         cb(bus, &sc->sc_hw.ls_ctl_start_pc, &sc->sc_hw.ls_ctl_start_pg,
 177             sizeof(uhci_qh_t), UHCI_QH_ALIGN);
 178
 179         cb(bus, &sc->sc_hw.fs_ctl_start_pc, &sc->sc_hw.fs_ctl_start_pg,
 180             sizeof(uhci_qh_t), UHCI_QH_ALIGN);
 181
 182         cb(bus, &sc->sc_hw.bulk_start_pc, &sc->sc_hw.bulk_start_pg,
 183             sizeof(uhci_qh_t), UHCI_QH_ALIGN);
 184
 185         cb(bus, &sc->sc_hw.last_qh_pc, &sc->sc_hw.last_qh_pg,
 186             sizeof(uhci_qh_t), UHCI_QH_ALIGN);
 187
 188         cb(bus, &sc->sc_hw.last_td_pc, &sc->sc_hw.last_td_pg,
 189             sizeof(uhci_td_t), UHCI_TD_ALIGN);
 190
 191         for (i = 0; i != UHCI_VFRAMELIST_COUNT; i++) {
 192                 cb(bus, sc->sc_hw.isoc_start_pc + i,
 193                     sc->sc_hw.isoc_start_pg + i,
 194                     sizeof(uhci_td_t), UHCI_TD_ALIGN);
 195         }
 196
 197         for (i = 0; i != UHCI_IFRAMELIST_COUNT; i++) {
 198                 cb(bus, sc->sc_hw.intr_start_pc + i,
 199                     sc->sc_hw.intr_start_pg + i,
 200                     sizeof(uhci_qh_t), UHCI_QH_ALIGN);
 201         }
 202 }
 203
 204 static void
 205 uhci_mem_layout_init(struct uhci_mem_layout *ml, struct usb_xfer *xfer)
 206 {
 207         ml->buf_pc = xfer->frbuffers + 0;
 208         ml->fix_pc = xfer->buf_fixup;
 209
 210         ml->buf_offset = 0;
 211
 212         ml->max_frame_size = xfer->max_frame_size;
 213 }
 214
 215 static void
 216 uhci_mem_layout_fixup(struct uhci_mem_layout *ml, struct uhci_td *td)
 217 {
 218         usbd_get_page(ml->buf_pc, ml->buf_offset, &ml->buf_res);
 219
 220         if (ml->buf_res.length < td->len) {
 221
 222                 /* need to do a fixup */
 223
 224                 usbd_get_page(ml->fix_pc, 0, &ml->fix_res);
 225
 226                 td->td_buffer = htole32(ml->fix_res.physaddr);
 227
 228                 /*
 229                  * The UHCI driver cannot handle
 230                  * page crossings, so a fixup is
 231                  * needed:
 232                  *
 233                  *  +----+----+ - - -
 234                  *  | YYY|Y   |
 235                  *  +----+----+ - - -
 236                  *     \    \
 237                  *      \    \
 238                  *       +----+
 239                  *       |YYYY|  (fixup)
 240                  *       +----+
 241                  */
 242
 243                 if ((td->td_token & htole32(UHCI_TD_PID)) ==
 244                     htole32(UHCI_TD_PID_IN)) {
 245                         td->fix_pc = ml->fix_pc;
 246                         usb_pc_cpu_invalidate(ml->fix_pc);
 247
 248                 } else {
 249                         td->fix_pc = NULL;
 250
 251                         /* copy data to fixup location */
 252
 253                         usbd_copy_out(ml->buf_pc, ml->buf_offset,
 254                             ml->fix_res.buffer, td->len);
 255
 256                         usb_pc_cpu_flush(ml->fix_pc);
 257                 }
 258
 259                 /* prepare next fixup */
 260
 261                 ml->fix_pc++;
 262
 263         } else {
 264
 265                 td->td_buffer = htole32(ml->buf_res.physaddr);
 266                 td->fix_pc = NULL;
 267         }
 268
 269         /* prepare next data location */
 270
 271         ml->buf_offset += td->len;
 272 }
 273
 274 /*
 275  * Return values:
 276  * 0: Success
 277  * Else: Failure
 278  */
 279 static uint8_t
 280 uhci_restart(uhci_softc_t *sc)
 281 {
 282         struct usb_page_search buf_res;
 283
 284         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
 285
 286         if (UREAD2(sc, UHCI_CMD) & UHCI_CMD_RS) {
 287                 DPRINTFN(2, "Already started\n");
 288                 return (0);
 289         }
 290
 291         DPRINTFN(2, "Restarting\n");
 292
 293         usbd_get_page(&sc->sc_hw.pframes_pc, 0, &buf_res);
 294
 295         /* Reload fresh base address */
 296         UWRITE4(sc, UHCI_FLBASEADDR, buf_res.physaddr);
 297
 298         /*
 299          * Assume 64 byte packets at frame end and start HC controller:
 300          */
 301         UHCICMD(sc, (UHCI_CMD_MAXP | UHCI_CMD_RS));
 302
 303         /* wait 10 milliseconds */
 304
 305         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 100);
 306
 307         /* check that controller has started */
 308
 309         if (UREAD2(sc, UHCI_STS) & UHCI_STS_HCH) {
 310                 DPRINTFN(2, "Failed\n");
 311                 return (1);
 312         }
 313         return (0);
 314 }
 315
 316 void
 317 uhci_reset(uhci_softc_t *sc)
 318 {
 319         uint16_t n;
 320
 321         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
 322
 323         DPRINTF("resetting the HC\n");
 324
 325         /* disable interrupts */
 326
 327         UWRITE2(sc, UHCI_INTR, 0);
 328
 329         /* global reset */
 330
 331         UHCICMD(sc, UHCI_CMD_GRESET);
 332
 333         /* wait */
 334
 335         usb_pause_mtx(&sc->sc_bus.bus_mtx,
 336             USB_MS_TO_TICKS(USB_BUS_RESET_DELAY));
 337
 338         /* terminate all transfers */
 339
 340         UHCICMD(sc, UHCI_CMD_HCRESET);
 341
 342         /* the reset bit goes low when the controller is done */
 343
 344         n = UHCI_RESET_TIMEOUT;
 345         while (n--) {
 346                 /* wait one millisecond */
 347
 348                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 1000);
 349
 350                 if (!(UREAD2(sc, UHCI_CMD) & UHCI_CMD_HCRESET)) {
 351                         goto done_1;
 352                 }
 353         }
 354
 355         device_printf(sc->sc_bus.bdev,
 356             "controller did not reset\n");
 357
 358 done_1:
 359
 360         n = 10;
 361         while (n--) {
 362                 /* wait one millisecond */
 363
 364                 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 1000);
 365
 366                 /* check if HC is stopped */
 367                 if (UREAD2(sc, UHCI_STS) & UHCI_STS_HCH) {
 368                         goto done_2;
 369                 }
 370         }
 371
 372         device_printf(sc->sc_bus.bdev,
 373             "controller did not stop\n");
 374
 375 done_2:
 376
 377         /* reload the configuration */
 378         UWRITE2(sc, UHCI_FRNUM, sc->sc_saved_frnum);
 379         UWRITE1(sc, UHCI_SOF, sc->sc_saved_sof);
 380
 381         USB_BUS_UNLOCK(&sc->sc_bus);
 382
 383         /* stop root interrupt */
 384         usb_callout_drain(&sc->sc_root_intr);
 385
 386         USB_BUS_LOCK(&sc->sc_bus);
 387 }
 388
 389 static void
 390 uhci_start(uhci_softc_t *sc)
 391 {
 392         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
 393
 394         DPRINTFN(2, "enabling\n");
 395
 396         /* enable interrupts */
 397
 398         UWRITE2(sc, UHCI_INTR,
 399             (UHCI_INTR_TOCRCIE |
 400             UHCI_INTR_RIE |
 401             UHCI_INTR_IOCE |
 402             UHCI_INTR_SPIE));
 403
 404         if (uhci_restart(sc)) {
 405                 device_printf(sc->sc_bus.bdev,
 406                     "cannot start HC controller\n");
 407         }
 408
 409         /* start root interrupt */
 410         uhci_root_intr(sc);
 411 }
 412
 413 static struct uhci_qh *
 414 uhci_init_qh(struct usb_page_cache *pc)
 415 {
 416         struct usb_page_search buf_res;
 417         struct uhci_qh *qh;
 418
 419         usbd_get_page(pc, 0, &buf_res);
 420
 421         qh = buf_res.buffer;
 422
 423         qh->qh_self =
 424             htole32(buf_res.physaddr) |
 425             htole32(UHCI_PTR_QH);
 426
 427         qh->page_cache = pc;
 428
 429         return (qh);
 430 }
 431
 432 static struct uhci_td *
 433 uhci_init_td(struct usb_page_cache *pc)
 434 {
 435         struct usb_page_search buf_res;
 436         struct uhci_td *td;
 437
 438         usbd_get_page(pc, 0, &buf_res);
 439
 440         td = buf_res.buffer;
 441
 442         td->td_self =
 443             htole32(buf_res.physaddr) |
 444             htole32(UHCI_PTR_TD);
 445
 446         td->page_cache = pc;
 447
 448         return (td);
 449 }
 450
 451 usb_error_t
 452 uhci_init(uhci_softc_t *sc)
 453 {
 454         uint16_t bit;
 455         uint16_t x;
 456         uint16_t y;
 457
 458         DPRINTF("start\n");
 459
 460         usb_callout_init_mtx(&sc->sc_root_intr, &sc->sc_bus.bus_mtx, 0);
 461
 462 #ifdef USB_DEBUG
 463         if (uhcidebug > 2) {
 464                 uhci_dumpregs(sc);
 465         }
 466 #endif
 467         sc->sc_saved_sof = 0x40;        /* default value */
 468         sc->sc_saved_frnum = 0;         /* default frame number */
 469
 470         /*
 471          * Setup QH's
 472          */
 473         sc->sc_ls_ctl_p_last =
 474             uhci_init_qh(&sc->sc_hw.ls_ctl_start_pc);
 475
 476         sc->sc_fs_ctl_p_last =
 477             uhci_init_qh(&sc->sc_hw.fs_ctl_start_pc);
 478
 479         sc->sc_bulk_p_last =
 480             uhci_init_qh(&sc->sc_hw.bulk_start_pc);
 481 #if 0
 482         sc->sc_reclaim_qh_p =
 483             sc->sc_fs_ctl_p_last;
 484 #else
 485         /* setup reclaim looping point */
 486         sc->sc_reclaim_qh_p =
 487             sc->sc_bulk_p_last;
 488 #endif
 489
 490         sc->sc_last_qh_p =
 491             uhci_init_qh(&sc->sc_hw.last_qh_pc);
 492
 493         sc->sc_last_td_p =
 494             uhci_init_td(&sc->sc_hw.last_td_pc);
 495
 496         for (x = 0; x != UHCI_VFRAMELIST_COUNT; x++) {
 497                 sc->sc_isoc_p_last[x] =
 498                     uhci_init_td(sc->sc_hw.isoc_start_pc + x);
 499         }
 500
 501         for (x = 0; x != UHCI_IFRAMELIST_COUNT; x++) {
 502                 sc->sc_intr_p_last[x] =
 503                     uhci_init_qh(sc->sc_hw.intr_start_pc + x);
 504         }
 505
 506         /*
 507          * the QHs are arranged to give poll intervals that are
 508          * powers of 2 times 1ms
 509          */
 510         bit = UHCI_IFRAMELIST_COUNT / 2;
 511         while (bit) {
 512                 x = bit;
 513                 while (x & bit) {
 514                         uhci_qh_t *qh_x;
 515                         uhci_qh_t *qh_y;
 516
 517                         y = (x ^ bit) | (bit / 2);
 518
 519                         /*
 520                          * the next QH has half the poll interval
 521                          */
 522                         qh_x = sc->sc_intr_p_last[x];
 523                         qh_y = sc->sc_intr_p_last[y];
 524
 525                         qh_x->h_next = NULL;
 526                         qh_x->qh_h_next = qh_y->qh_self;
 527                         qh_x->e_next = NULL;
 528                         qh_x->qh_e_next = htole32(UHCI_PTR_T);
 529                         x++;
 530                 }
 531                 bit >>= 1;
 532         }
 533
 534         if (1) {
 535                 uhci_qh_t *qh_ls;
 536                 uhci_qh_t *qh_intr;
 537
 538                 qh_ls = sc->sc_ls_ctl_p_last;
 539                 qh_intr = sc->sc_intr_p_last[0];
 540
 541                 /* start QH for interrupt traffic */
 542                 qh_intr->h_next = qh_ls;
 543                 qh_intr->qh_h_next = qh_ls->qh_self;
 544                 qh_intr->e_next = 0;
 545                 qh_intr->qh_e_next = htole32(UHCI_PTR_T);
 546         }
 547         for (x = 0; x != UHCI_VFRAMELIST_COUNT; x++) {
 548
 549                 uhci_td_t *td_x;
 550                 uhci_qh_t *qh_intr;
 551
 552                 td_x = sc->sc_isoc_p_last[x];
 553                 qh_intr = sc->sc_intr_p_last[x | (UHCI_IFRAMELIST_COUNT / 2)];
 554
 555                 /* start TD for isochronous traffic */
 556                 td_x->next = NULL;
 557                 td_x->td_next = qh_intr->qh_self;
 558                 td_x->td_status = htole32(UHCI_TD_IOS);
 559                 td_x->td_token = htole32(0);
 560                 td_x->td_buffer = htole32(0);
 561         }
 562
 563         if (1) {
 564                 uhci_qh_t *qh_ls;
 565                 uhci_qh_t *qh_fs;
 566
 567                 qh_ls = sc->sc_ls_ctl_p_last;
 568                 qh_fs = sc->sc_fs_ctl_p_last;
 569
 570                 /* start QH where low speed control traffic will be queued */
 571                 qh_ls->h_next = qh_fs;
 572                 qh_ls->qh_h_next = qh_fs->qh_self;
 573                 qh_ls->e_next = 0;
 574                 qh_ls->qh_e_next = htole32(UHCI_PTR_T);
 575         }
 576         if (1) {
 577                 uhci_qh_t *qh_ctl;
 578                 uhci_qh_t *qh_blk;
 579                 uhci_qh_t *qh_lst;
 580                 uhci_td_t *td_lst;
 581
 582                 qh_ctl = sc->sc_fs_ctl_p_last;
 583                 qh_blk = sc->sc_bulk_p_last;
 584
 585                 /* start QH where full speed control traffic will be queued */
 586                 qh_ctl->h_next = qh_blk;
 587                 qh_ctl->qh_h_next = qh_blk->qh_self;
 588                 qh_ctl->e_next = 0;
 589                 qh_ctl->qh_e_next = htole32(UHCI_PTR_T);
 590
 591                 qh_lst = sc->sc_last_qh_p;
 592
 593                 /* start QH where bulk traffic will be queued */
 594                 qh_blk->h_next = qh_lst;
 595                 qh_blk->qh_h_next = qh_lst->qh_self;
 596                 qh_blk->e_next = 0;
 597                 qh_blk->qh_e_next = htole32(UHCI_PTR_T);
 598
 599                 td_lst = sc->sc_last_td_p;
 600
 601                 /* end QH which is used for looping the QHs */
 602                 qh_lst->h_next = 0;
 603                 qh_lst->qh_h_next = htole32(UHCI_PTR_T);        /* end of QH chain */
 604                 qh_lst->e_next = td_lst;
 605                 qh_lst->qh_e_next = td_lst->td_self;
 606
 607                 /*
 608                  * end TD which hangs from the last QH, to avoid a bug in the PIIX
 609                  * that makes it run berserk otherwise
 610                  */
 611                 td_lst->next = 0;
 612                 td_lst->td_next = htole32(UHCI_PTR_T);
 613                 td_lst->td_status = htole32(0); /* inactive */
 614                 td_lst->td_token = htole32(0);
 615                 td_lst->td_buffer = htole32(0);
 616         }
 617         if (1) {
 618                 struct usb_page_search buf_res;
 619                 uint32_t *pframes;
 620
 621                 usbd_get_page(&sc->sc_hw.pframes_pc, 0, &buf_res);
 622
 623                 pframes = buf_res.buffer;
 624
 625
 626                 /*
 627                  * Setup UHCI framelist
 628                  *
 629                  * Execution order:
 630                  *
 631                  * pframes -> full speed isochronous -> interrupt QH's -> low
 632                  * speed control -> full speed control -> bulk transfers
 633                  *
 634                  */
 635
 636                 for (x = 0; x != UHCI_FRAMELIST_COUNT; x++) {
 637                         pframes[x] =
 638                             sc->sc_isoc_p_last[x % UHCI_VFRAMELIST_COUNT]->td_self;
 639                 }
 640         }
 641         /* flush all cache into memory */
 642
 643         usb_bus_mem_flush_all(&sc->sc_bus, &uhci_iterate_hw_softc);
 644
 645         /* set up the bus struct */
 646         sc->sc_bus.methods = &uhci_bus_methods;
 647
 648         USB_BUS_LOCK(&sc->sc_bus);
 649         /* reset the controller */
 650         uhci_reset(sc);
 651
 652         /* start the controller */
 653         uhci_start(sc);
 654         USB_BUS_UNLOCK(&sc->sc_bus);
 655
 656         /* catch lost interrupts */
 657         uhci_do_poll(&sc->sc_bus);
 658
 659         return (0);
 660 }
 661
 662 /* NOTE: suspend/resume is called from
 663  * interrupt context and cannot sleep!
 664  */
 665
 666 void
 667 uhci_suspend(uhci_softc_t *sc)
 668 {
 669         USB_BUS_LOCK(&sc->sc_bus);
 670
 671 #ifdef USB_DEBUG
 672         if (uhcidebug > 2) {
 673                 uhci_dumpregs(sc);
 674         }
 675 #endif
 676         /* save some state if BIOS doesn't */
 677
 678         sc->sc_saved_frnum = UREAD2(sc, UHCI_FRNUM);
 679         sc->sc_saved_sof = UREAD1(sc, UHCI_SOF);
 680
 681         /* stop the controller */
 682
 683         uhci_reset(sc);
 684
 685         /* enter global suspend */
 686
 687         UHCICMD(sc, UHCI_CMD_EGSM);
 688
 689         usb_pause_mtx(&sc->sc_bus.bus_mtx,
 690             USB_MS_TO_TICKS(USB_RESUME_WAIT));
 691
 692         USB_BUS_UNLOCK(&sc->sc_bus);
 693 }
 694
 695 void
 696 uhci_resume(uhci_softc_t *sc)
 697 {
 698         USB_BUS_LOCK(&sc->sc_bus);
 699
 700         /* reset the controller */
 701
 702         uhci_reset(sc);
 703
 704         /* force global resume */
 705
 706         UHCICMD(sc, UHCI_CMD_FGR);
 707
 708         usb_pause_mtx(&sc->sc_bus.bus_mtx,
 709             USB_MS_TO_TICKS(USB_RESUME_DELAY));
 710
 711         /* and start traffic again */
 712
 713         uhci_start(sc);
 714
 715 #ifdef USB_DEBUG
 716         if (uhcidebug > 2) {
 717                 uhci_dumpregs(sc);
 718         }
 719 #endif
 720
 721         USB_BUS_UNLOCK(&sc->sc_bus);
 722
 723         /* catch lost interrupts */
 724         uhci_do_poll(&sc->sc_bus);
 725 }
 726
 727 #ifdef USB_DEBUG
 728 static void
 729 uhci_dumpregs(uhci_softc_t *sc)
 730 {
 731         DPRINTFN(0, "%s regs: cmd=%04x, sts=%04x, intr=%04x, frnum=%04x, "
 732             "flbase=%08x, sof=%04x, portsc1=%04x, portsc2=%04x\n",
 733             device_get_nameunit(sc->sc_bus.bdev),
 734             UREAD2(sc, UHCI_CMD),
 735             UREAD2(sc, UHCI_STS),
 736             UREAD2(sc, UHCI_INTR),
 737             UREAD2(sc, UHCI_FRNUM),
 738             UREAD4(sc, UHCI_FLBASEADDR),
 739             UREAD1(sc, UHCI_SOF),
 740             UREAD2(sc, UHCI_PORTSC1),
 741             UREAD2(sc, UHCI_PORTSC2));
 742 }
 743
 744 static uint8_t
 745 uhci_dump_td(uhci_td_t *p)
 746 {
 747         uint32_t td_next;
 748         uint32_t td_status;
 749         uint32_t td_token;
 750         uint8_t temp;
 751
 752         usb_pc_cpu_invalidate(p->page_cache);
 753
 754         td_next = le32toh(p->td_next);
 755         td_status = le32toh(p->td_status);
 756         td_token = le32toh(p->td_token);
 757
 758         /*
 759          * Check whether the link pointer in this TD marks the link pointer
 760          * as end of queue:
 761          */
 762         temp = ((td_next & UHCI_PTR_T) || (td_next == 0));
 763
 764         printf("TD(%p) at 0x%08x = link=0x%08x status=0x%08x "
 765             "token=0x%08x buffer=0x%08x\n",
 766             p,
 767             le32toh(p->td_self),
 768             td_next,
 769             td_status,
 770             td_token,
 771             le32toh(p->td_buffer));
 772
 773         printf("TD(%p) td_next=%s%s%s td_status=%s%s%s%s%s%s%s%s%s%s%s, errcnt=%d, actlen=%d pid=%02x,"
 774             "addr=%d,endpt=%d,D=%d,maxlen=%d\n",
 775             p,
 776             (td_next & 1) ? "-T" : "",
 777             (td_next & 2) ? "-Q" : "",
 778             (td_next & 4) ? "-VF" : "",
 779             (td_status & UHCI_TD_BITSTUFF) ? "-BITSTUFF" : "",
 780             (td_status & UHCI_TD_CRCTO) ? "-CRCTO" : "",
 781             (td_status & UHCI_TD_NAK) ? "-NAK" : "",
 782             (td_status & UHCI_TD_BABBLE) ? "-BABBLE" : "",
 783             (td_status & UHCI_TD_DBUFFER) ? "-DBUFFER" : "",
 784             (td_status & UHCI_TD_STALLED) ? "-STALLED" : "",
 785             (td_status & UHCI_TD_ACTIVE) ? "-ACTIVE" : "",
 786             (td_status & UHCI_TD_IOC) ? "-IOC" : "",
 787             (td_status & UHCI_TD_IOS) ? "-IOS" : "",
 788             (td_status & UHCI_TD_LS) ? "-LS" : "",
 789             (td_status & UHCI_TD_SPD) ? "-SPD" : "",
 790             UHCI_TD_GET_ERRCNT(td_status),
 791             UHCI_TD_GET_ACTLEN(td_status),
 792             UHCI_TD_GET_PID(td_token),
 793             UHCI_TD_GET_DEVADDR(td_token),
 794             UHCI_TD_GET_ENDPT(td_token),
 795             UHCI_TD_GET_DT(td_token),
 796             UHCI_TD_GET_MAXLEN(td_token));
 797
 798         return (temp);
 799 }
 800
 801 static uint8_t
 802 uhci_dump_qh(uhci_qh_t *sqh)
 803 {
 804         uint8_t temp;
 805         uint32_t qh_h_next;
 806         uint32_t qh_e_next;
 807
 808         usb_pc_cpu_invalidate(sqh->page_cache);
 809
 810         qh_h_next = le32toh(sqh->qh_h_next);
 811         qh_e_next = le32toh(sqh->qh_e_next);
 812
 813         DPRINTFN(0, "QH(%p) at 0x%08x: h_next=0x%08x e_next=0x%08x\n", sqh,
 814             le32toh(sqh->qh_self), qh_h_next, qh_e_next);
 815
 816         temp = ((((sqh->h_next != NULL) && !(qh_h_next & UHCI_PTR_T)) ? 1 : 0) |
 817             (((sqh->e_next != NULL) && !(qh_e_next & UHCI_PTR_T)) ? 2 : 0));
 818
 819         return (temp);
 820 }
 821
 822 static void
 823 uhci_dump_all(uhci_softc_t *sc)
 824 {
 825         uhci_dumpregs(sc);
 826         uhci_dump_qh(sc->sc_ls_ctl_p_last);
 827         uhci_dump_qh(sc->sc_fs_ctl_p_last);
 828         uhci_dump_qh(sc->sc_bulk_p_last);
 829         uhci_dump_qh(sc->sc_last_qh_p);
 830 }
 831
 832 static void
 833 uhci_dump_qhs(uhci_qh_t *sqh)
 834 {
 835         uint8_t temp;
 836
 837         temp = uhci_dump_qh(sqh);
 838
 839         /*
 840          * uhci_dump_qhs displays all the QHs and TDs from the given QH
 841          * onwards Traverses sideways first, then down.
 842          *
 843          * QH1 QH2 No QH TD2.1 TD2.2 TD1.1 etc.
 844          *
 845          * TD2.x being the TDs queued at QH2 and QH1 being referenced from QH1.
 846          */
 847
 848         if (temp & 1)
 849                 uhci_dump_qhs(sqh->h_next);
 850         else
 851                 DPRINTF("No QH\n");
 852
 853         if (temp & 2)
 854                 uhci_dump_tds(sqh->e_next);
 855         else
 856                 DPRINTF("No TD\n");
 857 }
 858
 859 static void
 860 uhci_dump_tds(uhci_td_t *td)
 861 {
 862         for (;
 863             td != NULL;
 864             td = td->obj_next) {
 865                 if (uhci_dump_td(td)) {
 866                         break;
 867                 }
 868         }
 869 }
 870
 871 #endif
 872
 873 /*
 874  * Let the last QH loop back to the full speed control transfer QH.
 875  * This is what intel calls "bandwidth reclamation" and improves
 876  * USB performance a lot for some devices.
 877  * If we are already looping, just count it.
 878  */
 879 static void
 880 uhci_add_loop(uhci_softc_t *sc)
 881 {
 882         struct uhci_qh *qh_lst;
 883         struct uhci_qh *qh_rec;
 884
 885 #ifdef USB_DEBUG
 886         if (uhcinoloop) {
 887                 return;
 888         }
 889 #endif
 890         if (++(sc->sc_loops) == 1) {
 891                 DPRINTFN(6, "add\n");
 892
 893                 qh_lst = sc->sc_last_qh_p;
 894                 qh_rec = sc->sc_reclaim_qh_p;
 895
 896                 /* NOTE: we don't loop back the soft pointer */
 897
 898                 qh_lst->qh_h_next = qh_rec->qh_self;
 899                 usb_pc_cpu_flush(qh_lst->page_cache);
 900         }
 901 }
 902
 903 static void
 904 uhci_rem_loop(uhci_softc_t *sc)
 905 {
 906         struct uhci_qh *qh_lst;
 907
 908 #ifdef USB_DEBUG
 909         if (uhcinoloop) {
 910                 return;
 911         }
 912 #endif
 913         if (--(sc->sc_loops) == 0) {
 914                 DPRINTFN(6, "remove\n");
 915
 916                 qh_lst = sc->sc_last_qh_p;
 917                 qh_lst->qh_h_next = htole32(UHCI_PTR_T);
 918                 usb_pc_cpu_flush(qh_lst->page_cache);
 919         }
 920 }
 921
 922 static void
 923 uhci_transfer_intr_enqueue(struct usb_xfer *xfer)
 924 {
 925         /* check for early completion */
 926         if (uhci_check_transfer(xfer)) {
 927                 return;
 928         }
 929         /* put transfer on interrupt queue */
 930         usbd_transfer_enqueue(&xfer->xroot->bus->intr_q, xfer);
 931
 932         /* start timeout, if any */
 933         if (xfer->timeout != 0) {
 934                 usbd_transfer_timeout_ms(xfer, &uhci_timeout, xfer->timeout);
 935         }
 936 }
 937
 938 #define UHCI_APPEND_TD(std,last) (last) = _uhci_append_td(std,last)
 939 static uhci_td_t *
 940 _uhci_append_td(uhci_td_t *std, uhci_td_t *last)
 941 {
 942         DPRINTFN(11, "%p to %p\n", std, last);
 943
 944         /* (sc->sc_bus.mtx) must be locked */
 945
 946         std->next = last->next;
 947         std->td_next = last->td_next;
 948
 949         std->prev = last;
 950
 951         usb_pc_cpu_flush(std->page_cache);
 952
 953         /*
 954          * the last->next->prev is never followed: std->next->prev = std;
 955          */
 956         last->next = std;
 957         last->td_next = std->td_self;
 958
 959         usb_pc_cpu_flush(last->page_cache);
 960
 961         return (std);
 962 }
 963
 964 #define UHCI_APPEND_QH(sqh,last) (last) = _uhci_append_qh(sqh,last)
 965 static uhci_qh_t *
 966 _uhci_append_qh(uhci_qh_t *sqh, uhci_qh_t *last)
 967 {
 968         DPRINTFN(11, "%p to %p\n", sqh, last);
 969
 970         if (sqh->h_prev != NULL) {
 971                 /* should not happen */
 972                 DPRINTFN(0, "QH already linked!\n");
 973                 return (last);
 974         }
 975         /* (sc->sc_bus.mtx) must be locked */
 976
 977         sqh->h_next = last->h_next;
 978         sqh->qh_h_next = last->qh_h_next;
 979
 980         sqh->h_prev = last;
 981
 982         usb_pc_cpu_flush(sqh->page_cache);
 983
 984         /*
 985          * The "last->h_next->h_prev" is never followed:
 986          *
 987          * "sqh->h_next->h_prev" = sqh;
 988          */
 989
 990         last->h_next = sqh;
 991         last->qh_h_next = sqh->qh_self;
 992
 993         usb_pc_cpu_flush(last->page_cache);
 994
 995         return (sqh);
 996 }
 997
 998 /**/
 999
1000 #define UHCI_REMOVE_TD(std,last) (last) = _uhci_remove_td(std,last)
1001 static uhci_td_t *
1002 _uhci_remove_td(uhci_td_t *std, uhci_td_t *last)
1003 {
1004         DPRINTFN(11, "%p from %p\n", std, last);
1005
1006         /* (sc->sc_bus.mtx) must be locked */
1007
1008         std->prev->next = std->next;
1009         std->prev->td_next = std->td_next;
1010
1011         usb_pc_cpu_flush(std->prev->page_cache);
1012
1013         if (std->next) {
1014                 std->next->prev = std->prev;
1015                 usb_pc_cpu_flush(std->next->page_cache);
1016         }
1017         return ((last == std) ? std->prev : last);
1018 }
1019
1020 #define UHCI_REMOVE_QH(sqh,last) (last) = _uhci_remove_qh(sqh,last)
1021 static uhci_qh_t *
1022 _uhci_remove_qh(uhci_qh_t *sqh, uhci_qh_t *last)
1023 {
1024         DPRINTFN(11, "%p from %p\n", sqh, last);
1025
1026         /* (sc->sc_bus.mtx) must be locked */
1027
1028         /* only remove if not removed from a queue */
1029         if (sqh->h_prev) {
1030
1031                 sqh->h_prev->h_next = sqh->h_next;
1032                 sqh->h_prev->qh_h_next = sqh->qh_h_next;
1033
1034                 usb_pc_cpu_flush(sqh->h_prev->page_cache);
1035
1036                 if (sqh->h_next) {
1037                         sqh->h_next->h_prev = sqh->h_prev;
1038                         usb_pc_cpu_flush(sqh->h_next->page_cache);
1039                 }
1040                 last = ((last == sqh) ? sqh->h_prev : last);
1041
1042                 sqh->h_prev = 0;
1043
1044                 usb_pc_cpu_flush(sqh->page_cache);
1045         }
1046         return (last);
1047 }
1048
1049 static void
1050 uhci_isoc_done(uhci_softc_t *sc, struct usb_xfer *xfer)
1051 {
1052         struct usb_page_search res;
1053         uint32_t nframes = xfer->nframes;
1054         uint32_t status;
1055         uint32_t offset = 0;
1056         uint32_t *plen = xfer->frlengths;
1057         uint16_t len = 0;
1058         uhci_td_t *td = xfer->td_transfer_first;
1059         uhci_td_t **pp_last = &sc->sc_isoc_p_last[xfer->qh_pos];
1060
1061         DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
1062             xfer, xfer->endpoint);
1063
1064         /* sync any DMA memory before doing fixups */
1065
1066         usb_bdma_post_sync(xfer);
1067
1068         while (nframes--) {
1069                 if (td == NULL) {
1070                         panic("%s:%d: out of TD's\n",
1071                             __FUNCTION__, __LINE__);
1072                 }
1073                 if (pp_last >= &sc->sc_isoc_p_last[UHCI_VFRAMELIST_COUNT]) {
1074                         pp_last = &sc->sc_isoc_p_last[0];
1075                 }
1076 #ifdef USB_DEBUG
1077                 if (uhcidebug > 5) {
1078                         DPRINTF("isoc TD\n");
1079                         uhci_dump_td(td);
1080                 }
1081 #endif
1082                 usb_pc_cpu_invalidate(td->page_cache);
1083                 status = le32toh(td->td_status);
1084
1085                 len = UHCI_TD_GET_ACTLEN(status);
1086
1087                 if (len > *plen) {
1088                         len = *plen;
1089                 }
1090                 if (td->fix_pc) {
1091
1092                         usbd_get_page(td->fix_pc, 0, &res);
1093
1094                         /* copy data from fixup location to real location */
1095
1096                         usb_pc_cpu_invalidate(td->fix_pc);
1097
1098                         usbd_copy_in(xfer->frbuffers, offset,
1099                             res.buffer, len);
1100                 }
1101                 offset += *plen;
1102
1103                 *plen = len;
1104
1105                 /* remove TD from schedule */
1106                 UHCI_REMOVE_TD(td, *pp_last);
1107
1108                 pp_last++;
1109                 plen++;
1110                 td = td->obj_next;
1111         }
1112
1113         xfer->aframes = xfer->nframes;
1114 }
1115
1116 static usb_error_t
1117 uhci_non_isoc_done_sub(struct usb_xfer *xfer)
1118 {
1119         struct usb_page_search res;
1120         uhci_td_t *td;
1121         uhci_td_t *td_alt_next;
1122         uint32_t status;
1123         uint32_t token;
1124         uint16_t len;
1125
1126         td = xfer->td_transfer_cache;
1127         td_alt_next = td->alt_next;
1128
1129         if (xfer->aframes != xfer->nframes) {
1130                 usbd_xfer_set_frame_len(xfer, xfer->aframes, 0);
1131         }
1132         while (1) {
1133
1134                 usb_pc_cpu_invalidate(td->page_cache);
1135                 status = le32toh(td->td_status);
1136                 token = le32toh(td->td_token);
1137
1138                 /*
1139                  * Verify the status and add
1140                  * up the actual length:
1141                  */
1142
1143                 len = UHCI_TD_GET_ACTLEN(status);
1144                 if (len > td->len) {
1145                         /* should not happen */
1146                         DPRINTF("Invalid status length, "
1147                             "0x%04x/0x%04x bytes\n", len, td->len);
1148                         status |= UHCI_TD_STALLED;
1149
1150                 } else if ((xfer->aframes != xfer->nframes) && (len > 0)) {
1151
1152                         if (td->fix_pc) {
1153
1154                                 usbd_get_page(td->fix_pc, 0, &res);
1155
1156                                 /*
1157                                  * copy data from fixup location to real
1158                                  * location
1159                                  */
1160
1161                                 usb_pc_cpu_invalidate(td->fix_pc);
1162
1163                                 usbd_copy_in(xfer->frbuffers + xfer->aframes,
1164                                     xfer->frlengths[xfer->aframes], res.buffer, len);
1165                         }
1166                         /* update actual length */
1167
1168                         xfer->frlengths[xfer->aframes] += len;
1169                 }
1170                 /* Check for last transfer */
1171                 if (((void *)td) == xfer->td_transfer_last) {
1172                         td = NULL;
1173                         break;
1174                 }
1175                 if (status & UHCI_TD_STALLED) {
1176                         /* the transfer is finished */
1177                         td = NULL;
1178                         break;
1179                 }
1180                 /* Check for short transfer */
1181                 if (len != td->len) {
1182                         if (xfer->flags_int.short_frames_ok) {
1183                                 /* follow alt next */
1184                                 td = td->alt_next;
1185                         } else {
1186                                 /* the transfer is finished */
1187                                 td = NULL;
1188                         }
1189                         break;
1190                 }
1191                 td = td->obj_next;
1192
1193                 if (td->alt_next != td_alt_next) {
1194                         /* this USB frame is complete */
1195                         break;
1196                 }
1197         }
1198
1199         /* update transfer cache */
1200
1201         xfer->td_transfer_cache = td;
1202
1203         /* update data toggle */
1204
1205         xfer->endpoint->toggle_next = (token & UHCI_TD_SET_DT(1)) ? 0 : 1;
1206
1207 #ifdef USB_DEBUG
1208         if (status & UHCI_TD_ERROR) {
1209                 DPRINTFN(11, "error, addr=%d, endpt=0x%02x, frame=0x%02x "
1210                     "status=%s%s%s%s%s%s%s%s%s%s%s\n",
1211                     xfer->address, xfer->endpointno, xfer->aframes,
1212                     (status & UHCI_TD_BITSTUFF) ? "[BITSTUFF]" : "",
1213                     (status & UHCI_TD_CRCTO) ? "[CRCTO]" : "",
1214                     (status & UHCI_TD_NAK) ? "[NAK]" : "",
1215                     (status & UHCI_TD_BABBLE) ? "[BABBLE]" : "",
1216                     (status & UHCI_TD_DBUFFER) ? "[DBUFFER]" : "",
1217                     (status & UHCI_TD_STALLED) ? "[STALLED]" : "",
1218                     (status & UHCI_TD_ACTIVE) ? "[ACTIVE]" : "[NOT_ACTIVE]",
1219                     (status & UHCI_TD_IOC) ? "[IOC]" : "",
1220                     (status & UHCI_TD_IOS) ? "[IOS]" : "",
1221                     (status & UHCI_TD_LS) ? "[LS]" : "",
1222                     (status & UHCI_TD_SPD) ? "[SPD]" : "");
1223         }
1224 #endif
1225         return (status & UHCI_TD_STALLED) ?
1226             USB_ERR_STALLED : USB_ERR_NORMAL_COMPLETION;
1227 }
1228
1229 static void
1230 uhci_non_isoc_done(struct usb_xfer *xfer)
1231 {
1232         usb_error_t err = 0;
1233
1234         DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
1235             xfer, xfer->endpoint);
1236
1237 #ifdef USB_DEBUG
1238         if (uhcidebug > 10) {
1239                 uhci_dump_tds(xfer->td_transfer_first);
1240         }
1241 #endif
1242
1243         /* sync any DMA memory before doing fixups */
1244
1245         usb_bdma_post_sync(xfer);
1246
1247         /* reset scanner */
1248
1249         xfer->td_transfer_cache = xfer->td_transfer_first;
1250
1251         if (xfer->flags_int.control_xfr) {
1252                 if (xfer->flags_int.control_hdr) {
1253
1254                         err = uhci_non_isoc_done_sub(xfer);
1255                 }
1256                 xfer->aframes = 1;
1257
1258                 if (xfer->td_transfer_cache == NULL) {
1259                         goto done;
1260                 }
1261         }
1262         while (xfer->aframes != xfer->nframes) {
1263
1264                 err = uhci_non_isoc_done_sub(xfer);
1265                 xfer->aframes++;
1266
1267                 if (xfer->td_transfer_cache == NULL) {
1268                         goto done;
1269                 }
1270         }
1271
1272         if (xfer->flags_int.control_xfr &&
1273             !xfer->flags_int.control_act) {
1274
1275                 err = uhci_non_isoc_done_sub(xfer);
1276         }
1277 done:
1278         uhci_device_done(xfer, err);
1279 }
1280
1281 /*------------------------------------------------------------------------*
1282  *      uhci_check_transfer_sub
1283  *
1284  * The main purpose of this function is to update the data-toggle
1285  * in case it is wrong.
1286  *------------------------------------------------------------------------*/
1287 static void
1288 uhci_check_transfer_sub(struct usb_xfer *xfer)
1289 {
1290         uhci_qh_t *qh;
1291         uhci_td_t *td;
1292         uhci_td_t *td_alt_next;
1293
1294         uint32_t td_token;
1295         uint32_t td_self;
1296
1297         td = xfer->td_transfer_cache;
1298         qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
1299
1300         td_token = td->obj_next->td_token;
1301         td = td->alt_next;
1302         xfer->td_transfer_cache = td;
1303         td_self = td->td_self;
1304         td_alt_next = td->alt_next;
1305
1306         if (xfer->flags_int.control_xfr)
1307                 goto skip;      /* don't touch the DT value! */
1308
1309         if (!((td->td_token ^ td_token) & htole32(UHCI_TD_SET_DT(1))))
1310                 goto skip;      /* data toggle has correct value */
1311
1312         /*
1313          * The data toggle is wrong and we need to toggle it !
1314          */
1315         while (1) {
1316
1317                 td->td_token ^= htole32(UHCI_TD_SET_DT(1));
1318                 usb_pc_cpu_flush(td->page_cache);
1319
1320                 if (td == xfer->td_transfer_last) {
1321                         /* last transfer */
1322                         break;
1323                 }
1324                 td = td->obj_next;
1325
1326                 if (td->alt_next != td_alt_next) {
1327                         /* next frame */
1328                         break;
1329                 }
1330         }
1331 skip:
1332
1333         /* update the QH */
1334         qh->qh_e_next = td_self;
1335         usb_pc_cpu_flush(qh->page_cache);
1336
1337         DPRINTFN(13, "xfer=%p following alt next\n", xfer);
1338 }
1339
1340 /*------------------------------------------------------------------------*
1341  *      uhci_check_transfer
1342  *
1343  * Return values:
1344  *    0: USB transfer is not finished
1345  * Else: USB transfer is finished
1346  *------------------------------------------------------------------------*/
1347 static uint8_t
1348 uhci_check_transfer(struct usb_xfer *xfer)
1349 {
1350         uint32_t status;
1351         uint32_t token;
1352         uhci_td_t *td;
1353
1354         DPRINTFN(16, "xfer=%p checking transfer\n", xfer);
1355
1356         if (xfer->endpoint->methods == &uhci_device_isoc_methods) {
1357                 /* isochronous transfer */
1358
1359                 td = xfer->td_transfer_last;
1360
1361                 usb_pc_cpu_invalidate(td->page_cache);
1362                 status = le32toh(td->td_status);
1363
1364                 /* check also if the first is complete */
1365
1366                 td = xfer->td_transfer_first;
1367
1368                 usb_pc_cpu_invalidate(td->page_cache);
1369                 status |= le32toh(td->td_status);
1370
1371                 if (!(status & UHCI_TD_ACTIVE)) {
1372                         uhci_device_done(xfer, USB_ERR_NORMAL_COMPLETION);
1373                         goto transferred;
1374                 }
1375         } else {
1376                 /* non-isochronous transfer */
1377
1378                 /*
1379                  * check whether there is an error somewhere
1380                  * in the middle, or whether there was a short
1381                  * packet (SPD and not ACTIVE)
1382                  */
1383                 td = xfer->td_transfer_cache;
1384
1385                 while (1) {
1386                         usb_pc_cpu_invalidate(td->page_cache);
1387                         status = le32toh(td->td_status);
1388                         token = le32toh(td->td_token);
1389
1390                         /*
1391                          * if there is an active TD the transfer isn't done
1392                          */
1393                         if (status & UHCI_TD_ACTIVE) {
1394                                 /* update cache */
1395                                 xfer->td_transfer_cache = td;
1396                                 goto done;
1397                         }
1398                         /*
1399                          * last transfer descriptor makes the transfer done
1400                          */
1401                         if (((void *)td) == xfer->td_transfer_last) {
1402                                 break;
1403                         }
1404                         /*
1405                          * any kind of error makes the transfer done
1406                          */
1407                         if (status & UHCI_TD_STALLED) {
1408                                 break;
1409                         }
1410                         /*
1411                          * check if we reached the last packet
1412                          * or if there is a short packet:
1413                          */
1414                         if ((td->td_next == htole32(UHCI_PTR_T)) ||
1415                             (UHCI_TD_GET_ACTLEN(status) < td->len)) {
1416
1417                                 if (xfer->flags_int.short_frames_ok) {
1418                                         /* follow alt next */
1419                                         if (td->alt_next) {
1420                                                 /* update cache */
1421                                                 xfer->td_transfer_cache = td;
1422                                                 uhci_check_transfer_sub(xfer);
1423                                                 goto done;
1424                                         }
1425                                 }
1426                                 /* transfer is done */
1427                                 break;
1428                         }
1429                         td = td->obj_next;
1430                 }
1431                 uhci_non_isoc_done(xfer);
1432                 goto transferred;
1433         }
1434
1435 done:
1436         DPRINTFN(13, "xfer=%p is still active\n", xfer);
1437         return (0);
1438
1439 transferred:
1440         return (1);
1441 }
1442
1443 static void
1444 uhci_interrupt_poll(uhci_softc_t *sc)
1445 {
1446         struct usb_xfer *xfer;
1447
1448 repeat:
1449         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
1450                 /*
1451                  * check if transfer is transferred
1452                  */
1453                 if (uhci_check_transfer(xfer)) {
1454                         /* queue has been modified */
1455                         goto repeat;
1456                 }
1457         }
1458 }
1459
1460 /*------------------------------------------------------------------------*
1461  *      uhci_interrupt - UHCI interrupt handler
1462  *
1463  * NOTE: Do not access "sc->sc_bus.bdev" inside the interrupt handler,
1464  * hence the interrupt handler will be setup before "sc->sc_bus.bdev"
1465  * is present !
1466  *------------------------------------------------------------------------*/
1467 void
1468 uhci_interrupt(uhci_softc_t *sc)
1469 {
1470         uint32_t status;
1471
1472         USB_BUS_LOCK(&sc->sc_bus);
1473
1474         DPRINTFN(16, "real interrupt\n");
1475
1476 #ifdef USB_DEBUG
1477         if (uhcidebug > 15) {
1478                 uhci_dumpregs(sc);
1479         }
1480 #endif
1481         status = UREAD2(sc, UHCI_STS) & UHCI_STS_ALLINTRS;
1482         if (status == 0) {
1483                 /* the interrupt was not for us */
1484                 goto done;
1485         }
1486         if (status & (UHCI_STS_RD | UHCI_STS_HSE |
1487             UHCI_STS_HCPE | UHCI_STS_HCH)) {
1488
1489                 if (status & UHCI_STS_RD) {
1490 #ifdef USB_DEBUG
1491                         printf("%s: resume detect\n",
1492                             __FUNCTION__);
1493 #endif
1494                 }
1495                 if (status & UHCI_STS_HSE) {
1496                         printf("%s: host system error\n",
1497                             __FUNCTION__);
1498                 }
1499                 if (status & UHCI_STS_HCPE) {
1500                         printf("%s: host controller process error\n",
1501                             __FUNCTION__);
1502                 }
1503                 if (status & UHCI_STS_HCH) {
1504                         /* no acknowledge needed */
1505                         DPRINTF("%s: host controller halted\n",
1506                             __FUNCTION__);
1507 #ifdef USB_DEBUG
1508                         if (uhcidebug > 0) {
1509                                 uhci_dump_all(sc);
1510                         }
1511 #endif
1512                 }
1513         }
1514         /* get acknowledge bits */
1515         status &= (UHCI_STS_USBINT |
1516             UHCI_STS_USBEI |
1517             UHCI_STS_RD |
1518             UHCI_STS_HSE |
1519             UHCI_STS_HCPE);
1520
1521         if (status == 0) {
1522                 /* nothing to acknowledge */
1523                 goto done;
1524         }
1525         /* acknowledge interrupts */
1526         UWRITE2(sc, UHCI_STS, status);
1527
1528         /* poll all the USB transfers */
1529         uhci_interrupt_poll(sc);
1530
1531 done:
1532         USB_BUS_UNLOCK(&sc->sc_bus);
1533 }
1534
1535 /*
1536  * called when a request does not complete
1537  */
1538 static void
1539 uhci_timeout(void *arg)
1540 {
1541         struct usb_xfer *xfer = arg;
1542
1543         DPRINTF("xfer=%p\n", xfer);
1544
1545         USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
1546
1547         /* transfer is transferred */
1548         uhci_device_done(xfer, USB_ERR_TIMEOUT);
1549 }
1550
1551 static void
1552 uhci_do_poll(struct usb_bus *bus)
1553 {
1554         struct uhci_softc *sc = UHCI_BUS2SC(bus);
1555
1556         USB_BUS_LOCK(&sc->sc_bus);
1557         uhci_interrupt_poll(sc);
1558         USB_BUS_UNLOCK(&sc->sc_bus);
1559 }
1560
1561 static void
1562 uhci_setup_standard_chain_sub(struct uhci_std_temp *temp)
1563 {
1564         uhci_td_t *td;
1565         uhci_td_t *td_next;
1566         uhci_td_t *td_alt_next;
1567         uint32_t average;
1568         uint32_t len_old;
1569         uint8_t shortpkt_old;
1570         uint8_t precompute;
1571
1572         td_alt_next = NULL;
1573         shortpkt_old = temp->shortpkt;
1574         len_old = temp->len;
1575         precompute = 1;
1576
1577         /* software is used to detect short incoming transfers */
1578
1579         if ((temp->td_token & htole32(UHCI_TD_PID)) == htole32(UHCI_TD_PID_IN)) {
1580                 temp->td_status |= htole32(UHCI_TD_SPD);
1581         } else {
1582                 temp->td_status &= ~htole32(UHCI_TD_SPD);
1583         }
1584
1585         temp->ml.buf_offset = 0;
1586
1587 restart:
1588
1589         temp->td_token &= ~htole32(UHCI_TD_SET_MAXLEN(0));
1590         temp->td_token |= htole32(UHCI_TD_SET_MAXLEN(temp->average));
1591
1592         td = temp->td;
1593         td_next = temp->td_next;
1594
1595         while (1) {
1596
1597                 if (temp->len == 0) {
1598
1599                         if (temp->shortpkt) {
1600                                 break;
1601                         }
1602                         /* send a Zero Length Packet, ZLP, last */
1603
1604                         temp->shortpkt = 1;
1605                         temp->td_token |= htole32(UHCI_TD_SET_MAXLEN(0));
1606                         average = 0;
1607
1608                 } else {
1609
1610                         average = temp->average;
1611
1612                         if (temp->len < average) {
1613                                 temp->shortpkt = 1;
1614                                 temp->td_token &= ~htole32(UHCI_TD_SET_MAXLEN(0));
1615                                 temp->td_token |= htole32(UHCI_TD_SET_MAXLEN(temp->len));
1616                                 average = temp->len;
1617                         }
1618                 }
1619
1620                 if (td_next == NULL) {
1621                         panic("%s: out of UHCI transfer descriptors!", __FUNCTION__);
1622                 }
1623                 /* get next TD */
1624
1625                 td = td_next;
1626                 td_next = td->obj_next;
1627
1628                 /* check if we are pre-computing */
1629
1630                 if (precompute) {
1631
1632                         /* update remaining length */
1633
1634                         temp->len -= average;
1635
1636                         continue;
1637                 }
1638                 /* fill out current TD */
1639
1640                 td->td_status = temp->td_status;
1641                 td->td_token = temp->td_token;
1642
1643                 /* update data toggle */
1644
1645                 temp->td_token ^= htole32(UHCI_TD_SET_DT(1));
1646
1647                 if (average == 0) {
1648
1649                         td->len = 0;
1650                         td->td_buffer = 0;
1651                         td->fix_pc = NULL;
1652
1653                 } else {
1654
1655                         /* update remaining length */
1656
1657                         temp->len -= average;
1658
1659                         td->len = average;
1660
1661                         /* fill out buffer pointer and do fixup, if any */
1662
1663                         uhci_mem_layout_fixup(&temp->ml, td);
1664                 }
1665
1666                 td->alt_next = td_alt_next;
1667
1668                 if ((td_next == td_alt_next) && temp->setup_alt_next) {
1669                         /* we need to receive these frames one by one ! */
1670                         td->td_status |= htole32(UHCI_TD_IOC);
1671                         td->td_next = htole32(UHCI_PTR_T);
1672                 } else {
1673                         if (td_next) {
1674                                 /* link the current TD with the next one */
1675                                 td->td_next = td_next->td_self;
1676                         }
1677                 }
1678
1679                 usb_pc_cpu_flush(td->page_cache);
1680         }
1681
1682         if (precompute) {
1683                 precompute = 0;
1684
1685                 /* setup alt next pointer, if any */
1686                 if (temp->last_frame) {
1687                         td_alt_next = NULL;
1688                 } else {
1689                         /* we use this field internally */
1690                         td_alt_next = td_next;
1691                 }
1692
1693                 /* restore */
1694                 temp->shortpkt = shortpkt_old;
1695                 temp->len = len_old;
1696                 goto restart;
1697         }
1698         temp->td = td;
1699         temp->td_next = td_next;
1700 }
1701
1702 static uhci_td_t *
1703 uhci_setup_standard_chain(struct usb_xfer *xfer)
1704 {
1705         struct uhci_std_temp temp;
1706         uhci_td_t *td;
1707         uint32_t x;
1708
1709         DPRINTFN(9, "addr=%d endpt=%d sumlen=%d speed=%d\n",
1710             xfer->address, UE_GET_ADDR(xfer->endpointno),
1711             xfer->sumlen, usbd_get_speed(xfer->xroot->udev));
1712
1713         temp.average = xfer->max_frame_size;
1714         temp.max_frame_size = xfer->max_frame_size;
1715
1716         /* toggle the DMA set we are using */
1717         xfer->flags_int.curr_dma_set ^= 1;
1718
1719         /* get next DMA set */
1720         td = xfer->td_start[xfer->flags_int.curr_dma_set];
1721         xfer->td_transfer_first = td;
1722         xfer->td_transfer_cache = td;
1723
1724         temp.td = NULL;
1725         temp.td_next = td;
1726         temp.last_frame = 0;
1727         temp.setup_alt_next = xfer->flags_int.short_frames_ok;
1728
1729         uhci_mem_layout_init(&temp.ml, xfer);
1730
1731         temp.td_status =
1732             htole32(UHCI_TD_ZERO_ACTLEN(UHCI_TD_SET_ERRCNT(3) |
1733             UHCI_TD_ACTIVE));
1734
1735         if (xfer->xroot->udev->speed == USB_SPEED_LOW) {
1736                 temp.td_status |= htole32(UHCI_TD_LS);
1737         }
1738         temp.td_token =
1739             htole32(UHCI_TD_SET_ENDPT(xfer->endpointno) |
1740             UHCI_TD_SET_DEVADDR(xfer->address));
1741
1742         if (xfer->endpoint->toggle_next) {
1743                 /* DATA1 is next */
1744                 temp.td_token |= htole32(UHCI_TD_SET_DT(1));
1745         }
1746         /* check if we should prepend a setup message */
1747
1748         if (xfer->flags_int.control_xfr) {
1749
1750                 if (xfer->flags_int.control_hdr) {
1751
1752                         temp.td_token &= htole32(UHCI_TD_SET_DEVADDR(0x7F) |
1753                             UHCI_TD_SET_ENDPT(0xF));
1754                         temp.td_token |= htole32(UHCI_TD_PID_SETUP |
1755                             UHCI_TD_SET_DT(0));
1756
1757                         temp.len = xfer->frlengths[0];
1758                         temp.ml.buf_pc = xfer->frbuffers + 0;
1759                         temp.shortpkt = temp.len ? 1 : 0;
1760                         /* check for last frame */
1761                         if (xfer->nframes == 1) {
1762                                 /* no STATUS stage yet, SETUP is last */
1763                                 if (xfer->flags_int.control_act) {
1764                                         temp.last_frame = 1;
1765                                         temp.setup_alt_next = 0;
1766                                 }
1767                         }
1768                         uhci_setup_standard_chain_sub(&temp);
1769                 }
1770                 x = 1;
1771         } else {
1772                 x = 0;
1773         }
1774
1775         while (x != xfer->nframes) {
1776
1777                 /* DATA0 / DATA1 message */
1778
1779                 temp.len = xfer->frlengths[x];
1780                 temp.ml.buf_pc = xfer->frbuffers + x;
1781
1782                 x++;
1783
1784                 if (x == xfer->nframes) {
1785                         if (xfer->flags_int.control_xfr) {
1786                                 /* no STATUS stage yet, DATA is last */
1787                                 if (xfer->flags_int.control_act) {
1788                                         temp.last_frame = 1;
1789                                         temp.setup_alt_next = 0;
1790                                 }
1791                         } else {
1792                                 temp.last_frame = 1;
1793                                 temp.setup_alt_next = 0;
1794                         }
1795                 }
1796                 /*
1797                  * Keep previous data toggle,
1798                  * device address and endpoint number:
1799                  */
1800
1801                 temp.td_token &= htole32(UHCI_TD_SET_DEVADDR(0x7F) |
1802                     UHCI_TD_SET_ENDPT(0xF) |
1803                     UHCI_TD_SET_DT(1));
1804
1805                 if (temp.len == 0) {
1806
1807                         /* make sure that we send an USB packet */
1808
1809                         temp.shortpkt = 0;
1810
1811                 } else {
1812
1813                         /* regular data transfer */
1814
1815                         temp.shortpkt = (xfer->flags.force_short_xfer) ? 0 : 1;
1816                 }
1817
1818                 /* set endpoint direction */
1819
1820                 temp.td_token |=
1821                     (UE_GET_DIR(xfer->endpointno) == UE_DIR_IN) ?
1822                     htole32(UHCI_TD_PID_IN) :
1823                     htole32(UHCI_TD_PID_OUT);
1824
1825                 uhci_setup_standard_chain_sub(&temp);
1826         }
1827
1828         /* check if we should append a status stage */
1829
1830         if (xfer->flags_int.control_xfr &&
1831             !xfer->flags_int.control_act) {
1832
1833                 /*
1834                  * send a DATA1 message and reverse the current endpoint
1835                  * direction
1836                  */
1837
1838                 temp.td_token &= htole32(UHCI_TD_SET_DEVADDR(0x7F) |
1839                     UHCI_TD_SET_ENDPT(0xF) |
1840                     UHCI_TD_SET_DT(1));
1841                 temp.td_token |=
1842                     (UE_GET_DIR(xfer->endpointno) == UE_DIR_OUT) ?
1843                     htole32(UHCI_TD_PID_IN | UHCI_TD_SET_DT(1)) :
1844                     htole32(UHCI_TD_PID_OUT | UHCI_TD_SET_DT(1));
1845
1846                 temp.len = 0;
1847                 temp.ml.buf_pc = NULL;
1848                 temp.shortpkt = 0;
1849                 temp.last_frame = 1;
1850                 temp.setup_alt_next = 0;
1851
1852                 uhci_setup_standard_chain_sub(&temp);
1853         }
1854         td = temp.td;
1855
1856         /* Ensure that last TD is terminating: */
1857         td->td_next = htole32(UHCI_PTR_T);
1858
1859         /* set interrupt bit */
1860
1861         td->td_status |= htole32(UHCI_TD_IOC);
1862
1863         usb_pc_cpu_flush(td->page_cache);
1864
1865         /* must have at least one frame! */
1866
1867         xfer->td_transfer_last = td;
1868
1869 #ifdef USB_DEBUG
1870         if (uhcidebug > 8) {
1871                 DPRINTF("nexttog=%d; data before transfer:\n",
1872                     xfer->endpoint->toggle_next);
1873                 uhci_dump_tds(xfer->td_transfer_first);
1874         }
1875 #endif
1876         return (xfer->td_transfer_first);
1877 }
1878
1879 /* NOTE: "done" can be run two times in a row,
1880  * from close and from interrupt
1881  */
1882
1883 static void
1884 uhci_device_done(struct usb_xfer *xfer, usb_error_t error)
1885 {
1886         struct usb_pipe_methods *methods = xfer->endpoint->methods;
1887         uhci_softc_t *sc = UHCI_BUS2SC(xfer->xroot->bus);
1888         uhci_qh_t *qh;
1889
1890         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
1891
1892         DPRINTFN(2, "xfer=%p, endpoint=%p, error=%d\n",
1893             xfer, xfer->endpoint, error);
1894
1895         qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
1896         if (qh) {
1897                 usb_pc_cpu_invalidate(qh->page_cache);
1898         }
1899         if (xfer->flags_int.bandwidth_reclaimed) {
1900                 xfer->flags_int.bandwidth_reclaimed = 0;
1901                 uhci_rem_loop(sc);
1902         }
1903         if (methods == &uhci_device_bulk_methods) {
1904                 UHCI_REMOVE_QH(qh, sc->sc_bulk_p_last);
1905         }
1906         if (methods == &uhci_device_ctrl_methods) {
1907                 if (xfer->xroot->udev->speed == USB_SPEED_LOW) {
1908                         UHCI_REMOVE_QH(qh, sc->sc_ls_ctl_p_last);
1909                 } else {
1910                         UHCI_REMOVE_QH(qh, sc->sc_fs_ctl_p_last);
1911                 }
1912         }
1913         if (methods == &uhci_device_intr_methods) {
1914                 UHCI_REMOVE_QH(qh, sc->sc_intr_p_last[xfer->qh_pos]);
1915         }
1916         /*
1917          * Only finish isochronous transfers once
1918          * which will update "xfer->frlengths".
1919          */
1920         if (xfer->td_transfer_first &&
1921             xfer->td_transfer_last) {
1922                 if (methods == &uhci_device_isoc_methods) {
1923                         uhci_isoc_done(sc, xfer);
1924                 }
1925                 xfer->td_transfer_first = NULL;
1926                 xfer->td_transfer_last = NULL;
1927         }
1928         /* dequeue transfer and start next transfer */
1929         usbd_transfer_done(xfer, error);
1930 }
1931
1932 /*------------------------------------------------------------------------*
1933  * uhci bulk support
1934  *------------------------------------------------------------------------*/
1935 static void
1936 uhci_device_bulk_open(struct usb_xfer *xfer)
1937 {
1938         return;
1939 }
1940
1941 static void
1942 uhci_device_bulk_close(struct usb_xfer *xfer)
1943 {
1944         uhci_device_done(xfer, USB_ERR_CANCELLED);
1945 }
1946
1947 static void
1948 uhci_device_bulk_enter(struct usb_xfer *xfer)
1949 {
1950         return;
1951 }
1952
1953 static void
1954 uhci_device_bulk_start(struct usb_xfer *xfer)
1955 {
1956         uhci_softc_t *sc = UHCI_BUS2SC(xfer->xroot->bus);
1957         uhci_td_t *td;
1958         uhci_qh_t *qh;
1959
1960         /* setup TD's */
1961         td = uhci_setup_standard_chain(xfer);
1962
1963         /* setup QH */
1964         qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
1965
1966         qh->e_next = td;
1967         qh->qh_e_next = td->td_self;
1968
1969         if (xfer->xroot->udev->flags.self_suspended == 0) {
1970                 UHCI_APPEND_QH(qh, sc->sc_bulk_p_last);
1971                 uhci_add_loop(sc);
1972                 xfer->flags_int.bandwidth_reclaimed = 1;
1973         } else {
1974                 usb_pc_cpu_flush(qh->page_cache);
1975         }
1976
1977         /* put transfer on interrupt queue */
1978         uhci_transfer_intr_enqueue(xfer);
1979 }
1980
1981 struct usb_pipe_methods uhci_device_bulk_methods =
1982 {
1983         .open = uhci_device_bulk_open,
1984         .close = uhci_device_bulk_close,
1985         .enter = uhci_device_bulk_enter,
1986         .start = uhci_device_bulk_start,
1987 };
1988
1989 /*------------------------------------------------------------------------*
1990  * uhci control support
1991  *------------------------------------------------------------------------*/
1992 static void
1993 uhci_device_ctrl_open(struct usb_xfer *xfer)
1994 {
1995         return;
1996 }
1997
1998 static void
1999 uhci_device_ctrl_close(struct usb_xfer *xfer)
2000 {
2001         uhci_device_done(xfer, USB_ERR_CANCELLED);
2002 }
2003
2004 static void
2005 uhci_device_ctrl_enter(struct usb_xfer *xfer)
2006 {
2007         return;
2008 }
2009
2010 static void
2011 uhci_device_ctrl_start(struct usb_xfer *xfer)
2012 {
2013         uhci_softc_t *sc = UHCI_BUS2SC(xfer->xroot->bus);
2014         uhci_qh_t *qh;
2015         uhci_td_t *td;
2016
2017         /* setup TD's */
2018         td = uhci_setup_standard_chain(xfer);
2019
2020         /* setup QH */
2021         qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
2022
2023         qh->e_next = td;
2024         qh->qh_e_next = td->td_self;
2025
2026         /*
2027          * NOTE: some devices choke on bandwidth- reclamation for control
2028          * transfers
2029          */
2030         if (xfer->xroot->udev->flags.self_suspended == 0) {
2031                 if (xfer->xroot->udev->speed == USB_SPEED_LOW) {
2032                         UHCI_APPEND_QH(qh, sc->sc_ls_ctl_p_last);
2033                 } else {
2034                         UHCI_APPEND_QH(qh, sc->sc_fs_ctl_p_last);
2035                 }
2036         } else {
2037                 usb_pc_cpu_flush(qh->page_cache);
2038         }
2039         /* put transfer on interrupt queue */
2040         uhci_transfer_intr_enqueue(xfer);
2041 }
2042
2043 struct usb_pipe_methods uhci_device_ctrl_methods =
2044 {
2045         .open = uhci_device_ctrl_open,
2046         .close = uhci_device_ctrl_close,
2047         .enter = uhci_device_ctrl_enter,
2048         .start = uhci_device_ctrl_start,
2049 };
2050
2051 /*------------------------------------------------------------------------*
2052  * uhci interrupt support
2053  *------------------------------------------------------------------------*/
2054 static void
2055 uhci_device_intr_open(struct usb_xfer *xfer)
2056 {
2057         uhci_softc_t *sc = UHCI_BUS2SC(xfer->xroot->bus);
2058         uint16_t best;
2059         uint16_t bit;
2060         uint16_t x;
2061
2062         best = 0;
2063         bit = UHCI_IFRAMELIST_COUNT / 2;
2064         while (bit) {
2065                 if (xfer->interval >= bit) {
2066                         x = bit;
2067                         best = bit;
2068                         while (x & bit) {
2069                                 if (sc->sc_intr_stat[x] <
2070                                     sc->sc_intr_stat[best]) {
2071                                         best = x;
2072                                 }
2073                                 x++;
2074                         }
2075                         break;
2076                 }
2077                 bit >>= 1;
2078         }
2079
2080         sc->sc_intr_stat[best]++;
2081         xfer->qh_pos = best;
2082
2083         DPRINTFN(3, "best=%d interval=%d\n",
2084             best, xfer->interval);
2085 }
2086
2087 static void
2088 uhci_device_intr_close(struct usb_xfer *xfer)
2089 {
2090         uhci_softc_t *sc = UHCI_BUS2SC(xfer->xroot->bus);
2091
2092         sc->sc_intr_stat[xfer->qh_pos]--;
2093
2094         uhci_device_done(xfer, USB_ERR_CANCELLED);
2095 }
2096
2097 static void
2098 uhci_device_intr_enter(struct usb_xfer *xfer)
2099 {
2100         return;
2101 }
2102
2103 static void
2104 uhci_device_intr_start(struct usb_xfer *xfer)
2105 {
2106         uhci_softc_t *sc = UHCI_BUS2SC(xfer->xroot->bus);
2107         uhci_qh_t *qh;
2108         uhci_td_t *td;
2109
2110         /* setup TD's */
2111         td = uhci_setup_standard_chain(xfer);
2112
2113         /* setup QH */
2114         qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
2115
2116         qh->e_next = td;
2117         qh->qh_e_next = td->td_self;
2118
2119         if (xfer->xroot->udev->flags.self_suspended == 0) {
2120                 /* enter QHs into the controller data structures */
2121                 UHCI_APPEND_QH(qh, sc->sc_intr_p_last[xfer->qh_pos]);
2122         } else {
2123                 usb_pc_cpu_flush(qh->page_cache);
2124         }
2125
2126         /* put transfer on interrupt queue */
2127         uhci_transfer_intr_enqueue(xfer);
2128 }
2129
2130 struct usb_pipe_methods uhci_device_intr_methods =
2131 {
2132         .open = uhci_device_intr_open,
2133         .close = uhci_device_intr_close,
2134         .enter = uhci_device_intr_enter,
2135         .start = uhci_device_intr_start,
2136 };
2137
2138 /*------------------------------------------------------------------------*
2139  * uhci isochronous support
2140  *------------------------------------------------------------------------*/
2141 static void
2142 uhci_device_isoc_open(struct usb_xfer *xfer)
2143 {
2144         uhci_td_t *td;
2145         uint32_t td_token;
2146         uint8_t ds;
2147
2148         td_token =
2149             (UE_GET_DIR(xfer->endpointno) == UE_DIR_IN) ?
2150             UHCI_TD_IN(0, xfer->endpointno, xfer->address, 0) :
2151             UHCI_TD_OUT(0, xfer->endpointno, xfer->address, 0);
2152
2153         td_token = htole32(td_token);
2154
2155         /* initialize all TD's */
2156
2157         for (ds = 0; ds != 2; ds++) {
2158
2159                 for (td = xfer->td_start[ds]; td; td = td->obj_next) {
2160
2161                         /* mark TD as inactive */
2162                         td->td_status = htole32(UHCI_TD_IOS);
2163                         td->td_token = td_token;
2164
2165                         usb_pc_cpu_flush(td->page_cache);
2166                 }
2167         }
2168 }
2169
2170 static void
2171 uhci_device_isoc_close(struct usb_xfer *xfer)
2172 {
2173         uhci_device_done(xfer, USB_ERR_CANCELLED);
2174 }
2175
2176 static void
2177 uhci_device_isoc_enter(struct usb_xfer *xfer)
2178 {
2179         struct uhci_mem_layout ml;
2180         uhci_softc_t *sc = UHCI_BUS2SC(xfer->xroot->bus);
2181         uint32_t nframes;
2182         uint32_t temp;
2183         uint32_t *plen;
2184
2185 #ifdef USB_DEBUG
2186         uint8_t once = 1;
2187
2188 #endif
2189         uhci_td_t *td;
2190         uhci_td_t *td_last = NULL;
2191         uhci_td_t **pp_last;
2192
2193         DPRINTFN(6, "xfer=%p next=%d nframes=%d\n",
2194             xfer, xfer->endpoint->isoc_next, xfer->nframes);
2195
2196         nframes = UREAD2(sc, UHCI_FRNUM);
2197
2198         temp = (nframes - xfer->endpoint->isoc_next) &
2199             (UHCI_VFRAMELIST_COUNT - 1);
2200
2201         if ((xfer->endpoint->is_synced == 0) ||
2202             (temp < xfer->nframes)) {
2203                 /*
2204                  * If there is data underflow or the pipe queue is empty we
2205                  * schedule the transfer a few frames ahead of the current
2206                  * frame position. Else two isochronous transfers might
2207                  * overlap.
2208                  */
2209                 xfer->endpoint->isoc_next = (nframes + 3) & (UHCI_VFRAMELIST_COUNT - 1);
2210                 xfer->endpoint->is_synced = 1;
2211                 DPRINTFN(3, "start next=%d\n", xfer->endpoint->isoc_next);
2212         }
2213         /*
2214          * compute how many milliseconds the insertion is ahead of the
2215          * current frame position:
2216          */
2217         temp = (xfer->endpoint->isoc_next - nframes) &
2218             (UHCI_VFRAMELIST_COUNT - 1);
2219
2220         /*
2221          * pre-compute when the isochronous transfer will be finished:
2222          */
2223         xfer->isoc_time_complete =
2224             usb_isoc_time_expand(&sc->sc_bus, nframes) + temp +
2225             xfer->nframes;
2226
2227         /* get the real number of frames */
2228
2229         nframes = xfer->nframes;
2230
2231         uhci_mem_layout_init(&ml, xfer);
2232
2233         plen = xfer->frlengths;
2234
2235         /* toggle the DMA set we are using */
2236         xfer->flags_int.curr_dma_set ^= 1;
2237
2238         /* get next DMA set */
2239         td = xfer->td_start[xfer->flags_int.curr_dma_set];
2240         xfer->td_transfer_first = td;
2241
2242         pp_last = &sc->sc_isoc_p_last[xfer->endpoint->isoc_next];
2243
2244         /* store starting position */
2245
2246         xfer->qh_pos = xfer->endpoint->isoc_next;
2247
2248         while (nframes--) {
2249                 if (td == NULL) {
2250                         panic("%s:%d: out of TD's\n",
2251                             __FUNCTION__, __LINE__);
2252                 }
2253                 if (pp_last >= &sc->sc_isoc_p_last[UHCI_VFRAMELIST_COUNT]) {
2254                         pp_last = &sc->sc_isoc_p_last[0];
2255                 }
2256                 if (*plen > xfer->max_frame_size) {
2257 #ifdef USB_DEBUG
2258                         if (once) {
2259                                 once = 0;
2260                                 printf("%s: frame length(%d) exceeds %d "
2261                                     "bytes (frame truncated)\n",
2262                                     __FUNCTION__, *plen,
2263                                     xfer->max_frame_size);
2264                         }
2265 #endif
2266                         *plen = xfer->max_frame_size;
2267                 }
2268                 /* reuse td_token from last transfer */
2269
2270                 td->td_token &= htole32(~UHCI_TD_MAXLEN_MASK);
2271                 td->td_token |= htole32(UHCI_TD_SET_MAXLEN(*plen));
2272
2273                 td->len = *plen;
2274
2275                 if (td->len == 0) {
2276                         /*
2277                          * Do not call "uhci_mem_layout_fixup()" when the
2278                          * length is zero!
2279                          */
2280                         td->td_buffer = 0;
2281                         td->fix_pc = NULL;
2282
2283                 } else {
2284
2285                         /* fill out buffer pointer and do fixup, if any */
2286
2287                         uhci_mem_layout_fixup(&ml, td);
2288
2289                 }
2290
2291                 /* update status */
2292                 if (nframes == 0) {
2293                         td->td_status = htole32
2294                             (UHCI_TD_ZERO_ACTLEN
2295                             (UHCI_TD_SET_ERRCNT(0) |
2296                             UHCI_TD_ACTIVE |
2297                             UHCI_TD_IOS |
2298                             UHCI_TD_IOC));
2299                 } else {
2300                         td->td_status = htole32
2301                             (UHCI_TD_ZERO_ACTLEN
2302                             (UHCI_TD_SET_ERRCNT(0) |
2303                             UHCI_TD_ACTIVE |
2304                             UHCI_TD_IOS));
2305                 }
2306
2307                 usb_pc_cpu_flush(td->page_cache);
2308
2309 #ifdef USB_DEBUG
2310                 if (uhcidebug > 5) {
2311                         DPRINTF("TD %d\n", nframes);
2312                         uhci_dump_td(td);
2313                 }
2314 #endif
2315                 /* insert TD into schedule */
2316                 UHCI_APPEND_TD(td, *pp_last);
2317                 pp_last++;
2318
2319                 plen++;
2320                 td_last = td;
2321                 td = td->obj_next;
2322         }
2323
2324         xfer->td_transfer_last = td_last;
2325
2326         /* update isoc_next */
2327         xfer->endpoint->isoc_next = (pp_last - &sc->sc_isoc_p_last[0]) &
2328             (UHCI_VFRAMELIST_COUNT - 1);
2329 }
2330
2331 static void
2332 uhci_device_isoc_start(struct usb_xfer *xfer)
2333 {
2334         /* put transfer on interrupt queue */
2335         uhci_transfer_intr_enqueue(xfer);
2336 }
2337
2338 struct usb_pipe_methods uhci_device_isoc_methods =
2339 {
2340         .open = uhci_device_isoc_open,
2341         .close = uhci_device_isoc_close,
2342         .enter = uhci_device_isoc_enter,
2343         .start = uhci_device_isoc_start,
2344 };
2345
2346 /*------------------------------------------------------------------------*
2347  * uhci root control support
2348  *------------------------------------------------------------------------*
2349  * Simulate a hardware hub by handling all the necessary requests.
2350  *------------------------------------------------------------------------*/
2351
2352 static const
2353 struct usb_device_descriptor uhci_devd =
2354 {
2355         sizeof(struct usb_device_descriptor),
2356         UDESC_DEVICE,                   /* type */
2357         {0x00, 0x01},                   /* USB version */
2358         UDCLASS_HUB,                    /* class */
2359         UDSUBCLASS_HUB,                 /* subclass */
2360         UDPROTO_FSHUB,                  /* protocol */
2361         64,                             /* max packet */
2362         {0}, {0}, {0x00, 0x01},         /* device id */
2363         1, 2, 0,                        /* string indicies */
2364         1                               /* # of configurations */
2365 };
2366
2367 static const struct uhci_config_desc uhci_confd = {
2368         .confd = {
2369                 .bLength = sizeof(struct usb_config_descriptor),
2370                 .bDescriptorType = UDESC_CONFIG,
2371                 .wTotalLength[0] = sizeof(uhci_confd),
2372                 .bNumInterface = 1,
2373                 .bConfigurationValue = 1,
2374                 .iConfiguration = 0,
2375                 .bmAttributes = UC_SELF_POWERED,
2376                 .bMaxPower = 0          /* max power */
2377         },
2378         .ifcd = {
2379                 .bLength = sizeof(struct usb_interface_descriptor),
2380                 .bDescriptorType = UDESC_INTERFACE,
2381                 .bNumEndpoints = 1,
2382                 .bInterfaceClass = UICLASS_HUB,
2383                 .bInterfaceSubClass = UISUBCLASS_HUB,
2384                 .bInterfaceProtocol = UIPROTO_FSHUB,
2385         },
2386         .endpd = {
2387                 .bLength = sizeof(struct usb_endpoint_descriptor),
2388                 .bDescriptorType = UDESC_ENDPOINT,
2389                 .bEndpointAddress = UE_DIR_IN | UHCI_INTR_ENDPT,
2390                 .bmAttributes = UE_INTERRUPT,
2391                 .wMaxPacketSize[0] = 8, /* max packet (63 ports) */
2392                 .bInterval = 255,
2393         },
2394 };
2395
2396 static const
2397 struct usb_hub_descriptor_min uhci_hubd_piix =
2398 {
2399         sizeof(uhci_hubd_piix),
2400         UDESC_HUB,
2401         2,
2402         {UHD_PWR_NO_SWITCH | UHD_OC_INDIVIDUAL, 0},
2403         50,                             /* power on to power good */
2404         0,
2405         {0x00},                         /* both ports are removable */
2406 };
2407
2408 /*
2409  * The USB hub protocol requires that SET_FEATURE(PORT_RESET) also
2410  * enables the port, and also states that SET_FEATURE(PORT_ENABLE)
2411  * should not be used by the USB subsystem.  As we cannot issue a
2412  * SET_FEATURE(PORT_ENABLE) externally, we must ensure that the port
2413  * will be enabled as part of the reset.
2414  *
2415  * On the VT83C572, the port cannot be successfully enabled until the
2416  * outstanding "port enable change" and "connection status change"
2417  * events have been reset.
2418  */
2419 static usb_error_t
2420 uhci_portreset(uhci_softc_t *sc, uint16_t index)
2421 {
2422         uint16_t port;
2423         uint16_t x;
2424         uint8_t lim;
2425
2426         if (index == 1)
2427                 port = UHCI_PORTSC1;
2428         else if (index == 2)
2429                 port = UHCI_PORTSC2;
2430         else
2431                 return (USB_ERR_IOERROR);
2432
2433         /*
2434          * Before we do anything, turn on SOF messages on the USB
2435          * BUS. Some USB devices do not cope without them!
2436          */
2437         uhci_restart(sc);
2438
2439         x = URWMASK(UREAD2(sc, port));
2440         UWRITE2(sc, port, x | UHCI_PORTSC_PR);
2441
2442         usb_pause_mtx(&sc->sc_bus.bus_mtx,
2443             USB_MS_TO_TICKS(USB_PORT_ROOT_RESET_DELAY));
2444
2445         DPRINTFN(4, "uhci port %d reset, status0 = 0x%04x\n",
2446             index, UREAD2(sc, port));
2447
2448         x = URWMASK(UREAD2(sc, port));
2449         UWRITE2(sc, port, x & ~UHCI_PORTSC_PR);
2450
2451
2452         mtx_unlock(&sc->sc_bus.bus_mtx);
2453
2454         /*
2455          * This delay needs to be exactly 100us, else some USB devices
2456          * fail to attach!
2457          */
2458         DELAY(100);
2459
2460         mtx_lock(&sc->sc_bus.bus_mtx);
2461
2462         DPRINTFN(4, "uhci port %d reset, status1 = 0x%04x\n",
2463             index, UREAD2(sc, port));
2464
2465         x = URWMASK(UREAD2(sc, port));
2466         UWRITE2(sc, port, x | UHCI_PORTSC_PE);
2467
2468         for (lim = 0; lim < 12; lim++) {
2469
2470                 usb_pause_mtx(&sc->sc_bus.bus_mtx,
2471                     USB_MS_TO_TICKS(USB_PORT_RESET_DELAY));
2472
2473                 x = UREAD2(sc, port);
2474
2475                 DPRINTFN(4, "uhci port %d iteration %u, status = 0x%04x\n",
2476                     index, lim, x);
2477
2478                 if (!(x & UHCI_PORTSC_CCS)) {
2479                         /*
2480                          * No device is connected (or was disconnected
2481                          * during reset).  Consider the port reset.
2482                          * The delay must be long enough to ensure on
2483                          * the initial iteration that the device
2484                          * connection will have been registered.  50ms
2485                          * appears to be sufficient, but 20ms is not.
2486                          */
2487                         DPRINTFN(4, "uhci port %d loop %u, device detached\n",
2488                             index, lim);
2489                         goto done;
2490                 }
2491                 if (x & (UHCI_PORTSC_POEDC | UHCI_PORTSC_CSC)) {
2492                         /*
2493                          * Port enabled changed and/or connection
2494                          * status changed were set.  Reset either or
2495                          * both raised flags (by writing a 1 to that
2496                          * bit), and wait again for state to settle.
2497                          */
2498                         UWRITE2(sc, port, URWMASK(x) |
2499                             (x & (UHCI_PORTSC_POEDC | UHCI_PORTSC_CSC)));
2500                         continue;
2501                 }
2502                 if (x & UHCI_PORTSC_PE) {
2503                         /* port is enabled */
2504                         goto done;
2505                 }
2506                 UWRITE2(sc, port, URWMASK(x) | UHCI_PORTSC_PE);
2507         }
2508
2509         DPRINTFN(2, "uhci port %d reset timed out\n", index);
2510         return (USB_ERR_TIMEOUT);
2511
2512 done:
2513         DPRINTFN(4, "uhci port %d reset, status2 = 0x%04x\n",
2514             index, UREAD2(sc, port));
2515
2516         sc->sc_isreset = 1;
2517         return (USB_ERR_NORMAL_COMPLETION);
2518 }
2519
2520 static usb_error_t
2521 uhci_roothub_exec(struct usb_device *udev,
2522     struct usb_device_request *req, const void **pptr, uint16_t *plength)
2523 {
2524         uhci_softc_t *sc = UHCI_BUS2SC(udev->bus);
2525         const void *ptr;
2526         const char *str_ptr;
2527         uint16_t x;
2528         uint16_t port;
2529         uint16_t value;
2530         uint16_t index;
2531         uint16_t status;
2532         uint16_t change;
2533         uint16_t len;
2534         usb_error_t err;
2535
2536         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
2537
2538         /* buffer reset */
2539         ptr = (const void *)&sc->sc_hub_desc.temp;
2540         len = 0;
2541         err = 0;
2542
2543         value = UGETW(req->wValue);
2544         index = UGETW(req->wIndex);
2545
2546         DPRINTFN(3, "type=0x%02x request=0x%02x wLen=0x%04x "
2547             "wValue=0x%04x wIndex=0x%04x\n",
2548             req->bmRequestType, req->bRequest,
2549             UGETW(req->wLength), value, index);
2550
2551 #define C(x,y) ((x) | ((y) << 8))
2552         switch (C(req->bRequest, req->bmRequestType)) {
2553         case C(UR_CLEAR_FEATURE, UT_WRITE_DEVICE):
2554         case C(UR_CLEAR_FEATURE, UT_WRITE_INTERFACE):
2555         case C(UR_CLEAR_FEATURE, UT_WRITE_ENDPOINT):
2556                 /*
2557                  * DEVICE_REMOTE_WAKEUP and ENDPOINT_HALT are no-ops
2558                  * for the integrated root hub.
2559                  */
2560                 break;
2561         case C(UR_GET_CONFIG, UT_READ_DEVICE):
2562                 len = 1;
2563                 sc->sc_hub_desc.temp[0] = sc->sc_conf;
2564                 break;
2565         case C(UR_GET_DESCRIPTOR, UT_READ_DEVICE):
2566                 switch (value >> 8) {
2567                 case UDESC_DEVICE:
2568                         if ((value & 0xff) != 0) {
2569                                 err = USB_ERR_IOERROR;
2570                                 goto done;
2571                         }
2572                         len = sizeof(uhci_devd);
2573                         ptr = (const void *)&uhci_devd;
2574                         break;
2575
2576                 case UDESC_CONFIG:
2577                         if ((value & 0xff) != 0) {
2578                                 err = USB_ERR_IOERROR;
2579                                 goto done;
2580                         }
2581                         len = sizeof(uhci_confd);
2582                         ptr = (const void *)&uhci_confd;
2583                         break;
2584
2585                 case UDESC_STRING:
2586                         switch (value & 0xff) {
2587                         case 0: /* Language table */
2588                                 str_ptr = "\001";
2589                                 break;
2590
2591                         case 1: /* Vendor */
2592                                 str_ptr = sc->sc_vendor;
2593                                 break;
2594
2595                         case 2: /* Product */
2596                                 str_ptr = "UHCI root HUB";
2597                                 break;
2598
2599                         default:
2600                                 str_ptr = "";
2601                                 break;
2602                         }
2603
2604                         len = usb_make_str_desc
2605                             (sc->sc_hub_desc.temp,
2606                             sizeof(sc->sc_hub_desc.temp),
2607                             str_ptr);
2608                         break;
2609
2610                 default:
2611                         err = USB_ERR_IOERROR;
2612                         goto done;
2613                 }
2614                 break;
2615         case C(UR_GET_INTERFACE, UT_READ_INTERFACE):
2616                 len = 1;
2617                 sc->sc_hub_desc.temp[0] = 0;
2618                 break;
2619         case C(UR_GET_STATUS, UT_READ_DEVICE):
2620                 len = 2;
2621                 USETW(sc->sc_hub_desc.stat.wStatus, UDS_SELF_POWERED);
2622                 break;
2623         case C(UR_GET_STATUS, UT_READ_INTERFACE):
2624         case C(UR_GET_STATUS, UT_READ_ENDPOINT):
2625                 len = 2;
2626                 USETW(sc->sc_hub_desc.stat.wStatus, 0);
2627                 break;
2628         case C(UR_SET_ADDRESS, UT_WRITE_DEVICE):
2629                 if (value >= UHCI_MAX_DEVICES) {
2630                         err = USB_ERR_IOERROR;
2631                         goto done;
2632                 }
2633                 sc->sc_addr = value;
2634                 break;
2635         case C(UR_SET_CONFIG, UT_WRITE_DEVICE):
2636                 if ((value != 0) && (value != 1)) {
2637                         err = USB_ERR_IOERROR;
2638                         goto done;
2639                 }
2640                 sc->sc_conf = value;
2641                 break;
2642         case C(UR_SET_DESCRIPTOR, UT_WRITE_DEVICE):
2643                 break;
2644         case C(UR_SET_FEATURE, UT_WRITE_DEVICE):
2645         case C(UR_SET_FEATURE, UT_WRITE_INTERFACE):
2646         case C(UR_SET_FEATURE, UT_WRITE_ENDPOINT):
2647                 err = USB_ERR_IOERROR;
2648                 goto done;
2649         case C(UR_SET_INTERFACE, UT_WRITE_INTERFACE):
2650                 break;
2651         case C(UR_SYNCH_FRAME, UT_WRITE_ENDPOINT):
2652                 break;
2653                 /* Hub requests */
2654         case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_DEVICE):
2655                 break;
2656         case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_OTHER):
2657                 DPRINTFN(4, "UR_CLEAR_PORT_FEATURE "
2658                     "port=%d feature=%d\n",
2659                     index, value);
2660                 if (index == 1)
2661                         port = UHCI_PORTSC1;
2662                 else if (index == 2)
2663                         port = UHCI_PORTSC2;
2664                 else {
2665                         err = USB_ERR_IOERROR;
2666                         goto done;
2667                 }
2668                 switch (value) {
2669                 case UHF_PORT_ENABLE:
2670                         x = URWMASK(UREAD2(sc, port));
2671                         UWRITE2(sc, port, x & ~UHCI_PORTSC_PE);
2672                         break;
2673                 case UHF_PORT_SUSPEND:
2674                         x = URWMASK(UREAD2(sc, port));
2675                         UWRITE2(sc, port, x & ~(UHCI_PORTSC_SUSP));
2676                         break;
2677                 case UHF_PORT_RESET:
2678                         x = URWMASK(UREAD2(sc, port));
2679                         UWRITE2(sc, port, x & ~UHCI_PORTSC_PR);
2680                         break;
2681                 case UHF_C_PORT_CONNECTION:
2682                         x = URWMASK(UREAD2(sc, port));
2683                         UWRITE2(sc, port, x | UHCI_PORTSC_CSC);
2684                         break;
2685                 case UHF_C_PORT_ENABLE:
2686                         x = URWMASK(UREAD2(sc, port));
2687                         UWRITE2(sc, port, x | UHCI_PORTSC_POEDC);
2688                         break;
2689                 case UHF_C_PORT_OVER_CURRENT:
2690                         x = URWMASK(UREAD2(sc, port));
2691                         UWRITE2(sc, port, x | UHCI_PORTSC_OCIC);
2692                         break;
2693                 case UHF_C_PORT_RESET:
2694                         sc->sc_isreset = 0;
2695                         err = USB_ERR_NORMAL_COMPLETION;
2696                         goto done;
2697                 case UHF_C_PORT_SUSPEND:
2698                         sc->sc_isresumed &= ~(1 << index);
2699                         break;
2700                 case UHF_PORT_CONNECTION:
2701                 case UHF_PORT_OVER_CURRENT:
2702                 case UHF_PORT_POWER:
2703                 case UHF_PORT_LOW_SPEED:
2704                 default:
2705                         err = USB_ERR_IOERROR;
2706                         goto done;
2707                 }
2708                 break;
2709         case C(UR_GET_BUS_STATE, UT_READ_CLASS_OTHER):
2710                 if (index == 1)
2711                         port = UHCI_PORTSC1;
2712                 else if (index == 2)
2713                         port = UHCI_PORTSC2;
2714                 else {
2715                         err = USB_ERR_IOERROR;
2716                         goto done;
2717                 }
2718                 len = 1;
2719                 sc->sc_hub_desc.temp[0] =
2720                     ((UREAD2(sc, port) & UHCI_PORTSC_LS) >>
2721                     UHCI_PORTSC_LS_SHIFT);
2722                 break;
2723         case C(UR_GET_DESCRIPTOR, UT_READ_CLASS_DEVICE):
2724                 if ((value & 0xff) != 0) {
2725                         err = USB_ERR_IOERROR;
2726                         goto done;
2727                 }
2728                 len = sizeof(uhci_hubd_piix);
2729                 ptr = (const void *)&uhci_hubd_piix;
2730                 break;
2731         case C(UR_GET_STATUS, UT_READ_CLASS_DEVICE):
2732                 len = 16;
2733                 bzero(sc->sc_hub_desc.temp, 16);
2734                 break;
2735         case C(UR_GET_STATUS, UT_READ_CLASS_OTHER):
2736                 if (index == 1)
2737                         port = UHCI_PORTSC1;
2738                 else if (index == 2)
2739                         port = UHCI_PORTSC2;
2740                 else {
2741                         err = USB_ERR_IOERROR;
2742                         goto done;
2743                 }
2744                 x = UREAD2(sc, port);
2745                 status = change = 0;
2746                 if (x & UHCI_PORTSC_CCS)
2747                         status |= UPS_CURRENT_CONNECT_STATUS;
2748                 if (x & UHCI_PORTSC_CSC)
2749                         change |= UPS_C_CONNECT_STATUS;
2750                 if (x & UHCI_PORTSC_PE)
2751                         status |= UPS_PORT_ENABLED;
2752                 if (x & UHCI_PORTSC_POEDC)
2753                         change |= UPS_C_PORT_ENABLED;
2754                 if (x & UHCI_PORTSC_OCI)
2755                         status |= UPS_OVERCURRENT_INDICATOR;
2756                 if (x & UHCI_PORTSC_OCIC)
2757                         change |= UPS_C_OVERCURRENT_INDICATOR;
2758                 if (x & UHCI_PORTSC_LSDA)
2759                         status |= UPS_LOW_SPEED;
2760                 if ((x & UHCI_PORTSC_PE) && (x & UHCI_PORTSC_RD)) {
2761                         /* need to do a write back */
2762                         UWRITE2(sc, port, URWMASK(x));
2763
2764                         /* wait 20ms for resume sequence to complete */
2765                         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 50);
2766
2767                         /* clear suspend and resume detect */
2768                         UWRITE2(sc, port, URWMASK(x) & ~(UHCI_PORTSC_RD |
2769                             UHCI_PORTSC_SUSP));
2770
2771                         /* wait a little bit */
2772                         usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 500);
2773
2774                         sc->sc_isresumed |= (1 << index);
2775
2776                 } else if (x & UHCI_PORTSC_SUSP) {
2777                         status |= UPS_SUSPEND;
2778                 }
2779                 status |= UPS_PORT_POWER;
2780                 if (sc->sc_isresumed & (1 << index))
2781                         change |= UPS_C_SUSPEND;
2782                 if (sc->sc_isreset)
2783                         change |= UPS_C_PORT_RESET;
2784                 USETW(sc->sc_hub_desc.ps.wPortStatus, status);
2785                 USETW(sc->sc_hub_desc.ps.wPortChange, change);
2786                 len = sizeof(sc->sc_hub_desc.ps);
2787                 break;
2788         case C(UR_SET_DESCRIPTOR, UT_WRITE_CLASS_DEVICE):
2789                 err = USB_ERR_IOERROR;
2790                 goto done;
2791         case C(UR_SET_FEATURE, UT_WRITE_CLASS_DEVICE):
2792                 break;
2793         case C(UR_SET_FEATURE, UT_WRITE_CLASS_OTHER):
2794                 if (index == 1)
2795                         port = UHCI_PORTSC1;
2796                 else if (index == 2)
2797                         port = UHCI_PORTSC2;
2798                 else {
2799                         err = USB_ERR_IOERROR;
2800                         goto done;
2801                 }
2802                 switch (value) {
2803                 case UHF_PORT_ENABLE:
2804                         x = URWMASK(UREAD2(sc, port));
2805                         UWRITE2(sc, port, x | UHCI_PORTSC_PE);
2806                         break;
2807                 case UHF_PORT_SUSPEND:
2808                         x = URWMASK(UREAD2(sc, port));
2809                         UWRITE2(sc, port, x | UHCI_PORTSC_SUSP);
2810                         break;
2811                 case UHF_PORT_RESET:
2812                         err = uhci_portreset(sc, index);
2813                         goto done;
2814                 case UHF_PORT_POWER:
2815                         /* pretend we turned on power */
2816                         err = USB_ERR_NORMAL_COMPLETION;
2817                         goto done;
2818                 case UHF_C_PORT_CONNECTION:
2819                 case UHF_C_PORT_ENABLE:
2820                 case UHF_C_PORT_OVER_CURRENT:
2821                 case UHF_PORT_CONNECTION:
2822                 case UHF_PORT_OVER_CURRENT:
2823                 case UHF_PORT_LOW_SPEED:
2824                 case UHF_C_PORT_SUSPEND:
2825                 case UHF_C_PORT_RESET:
2826                 default:
2827                         err = USB_ERR_IOERROR;
2828                         goto done;
2829                 }
2830                 break;
2831         default:
2832                 err = USB_ERR_IOERROR;
2833                 goto done;
2834         }
2835 done:
2836         *plength = len;
2837         *pptr = ptr;
2838         return (err);
2839 }
2840
2841 /*
2842  * This routine is executed periodically and simulates interrupts from
2843  * the root controller interrupt pipe for port status change:
2844  */
2845 static void
2846 uhci_root_intr(uhci_softc_t *sc)
2847 {
2848         DPRINTFN(21, "\n");
2849
2850         USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
2851
2852         sc->sc_hub_idata[0] = 0;
2853
2854         if (UREAD2(sc, UHCI_PORTSC1) & (UHCI_PORTSC_CSC |
2855             UHCI_PORTSC_OCIC | UHCI_PORTSC_RD)) {
2856                 sc->sc_hub_idata[0] |= 1 << 1;
2857         }
2858         if (UREAD2(sc, UHCI_PORTSC2) & (UHCI_PORTSC_CSC |
2859             UHCI_PORTSC_OCIC | UHCI_PORTSC_RD)) {
2860                 sc->sc_hub_idata[0] |= 1 << 2;
2861         }
2862
2863         /* restart timer */
2864         usb_callout_reset(&sc->sc_root_intr, hz,
2865             (void *)&uhci_root_intr, sc);
2866
2867         if (sc->sc_hub_idata[0] != 0) {
2868                 uhub_root_intr(&sc->sc_bus, sc->sc_hub_idata,
2869                     sizeof(sc->sc_hub_idata));
2870         }
2871 }
2872
2873 static void
2874 uhci_xfer_setup(struct usb_setup_params *parm)
2875 {
2876         struct usb_page_search page_info;
2877         struct usb_page_cache *pc;
2878         uhci_softc_t *sc;
2879         struct usb_xfer *xfer;
2880         void *last_obj;
2881         uint32_t ntd;
2882         uint32_t nqh;
2883         uint32_t nfixup;
2884         uint32_t n;
2885         uint16_t align;
2886
2887         sc = UHCI_BUS2SC(parm->udev->bus);
2888         xfer = parm->curr_xfer;
2889
2890         parm->hc_max_packet_size = 0x500;
2891         parm->hc_max_packet_count = 1;
2892         parm->hc_max_frame_size = 0x500;
2893
2894         /*
2895          * compute ntd and nqh
2896          */
2897         if (parm->methods == &uhci_device_ctrl_methods) {
2898                 xfer->flags_int.bdma_enable = 1;
2899                 xfer->flags_int.bdma_no_post_sync = 1;
2900
2901                 usbd_transfer_setup_sub(parm);
2902
2903                 /* see EHCI HC driver for proof of "ntd" formula */
2904
2905                 nqh = 1;
2906                 ntd = ((2 * xfer->nframes) + 1  /* STATUS */
2907                     + (xfer->max_data_length / xfer->max_frame_size));
2908
2909         } else if (parm->methods == &uhci_device_bulk_methods) {
2910                 xfer->flags_int.bdma_enable = 1;
2911                 xfer->flags_int.bdma_no_post_sync = 1;
2912
2913                 usbd_transfer_setup_sub(parm);
2914
2915                 nqh = 1;
2916                 ntd = ((2 * xfer->nframes)
2917                     + (xfer->max_data_length / xfer->max_frame_size));
2918
2919         } else if (parm->methods == &uhci_device_intr_methods) {
2920                 xfer->flags_int.bdma_enable = 1;
2921                 xfer->flags_int.bdma_no_post_sync = 1;
2922
2923                 usbd_transfer_setup_sub(parm);
2924
2925                 nqh = 1;
2926                 ntd = ((2 * xfer->nframes)
2927                     + (xfer->max_data_length / xfer->max_frame_size));
2928
2929         } else if (parm->methods == &uhci_device_isoc_methods) {
2930                 xfer->flags_int.bdma_enable = 1;
2931                 xfer->flags_int.bdma_no_post_sync = 1;
2932
2933                 usbd_transfer_setup_sub(parm);
2934
2935                 nqh = 0;
2936                 ntd = xfer->nframes;
2937
2938         } else {
2939
2940                 usbd_transfer_setup_sub(parm);
2941
2942                 nqh = 0;
2943                 ntd = 0;
2944         }
2945
2946         if (parm->err) {
2947                 return;
2948         }
2949         /*
2950          * NOTE: the UHCI controller requires that
2951          * every packet must be contiguous on
2952          * the same USB memory page !
2953          */
2954         nfixup = (parm->bufsize / USB_PAGE_SIZE) + 1;
2955
2956         /*
2957          * Compute a suitable power of two alignment
2958          * for our "max_frame_size" fixup buffer(s):
2959          */
2960         align = xfer->max_frame_size;
2961         n = 0;
2962         while (align) {
2963                 align >>= 1;
2964                 n++;
2965         }
2966
2967         /* check for power of two */
2968         if (!(xfer->max_frame_size &
2969             (xfer->max_frame_size - 1))) {
2970                 n--;
2971         }
2972         /*
2973          * We don't allow alignments of
2974          * less than 8 bytes:
2975          *
2976          * NOTE: Allocating using an aligment
2977          * of 1 byte has special meaning!
2978          */
2979         if (n < 3) {
2980                 n = 3;
2981         }
2982         align = (1 << n);
2983
2984         if (usbd_transfer_setup_sub_malloc(
2985             parm, &pc, xfer->max_frame_size,
2986             align, nfixup)) {
2987                 parm->err = USB_ERR_NOMEM;
2988                 return;
2989         }
2990         xfer->buf_fixup = pc;
2991
2992 alloc_dma_set:
2993
2994         if (parm->err) {
2995                 return;
2996         }
2997         last_obj = NULL;
2998
2999         if (usbd_transfer_setup_sub_malloc(
3000             parm, &pc, sizeof(uhci_td_t),
3001             UHCI_TD_ALIGN, ntd)) {
3002                 parm->err = USB_ERR_NOMEM;
3003                 return;
3004         }
3005         if (parm->buf) {
3006                 for (n = 0; n != ntd; n++) {
3007                         uhci_td_t *td;
3008
3009                         usbd_get_page(pc + n, 0, &page_info);
3010
3011                         td = page_info.buffer;
3012
3013                         /* init TD */
3014                         if ((parm->methods == &uhci_device_bulk_methods) ||
3015                             (parm->methods == &uhci_device_ctrl_methods) ||
3016                             (parm->methods == &uhci_device_intr_methods)) {
3017                                 /* set depth first bit */
3018                                 td->td_self = htole32(page_info.physaddr |
3019                                     UHCI_PTR_TD | UHCI_PTR_VF);
3020                         } else {
3021                                 td->td_self = htole32(page_info.physaddr |
3022                                     UHCI_PTR_TD);
3023                         }
3024
3025                         td->obj_next = last_obj;
3026                         td->page_cache = pc + n;
3027
3028                         last_obj = td;
3029
3030                         usb_pc_cpu_flush(pc + n);
3031                 }
3032         }
3033         xfer->td_start[xfer->flags_int.curr_dma_set] = last_obj;
3034
3035         last_obj = NULL;
3036
3037         if (usbd_transfer_setup_sub_malloc(
3038             parm, &pc, sizeof(uhci_qh_t),
3039             UHCI_QH_ALIGN, nqh)) {
3040                 parm->err = USB_ERR_NOMEM;
3041                 return;
3042         }
3043         if (parm->buf) {
3044                 for (n = 0; n != nqh; n++) {
3045                         uhci_qh_t *qh;
3046
3047                         usbd_get_page(pc + n, 0, &page_info);
3048
3049                         qh = page_info.buffer;
3050
3051                         /* init QH */
3052                         qh->qh_self = htole32(page_info.physaddr | UHCI_PTR_QH);
3053                         qh->obj_next = last_obj;
3054                         qh->page_cache = pc + n;
3055
3056                         last_obj = qh;
3057
3058                         usb_pc_cpu_flush(pc + n);
3059                 }
3060         }
3061         xfer->qh_start[xfer->flags_int.curr_dma_set] = last_obj;
3062
3063         if (!xfer->flags_int.curr_dma_set) {
3064                 xfer->flags_int.curr_dma_set = 1;
3065                 goto alloc_dma_set;
3066         }
3067 }
3068
3069 static void
3070 uhci_ep_init(struct usb_device *udev, struct usb_endpoint_descriptor *edesc,
3071     struct usb_endpoint *ep)
3072 {
3073         uhci_softc_t *sc = UHCI_BUS2SC(udev->bus);
3074
3075         DPRINTFN(2, "endpoint=%p, addr=%d, endpt=%d, mode=%d (%d)\n",
3076             ep, udev->address,
3077             edesc->bEndpointAddress, udev->flags.usb_mode,
3078             sc->sc_addr);
3079
3080         if (udev->flags.usb_mode != USB_MODE_HOST) {
3081                 /* not supported */
3082                 return;
3083         }
3084         if (udev->device_index != sc->sc_addr) {
3085                 switch (edesc->bmAttributes & UE_XFERTYPE) {
3086                 case UE_CONTROL:
3087                         ep->methods = &uhci_device_ctrl_methods;
3088                         break;
3089                 case UE_INTERRUPT:
3090                         ep->methods = &uhci_device_intr_methods;
3091                         break;
3092                 case UE_ISOCHRONOUS:
3093                         if (udev->speed == USB_SPEED_FULL) {
3094                                 ep->methods = &uhci_device_isoc_methods;
3095                         }
3096                         break;
3097                 case UE_BULK:
3098                         if (udev->speed != USB_SPEED_LOW) {
3099                                 ep->methods = &uhci_device_bulk_methods;
3100                         }
3101                         break;
3102                 default:
3103                         /* do nothing */
3104                         break;
3105                 }
3106         }
3107 }
3108
3109 static void
3110 uhci_xfer_unsetup(struct usb_xfer *xfer)
3111 {
3112         return;
3113 }
3114
3115 static void
3116 uhci_get_dma_delay(struct usb_bus *bus, uint32_t *pus)
3117 {
3118         /*
3119          * Wait until hardware has finished any possible use of the
3120          * transfer descriptor(s) and QH
3121          */
3122         *pus = (1125);                  /* microseconds */
3123 }
3124
3125 static void
3126 uhci_device_resume(struct usb_device *udev)
3127 {
3128         struct uhci_softc *sc = UHCI_BUS2SC(udev->bus);
3129         struct usb_xfer *xfer;
3130         struct usb_pipe_methods *methods;
3131         uhci_qh_t *qh;
3132
3133         DPRINTF("\n");
3134
3135         USB_BUS_LOCK(udev->bus);
3136
3137         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
3138
3139                 if (xfer->xroot->udev == udev) {
3140
3141                         methods = xfer->endpoint->methods;
3142                         qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
3143
3144                         if (methods == &uhci_device_bulk_methods) {
3145                                 UHCI_APPEND_QH(qh, sc->sc_bulk_p_last);
3146                                 uhci_add_loop(sc);
3147                                 xfer->flags_int.bandwidth_reclaimed = 1;
3148                         }
3149                         if (methods == &uhci_device_ctrl_methods) {
3150                                 if (xfer->xroot->udev->speed == USB_SPEED_LOW) {
3151                                         UHCI_APPEND_QH(qh, sc->sc_ls_ctl_p_last);
3152                                 } else {
3153                                         UHCI_APPEND_QH(qh, sc->sc_fs_ctl_p_last);
3154                                 }
3155                         }
3156                         if (methods == &uhci_device_intr_methods) {
3157                                 UHCI_APPEND_QH(qh, sc->sc_intr_p_last[xfer->qh_pos]);
3158                         }
3159                 }
3160         }
3161
3162         USB_BUS_UNLOCK(udev->bus);
3163
3164         return;
3165 }
3166
3167 static void
3168 uhci_device_suspend(struct usb_device *udev)
3169 {
3170         struct uhci_softc *sc = UHCI_BUS2SC(udev->bus);
3171         struct usb_xfer *xfer;
3172         struct usb_pipe_methods *methods;
3173         uhci_qh_t *qh;
3174
3175         DPRINTF("\n");
3176
3177         USB_BUS_LOCK(udev->bus);
3178
3179         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
3180
3181                 if (xfer->xroot->udev == udev) {
3182
3183                         methods = xfer->endpoint->methods;
3184                         qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
3185
3186                         if (xfer->flags_int.bandwidth_reclaimed) {
3187                                 xfer->flags_int.bandwidth_reclaimed = 0;
3188                                 uhci_rem_loop(sc);
3189                         }
3190                         if (methods == &uhci_device_bulk_methods) {
3191                                 UHCI_REMOVE_QH(qh, sc->sc_bulk_p_last);
3192                         }
3193                         if (methods == &uhci_device_ctrl_methods) {
3194                                 if (xfer->xroot->udev->speed == USB_SPEED_LOW) {
3195                                         UHCI_REMOVE_QH(qh, sc->sc_ls_ctl_p_last);
3196                                 } else {
3197                                         UHCI_REMOVE_QH(qh, sc->sc_fs_ctl_p_last);
3198                                 }
3199                         }
3200                         if (methods == &uhci_device_intr_methods) {
3201                                 UHCI_REMOVE_QH(qh, sc->sc_intr_p_last[xfer->qh_pos]);
3202                         }
3203                 }
3204         }
3205
3206         USB_BUS_UNLOCK(udev->bus);
3207
3208         return;
3209 }
3210
3211 static void
3212 uhci_set_hw_power(struct usb_bus *bus)
3213 {
3214         struct uhci_softc *sc = UHCI_BUS2SC(bus);
3215         uint32_t flags;
3216
3217         DPRINTF("\n");
3218
3219         USB_BUS_LOCK(bus);
3220
3221         flags = bus->hw_power_state;
3222
3223         /*
3224          * WARNING: Some FULL speed USB devices require periodic SOF
3225          * messages! If any USB devices are connected through the
3226          * UHCI, power save will be disabled!
3227          */
3228         if (flags & (USB_HW_POWER_CONTROL |
3229             USB_HW_POWER_NON_ROOT_HUB |
3230             USB_HW_POWER_BULK |
3231             USB_HW_POWER_INTERRUPT |
3232             USB_HW_POWER_ISOC)) {
3233                 DPRINTF("Some USB transfer is "
3234                     "active on unit %u.\n",
3235                     device_get_unit(sc->sc_bus.bdev));
3236                 uhci_restart(sc);
3237         } else {
3238                 DPRINTF("Power save on unit %u.\n",
3239                     device_get_unit(sc->sc_bus.bdev));
3240                 UHCICMD(sc, UHCI_CMD_MAXP);
3241         }
3242
3243         USB_BUS_UNLOCK(bus);
3244
3245         return;
3246 }
3247
3248
3249 struct usb_bus_methods uhci_bus_methods =
3250 {
3251         .endpoint_init = uhci_ep_init,
3252         .xfer_setup = uhci_xfer_setup,
3253         .xfer_unsetup = uhci_xfer_unsetup,
3254         .get_dma_delay = uhci_get_dma_delay,
3255         .device_resume = uhci_device_resume,
3256         .device_suspend = uhci_device_suspend,
3257         .set_hw_power = uhci_set_hw_power,
3258         .roothub_exec = uhci_roothub_exec,
3259         .xfer_poll = uhci_do_poll,
3260 };