sys/dev/twe/twe_freebsd.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2000 Michael Smith
   5  * Copyright (c) 2003 Paul Saab
   6  * Copyright (c) 2003 Vinod Kashyap
   7  * Copyright (c) 2000 BSDi
   8  * All rights reserved.
   9  *
  10  * Redistribution and use in source and binary forms, with or without
  11  * modification, are permitted provided that the following conditions
  12  * are met:
  13  * 1. Redistributions of source code must retain the above copyright
  14  *    notice, this list of conditions and the following disclaimer.
  15  * 2. Redistributions in binary form must reproduce the above copyright
  16  *    notice, this list of conditions and the following disclaimer in the
  17  *    documentation and/or other materials provided with the distribution.
  18  *
  19  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  20  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  21  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  22  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  23  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  24  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  25  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  26  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  27  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  28  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  29  * SUCH DAMAGE.
  30  */
  31
  32 #include <sys/cdefs.h>
  33 __FBSDID("$FreeBSD$");
  34
  35 /*
  36  * FreeBSD-specific code.
  37  */
  38
  39 #include <dev/twe/twe_compat.h>
  40 #include <dev/twe/twereg.h>
  41 #include <dev/twe/tweio.h>
  42 #include <dev/twe/twevar.h>
  43 #include <dev/twe/twe_tables.h>
  44
  45 #include <vm/vm.h>
  46
  47 static devclass_t       twe_devclass;
  48
  49 #ifdef TWE_DEBUG
  50 static u_int32_t        twed_bio_in;
  51 #define TWED_BIO_IN     twed_bio_in++
  52 static u_int32_t        twed_bio_out;
  53 #define TWED_BIO_OUT    twed_bio_out++
  54 #else
  55 #define TWED_BIO_IN
  56 #define TWED_BIO_OUT
  57 #endif
  58
  59 static void     twe_setup_data_dmamap(void *arg, bus_dma_segment_t *segs, int nsegments, int error);
  60 static void     twe_setup_request_dmamap(void *arg, bus_dma_segment_t *segs, int nsegments, int error);
  61
  62 /********************************************************************************
  63  ********************************************************************************
  64                                                          Control device interface
  65  ********************************************************************************
  66  ********************************************************************************/
  67
  68 static  d_open_t                twe_open;
  69 static  d_close_t               twe_close;
  70 static  d_ioctl_t               twe_ioctl_wrapper;
  71
  72 static struct cdevsw twe_cdevsw = {
  73         .d_version =    D_VERSION,
  74         .d_open =       twe_open,
  75         .d_close =      twe_close,
  76         .d_ioctl =      twe_ioctl_wrapper,
  77         .d_name =       "twe",
  78 };
  79
  80 /********************************************************************************
  81  * Accept an open operation on the control device.
  82  */
  83 static int
  84 twe_open(struct cdev *dev, int flags, int fmt, struct thread *td)
  85 {
  86     struct twe_softc            *sc = (struct twe_softc *)dev->si_drv1;
  87
  88     TWE_IO_LOCK(sc);
  89     if (sc->twe_state & TWE_STATE_DETACHING) {
  90         TWE_IO_UNLOCK(sc);
  91         return (ENXIO);
  92     }
  93     sc->twe_state |= TWE_STATE_OPEN;
  94     TWE_IO_UNLOCK(sc);
  95     return(0);
  96 }
  97
  98 /********************************************************************************
  99  * Accept the last close on the control device.
 100  */
 101 static int
 102 twe_close(struct cdev *dev, int flags, int fmt, struct thread *td)
 103 {
 104     struct twe_softc            *sc = (struct twe_softc *)dev->si_drv1;
 105
 106     TWE_IO_LOCK(sc);
 107     sc->twe_state &= ~TWE_STATE_OPEN;
 108     TWE_IO_UNLOCK(sc);
 109     return (0);
 110 }
 111
 112 /********************************************************************************
 113  * Handle controller-specific control operations.
 114  */
 115 static int
 116 twe_ioctl_wrapper(struct cdev *dev, u_long cmd, caddr_t addr, int32_t flag, struct thread *td)
 117 {
 118     struct twe_softc            *sc = (struct twe_softc *)dev->si_drv1;
 119
 120     return(twe_ioctl(sc, cmd, addr));
 121 }
 122
 123 /********************************************************************************
 124  ********************************************************************************
 125                                                              PCI device interface
 126  ********************************************************************************
 127  ********************************************************************************/
 128
 129 static int      twe_probe(device_t dev);
 130 static int      twe_attach(device_t dev);
 131 static void     twe_free(struct twe_softc *sc);
 132 static int      twe_detach(device_t dev);
 133 static int      twe_shutdown(device_t dev);
 134 static int      twe_suspend(device_t dev);
 135 static int      twe_resume(device_t dev);
 136 static void     twe_pci_intr(void *arg);
 137 static void     twe_intrhook(void *arg);
 138
 139 static device_method_t twe_methods[] = {
 140     /* Device interface */
 141     DEVMETHOD(device_probe,     twe_probe),
 142     DEVMETHOD(device_attach,    twe_attach),
 143     DEVMETHOD(device_detach,    twe_detach),
 144     DEVMETHOD(device_shutdown,  twe_shutdown),
 145     DEVMETHOD(device_suspend,   twe_suspend),
 146     DEVMETHOD(device_resume,    twe_resume),
 147
 148     DEVMETHOD_END
 149 };
 150
 151 static driver_t twe_pci_driver = {
 152         "twe",
 153         twe_methods,
 154         sizeof(struct twe_softc)
 155 };
 156
 157 DRIVER_MODULE(twe, pci, twe_pci_driver, twe_devclass, 0, 0);
 158
 159 /********************************************************************************
 160  * Match a 3ware Escalade ATA RAID controller.
 161  */
 162 static int
 163 twe_probe(device_t dev)
 164 {
 165
 166     debug_called(4);
 167
 168     if ((pci_get_vendor(dev) == TWE_VENDOR_ID) &&
 169         ((pci_get_device(dev) == TWE_DEVICE_ID) ||
 170          (pci_get_device(dev) == TWE_DEVICE_ID_ASIC))) {
 171         device_set_desc_copy(dev, TWE_DEVICE_NAME ". Driver version " TWE_DRIVER_VERSION_STRING);
 172         return(BUS_PROBE_DEFAULT);
 173     }
 174     return(ENXIO);
 175 }
 176
 177 /********************************************************************************
 178  * Allocate resources, initialise the controller.
 179  */
 180 static int
 181 twe_attach(device_t dev)
 182 {
 183     struct twe_softc    *sc;
 184     struct sysctl_oid   *sysctl_tree;
 185     int                 rid, error;
 186
 187     debug_called(4);
 188
 189     /*
 190      * Initialise the softc structure.
 191      */
 192     sc = device_get_softc(dev);
 193     sc->twe_dev = dev;
 194     mtx_init(&sc->twe_io_lock, "twe I/O", NULL, MTX_DEF);
 195     sx_init(&sc->twe_config_lock, "twe config");
 196
 197     /*
 198      * XXX: This sysctl tree must stay at hw.tweX rather than using
 199      * the device_get_sysctl_tree() created by new-bus because
 200      * existing 3rd party binary tools such as tw_cli and 3dm2 use the
 201      * existence of this sysctl node to discover controllers.
 202      */
 203     sysctl_tree = SYSCTL_ADD_NODE(device_get_sysctl_ctx(dev),
 204         SYSCTL_STATIC_CHILDREN(_hw), OID_AUTO,
 205         device_get_nameunit(dev), CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "");
 206     if (sysctl_tree == NULL) {
 207         twe_printf(sc, "cannot add sysctl tree node\n");
 208         return (ENXIO);
 209     }
 210     SYSCTL_ADD_STRING(device_get_sysctl_ctx(dev), SYSCTL_CHILDREN(sysctl_tree),
 211         OID_AUTO, "driver_version", CTLFLAG_RD, TWE_DRIVER_VERSION_STRING, 0,
 212         "TWE driver version");
 213
 214     /*
 215      * Force the busmaster enable bit on, in case the BIOS forgot.
 216      */
 217     pci_enable_busmaster(dev);
 218
 219     /*
 220      * Allocate the PCI register window.
 221      */
 222     rid = TWE_IO_CONFIG_REG;
 223     if ((sc->twe_io = bus_alloc_resource_any(dev, SYS_RES_IOPORT, &rid,
 224         RF_ACTIVE)) == NULL) {
 225         twe_printf(sc, "can't allocate register window\n");
 226         twe_free(sc);
 227         return(ENXIO);
 228     }
 229
 230     /*
 231      * Allocate the parent bus DMA tag appropriate for PCI.
 232      */
 233     if (bus_dma_tag_create(bus_get_dma_tag(dev),                /* PCI parent */
 234                            1, 0,                                /* alignment, boundary */
 235                            BUS_SPACE_MAXADDR_32BIT,             /* lowaddr */
 236                            BUS_SPACE_MAXADDR,                   /* highaddr */
 237                            NULL, NULL,                          /* filter, filterarg */
 238                            BUS_SPACE_MAXSIZE_32BIT,             /* maxsize */
 239                            BUS_SPACE_UNRESTRICTED,              /* nsegments */
 240                            BUS_SPACE_MAXSIZE_32BIT,             /* maxsegsize */
 241                            0,                                   /* flags */
 242                            NULL,                                /* lockfunc */
 243                            NULL,                                /* lockarg */
 244                            &sc->twe_parent_dmat)) {
 245         twe_printf(sc, "can't allocate parent DMA tag\n");
 246         twe_free(sc);
 247         return(ENOMEM);
 248     }
 249
 250     /*
 251      * Allocate and connect our interrupt.
 252      */
 253     rid = 0;
 254     if ((sc->twe_irq = bus_alloc_resource_any(sc->twe_dev, SYS_RES_IRQ,
 255         &rid, RF_SHAREABLE | RF_ACTIVE)) == NULL) {
 256         twe_printf(sc, "can't allocate interrupt\n");
 257         twe_free(sc);
 258         return(ENXIO);
 259     }
 260     if (bus_setup_intr(sc->twe_dev, sc->twe_irq, INTR_TYPE_BIO | INTR_ENTROPY | INTR_MPSAFE,
 261                        NULL, twe_pci_intr, sc, &sc->twe_intr)) {
 262         twe_printf(sc, "can't set up interrupt\n");
 263         twe_free(sc);
 264         return(ENXIO);
 265     }
 266
 267     /*
 268      * Create DMA tag for mapping command's into controller-addressable space.
 269      */
 270     if (bus_dma_tag_create(sc->twe_parent_dmat,         /* parent */
 271                            1, 0,                        /* alignment, boundary */
 272                            BUS_SPACE_MAXADDR_32BIT,     /* lowaddr */
 273                            BUS_SPACE_MAXADDR,           /* highaddr */
 274                            NULL, NULL,                  /* filter, filterarg */
 275                            sizeof(TWE_Command) *
 276                            TWE_Q_LENGTH, 1,             /* maxsize, nsegments */
 277                            BUS_SPACE_MAXSIZE_32BIT,     /* maxsegsize */
 278                            0,                           /* flags */
 279                            NULL,                        /* lockfunc */
 280                            NULL,                        /* lockarg */
 281                            &sc->twe_cmd_dmat)) {
 282         twe_printf(sc, "can't allocate data buffer DMA tag\n");
 283         twe_free(sc);
 284         return(ENOMEM);
 285     }
 286     /*
 287      * Allocate memory and make it available for DMA.
 288      */
 289     if (bus_dmamem_alloc(sc->twe_cmd_dmat, (void **)&sc->twe_cmd,
 290                          BUS_DMA_NOWAIT, &sc->twe_cmdmap)) {
 291         twe_printf(sc, "can't allocate command memory\n");
 292         return(ENOMEM);
 293     }
 294     bus_dmamap_load(sc->twe_cmd_dmat, sc->twe_cmdmap, sc->twe_cmd,
 295                     sizeof(TWE_Command) * TWE_Q_LENGTH,
 296                     twe_setup_request_dmamap, sc, 0);
 297     bzero(sc->twe_cmd, sizeof(TWE_Command) * TWE_Q_LENGTH);
 298
 299     /*
 300      * Create DMA tag for mapping objects into controller-addressable space.
 301      */
 302     if (bus_dma_tag_create(sc->twe_parent_dmat,         /* parent */
 303                            1, 0,                        /* alignment, boundary */
 304                            BUS_SPACE_MAXADDR_32BIT,     /* lowaddr */
 305                            BUS_SPACE_MAXADDR,           /* highaddr */
 306                            NULL, NULL,                  /* filter, filterarg */
 307                            (TWE_MAX_SGL_LENGTH - 1) * PAGE_SIZE,/* maxsize */
 308                            TWE_MAX_SGL_LENGTH,          /* nsegments */
 309                            BUS_SPACE_MAXSIZE_32BIT,     /* maxsegsize */
 310                            BUS_DMA_ALLOCNOW,            /* flags */
 311                            busdma_lock_mutex,           /* lockfunc */
 312                            &sc->twe_io_lock,            /* lockarg */
 313                            &sc->twe_buffer_dmat)) {
 314         twe_printf(sc, "can't allocate data buffer DMA tag\n");
 315         twe_free(sc);
 316         return(ENOMEM);
 317     }
 318
 319     /*
 320      * Create DMA tag for mapping objects into controller-addressable space.
 321      */
 322     if (bus_dma_tag_create(sc->twe_parent_dmat,         /* parent */
 323                            1, 0,                        /* alignment, boundary */
 324                            BUS_SPACE_MAXADDR_32BIT,     /* lowaddr */
 325                            BUS_SPACE_MAXADDR,           /* highaddr */
 326                            NULL, NULL,                  /* filter, filterarg */
 327                            DFLTPHYS, 1,                 /* maxsize, nsegments */
 328                            BUS_SPACE_MAXSIZE_32BIT,     /* maxsegsize */
 329                            0,                           /* flags */
 330                            NULL,                        /* lockfunc */
 331                            NULL,                        /* lockarg */
 332                            &sc->twe_immediate_dmat)) {
 333         twe_printf(sc, "can't allocate data buffer DMA tag\n");
 334         twe_free(sc);
 335         return(ENOMEM);
 336     }
 337     /*
 338      * Allocate memory for requests which cannot sleep or support continuation.
 339      */
 340      if (bus_dmamem_alloc(sc->twe_immediate_dmat, (void **)&sc->twe_immediate,
 341                           BUS_DMA_NOWAIT, &sc->twe_immediate_map)) {
 342         twe_printf(sc, "can't allocate memory for immediate requests\n");
 343         return(ENOMEM);
 344      }
 345
 346     /*
 347      * Initialise the controller and driver core.
 348      */
 349     if ((error = twe_setup(sc))) {
 350         twe_free(sc);
 351         return(error);
 352     }
 353
 354     /*
 355      * Print some information about the controller and configuration.
 356      */
 357     twe_describe_controller(sc);
 358
 359     /*
 360      * Create the control device.
 361      */
 362     sc->twe_dev_t = make_dev(&twe_cdevsw, device_get_unit(sc->twe_dev), UID_ROOT, GID_OPERATOR,
 363                              S_IRUSR | S_IWUSR, "twe%d", device_get_unit(sc->twe_dev));
 364     sc->twe_dev_t->si_drv1 = sc;
 365     /*
 366      * Schedule ourselves to bring the controller up once interrupts are available.
 367      * This isn't strictly necessary, since we disable interrupts while probing the
 368      * controller, but it is more in keeping with common practice for other disk
 369      * devices.
 370      */
 371     sc->twe_ich.ich_func = twe_intrhook;
 372     sc->twe_ich.ich_arg = sc;
 373     if (config_intrhook_establish(&sc->twe_ich) != 0) {
 374         twe_printf(sc, "can't establish configuration hook\n");
 375         twe_free(sc);
 376         return(ENXIO);
 377     }
 378
 379     return(0);
 380 }
 381
 382 /********************************************************************************
 383  * Free all of the resources associated with (sc).
 384  *
 385  * Should not be called if the controller is active.
 386  */
 387 static void
 388 twe_free(struct twe_softc *sc)
 389 {
 390     struct twe_request  *tr;
 391
 392     debug_called(4);
 393
 394     /* throw away any command buffers */
 395     while ((tr = twe_dequeue_free(sc)) != NULL)
 396         twe_free_request(tr);
 397
 398     if (sc->twe_cmd != NULL) {
 399         bus_dmamap_unload(sc->twe_cmd_dmat, sc->twe_cmdmap);
 400         bus_dmamem_free(sc->twe_cmd_dmat, sc->twe_cmd, sc->twe_cmdmap);
 401     }
 402
 403     if (sc->twe_immediate != NULL) {
 404         bus_dmamap_unload(sc->twe_immediate_dmat, sc->twe_immediate_map);
 405         bus_dmamem_free(sc->twe_immediate_dmat, sc->twe_immediate,
 406                         sc->twe_immediate_map);
 407     }
 408
 409     if (sc->twe_immediate_dmat)
 410         bus_dma_tag_destroy(sc->twe_immediate_dmat);
 411
 412     /* destroy the data-transfer DMA tag */
 413     if (sc->twe_buffer_dmat)
 414         bus_dma_tag_destroy(sc->twe_buffer_dmat);
 415
 416     /* disconnect the interrupt handler */
 417     if (sc->twe_intr)
 418         bus_teardown_intr(sc->twe_dev, sc->twe_irq, sc->twe_intr);
 419     if (sc->twe_irq != NULL)
 420         bus_release_resource(sc->twe_dev, SYS_RES_IRQ, 0, sc->twe_irq);
 421
 422     /* destroy the parent DMA tag */
 423     if (sc->twe_parent_dmat)
 424         bus_dma_tag_destroy(sc->twe_parent_dmat);
 425
 426     /* release the register window mapping */
 427     if (sc->twe_io != NULL)
 428         bus_release_resource(sc->twe_dev, SYS_RES_IOPORT, TWE_IO_CONFIG_REG, sc->twe_io);
 429
 430     /* destroy control device */
 431     if (sc->twe_dev_t != (struct cdev *)NULL)
 432         destroy_dev(sc->twe_dev_t);
 433
 434     sx_destroy(&sc->twe_config_lock);
 435     mtx_destroy(&sc->twe_io_lock);
 436 }
 437
 438 /********************************************************************************
 439  * Disconnect from the controller completely, in preparation for unload.
 440  */
 441 static int
 442 twe_detach(device_t dev)
 443 {
 444     struct twe_softc    *sc = device_get_softc(dev);
 445
 446     debug_called(4);
 447
 448     TWE_IO_LOCK(sc);
 449     if (sc->twe_state & TWE_STATE_OPEN) {
 450         TWE_IO_UNLOCK(sc);
 451         return (EBUSY);
 452     }
 453     sc->twe_state |= TWE_STATE_DETACHING;
 454     TWE_IO_UNLOCK(sc);
 455
 456     /*
 457      * Shut the controller down.
 458      */
 459     if (twe_shutdown(dev)) {
 460         TWE_IO_LOCK(sc);
 461         sc->twe_state &= ~TWE_STATE_DETACHING;
 462         TWE_IO_UNLOCK(sc);
 463         return (EBUSY);
 464     }
 465
 466     twe_free(sc);
 467
 468     return(0);
 469 }
 470
 471 /********************************************************************************
 472  * Bring the controller down to a dormant state and detach all child devices.
 473  *
 474  * Note that we can assume that the bioq on the controller is empty, as we won't
 475  * allow shutdown if any device is open.
 476  */
 477 static int
 478 twe_shutdown(device_t dev)
 479 {
 480     struct twe_softc    *sc = device_get_softc(dev);
 481     int                 i, error = 0;
 482
 483     debug_called(4);
 484
 485     /*
 486      * Delete all our child devices.
 487      */
 488     TWE_CONFIG_LOCK(sc);
 489     for (i = 0; i < TWE_MAX_UNITS; i++) {
 490         if (sc->twe_drive[i].td_disk != 0) {
 491             if ((error = twe_detach_drive(sc, i)) != 0) {
 492                 TWE_CONFIG_UNLOCK(sc);
 493                 return (error);
 494             }
 495         }
 496     }
 497     TWE_CONFIG_UNLOCK(sc);
 498
 499     /*
 500      * Bring the controller down.
 501      */
 502     TWE_IO_LOCK(sc);
 503     twe_deinit(sc);
 504     TWE_IO_UNLOCK(sc);
 505
 506     return(0);
 507 }
 508
 509 /********************************************************************************
 510  * Bring the controller to a quiescent state, ready for system suspend.
 511  */
 512 static int
 513 twe_suspend(device_t dev)
 514 {
 515     struct twe_softc    *sc = device_get_softc(dev);
 516
 517     debug_called(4);
 518
 519     TWE_IO_LOCK(sc);
 520     sc->twe_state |= TWE_STATE_SUSPEND;
 521
 522     twe_disable_interrupts(sc);
 523     TWE_IO_UNLOCK(sc);
 524
 525     return(0);
 526 }
 527
 528 /********************************************************************************
 529  * Bring the controller back to a state ready for operation.
 530  */
 531 static int
 532 twe_resume(device_t dev)
 533 {
 534     struct twe_softc    *sc = device_get_softc(dev);
 535
 536     debug_called(4);
 537
 538     TWE_IO_LOCK(sc);
 539     sc->twe_state &= ~TWE_STATE_SUSPEND;
 540     twe_enable_interrupts(sc);
 541     TWE_IO_UNLOCK(sc);
 542
 543     return(0);
 544 }
 545
 546 /*******************************************************************************
 547  * Take an interrupt, or be poked by other code to look for interrupt-worthy
 548  * status.
 549  */
 550 static void
 551 twe_pci_intr(void *arg)
 552 {
 553     struct twe_softc *sc = arg;
 554
 555     TWE_IO_LOCK(sc);
 556     twe_intr(sc);
 557     TWE_IO_UNLOCK(sc);
 558 }
 559
 560 /********************************************************************************
 561  * Delayed-startup hook
 562  */
 563 static void
 564 twe_intrhook(void *arg)
 565 {
 566     struct twe_softc            *sc = (struct twe_softc *)arg;
 567
 568     /* pull ourselves off the intrhook chain */
 569     config_intrhook_disestablish(&sc->twe_ich);
 570
 571     /* call core startup routine */
 572     twe_init(sc);
 573 }
 574
 575 /********************************************************************************
 576  * Given a detected drive, attach it to the bio interface.
 577  *
 578  * This is called from twe_add_unit.
 579  */
 580 int
 581 twe_attach_drive(struct twe_softc *sc, struct twe_drive *dr)
 582 {
 583     char        buf[80];
 584     int         error;
 585
 586     mtx_lock(&Giant);
 587     dr->td_disk =  device_add_child(sc->twe_dev, NULL, -1);
 588     if (dr->td_disk == NULL) {
 589         mtx_unlock(&Giant);
 590         twe_printf(sc, "Cannot add unit\n");
 591         return (EIO);
 592     }
 593     device_set_ivars(dr->td_disk, dr);
 594
 595     /*
 596      * XXX It would make sense to test the online/initialising bits, but they seem to be
 597      * always set...
 598      */
 599     sprintf(buf, "Unit %d, %s, %s",
 600             dr->td_twe_unit,
 601             twe_describe_code(twe_table_unittype, dr->td_type),
 602             twe_describe_code(twe_table_unitstate, dr->td_state & TWE_PARAM_UNITSTATUS_MASK));
 603     device_set_desc_copy(dr->td_disk, buf);
 604
 605     error = device_probe_and_attach(dr->td_disk);
 606     mtx_unlock(&Giant);
 607     if (error != 0) {
 608         twe_printf(sc, "Cannot attach unit to controller. error = %d\n", error);
 609         return (EIO);
 610     }
 611     return (0);
 612 }
 613
 614 /********************************************************************************
 615  * Detach the specified unit if it exsists
 616  *
 617  * This is called from twe_del_unit.
 618  */
 619 int
 620 twe_detach_drive(struct twe_softc *sc, int unit)
 621 {
 622     int error = 0;
 623
 624     TWE_CONFIG_ASSERT_LOCKED(sc);
 625     mtx_lock(&Giant);
 626     error = device_delete_child(sc->twe_dev, sc->twe_drive[unit].td_disk);
 627     mtx_unlock(&Giant);
 628     if (error != 0) {
 629         twe_printf(sc, "failed to delete unit %d\n", unit);
 630         return(error);
 631     }
 632     bzero(&sc->twe_drive[unit], sizeof(sc->twe_drive[unit]));
 633     return(error);
 634 }
 635
 636 /********************************************************************************
 637  * Clear a PCI parity error.
 638  */
 639 void
 640 twe_clear_pci_parity_error(struct twe_softc *sc)
 641 {
 642     TWE_CONTROL(sc, TWE_CONTROL_CLEAR_PARITY_ERROR);
 643     pci_write_config(sc->twe_dev, PCIR_STATUS, TWE_PCI_CLEAR_PARITY_ERROR, 2);
 644 }
 645
 646 /********************************************************************************
 647  * Clear a PCI abort.
 648  */
 649 void
 650 twe_clear_pci_abort(struct twe_softc *sc)
 651 {
 652     TWE_CONTROL(sc, TWE_CONTROL_CLEAR_PCI_ABORT);
 653     pci_write_config(sc->twe_dev, PCIR_STATUS, TWE_PCI_CLEAR_PCI_ABORT, 2);
 654 }
 655
 656 /********************************************************************************
 657  ********************************************************************************
 658                                                                       Disk device
 659  ********************************************************************************
 660  ********************************************************************************/
 661
 662 /*
 663  * Disk device softc
 664  */
 665 struct twed_softc
 666 {
 667     device_t            twed_dev;
 668     struct twe_softc    *twed_controller;       /* parent device softc */
 669     struct twe_drive    *twed_drive;            /* drive data in parent softc */
 670     struct disk         *twed_disk;             /* generic disk handle */
 671 };
 672
 673 /*
 674  * Disk device bus interface
 675  */
 676 static int twed_probe(device_t dev);
 677 static int twed_attach(device_t dev);
 678 static int twed_detach(device_t dev);
 679
 680 static device_method_t twed_methods[] = {
 681     DEVMETHOD(device_probe,     twed_probe),
 682     DEVMETHOD(device_attach,    twed_attach),
 683     DEVMETHOD(device_detach,    twed_detach),
 684     { 0, 0 }
 685 };
 686
 687 static driver_t twed_driver = {
 688     "twed",
 689     twed_methods,
 690     sizeof(struct twed_softc)
 691 };
 692
 693 static devclass_t       twed_devclass;
 694 DRIVER_MODULE(twed, twe, twed_driver, twed_devclass, 0, 0);
 695
 696 /*
 697  * Disk device control interface.
 698  */
 699
 700 /********************************************************************************
 701  * Handle open from generic layer.
 702  *
 703  * Note that this is typically only called by the diskslice code, and not
 704  * for opens on subdevices (eg. slices, partitions).
 705  */
 706 static int
 707 twed_open(struct disk *dp)
 708 {
 709     struct twed_softc   *sc = (struct twed_softc *)dp->d_drv1;
 710
 711     debug_called(4);
 712
 713     if (sc == NULL)
 714         return (ENXIO);
 715
 716     /* check that the controller is up and running */
 717     if (sc->twed_controller->twe_state & TWE_STATE_SHUTDOWN)
 718         return(ENXIO);
 719
 720     return (0);
 721 }
 722
 723 /********************************************************************************
 724  * Handle an I/O request.
 725  */
 726 static void
 727 twed_strategy(struct bio *bp)
 728 {
 729     struct twed_softc   *sc = bp->bio_disk->d_drv1;
 730
 731     debug_called(4);
 732
 733     bp->bio_driver1 = &sc->twed_drive->td_twe_unit;
 734     TWED_BIO_IN;
 735
 736     /* bogus disk? */
 737     if (sc == NULL || sc->twed_drive->td_disk == NULL) {
 738         bp->bio_error = EINVAL;
 739         bp->bio_flags |= BIO_ERROR;
 740         printf("twe: bio for invalid disk!\n");
 741         biodone(bp);
 742         TWED_BIO_OUT;
 743         return;
 744     }
 745
 746     /* queue the bio on the controller */
 747     TWE_IO_LOCK(sc->twed_controller);
 748     twe_enqueue_bio(sc->twed_controller, bp);
 749
 750     /* poke the controller to start I/O */
 751     twe_startio(sc->twed_controller);
 752     TWE_IO_UNLOCK(sc->twed_controller);
 753     return;
 754 }
 755
 756 /********************************************************************************
 757  * System crashdump support
 758  */
 759 static int
 760 twed_dump(void *arg, void *virtual, vm_offset_t physical, off_t offset, size_t length)
 761 {
 762     struct twed_softc   *twed_sc;
 763     struct twe_softc    *twe_sc;
 764     int                 error;
 765     struct disk         *dp;
 766
 767     dp = arg;
 768     twed_sc = (struct twed_softc *)dp->d_drv1;
 769     if (twed_sc == NULL)
 770         return(ENXIO);
 771     twe_sc  = (struct twe_softc *)twed_sc->twed_controller;
 772
 773     if (length > 0) {
 774         if ((error = twe_dump_blocks(twe_sc, twed_sc->twed_drive->td_twe_unit, offset / TWE_BLOCK_SIZE, virtual, length / TWE_BLOCK_SIZE)) != 0)
 775             return(error);
 776     }
 777     return(0);
 778 }
 779
 780 /********************************************************************************
 781  * Handle completion of an I/O request.
 782  */
 783 void
 784 twed_intr(struct bio *bp)
 785 {
 786     debug_called(4);
 787
 788     /* if no error, transfer completed */
 789     if (!(bp->bio_flags & BIO_ERROR))
 790         bp->bio_resid = 0;
 791
 792     biodone(bp);
 793     TWED_BIO_OUT;
 794 }
 795
 796 /********************************************************************************
 797  * Default probe stub.
 798  */
 799 static int
 800 twed_probe(device_t dev)
 801 {
 802     return (0);
 803 }
 804
 805 /********************************************************************************
 806  * Attach a unit to the controller.
 807  */
 808 static int
 809 twed_attach(device_t dev)
 810 {
 811     struct twed_softc   *sc;
 812     device_t            parent;
 813
 814     debug_called(4);
 815
 816     /* initialise our softc */
 817     sc = device_get_softc(dev);
 818     parent = device_get_parent(dev);
 819     sc->twed_controller = (struct twe_softc *)device_get_softc(parent);
 820     sc->twed_drive = device_get_ivars(dev);
 821     sc->twed_dev = dev;
 822
 823     /* report the drive */
 824     twed_printf(sc, "%uMB (%u sectors)\n",
 825                 sc->twed_drive->td_size / ((1024 * 1024) / TWE_BLOCK_SIZE),
 826                 sc->twed_drive->td_size);
 827
 828     /* attach a generic disk device to ourselves */
 829
 830     sc->twed_drive->td_sys_unit = device_get_unit(dev);
 831
 832     sc->twed_disk = disk_alloc();
 833     sc->twed_disk->d_open = twed_open;
 834     sc->twed_disk->d_strategy = twed_strategy;
 835     sc->twed_disk->d_dump = (dumper_t *)twed_dump;
 836     sc->twed_disk->d_name = "twed";
 837     sc->twed_disk->d_drv1 = sc;
 838     sc->twed_disk->d_maxsize = (TWE_MAX_SGL_LENGTH - 1) * PAGE_SIZE;
 839     sc->twed_disk->d_sectorsize = TWE_BLOCK_SIZE;
 840     sc->twed_disk->d_mediasize = TWE_BLOCK_SIZE * (off_t)sc->twed_drive->td_size;
 841     if (sc->twed_drive->td_type == TWE_UD_CONFIG_RAID0 ||
 842         sc->twed_drive->td_type == TWE_UD_CONFIG_RAID5 ||
 843         sc->twed_drive->td_type == TWE_UD_CONFIG_RAID10) {
 844             sc->twed_disk->d_stripesize =
 845                 TWE_BLOCK_SIZE << sc->twed_drive->td_stripe;
 846             sc->twed_disk->d_stripeoffset = 0;
 847     }
 848     sc->twed_disk->d_fwsectors = sc->twed_drive->td_sectors;
 849     sc->twed_disk->d_fwheads = sc->twed_drive->td_heads;
 850     sc->twed_disk->d_unit = sc->twed_drive->td_sys_unit;
 851
 852     disk_create(sc->twed_disk, DISK_VERSION);
 853
 854     /* set the maximum I/O size to the theoretical maximum allowed by the S/G list size */
 855
 856     return (0);
 857 }
 858
 859 /********************************************************************************
 860  * Disconnect ourselves from the system.
 861  */
 862 static int
 863 twed_detach(device_t dev)
 864 {
 865     struct twed_softc *sc = (struct twed_softc *)device_get_softc(dev);
 866
 867     debug_called(4);
 868
 869     if (sc->twed_disk->d_flags & DISKFLAG_OPEN)
 870         return(EBUSY);
 871
 872     disk_destroy(sc->twed_disk);
 873
 874     return(0);
 875 }
 876
 877 /********************************************************************************
 878  ********************************************************************************
 879                                                                              Misc
 880  ********************************************************************************
 881  ********************************************************************************/
 882
 883 /********************************************************************************
 884  * Allocate a command buffer
 885  */
 886 static MALLOC_DEFINE(TWE_MALLOC_CLASS, "twe_commands", "twe commands");
 887
 888 struct twe_request *
 889 twe_allocate_request(struct twe_softc *sc, int tag)
 890 {
 891     struct twe_request  *tr;
 892
 893     tr = malloc(sizeof(struct twe_request), TWE_MALLOC_CLASS, M_WAITOK | M_ZERO);
 894     tr->tr_sc = sc;
 895     tr->tr_tag = tag;
 896     if (bus_dmamap_create(sc->twe_buffer_dmat, 0, &tr->tr_dmamap)) {
 897         twe_free_request(tr);
 898         twe_printf(sc, "unable to allocate dmamap for tag %d\n", tag);
 899         return(NULL);
 900     }
 901     return(tr);
 902 }
 903
 904 /********************************************************************************
 905  * Permanently discard a command buffer.
 906  */
 907 void
 908 twe_free_request(struct twe_request *tr)
 909 {
 910     struct twe_softc    *sc = tr->tr_sc;
 911
 912     debug_called(4);
 913
 914     bus_dmamap_destroy(sc->twe_buffer_dmat, tr->tr_dmamap);
 915     free(tr, TWE_MALLOC_CLASS);
 916 }
 917
 918 /********************************************************************************
 919  * Map/unmap (tr)'s command and data in the controller's addressable space.
 920  *
 921  * These routines ensure that the data which the controller is going to try to
 922  * access is actually visible to the controller, in a machine-independant
 923  * fashion.  Due to a hardware limitation, I/O buffers must be 512-byte aligned
 924  * and we take care of that here as well.
 925  */
 926 static void
 927 twe_fillin_sgl(TWE_SG_Entry *sgl, bus_dma_segment_t *segs, int nsegments, int max_sgl)
 928 {
 929     int i;
 930
 931     for (i = 0; i < nsegments; i++) {
 932         sgl[i].address = segs[i].ds_addr;
 933         sgl[i].length = segs[i].ds_len;
 934     }
 935     for (; i < max_sgl; i++) {                          /* XXX necessary? */
 936         sgl[i].address = 0;
 937         sgl[i].length = 0;
 938     }
 939 }
 940
 941 static void
 942 twe_setup_data_dmamap(void *arg, bus_dma_segment_t *segs, int nsegments, int error)
 943 {
 944     struct twe_request  *tr = (struct twe_request *)arg;
 945     struct twe_softc    *sc = tr->tr_sc;
 946     TWE_Command         *cmd = TWE_FIND_COMMAND(tr);
 947
 948     debug_called(4);
 949
 950     if (tr->tr_flags & TWE_CMD_MAPPED)
 951         panic("already mapped command");
 952
 953     tr->tr_flags |= TWE_CMD_MAPPED;
 954
 955     if (tr->tr_flags & TWE_CMD_IN_PROGRESS)
 956         sc->twe_state &= ~TWE_STATE_FRZN;
 957     /* save base of first segment in command (applicable if there only one segment) */
 958     tr->tr_dataphys = segs[0].ds_addr;
 959
 960     /* correct command size for s/g list size */
 961     cmd->generic.size += 2 * nsegments;
 962
 963     /*
 964      * Due to the fact that parameter and I/O commands have the scatter/gather list in
 965      * different places, we need to determine which sort of command this actually is
 966      * before we can populate it correctly.
 967      */
 968     switch(cmd->generic.opcode) {
 969     case TWE_OP_GET_PARAM:
 970     case TWE_OP_SET_PARAM:
 971         cmd->generic.sgl_offset = 2;
 972         twe_fillin_sgl(&cmd->param.sgl[0], segs, nsegments, TWE_MAX_SGL_LENGTH);
 973         break;
 974     case TWE_OP_READ:
 975     case TWE_OP_WRITE:
 976         cmd->generic.sgl_offset = 3;
 977         twe_fillin_sgl(&cmd->io.sgl[0], segs, nsegments, TWE_MAX_SGL_LENGTH);
 978         break;
 979     case TWE_OP_ATA_PASSTHROUGH:
 980         cmd->generic.sgl_offset = 5;
 981         twe_fillin_sgl(&cmd->ata.sgl[0], segs, nsegments, TWE_MAX_ATA_SGL_LENGTH);
 982         break;
 983     default:
 984         /*
 985          * Fall back to what the linux driver does.
 986          * Do this because the API may send an opcode
 987          * the driver knows nothing about and this will
 988          * at least stop PCIABRT's from hosing us.
 989          */
 990         switch (cmd->generic.sgl_offset) {
 991         case 2:
 992             twe_fillin_sgl(&cmd->param.sgl[0], segs, nsegments, TWE_MAX_SGL_LENGTH);
 993             break;
 994         case 3:
 995             twe_fillin_sgl(&cmd->io.sgl[0], segs, nsegments, TWE_MAX_SGL_LENGTH);
 996             break;
 997         case 5:
 998             twe_fillin_sgl(&cmd->ata.sgl[0], segs, nsegments, TWE_MAX_ATA_SGL_LENGTH);
 999             break;
1000         }
1001     }
1002
1003     if (tr->tr_flags & TWE_CMD_DATAIN) {
1004         if (tr->tr_flags & TWE_CMD_IMMEDIATE) {
1005             bus_dmamap_sync(sc->twe_immediate_dmat, sc->twe_immediate_map,
1006                             BUS_DMASYNC_PREREAD);
1007         } else {
1008             bus_dmamap_sync(sc->twe_buffer_dmat, tr->tr_dmamap,
1009                             BUS_DMASYNC_PREREAD);
1010         }
1011     }
1012
1013     if (tr->tr_flags & TWE_CMD_DATAOUT) {
1014         /*
1015          * if we're using an alignment buffer, and we're writing data
1016          * copy the real data out
1017          */
1018         if (tr->tr_flags & TWE_CMD_ALIGNBUF)
1019             bcopy(tr->tr_realdata, tr->tr_data, tr->tr_length);
1020
1021         if (tr->tr_flags & TWE_CMD_IMMEDIATE) {
1022             bus_dmamap_sync(sc->twe_immediate_dmat, sc->twe_immediate_map,
1023                             BUS_DMASYNC_PREWRITE);
1024         } else {
1025             bus_dmamap_sync(sc->twe_buffer_dmat, tr->tr_dmamap,
1026                             BUS_DMASYNC_PREWRITE);
1027         }
1028     }
1029
1030     if (twe_start(tr) == EBUSY) {
1031         tr->tr_sc->twe_state |= TWE_STATE_CTLR_BUSY;
1032         twe_requeue_ready(tr);
1033     }
1034 }
1035
1036 static void
1037 twe_setup_request_dmamap(void *arg, bus_dma_segment_t *segs, int nsegments, int error)
1038 {
1039     struct twe_softc    *sc = (struct twe_softc *)arg;
1040
1041     debug_called(4);
1042
1043     /* command can't cross a page boundary */
1044     sc->twe_cmdphys = segs[0].ds_addr;
1045 }
1046
1047 int
1048 twe_map_request(struct twe_request *tr)
1049 {
1050     struct twe_softc    *sc = tr->tr_sc;
1051     int                 error = 0;
1052
1053     debug_called(4);
1054
1055     if (!dumping)
1056         TWE_IO_ASSERT_LOCKED(sc);
1057     if (sc->twe_state & (TWE_STATE_CTLR_BUSY | TWE_STATE_FRZN)) {
1058         twe_requeue_ready(tr);
1059         return (EBUSY);
1060     }
1061
1062     bus_dmamap_sync(sc->twe_cmd_dmat, sc->twe_cmdmap, BUS_DMASYNC_PREWRITE);
1063
1064     /*
1065      * If the command involves data, map that too.
1066      */
1067     if (tr->tr_data != NULL && ((tr->tr_flags & TWE_CMD_MAPPED) == 0)) {
1068         /*
1069          * Data must be 64-byte aligned; allocate a fixup buffer if it's not.
1070          */
1071         if (((vm_offset_t)tr->tr_data % TWE_ALIGNMENT) != 0) {
1072             tr->tr_realdata = tr->tr_data;                              /* save pointer to 'real' data */
1073             tr->tr_flags |= TWE_CMD_ALIGNBUF;
1074             tr->tr_data = malloc(tr->tr_length, TWE_MALLOC_CLASS, M_NOWAIT);
1075             if (tr->tr_data == NULL) {
1076                 twe_printf(sc, "%s: malloc failed\n", __func__);
1077                 tr->tr_data = tr->tr_realdata; /* restore original data pointer */
1078                 return(ENOMEM);
1079             }
1080         }
1081
1082         /*
1083          * Map the data buffer into bus space and build the s/g list.
1084          */
1085         if (tr->tr_flags & TWE_CMD_IMMEDIATE) {
1086             error = bus_dmamap_load(sc->twe_immediate_dmat, sc->twe_immediate_map, sc->twe_immediate,
1087                             tr->tr_length, twe_setup_data_dmamap, tr, BUS_DMA_NOWAIT);
1088         } else {
1089             error = bus_dmamap_load(sc->twe_buffer_dmat, tr->tr_dmamap, tr->tr_data, tr->tr_length,
1090                                     twe_setup_data_dmamap, tr, 0);
1091         }
1092         if (error == EINPROGRESS) {
1093             tr->tr_flags |= TWE_CMD_IN_PROGRESS;
1094             sc->twe_state |= TWE_STATE_FRZN;
1095             error = 0;
1096         }
1097     } else
1098         if ((error = twe_start(tr)) == EBUSY) {
1099             sc->twe_state |= TWE_STATE_CTLR_BUSY;
1100             twe_requeue_ready(tr);
1101         }
1102
1103     return(error);
1104 }
1105
1106 void
1107 twe_unmap_request(struct twe_request *tr)
1108 {
1109     struct twe_softc    *sc = tr->tr_sc;
1110
1111     debug_called(4);
1112
1113     if (!dumping)
1114         TWE_IO_ASSERT_LOCKED(sc);
1115     bus_dmamap_sync(sc->twe_cmd_dmat, sc->twe_cmdmap, BUS_DMASYNC_POSTWRITE);
1116
1117     /*
1118      * If the command involved data, unmap that too.
1119      */
1120     if (tr->tr_data != NULL) {
1121         if (tr->tr_flags & TWE_CMD_DATAIN) {
1122             if (tr->tr_flags & TWE_CMD_IMMEDIATE) {
1123                 bus_dmamap_sync(sc->twe_immediate_dmat, sc->twe_immediate_map,
1124                                 BUS_DMASYNC_POSTREAD);
1125             } else {
1126                 bus_dmamap_sync(sc->twe_buffer_dmat, tr->tr_dmamap,
1127                                 BUS_DMASYNC_POSTREAD);
1128             }
1129
1130             /* if we're using an alignment buffer, and we're reading data, copy the real data in */
1131             if (tr->tr_flags & TWE_CMD_ALIGNBUF)
1132                 bcopy(tr->tr_data, tr->tr_realdata, tr->tr_length);
1133         }
1134         if (tr->tr_flags & TWE_CMD_DATAOUT) {
1135             if (tr->tr_flags & TWE_CMD_IMMEDIATE) {
1136                 bus_dmamap_sync(sc->twe_immediate_dmat, sc->twe_immediate_map,
1137                                 BUS_DMASYNC_POSTWRITE);
1138             } else {
1139                 bus_dmamap_sync(sc->twe_buffer_dmat, tr->tr_dmamap,
1140                                 BUS_DMASYNC_POSTWRITE);
1141             }
1142         }
1143
1144         if (tr->tr_flags & TWE_CMD_IMMEDIATE) {
1145             bus_dmamap_unload(sc->twe_immediate_dmat, sc->twe_immediate_map);
1146         } else {
1147             bus_dmamap_unload(sc->twe_buffer_dmat, tr->tr_dmamap);
1148         }
1149     }
1150
1151     /* free alignment buffer if it was used */
1152     if (tr->tr_flags & TWE_CMD_ALIGNBUF) {
1153         free(tr->tr_data, TWE_MALLOC_CLASS);
1154         tr->tr_data = tr->tr_realdata;          /* restore 'real' data pointer */
1155     }
1156 }
1157
1158 #ifdef TWE_DEBUG
1159 void twe_report(void);
1160 /********************************************************************************
1161  * Print current controller status, call from DDB.
1162  */
1163 void
1164 twe_report(void)
1165 {
1166     struct twe_softc    *sc;
1167     int                 i;
1168
1169     for (i = 0; (sc = devclass_get_softc(twe_devclass, i)) != NULL; i++)
1170         twe_print_controller(sc);
1171     printf("twed: total bio count in %u  out %u\n", twed_bio_in, twed_bio_out);
1172 }
1173 #endif