sys/dev/twe/twe_freebsd.c

   1 /*-
   2  * Copyright (c) 2000 Michael Smith
   3  * Copyright (c) 2003 Paul Saab
   4  * Copyright (c) 2003 Vinod Kashyap
   5  * Copyright (c) 2000 BSDi
   6  * All rights reserved.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  27  * SUCH DAMAGE.
  28  *
  29  * $FreeBSD$
  30  */
  31
  32 /*
  33  * FreeBSD-specific code.
  34  */
  35
  36 #include <dev/twe/twe_compat.h>
  37 #include <dev/twe/twereg.h>
  38 #include <dev/twe/tweio.h>
  39 #include <dev/twe/twevar.h>
  40 #include <dev/twe/twe_tables.h>
  41
  42 #include <vm/vm.h>
  43
  44 static devclass_t       twe_devclass;
  45
  46 #ifdef TWE_DEBUG
  47 static u_int32_t        twed_bio_in;
  48 #define TWED_BIO_IN     twed_bio_in++
  49 static u_int32_t        twed_bio_out;
  50 #define TWED_BIO_OUT    twed_bio_out++
  51 #else
  52 #define TWED_BIO_IN
  53 #define TWED_BIO_OUT
  54 #endif
  55
  56 static void     twe_setup_data_dmamap(void *arg, bus_dma_segment_t *segs, int nsegments, int error);
  57 static void     twe_setup_request_dmamap(void *arg, bus_dma_segment_t *segs, int nsegments, int error);
  58
  59 /********************************************************************************
  60  ********************************************************************************
  61                                                          Control device interface
  62  ********************************************************************************
  63  ********************************************************************************/
  64
  65 static  d_open_t                twe_open;
  66 static  d_close_t               twe_close;
  67 static  d_ioctl_t               twe_ioctl_wrapper;
  68
  69 static struct cdevsw twe_cdevsw = {
  70         .d_version =    D_VERSION,
  71         .d_flags =      D_NEEDGIANT,
  72         .d_open =       twe_open,
  73         .d_close =      twe_close,
  74         .d_ioctl =      twe_ioctl_wrapper,
  75         .d_name =       "twe",
  76 };
  77
  78 /********************************************************************************
  79  * Accept an open operation on the control device.
  80  */
  81 static int
  82 twe_open(struct cdev *dev, int flags, int fmt, struct thread *td)
  83 {
  84     struct twe_softc            *sc = (struct twe_softc *)dev->si_drv1;
  85
  86     sc->twe_state |= TWE_STATE_OPEN;
  87     return(0);
  88 }
  89
  90 /********************************************************************************
  91  * Accept the last close on the control device.
  92  */
  93 static int
  94 twe_close(struct cdev *dev, int flags, int fmt, struct thread *td)
  95 {
  96     struct twe_softc            *sc = (struct twe_softc *)dev->si_drv1;
  97
  98     sc->twe_state &= ~TWE_STATE_OPEN;
  99     return (0);
 100 }
 101
 102 /********************************************************************************
 103  * Handle controller-specific control operations.
 104  */
 105 static int
 106 twe_ioctl_wrapper(struct cdev *dev, u_long cmd, caddr_t addr, int32_t flag, struct thread *td)
 107 {
 108     struct twe_softc            *sc = (struct twe_softc *)dev->si_drv1;
 109
 110     return(twe_ioctl(sc, cmd, addr));
 111 }
 112
 113 /********************************************************************************
 114  ********************************************************************************
 115                                                              PCI device interface
 116  ********************************************************************************
 117  ********************************************************************************/
 118
 119 static int      twe_probe(device_t dev);
 120 static int      twe_attach(device_t dev);
 121 static void     twe_free(struct twe_softc *sc);
 122 static int      twe_detach(device_t dev);
 123 static int      twe_shutdown(device_t dev);
 124 static int      twe_suspend(device_t dev);
 125 static int      twe_resume(device_t dev);
 126 static void     twe_pci_intr(void *arg);
 127 static void     twe_intrhook(void *arg);
 128
 129 static device_method_t twe_methods[] = {
 130     /* Device interface */
 131     DEVMETHOD(device_probe,     twe_probe),
 132     DEVMETHOD(device_attach,    twe_attach),
 133     DEVMETHOD(device_detach,    twe_detach),
 134     DEVMETHOD(device_shutdown,  twe_shutdown),
 135     DEVMETHOD(device_suspend,   twe_suspend),
 136     DEVMETHOD(device_resume,    twe_resume),
 137
 138     DEVMETHOD(bus_print_child,  bus_generic_print_child),
 139     DEVMETHOD(bus_driver_added, bus_generic_driver_added),
 140     { 0, 0 }
 141 };
 142
 143 static driver_t twe_pci_driver = {
 144         "twe",
 145         twe_methods,
 146         sizeof(struct twe_softc)
 147 };
 148
 149 DRIVER_MODULE(twe, pci, twe_pci_driver, twe_devclass, 0, 0);
 150
 151 /********************************************************************************
 152  * Match a 3ware Escalade ATA RAID controller.
 153  */
 154 static int
 155 twe_probe(device_t dev)
 156 {
 157
 158     debug_called(4);
 159
 160     if ((pci_get_vendor(dev) == TWE_VENDOR_ID) &&
 161         ((pci_get_device(dev) == TWE_DEVICE_ID) ||
 162          (pci_get_device(dev) == TWE_DEVICE_ID_ASIC))) {
 163         device_set_desc_copy(dev, TWE_DEVICE_NAME ". Driver version " TWE_DRIVER_VERSION_STRING);
 164         return(BUS_PROBE_DEFAULT);
 165     }
 166     return(ENXIO);
 167 }
 168
 169 /********************************************************************************
 170  * Allocate resources, initialise the controller.
 171  */
 172 static int
 173 twe_attach(device_t dev)
 174 {
 175     struct twe_softc    *sc;
 176     int                 rid, error;
 177     u_int32_t           command;
 178
 179     debug_called(4);
 180
 181     /*
 182      * Initialise the softc structure.
 183      */
 184     sc = device_get_softc(dev);
 185     sc->twe_dev = dev;
 186
 187     sysctl_ctx_init(&sc->sysctl_ctx);
 188     sc->sysctl_tree = SYSCTL_ADD_NODE(&sc->sysctl_ctx,
 189         SYSCTL_STATIC_CHILDREN(_hw), OID_AUTO,
 190         device_get_nameunit(dev), CTLFLAG_RD, 0, "");
 191     if (sc->sysctl_tree == NULL) {
 192         twe_printf(sc, "cannot add sysctl tree node\n");
 193         return (ENXIO);
 194     }
 195     SYSCTL_ADD_STRING(&sc->sysctl_ctx, SYSCTL_CHILDREN(sc->sysctl_tree),
 196         OID_AUTO, "driver_version", CTLFLAG_RD, TWE_DRIVER_VERSION_STRING, 0,
 197         "TWE driver version");
 198
 199     /*
 200      * Make sure we are going to be able to talk to this board.
 201      */
 202     command = pci_read_config(dev, PCIR_COMMAND, 2);
 203     if ((command & PCIM_CMD_PORTEN) == 0) {
 204         twe_printf(sc, "register window not available\n");
 205         return(ENXIO);
 206     }
 207     /*
 208      * Force the busmaster enable bit on, in case the BIOS forgot.
 209      */
 210     command |= PCIM_CMD_BUSMASTEREN;
 211     pci_write_config(dev, PCIR_COMMAND, command, 2);
 212
 213     /*
 214      * Allocate the PCI register window.
 215      */
 216     rid = TWE_IO_CONFIG_REG;
 217     if ((sc->twe_io = bus_alloc_resource_any(dev, SYS_RES_IOPORT, &rid,
 218         RF_ACTIVE)) == NULL) {
 219         twe_printf(sc, "can't allocate register window\n");
 220         twe_free(sc);
 221         return(ENXIO);
 222     }
 223     sc->twe_btag = rman_get_bustag(sc->twe_io);
 224     sc->twe_bhandle = rman_get_bushandle(sc->twe_io);
 225
 226     /*
 227      * Allocate the parent bus DMA tag appropriate for PCI.
 228      */
 229     if (bus_dma_tag_create(NULL,                                /* parent */
 230                            1, 0,                                /* alignment, boundary */
 231                            BUS_SPACE_MAXADDR_32BIT,             /* lowaddr */
 232                            BUS_SPACE_MAXADDR,                   /* highaddr */
 233                            NULL, NULL,                          /* filter, filterarg */
 234                            MAXBSIZE, TWE_MAX_SGL_LENGTH,        /* maxsize, nsegments */
 235                            BUS_SPACE_MAXSIZE_32BIT,             /* maxsegsize */
 236                            0,                                   /* flags */
 237                            NULL,                                /* lockfunc */
 238                            NULL,                                /* lockarg */
 239                            &sc->twe_parent_dmat)) {
 240         twe_printf(sc, "can't allocate parent DMA tag\n");
 241         twe_free(sc);
 242         return(ENOMEM);
 243     }
 244
 245     /*
 246      * Allocate and connect our interrupt.
 247      */
 248     rid = 0;
 249     if ((sc->twe_irq = bus_alloc_resource_any(sc->twe_dev, SYS_RES_IRQ,
 250         &rid, RF_SHAREABLE | RF_ACTIVE)) == NULL) {
 251         twe_printf(sc, "can't allocate interrupt\n");
 252         twe_free(sc);
 253         return(ENXIO);
 254     }
 255     if (bus_setup_intr(sc->twe_dev, sc->twe_irq, INTR_TYPE_BIO | INTR_ENTROPY,
 256                        NULL, twe_pci_intr, sc, &sc->twe_intr)) {
 257         twe_printf(sc, "can't set up interrupt\n");
 258         twe_free(sc);
 259         return(ENXIO);
 260     }
 261
 262     /*
 263      * Create DMA tag for mapping command's into controller-addressable space.
 264      */
 265     if (bus_dma_tag_create(sc->twe_parent_dmat,         /* parent */
 266                            1, 0,                        /* alignment, boundary */
 267                            BUS_SPACE_MAXADDR_32BIT,     /* lowaddr */
 268                            BUS_SPACE_MAXADDR,           /* highaddr */
 269                            NULL, NULL,                  /* filter, filterarg */
 270                            sizeof(TWE_Command) *
 271                            TWE_Q_LENGTH, 1,             /* maxsize, nsegments */
 272                            BUS_SPACE_MAXSIZE_32BIT,     /* maxsegsize */
 273                            0,                           /* flags */
 274                            NULL,                        /* lockfunc */
 275                            NULL,                        /* lockarg */
 276                            &sc->twe_cmd_dmat)) {
 277         twe_printf(sc, "can't allocate data buffer DMA tag\n");
 278         twe_free(sc);
 279         return(ENOMEM);
 280     }
 281     /*
 282      * Allocate memory and make it available for DMA.
 283      */
 284     if (bus_dmamem_alloc(sc->twe_cmd_dmat, (void **)&sc->twe_cmd,
 285                          BUS_DMA_NOWAIT, &sc->twe_cmdmap)) {
 286         twe_printf(sc, "can't allocate command memory\n");
 287         return(ENOMEM);
 288     }
 289     bus_dmamap_load(sc->twe_cmd_dmat, sc->twe_cmdmap, sc->twe_cmd,
 290                     sizeof(TWE_Command) * TWE_Q_LENGTH,
 291                     twe_setup_request_dmamap, sc, 0);
 292     bzero(sc->twe_cmd, sizeof(TWE_Command) * TWE_Q_LENGTH);
 293
 294     /*
 295      * Create DMA tag for mapping objects into controller-addressable space.
 296      */
 297     if (bus_dma_tag_create(sc->twe_parent_dmat,         /* parent */
 298                            1, 0,                        /* alignment, boundary */
 299                            BUS_SPACE_MAXADDR_32BIT,     /* lowaddr */
 300                            BUS_SPACE_MAXADDR,           /* highaddr */
 301                            NULL, NULL,                  /* filter, filterarg */
 302                            MAXBSIZE, TWE_MAX_SGL_LENGTH,/* maxsize, nsegments */
 303                            BUS_SPACE_MAXSIZE_32BIT,     /* maxsegsize */
 304                            BUS_DMA_ALLOCNOW,            /* flags */
 305                            busdma_lock_mutex,           /* lockfunc */
 306                            &Giant,                      /* lockarg */
 307                            &sc->twe_buffer_dmat)) {
 308         twe_printf(sc, "can't allocate data buffer DMA tag\n");
 309         twe_free(sc);
 310         return(ENOMEM);
 311     }
 312
 313     /*
 314      * Create DMA tag for mapping objects into controller-addressable space.
 315      */
 316     if (bus_dma_tag_create(sc->twe_parent_dmat,         /* parent */
 317                            1, 0,                        /* alignment, boundary */
 318                            BUS_SPACE_MAXADDR_32BIT,     /* lowaddr */
 319                            BUS_SPACE_MAXADDR,           /* highaddr */
 320                            NULL, NULL,                  /* filter, filterarg */
 321                            MAXBSIZE, 1,                 /* maxsize, nsegments */
 322                            BUS_SPACE_MAXSIZE_32BIT,     /* maxsegsize */
 323                            0,                           /* flags */
 324                            NULL,                        /* lockfunc */
 325                            NULL,                        /* lockarg */
 326                            &sc->twe_immediate_dmat)) {
 327         twe_printf(sc, "can't allocate data buffer DMA tag\n");
 328         twe_free(sc);
 329         return(ENOMEM);
 330     }
 331     /*
 332      * Allocate memory for requests which cannot sleep or support continuation.
 333      */
 334      if (bus_dmamem_alloc(sc->twe_immediate_dmat, (void **)&sc->twe_immediate,
 335                           BUS_DMA_NOWAIT, &sc->twe_immediate_map)) {
 336         twe_printf(sc, "can't allocate memory for immediate requests\n");
 337         return(ENOMEM);
 338      }
 339
 340     /*
 341      * Initialise the controller and driver core.
 342      */
 343     if ((error = twe_setup(sc))) {
 344         twe_free(sc);
 345         return(error);
 346     }
 347
 348     /*
 349      * Print some information about the controller and configuration.
 350      */
 351     twe_describe_controller(sc);
 352
 353     /*
 354      * Create the control device.
 355      */
 356     sc->twe_dev_t = make_dev(&twe_cdevsw, device_get_unit(sc->twe_dev), UID_ROOT, GID_OPERATOR,
 357                              S_IRUSR | S_IWUSR, "twe%d", device_get_unit(sc->twe_dev));
 358     sc->twe_dev_t->si_drv1 = sc;
 359     /*
 360      * Schedule ourselves to bring the controller up once interrupts are available.
 361      * This isn't strictly necessary, since we disable interrupts while probing the
 362      * controller, but it is more in keeping with common practice for other disk
 363      * devices.
 364      */
 365     sc->twe_ich.ich_func = twe_intrhook;
 366     sc->twe_ich.ich_arg = sc;
 367     if (config_intrhook_establish(&sc->twe_ich) != 0) {
 368         twe_printf(sc, "can't establish configuration hook\n");
 369         twe_free(sc);
 370         return(ENXIO);
 371     }
 372
 373     return(0);
 374 }
 375
 376 /********************************************************************************
 377  * Free all of the resources associated with (sc).
 378  *
 379  * Should not be called if the controller is active.
 380  */
 381 static void
 382 twe_free(struct twe_softc *sc)
 383 {
 384     struct twe_request  *tr;
 385
 386     debug_called(4);
 387
 388     /* throw away any command buffers */
 389     while ((tr = twe_dequeue_free(sc)) != NULL)
 390         twe_free_request(tr);
 391
 392     if (sc->twe_cmd != NULL) {
 393         bus_dmamap_unload(sc->twe_cmd_dmat, sc->twe_cmdmap);
 394         bus_dmamem_free(sc->twe_cmd_dmat, sc->twe_cmd, sc->twe_cmdmap);
 395     }
 396
 397     if (sc->twe_immediate != NULL) {
 398         bus_dmamap_unload(sc->twe_immediate_dmat, sc->twe_immediate_map);
 399         bus_dmamem_free(sc->twe_immediate_dmat, sc->twe_immediate,
 400                         sc->twe_immediate_map);
 401     }
 402
 403     if (sc->twe_immediate_dmat)
 404         bus_dma_tag_destroy(sc->twe_immediate_dmat);
 405
 406     /* destroy the data-transfer DMA tag */
 407     if (sc->twe_buffer_dmat)
 408         bus_dma_tag_destroy(sc->twe_buffer_dmat);
 409
 410     /* disconnect the interrupt handler */
 411     if (sc->twe_intr)
 412         bus_teardown_intr(sc->twe_dev, sc->twe_irq, sc->twe_intr);
 413     if (sc->twe_irq != NULL)
 414         bus_release_resource(sc->twe_dev, SYS_RES_IRQ, 0, sc->twe_irq);
 415
 416     /* destroy the parent DMA tag */
 417     if (sc->twe_parent_dmat)
 418         bus_dma_tag_destroy(sc->twe_parent_dmat);
 419
 420     /* release the register window mapping */
 421     if (sc->twe_io != NULL)
 422         bus_release_resource(sc->twe_dev, SYS_RES_IOPORT, TWE_IO_CONFIG_REG, sc->twe_io);
 423
 424     /* destroy control device */
 425     if (sc->twe_dev_t != (struct cdev *)NULL)
 426         destroy_dev(sc->twe_dev_t);
 427
 428     sysctl_ctx_free(&sc->sysctl_ctx);
 429 }
 430
 431 /********************************************************************************
 432  * Disconnect from the controller completely, in preparation for unload.
 433  */
 434 static int
 435 twe_detach(device_t dev)
 436 {
 437     struct twe_softc    *sc = device_get_softc(dev);
 438     int                 s, error;
 439
 440     debug_called(4);
 441
 442     error = EBUSY;
 443     s = splbio();
 444     if (sc->twe_state & TWE_STATE_OPEN)
 445         goto out;
 446
 447     /*
 448      * Shut the controller down.
 449      */
 450     if (twe_shutdown(dev))
 451         goto out;
 452
 453     twe_free(sc);
 454
 455     error = 0;
 456  out:
 457     splx(s);
 458     return(error);
 459 }
 460
 461 /********************************************************************************
 462  * Bring the controller down to a dormant state and detach all child devices.
 463  *
 464  * Note that we can assume that the bioq on the controller is empty, as we won't
 465  * allow shutdown if any device is open.
 466  */
 467 static int
 468 twe_shutdown(device_t dev)
 469 {
 470     struct twe_softc    *sc = device_get_softc(dev);
 471     int                 i, s, error = 0;
 472
 473     debug_called(4);
 474
 475     s = splbio();
 476
 477     /*
 478      * Delete all our child devices.
 479      */
 480     for (i = 0; i < TWE_MAX_UNITS; i++) {
 481         if (sc->twe_drive[i].td_disk != 0) {
 482             if ((error = twe_detach_drive(sc, i)) != 0)
 483                 goto out;
 484         }
 485     }
 486
 487     /*
 488      * Bring the controller down.
 489      */
 490     twe_deinit(sc);
 491
 492 out:
 493     splx(s);
 494     return(error);
 495 }
 496
 497 /********************************************************************************
 498  * Bring the controller to a quiescent state, ready for system suspend.
 499  */
 500 static int
 501 twe_suspend(device_t dev)
 502 {
 503     struct twe_softc    *sc = device_get_softc(dev);
 504     int                 s;
 505
 506     debug_called(4);
 507
 508     s = splbio();
 509     sc->twe_state |= TWE_STATE_SUSPEND;
 510
 511     twe_disable_interrupts(sc);
 512     splx(s);
 513
 514     return(0);
 515 }
 516
 517 /********************************************************************************
 518  * Bring the controller back to a state ready for operation.
 519  */
 520 static int
 521 twe_resume(device_t dev)
 522 {
 523     struct twe_softc    *sc = device_get_softc(dev);
 524
 525     debug_called(4);
 526
 527     sc->twe_state &= ~TWE_STATE_SUSPEND;
 528     twe_enable_interrupts(sc);
 529
 530     return(0);
 531 }
 532
 533 /*******************************************************************************
 534  * Take an interrupt, or be poked by other code to look for interrupt-worthy
 535  * status.
 536  */
 537 static void
 538 twe_pci_intr(void *arg)
 539 {
 540     twe_intr((struct twe_softc *)arg);
 541 }
 542
 543 /********************************************************************************
 544  * Delayed-startup hook
 545  */
 546 static void
 547 twe_intrhook(void *arg)
 548 {
 549     struct twe_softc            *sc = (struct twe_softc *)arg;
 550
 551     /* pull ourselves off the intrhook chain */
 552     config_intrhook_disestablish(&sc->twe_ich);
 553
 554     /* call core startup routine */
 555     twe_init(sc);
 556 }
 557
 558 /********************************************************************************
 559  * Given a detected drive, attach it to the bio interface.
 560  *
 561  * This is called from twe_add_unit.
 562  */
 563 int
 564 twe_attach_drive(struct twe_softc *sc, struct twe_drive *dr)
 565 {
 566     char        buf[80];
 567     int         error;
 568
 569     dr->td_disk =  device_add_child(sc->twe_dev, NULL, -1);
 570     if (dr->td_disk == NULL) {
 571         twe_printf(sc, "Cannot add unit\n");
 572         return (EIO);
 573     }
 574     device_set_ivars(dr->td_disk, dr);
 575
 576     /*
 577      * XXX It would make sense to test the online/initialising bits, but they seem to be
 578      * always set...
 579      */
 580     sprintf(buf, "Unit %d, %s, %s",
 581             dr->td_twe_unit,
 582             twe_describe_code(twe_table_unittype, dr->td_type),
 583             twe_describe_code(twe_table_unitstate, dr->td_state & TWE_PARAM_UNITSTATUS_MASK));
 584     device_set_desc_copy(dr->td_disk, buf);
 585
 586     if ((error = bus_generic_attach(sc->twe_dev)) != 0) {
 587         twe_printf(sc, "Cannot attach unit to controller. error = %d\n", error);
 588         return (EIO);
 589     }
 590     return (0);
 591 }
 592
 593 /********************************************************************************
 594  * Detach the specified unit if it exsists
 595  *
 596  * This is called from twe_del_unit.
 597  */
 598 int
 599 twe_detach_drive(struct twe_softc *sc, int unit)
 600 {
 601     int error = 0;
 602
 603     if ((error = device_delete_child(sc->twe_dev, sc->twe_drive[unit].td_disk)) != 0) {
 604         twe_printf(sc, "failed to delete unit %d\n", unit);
 605         return(error);
 606     }
 607     bzero(&sc->twe_drive[unit], sizeof(sc->twe_drive[unit]));
 608     return(error);
 609 }
 610
 611 /********************************************************************************
 612  * Clear a PCI parity error.
 613  */
 614 void
 615 twe_clear_pci_parity_error(struct twe_softc *sc)
 616 {
 617     TWE_CONTROL(sc, TWE_CONTROL_CLEAR_PARITY_ERROR);
 618     pci_write_config(sc->twe_dev, PCIR_STATUS, TWE_PCI_CLEAR_PARITY_ERROR, 2);
 619 }
 620
 621 /********************************************************************************
 622  * Clear a PCI abort.
 623  */
 624 void
 625 twe_clear_pci_abort(struct twe_softc *sc)
 626 {
 627     TWE_CONTROL(sc, TWE_CONTROL_CLEAR_PCI_ABORT);
 628     pci_write_config(sc->twe_dev, PCIR_STATUS, TWE_PCI_CLEAR_PCI_ABORT, 2);
 629 }
 630
 631 /********************************************************************************
 632  ********************************************************************************
 633                                                                       Disk device
 634  ********************************************************************************
 635  ********************************************************************************/
 636
 637 /*
 638  * Disk device softc
 639  */
 640 struct twed_softc
 641 {
 642     device_t            twed_dev;
 643     struct twe_softc    *twed_controller;       /* parent device softc */
 644     struct twe_drive    *twed_drive;            /* drive data in parent softc */
 645     struct disk         *twed_disk;             /* generic disk handle */
 646 };
 647
 648 /*
 649  * Disk device bus interface
 650  */
 651 static int twed_probe(device_t dev);
 652 static int twed_attach(device_t dev);
 653 static int twed_detach(device_t dev);
 654
 655 static device_method_t twed_methods[] = {
 656     DEVMETHOD(device_probe,     twed_probe),
 657     DEVMETHOD(device_attach,    twed_attach),
 658     DEVMETHOD(device_detach,    twed_detach),
 659     { 0, 0 }
 660 };
 661
 662 static driver_t twed_driver = {
 663     "twed",
 664     twed_methods,
 665     sizeof(struct twed_softc)
 666 };
 667
 668 static devclass_t       twed_devclass;
 669 DRIVER_MODULE(twed, twe, twed_driver, twed_devclass, 0, 0);
 670
 671 /*
 672  * Disk device control interface.
 673  */
 674
 675 #ifdef FREEBSD_4
 676 static int              disks_registered = 0;
 677 #endif
 678
 679 /********************************************************************************
 680  * Handle open from generic layer.
 681  *
 682  * Note that this is typically only called by the diskslice code, and not
 683  * for opens on subdevices (eg. slices, partitions).
 684  */
 685 static int
 686 twed_open(struct disk *dp)
 687 {
 688     struct twed_softc   *sc = (struct twed_softc *)dp->d_drv1;
 689
 690     debug_called(4);
 691
 692     if (sc == NULL)
 693         return (ENXIO);
 694
 695     /* check that the controller is up and running */
 696     if (sc->twed_controller->twe_state & TWE_STATE_SHUTDOWN)
 697         return(ENXIO);
 698
 699     return (0);
 700 }
 701
 702 /********************************************************************************
 703  * Handle an I/O request.
 704  */
 705 static void
 706 twed_strategy(twe_bio *bp)
 707 {
 708     struct twed_softc   *sc = (struct twed_softc *)TWE_BIO_SOFTC(bp);
 709
 710     debug_called(4);
 711
 712     bp->bio_driver1 = &sc->twed_drive->td_twe_unit;
 713     TWED_BIO_IN;
 714
 715     /* bogus disk? */
 716     if (sc == NULL || sc->twed_drive->td_disk == NULL) {
 717         TWE_BIO_SET_ERROR(bp, EINVAL);
 718         printf("twe: bio for invalid disk!\n");
 719         TWE_BIO_DONE(bp);
 720         TWED_BIO_OUT;
 721         return;
 722     }
 723
 724     /* perform accounting */
 725     TWE_BIO_STATS_START(bp);
 726
 727     /* queue the bio on the controller */
 728     twe_enqueue_bio(sc->twed_controller, bp);
 729
 730     /* poke the controller to start I/O */
 731     twe_startio(sc->twed_controller);
 732     return;
 733 }
 734
 735 /********************************************************************************
 736  * System crashdump support
 737  */
 738 static int
 739 twed_dump(void *arg, void *virtual, vm_offset_t physical, off_t offset, size_t length)
 740 {
 741     struct twed_softc   *twed_sc;
 742     struct twe_softc    *twe_sc;
 743     int                 error;
 744     struct disk         *dp;
 745
 746     dp = arg;
 747     twed_sc = (struct twed_softc *)dp->d_drv1;
 748     if (twed_sc == NULL)
 749         return(ENXIO);
 750     twe_sc  = (struct twe_softc *)twed_sc->twed_controller;
 751
 752     if (length > 0) {
 753         if ((error = twe_dump_blocks(twe_sc, twed_sc->twed_drive->td_twe_unit, offset / TWE_BLOCK_SIZE, virtual, length / TWE_BLOCK_SIZE)) != 0)
 754             return(error);
 755     }
 756     return(0);
 757 }
 758
 759 /********************************************************************************
 760  * Handle completion of an I/O request.
 761  */
 762 void
 763 twed_intr(twe_bio *bp)
 764 {
 765     debug_called(4);
 766
 767     /* if no error, transfer completed */
 768     if (!TWE_BIO_HAS_ERROR(bp))
 769         TWE_BIO_RESID(bp) = 0;
 770
 771     TWE_BIO_STATS_END(bp);
 772     TWE_BIO_DONE(bp);
 773     TWED_BIO_OUT;
 774 }
 775
 776 /********************************************************************************
 777  * Default probe stub.
 778  */
 779 static int
 780 twed_probe(device_t dev)
 781 {
 782     return (0);
 783 }
 784
 785 /********************************************************************************
 786  * Attach a unit to the controller.
 787  */
 788 static int
 789 twed_attach(device_t dev)
 790 {
 791     struct twed_softc   *sc;
 792     device_t            parent;
 793
 794     debug_called(4);
 795
 796     /* initialise our softc */
 797     sc = device_get_softc(dev);
 798     parent = device_get_parent(dev);
 799     sc->twed_controller = (struct twe_softc *)device_get_softc(parent);
 800     sc->twed_drive = device_get_ivars(dev);
 801     sc->twed_dev = dev;
 802
 803     /* report the drive */
 804     twed_printf(sc, "%uMB (%u sectors)\n",
 805                 sc->twed_drive->td_size / ((1024 * 1024) / TWE_BLOCK_SIZE),
 806                 sc->twed_drive->td_size);
 807
 808     /* attach a generic disk device to ourselves */
 809
 810     sc->twed_drive->td_sys_unit = device_get_unit(dev);
 811
 812     sc->twed_disk = disk_alloc();
 813     sc->twed_disk->d_open = twed_open;
 814     sc->twed_disk->d_strategy = twed_strategy;
 815     sc->twed_disk->d_dump = (dumper_t *)twed_dump;
 816     sc->twed_disk->d_name = "twed";
 817     sc->twed_disk->d_drv1 = sc;
 818     sc->twed_disk->d_maxsize = (TWE_MAX_SGL_LENGTH - 1) * PAGE_SIZE;
 819     sc->twed_disk->d_sectorsize = TWE_BLOCK_SIZE;
 820     sc->twed_disk->d_mediasize = TWE_BLOCK_SIZE * (off_t)sc->twed_drive->td_size;
 821     if (sc->twed_drive->td_type == TWE_UD_CONFIG_RAID0 ||
 822         sc->twed_drive->td_type == TWE_UD_CONFIG_RAID5 ||
 823         sc->twed_drive->td_type == TWE_UD_CONFIG_RAID10) {
 824             sc->twed_disk->d_stripesize =
 825                 TWE_BLOCK_SIZE << sc->twed_drive->td_stripe;
 826             sc->twed_disk->d_stripeoffset = 0;
 827     }
 828     sc->twed_disk->d_fwsectors = sc->twed_drive->td_sectors;
 829     sc->twed_disk->d_fwheads = sc->twed_drive->td_heads;
 830     sc->twed_disk->d_unit = sc->twed_drive->td_sys_unit;
 831     sc->twed_disk->d_flags = DISKFLAG_NEEDSGIANT;
 832
 833     disk_create(sc->twed_disk, DISK_VERSION);
 834
 835 #ifdef FREEBSD_4
 836     disks_registered++;
 837 #endif
 838
 839     /* set the maximum I/O size to the theoretical maximum allowed by the S/G list size */
 840
 841     return (0);
 842 }
 843
 844 /********************************************************************************
 845  * Disconnect ourselves from the system.
 846  */
 847 static int
 848 twed_detach(device_t dev)
 849 {
 850     struct twed_softc *sc = (struct twed_softc *)device_get_softc(dev);
 851
 852     debug_called(4);
 853
 854     if (sc->twed_disk->d_flags & DISKFLAG_OPEN)
 855         return(EBUSY);
 856
 857     disk_destroy(sc->twed_disk);
 858
 859 #ifdef FREEBSD_4
 860     if (--disks_registered == 0)
 861         cdevsw_remove(&tweddisk_cdevsw);
 862 #endif
 863     return(0);
 864 }
 865
 866 /********************************************************************************
 867  ********************************************************************************
 868                                                                              Misc
 869  ********************************************************************************
 870  ********************************************************************************/
 871
 872 /********************************************************************************
 873  * Allocate a command buffer
 874  */
 875 MALLOC_DEFINE(TWE_MALLOC_CLASS, "twe_commands", "twe commands");
 876
 877 struct twe_request *
 878 twe_allocate_request(struct twe_softc *sc, int tag)
 879 {
 880     struct twe_request  *tr;
 881
 882     if ((tr = malloc(sizeof(struct twe_request), TWE_MALLOC_CLASS, M_WAITOK)) == NULL) {
 883         twe_printf(sc, "unable to allocate memory for tag %d\n", tag);
 884         return(NULL);
 885     }
 886     bzero(tr, sizeof(*tr));
 887     tr->tr_sc = sc;
 888     tr->tr_tag = tag;
 889     if (bus_dmamap_create(sc->twe_buffer_dmat, 0, &tr->tr_dmamap)) {
 890         twe_free_request(tr);
 891         twe_printf(sc, "unable to allocate dmamap for tag %d\n", tag);
 892         return(NULL);
 893     }
 894     return(tr);
 895 }
 896
 897 /********************************************************************************
 898  * Permanently discard a command buffer.
 899  */
 900 void
 901 twe_free_request(struct twe_request *tr)
 902 {
 903     struct twe_softc    *sc = tr->tr_sc;
 904
 905     debug_called(4);
 906
 907     bus_dmamap_destroy(sc->twe_buffer_dmat, tr->tr_dmamap);
 908     free(tr, TWE_MALLOC_CLASS);
 909 }
 910
 911 /********************************************************************************
 912  * Map/unmap (tr)'s command and data in the controller's addressable space.
 913  *
 914  * These routines ensure that the data which the controller is going to try to
 915  * access is actually visible to the controller, in a machine-independant
 916  * fashion.  Due to a hardware limitation, I/O buffers must be 512-byte aligned
 917  * and we take care of that here as well.
 918  */
 919 static void
 920 twe_fillin_sgl(TWE_SG_Entry *sgl, bus_dma_segment_t *segs, int nsegments, int max_sgl)
 921 {
 922     int i;
 923
 924     for (i = 0; i < nsegments; i++) {
 925         sgl[i].address = segs[i].ds_addr;
 926         sgl[i].length = segs[i].ds_len;
 927     }
 928     for (; i < max_sgl; i++) {                          /* XXX necessary? */
 929         sgl[i].address = 0;
 930         sgl[i].length = 0;
 931     }
 932 }
 933
 934 static void
 935 twe_setup_data_dmamap(void *arg, bus_dma_segment_t *segs, int nsegments, int error)
 936 {
 937     struct twe_request  *tr = (struct twe_request *)arg;
 938     struct twe_softc    *sc = tr->tr_sc;
 939     TWE_Command         *cmd = TWE_FIND_COMMAND(tr);
 940
 941     debug_called(4);
 942
 943     if (tr->tr_flags & TWE_CMD_MAPPED)
 944         panic("already mapped command");
 945
 946     tr->tr_flags |= TWE_CMD_MAPPED;
 947
 948     if (tr->tr_flags & TWE_CMD_IN_PROGRESS)
 949         sc->twe_state &= ~TWE_STATE_FRZN;
 950     /* save base of first segment in command (applicable if there only one segment) */
 951     tr->tr_dataphys = segs[0].ds_addr;
 952
 953     /* correct command size for s/g list size */
 954     cmd->generic.size += 2 * nsegments;
 955
 956     /*
 957      * Due to the fact that parameter and I/O commands have the scatter/gather list in
 958      * different places, we need to determine which sort of command this actually is
 959      * before we can populate it correctly.
 960      */
 961     switch(cmd->generic.opcode) {
 962     case TWE_OP_GET_PARAM:
 963     case TWE_OP_SET_PARAM:
 964         cmd->generic.sgl_offset = 2;
 965         twe_fillin_sgl(&cmd->param.sgl[0], segs, nsegments, TWE_MAX_SGL_LENGTH);
 966         break;
 967     case TWE_OP_READ:
 968     case TWE_OP_WRITE:
 969         cmd->generic.sgl_offset = 3;
 970         twe_fillin_sgl(&cmd->io.sgl[0], segs, nsegments, TWE_MAX_SGL_LENGTH);
 971         break;
 972     case TWE_OP_ATA_PASSTHROUGH:
 973         cmd->generic.sgl_offset = 5;
 974         twe_fillin_sgl(&cmd->ata.sgl[0], segs, nsegments, TWE_MAX_ATA_SGL_LENGTH);
 975         break;
 976     default:
 977         /*
 978          * Fall back to what the linux driver does.
 979          * Do this because the API may send an opcode
 980          * the driver knows nothing about and this will
 981          * at least stop PCIABRT's from hosing us.
 982          */
 983         switch (cmd->generic.sgl_offset) {
 984         case 2:
 985             twe_fillin_sgl(&cmd->param.sgl[0], segs, nsegments, TWE_MAX_SGL_LENGTH);
 986             break;
 987         case 3:
 988             twe_fillin_sgl(&cmd->io.sgl[0], segs, nsegments, TWE_MAX_SGL_LENGTH);
 989             break;
 990         case 5:
 991             twe_fillin_sgl(&cmd->ata.sgl[0], segs, nsegments, TWE_MAX_ATA_SGL_LENGTH);
 992             break;
 993         }
 994     }
 995
 996     if (tr->tr_flags & TWE_CMD_DATAIN) {
 997         if (tr->tr_flags & TWE_CMD_IMMEDIATE) {
 998             bus_dmamap_sync(sc->twe_immediate_dmat, sc->twe_immediate_map,
 999                             BUS_DMASYNC_PREREAD);
1000         } else {
1001             bus_dmamap_sync(sc->twe_buffer_dmat, tr->tr_dmamap,
1002                             BUS_DMASYNC_PREREAD);
1003         }
1004     }
1005
1006     if (tr->tr_flags & TWE_CMD_DATAOUT) {
1007         /*
1008          * if we're using an alignment buffer, and we're writing data
1009          * copy the real data out
1010          */
1011         if (tr->tr_flags & TWE_CMD_ALIGNBUF)
1012             bcopy(tr->tr_realdata, tr->tr_data, tr->tr_length);
1013
1014         if (tr->tr_flags & TWE_CMD_IMMEDIATE) {
1015             bus_dmamap_sync(sc->twe_immediate_dmat, sc->twe_immediate_map,
1016                             BUS_DMASYNC_PREWRITE);
1017         } else {
1018             bus_dmamap_sync(sc->twe_buffer_dmat, tr->tr_dmamap,
1019                             BUS_DMASYNC_PREWRITE);
1020         }
1021     }
1022
1023     if (twe_start(tr) == EBUSY) {
1024         tr->tr_sc->twe_state |= TWE_STATE_CTLR_BUSY;
1025         twe_requeue_ready(tr);
1026     }
1027 }
1028
1029 static void
1030 twe_setup_request_dmamap(void *arg, bus_dma_segment_t *segs, int nsegments, int error)
1031 {
1032     struct twe_softc    *sc = (struct twe_softc *)arg;
1033
1034     debug_called(4);
1035
1036     /* command can't cross a page boundary */
1037     sc->twe_cmdphys = segs[0].ds_addr;
1038 }
1039
1040 int
1041 twe_map_request(struct twe_request *tr)
1042 {
1043     struct twe_softc    *sc = tr->tr_sc;
1044     int                 error = 0;
1045
1046     debug_called(4);
1047
1048     if (sc->twe_state & (TWE_STATE_CTLR_BUSY | TWE_STATE_FRZN)) {
1049         twe_requeue_ready(tr);
1050         return (EBUSY);
1051     }
1052
1053     bus_dmamap_sync(sc->twe_cmd_dmat, sc->twe_cmdmap, BUS_DMASYNC_PREWRITE);
1054
1055     /*
1056      * If the command involves data, map that too.
1057      */
1058     if (tr->tr_data != NULL && ((tr->tr_flags & TWE_CMD_MAPPED) == 0)) {
1059
1060         /*
1061          * Data must be 64-byte aligned; allocate a fixup buffer if it's not.
1062          */
1063         if (((vm_offset_t)tr->tr_data % TWE_ALIGNMENT) != 0) {
1064             tr->tr_realdata = tr->tr_data;                              /* save pointer to 'real' data */
1065             tr->tr_flags |= TWE_CMD_ALIGNBUF;
1066             tr->tr_data = malloc(tr->tr_length, TWE_MALLOC_CLASS, M_NOWAIT);
1067             if (tr->tr_data == NULL) {
1068                 twe_printf(sc, "%s: malloc failed\n", __func__);
1069                 tr->tr_data = tr->tr_realdata; /* restore original data pointer */
1070                 return(ENOMEM);
1071             }
1072         }
1073
1074         /*
1075          * Map the data buffer into bus space and build the s/g list.
1076          */
1077         if (tr->tr_flags & TWE_CMD_IMMEDIATE) {
1078             error = bus_dmamap_load(sc->twe_immediate_dmat, sc->twe_immediate_map, sc->twe_immediate,
1079                             tr->tr_length, twe_setup_data_dmamap, tr, BUS_DMA_NOWAIT);
1080         } else {
1081             error = bus_dmamap_load(sc->twe_buffer_dmat, tr->tr_dmamap, tr->tr_data, tr->tr_length,
1082                                     twe_setup_data_dmamap, tr, 0);
1083         }
1084         if (error == EINPROGRESS) {
1085             tr->tr_flags |= TWE_CMD_IN_PROGRESS;
1086             sc->twe_state |= TWE_STATE_FRZN;
1087             error = 0;
1088         }
1089     } else
1090         if ((error = twe_start(tr)) == EBUSY) {
1091             sc->twe_state |= TWE_STATE_CTLR_BUSY;
1092             twe_requeue_ready(tr);
1093         }
1094
1095     return(error);
1096 }
1097
1098 void
1099 twe_unmap_request(struct twe_request *tr)
1100 {
1101     struct twe_softc    *sc = tr->tr_sc;
1102
1103     debug_called(4);
1104
1105     bus_dmamap_sync(sc->twe_cmd_dmat, sc->twe_cmdmap, BUS_DMASYNC_POSTWRITE);
1106
1107     /*
1108      * If the command involved data, unmap that too.
1109      */
1110     if (tr->tr_data != NULL) {
1111         if (tr->tr_flags & TWE_CMD_DATAIN) {
1112             if (tr->tr_flags & TWE_CMD_IMMEDIATE) {
1113                 bus_dmamap_sync(sc->twe_immediate_dmat, sc->twe_immediate_map,
1114                                 BUS_DMASYNC_POSTREAD);
1115             } else {
1116                 bus_dmamap_sync(sc->twe_buffer_dmat, tr->tr_dmamap,
1117                                 BUS_DMASYNC_POSTREAD);
1118             }
1119
1120             /* if we're using an alignment buffer, and we're reading data, copy the real data in */
1121             if (tr->tr_flags & TWE_CMD_ALIGNBUF)
1122                 bcopy(tr->tr_data, tr->tr_realdata, tr->tr_length);
1123         }
1124         if (tr->tr_flags & TWE_CMD_DATAOUT) {
1125             if (tr->tr_flags & TWE_CMD_IMMEDIATE) {
1126                 bus_dmamap_sync(sc->twe_immediate_dmat, sc->twe_immediate_map,
1127                                 BUS_DMASYNC_POSTWRITE);
1128             } else {
1129                 bus_dmamap_sync(sc->twe_buffer_dmat, tr->tr_dmamap,
1130                                 BUS_DMASYNC_POSTWRITE);
1131             }
1132         }
1133
1134         if (tr->tr_flags & TWE_CMD_IMMEDIATE) {
1135             bus_dmamap_unload(sc->twe_immediate_dmat, sc->twe_immediate_map);
1136         } else {
1137             bus_dmamap_unload(sc->twe_buffer_dmat, tr->tr_dmamap);
1138         }
1139     }
1140
1141     /* free alignment buffer if it was used */
1142     if (tr->tr_flags & TWE_CMD_ALIGNBUF) {
1143         free(tr->tr_data, TWE_MALLOC_CLASS);
1144         tr->tr_data = tr->tr_realdata;          /* restore 'real' data pointer */
1145     }
1146 }
1147
1148 #ifdef TWE_DEBUG
1149 void twe_report(void);
1150 /********************************************************************************
1151  * Print current controller status, call from DDB.
1152  */
1153 void
1154 twe_report(void)
1155 {
1156     struct twe_softc    *sc;
1157     int                 i, s;
1158
1159     s = splbio();
1160     for (i = 0; (sc = devclass_get_softc(twe_devclass, i)) != NULL; i++)
1161         twe_print_controller(sc);
1162     printf("twed: total bio count in %u  out %u\n", twed_bio_in, twed_bio_out);
1163     splx(s);
1164 }
1165 #endif