sys/cam/ata/ata_da.c

   1 /*-
   2  * Copyright (c) 2009 Alexander Motin <mav@FreeBSD.org>
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer,
  10  *    without modification, immediately at the beginning of the file.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  18  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  20  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  21  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  22  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  24  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  */
  26
  27 #include <sys/cdefs.h>
  28 __FBSDID("$FreeBSD$");
  29
  30 #include "opt_ada.h"
  31
  32 #include <sys/param.h>
  33
  34 #ifdef _KERNEL
  35 #include <sys/systm.h>
  36 #include <sys/kernel.h>
  37 #include <sys/bio.h>
  38 #include <sys/sysctl.h>
  39 #include <sys/taskqueue.h>
  40 #include <sys/lock.h>
  41 #include <sys/mutex.h>
  42 #include <sys/conf.h>
  43 #include <sys/devicestat.h>
  44 #include <sys/eventhandler.h>
  45 #include <sys/malloc.h>
  46 #include <sys/cons.h>
  47 #include <sys/proc.h>
  48 #include <sys/reboot.h>
  49 #include <geom/geom_disk.h>
  50 #endif /* _KERNEL */
  51
  52 #ifndef _KERNEL
  53 #include <stdio.h>
  54 #include <string.h>
  55 #endif /* _KERNEL */
  56
  57 #include <cam/cam.h>
  58 #include <cam/cam_ccb.h>
  59 #include <cam/cam_periph.h>
  60 #include <cam/cam_xpt_periph.h>
  61 #include <cam/cam_sim.h>
  62
  63 #include <cam/ata/ata_all.h>
  64
  65 #include <machine/md_var.h>     /* geometry translation */
  66
  67 #ifdef _KERNEL
  68
  69 #define ATA_MAX_28BIT_LBA               268435455UL
  70
  71 typedef enum {
  72         ADA_STATE_RAHEAD,
  73         ADA_STATE_WCACHE,
  74         ADA_STATE_NORMAL
  75 } ada_state;
  76
  77 typedef enum {
  78         ADA_FLAG_CAN_48BIT      = 0x0002,
  79         ADA_FLAG_CAN_FLUSHCACHE = 0x0004,
  80         ADA_FLAG_CAN_NCQ        = 0x0008,
  81         ADA_FLAG_CAN_DMA        = 0x0010,
  82         ADA_FLAG_NEED_OTAG      = 0x0020,
  83         ADA_FLAG_WAS_OTAG       = 0x0040,
  84         ADA_FLAG_CAN_TRIM       = 0x0080,
  85         ADA_FLAG_OPEN           = 0x0100,
  86         ADA_FLAG_SCTX_INIT      = 0x0200,
  87         ADA_FLAG_CAN_CFA        = 0x0400,
  88         ADA_FLAG_CAN_POWERMGT   = 0x0800,
  89         ADA_FLAG_CAN_DMA48      = 0x1000,
  90         ADA_FLAG_DIRTY          = 0x2000
  91 } ada_flags;
  92
  93 typedef enum {
  94         ADA_Q_NONE              = 0x00,
  95         ADA_Q_4K                = 0x01,
  96 } ada_quirks;
  97
  98 #define ADA_Q_BIT_STRING        \
  99         "\020"                  \
 100         "\0014K"
 101
 102 typedef enum {
 103         ADA_CCB_RAHEAD          = 0x01,
 104         ADA_CCB_WCACHE          = 0x02,
 105         ADA_CCB_BUFFER_IO       = 0x03,
 106         ADA_CCB_DUMP            = 0x05,
 107         ADA_CCB_TRIM            = 0x06,
 108         ADA_CCB_TYPE_MASK       = 0x0F,
 109 } ada_ccb_state;
 110
 111 /* Offsets into our private area for storing information */
 112 #define ccb_state       ppriv_field0
 113 #define ccb_bp          ppriv_ptr1
 114
 115 struct disk_params {
 116         u_int8_t  heads;
 117         u_int8_t  secs_per_track;
 118         u_int32_t cylinders;
 119         u_int32_t secsize;      /* Number of bytes/logical sector */
 120         u_int64_t sectors;      /* Total number sectors */
 121 };
 122
 123 #define TRIM_MAX_BLOCKS 8
 124 #define TRIM_MAX_RANGES (TRIM_MAX_BLOCKS * ATA_DSM_BLK_RANGES)
 125 struct trim_request {
 126         uint8_t         data[TRIM_MAX_RANGES * ATA_DSM_RANGE_SIZE];
 127         TAILQ_HEAD(, bio) bps;
 128 };
 129
 130 struct ada_softc {
 131         struct   bio_queue_head bio_queue;
 132         struct   bio_queue_head trim_queue;
 133         int      outstanding_cmds;      /* Number of active commands */
 134         int      refcount;              /* Active xpt_action() calls */
 135         ada_state state;
 136         ada_flags flags;
 137         ada_quirks quirks;
 138         int      sort_io_queue;
 139         int      trim_max_ranges;
 140         int      trim_running;
 141         int      read_ahead;
 142         int      write_cache;
 143 #ifdef ADA_TEST_FAILURE
 144         int      force_read_error;
 145         int      force_write_error;
 146         int      periodic_read_error;
 147         int      periodic_read_count;
 148 #endif
 149         struct   disk_params params;
 150         struct   disk *disk;
 151         struct task             sysctl_task;
 152         struct sysctl_ctx_list  sysctl_ctx;
 153         struct sysctl_oid       *sysctl_tree;
 154         struct callout          sendordered_c;
 155         struct trim_request     trim_req;
 156 };
 157
 158 struct ada_quirk_entry {
 159         struct scsi_inquiry_pattern inq_pat;
 160         ada_quirks quirks;
 161 };
 162
 163 static struct ada_quirk_entry ada_quirk_table[] =
 164 {
 165         {
 166                 /* Hitachi Advanced Format (4k) drives */
 167                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Hitachi H??????????E3*", "*" },
 168                 /*quirks*/ADA_Q_4K
 169         },
 170         {
 171                 /* Samsung Advanced Format (4k) drives */
 172                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG HD155UI*", "*" },
 173                 /*quirks*/ADA_Q_4K
 174         },
 175         {
 176                 /* Samsung Advanced Format (4k) drives */
 177                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG HD204UI*", "*" },
 178                 /*quirks*/ADA_Q_4K
 179         },
 180         {
 181                 /* Seagate Barracuda Green Advanced Format (4k) drives */
 182                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST????DL*", "*" },
 183                 /*quirks*/ADA_Q_4K
 184         },
 185         {
 186                 /* Seagate Barracuda Advanced Format (4k) drives */
 187                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST???DM*", "*" },
 188                 /*quirks*/ADA_Q_4K
 189         },
 190         {
 191                 /* Seagate Barracuda Advanced Format (4k) drives */
 192                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST????DM*", "*" },
 193                 /*quirks*/ADA_Q_4K
 194         },
 195         {
 196                 /* Seagate Momentus Advanced Format (4k) drives */
 197                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9500423AS*", "*" },
 198                 /*quirks*/ADA_Q_4K
 199         },
 200         {
 201                 /* Seagate Momentus Advanced Format (4k) drives */
 202                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9500424AS*", "*" },
 203                 /*quirks*/ADA_Q_4K
 204         },
 205         {
 206                 /* Seagate Momentus Advanced Format (4k) drives */
 207                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9640423AS*", "*" },
 208                 /*quirks*/ADA_Q_4K
 209         },
 210         {
 211                 /* Seagate Momentus Advanced Format (4k) drives */
 212                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9640424AS*", "*" },
 213                 /*quirks*/ADA_Q_4K
 214         },
 215         {
 216                 /* Seagate Momentus Advanced Format (4k) drives */
 217                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9750420AS*", "*" },
 218                 /*quirks*/ADA_Q_4K
 219         },
 220         {
 221                 /* Seagate Momentus Advanced Format (4k) drives */
 222                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9750422AS*", "*" },
 223                 /*quirks*/ADA_Q_4K
 224         },
 225         {
 226                 /* Seagate Momentus Advanced Format (4k) drives */
 227                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9750423AS*", "*" },
 228                 /*quirks*/ADA_Q_4K
 229         },
 230         {
 231                 /* Seagate Momentus Thin Advanced Format (4k) drives */
 232                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST???LT*", "*" },
 233                 /*quirks*/ADA_Q_4K
 234         },
 235         {
 236                 /* WDC Caviar Green Advanced Format (4k) drives */
 237                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD????RS*", "*" },
 238                 /*quirks*/ADA_Q_4K
 239         },
 240         {
 241                 /* WDC Caviar Green Advanced Format (4k) drives */
 242                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD????RX*", "*" },
 243                 /*quirks*/ADA_Q_4K
 244         },
 245         {
 246                 /* WDC Caviar Green Advanced Format (4k) drives */
 247                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD??????RS*", "*" },
 248                 /*quirks*/ADA_Q_4K
 249         },
 250         {
 251                 /* WDC Caviar Green Advanced Format (4k) drives */
 252                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD??????RX*", "*" },
 253                 /*quirks*/ADA_Q_4K
 254         },
 255         {
 256                 /* WDC Scorpio Black Advanced Format (4k) drives */
 257                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD???PKT*", "*" },
 258                 /*quirks*/ADA_Q_4K
 259         },
 260         {
 261                 /* WDC Scorpio Black Advanced Format (4k) drives */
 262                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD?????PKT*", "*" },
 263                 /*quirks*/ADA_Q_4K
 264         },
 265         {
 266                 /* WDC Scorpio Blue Advanced Format (4k) drives */
 267                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD???PVT*", "*" },
 268                 /*quirks*/ADA_Q_4K
 269         },
 270         {
 271                 /* WDC Scorpio Blue Advanced Format (4k) drives */
 272                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD?????PVT*", "*" },
 273                 /*quirks*/ADA_Q_4K
 274         },
 275         /* SSDs */
 276         {
 277                 /*
 278                  * Corsair Force 2 SSDs
 279                  * 4k optimised & trim only works in 4k requests + 4k aligned
 280                  */
 281                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair CSSD-F*", "*" },
 282                 /*quirks*/ADA_Q_4K
 283         },
 284         {
 285                 /*
 286                  * Corsair Force 3 SSDs
 287                  * 4k optimised & trim only works in 4k requests + 4k aligned
 288                  */
 289                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair Force 3*", "*" },
 290                 /*quirks*/ADA_Q_4K
 291         },
 292         {
 293                 /*
 294                  * Corsair Neutron GTX SSDs
 295                  * 4k optimised & trim only works in 4k requests + 4k aligned
 296                  */
 297                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair Neutron GTX*", "*" },
 298                 /*quirks*/ADA_Q_4K
 299         },
 300         {
 301                 /*
 302                  * Corsair Force GT SSDs
 303                  * 4k optimised & trim only works in 4k requests + 4k aligned
 304                  */
 305                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair Force GT*", "*" },
 306                 /*quirks*/ADA_Q_4K
 307         },
 308         {
 309                 /*
 310                  * Crucial M4 SSDs
 311                  * 4k optimised & trim only works in 4k requests + 4k aligned
 312                  */
 313                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "M4-CT???M4SSD2*", "*" },
 314                 /*quirks*/ADA_Q_4K
 315         },
 316         {
 317                 /*
 318                  * Crucial RealSSD C300 SSDs
 319                  * 4k optimised
 320                  */
 321                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "C300-CTFDDAC???MAG*",
 322                 "*" }, /*quirks*/ADA_Q_4K
 323         },
 324         {
 325                 /*
 326                  * Intel 320 Series SSDs
 327                  * 4k optimised & trim only works in 4k requests + 4k aligned
 328                  */
 329                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSA2CW*", "*" },
 330                 /*quirks*/ADA_Q_4K
 331         },
 332         {
 333                 /*
 334                  * Intel 330 Series SSDs
 335                  * 4k optimised & trim only works in 4k requests + 4k aligned
 336                  */
 337                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSC2CT*", "*" },
 338                 /*quirks*/ADA_Q_4K
 339         },
 340         {
 341                 /*
 342                  * Intel 510 Series SSDs
 343                  * 4k optimised & trim only works in 4k requests + 4k aligned
 344                  */
 345                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSC2MH*", "*" },
 346                 /*quirks*/ADA_Q_4K
 347         },
 348         {
 349                 /*
 350                  * Intel 520 Series SSDs
 351                  * 4k optimised & trim only works in 4k requests + 4k aligned
 352                  */
 353                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSC2BW*", "*" },
 354                 /*quirks*/ADA_Q_4K
 355         },
 356         {
 357                 /*
 358                  * Intel X25-M Series SSDs
 359                  * 4k optimised & trim only works in 4k requests + 4k aligned
 360                  */
 361                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSA2M*", "*" },
 362                 /*quirks*/ADA_Q_4K
 363         },
 364         {
 365                 /*
 366                  * Kingston E100 Series SSDs
 367                  * 4k optimised & trim only works in 4k requests + 4k aligned
 368                  */
 369                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "KINGSTON SE100S3*", "*" },
 370                 /*quirks*/ADA_Q_4K
 371         },
 372         {
 373                 /*
 374                  * Kingston HyperX 3k SSDs
 375                  * 4k optimised & trim only works in 4k requests + 4k aligned
 376                  */
 377                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "KINGSTON SH103S3*", "*" },
 378                 /*quirks*/ADA_Q_4K
 379         },
 380         {
 381                 /*
 382                  * Marvell SSDs (entry taken from OpenSolaris)
 383                  * 4k optimised & trim only works in 4k requests + 4k aligned
 384                  */
 385                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "MARVELL SD88SA02*", "*" },
 386                 /*quirks*/ADA_Q_4K
 387         },
 388         {
 389                 /*
 390                  * OCZ Agility 2 SSDs
 391                  * 4k optimised & trim only works in 4k requests + 4k aligned
 392                  */
 393                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-AGILITY2*", "*" },
 394                 /*quirks*/ADA_Q_4K
 395         },
 396         {
 397                 /*
 398                  * OCZ Agility 3 SSDs
 399                  * 4k optimised & trim only works in 4k requests + 4k aligned
 400                  */
 401                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-AGILITY3*", "*" },
 402                 /*quirks*/ADA_Q_4K
 403         },
 404         {
 405                 /*
 406                  * OCZ Deneva R Series SSDs
 407                  * 4k optimised & trim only works in 4k requests + 4k aligned
 408                  */
 409                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "DENRSTE251M45*", "*" },
 410                 /*quirks*/ADA_Q_4K
 411         },
 412         {
 413                 /*
 414                  * OCZ Vertex 2 SSDs (inc pro series)
 415                  * 4k optimised & trim only works in 4k requests + 4k aligned
 416                  */
 417                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ?VERTEX2*", "*" },
 418                 /*quirks*/ADA_Q_4K
 419         },
 420         {
 421                 /*
 422                  * OCZ Vertex 3 SSDs
 423                  * 4k optimised & trim only works in 4k requests + 4k aligned
 424                  */
 425                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-VERTEX3*", "*" },
 426                 /*quirks*/ADA_Q_4K
 427         },
 428         {
 429                 /*
 430                  * OCZ Vertex 4 SSDs
 431                  * 4k optimised & trim only works in 4k requests + 4k aligned
 432                  */
 433                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-VERTEX4*", "*" },
 434                 /*quirks*/ADA_Q_4K
 435         },
 436         {
 437                 /*
 438                  * Samsung 830 Series SSDs
 439                  * 4k optimised
 440                  */
 441                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG SSD 830 Series*", "*" },
 442                 /*quirks*/ADA_Q_4K
 443         },
 444         {
 445                 /*
 446                  * SuperTalent TeraDrive CT SSDs
 447                  * 4k optimised & trim only works in 4k requests + 4k aligned
 448                  */
 449                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "FTM??CT25H*", "*" },
 450                 /*quirks*/ADA_Q_4K
 451         },
 452         {
 453                 /*
 454                  * XceedIOPS SATA SSDs
 455                  * 4k optimised
 456                  */
 457                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SG9XCS2D*", "*" },
 458                 /*quirks*/ADA_Q_4K
 459         },
 460         {
 461                 /* Default */
 462                 {
 463                   T_ANY, SIP_MEDIA_REMOVABLE|SIP_MEDIA_FIXED,
 464                   /*vendor*/"*", /*product*/"*", /*revision*/"*"
 465                 },
 466                 /*quirks*/0
 467         },
 468 };
 469
 470 static  disk_strategy_t adastrategy;
 471 static  dumper_t        adadump;
 472 static  periph_init_t   adainit;
 473 static  void            adaasync(void *callback_arg, u_int32_t code,
 474                                 struct cam_path *path, void *arg);
 475 static  void            adasysctlinit(void *context, int pending);
 476 static  periph_ctor_t   adaregister;
 477 static  periph_dtor_t   adacleanup;
 478 static  periph_start_t  adastart;
 479 static  periph_oninv_t  adaoninvalidate;
 480 static  void            adadone(struct cam_periph *periph,
 481                                union ccb *done_ccb);
 482 static  int             adaerror(union ccb *ccb, u_int32_t cam_flags,
 483                                 u_int32_t sense_flags);
 484 static void             adagetparams(struct cam_periph *periph,
 485                                 struct ccb_getdev *cgd);
 486 static timeout_t        adasendorderedtag;
 487 static void             adashutdown(void *arg, int howto);
 488 static void             adasuspend(void *arg);
 489 static void             adaresume(void *arg);
 490
 491 #ifndef ADA_DEFAULT_LEGACY_ALIASES
 492 #define ADA_DEFAULT_LEGACY_ALIASES      1
 493 #endif
 494
 495 #ifndef ADA_DEFAULT_TIMEOUT
 496 #define ADA_DEFAULT_TIMEOUT 30  /* Timeout in seconds */
 497 #endif
 498
 499 #ifndef ADA_DEFAULT_RETRY
 500 #define ADA_DEFAULT_RETRY       4
 501 #endif
 502
 503 #ifndef ADA_DEFAULT_SEND_ORDERED
 504 #define ADA_DEFAULT_SEND_ORDERED        1
 505 #endif
 506
 507 #ifndef ADA_DEFAULT_SPINDOWN_SHUTDOWN
 508 #define ADA_DEFAULT_SPINDOWN_SHUTDOWN   1
 509 #endif
 510
 511 #ifndef ADA_DEFAULT_SPINDOWN_SUSPEND
 512 #define ADA_DEFAULT_SPINDOWN_SUSPEND    1
 513 #endif
 514
 515 #ifndef ADA_DEFAULT_READ_AHEAD
 516 #define ADA_DEFAULT_READ_AHEAD  1
 517 #endif
 518
 519 #ifndef ADA_DEFAULT_WRITE_CACHE
 520 #define ADA_DEFAULT_WRITE_CACHE 1
 521 #endif
 522
 523 #define ADA_RA  (softc->read_ahead >= 0 ? \
 524                  softc->read_ahead : ada_read_ahead)
 525 #define ADA_WC  (softc->write_cache >= 0 ? \
 526                  softc->write_cache : ada_write_cache)
 527 #define ADA_SIO (softc->sort_io_queue >= 0 ? \
 528                  softc->sort_io_queue : cam_sort_io_queues)
 529
 530 /*
 531  * Most platforms map firmware geometry to actual, but some don't.  If
 532  * not overridden, default to nothing.
 533  */
 534 #ifndef ata_disk_firmware_geom_adjust
 535 #define ata_disk_firmware_geom_adjust(disk)
 536 #endif
 537
 538 static int ada_legacy_aliases = ADA_DEFAULT_LEGACY_ALIASES;
 539 static int ada_retry_count = ADA_DEFAULT_RETRY;
 540 static int ada_default_timeout = ADA_DEFAULT_TIMEOUT;
 541 static int ada_send_ordered = ADA_DEFAULT_SEND_ORDERED;
 542 static int ada_spindown_shutdown = ADA_DEFAULT_SPINDOWN_SHUTDOWN;
 543 static int ada_spindown_suspend = ADA_DEFAULT_SPINDOWN_SUSPEND;
 544 static int ada_read_ahead = ADA_DEFAULT_READ_AHEAD;
 545 static int ada_write_cache = ADA_DEFAULT_WRITE_CACHE;
 546
 547 static SYSCTL_NODE(_kern_cam, OID_AUTO, ada, CTLFLAG_RD, 0,
 548             "CAM Direct Access Disk driver");
 549 SYSCTL_INT(_kern_cam_ada, OID_AUTO, legacy_aliases, CTLFLAG_RW,
 550            &ada_legacy_aliases, 0, "Create legacy-like device aliases");
 551 TUNABLE_INT("kern.cam.ada.legacy_aliases", &ada_legacy_aliases);
 552 SYSCTL_INT(_kern_cam_ada, OID_AUTO, retry_count, CTLFLAG_RW,
 553            &ada_retry_count, 0, "Normal I/O retry count");
 554 TUNABLE_INT("kern.cam.ada.retry_count", &ada_retry_count);
 555 SYSCTL_INT(_kern_cam_ada, OID_AUTO, default_timeout, CTLFLAG_RW,
 556            &ada_default_timeout, 0, "Normal I/O timeout (in seconds)");
 557 TUNABLE_INT("kern.cam.ada.default_timeout", &ada_default_timeout);
 558 SYSCTL_INT(_kern_cam_ada, OID_AUTO, send_ordered, CTLFLAG_RW,
 559            &ada_send_ordered, 0, "Send Ordered Tags");
 560 TUNABLE_INT("kern.cam.ada.send_ordered", &ada_send_ordered);
 561 SYSCTL_INT(_kern_cam_ada, OID_AUTO, spindown_shutdown, CTLFLAG_RW,
 562            &ada_spindown_shutdown, 0, "Spin down upon shutdown");
 563 TUNABLE_INT("kern.cam.ada.spindown_shutdown", &ada_spindown_shutdown);
 564 SYSCTL_INT(_kern_cam_ada, OID_AUTO, spindown_suspend, CTLFLAG_RW,
 565            &ada_spindown_suspend, 0, "Spin down upon suspend");
 566 TUNABLE_INT("kern.cam.ada.spindown_suspend", &ada_spindown_suspend);
 567 SYSCTL_INT(_kern_cam_ada, OID_AUTO, read_ahead, CTLFLAG_RW,
 568            &ada_read_ahead, 0, "Enable disk read-ahead");
 569 TUNABLE_INT("kern.cam.ada.read_ahead", &ada_read_ahead);
 570 SYSCTL_INT(_kern_cam_ada, OID_AUTO, write_cache, CTLFLAG_RW,
 571            &ada_write_cache, 0, "Enable disk write cache");
 572 TUNABLE_INT("kern.cam.ada.write_cache", &ada_write_cache);
 573
 574 /*
 575  * ADA_ORDEREDTAG_INTERVAL determines how often, relative
 576  * to the default timeout, we check to see whether an ordered
 577  * tagged transaction is appropriate to prevent simple tag
 578  * starvation.  Since we'd like to ensure that there is at least
 579  * 1/2 of the timeout length left for a starved transaction to
 580  * complete after we've sent an ordered tag, we must poll at least
 581  * four times in every timeout period.  This takes care of the worst
 582  * case where a starved transaction starts during an interval that
 583  * meets the requirement "don't send an ordered tag" test so it takes
 584  * us two intervals to determine that a tag must be sent.
 585  */
 586 #ifndef ADA_ORDEREDTAG_INTERVAL
 587 #define ADA_ORDEREDTAG_INTERVAL 4
 588 #endif
 589
 590 static struct periph_driver adadriver =
 591 {
 592         adainit, "ada",
 593         TAILQ_HEAD_INITIALIZER(adadriver.units), /* generation */ 0
 594 };
 595
 596 PERIPHDRIVER_DECLARE(ada, adadriver);
 597
 598 static MALLOC_DEFINE(M_ATADA, "ata_da", "ata_da buffers");
 599
 600 static int
 601 adaopen(struct disk *dp)
 602 {
 603         struct cam_periph *periph;
 604         struct ada_softc *softc;
 605         int error;
 606
 607         periph = (struct cam_periph *)dp->d_drv1;
 608         if (cam_periph_acquire(periph) != CAM_REQ_CMP) {
 609                 return(ENXIO);
 610         }
 611
 612         cam_periph_lock(periph);
 613         if ((error = cam_periph_hold(periph, PRIBIO|PCATCH)) != 0) {
 614                 cam_periph_unlock(periph);
 615                 cam_periph_release(periph);
 616                 return (error);
 617         }
 618
 619         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE | CAM_DEBUG_PERIPH,
 620             ("adaopen\n"));
 621
 622         softc = (struct ada_softc *)periph->softc;
 623         softc->flags |= ADA_FLAG_OPEN;
 624
 625         cam_periph_unhold(periph);
 626         cam_periph_unlock(periph);
 627         return (0);
 628 }
 629
 630 static int
 631 adaclose(struct disk *dp)
 632 {
 633         struct  cam_periph *periph;
 634         struct  ada_softc *softc;
 635         union ccb *ccb;
 636         int error;
 637
 638         periph = (struct cam_periph *)dp->d_drv1;
 639         softc = (struct ada_softc *)periph->softc;
 640         cam_periph_lock(periph);
 641
 642         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE | CAM_DEBUG_PERIPH,
 643             ("adaclose\n"));
 644
 645         /* We only sync the cache if the drive is capable of it. */
 646         if ((softc->flags & ADA_FLAG_DIRTY) != 0 &&
 647             (softc->flags & ADA_FLAG_CAN_FLUSHCACHE) != 0 &&
 648             (periph->flags & CAM_PERIPH_INVALID) == 0 &&
 649             cam_periph_hold(periph, PRIBIO) == 0) {
 650
 651                 ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL);
 652                 cam_fill_ataio(&ccb->ataio,
 653                                     1,
 654                                     adadone,
 655                                     CAM_DIR_NONE,
 656                                     0,
 657                                     NULL,
 658                                     0,
 659                                     ada_default_timeout*1000);
 660
 661                 if (softc->flags & ADA_FLAG_CAN_48BIT)
 662                         ata_48bit_cmd(&ccb->ataio, ATA_FLUSHCACHE48, 0, 0, 0);
 663                 else
 664                         ata_28bit_cmd(&ccb->ataio, ATA_FLUSHCACHE, 0, 0, 0);
 665                 error = cam_periph_runccb(ccb, adaerror, /*cam_flags*/0,
 666                     /*sense_flags*/0, softc->disk->d_devstat);
 667
 668                 if (error != 0)
 669                         xpt_print(periph->path, "Synchronize cache failed\n");
 670                 else
 671                         softc->flags &= ~ADA_FLAG_DIRTY;
 672                 xpt_release_ccb(ccb);
 673                 cam_periph_unhold(periph);
 674         }
 675
 676         softc->flags &= ~ADA_FLAG_OPEN;
 677
 678         while (softc->refcount != 0)
 679                 cam_periph_sleep(periph, &softc->refcount, PRIBIO, "adaclose", 1);
 680         cam_periph_unlock(periph);
 681         cam_periph_release(periph);
 682         return (0);
 683 }
 684
 685 static void
 686 adaschedule(struct cam_periph *periph)
 687 {
 688         struct ada_softc *softc = (struct ada_softc *)periph->softc;
 689
 690         if (softc->state != ADA_STATE_NORMAL)
 691                 return;
 692
 693         /* Check if we have more work to do. */
 694         if (bioq_first(&softc->bio_queue) ||
 695             (!softc->trim_running && bioq_first(&softc->trim_queue))) {
 696                 xpt_schedule(periph, CAM_PRIORITY_NORMAL);
 697         }
 698 }
 699
 700 /*
 701  * Actually translate the requested transfer into one the physical driver
 702  * can understand.  The transfer is described by a buf and will include
 703  * only one physical transfer.
 704  */
 705 static void
 706 adastrategy(struct bio *bp)
 707 {
 708         struct cam_periph *periph;
 709         struct ada_softc *softc;
 710
 711         periph = (struct cam_periph *)bp->bio_disk->d_drv1;
 712         softc = (struct ada_softc *)periph->softc;
 713
 714         cam_periph_lock(periph);
 715
 716         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("adastrategy(%p)\n", bp));
 717
 718         /*
 719          * If the device has been made invalid, error out
 720          */
 721         if ((periph->flags & CAM_PERIPH_INVALID) != 0) {
 722                 cam_periph_unlock(periph);
 723                 biofinish(bp, NULL, ENXIO);
 724                 return;
 725         }
 726
 727         /*
 728          * Place it in the queue of disk activities for this disk
 729          */
 730         if (bp->bio_cmd == BIO_DELETE &&
 731             (softc->flags & ADA_FLAG_CAN_TRIM)) {
 732                 if (ADA_SIO)
 733                     bioq_disksort(&softc->trim_queue, bp);
 734                 else
 735                     bioq_insert_tail(&softc->trim_queue, bp);
 736         } else {
 737                 if (ADA_SIO)
 738                     bioq_disksort(&softc->bio_queue, bp);
 739                 else
 740                     bioq_insert_tail(&softc->bio_queue, bp);
 741         }
 742
 743         /*
 744          * Schedule ourselves for performing the work.
 745          */
 746         adaschedule(periph);
 747         cam_periph_unlock(periph);
 748
 749         return;
 750 }
 751
 752 static int
 753 adadump(void *arg, void *virtual, vm_offset_t physical, off_t offset, size_t length)
 754 {
 755         struct      cam_periph *periph;
 756         struct      ada_softc *softc;
 757         u_int       secsize;
 758         union       ccb ccb;
 759         struct      disk *dp;
 760         uint64_t    lba;
 761         uint16_t    count;
 762         int         error = 0;
 763
 764         dp = arg;
 765         periph = dp->d_drv1;
 766         softc = (struct ada_softc *)periph->softc;
 767         cam_periph_lock(periph);
 768         secsize = softc->params.secsize;
 769         lba = offset / secsize;
 770         count = length / secsize;
 771
 772         if ((periph->flags & CAM_PERIPH_INVALID) != 0) {
 773                 cam_periph_unlock(periph);
 774                 return (ENXIO);
 775         }
 776
 777         if (length > 0) {
 778                 xpt_setup_ccb(&ccb.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
 779                 ccb.ccb_h.ccb_state = ADA_CCB_DUMP;
 780                 cam_fill_ataio(&ccb.ataio,
 781                     0,
 782                     adadone,
 783                     CAM_DIR_OUT,
 784                     0,
 785                     (u_int8_t *) virtual,
 786                     length,
 787                     ada_default_timeout*1000);
 788                 if ((softc->flags & ADA_FLAG_CAN_48BIT) &&
 789                     (lba + count >= ATA_MAX_28BIT_LBA ||
 790                     count >= 256)) {
 791                         ata_48bit_cmd(&ccb.ataio, ATA_WRITE_DMA48,
 792                             0, lba, count);
 793                 } else {
 794                         ata_28bit_cmd(&ccb.ataio, ATA_WRITE_DMA,
 795                             0, lba, count);
 796                 }
 797                 xpt_polled_action(&ccb);
 798
 799                 error = cam_periph_error(&ccb,
 800                     0, SF_NO_RECOVERY | SF_NO_RETRY, NULL);
 801                 if ((ccb.ccb_h.status & CAM_DEV_QFRZN) != 0)
 802                         cam_release_devq(ccb.ccb_h.path, /*relsim_flags*/0,
 803                             /*reduction*/0, /*timeout*/0, /*getcount_only*/0);
 804                 if (error != 0)
 805                         printf("Aborting dump due to I/O error.\n");
 806
 807                 cam_periph_unlock(periph);
 808                 return (error);
 809         }
 810
 811         if (softc->flags & ADA_FLAG_CAN_FLUSHCACHE) {
 812                 xpt_setup_ccb(&ccb.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
 813
 814                 ccb.ccb_h.ccb_state = ADA_CCB_DUMP;
 815                 cam_fill_ataio(&ccb.ataio,
 816                                     0,
 817                                     adadone,
 818                                     CAM_DIR_NONE,
 819                                     0,
 820                                     NULL,
 821                                     0,
 822                                     ada_default_timeout*1000);
 823
 824                 if (softc->flags & ADA_FLAG_CAN_48BIT)
 825                         ata_48bit_cmd(&ccb.ataio, ATA_FLUSHCACHE48, 0, 0, 0);
 826                 else
 827                         ata_28bit_cmd(&ccb.ataio, ATA_FLUSHCACHE, 0, 0, 0);
 828                 xpt_polled_action(&ccb);
 829
 830                 error = cam_periph_error(&ccb,
 831                     0, SF_NO_RECOVERY | SF_NO_RETRY, NULL);
 832                 if ((ccb.ccb_h.status & CAM_DEV_QFRZN) != 0)
 833                         cam_release_devq(ccb.ccb_h.path, /*relsim_flags*/0,
 834                             /*reduction*/0, /*timeout*/0, /*getcount_only*/0);
 835                 if (error != 0)
 836                         xpt_print(periph->path, "Synchronize cache failed\n");
 837         }
 838         cam_periph_unlock(periph);
 839         return (error);
 840 }
 841
 842 static void
 843 adainit(void)
 844 {
 845         cam_status status;
 846
 847         /*
 848          * Install a global async callback.  This callback will
 849          * receive async callbacks like "new device found".
 850          */
 851         status = xpt_register_async(AC_FOUND_DEVICE, adaasync, NULL, NULL);
 852
 853         if (status != CAM_REQ_CMP) {
 854                 printf("ada: Failed to attach master async callback "
 855                        "due to status 0x%x!\n", status);
 856         } else if (ada_send_ordered) {
 857
 858                 /* Register our event handlers */
 859                 if ((EVENTHANDLER_REGISTER(power_suspend, adasuspend,
 860                                            NULL, EVENTHANDLER_PRI_LAST)) == NULL)
 861                     printf("adainit: power event registration failed!\n");
 862                 if ((EVENTHANDLER_REGISTER(power_resume, adaresume,
 863                                            NULL, EVENTHANDLER_PRI_LAST)) == NULL)
 864                     printf("adainit: power event registration failed!\n");
 865                 if ((EVENTHANDLER_REGISTER(shutdown_post_sync, adashutdown,
 866                                            NULL, SHUTDOWN_PRI_DEFAULT)) == NULL)
 867                     printf("adainit: shutdown event registration failed!\n");
 868         }
 869 }
 870
 871 /*
 872  * Callback from GEOM, called when it has finished cleaning up its
 873  * resources.
 874  */
 875 static void
 876 adadiskgonecb(struct disk *dp)
 877 {
 878         struct cam_periph *periph;
 879
 880         periph = (struct cam_periph *)dp->d_drv1;
 881
 882         cam_periph_release(periph);
 883 }
 884
 885 static void
 886 adaoninvalidate(struct cam_periph *periph)
 887 {
 888         struct ada_softc *softc;
 889
 890         softc = (struct ada_softc *)periph->softc;
 891
 892         /*
 893          * De-register any async callbacks.
 894          */
 895         xpt_register_async(0, adaasync, periph, periph->path);
 896
 897         /*
 898          * Return all queued I/O with ENXIO.
 899          * XXX Handle any transactions queued to the card
 900          *     with XPT_ABORT_CCB.
 901          */
 902         bioq_flush(&softc->bio_queue, NULL, ENXIO);
 903         bioq_flush(&softc->trim_queue, NULL, ENXIO);
 904
 905         disk_gone(softc->disk);
 906 }
 907
 908 static void
 909 adacleanup(struct cam_periph *periph)
 910 {
 911         struct ada_softc *softc;
 912
 913         softc = (struct ada_softc *)periph->softc;
 914
 915         cam_periph_unlock(periph);
 916
 917         /*
 918          * If we can't free the sysctl tree, oh well...
 919          */
 920         if ((softc->flags & ADA_FLAG_SCTX_INIT) != 0
 921             && sysctl_ctx_free(&softc->sysctl_ctx) != 0) {
 922                 xpt_print(periph->path, "can't remove sysctl context\n");
 923         }
 924
 925         disk_destroy(softc->disk);
 926         callout_drain(&softc->sendordered_c);
 927         free(softc, M_DEVBUF);
 928         cam_periph_lock(periph);
 929 }
 930
 931 static void
 932 adaasync(void *callback_arg, u_int32_t code,
 933         struct cam_path *path, void *arg)
 934 {
 935         struct ccb_getdev cgd;
 936         struct cam_periph *periph;
 937         struct ada_softc *softc;
 938
 939         periph = (struct cam_periph *)callback_arg;
 940         switch (code) {
 941         case AC_FOUND_DEVICE:
 942         {
 943                 struct ccb_getdev *cgd;
 944                 cam_status status;
 945
 946                 cgd = (struct ccb_getdev *)arg;
 947                 if (cgd == NULL)
 948                         break;
 949
 950                 if (cgd->protocol != PROTO_ATA)
 951                         break;
 952
 953                 /*
 954                  * Allocate a peripheral instance for
 955                  * this device and start the probe
 956                  * process.
 957                  */
 958                 status = cam_periph_alloc(adaregister, adaoninvalidate,
 959                                           adacleanup, adastart,
 960                                           "ada", CAM_PERIPH_BIO,
 961                                           path, adaasync,
 962                                           AC_FOUND_DEVICE, cgd);
 963
 964                 if (status != CAM_REQ_CMP
 965                  && status != CAM_REQ_INPROG)
 966                         printf("adaasync: Unable to attach to new device "
 967                                 "due to status 0x%x\n", status);
 968                 break;
 969         }
 970         case AC_GETDEV_CHANGED:
 971         {
 972                 softc = (struct ada_softc *)periph->softc;
 973                 xpt_setup_ccb(&cgd.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
 974                 cgd.ccb_h.func_code = XPT_GDEV_TYPE;
 975                 xpt_action((union ccb *)&cgd);
 976
 977                 if ((cgd.ident_data.capabilities1 & ATA_SUPPORT_DMA) &&
 978                     (cgd.inq_flags & SID_DMA))
 979                         softc->flags |= ADA_FLAG_CAN_DMA;
 980                 else
 981                         softc->flags &= ~ADA_FLAG_CAN_DMA;
 982                 if (cgd.ident_data.support.command2 & ATA_SUPPORT_ADDRESS48) {
 983                         softc->flags |= ADA_FLAG_CAN_48BIT;
 984                         if (cgd.inq_flags & SID_DMA48)
 985                                 softc->flags |= ADA_FLAG_CAN_DMA48;
 986                         else
 987                                 softc->flags &= ~ADA_FLAG_CAN_DMA48;
 988                 } else
 989                         softc->flags &= ~(ADA_FLAG_CAN_48BIT |
 990                             ADA_FLAG_CAN_DMA48);
 991                 if ((cgd.ident_data.satacapabilities & ATA_SUPPORT_NCQ) &&
 992                     (cgd.inq_flags & SID_DMA) && (cgd.inq_flags & SID_CmdQue))
 993                         softc->flags |= ADA_FLAG_CAN_NCQ;
 994                 else
 995                         softc->flags &= ~ADA_FLAG_CAN_NCQ;
 996                 if ((cgd.ident_data.support_dsm & ATA_SUPPORT_DSM_TRIM) &&
 997                     (cgd.inq_flags & SID_DMA))
 998                         softc->flags |= ADA_FLAG_CAN_TRIM;
 999                 else
1000                         softc->flags &= ~ADA_FLAG_CAN_TRIM;
1001
1002                 cam_periph_async(periph, code, path, arg);
1003                 break;
1004         }
1005         case AC_ADVINFO_CHANGED:
1006         {
1007                 uintptr_t buftype;
1008
1009                 buftype = (uintptr_t)arg;
1010                 if (buftype == CDAI_TYPE_PHYS_PATH) {
1011                         struct ada_softc *softc;
1012
1013                         softc = periph->softc;
1014                         disk_attr_changed(softc->disk, "GEOM::physpath",
1015                                           M_NOWAIT);
1016                 }
1017                 break;
1018         }
1019         case AC_SENT_BDR:
1020         case AC_BUS_RESET:
1021         {
1022                 softc = (struct ada_softc *)periph->softc;
1023                 cam_periph_async(periph, code, path, arg);
1024                 if (softc->state != ADA_STATE_NORMAL)
1025                         break;
1026                 xpt_setup_ccb(&cgd.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
1027                 cgd.ccb_h.func_code = XPT_GDEV_TYPE;
1028                 xpt_action((union ccb *)&cgd);
1029                 if (ADA_RA >= 0 &&
1030                     cgd.ident_data.support.command1 & ATA_SUPPORT_LOOKAHEAD)
1031                         softc->state = ADA_STATE_RAHEAD;
1032                 else if (ADA_WC >= 0 &&
1033                     cgd.ident_data.support.command1 & ATA_SUPPORT_WRITECACHE)
1034                         softc->state = ADA_STATE_WCACHE;
1035                 else
1036                     break;
1037                 if (cam_periph_acquire(periph) != CAM_REQ_CMP)
1038                         softc->state = ADA_STATE_NORMAL;
1039                 else
1040                         xpt_schedule(periph, CAM_PRIORITY_DEV);
1041         }
1042         default:
1043                 cam_periph_async(periph, code, path, arg);
1044                 break;
1045         }
1046 }
1047
1048 static void
1049 adasysctlinit(void *context, int pending)
1050 {
1051         struct cam_periph *periph;
1052         struct ada_softc *softc;
1053         char tmpstr[80], tmpstr2[80];
1054
1055         periph = (struct cam_periph *)context;
1056
1057         /* periph was held for us when this task was enqueued */
1058         if ((periph->flags & CAM_PERIPH_INVALID) != 0) {
1059                 cam_periph_release(periph);
1060                 return;
1061         }
1062
1063         softc = (struct ada_softc *)periph->softc;
1064         snprintf(tmpstr, sizeof(tmpstr), "CAM ADA unit %d", periph->unit_number);
1065         snprintf(tmpstr2, sizeof(tmpstr2), "%d", periph->unit_number);
1066
1067         sysctl_ctx_init(&softc->sysctl_ctx);
1068         softc->flags |= ADA_FLAG_SCTX_INIT;
1069         softc->sysctl_tree = SYSCTL_ADD_NODE(&softc->sysctl_ctx,
1070                 SYSCTL_STATIC_CHILDREN(_kern_cam_ada), OID_AUTO, tmpstr2,
1071                 CTLFLAG_RD, 0, tmpstr);
1072         if (softc->sysctl_tree == NULL) {
1073                 printf("adasysctlinit: unable to allocate sysctl tree\n");
1074                 cam_periph_release(periph);
1075                 return;
1076         }
1077
1078         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1079                 OID_AUTO, "read_ahead", CTLFLAG_RW | CTLFLAG_MPSAFE,
1080                 &softc->read_ahead, 0, "Enable disk read ahead.");
1081         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1082                 OID_AUTO, "write_cache", CTLFLAG_RW | CTLFLAG_MPSAFE,
1083                 &softc->write_cache, 0, "Enable disk write cache.");
1084         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1085                 OID_AUTO, "sort_io_queue", CTLFLAG_RW | CTLFLAG_MPSAFE,
1086                 &softc->sort_io_queue, 0,
1087                 "Sort IO queue to try and optimise disk access patterns");
1088 #ifdef ADA_TEST_FAILURE
1089         /*
1090          * Add a 'door bell' sysctl which allows one to set it from userland
1091          * and cause something bad to happen.  For the moment, we only allow
1092          * whacking the next read or write.
1093          */
1094         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1095                 OID_AUTO, "force_read_error", CTLFLAG_RW | CTLFLAG_MPSAFE,
1096                 &softc->force_read_error, 0,
1097                 "Force a read error for the next N reads.");
1098         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1099                 OID_AUTO, "force_write_error", CTLFLAG_RW | CTLFLAG_MPSAFE,
1100                 &softc->force_write_error, 0,
1101                 "Force a write error for the next N writes.");
1102         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1103                 OID_AUTO, "periodic_read_error", CTLFLAG_RW | CTLFLAG_MPSAFE,
1104                 &softc->periodic_read_error, 0,
1105                 "Force a read error every N reads (don't set too low).");
1106 #endif
1107         cam_periph_release(periph);
1108 }
1109
1110 static int
1111 adagetattr(struct bio *bp)
1112 {
1113         int ret;
1114         struct cam_periph *periph;
1115
1116         periph = (struct cam_periph *)bp->bio_disk->d_drv1;
1117         cam_periph_lock(periph);
1118         ret = xpt_getattr(bp->bio_data, bp->bio_length, bp->bio_attribute,
1119             periph->path);
1120         cam_periph_unlock(periph);
1121         if (ret == 0)
1122                 bp->bio_completed = bp->bio_length;
1123         return ret;
1124 }
1125
1126 static cam_status
1127 adaregister(struct cam_periph *periph, void *arg)
1128 {
1129         struct ada_softc *softc;
1130         struct ccb_pathinq cpi;
1131         struct ccb_getdev *cgd;
1132         char   announce_buf[80], buf1[32];
1133         struct disk_params *dp;
1134         caddr_t match;
1135         u_int maxio;
1136         int legacy_id, quirks;
1137
1138         cgd = (struct ccb_getdev *)arg;
1139         if (cgd == NULL) {
1140                 printf("adaregister: no getdev CCB, can't register device\n");
1141                 return(CAM_REQ_CMP_ERR);
1142         }
1143
1144         softc = (struct ada_softc *)malloc(sizeof(*softc), M_DEVBUF,
1145             M_NOWAIT|M_ZERO);
1146
1147         if (softc == NULL) {
1148                 printf("adaregister: Unable to probe new device. "
1149                     "Unable to allocate softc\n");
1150                 return(CAM_REQ_CMP_ERR);
1151         }
1152
1153         bioq_init(&softc->bio_queue);
1154         bioq_init(&softc->trim_queue);
1155
1156         if ((cgd->ident_data.capabilities1 & ATA_SUPPORT_DMA) &&
1157             (cgd->inq_flags & SID_DMA))
1158                 softc->flags |= ADA_FLAG_CAN_DMA;
1159         if (cgd->ident_data.support.command2 & ATA_SUPPORT_ADDRESS48) {
1160                 softc->flags |= ADA_FLAG_CAN_48BIT;
1161                 if (cgd->inq_flags & SID_DMA48)
1162                         softc->flags |= ADA_FLAG_CAN_DMA48;
1163         }
1164         if (cgd->ident_data.support.command2 & ATA_SUPPORT_FLUSHCACHE)
1165                 softc->flags |= ADA_FLAG_CAN_FLUSHCACHE;
1166         if (cgd->ident_data.support.command1 & ATA_SUPPORT_POWERMGT)
1167                 softc->flags |= ADA_FLAG_CAN_POWERMGT;
1168         if ((cgd->ident_data.satacapabilities & ATA_SUPPORT_NCQ) &&
1169             (cgd->inq_flags & SID_DMA) && (cgd->inq_flags & SID_CmdQue))
1170                 softc->flags |= ADA_FLAG_CAN_NCQ;
1171         if ((cgd->ident_data.support_dsm & ATA_SUPPORT_DSM_TRIM) &&
1172             (cgd->inq_flags & SID_DMA)) {
1173                 softc->flags |= ADA_FLAG_CAN_TRIM;
1174                 softc->trim_max_ranges = TRIM_MAX_RANGES;
1175                 if (cgd->ident_data.max_dsm_blocks != 0) {
1176                         softc->trim_max_ranges =
1177                             min(cgd->ident_data.max_dsm_blocks *
1178                                 ATA_DSM_BLK_RANGES, softc->trim_max_ranges);
1179                 }
1180         }
1181         if (cgd->ident_data.support.command2 & ATA_SUPPORT_CFA)
1182                 softc->flags |= ADA_FLAG_CAN_CFA;
1183
1184         periph->softc = softc;
1185
1186         /*
1187          * See if this device has any quirks.
1188          */
1189         match = cam_quirkmatch((caddr_t)&cgd->ident_data,
1190                                (caddr_t)ada_quirk_table,
1191                                sizeof(ada_quirk_table)/sizeof(*ada_quirk_table),
1192                                sizeof(*ada_quirk_table), ata_identify_match);
1193         if (match != NULL)
1194                 softc->quirks = ((struct ada_quirk_entry *)match)->quirks;
1195         else
1196                 softc->quirks = ADA_Q_NONE;
1197
1198         bzero(&cpi, sizeof(cpi));
1199         xpt_setup_ccb(&cpi.ccb_h, periph->path, CAM_PRIORITY_NONE);
1200         cpi.ccb_h.func_code = XPT_PATH_INQ;
1201         xpt_action((union ccb *)&cpi);
1202
1203         TASK_INIT(&softc->sysctl_task, 0, adasysctlinit, periph);
1204
1205         /*
1206          * Register this media as a disk
1207          */
1208         (void)cam_periph_hold(periph, PRIBIO);
1209         cam_periph_unlock(periph);
1210         snprintf(announce_buf, sizeof(announce_buf),
1211             "kern.cam.ada.%d.quirks", periph->unit_number);
1212         quirks = softc->quirks;
1213         TUNABLE_INT_FETCH(announce_buf, &quirks);
1214         softc->quirks = quirks;
1215         softc->read_ahead = -1;
1216         snprintf(announce_buf, sizeof(announce_buf),
1217             "kern.cam.ada.%d.read_ahead", periph->unit_number);
1218         TUNABLE_INT_FETCH(announce_buf, &softc->read_ahead);
1219         softc->write_cache = -1;
1220         snprintf(announce_buf, sizeof(announce_buf),
1221             "kern.cam.ada.%d.write_cache", periph->unit_number);
1222         TUNABLE_INT_FETCH(announce_buf, &softc->write_cache);
1223         /* Disable queue sorting for non-rotational media by default. */
1224         if (cgd->ident_data.media_rotation_rate == 1)
1225                 softc->sort_io_queue = 0;
1226         else
1227                 softc->sort_io_queue = -1;
1228         adagetparams(periph, cgd);
1229         softc->disk = disk_alloc();
1230         softc->disk->d_devstat = devstat_new_entry(periph->periph_name,
1231                           periph->unit_number, softc->params.secsize,
1232                           DEVSTAT_ALL_SUPPORTED,
1233                           DEVSTAT_TYPE_DIRECT |
1234                           XPORT_DEVSTAT_TYPE(cpi.transport),
1235                           DEVSTAT_PRIORITY_DISK);
1236         softc->disk->d_open = adaopen;
1237         softc->disk->d_close = adaclose;
1238         softc->disk->d_strategy = adastrategy;
1239         softc->disk->d_getattr = adagetattr;
1240         softc->disk->d_dump = adadump;
1241         softc->disk->d_gone = adadiskgonecb;
1242         softc->disk->d_name = "ada";
1243         softc->disk->d_drv1 = periph;
1244         maxio = cpi.maxio;              /* Honor max I/O size of SIM */
1245         if (maxio == 0)
1246                 maxio = DFLTPHYS;       /* traditional default */
1247         else if (maxio > MAXPHYS)
1248                 maxio = MAXPHYS;        /* for safety */
1249         if (softc->flags & ADA_FLAG_CAN_48BIT)
1250                 maxio = min(maxio, 65536 * softc->params.secsize);
1251         else                                    /* 28bit ATA command limit */
1252                 maxio = min(maxio, 256 * softc->params.secsize);
1253         softc->disk->d_maxsize = maxio;
1254         softc->disk->d_unit = periph->unit_number;
1255         softc->disk->d_flags = DISKFLAG_DIRECT_COMPLETION;
1256         if (softc->flags & ADA_FLAG_CAN_FLUSHCACHE)
1257                 softc->disk->d_flags |= DISKFLAG_CANFLUSHCACHE;
1258         if (softc->flags & ADA_FLAG_CAN_TRIM) {
1259                 softc->disk->d_flags |= DISKFLAG_CANDELETE;
1260                 softc->disk->d_delmaxsize = softc->params.secsize *
1261                                             ATA_DSM_RANGE_MAX *
1262                                             softc->trim_max_ranges;
1263         } else if ((softc->flags & ADA_FLAG_CAN_CFA) &&
1264             !(softc->flags & ADA_FLAG_CAN_48BIT)) {
1265                 softc->disk->d_flags |= DISKFLAG_CANDELETE;
1266                 softc->disk->d_delmaxsize = 256 * softc->params.secsize;
1267         } else
1268                 softc->disk->d_delmaxsize = maxio;
1269         if ((cpi.hba_misc & PIM_UNMAPPED) != 0)
1270                 softc->disk->d_flags |= DISKFLAG_UNMAPPED_BIO;
1271         strlcpy(softc->disk->d_descr, cgd->ident_data.model,
1272             MIN(sizeof(softc->disk->d_descr), sizeof(cgd->ident_data.model)));
1273         strlcpy(softc->disk->d_ident, cgd->ident_data.serial,
1274             MIN(sizeof(softc->disk->d_ident), sizeof(cgd->ident_data.serial)));
1275         softc->disk->d_hba_vendor = cpi.hba_vendor;
1276         softc->disk->d_hba_device = cpi.hba_device;
1277         softc->disk->d_hba_subvendor = cpi.hba_subvendor;
1278         softc->disk->d_hba_subdevice = cpi.hba_subdevice;
1279
1280         softc->disk->d_sectorsize = softc->params.secsize;
1281         softc->disk->d_mediasize = (off_t)softc->params.sectors *
1282             softc->params.secsize;
1283         if (ata_physical_sector_size(&cgd->ident_data) !=
1284             softc->params.secsize) {
1285                 softc->disk->d_stripesize =
1286                     ata_physical_sector_size(&cgd->ident_data);
1287                 softc->disk->d_stripeoffset = (softc->disk->d_stripesize -
1288                     ata_logical_sector_offset(&cgd->ident_data)) %
1289                     softc->disk->d_stripesize;
1290         } else if (softc->quirks & ADA_Q_4K) {
1291                 softc->disk->d_stripesize = 4096;
1292                 softc->disk->d_stripeoffset = 0;
1293         }
1294         softc->disk->d_fwsectors = softc->params.secs_per_track;
1295         softc->disk->d_fwheads = softc->params.heads;
1296         ata_disk_firmware_geom_adjust(softc->disk);
1297
1298         if (ada_legacy_aliases) {
1299 #ifdef ATA_STATIC_ID
1300                 legacy_id = xpt_path_legacy_ata_id(periph->path);
1301 #else
1302                 legacy_id = softc->disk->d_unit;
1303 #endif
1304                 if (legacy_id >= 0) {
1305                         snprintf(announce_buf, sizeof(announce_buf),
1306                             "kern.devalias.%s%d",
1307                             softc->disk->d_name, softc->disk->d_unit);
1308                         snprintf(buf1, sizeof(buf1),
1309                             "ad%d", legacy_id);
1310                         setenv(announce_buf, buf1);
1311                 }
1312         } else
1313                 legacy_id = -1;
1314         /*
1315          * Acquire a reference to the periph before we register with GEOM.
1316          * We'll release this reference once GEOM calls us back (via
1317          * adadiskgonecb()) telling us that our provider has been freed.
1318          */
1319         if (cam_periph_acquire(periph) != CAM_REQ_CMP) {
1320                 xpt_print(periph->path, "%s: lost periph during "
1321                           "registration!\n", __func__);
1322                 cam_periph_lock(periph);
1323                 return (CAM_REQ_CMP_ERR);
1324         }
1325         disk_create(softc->disk, DISK_VERSION);
1326         cam_periph_lock(periph);
1327         cam_periph_unhold(periph);
1328
1329         dp = &softc->params;
1330         snprintf(announce_buf, sizeof(announce_buf),
1331                 "%juMB (%ju %u byte sectors: %dH %dS/T %dC)",
1332                 (uintmax_t)(((uintmax_t)dp->secsize *
1333                 dp->sectors) / (1024*1024)),
1334                 (uintmax_t)dp->sectors,
1335                 dp->secsize, dp->heads,
1336                 dp->secs_per_track, dp->cylinders);
1337         xpt_announce_periph(periph, announce_buf);
1338         xpt_announce_quirks(periph, softc->quirks, ADA_Q_BIT_STRING);
1339         if (legacy_id >= 0)
1340                 printf("%s%d: Previously was known as ad%d\n",
1341                        periph->periph_name, periph->unit_number, legacy_id);
1342
1343         /*
1344          * Create our sysctl variables, now that we know
1345          * we have successfully attached.
1346          */
1347         if (cam_periph_acquire(periph) == CAM_REQ_CMP)
1348                 taskqueue_enqueue(taskqueue_thread, &softc->sysctl_task);
1349
1350         /*
1351          * Add async callbacks for bus reset and
1352          * bus device reset calls.  I don't bother
1353          * checking if this fails as, in most cases,
1354          * the system will function just fine without
1355          * them and the only alternative would be to
1356          * not attach the device on failure.
1357          */
1358         xpt_register_async(AC_SENT_BDR | AC_BUS_RESET | AC_LOST_DEVICE |
1359             AC_GETDEV_CHANGED | AC_ADVINFO_CHANGED,
1360             adaasync, periph, periph->path);
1361
1362         /*
1363          * Schedule a periodic event to occasionally send an
1364          * ordered tag to a device.
1365          */
1366         callout_init_mtx(&softc->sendordered_c, cam_periph_mtx(periph), 0);
1367         callout_reset(&softc->sendordered_c,
1368             (ada_default_timeout * hz) / ADA_ORDEREDTAG_INTERVAL,
1369             adasendorderedtag, softc);
1370
1371         if (ADA_RA >= 0 &&
1372             cgd->ident_data.support.command1 & ATA_SUPPORT_LOOKAHEAD) {
1373                 softc->state = ADA_STATE_RAHEAD;
1374         } else if (ADA_WC >= 0 &&
1375             cgd->ident_data.support.command1 & ATA_SUPPORT_WRITECACHE) {
1376                 softc->state = ADA_STATE_WCACHE;
1377         } else {
1378                 softc->state = ADA_STATE_NORMAL;
1379                 return(CAM_REQ_CMP);
1380         }
1381         if (cam_periph_acquire(periph) != CAM_REQ_CMP)
1382                 softc->state = ADA_STATE_NORMAL;
1383         else
1384                 xpt_schedule(periph, CAM_PRIORITY_DEV);
1385         return(CAM_REQ_CMP);
1386 }
1387
1388 static void
1389 adastart(struct cam_periph *periph, union ccb *start_ccb)
1390 {
1391         struct ada_softc *softc = (struct ada_softc *)periph->softc;
1392         struct ccb_ataio *ataio = &start_ccb->ataio;
1393
1394         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("adastart\n"));
1395
1396         switch (softc->state) {
1397         case ADA_STATE_NORMAL:
1398         {
1399                 struct bio *bp;
1400                 u_int8_t tag_code;
1401
1402                 /* Run TRIM if not running yet. */
1403                 if (!softc->trim_running &&
1404                     (bp = bioq_first(&softc->trim_queue)) != 0) {
1405                         struct trim_request *req = &softc->trim_req;
1406                         struct bio *bp1;
1407                         uint64_t lastlba = (uint64_t)-1;
1408                         int c, lastcount = 0, off, ranges = 0;
1409
1410                         softc->trim_running = 1;
1411                         bzero(req, sizeof(*req));
1412                         TAILQ_INIT(&req->bps);
1413                         bp1 = bp;
1414                         do {
1415                                 uint64_t lba = bp1->bio_pblkno;
1416                                 int count = bp1->bio_bcount /
1417                                     softc->params.secsize;
1418
1419                                 bioq_remove(&softc->trim_queue, bp1);
1420
1421                                 /* Try to extend the previous range. */
1422                                 if (lba == lastlba) {
1423                                         c = min(count, ATA_DSM_RANGE_MAX - lastcount);
1424                                         lastcount += c;
1425                                         off = (ranges - 1) * ATA_DSM_RANGE_SIZE;
1426                                         req->data[off + 6] = lastcount & 0xff;
1427                                         req->data[off + 7] =
1428                                             (lastcount >> 8) & 0xff;
1429                                         count -= c;
1430                                         lba += c;
1431                                 }
1432
1433                                 while (count > 0) {
1434                                         c = min(count, ATA_DSM_RANGE_MAX);
1435                                         off = ranges * ATA_DSM_RANGE_SIZE;
1436                                         req->data[off + 0] = lba & 0xff;
1437                                         req->data[off + 1] = (lba >> 8) & 0xff;
1438                                         req->data[off + 2] = (lba >> 16) & 0xff;
1439                                         req->data[off + 3] = (lba >> 24) & 0xff;
1440                                         req->data[off + 4] = (lba >> 32) & 0xff;
1441                                         req->data[off + 5] = (lba >> 40) & 0xff;
1442                                         req->data[off + 6] = c & 0xff;
1443                                         req->data[off + 7] = (c >> 8) & 0xff;
1444                                         lba += c;
1445                                         count -= c;
1446                                         lastcount = c;
1447                                         ranges++;
1448                                         /*
1449                                          * Its the caller's responsibility to ensure the
1450                                          * request will fit so we don't need to check for
1451                                          * overrun here
1452                                          */
1453                                 }
1454                                 lastlba = lba;
1455                                 TAILQ_INSERT_TAIL(&req->bps, bp1, bio_queue);
1456                                 bp1 = bioq_first(&softc->trim_queue);
1457                                 if (bp1 == NULL ||
1458                                     bp1->bio_bcount / softc->params.secsize >
1459                                     (softc->trim_max_ranges - ranges) *
1460                                     ATA_DSM_RANGE_MAX)
1461                                         break;
1462                         } while (1);
1463                         cam_fill_ataio(ataio,
1464                             ada_retry_count,
1465                             adadone,
1466                             CAM_DIR_OUT,
1467                             0,
1468                             req->data,
1469                             ((ranges + ATA_DSM_BLK_RANGES - 1) /
1470                                 ATA_DSM_BLK_RANGES) * ATA_DSM_BLK_SIZE,
1471                             ada_default_timeout * 1000);
1472                         ata_48bit_cmd(ataio, ATA_DATA_SET_MANAGEMENT,
1473                             ATA_DSM_TRIM, 0, (ranges + ATA_DSM_BLK_RANGES -
1474                             1) / ATA_DSM_BLK_RANGES);
1475                         start_ccb->ccb_h.ccb_state = ADA_CCB_TRIM;
1476                         start_ccb->ccb_h.flags |= CAM_UNLOCKED;
1477                         goto out;
1478                 }
1479                 /* Run regular command. */
1480                 bp = bioq_first(&softc->bio_queue);
1481                 if (bp == NULL) {
1482                         xpt_release_ccb(start_ccb);
1483                         break;
1484                 }
1485                 bioq_remove(&softc->bio_queue, bp);
1486
1487                 if ((bp->bio_flags & BIO_ORDERED) != 0
1488                  || (softc->flags & ADA_FLAG_NEED_OTAG) != 0) {
1489                         softc->flags &= ~ADA_FLAG_NEED_OTAG;
1490                         softc->flags |= ADA_FLAG_WAS_OTAG;
1491                         tag_code = 0;
1492                 } else {
1493                         tag_code = 1;
1494                 }
1495                 switch (bp->bio_cmd) {
1496                 case BIO_WRITE:
1497                         softc->flags |= ADA_FLAG_DIRTY;
1498                         /* FALLTHROUGH */
1499                 case BIO_READ:
1500                 {
1501                         uint64_t lba = bp->bio_pblkno;
1502                         uint16_t count = bp->bio_bcount / softc->params.secsize;
1503 #ifdef ADA_TEST_FAILURE
1504                         int fail = 0;
1505
1506                         /*
1507                          * Support the failure ioctls.  If the command is a
1508                          * read, and there are pending forced read errors, or
1509                          * if a write and pending write errors, then fail this
1510                          * operation with EIO.  This is useful for testing
1511                          * purposes.  Also, support having every Nth read fail.
1512                          *
1513                          * This is a rather blunt tool.
1514                          */
1515                         if (bp->bio_cmd == BIO_READ) {
1516                                 if (softc->force_read_error) {
1517                                         softc->force_read_error--;
1518                                         fail = 1;
1519                                 }
1520                                 if (softc->periodic_read_error > 0) {
1521                                         if (++softc->periodic_read_count >=
1522                                             softc->periodic_read_error) {
1523                                                 softc->periodic_read_count = 0;
1524                                                 fail = 1;
1525                                         }
1526                                 }
1527                         } else {
1528                                 if (softc->force_write_error) {
1529                                         softc->force_write_error--;
1530                                         fail = 1;
1531                                 }
1532                         }
1533                         if (fail) {
1534                                 bp->bio_error = EIO;
1535                                 bp->bio_flags |= BIO_ERROR;
1536                                 biodone(bp);
1537                                 xpt_release_ccb(start_ccb);
1538                                 adaschedule(periph);
1539                                 return;
1540                         }
1541 #endif
1542                         KASSERT((bp->bio_flags & BIO_UNMAPPED) == 0 ||
1543                             round_page(bp->bio_bcount + bp->bio_ma_offset) /
1544                             PAGE_SIZE == bp->bio_ma_n,
1545                             ("Short bio %p", bp));
1546                         cam_fill_ataio(ataio,
1547                             ada_retry_count,
1548                             adadone,
1549                             (bp->bio_cmd == BIO_READ ? CAM_DIR_IN :
1550                                 CAM_DIR_OUT) | ((bp->bio_flags & BIO_UNMAPPED)
1551                                 != 0 ? CAM_DATA_BIO : 0),
1552                             tag_code,
1553                             ((bp->bio_flags & BIO_UNMAPPED) != 0) ? (void *)bp :
1554                                 bp->bio_data,
1555                             bp->bio_bcount,
1556                             ada_default_timeout*1000);
1557
1558                         if ((softc->flags & ADA_FLAG_CAN_NCQ) && tag_code) {
1559                                 if (bp->bio_cmd == BIO_READ) {
1560                                         ata_ncq_cmd(ataio, ATA_READ_FPDMA_QUEUED,
1561                                             lba, count);
1562                                 } else {
1563                                         ata_ncq_cmd(ataio, ATA_WRITE_FPDMA_QUEUED,
1564                                             lba, count);
1565                                 }
1566                         } else if ((softc->flags & ADA_FLAG_CAN_48BIT) &&
1567                             (lba + count >= ATA_MAX_28BIT_LBA ||
1568                             count > 256)) {
1569                                 if (softc->flags & ADA_FLAG_CAN_DMA48) {
1570                                         if (bp->bio_cmd == BIO_READ) {
1571                                                 ata_48bit_cmd(ataio, ATA_READ_DMA48,
1572                                                     0, lba, count);
1573                                         } else {
1574                                                 ata_48bit_cmd(ataio, ATA_WRITE_DMA48,
1575                                                     0, lba, count);
1576                                         }
1577                                 } else {
1578                                         if (bp->bio_cmd == BIO_READ) {
1579                                                 ata_48bit_cmd(ataio, ATA_READ_MUL48,
1580                                                     0, lba, count);
1581                                         } else {
1582                                                 ata_48bit_cmd(ataio, ATA_WRITE_MUL48,
1583                                                     0, lba, count);
1584                                         }
1585                                 }
1586                         } else {
1587                                 if (count == 256)
1588                                         count = 0;
1589                                 if (softc->flags & ADA_FLAG_CAN_DMA) {
1590                                         if (bp->bio_cmd == BIO_READ) {
1591                                                 ata_28bit_cmd(ataio, ATA_READ_DMA,
1592                                                     0, lba, count);
1593                                         } else {
1594                                                 ata_28bit_cmd(ataio, ATA_WRITE_DMA,
1595                                                     0, lba, count);
1596                                         }
1597                                 } else {
1598                                         if (bp->bio_cmd == BIO_READ) {
1599                                                 ata_28bit_cmd(ataio, ATA_READ_MUL,
1600                                                     0, lba, count);
1601                                         } else {
1602                                                 ata_28bit_cmd(ataio, ATA_WRITE_MUL,
1603                                                     0, lba, count);
1604                                         }
1605                                 }
1606                         }
1607                         break;
1608                 }
1609                 case BIO_DELETE:
1610                 {
1611                         uint64_t lba = bp->bio_pblkno;
1612                         uint16_t count = bp->bio_bcount / softc->params.secsize;
1613
1614                         cam_fill_ataio(ataio,
1615                             ada_retry_count,
1616                             adadone,
1617                             CAM_DIR_NONE,
1618                             0,
1619                             NULL,
1620                             0,
1621                             ada_default_timeout*1000);
1622
1623                         if (count >= 256)
1624                                 count = 0;
1625                         ata_28bit_cmd(ataio, ATA_CFA_ERASE, 0, lba, count);
1626                         break;
1627                 }
1628                 case BIO_FLUSH:
1629                         cam_fill_ataio(ataio,
1630                             1,
1631                             adadone,
1632                             CAM_DIR_NONE,
1633                             0,
1634                             NULL,
1635                             0,
1636                             ada_default_timeout*1000);
1637
1638                         if (softc->flags & ADA_FLAG_CAN_48BIT)
1639                                 ata_48bit_cmd(ataio, ATA_FLUSHCACHE48, 0, 0, 0);
1640                         else
1641                                 ata_28bit_cmd(ataio, ATA_FLUSHCACHE, 0, 0, 0);
1642                         break;
1643                 }
1644                 start_ccb->ccb_h.ccb_state = ADA_CCB_BUFFER_IO;
1645                 start_ccb->ccb_h.flags |= CAM_UNLOCKED;
1646 out:
1647                 start_ccb->ccb_h.ccb_bp = bp;
1648                 softc->outstanding_cmds++;
1649                 softc->refcount++;
1650                 cam_periph_unlock(periph);
1651                 xpt_action(start_ccb);
1652                 cam_periph_lock(periph);
1653                 softc->refcount--;
1654
1655                 /* May have more work to do, so ensure we stay scheduled */
1656                 adaschedule(periph);
1657                 break;
1658         }
1659         case ADA_STATE_RAHEAD:
1660         case ADA_STATE_WCACHE:
1661         {
1662                 cam_fill_ataio(ataio,
1663                     1,
1664                     adadone,
1665                     CAM_DIR_NONE,
1666                     0,
1667                     NULL,
1668                     0,
1669                     ada_default_timeout*1000);
1670
1671                 if (softc->state == ADA_STATE_RAHEAD) {
1672                         ata_28bit_cmd(ataio, ATA_SETFEATURES, ADA_RA ?
1673                             ATA_SF_ENAB_RCACHE : ATA_SF_DIS_RCACHE, 0, 0);
1674                         start_ccb->ccb_h.ccb_state = ADA_CCB_RAHEAD;
1675                 } else {
1676                         ata_28bit_cmd(ataio, ATA_SETFEATURES, ADA_WC ?
1677                             ATA_SF_ENAB_WCACHE : ATA_SF_DIS_WCACHE, 0, 0);
1678                         start_ccb->ccb_h.ccb_state = ADA_CCB_WCACHE;
1679                 }
1680                 start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
1681                 xpt_action(start_ccb);
1682                 break;
1683         }
1684         }
1685 }
1686
1687 static void
1688 adadone(struct cam_periph *periph, union ccb *done_ccb)
1689 {
1690         struct ada_softc *softc;
1691         struct ccb_ataio *ataio;
1692         struct ccb_getdev *cgd;
1693         struct cam_path *path;
1694         int state;
1695
1696         softc = (struct ada_softc *)periph->softc;
1697         ataio = &done_ccb->ataio;
1698         path = done_ccb->ccb_h.path;
1699
1700         CAM_DEBUG(path, CAM_DEBUG_TRACE, ("adadone\n"));
1701
1702         state = ataio->ccb_h.ccb_state & ADA_CCB_TYPE_MASK;
1703         switch (state) {
1704         case ADA_CCB_BUFFER_IO:
1705         case ADA_CCB_TRIM:
1706         {
1707                 struct bio *bp;
1708                 int error;
1709
1710                 cam_periph_lock(periph);
1711                 if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
1712                         error = adaerror(done_ccb, 0, 0);
1713                         if (error == ERESTART) {
1714                                 /* A retry was scheduled, so just return. */
1715                                 cam_periph_unlock(periph);
1716                                 return;
1717                         }
1718                         if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0)
1719                                 cam_release_devq(path,
1720                                                  /*relsim_flags*/0,
1721                                                  /*reduction*/0,
1722                                                  /*timeout*/0,
1723                                                  /*getcount_only*/0);
1724                 } else {
1725                         if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0)
1726                                 panic("REQ_CMP with QFRZN");
1727                         error = 0;
1728                 }
1729                 bp = (struct bio *)done_ccb->ccb_h.ccb_bp;
1730                 bp->bio_error = error;
1731                 if (error != 0) {
1732                         bp->bio_resid = bp->bio_bcount;
1733                         bp->bio_flags |= BIO_ERROR;
1734                 } else {
1735                         if (state == ADA_CCB_TRIM)
1736                                 bp->bio_resid = 0;
1737                         else
1738                                 bp->bio_resid = ataio->resid;
1739                         if (bp->bio_resid > 0)
1740                                 bp->bio_flags |= BIO_ERROR;
1741                 }
1742                 softc->outstanding_cmds--;
1743                 if (softc->outstanding_cmds == 0)
1744                         softc->flags |= ADA_FLAG_WAS_OTAG;
1745                 xpt_release_ccb(done_ccb);
1746                 if (state == ADA_CCB_TRIM) {
1747                         TAILQ_HEAD(, bio) queue;
1748                         struct bio *bp1;
1749
1750                         TAILQ_INIT(&queue);
1751                         TAILQ_CONCAT(&queue, &softc->trim_req.bps, bio_queue);
1752                         softc->trim_running = 0;
1753                         adaschedule(periph);
1754                         cam_periph_unlock(periph);
1755                         while ((bp1 = TAILQ_FIRST(&queue)) != NULL) {
1756                                 TAILQ_REMOVE(&queue, bp1, bio_queue);
1757                                 bp1->bio_error = error;
1758                                 if (error != 0) {
1759                                         bp1->bio_flags |= BIO_ERROR;
1760                                         bp1->bio_resid = bp1->bio_bcount;
1761                                 } else
1762                                         bp1->bio_resid = 0;
1763                                 biodone(bp1);
1764                         }
1765                 } else {
1766                         cam_periph_unlock(periph);
1767                         biodone(bp);
1768                 }
1769                 return;
1770         }
1771         case ADA_CCB_RAHEAD:
1772         {
1773                 if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
1774                         if (adaerror(done_ccb, 0, 0) == ERESTART) {
1775 out:
1776                                 /* Drop freeze taken due to CAM_DEV_QFREEZE */
1777                                 cam_release_devq(path, 0, 0, 0, FALSE);
1778                                 return;
1779                         } else if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
1780                                 cam_release_devq(path,
1781                                     /*relsim_flags*/0,
1782                                     /*reduction*/0,
1783                                     /*timeout*/0,
1784                                     /*getcount_only*/0);
1785                         }
1786                 }
1787
1788                 /*
1789                  * Since our peripheral may be invalidated by an error
1790                  * above or an external event, we must release our CCB
1791                  * before releasing the reference on the peripheral.
1792                  * The peripheral will only go away once the last reference
1793                  * is removed, and we need it around for the CCB release
1794                  * operation.
1795                  */
1796                 cgd = (struct ccb_getdev *)done_ccb;
1797                 xpt_setup_ccb(&cgd->ccb_h, path, CAM_PRIORITY_NORMAL);
1798                 cgd->ccb_h.func_code = XPT_GDEV_TYPE;
1799                 xpt_action((union ccb *)cgd);
1800                 if (ADA_WC >= 0 &&
1801                     cgd->ident_data.support.command1 & ATA_SUPPORT_WRITECACHE) {
1802                         softc->state = ADA_STATE_WCACHE;
1803                         xpt_release_ccb(done_ccb);
1804                         xpt_schedule(periph, CAM_PRIORITY_DEV);
1805                         goto out;
1806                 }
1807                 softc->state = ADA_STATE_NORMAL;
1808                 xpt_release_ccb(done_ccb);
1809                 /* Drop freeze taken due to CAM_DEV_QFREEZE */
1810                 cam_release_devq(path, 0, 0, 0, FALSE);
1811                 adaschedule(periph);
1812                 cam_periph_release_locked(periph);
1813                 return;
1814         }
1815         case ADA_CCB_WCACHE:
1816         {
1817                 if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
1818                         if (adaerror(done_ccb, 0, 0) == ERESTART) {
1819                                 goto out;
1820                         } else if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
1821                                 cam_release_devq(path,
1822                                     /*relsim_flags*/0,
1823                                     /*reduction*/0,
1824                                     /*timeout*/0,
1825                                     /*getcount_only*/0);
1826                         }
1827                 }
1828
1829                 softc->state = ADA_STATE_NORMAL;
1830                 /*
1831                  * Since our peripheral may be invalidated by an error
1832                  * above or an external event, we must release our CCB
1833                  * before releasing the reference on the peripheral.
1834                  * The peripheral will only go away once the last reference
1835                  * is removed, and we need it around for the CCB release
1836                  * operation.
1837                  */
1838                 xpt_release_ccb(done_ccb);
1839                 /* Drop freeze taken due to CAM_DEV_QFREEZE */
1840                 cam_release_devq(path, 0, 0, 0, FALSE);
1841                 adaschedule(periph);
1842                 cam_periph_release_locked(periph);
1843                 return;
1844         }
1845         case ADA_CCB_DUMP:
1846                 /* No-op.  We're polling */
1847                 return;
1848         default:
1849                 break;
1850         }
1851         xpt_release_ccb(done_ccb);
1852 }
1853
1854 static int
1855 adaerror(union ccb *ccb, u_int32_t cam_flags, u_int32_t sense_flags)
1856 {
1857
1858         return(cam_periph_error(ccb, cam_flags, sense_flags, NULL));
1859 }
1860
1861 static void
1862 adagetparams(struct cam_periph *periph, struct ccb_getdev *cgd)
1863 {
1864         struct ada_softc *softc = (struct ada_softc *)periph->softc;
1865         struct disk_params *dp = &softc->params;
1866         u_int64_t lbasize48;
1867         u_int32_t lbasize;
1868
1869         dp->secsize = ata_logical_sector_size(&cgd->ident_data);
1870         if ((cgd->ident_data.atavalid & ATA_FLAG_54_58) &&
1871                 cgd->ident_data.current_heads && cgd->ident_data.current_sectors) {
1872                 dp->heads = cgd->ident_data.current_heads;
1873                 dp->secs_per_track = cgd->ident_data.current_sectors;
1874                 dp->cylinders = cgd->ident_data.cylinders;
1875                 dp->sectors = (u_int32_t)cgd->ident_data.current_size_1 |
1876                           ((u_int32_t)cgd->ident_data.current_size_2 << 16);
1877         } else {
1878                 dp->heads = cgd->ident_data.heads;
1879                 dp->secs_per_track = cgd->ident_data.sectors;
1880                 dp->cylinders = cgd->ident_data.cylinders;
1881                 dp->sectors = cgd->ident_data.cylinders * dp->heads * dp->secs_per_track;
1882         }
1883         lbasize = (u_int32_t)cgd->ident_data.lba_size_1 |
1884                   ((u_int32_t)cgd->ident_data.lba_size_2 << 16);
1885
1886         /* use the 28bit LBA size if valid or bigger than the CHS mapping */
1887         if (cgd->ident_data.cylinders == 16383 || dp->sectors < lbasize)
1888                 dp->sectors = lbasize;
1889
1890         /* use the 48bit LBA size if valid */
1891         lbasize48 = ((u_int64_t)cgd->ident_data.lba_size48_1) |
1892                     ((u_int64_t)cgd->ident_data.lba_size48_2 << 16) |
1893                     ((u_int64_t)cgd->ident_data.lba_size48_3 << 32) |
1894                     ((u_int64_t)cgd->ident_data.lba_size48_4 << 48);
1895         if ((cgd->ident_data.support.command2 & ATA_SUPPORT_ADDRESS48) &&
1896             lbasize48 > ATA_MAX_28BIT_LBA)
1897                 dp->sectors = lbasize48;
1898 }
1899
1900 static void
1901 adasendorderedtag(void *arg)
1902 {
1903         struct ada_softc *softc = arg;
1904
1905         if (ada_send_ordered) {
1906                 if (softc->outstanding_cmds > 0) {
1907                         if ((softc->flags & ADA_FLAG_WAS_OTAG) == 0)
1908                                 softc->flags |= ADA_FLAG_NEED_OTAG;
1909                         softc->flags &= ~ADA_FLAG_WAS_OTAG;
1910                 }
1911         }
1912         /* Queue us up again */
1913         callout_reset(&softc->sendordered_c,
1914             (ada_default_timeout * hz) / ADA_ORDEREDTAG_INTERVAL,
1915             adasendorderedtag, softc);
1916 }
1917
1918 /*
1919  * Step through all ADA peripheral drivers, and if the device is still open,
1920  * sync the disk cache to physical media.
1921  */
1922 static void
1923 adaflush(void)
1924 {
1925         struct cam_periph *periph;
1926         struct ada_softc *softc;
1927         union ccb *ccb;
1928         int error;
1929
1930         CAM_PERIPH_FOREACH(periph, &adadriver) {
1931                 softc = (struct ada_softc *)periph->softc;
1932                 if (SCHEDULER_STOPPED()) {
1933                         /* If we paniced with the lock held, do not recurse. */
1934                         if (!cam_periph_owned(periph) &&
1935                             (softc->flags & ADA_FLAG_OPEN)) {
1936                                 adadump(softc->disk, NULL, 0, 0, 0);
1937                         }
1938                         continue;
1939                 }
1940                 cam_periph_lock(periph);
1941                 /*
1942                  * We only sync the cache if the drive is still open, and
1943                  * if the drive is capable of it..
1944                  */
1945                 if (((softc->flags & ADA_FLAG_OPEN) == 0) ||
1946                     (softc->flags & ADA_FLAG_CAN_FLUSHCACHE) == 0) {
1947                         cam_periph_unlock(periph);
1948                         continue;
1949                 }
1950
1951                 ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL);
1952                 cam_fill_ataio(&ccb->ataio,
1953                                     0,
1954                                     adadone,
1955                                     CAM_DIR_NONE,
1956                                     0,
1957                                     NULL,
1958                                     0,
1959                                     ada_default_timeout*1000);
1960                 if (softc->flags & ADA_FLAG_CAN_48BIT)
1961                         ata_48bit_cmd(&ccb->ataio, ATA_FLUSHCACHE48, 0, 0, 0);
1962                 else
1963                         ata_28bit_cmd(&ccb->ataio, ATA_FLUSHCACHE, 0, 0, 0);
1964
1965                 error = cam_periph_runccb(ccb, adaerror, /*cam_flags*/0,
1966                     /*sense_flags*/ SF_NO_RECOVERY | SF_NO_RETRY,
1967                     softc->disk->d_devstat);
1968                 if (error != 0)
1969                         xpt_print(periph->path, "Synchronize cache failed\n");
1970                 xpt_release_ccb(ccb);
1971                 cam_periph_unlock(periph);
1972         }
1973 }
1974
1975 static void
1976 adaspindown(uint8_t cmd, int flags)
1977 {
1978         struct cam_periph *periph;
1979         struct ada_softc *softc;
1980         union ccb *ccb;
1981         int error;
1982
1983         CAM_PERIPH_FOREACH(periph, &adadriver) {
1984                 /* If we paniced with lock held - not recurse here. */
1985                 if (cam_periph_owned(periph))
1986                         continue;
1987                 cam_periph_lock(periph);
1988                 softc = (struct ada_softc *)periph->softc;
1989                 /*
1990                  * We only spin-down the drive if it is capable of it..
1991                  */
1992                 if ((softc->flags & ADA_FLAG_CAN_POWERMGT) == 0) {
1993                         cam_periph_unlock(periph);
1994                         continue;
1995                 }
1996
1997                 if (bootverbose)
1998                         xpt_print(periph->path, "spin-down\n");
1999
2000                 ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL);
2001                 cam_fill_ataio(&ccb->ataio,
2002                                     0,
2003                                     adadone,
2004                                     CAM_DIR_NONE | flags,
2005                                     0,
2006                                     NULL,
2007                                     0,
2008                                     ada_default_timeout*1000);
2009                 ata_28bit_cmd(&ccb->ataio, cmd, 0, 0, 0);
2010
2011                 error = cam_periph_runccb(ccb, adaerror, /*cam_flags*/0,
2012                     /*sense_flags*/ SF_NO_RECOVERY | SF_NO_RETRY,
2013                     softc->disk->d_devstat);
2014                 if (error != 0)
2015                         xpt_print(periph->path, "Spin-down disk failed\n");
2016                 xpt_release_ccb(ccb);
2017                 cam_periph_unlock(periph);
2018         }
2019 }
2020
2021 static void
2022 adashutdown(void *arg, int howto)
2023 {
2024
2025         adaflush();
2026         if (ada_spindown_shutdown != 0 &&
2027             (howto & (RB_HALT | RB_POWEROFF)) != 0)
2028                 adaspindown(ATA_STANDBY_IMMEDIATE, 0);
2029 }
2030
2031 static void
2032 adasuspend(void *arg)
2033 {
2034
2035         adaflush();
2036         if (ada_spindown_suspend != 0)
2037                 adaspindown(ATA_SLEEP, CAM_DEV_QFREEZE);
2038 }
2039
2040 static void
2041 adaresume(void *arg)
2042 {
2043         struct cam_periph *periph;
2044         struct ada_softc *softc;
2045
2046         if (ada_spindown_suspend == 0)
2047                 return;
2048
2049         CAM_PERIPH_FOREACH(periph, &adadriver) {
2050                 cam_periph_lock(periph);
2051                 softc = (struct ada_softc *)periph->softc;
2052                 /*
2053                  * We only spin-down the drive if it is capable of it..
2054                  */
2055                 if ((softc->flags & ADA_FLAG_CAN_POWERMGT) == 0) {
2056                         cam_periph_unlock(periph);
2057                         continue;
2058                 }
2059
2060                 if (bootverbose)
2061                         xpt_print(periph->path, "resume\n");
2062
2063                 /*
2064                  * Drop freeze taken due to CAM_DEV_QFREEZE flag set on
2065                  * sleep request.
2066                  */
2067                 cam_release_devq(periph->path,
2068                          /*relsim_flags*/0,
2069                          /*openings*/0,
2070                          /*timeout*/0,
2071                          /*getcount_only*/0);
2072
2073                 cam_periph_unlock(periph);
2074         }
2075 }
2076
2077 #endif /* _KERNEL */