sys/cam/ata/ata_da.c

   1 /*-
   2  * Copyright (c) 2009 Alexander Motin <mav@FreeBSD.org>
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer,
  10  *    without modification, immediately at the beginning of the file.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  18  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  20  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  21  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  22  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  24  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  */
  26
  27 #include <sys/cdefs.h>
  28 __FBSDID("$FreeBSD$");
  29
  30 #include "opt_ada.h"
  31 #include "opt_ata.h"
  32
  33 #include <sys/param.h>
  34
  35 #ifdef _KERNEL
  36 #include <sys/systm.h>
  37 #include <sys/kernel.h>
  38 #include <sys/bio.h>
  39 #include <sys/sysctl.h>
  40 #include <sys/taskqueue.h>
  41 #include <sys/lock.h>
  42 #include <sys/mutex.h>
  43 #include <sys/conf.h>
  44 #include <sys/devicestat.h>
  45 #include <sys/eventhandler.h>
  46 #include <sys/malloc.h>
  47 #include <sys/cons.h>
  48 #include <sys/proc.h>
  49 #include <sys/reboot.h>
  50 #include <geom/geom_disk.h>
  51 #endif /* _KERNEL */
  52
  53 #ifndef _KERNEL
  54 #include <stdio.h>
  55 #include <string.h>
  56 #endif /* _KERNEL */
  57
  58 #include <cam/cam.h>
  59 #include <cam/cam_ccb.h>
  60 #include <cam/cam_periph.h>
  61 #include <cam/cam_xpt_periph.h>
  62 #include <cam/cam_sim.h>
  63
  64 #include <cam/ata/ata_all.h>
  65
  66 #include <machine/md_var.h>     /* geometry translation */
  67
  68 #ifdef _KERNEL
  69
  70 #define ATA_MAX_28BIT_LBA               268435455UL
  71
  72 typedef enum {
  73         ADA_STATE_RAHEAD,
  74         ADA_STATE_WCACHE,
  75         ADA_STATE_NORMAL
  76 } ada_state;
  77
  78 typedef enum {
  79         ADA_FLAG_CAN_48BIT      = 0x0002,
  80         ADA_FLAG_CAN_FLUSHCACHE = 0x0004,
  81         ADA_FLAG_CAN_NCQ        = 0x0008,
  82         ADA_FLAG_CAN_DMA        = 0x0010,
  83         ADA_FLAG_NEED_OTAG      = 0x0020,
  84         ADA_FLAG_WENT_IDLE      = 0x0040,
  85         ADA_FLAG_CAN_TRIM       = 0x0080,
  86         ADA_FLAG_OPEN           = 0x0100,
  87         ADA_FLAG_SCTX_INIT      = 0x0200,
  88         ADA_FLAG_CAN_CFA        = 0x0400,
  89         ADA_FLAG_CAN_POWERMGT   = 0x0800,
  90         ADA_FLAG_CAN_DMA48      = 0x1000,
  91         ADA_FLAG_DIRTY          = 0x2000
  92 } ada_flags;
  93
  94 typedef enum {
  95         ADA_Q_NONE              = 0x00,
  96         ADA_Q_4K                = 0x01,
  97 } ada_quirks;
  98
  99 #define ADA_Q_BIT_STRING        \
 100         "\020"                  \
 101         "\0014K"
 102
 103 typedef enum {
 104         ADA_CCB_RAHEAD          = 0x01,
 105         ADA_CCB_WCACHE          = 0x02,
 106         ADA_CCB_BUFFER_IO       = 0x03,
 107         ADA_CCB_WAITING         = 0x04,
 108         ADA_CCB_DUMP            = 0x05,
 109         ADA_CCB_TRIM            = 0x06,
 110         ADA_CCB_TYPE_MASK       = 0x0F,
 111 } ada_ccb_state;
 112
 113 /* Offsets into our private area for storing information */
 114 #define ccb_state       ppriv_field0
 115 #define ccb_bp          ppriv_ptr1
 116
 117 struct disk_params {
 118         u_int8_t  heads;
 119         u_int8_t  secs_per_track;
 120         u_int32_t cylinders;
 121         u_int32_t secsize;      /* Number of bytes/logical sector */
 122         u_int64_t sectors;      /* Total number sectors */
 123 };
 124
 125 #define TRIM_MAX_BLOCKS 8
 126 #define TRIM_MAX_RANGES (TRIM_MAX_BLOCKS * ATA_DSM_BLK_RANGES)
 127 #define TRIM_MAX_BIOS   (TRIM_MAX_RANGES * 4)
 128 struct trim_request {
 129         uint8_t         data[TRIM_MAX_RANGES * ATA_DSM_RANGE_SIZE];
 130         struct bio      *bps[TRIM_MAX_BIOS];
 131 };
 132
 133 struct ada_softc {
 134         struct   bio_queue_head bio_queue;
 135         struct   bio_queue_head trim_queue;
 136         ada_state state;
 137         ada_flags flags;
 138         ada_quirks quirks;
 139         int      sort_io_queue;
 140         int      ordered_tag_count;
 141         int      outstanding_cmds;
 142         int      trim_max_ranges;
 143         int      trim_running;
 144         int      read_ahead;
 145         int      write_cache;
 146 #ifdef ADA_TEST_FAILURE
 147         int      force_read_error;
 148         int      force_write_error;
 149         int      periodic_read_error;
 150         int      periodic_read_count;
 151 #endif
 152         struct   disk_params params;
 153         struct   disk *disk;
 154         struct task             sysctl_task;
 155         struct sysctl_ctx_list  sysctl_ctx;
 156         struct sysctl_oid       *sysctl_tree;
 157         struct callout          sendordered_c;
 158         struct trim_request     trim_req;
 159 };
 160
 161 struct ada_quirk_entry {
 162         struct scsi_inquiry_pattern inq_pat;
 163         ada_quirks quirks;
 164 };
 165
 166 static struct ada_quirk_entry ada_quirk_table[] =
 167 {
 168         {
 169                 /* Hitachi Advanced Format (4k) drives */
 170                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Hitachi H??????????E3*", "*" },
 171                 /*quirks*/ADA_Q_4K
 172         },
 173         {
 174                 /* Samsung Advanced Format (4k) drives */
 175                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG HD155UI*", "*" },
 176                 /*quirks*/ADA_Q_4K
 177         },
 178         {
 179                 /* Samsung Advanced Format (4k) drives */
 180                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG HD204UI*", "*" },
 181                 /*quirks*/ADA_Q_4K
 182         },
 183         {
 184                 /* Seagate Barracuda Green Advanced Format (4k) drives */
 185                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST????DL*", "*" },
 186                 /*quirks*/ADA_Q_4K
 187         },
 188         {
 189                 /* Seagate Barracuda Advanced Format (4k) drives */
 190                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST???DM*", "*" },
 191                 /*quirks*/ADA_Q_4K
 192         },
 193         {
 194                 /* Seagate Barracuda Advanced Format (4k) drives */
 195                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST????DM*", "*" },
 196                 /*quirks*/ADA_Q_4K
 197         },
 198         {
 199                 /* Seagate Momentus Advanced Format (4k) drives */
 200                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9500423AS*", "*" },
 201                 /*quirks*/ADA_Q_4K
 202         },
 203         {
 204                 /* Seagate Momentus Advanced Format (4k) drives */
 205                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9500424AS*", "*" },
 206                 /*quirks*/ADA_Q_4K
 207         },
 208         {
 209                 /* Seagate Momentus Advanced Format (4k) drives */
 210                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9640423AS*", "*" },
 211                 /*quirks*/ADA_Q_4K
 212         },
 213         {
 214                 /* Seagate Momentus Advanced Format (4k) drives */
 215                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9640424AS*", "*" },
 216                 /*quirks*/ADA_Q_4K
 217         },
 218         {
 219                 /* Seagate Momentus Advanced Format (4k) drives */
 220                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9750420AS*", "*" },
 221                 /*quirks*/ADA_Q_4K
 222         },
 223         {
 224                 /* Seagate Momentus Advanced Format (4k) drives */
 225                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9750422AS*", "*" },
 226                 /*quirks*/ADA_Q_4K
 227         },
 228         {
 229                 /* Seagate Momentus Advanced Format (4k) drives */
 230                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9750423AS*", "*" },
 231                 /*quirks*/ADA_Q_4K
 232         },
 233         {
 234                 /* Seagate Momentus Thin Advanced Format (4k) drives */
 235                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST???LT*", "*" },
 236                 /*quirks*/ADA_Q_4K
 237         },
 238         {
 239                 /* WDC Caviar Green Advanced Format (4k) drives */
 240                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD????RS*", "*" },
 241                 /*quirks*/ADA_Q_4K
 242         },
 243         {
 244                 /* WDC Caviar Green Advanced Format (4k) drives */
 245                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD????RX*", "*" },
 246                 /*quirks*/ADA_Q_4K
 247         },
 248         {
 249                 /* WDC Caviar Green Advanced Format (4k) drives */
 250                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD??????RS*", "*" },
 251                 /*quirks*/ADA_Q_4K
 252         },
 253         {
 254                 /* WDC Caviar Green Advanced Format (4k) drives */
 255                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD??????RX*", "*" },
 256                 /*quirks*/ADA_Q_4K
 257         },
 258         {
 259                 /* WDC Scorpio Black Advanced Format (4k) drives */
 260                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD???PKT*", "*" },
 261                 /*quirks*/ADA_Q_4K
 262         },
 263         {
 264                 /* WDC Scorpio Black Advanced Format (4k) drives */
 265                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD?????PKT*", "*" },
 266                 /*quirks*/ADA_Q_4K
 267         },
 268         {
 269                 /* WDC Scorpio Blue Advanced Format (4k) drives */
 270                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD???PVT*", "*" },
 271                 /*quirks*/ADA_Q_4K
 272         },
 273         {
 274                 /* WDC Scorpio Blue Advanced Format (4k) drives */
 275                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD?????PVT*", "*" },
 276                 /*quirks*/ADA_Q_4K
 277         },
 278         /* SSDs */
 279         {
 280                 /*
 281                  * Corsair Force 2 SSDs
 282                  * 4k optimised & trim only works in 4k requests + 4k aligned
 283                  */
 284                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair CSSD-F*", "*" },
 285                 /*quirks*/ADA_Q_4K
 286         },
 287         {
 288                 /*
 289                  * Corsair Force 3 SSDs
 290                  * 4k optimised & trim only works in 4k requests + 4k aligned
 291                  */
 292                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair Force 3*", "*" },
 293                 /*quirks*/ADA_Q_4K
 294         },
 295         {
 296                 /*
 297                  * Corsair Force GT SSDs
 298                  * 4k optimised & trim only works in 4k requests + 4k aligned
 299                  */
 300                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair Force GT*", "*" },
 301                 /*quirks*/ADA_Q_4K
 302         },
 303         {
 304                 /*
 305                  * Crucial M4 SSDs
 306                  * 4k optimised & trim only works in 4k requests + 4k aligned
 307                  */
 308                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "M4-CT???M4SSD2*", "*" },
 309                 /*quirks*/ADA_Q_4K
 310         },
 311         {
 312                 /*
 313                  * Crucial RealSSD C300 SSDs
 314                  * 4k optimised
 315                  */
 316                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "C300-CTFDDAC???MAG*",
 317                 "*" }, /*quirks*/ADA_Q_4K
 318         },
 319         {
 320                 /*
 321                  * Intel 320 Series SSDs
 322                  * 4k optimised & trim only works in 4k requests + 4k aligned
 323                  */
 324                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSA2CW*", "*" },
 325                 /*quirks*/ADA_Q_4K
 326         },
 327         {
 328                 /*
 329                  * Intel 330 Series SSDs
 330                  * 4k optimised & trim only works in 4k requests + 4k aligned
 331                  */
 332                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSC2CT*", "*" },
 333                 /*quirks*/ADA_Q_4K
 334         },
 335         {
 336                 /*
 337                  * Intel 510 Series SSDs
 338                  * 4k optimised & trim only works in 4k requests + 4k aligned
 339                  */
 340                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSC2MH*", "*" },
 341                 /*quirks*/ADA_Q_4K
 342         },
 343         {
 344                 /*
 345                  * Intel 520 Series SSDs
 346                  * 4k optimised & trim only works in 4k requests + 4k aligned
 347                  */
 348                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSC2BW*", "*" },
 349                 /*quirks*/ADA_Q_4K
 350         },
 351         {
 352                 /*
 353                  * Intel X25-M Series SSDs
 354                  * 4k optimised & trim only works in 4k requests + 4k aligned
 355                  */
 356                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSA2M*", "*" },
 357                 /*quirks*/ADA_Q_4K
 358         },
 359         {
 360                 /*
 361                  * Kingston E100 Series SSDs
 362                  * 4k optimised & trim only works in 4k requests + 4k aligned
 363                  */
 364                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "KINGSTON SE100S3*", "*" },
 365                 /*quirks*/ADA_Q_4K
 366         },
 367         {
 368                 /*
 369                  * Kingston HyperX 3k SSDs
 370                  * 4k optimised & trim only works in 4k requests + 4k aligned
 371                  */
 372                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "KINGSTON SH103S3*", "*" },
 373                 /*quirks*/ADA_Q_4K
 374         },
 375         {
 376                 /*
 377                  * Marvell SSDs (entry taken from OpenSolaris)
 378                  * 4k optimised & trim only works in 4k requests + 4k aligned
 379                  */
 380                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "MARVELL SD88SA02*", "*" },
 381                 /*quirks*/ADA_Q_4K
 382         },
 383         {
 384                 /*
 385                  * OCZ Agility 2 SSDs
 386                  * 4k optimised & trim only works in 4k requests + 4k aligned
 387                  */
 388                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-AGILITY2*", "*" },
 389                 /*quirks*/ADA_Q_4K
 390         },
 391         {
 392                 /*
 393                  * OCZ Agility 3 SSDs
 394                  * 4k optimised & trim only works in 4k requests + 4k aligned
 395                  */
 396                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-AGILITY3*", "*" },
 397                 /*quirks*/ADA_Q_4K
 398         },
 399         {
 400                 /*
 401                  * OCZ Deneva R Series SSDs
 402                  * 4k optimised & trim only works in 4k requests + 4k aligned
 403                  */
 404                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "DENRSTE251M45*", "*" },
 405                 /*quirks*/ADA_Q_4K
 406         },
 407         {
 408                 /*
 409                  * OCZ Vertex 2 SSDs (inc pro series)
 410                  * 4k optimised & trim only works in 4k requests + 4k aligned
 411                  */
 412                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ?VERTEX2*", "*" },
 413                 /*quirks*/ADA_Q_4K
 414         },
 415         {
 416                 /*
 417                  * OCZ Vertex 3 SSDs
 418                  * 4k optimised & trim only works in 4k requests + 4k aligned
 419                  */
 420                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-VERTEX3*", "*" },
 421                 /*quirks*/ADA_Q_4K
 422         },
 423         {
 424                 /*
 425                  * OCZ Vertex 4 SSDs
 426                  * 4k optimised & trim only works in 4k requests + 4k aligned
 427                  */
 428                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-VERTEX4*", "*" },
 429                 /*quirks*/ADA_Q_4K
 430         },
 431         {
 432                 /*
 433                  * Samsung 830 Series SSDs
 434                  * 4k optimised
 435                  */
 436                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG SSD 830 Series*", "*" },
 437                 /*quirks*/ADA_Q_4K
 438         },
 439         {
 440                 /*
 441                  * SuperTalent TeraDrive CT SSDs
 442                  * 4k optimised & trim only works in 4k requests + 4k aligned
 443                  */
 444                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "FTM??CT25H*", "*" },
 445                 /*quirks*/ADA_Q_4K
 446         },
 447         {
 448                 /*
 449                  * XceedIOPS SATA SSDs
 450                  * 4k optimised
 451                  */
 452                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SG9XCS2D*", "*" },
 453                 /*quirks*/ADA_Q_4K
 454         },
 455         {
 456                 /* Default */
 457                 {
 458                   T_ANY, SIP_MEDIA_REMOVABLE|SIP_MEDIA_FIXED,
 459                   /*vendor*/"*", /*product*/"*", /*revision*/"*"
 460                 },
 461                 /*quirks*/0
 462         },
 463 };
 464
 465 static  disk_strategy_t adastrategy;
 466 static  dumper_t        adadump;
 467 static  periph_init_t   adainit;
 468 static  void            adaasync(void *callback_arg, u_int32_t code,
 469                                 struct cam_path *path, void *arg);
 470 static  void            adasysctlinit(void *context, int pending);
 471 static  periph_ctor_t   adaregister;
 472 static  periph_dtor_t   adacleanup;
 473 static  periph_start_t  adastart;
 474 static  periph_oninv_t  adaoninvalidate;
 475 static  void            adadone(struct cam_periph *periph,
 476                                union ccb *done_ccb);
 477 static  int             adaerror(union ccb *ccb, u_int32_t cam_flags,
 478                                 u_int32_t sense_flags);
 479 static void             adagetparams(struct cam_periph *periph,
 480                                 struct ccb_getdev *cgd);
 481 static timeout_t        adasendorderedtag;
 482 static void             adashutdown(void *arg, int howto);
 483 static void             adasuspend(void *arg);
 484 static void             adaresume(void *arg);
 485
 486 #ifndef ADA_DEFAULT_LEGACY_ALIASES
 487 #ifdef ATA_CAM
 488 #define ADA_DEFAULT_LEGACY_ALIASES      1
 489 #else
 490 #define ADA_DEFAULT_LEGACY_ALIASES      0
 491 #endif
 492 #endif
 493
 494 #ifndef ADA_DEFAULT_TIMEOUT
 495 #define ADA_DEFAULT_TIMEOUT 30  /* Timeout in seconds */
 496 #endif
 497
 498 #ifndef ADA_DEFAULT_RETRY
 499 #define ADA_DEFAULT_RETRY       4
 500 #endif
 501
 502 #ifndef ADA_DEFAULT_SEND_ORDERED
 503 #define ADA_DEFAULT_SEND_ORDERED        1
 504 #endif
 505
 506 #ifndef ADA_DEFAULT_SPINDOWN_SHUTDOWN
 507 #define ADA_DEFAULT_SPINDOWN_SHUTDOWN   1
 508 #endif
 509
 510 #ifndef ADA_DEFAULT_SPINDOWN_SUSPEND
 511 #define ADA_DEFAULT_SPINDOWN_SUSPEND    1
 512 #endif
 513
 514 #ifndef ADA_DEFAULT_READ_AHEAD
 515 #define ADA_DEFAULT_READ_AHEAD  1
 516 #endif
 517
 518 #ifndef ADA_DEFAULT_WRITE_CACHE
 519 #define ADA_DEFAULT_WRITE_CACHE 1
 520 #endif
 521
 522 #define ADA_RA  (softc->read_ahead >= 0 ? \
 523                  softc->read_ahead : ada_read_ahead)
 524 #define ADA_WC  (softc->write_cache >= 0 ? \
 525                  softc->write_cache : ada_write_cache)
 526 #define ADA_SIO (softc->sort_io_queue >= 0 ? \
 527                  softc->sort_io_queue : cam_sort_io_queues)
 528
 529 /*
 530  * Most platforms map firmware geometry to actual, but some don't.  If
 531  * not overridden, default to nothing.
 532  */
 533 #ifndef ata_disk_firmware_geom_adjust
 534 #define ata_disk_firmware_geom_adjust(disk)
 535 #endif
 536
 537 static int ada_legacy_aliases = ADA_DEFAULT_LEGACY_ALIASES;
 538 static int ada_retry_count = ADA_DEFAULT_RETRY;
 539 static int ada_default_timeout = ADA_DEFAULT_TIMEOUT;
 540 static int ada_send_ordered = ADA_DEFAULT_SEND_ORDERED;
 541 static int ada_spindown_shutdown = ADA_DEFAULT_SPINDOWN_SHUTDOWN;
 542 static int ada_spindown_suspend = ADA_DEFAULT_SPINDOWN_SUSPEND;
 543 static int ada_read_ahead = ADA_DEFAULT_READ_AHEAD;
 544 static int ada_write_cache = ADA_DEFAULT_WRITE_CACHE;
 545
 546 static SYSCTL_NODE(_kern_cam, OID_AUTO, ada, CTLFLAG_RD, 0,
 547             "CAM Direct Access Disk driver");
 548 SYSCTL_INT(_kern_cam_ada, OID_AUTO, legacy_aliases, CTLFLAG_RW,
 549            &ada_legacy_aliases, 0, "Create legacy-like device aliases");
 550 TUNABLE_INT("kern.cam.ada.legacy_aliases", &ada_legacy_aliases);
 551 SYSCTL_INT(_kern_cam_ada, OID_AUTO, retry_count, CTLFLAG_RW,
 552            &ada_retry_count, 0, "Normal I/O retry count");
 553 TUNABLE_INT("kern.cam.ada.retry_count", &ada_retry_count);
 554 SYSCTL_INT(_kern_cam_ada, OID_AUTO, default_timeout, CTLFLAG_RW,
 555            &ada_default_timeout, 0, "Normal I/O timeout (in seconds)");
 556 TUNABLE_INT("kern.cam.ada.default_timeout", &ada_default_timeout);
 557 SYSCTL_INT(_kern_cam_ada, OID_AUTO, send_ordered, CTLFLAG_RW,
 558            &ada_send_ordered, 0, "Send Ordered Tags");
 559 TUNABLE_INT("kern.cam.ada.send_ordered", &ada_send_ordered);
 560 SYSCTL_INT(_kern_cam_ada, OID_AUTO, spindown_shutdown, CTLFLAG_RW,
 561            &ada_spindown_shutdown, 0, "Spin down upon shutdown");
 562 TUNABLE_INT("kern.cam.ada.spindown_shutdown", &ada_spindown_shutdown);
 563 SYSCTL_INT(_kern_cam_ada, OID_AUTO, spindown_suspend, CTLFLAG_RW,
 564            &ada_spindown_suspend, 0, "Spin down upon suspend");
 565 TUNABLE_INT("kern.cam.ada.spindown_suspend", &ada_spindown_suspend);
 566 SYSCTL_INT(_kern_cam_ada, OID_AUTO, read_ahead, CTLFLAG_RW,
 567            &ada_read_ahead, 0, "Enable disk read-ahead");
 568 TUNABLE_INT("kern.cam.ada.read_ahead", &ada_read_ahead);
 569 SYSCTL_INT(_kern_cam_ada, OID_AUTO, write_cache, CTLFLAG_RW,
 570            &ada_write_cache, 0, "Enable disk write cache");
 571 TUNABLE_INT("kern.cam.ada.write_cache", &ada_write_cache);
 572
 573 /*
 574  * ADA_ORDEREDTAG_INTERVAL determines how often, relative
 575  * to the default timeout, we check to see whether an ordered
 576  * tagged transaction is appropriate to prevent simple tag
 577  * starvation.  Since we'd like to ensure that there is at least
 578  * 1/2 of the timeout length left for a starved transaction to
 579  * complete after we've sent an ordered tag, we must poll at least
 580  * four times in every timeout period.  This takes care of the worst
 581  * case where a starved transaction starts during an interval that
 582  * meets the requirement "don't send an ordered tag" test so it takes
 583  * us two intervals to determine that a tag must be sent.
 584  */
 585 #ifndef ADA_ORDEREDTAG_INTERVAL
 586 #define ADA_ORDEREDTAG_INTERVAL 4
 587 #endif
 588
 589 static struct periph_driver adadriver =
 590 {
 591         adainit, "ada",
 592         TAILQ_HEAD_INITIALIZER(adadriver.units), /* generation */ 0
 593 };
 594
 595 PERIPHDRIVER_DECLARE(ada, adadriver);
 596
 597 static MALLOC_DEFINE(M_ATADA, "ata_da", "ata_da buffers");
 598
 599 static int
 600 adaopen(struct disk *dp)
 601 {
 602         struct cam_periph *periph;
 603         struct ada_softc *softc;
 604         int error;
 605
 606         periph = (struct cam_periph *)dp->d_drv1;
 607         if (cam_periph_acquire(periph) != CAM_REQ_CMP) {
 608                 return(ENXIO);
 609         }
 610
 611         cam_periph_lock(periph);
 612         if ((error = cam_periph_hold(periph, PRIBIO|PCATCH)) != 0) {
 613                 cam_periph_unlock(periph);
 614                 cam_periph_release(periph);
 615                 return (error);
 616         }
 617
 618         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE | CAM_DEBUG_PERIPH,
 619             ("adaopen\n"));
 620
 621         softc = (struct ada_softc *)periph->softc;
 622         softc->flags |= ADA_FLAG_OPEN;
 623
 624         cam_periph_unhold(periph);
 625         cam_periph_unlock(periph);
 626         return (0);
 627 }
 628
 629 static int
 630 adaclose(struct disk *dp)
 631 {
 632         struct  cam_periph *periph;
 633         struct  ada_softc *softc;
 634         union ccb *ccb;
 635         int error;
 636
 637         periph = (struct cam_periph *)dp->d_drv1;
 638         cam_periph_lock(periph);
 639         if (cam_periph_hold(periph, PRIBIO) != 0) {
 640                 cam_periph_unlock(periph);
 641                 cam_periph_release(periph);
 642                 return (0);
 643         }
 644
 645         softc = (struct ada_softc *)periph->softc;
 646
 647         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE | CAM_DEBUG_PERIPH,
 648             ("adaclose\n"));
 649
 650         /* We only sync the cache if the drive is capable of it. */
 651         if ((softc->flags & ADA_FLAG_DIRTY) != 0 &&
 652             (softc->flags & ADA_FLAG_CAN_FLUSHCACHE) != 0 &&
 653             (periph->flags & CAM_PERIPH_INVALID) == 0) {
 654
 655                 ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL);
 656                 cam_fill_ataio(&ccb->ataio,
 657                                     1,
 658                                     adadone,
 659                                     CAM_DIR_NONE,
 660                                     0,
 661                                     NULL,
 662                                     0,
 663                                     ada_default_timeout*1000);
 664
 665                 if (softc->flags & ADA_FLAG_CAN_48BIT)
 666                         ata_48bit_cmd(&ccb->ataio, ATA_FLUSHCACHE48, 0, 0, 0);
 667                 else
 668                         ata_28bit_cmd(&ccb->ataio, ATA_FLUSHCACHE, 0, 0, 0);
 669                 error = cam_periph_runccb(ccb, adaerror, /*cam_flags*/0,
 670                     /*sense_flags*/0, softc->disk->d_devstat);
 671
 672                 if (error != 0)
 673                         xpt_print(periph->path, "Synchronize cache failed\n");
 674                 else
 675                         softc->flags &= ~ADA_FLAG_DIRTY;
 676                 xpt_release_ccb(ccb);
 677         }
 678
 679         softc->flags &= ~ADA_FLAG_OPEN;
 680         cam_periph_unhold(periph);
 681         cam_periph_unlock(periph);
 682         cam_periph_release(periph);
 683         return (0);
 684 }
 685
 686 static void
 687 adaschedule(struct cam_periph *periph)
 688 {
 689         struct ada_softc *softc = (struct ada_softc *)periph->softc;
 690         uint32_t prio;
 691
 692         if (softc->state != ADA_STATE_NORMAL)
 693                 return;
 694
 695         /* Check if cam_periph_getccb() was called. */
 696         prio = periph->immediate_priority;
 697
 698         /* Check if we have more work to do. */
 699         if (bioq_first(&softc->bio_queue) ||
 700             (!softc->trim_running && bioq_first(&softc->trim_queue))) {
 701                 prio = CAM_PRIORITY_NORMAL;
 702         }
 703
 704         /* Schedule CCB if any of above is true. */
 705         if (prio != CAM_PRIORITY_NONE)
 706                 xpt_schedule(periph, prio);
 707 }
 708
 709 /*
 710  * Actually translate the requested transfer into one the physical driver
 711  * can understand.  The transfer is described by a buf and will include
 712  * only one physical transfer.
 713  */
 714 static void
 715 adastrategy(struct bio *bp)
 716 {
 717         struct cam_periph *periph;
 718         struct ada_softc *softc;
 719
 720         periph = (struct cam_periph *)bp->bio_disk->d_drv1;
 721         softc = (struct ada_softc *)periph->softc;
 722
 723         cam_periph_lock(periph);
 724
 725         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("adastrategy(%p)\n", bp));
 726
 727         /*
 728          * If the device has been made invalid, error out
 729          */
 730         if ((periph->flags & CAM_PERIPH_INVALID) != 0) {
 731                 cam_periph_unlock(periph);
 732                 biofinish(bp, NULL, ENXIO);
 733                 return;
 734         }
 735
 736         /*
 737          * Place it in the queue of disk activities for this disk
 738          */
 739         if (bp->bio_cmd == BIO_DELETE &&
 740             (softc->flags & ADA_FLAG_CAN_TRIM)) {
 741                 if (ADA_SIO)
 742                     bioq_disksort(&softc->trim_queue, bp);
 743                 else
 744                     bioq_insert_tail(&softc->trim_queue, bp);
 745         } else {
 746                 if (ADA_SIO)
 747                     bioq_disksort(&softc->bio_queue, bp);
 748                 else
 749                     bioq_insert_tail(&softc->bio_queue, bp);
 750         }
 751
 752         /*
 753          * Schedule ourselves for performing the work.
 754          */
 755         adaschedule(periph);
 756         cam_periph_unlock(periph);
 757
 758         return;
 759 }
 760
 761 static int
 762 adadump(void *arg, void *virtual, vm_offset_t physical, off_t offset, size_t length)
 763 {
 764         struct      cam_periph *periph;
 765         struct      ada_softc *softc;
 766         u_int       secsize;
 767         union       ccb ccb;
 768         struct      disk *dp;
 769         uint64_t    lba;
 770         uint16_t    count;
 771         int         error = 0;
 772
 773         dp = arg;
 774         periph = dp->d_drv1;
 775         softc = (struct ada_softc *)periph->softc;
 776         cam_periph_lock(periph);
 777         secsize = softc->params.secsize;
 778         lba = offset / secsize;
 779         count = length / secsize;
 780
 781         if ((periph->flags & CAM_PERIPH_INVALID) != 0) {
 782                 cam_periph_unlock(periph);
 783                 return (ENXIO);
 784         }
 785
 786         if (length > 0) {
 787                 xpt_setup_ccb(&ccb.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
 788                 ccb.ccb_h.ccb_state = ADA_CCB_DUMP;
 789                 cam_fill_ataio(&ccb.ataio,
 790                     0,
 791                     adadone,
 792                     CAM_DIR_OUT,
 793                     0,
 794                     (u_int8_t *) virtual,
 795                     length,
 796                     ada_default_timeout*1000);
 797                 if ((softc->flags & ADA_FLAG_CAN_48BIT) &&
 798                     (lba + count >= ATA_MAX_28BIT_LBA ||
 799                     count >= 256)) {
 800                         ata_48bit_cmd(&ccb.ataio, ATA_WRITE_DMA48,
 801                             0, lba, count);
 802                 } else {
 803                         ata_28bit_cmd(&ccb.ataio, ATA_WRITE_DMA,
 804                             0, lba, count);
 805                 }
 806                 xpt_polled_action(&ccb);
 807
 808                 error = cam_periph_error(&ccb,
 809                     0, SF_NO_RECOVERY | SF_NO_RETRY, NULL);
 810                 if ((ccb.ccb_h.status & CAM_DEV_QFRZN) != 0)
 811                         cam_release_devq(ccb.ccb_h.path, /*relsim_flags*/0,
 812                             /*reduction*/0, /*timeout*/0, /*getcount_only*/0);
 813                 if (error != 0)
 814                         printf("Aborting dump due to I/O error.\n");
 815
 816                 cam_periph_unlock(periph);
 817                 return (error);
 818         }
 819
 820         if (softc->flags & ADA_FLAG_CAN_FLUSHCACHE) {
 821                 xpt_setup_ccb(&ccb.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
 822
 823                 ccb.ccb_h.ccb_state = ADA_CCB_DUMP;
 824                 cam_fill_ataio(&ccb.ataio,
 825                                     0,
 826                                     adadone,
 827                                     CAM_DIR_NONE,
 828                                     0,
 829                                     NULL,
 830                                     0,
 831                                     ada_default_timeout*1000);
 832
 833                 if (softc->flags & ADA_FLAG_CAN_48BIT)
 834                         ata_48bit_cmd(&ccb.ataio, ATA_FLUSHCACHE48, 0, 0, 0);
 835                 else
 836                         ata_28bit_cmd(&ccb.ataio, ATA_FLUSHCACHE, 0, 0, 0);
 837                 xpt_polled_action(&ccb);
 838
 839                 error = cam_periph_error(&ccb,
 840                     0, SF_NO_RECOVERY | SF_NO_RETRY, NULL);
 841                 if ((ccb.ccb_h.status & CAM_DEV_QFRZN) != 0)
 842                         cam_release_devq(ccb.ccb_h.path, /*relsim_flags*/0,
 843                             /*reduction*/0, /*timeout*/0, /*getcount_only*/0);
 844                 if (error != 0)
 845                         xpt_print(periph->path, "Synchronize cache failed\n");
 846         }
 847         cam_periph_unlock(periph);
 848         return (error);
 849 }
 850
 851 static void
 852 adainit(void)
 853 {
 854         cam_status status;
 855
 856         /*
 857          * Install a global async callback.  This callback will
 858          * receive async callbacks like "new device found".
 859          */
 860         status = xpt_register_async(AC_FOUND_DEVICE, adaasync, NULL, NULL);
 861
 862         if (status != CAM_REQ_CMP) {
 863                 printf("ada: Failed to attach master async callback "
 864                        "due to status 0x%x!\n", status);
 865         } else if (ada_send_ordered) {
 866
 867                 /* Register our event handlers */
 868                 if ((EVENTHANDLER_REGISTER(power_suspend, adasuspend,
 869                                            NULL, EVENTHANDLER_PRI_LAST)) == NULL)
 870                     printf("adainit: power event registration failed!\n");
 871                 if ((EVENTHANDLER_REGISTER(power_resume, adaresume,
 872                                            NULL, EVENTHANDLER_PRI_LAST)) == NULL)
 873                     printf("adainit: power event registration failed!\n");
 874                 if ((EVENTHANDLER_REGISTER(shutdown_post_sync, adashutdown,
 875                                            NULL, SHUTDOWN_PRI_DEFAULT)) == NULL)
 876                     printf("adainit: shutdown event registration failed!\n");
 877         }
 878 }
 879
 880 /*
 881  * Callback from GEOM, called when it has finished cleaning up its
 882  * resources.
 883  */
 884 static void
 885 adadiskgonecb(struct disk *dp)
 886 {
 887         struct cam_periph *periph;
 888
 889         periph = (struct cam_periph *)dp->d_drv1;
 890
 891         cam_periph_release(periph);
 892 }
 893
 894 static void
 895 adaoninvalidate(struct cam_periph *periph)
 896 {
 897         struct ada_softc *softc;
 898
 899         softc = (struct ada_softc *)periph->softc;
 900
 901         /*
 902          * De-register any async callbacks.
 903          */
 904         xpt_register_async(0, adaasync, periph, periph->path);
 905
 906         /*
 907          * Return all queued I/O with ENXIO.
 908          * XXX Handle any transactions queued to the card
 909          *     with XPT_ABORT_CCB.
 910          */
 911         bioq_flush(&softc->bio_queue, NULL, ENXIO);
 912         bioq_flush(&softc->trim_queue, NULL, ENXIO);
 913
 914         disk_gone(softc->disk);
 915         xpt_print(periph->path, "lost device\n");
 916 }
 917
 918 static void
 919 adacleanup(struct cam_periph *periph)
 920 {
 921         struct ada_softc *softc;
 922
 923         softc = (struct ada_softc *)periph->softc;
 924
 925         xpt_print(periph->path, "removing device entry\n");
 926         cam_periph_unlock(periph);
 927
 928         /*
 929          * If we can't free the sysctl tree, oh well...
 930          */
 931         if ((softc->flags & ADA_FLAG_SCTX_INIT) != 0
 932             && sysctl_ctx_free(&softc->sysctl_ctx) != 0) {
 933                 xpt_print(periph->path, "can't remove sysctl context\n");
 934         }
 935
 936         disk_destroy(softc->disk);
 937         callout_drain(&softc->sendordered_c);
 938         free(softc, M_DEVBUF);
 939         cam_periph_lock(periph);
 940 }
 941
 942 static void
 943 adaasync(void *callback_arg, u_int32_t code,
 944         struct cam_path *path, void *arg)
 945 {
 946         struct ccb_getdev cgd;
 947         struct cam_periph *periph;
 948         struct ada_softc *softc;
 949
 950         periph = (struct cam_periph *)callback_arg;
 951         switch (code) {
 952         case AC_FOUND_DEVICE:
 953         {
 954                 struct ccb_getdev *cgd;
 955                 cam_status status;
 956
 957                 cgd = (struct ccb_getdev *)arg;
 958                 if (cgd == NULL)
 959                         break;
 960
 961                 if (cgd->protocol != PROTO_ATA)
 962                         break;
 963
 964                 /*
 965                  * Allocate a peripheral instance for
 966                  * this device and start the probe
 967                  * process.
 968                  */
 969                 status = cam_periph_alloc(adaregister, adaoninvalidate,
 970                                           adacleanup, adastart,
 971                                           "ada", CAM_PERIPH_BIO,
 972                                           cgd->ccb_h.path, adaasync,
 973                                           AC_FOUND_DEVICE, cgd);
 974
 975                 if (status != CAM_REQ_CMP
 976                  && status != CAM_REQ_INPROG)
 977                         printf("adaasync: Unable to attach to new device "
 978                                 "due to status 0x%x\n", status);
 979                 break;
 980         }
 981         case AC_GETDEV_CHANGED:
 982         {
 983                 softc = (struct ada_softc *)periph->softc;
 984                 xpt_setup_ccb(&cgd.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
 985                 cgd.ccb_h.func_code = XPT_GDEV_TYPE;
 986                 xpt_action((union ccb *)&cgd);
 987
 988                 if ((cgd.ident_data.capabilities1 & ATA_SUPPORT_DMA) &&
 989                     (cgd.inq_flags & SID_DMA))
 990                         softc->flags |= ADA_FLAG_CAN_DMA;
 991                 else
 992                         softc->flags &= ~ADA_FLAG_CAN_DMA;
 993                 if (cgd.ident_data.support.command2 & ATA_SUPPORT_ADDRESS48) {
 994                         softc->flags |= ADA_FLAG_CAN_48BIT;
 995                         if (cgd.inq_flags & SID_DMA48)
 996                                 softc->flags |= ADA_FLAG_CAN_DMA48;
 997                         else
 998                                 softc->flags &= ~ADA_FLAG_CAN_DMA48;
 999                 } else
1000                         softc->flags &= ~(ADA_FLAG_CAN_48BIT |
1001                             ADA_FLAG_CAN_DMA48);
1002                 if ((cgd.ident_data.satacapabilities & ATA_SUPPORT_NCQ) &&
1003                     (cgd.inq_flags & SID_DMA) && (cgd.inq_flags & SID_CmdQue))
1004                         softc->flags |= ADA_FLAG_CAN_NCQ;
1005                 else
1006                         softc->flags &= ~ADA_FLAG_CAN_NCQ;
1007                 if ((cgd.ident_data.support_dsm & ATA_SUPPORT_DSM_TRIM) &&
1008                     (cgd.inq_flags & SID_DMA))
1009                         softc->flags |= ADA_FLAG_CAN_TRIM;
1010                 else
1011                         softc->flags &= ~ADA_FLAG_CAN_TRIM;
1012
1013                 cam_periph_async(periph, code, path, arg);
1014                 break;
1015         }
1016         case AC_ADVINFO_CHANGED:
1017         {
1018                 uintptr_t buftype;
1019
1020                 buftype = (uintptr_t)arg;
1021                 if (buftype == CDAI_TYPE_PHYS_PATH) {
1022                         struct ada_softc *softc;
1023
1024                         softc = periph->softc;
1025                         disk_attr_changed(softc->disk, "GEOM::physpath",
1026                                           M_NOWAIT);
1027                 }
1028                 break;
1029         }
1030         case AC_SENT_BDR:
1031         case AC_BUS_RESET:
1032         {
1033                 softc = (struct ada_softc *)periph->softc;
1034                 cam_periph_async(periph, code, path, arg);
1035                 if (softc->state != ADA_STATE_NORMAL)
1036                         break;
1037                 xpt_setup_ccb(&cgd.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
1038                 cgd.ccb_h.func_code = XPT_GDEV_TYPE;
1039                 xpt_action((union ccb *)&cgd);
1040                 if (ADA_RA >= 0 &&
1041                     cgd.ident_data.support.command1 & ATA_SUPPORT_LOOKAHEAD)
1042                         softc->state = ADA_STATE_RAHEAD;
1043                 else if (ADA_WC >= 0 &&
1044                     cgd.ident_data.support.command1 & ATA_SUPPORT_WRITECACHE)
1045                         softc->state = ADA_STATE_WCACHE;
1046                 else
1047                     break;
1048                 cam_periph_acquire(periph);
1049                 xpt_schedule(periph, CAM_PRIORITY_DEV);
1050         }
1051         default:
1052                 cam_periph_async(periph, code, path, arg);
1053                 break;
1054         }
1055 }
1056
1057 static void
1058 adasysctlinit(void *context, int pending)
1059 {
1060         struct cam_periph *periph;
1061         struct ada_softc *softc;
1062         char tmpstr[80], tmpstr2[80];
1063
1064         periph = (struct cam_periph *)context;
1065
1066         /* periph was held for us when this task was enqueued */
1067         if ((periph->flags & CAM_PERIPH_INVALID) != 0) {
1068                 cam_periph_release(periph);
1069                 return;
1070         }
1071
1072         softc = (struct ada_softc *)periph->softc;
1073         snprintf(tmpstr, sizeof(tmpstr), "CAM ADA unit %d", periph->unit_number);
1074         snprintf(tmpstr2, sizeof(tmpstr2), "%d", periph->unit_number);
1075
1076         sysctl_ctx_init(&softc->sysctl_ctx);
1077         softc->flags |= ADA_FLAG_SCTX_INIT;
1078         softc->sysctl_tree = SYSCTL_ADD_NODE(&softc->sysctl_ctx,
1079                 SYSCTL_STATIC_CHILDREN(_kern_cam_ada), OID_AUTO, tmpstr2,
1080                 CTLFLAG_RD, 0, tmpstr);
1081         if (softc->sysctl_tree == NULL) {
1082                 printf("adasysctlinit: unable to allocate sysctl tree\n");
1083                 cam_periph_release(periph);
1084                 return;
1085         }
1086
1087         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1088                 OID_AUTO, "read_ahead", CTLFLAG_RW | CTLFLAG_MPSAFE,
1089                 &softc->read_ahead, 0, "Enable disk read ahead.");
1090         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1091                 OID_AUTO, "write_cache", CTLFLAG_RW | CTLFLAG_MPSAFE,
1092                 &softc->write_cache, 0, "Enable disk write cache.");
1093         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1094                 OID_AUTO, "sort_io_queue", CTLFLAG_RW | CTLFLAG_MPSAFE,
1095                 &softc->sort_io_queue, 0,
1096                 "Sort IO queue to try and optimise disk access patterns");
1097 #ifdef ADA_TEST_FAILURE
1098         /*
1099          * Add a 'door bell' sysctl which allows one to set it from userland
1100          * and cause something bad to happen.  For the moment, we only allow
1101          * whacking the next read or write.
1102          */
1103         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1104                 OID_AUTO, "force_read_error", CTLFLAG_RW | CTLFLAG_MPSAFE,
1105                 &softc->force_read_error, 0,
1106                 "Force a read error for the next N reads.");
1107         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1108                 OID_AUTO, "force_write_error", CTLFLAG_RW | CTLFLAG_MPSAFE,
1109                 &softc->force_write_error, 0,
1110                 "Force a write error for the next N writes.");
1111         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1112                 OID_AUTO, "periodic_read_error", CTLFLAG_RW | CTLFLAG_MPSAFE,
1113                 &softc->periodic_read_error, 0,
1114                 "Force a read error every N reads (don't set too low).");
1115 #endif
1116         cam_periph_release(periph);
1117 }
1118
1119 static int
1120 adagetattr(struct bio *bp)
1121 {
1122         int ret;
1123         struct cam_periph *periph;
1124
1125         periph = (struct cam_periph *)bp->bio_disk->d_drv1;
1126         cam_periph_lock(periph);
1127         ret = xpt_getattr(bp->bio_data, bp->bio_length, bp->bio_attribute,
1128             periph->path);
1129         cam_periph_unlock(periph);
1130         if (ret == 0)
1131                 bp->bio_completed = bp->bio_length;
1132         return ret;
1133 }
1134
1135 static cam_status
1136 adaregister(struct cam_periph *periph, void *arg)
1137 {
1138         struct ada_softc *softc;
1139         struct ccb_pathinq cpi;
1140         struct ccb_getdev *cgd;
1141         char   announce_buf[80], buf1[32];
1142         struct disk_params *dp;
1143         caddr_t match;
1144         u_int maxio;
1145         int legacy_id, quirks;
1146
1147         cgd = (struct ccb_getdev *)arg;
1148         if (cgd == NULL) {
1149                 printf("adaregister: no getdev CCB, can't register device\n");
1150                 return(CAM_REQ_CMP_ERR);
1151         }
1152
1153         softc = (struct ada_softc *)malloc(sizeof(*softc), M_DEVBUF,
1154             M_NOWAIT|M_ZERO);
1155
1156         if (softc == NULL) {
1157                 printf("adaregister: Unable to probe new device. "
1158                     "Unable to allocate softc\n");
1159                 return(CAM_REQ_CMP_ERR);
1160         }
1161
1162         bioq_init(&softc->bio_queue);
1163         bioq_init(&softc->trim_queue);
1164
1165         if ((cgd->ident_data.capabilities1 & ATA_SUPPORT_DMA) &&
1166             (cgd->inq_flags & SID_DMA))
1167                 softc->flags |= ADA_FLAG_CAN_DMA;
1168         if (cgd->ident_data.support.command2 & ATA_SUPPORT_ADDRESS48) {
1169                 softc->flags |= ADA_FLAG_CAN_48BIT;
1170                 if (cgd->inq_flags & SID_DMA48)
1171                         softc->flags |= ADA_FLAG_CAN_DMA48;
1172         }
1173         if (cgd->ident_data.support.command2 & ATA_SUPPORT_FLUSHCACHE)
1174                 softc->flags |= ADA_FLAG_CAN_FLUSHCACHE;
1175         if (cgd->ident_data.support.command1 & ATA_SUPPORT_POWERMGT)
1176                 softc->flags |= ADA_FLAG_CAN_POWERMGT;
1177         if ((cgd->ident_data.satacapabilities & ATA_SUPPORT_NCQ) &&
1178             (cgd->inq_flags & SID_DMA) && (cgd->inq_flags & SID_CmdQue))
1179                 softc->flags |= ADA_FLAG_CAN_NCQ;
1180         if ((cgd->ident_data.support_dsm & ATA_SUPPORT_DSM_TRIM) &&
1181             (cgd->inq_flags & SID_DMA)) {
1182                 softc->flags |= ADA_FLAG_CAN_TRIM;
1183                 softc->trim_max_ranges = TRIM_MAX_RANGES;
1184                 if (cgd->ident_data.max_dsm_blocks != 0) {
1185                         softc->trim_max_ranges =
1186                             min(cgd->ident_data.max_dsm_blocks *
1187                                 ATA_DSM_BLK_RANGES, softc->trim_max_ranges);
1188                 }
1189         }
1190         if (cgd->ident_data.support.command2 & ATA_SUPPORT_CFA)
1191                 softc->flags |= ADA_FLAG_CAN_CFA;
1192
1193         periph->softc = softc;
1194
1195         /*
1196          * See if this device has any quirks.
1197          */
1198         match = cam_quirkmatch((caddr_t)&cgd->ident_data,
1199                                (caddr_t)ada_quirk_table,
1200                                sizeof(ada_quirk_table)/sizeof(*ada_quirk_table),
1201                                sizeof(*ada_quirk_table), ata_identify_match);
1202         if (match != NULL)
1203                 softc->quirks = ((struct ada_quirk_entry *)match)->quirks;
1204         else
1205                 softc->quirks = ADA_Q_NONE;
1206
1207         bzero(&cpi, sizeof(cpi));
1208         xpt_setup_ccb(&cpi.ccb_h, periph->path, CAM_PRIORITY_NONE);
1209         cpi.ccb_h.func_code = XPT_PATH_INQ;
1210         xpt_action((union ccb *)&cpi);
1211
1212         TASK_INIT(&softc->sysctl_task, 0, adasysctlinit, periph);
1213
1214         /*
1215          * Register this media as a disk
1216          */
1217         (void)cam_periph_hold(periph, PRIBIO);
1218         cam_periph_unlock(periph);
1219         snprintf(announce_buf, sizeof(announce_buf),
1220             "kern.cam.ada.%d.quirks", periph->unit_number);
1221         quirks = softc->quirks;
1222         TUNABLE_INT_FETCH(announce_buf, &quirks);
1223         softc->quirks = quirks;
1224         softc->read_ahead = -1;
1225         snprintf(announce_buf, sizeof(announce_buf),
1226             "kern.cam.ada.%d.read_ahead", periph->unit_number);
1227         TUNABLE_INT_FETCH(announce_buf, &softc->read_ahead);
1228         softc->write_cache = -1;
1229         snprintf(announce_buf, sizeof(announce_buf),
1230             "kern.cam.ada.%d.write_cache", periph->unit_number);
1231         TUNABLE_INT_FETCH(announce_buf, &softc->write_cache);
1232         /* Disable queue sorting for non-rotational media by default. */
1233         if (cgd->ident_data.media_rotation_rate == 1)
1234                 softc->sort_io_queue = 0;
1235         else
1236                 softc->sort_io_queue = -1;
1237         adagetparams(periph, cgd);
1238         softc->disk = disk_alloc();
1239         softc->disk->d_devstat = devstat_new_entry(periph->periph_name,
1240                           periph->unit_number, softc->params.secsize,
1241                           DEVSTAT_ALL_SUPPORTED,
1242                           DEVSTAT_TYPE_DIRECT |
1243                           XPORT_DEVSTAT_TYPE(cpi.transport),
1244                           DEVSTAT_PRIORITY_DISK);
1245         softc->disk->d_open = adaopen;
1246         softc->disk->d_close = adaclose;
1247         softc->disk->d_strategy = adastrategy;
1248         softc->disk->d_getattr = adagetattr;
1249         softc->disk->d_dump = adadump;
1250         softc->disk->d_gone = adadiskgonecb;
1251         softc->disk->d_name = "ada";
1252         softc->disk->d_drv1 = periph;
1253         maxio = cpi.maxio;              /* Honor max I/O size of SIM */
1254         if (maxio == 0)
1255                 maxio = DFLTPHYS;       /* traditional default */
1256         else if (maxio > MAXPHYS)
1257                 maxio = MAXPHYS;        /* for safety */
1258         if (softc->flags & ADA_FLAG_CAN_48BIT)
1259                 maxio = min(maxio, 65536 * softc->params.secsize);
1260         else                                    /* 28bit ATA command limit */
1261                 maxio = min(maxio, 256 * softc->params.secsize);
1262         softc->disk->d_maxsize = maxio;
1263         softc->disk->d_unit = periph->unit_number;
1264         softc->disk->d_flags = 0;
1265         if (softc->flags & ADA_FLAG_CAN_FLUSHCACHE)
1266                 softc->disk->d_flags |= DISKFLAG_CANFLUSHCACHE;
1267         if (softc->flags & ADA_FLAG_CAN_TRIM) {
1268                 softc->disk->d_flags |= DISKFLAG_CANDELETE;
1269                 softc->disk->d_delmaxsize = softc->params.secsize *
1270                                             ATA_DSM_RANGE_MAX *
1271                                             softc->trim_max_ranges;
1272         } else if ((softc->flags & ADA_FLAG_CAN_CFA) &&
1273             !(softc->flags & ADA_FLAG_CAN_48BIT)) {
1274                 softc->disk->d_flags |= DISKFLAG_CANDELETE;
1275                 softc->disk->d_delmaxsize = 256 * softc->params.secsize;
1276         } else
1277                 softc->disk->d_delmaxsize = maxio;
1278         if ((cpi.hba_misc & PIM_UNMAPPED) != 0)
1279                 softc->disk->d_flags |= DISKFLAG_UNMAPPED_BIO;
1280         strlcpy(softc->disk->d_descr, cgd->ident_data.model,
1281             MIN(sizeof(softc->disk->d_descr), sizeof(cgd->ident_data.model)));
1282         strlcpy(softc->disk->d_ident, cgd->ident_data.serial,
1283             MIN(sizeof(softc->disk->d_ident), sizeof(cgd->ident_data.serial)));
1284         softc->disk->d_hba_vendor = cpi.hba_vendor;
1285         softc->disk->d_hba_device = cpi.hba_device;
1286         softc->disk->d_hba_subvendor = cpi.hba_subvendor;
1287         softc->disk->d_hba_subdevice = cpi.hba_subdevice;
1288
1289         softc->disk->d_sectorsize = softc->params.secsize;
1290         softc->disk->d_mediasize = (off_t)softc->params.sectors *
1291             softc->params.secsize;
1292         if (ata_physical_sector_size(&cgd->ident_data) !=
1293             softc->params.secsize) {
1294                 softc->disk->d_stripesize =
1295                     ata_physical_sector_size(&cgd->ident_data);
1296                 softc->disk->d_stripeoffset = (softc->disk->d_stripesize -
1297                     ata_logical_sector_offset(&cgd->ident_data)) %
1298                     softc->disk->d_stripesize;
1299         } else if (softc->quirks & ADA_Q_4K) {
1300                 softc->disk->d_stripesize = 4096;
1301                 softc->disk->d_stripeoffset = 0;
1302         }
1303         softc->disk->d_fwsectors = softc->params.secs_per_track;
1304         softc->disk->d_fwheads = softc->params.heads;
1305         ata_disk_firmware_geom_adjust(softc->disk);
1306
1307         if (ada_legacy_aliases) {
1308 #ifdef ATA_STATIC_ID
1309                 legacy_id = xpt_path_legacy_ata_id(periph->path);
1310 #else
1311                 legacy_id = softc->disk->d_unit;
1312 #endif
1313                 if (legacy_id >= 0) {
1314                         snprintf(announce_buf, sizeof(announce_buf),
1315                             "kern.devalias.%s%d",
1316                             softc->disk->d_name, softc->disk->d_unit);
1317                         snprintf(buf1, sizeof(buf1),
1318                             "ad%d", legacy_id);
1319                         setenv(announce_buf, buf1);
1320                 }
1321         } else
1322                 legacy_id = -1;
1323         /*
1324          * Acquire a reference to the periph before we register with GEOM.
1325          * We'll release this reference once GEOM calls us back (via
1326          * adadiskgonecb()) telling us that our provider has been freed.
1327          */
1328         if (cam_periph_acquire(periph) != CAM_REQ_CMP) {
1329                 xpt_print(periph->path, "%s: lost periph during "
1330                           "registration!\n", __func__);
1331                 cam_periph_lock(periph);
1332                 return (CAM_REQ_CMP_ERR);
1333         }
1334         disk_create(softc->disk, DISK_VERSION);
1335         cam_periph_lock(periph);
1336         cam_periph_unhold(periph);
1337
1338         dp = &softc->params;
1339         snprintf(announce_buf, sizeof(announce_buf),
1340                 "%juMB (%ju %u byte sectors: %dH %dS/T %dC)",
1341                 (uintmax_t)(((uintmax_t)dp->secsize *
1342                 dp->sectors) / (1024*1024)),
1343                 (uintmax_t)dp->sectors,
1344                 dp->secsize, dp->heads,
1345                 dp->secs_per_track, dp->cylinders);
1346         xpt_announce_periph(periph, announce_buf);
1347         xpt_announce_quirks(periph, softc->quirks, ADA_Q_BIT_STRING);
1348         if (legacy_id >= 0)
1349                 printf("%s%d: Previously was known as ad%d\n",
1350                        periph->periph_name, periph->unit_number, legacy_id);
1351
1352         /*
1353          * Create our sysctl variables, now that we know
1354          * we have successfully attached.
1355          */
1356         cam_periph_acquire(periph);
1357         taskqueue_enqueue(taskqueue_thread, &softc->sysctl_task);
1358
1359         /*
1360          * Add async callbacks for bus reset and
1361          * bus device reset calls.  I don't bother
1362          * checking if this fails as, in most cases,
1363          * the system will function just fine without
1364          * them and the only alternative would be to
1365          * not attach the device on failure.
1366          */
1367         xpt_register_async(AC_SENT_BDR | AC_BUS_RESET | AC_LOST_DEVICE |
1368             AC_GETDEV_CHANGED | AC_ADVINFO_CHANGED,
1369             adaasync, periph, periph->path);
1370
1371         /*
1372          * Schedule a periodic event to occasionally send an
1373          * ordered tag to a device.
1374          */
1375         callout_init_mtx(&softc->sendordered_c, periph->sim->mtx, 0);
1376         callout_reset(&softc->sendordered_c,
1377             (ada_default_timeout * hz) / ADA_ORDEREDTAG_INTERVAL,
1378             adasendorderedtag, softc);
1379
1380         if (ADA_RA >= 0 &&
1381             cgd->ident_data.support.command1 & ATA_SUPPORT_LOOKAHEAD) {
1382                 softc->state = ADA_STATE_RAHEAD;
1383                 cam_periph_acquire(periph);
1384                 xpt_schedule(periph, CAM_PRIORITY_DEV);
1385         } else if (ADA_WC >= 0 &&
1386             cgd->ident_data.support.command1 & ATA_SUPPORT_WRITECACHE) {
1387                 softc->state = ADA_STATE_WCACHE;
1388                 cam_periph_acquire(periph);
1389                 xpt_schedule(periph, CAM_PRIORITY_DEV);
1390         } else
1391                 softc->state = ADA_STATE_NORMAL;
1392
1393         return(CAM_REQ_CMP);
1394 }
1395
1396 static void
1397 adastart(struct cam_periph *periph, union ccb *start_ccb)
1398 {
1399         struct ada_softc *softc = (struct ada_softc *)periph->softc;
1400         struct ccb_ataio *ataio = &start_ccb->ataio;
1401
1402         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("adastart\n"));
1403
1404         switch (softc->state) {
1405         case ADA_STATE_NORMAL:
1406         {
1407                 struct bio *bp;
1408                 u_int8_t tag_code;
1409
1410                 /* Execute immediate CCB if waiting. */
1411                 if (periph->immediate_priority <= periph->pinfo.priority) {
1412                         CAM_DEBUG(periph->path, CAM_DEBUG_SUBTRACE,
1413                                         ("queuing for immediate ccb\n"));
1414                         start_ccb->ccb_h.ccb_state = ADA_CCB_WAITING;
1415                         SLIST_INSERT_HEAD(&periph->ccb_list, &start_ccb->ccb_h,
1416                                           periph_links.sle);
1417                         periph->immediate_priority = CAM_PRIORITY_NONE;
1418                         wakeup(&periph->ccb_list);
1419                         /* Have more work to do, so ensure we stay scheduled */
1420                         adaschedule(periph);
1421                         break;
1422                 }
1423                 /* Run TRIM if not running yet. */
1424                 if (!softc->trim_running &&
1425                     (bp = bioq_first(&softc->trim_queue)) != 0) {
1426                         struct trim_request *req = &softc->trim_req;
1427                         struct bio *bp1;
1428                         uint64_t lastlba = (uint64_t)-1;
1429                         int bps = 0, c, lastcount = 0, off, ranges = 0;
1430
1431                         softc->trim_running = 1;
1432                         bzero(req, sizeof(*req));
1433                         bp1 = bp;
1434                         do {
1435                                 uint64_t lba = bp1->bio_pblkno;
1436                                 int count = bp1->bio_bcount /
1437                                     softc->params.secsize;
1438
1439                                 bioq_remove(&softc->trim_queue, bp1);
1440
1441                                 /* Try to extend the previous range. */
1442                                 if (lba == lastlba) {
1443                                         c = min(count, ATA_DSM_RANGE_MAX - lastcount);
1444                                         lastcount += c;
1445                                         off = (ranges - 1) * ATA_DSM_RANGE_SIZE;
1446                                         req->data[off + 6] = lastcount & 0xff;
1447                                         req->data[off + 7] =
1448                                             (lastcount >> 8) & 0xff;
1449                                         count -= c;
1450                                         lba += c;
1451                                 }
1452
1453                                 while (count > 0) {
1454                                         c = min(count, ATA_DSM_RANGE_MAX);
1455                                         off = ranges * ATA_DSM_RANGE_SIZE;
1456                                         req->data[off + 0] = lba & 0xff;
1457                                         req->data[off + 1] = (lba >> 8) & 0xff;
1458                                         req->data[off + 2] = (lba >> 16) & 0xff;
1459                                         req->data[off + 3] = (lba >> 24) & 0xff;
1460                                         req->data[off + 4] = (lba >> 32) & 0xff;
1461                                         req->data[off + 5] = (lba >> 40) & 0xff;
1462                                         req->data[off + 6] = c & 0xff;
1463                                         req->data[off + 7] = (c >> 8) & 0xff;
1464                                         lba += c;
1465                                         count -= c;
1466                                         lastcount = c;
1467                                         ranges++;
1468                                         /*
1469                                          * Its the caller's responsibility to ensure the
1470                                          * request will fit so we don't need to check for
1471                                          * overrun here
1472                                          */
1473                                 }
1474                                 lastlba = lba;
1475                                 req->bps[bps++] = bp1;
1476                                 bp1 = bioq_first(&softc->trim_queue);
1477                                 if (bps >= TRIM_MAX_BIOS ||
1478                                     bp1 == NULL ||
1479                                     bp1->bio_bcount / softc->params.secsize >
1480                                     (softc->trim_max_ranges - ranges) *
1481                                     ATA_DSM_RANGE_MAX)
1482                                         break;
1483                         } while (1);
1484                         cam_fill_ataio(ataio,
1485                             ada_retry_count,
1486                             adadone,
1487                             CAM_DIR_OUT,
1488                             0,
1489                             req->data,
1490                             ((ranges + ATA_DSM_BLK_RANGES - 1) /
1491                                 ATA_DSM_BLK_RANGES) * ATA_DSM_BLK_SIZE,
1492                             ada_default_timeout * 1000);
1493                         ata_48bit_cmd(ataio, ATA_DATA_SET_MANAGEMENT,
1494                             ATA_DSM_TRIM, 0, (ranges + ATA_DSM_BLK_RANGES -
1495                             1) / ATA_DSM_BLK_RANGES);
1496                         start_ccb->ccb_h.ccb_state = ADA_CCB_TRIM;
1497                         goto out;
1498                 }
1499                 /* Run regular command. */
1500                 bp = bioq_first(&softc->bio_queue);
1501                 if (bp == NULL) {
1502                         xpt_release_ccb(start_ccb);
1503                         break;
1504                 }
1505                 bioq_remove(&softc->bio_queue, bp);
1506
1507                 if ((bp->bio_flags & BIO_ORDERED) != 0
1508                  || (softc->flags & ADA_FLAG_NEED_OTAG) != 0) {
1509                         softc->flags &= ~ADA_FLAG_NEED_OTAG;
1510                         softc->ordered_tag_count++;
1511                         tag_code = 0;
1512                 } else {
1513                         tag_code = 1;
1514                 }
1515                 switch (bp->bio_cmd) {
1516                 case BIO_WRITE:
1517                         softc->flags |= ADA_FLAG_DIRTY;
1518                         /* FALLTHROUGH */
1519                 case BIO_READ:
1520                 {
1521                         uint64_t lba = bp->bio_pblkno;
1522                         uint16_t count = bp->bio_bcount / softc->params.secsize;
1523 #ifdef ADA_TEST_FAILURE
1524                         int fail = 0;
1525
1526                         /*
1527                          * Support the failure ioctls.  If the command is a
1528                          * read, and there are pending forced read errors, or
1529                          * if a write and pending write errors, then fail this
1530                          * operation with EIO.  This is useful for testing
1531                          * purposes.  Also, support having every Nth read fail.
1532                          *
1533                          * This is a rather blunt tool.
1534                          */
1535                         if (bp->bio_cmd == BIO_READ) {
1536                                 if (softc->force_read_error) {
1537                                         softc->force_read_error--;
1538                                         fail = 1;
1539                                 }
1540                                 if (softc->periodic_read_error > 0) {
1541                                         if (++softc->periodic_read_count >=
1542                                             softc->periodic_read_error) {
1543                                                 softc->periodic_read_count = 0;
1544                                                 fail = 1;
1545                                         }
1546                                 }
1547                         } else {
1548                                 if (softc->force_write_error) {
1549                                         softc->force_write_error--;
1550                                         fail = 1;
1551                                 }
1552                         }
1553                         if (fail) {
1554                                 bp->bio_error = EIO;
1555                                 bp->bio_flags |= BIO_ERROR;
1556                                 biodone(bp);
1557                                 xpt_release_ccb(start_ccb);
1558                                 adaschedule(periph);
1559                                 return;
1560                         }
1561 #endif
1562                         KASSERT((bp->bio_flags & BIO_UNMAPPED) == 0 ||
1563                             round_page(bp->bio_bcount + bp->bio_ma_offset) /
1564                             PAGE_SIZE == bp->bio_ma_n,
1565                             ("Short bio %p", bp));
1566                         cam_fill_ataio(ataio,
1567                             ada_retry_count,
1568                             adadone,
1569                             (bp->bio_cmd == BIO_READ ? CAM_DIR_IN :
1570                                 CAM_DIR_OUT) | ((bp->bio_flags & BIO_UNMAPPED)
1571                                 != 0 ? CAM_DATA_BIO : 0),
1572                             tag_code,
1573                             ((bp->bio_flags & BIO_UNMAPPED) != 0) ? (void *)bp :
1574                                 bp->bio_data,
1575                             bp->bio_bcount,
1576                             ada_default_timeout*1000);
1577
1578                         if ((softc->flags & ADA_FLAG_CAN_NCQ) && tag_code) {
1579                                 if (bp->bio_cmd == BIO_READ) {
1580                                         ata_ncq_cmd(ataio, ATA_READ_FPDMA_QUEUED,
1581                                             lba, count);
1582                                 } else {
1583                                         ata_ncq_cmd(ataio, ATA_WRITE_FPDMA_QUEUED,
1584                                             lba, count);
1585                                 }
1586                         } else if ((softc->flags & ADA_FLAG_CAN_48BIT) &&
1587                             (lba + count >= ATA_MAX_28BIT_LBA ||
1588                             count > 256)) {
1589                                 if (softc->flags & ADA_FLAG_CAN_DMA48) {
1590                                         if (bp->bio_cmd == BIO_READ) {
1591                                                 ata_48bit_cmd(ataio, ATA_READ_DMA48,
1592                                                     0, lba, count);
1593                                         } else {
1594                                                 ata_48bit_cmd(ataio, ATA_WRITE_DMA48,
1595                                                     0, lba, count);
1596                                         }
1597                                 } else {
1598                                         if (bp->bio_cmd == BIO_READ) {
1599                                                 ata_48bit_cmd(ataio, ATA_READ_MUL48,
1600                                                     0, lba, count);
1601                                         } else {
1602                                                 ata_48bit_cmd(ataio, ATA_WRITE_MUL48,
1603                                                     0, lba, count);
1604                                         }
1605                                 }
1606                         } else {
1607                                 if (count == 256)
1608                                         count = 0;
1609                                 if (softc->flags & ADA_FLAG_CAN_DMA) {
1610                                         if (bp->bio_cmd == BIO_READ) {
1611                                                 ata_28bit_cmd(ataio, ATA_READ_DMA,
1612                                                     0, lba, count);
1613                                         } else {
1614                                                 ata_28bit_cmd(ataio, ATA_WRITE_DMA,
1615                                                     0, lba, count);
1616                                         }
1617                                 } else {
1618                                         if (bp->bio_cmd == BIO_READ) {
1619                                                 ata_28bit_cmd(ataio, ATA_READ_MUL,
1620                                                     0, lba, count);
1621                                         } else {
1622                                                 ata_28bit_cmd(ataio, ATA_WRITE_MUL,
1623                                                     0, lba, count);
1624                                         }
1625                                 }
1626                         }
1627                         break;
1628                 }
1629                 case BIO_DELETE:
1630                 {
1631                         uint64_t lba = bp->bio_pblkno;
1632                         uint16_t count = bp->bio_bcount / softc->params.secsize;
1633
1634                         cam_fill_ataio(ataio,
1635                             ada_retry_count,
1636                             adadone,
1637                             CAM_DIR_NONE,
1638                             0,
1639                             NULL,
1640                             0,
1641                             ada_default_timeout*1000);
1642
1643                         if (count >= 256)
1644                                 count = 0;
1645                         ata_28bit_cmd(ataio, ATA_CFA_ERASE, 0, lba, count);
1646                         break;
1647                 }
1648                 case BIO_FLUSH:
1649                         cam_fill_ataio(ataio,
1650                             1,
1651                             adadone,
1652                             CAM_DIR_NONE,
1653                             0,
1654                             NULL,
1655                             0,
1656                             ada_default_timeout*1000);
1657
1658                         if (softc->flags & ADA_FLAG_CAN_48BIT)
1659                                 ata_48bit_cmd(ataio, ATA_FLUSHCACHE48, 0, 0, 0);
1660                         else
1661                                 ata_28bit_cmd(ataio, ATA_FLUSHCACHE, 0, 0, 0);
1662                         break;
1663                 }
1664                 start_ccb->ccb_h.ccb_state = ADA_CCB_BUFFER_IO;
1665 out:
1666                 start_ccb->ccb_h.ccb_bp = bp;
1667                 softc->outstanding_cmds++;
1668                 xpt_action(start_ccb);
1669
1670                 /* May have more work to do, so ensure we stay scheduled */
1671                 adaschedule(periph);
1672                 break;
1673         }
1674         case ADA_STATE_RAHEAD:
1675         case ADA_STATE_WCACHE:
1676         {
1677                 if ((periph->flags & CAM_PERIPH_INVALID) != 0) {
1678                         softc->state = ADA_STATE_NORMAL;
1679                         xpt_release_ccb(start_ccb);
1680                         adaschedule(periph);
1681                         cam_periph_release_locked(periph);
1682                         return;
1683                 }
1684
1685                 cam_fill_ataio(ataio,
1686                     1,
1687                     adadone,
1688                     CAM_DIR_NONE,
1689                     0,
1690                     NULL,
1691                     0,
1692                     ada_default_timeout*1000);
1693
1694                 if (softc->state == ADA_STATE_RAHEAD) {
1695                         ata_28bit_cmd(ataio, ATA_SETFEATURES, ADA_RA ?
1696                             ATA_SF_ENAB_RCACHE : ATA_SF_DIS_RCACHE, 0, 0);
1697                         start_ccb->ccb_h.ccb_state = ADA_CCB_RAHEAD;
1698                 } else {
1699                         ata_28bit_cmd(ataio, ATA_SETFEATURES, ADA_WC ?
1700                             ATA_SF_ENAB_WCACHE : ATA_SF_DIS_WCACHE, 0, 0);
1701                         start_ccb->ccb_h.ccb_state = ADA_CCB_WCACHE;
1702                 }
1703                 start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
1704                 xpt_action(start_ccb);
1705                 break;
1706         }
1707         }
1708 }
1709
1710 static void
1711 adadone(struct cam_periph *periph, union ccb *done_ccb)
1712 {
1713         struct ada_softc *softc;
1714         struct ccb_ataio *ataio;
1715         struct ccb_getdev *cgd;
1716         struct cam_path *path;
1717         int state;
1718
1719         softc = (struct ada_softc *)periph->softc;
1720         ataio = &done_ccb->ataio;
1721         path = done_ccb->ccb_h.path;
1722
1723         CAM_DEBUG(path, CAM_DEBUG_TRACE, ("adadone\n"));
1724
1725         state = ataio->ccb_h.ccb_state & ADA_CCB_TYPE_MASK;
1726         switch (state) {
1727         case ADA_CCB_BUFFER_IO:
1728         case ADA_CCB_TRIM:
1729         {
1730                 struct bio *bp;
1731                 int error;
1732
1733                 if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
1734                         error = adaerror(done_ccb, 0, 0);
1735                         if (error == ERESTART) {
1736                                 /* A retry was scheduled, so just return. */
1737                                 return;
1738                         }
1739                         if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0)
1740                                 cam_release_devq(path,
1741                                                  /*relsim_flags*/0,
1742                                                  /*reduction*/0,
1743                                                  /*timeout*/0,
1744                                                  /*getcount_only*/0);
1745                 } else {
1746                         if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0)
1747                                 panic("REQ_CMP with QFRZN");
1748                         error = 0;
1749                 }
1750                 bp = (struct bio *)done_ccb->ccb_h.ccb_bp;
1751                 bp->bio_error = error;
1752                 if (error != 0) {
1753                         bp->bio_resid = bp->bio_bcount;
1754                         bp->bio_flags |= BIO_ERROR;
1755                 } else {
1756                         if (state == ADA_CCB_TRIM)
1757                                 bp->bio_resid = 0;
1758                         else
1759                                 bp->bio_resid = ataio->resid;
1760                         if (bp->bio_resid > 0)
1761                                 bp->bio_flags |= BIO_ERROR;
1762                 }
1763                 softc->outstanding_cmds--;
1764                 if (softc->outstanding_cmds == 0)
1765                         softc->flags |= ADA_FLAG_WENT_IDLE;
1766                 if (state == ADA_CCB_TRIM) {
1767                         struct trim_request *req =
1768                             (struct trim_request *)ataio->data_ptr;
1769                         int i;
1770
1771                         for (i = 1; i < TRIM_MAX_BIOS && req->bps[i]; i++) {
1772                                 struct bio *bp1 = req->bps[i];
1773
1774                                 bp1->bio_error = bp->bio_error;
1775                                 if (bp->bio_flags & BIO_ERROR) {
1776                                         bp1->bio_flags |= BIO_ERROR;
1777                                         bp1->bio_resid = bp1->bio_bcount;
1778                                 } else
1779                                         bp1->bio_resid = 0;
1780                                 biodone(bp1);
1781                         }
1782                         softc->trim_running = 0;
1783                         biodone(bp);
1784                         adaschedule(periph);
1785                 } else
1786                         biodone(bp);
1787                 break;
1788         }
1789         case ADA_CCB_RAHEAD:
1790         {
1791                 if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
1792                         if (adaerror(done_ccb, 0, 0) == ERESTART) {
1793 out:
1794                                 /* Drop freeze taken due to CAM_DEV_QFREEZE */
1795                                 cam_release_devq(path, 0, 0, 0, FALSE);
1796                                 return;
1797                         } else if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
1798                                 cam_release_devq(path,
1799                                     /*relsim_flags*/0,
1800                                     /*reduction*/0,
1801                                     /*timeout*/0,
1802                                     /*getcount_only*/0);
1803                         }
1804                 }
1805
1806                 /*
1807                  * Since our peripheral may be invalidated by an error
1808                  * above or an external event, we must release our CCB
1809                  * before releasing the reference on the peripheral.
1810                  * The peripheral will only go away once the last reference
1811                  * is removed, and we need it around for the CCB release
1812                  * operation.
1813                  */
1814                 cgd = (struct ccb_getdev *)done_ccb;
1815                 xpt_setup_ccb(&cgd->ccb_h, path, CAM_PRIORITY_NORMAL);
1816                 cgd->ccb_h.func_code = XPT_GDEV_TYPE;
1817                 xpt_action((union ccb *)cgd);
1818                 if (ADA_WC >= 0 &&
1819                     cgd->ident_data.support.command1 & ATA_SUPPORT_WRITECACHE) {
1820                         softc->state = ADA_STATE_WCACHE;
1821                         xpt_release_ccb(done_ccb);
1822                         xpt_schedule(periph, CAM_PRIORITY_DEV);
1823                         goto out;
1824                 }
1825                 softc->state = ADA_STATE_NORMAL;
1826                 xpt_release_ccb(done_ccb);
1827                 /* Drop freeze taken due to CAM_DEV_QFREEZE */
1828                 cam_release_devq(path, 0, 0, 0, FALSE);
1829                 adaschedule(periph);
1830                 cam_periph_release_locked(periph);
1831                 return;
1832         }
1833         case ADA_CCB_WCACHE:
1834         {
1835                 if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
1836                         if (adaerror(done_ccb, 0, 0) == ERESTART) {
1837                                 goto out;
1838                         } else if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
1839                                 cam_release_devq(path,
1840                                     /*relsim_flags*/0,
1841                                     /*reduction*/0,
1842                                     /*timeout*/0,
1843                                     /*getcount_only*/0);
1844                         }
1845                 }
1846
1847                 softc->state = ADA_STATE_NORMAL;
1848                 /*
1849                  * Since our peripheral may be invalidated by an error
1850                  * above or an external event, we must release our CCB
1851                  * before releasing the reference on the peripheral.
1852                  * The peripheral will only go away once the last reference
1853                  * is removed, and we need it around for the CCB release
1854                  * operation.
1855                  */
1856                 xpt_release_ccb(done_ccb);
1857                 /* Drop freeze taken due to CAM_DEV_QFREEZE */
1858                 cam_release_devq(path, 0, 0, 0, FALSE);
1859                 adaschedule(periph);
1860                 cam_periph_release_locked(periph);
1861                 return;
1862         }
1863         case ADA_CCB_WAITING:
1864         {
1865                 /* Caller will release the CCB */
1866                 wakeup(&done_ccb->ccb_h.cbfcnp);
1867                 return;
1868         }
1869         case ADA_CCB_DUMP:
1870                 /* No-op.  We're polling */
1871                 return;
1872         default:
1873                 break;
1874         }
1875         xpt_release_ccb(done_ccb);
1876 }
1877
1878 static int
1879 adaerror(union ccb *ccb, u_int32_t cam_flags, u_int32_t sense_flags)
1880 {
1881
1882         return(cam_periph_error(ccb, cam_flags, sense_flags, NULL));
1883 }
1884
1885 static void
1886 adagetparams(struct cam_periph *periph, struct ccb_getdev *cgd)
1887 {
1888         struct ada_softc *softc = (struct ada_softc *)periph->softc;
1889         struct disk_params *dp = &softc->params;
1890         u_int64_t lbasize48;
1891         u_int32_t lbasize;
1892
1893         dp->secsize = ata_logical_sector_size(&cgd->ident_data);
1894         if ((cgd->ident_data.atavalid & ATA_FLAG_54_58) &&
1895                 cgd->ident_data.current_heads && cgd->ident_data.current_sectors) {
1896                 dp->heads = cgd->ident_data.current_heads;
1897                 dp->secs_per_track = cgd->ident_data.current_sectors;
1898                 dp->cylinders = cgd->ident_data.cylinders;
1899                 dp->sectors = (u_int32_t)cgd->ident_data.current_size_1 |
1900                           ((u_int32_t)cgd->ident_data.current_size_2 << 16);
1901         } else {
1902                 dp->heads = cgd->ident_data.heads;
1903                 dp->secs_per_track = cgd->ident_data.sectors;
1904                 dp->cylinders = cgd->ident_data.cylinders;
1905                 dp->sectors = cgd->ident_data.cylinders * dp->heads * dp->secs_per_track;
1906         }
1907         lbasize = (u_int32_t)cgd->ident_data.lba_size_1 |
1908                   ((u_int32_t)cgd->ident_data.lba_size_2 << 16);
1909
1910         /* use the 28bit LBA size if valid or bigger than the CHS mapping */
1911         if (cgd->ident_data.cylinders == 16383 || dp->sectors < lbasize)
1912                 dp->sectors = lbasize;
1913
1914         /* use the 48bit LBA size if valid */
1915         lbasize48 = ((u_int64_t)cgd->ident_data.lba_size48_1) |
1916                     ((u_int64_t)cgd->ident_data.lba_size48_2 << 16) |
1917                     ((u_int64_t)cgd->ident_data.lba_size48_3 << 32) |
1918                     ((u_int64_t)cgd->ident_data.lba_size48_4 << 48);
1919         if ((cgd->ident_data.support.command2 & ATA_SUPPORT_ADDRESS48) &&
1920             lbasize48 > ATA_MAX_28BIT_LBA)
1921                 dp->sectors = lbasize48;
1922 }
1923
1924 static void
1925 adasendorderedtag(void *arg)
1926 {
1927         struct ada_softc *softc = arg;
1928
1929         if (ada_send_ordered) {
1930                 if ((softc->ordered_tag_count == 0)
1931                  && ((softc->flags & ADA_FLAG_WENT_IDLE) == 0)) {
1932                         softc->flags |= ADA_FLAG_NEED_OTAG;
1933                 }
1934                 if (softc->outstanding_cmds > 0)
1935                         softc->flags &= ~ADA_FLAG_WENT_IDLE;
1936
1937                 softc->ordered_tag_count = 0;
1938         }
1939         /* Queue us up again */
1940         callout_reset(&softc->sendordered_c,
1941             (ada_default_timeout * hz) / ADA_ORDEREDTAG_INTERVAL,
1942             adasendorderedtag, softc);
1943 }
1944
1945 /*
1946  * Step through all ADA peripheral drivers, and if the device is still open,
1947  * sync the disk cache to physical media.
1948  */
1949 static void
1950 adaflush(void)
1951 {
1952         struct cam_periph *periph;
1953         struct ada_softc *softc;
1954         union ccb *ccb;
1955         int error;
1956
1957         CAM_PERIPH_FOREACH(periph, &adadriver) {
1958                 softc = (struct ada_softc *)periph->softc;
1959                 if (SCHEDULER_STOPPED()) {
1960                         /* If we paniced with the lock held, do not recurse. */
1961                         if (!cam_periph_owned(periph) &&
1962                             (softc->flags & ADA_FLAG_OPEN)) {
1963                                 adadump(softc->disk, NULL, 0, 0, 0);
1964                         }
1965                         continue;
1966                 }
1967                 cam_periph_lock(periph);
1968                 /*
1969                  * We only sync the cache if the drive is still open, and
1970                  * if the drive is capable of it..
1971                  */
1972                 if (((softc->flags & ADA_FLAG_OPEN) == 0) ||
1973                     (softc->flags & ADA_FLAG_CAN_FLUSHCACHE) == 0) {
1974                         cam_periph_unlock(periph);
1975                         continue;
1976                 }
1977
1978                 ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL);
1979                 cam_fill_ataio(&ccb->ataio,
1980                                     0,
1981                                     adadone,
1982                                     CAM_DIR_NONE,
1983                                     0,
1984                                     NULL,
1985                                     0,
1986                                     ada_default_timeout*1000);
1987                 if (softc->flags & ADA_FLAG_CAN_48BIT)
1988                         ata_48bit_cmd(&ccb->ataio, ATA_FLUSHCACHE48, 0, 0, 0);
1989                 else
1990                         ata_28bit_cmd(&ccb->ataio, ATA_FLUSHCACHE, 0, 0, 0);
1991
1992                 error = cam_periph_runccb(ccb, adaerror, /*cam_flags*/0,
1993                     /*sense_flags*/ SF_NO_RECOVERY | SF_NO_RETRY,
1994                     softc->disk->d_devstat);
1995                 if (error != 0)
1996                         xpt_print(periph->path, "Synchronize cache failed\n");
1997                 xpt_release_ccb(ccb);
1998                 cam_periph_unlock(periph);
1999         }
2000 }
2001
2002 static void
2003 adaspindown(uint8_t cmd, int flags)
2004 {
2005         struct cam_periph *periph;
2006         struct ada_softc *softc;
2007         union ccb *ccb;
2008         int error;
2009
2010         CAM_PERIPH_FOREACH(periph, &adadriver) {
2011                 /* If we paniced with lock held - not recurse here. */
2012                 if (cam_periph_owned(periph))
2013                         continue;
2014                 cam_periph_lock(periph);
2015                 softc = (struct ada_softc *)periph->softc;
2016                 /*
2017                  * We only spin-down the drive if it is capable of it..
2018                  */
2019                 if ((softc->flags & ADA_FLAG_CAN_POWERMGT) == 0) {
2020                         cam_periph_unlock(periph);
2021                         continue;
2022                 }
2023
2024                 if (bootverbose)
2025                         xpt_print(periph->path, "spin-down\n");
2026
2027                 ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL);
2028                 cam_fill_ataio(&ccb->ataio,
2029                                     0,
2030                                     adadone,
2031                                     CAM_DIR_NONE | flags,
2032                                     0,
2033                                     NULL,
2034                                     0,
2035                                     ada_default_timeout*1000);
2036                 ata_28bit_cmd(&ccb->ataio, cmd, 0, 0, 0);
2037
2038                 error = cam_periph_runccb(ccb, adaerror, /*cam_flags*/0,
2039                     /*sense_flags*/ SF_NO_RECOVERY | SF_NO_RETRY,
2040                     softc->disk->d_devstat);
2041                 if (error != 0)
2042                         xpt_print(periph->path, "Spin-down disk failed\n");
2043                 xpt_release_ccb(ccb);
2044                 cam_periph_unlock(periph);
2045         }
2046 }
2047
2048 static void
2049 adashutdown(void *arg, int howto)
2050 {
2051
2052         adaflush();
2053         if (ada_spindown_shutdown != 0 &&
2054             (howto & (RB_HALT | RB_POWEROFF)) != 0)
2055                 adaspindown(ATA_STANDBY_IMMEDIATE, 0);
2056 }
2057
2058 static void
2059 adasuspend(void *arg)
2060 {
2061
2062         adaflush();
2063         if (ada_spindown_suspend != 0)
2064                 adaspindown(ATA_SLEEP, CAM_DEV_QFREEZE);
2065 }
2066
2067 static void
2068 adaresume(void *arg)
2069 {
2070         struct cam_periph *periph;
2071         struct ada_softc *softc;
2072
2073         if (ada_spindown_suspend == 0)
2074                 return;
2075
2076         CAM_PERIPH_FOREACH(periph, &adadriver) {
2077                 cam_periph_lock(periph);
2078                 softc = (struct ada_softc *)periph->softc;
2079                 /*
2080                  * We only spin-down the drive if it is capable of it..
2081                  */
2082                 if ((softc->flags & ADA_FLAG_CAN_POWERMGT) == 0) {
2083                         cam_periph_unlock(periph);
2084                         continue;
2085                 }
2086
2087                 if (bootverbose)
2088                         xpt_print(periph->path, "resume\n");
2089
2090                 /*
2091                  * Drop freeze taken due to CAM_DEV_QFREEZE flag set on
2092                  * sleep request.
2093                  */
2094                 cam_release_devq(periph->path,
2095                          /*relsim_flags*/0,
2096                          /*openings*/0,
2097                          /*timeout*/0,
2098                          /*getcount_only*/0);
2099
2100                 cam_periph_unlock(periph);
2101         }
2102 }
2103
2104 #endif /* _KERNEL */