sys/cam/scsi/scsi_da.c

   1 /*-
   2  * Implementation of SCSI Direct Access Peripheral driver for CAM.
   3  *
   4  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   5  *
   6  * Copyright (c) 1997 Justin T. Gibbs.
   7  * All rights reserved.
   8  *
   9  * Redistribution and use in source and binary forms, with or without
  10  * modification, are permitted provided that the following conditions
  11  * are met:
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions, and the following disclaimer,
  14  *    without modification, immediately at the beginning of the file.
  15  * 2. The name of the author may not be used to endorse or promote products
  16  *    derived from this software without specific prior written permission.
  17  *
  18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  21  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
  22  * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  28  * SUCH DAMAGE.
  29  */
  30
  31 #include <sys/cdefs.h>
  32 __FBSDID("$FreeBSD$");
  33
  34 #include <sys/param.h>
  35
  36 #ifdef _KERNEL
  37 #include "opt_da.h"
  38 #include <sys/systm.h>
  39 #include <sys/kernel.h>
  40 #include <sys/bio.h>
  41 #include <sys/sysctl.h>
  42 #include <sys/taskqueue.h>
  43 #include <sys/lock.h>
  44 #include <sys/mutex.h>
  45 #include <sys/conf.h>
  46 #include <sys/devicestat.h>
  47 #include <sys/eventhandler.h>
  48 #include <sys/malloc.h>
  49 #include <sys/cons.h>
  50 #include <sys/endian.h>
  51 #include <sys/proc.h>
  52 #include <sys/sbuf.h>
  53 #include <geom/geom.h>
  54 #include <geom/geom_disk.h>
  55 #include <machine/atomic.h>
  56 #endif /* _KERNEL */
  57
  58 #ifndef _KERNEL
  59 #include <stdio.h>
  60 #include <string.h>
  61 #endif /* _KERNEL */
  62
  63 #include <cam/cam.h>
  64 #include <cam/cam_ccb.h>
  65 #include <cam/cam_periph.h>
  66 #include <cam/cam_xpt_periph.h>
  67 #include <cam/cam_sim.h>
  68 #include <cam/cam_iosched.h>
  69
  70 #include <cam/scsi/scsi_message.h>
  71 #include <cam/scsi/scsi_da.h>
  72
  73 #ifdef _KERNEL
  74 /*
  75  * Note that there are probe ordering dependencies here.  The order isn't
  76  * controlled by this enumeration, but by explicit state transitions in
  77  * dastart() and dadone().  Here are some of the dependencies:
  78  *
  79  * 1. RC should come first, before RC16, unless there is evidence that RC16
  80  *    is supported.
  81  * 2. BDC needs to come before any of the ATA probes, or the ZONE probe.
  82  * 3. The ATA probes should go in this order:
  83  *    ATA -> LOGDIR -> IDDIR -> SUP -> ATA_ZONE
  84  */
  85 typedef enum {
  86         DA_STATE_PROBE_WP,
  87         DA_STATE_PROBE_RC,
  88         DA_STATE_PROBE_RC16,
  89         DA_STATE_PROBE_LBP,
  90         DA_STATE_PROBE_BLK_LIMITS,
  91         DA_STATE_PROBE_BDC,
  92         DA_STATE_PROBE_ATA,
  93         DA_STATE_PROBE_ATA_LOGDIR,
  94         DA_STATE_PROBE_ATA_IDDIR,
  95         DA_STATE_PROBE_ATA_SUP,
  96         DA_STATE_PROBE_ATA_ZONE,
  97         DA_STATE_PROBE_ZONE,
  98         DA_STATE_NORMAL
  99 } da_state;
 100
 101 typedef enum {
 102         DA_FLAG_PACK_INVALID    = 0x000001,
 103         DA_FLAG_NEW_PACK        = 0x000002,
 104         DA_FLAG_PACK_LOCKED     = 0x000004,
 105         DA_FLAG_PACK_REMOVABLE  = 0x000008,
 106         DA_FLAG_NEED_OTAG       = 0x000020,
 107         DA_FLAG_WAS_OTAG        = 0x000040,
 108         DA_FLAG_RETRY_UA        = 0x000080,
 109         DA_FLAG_OPEN            = 0x000100,
 110         DA_FLAG_SCTX_INIT       = 0x000200,
 111         DA_FLAG_CAN_RC16        = 0x000400,
 112         DA_FLAG_PROBED          = 0x000800,
 113         DA_FLAG_DIRTY           = 0x001000,
 114         DA_FLAG_ANNOUNCED       = 0x002000,
 115         DA_FLAG_CAN_ATA_DMA     = 0x004000,
 116         DA_FLAG_CAN_ATA_LOG     = 0x008000,
 117         DA_FLAG_CAN_ATA_IDLOG   = 0x010000,
 118         DA_FLAG_CAN_ATA_SUPCAP  = 0x020000,
 119         DA_FLAG_CAN_ATA_ZONE    = 0x040000,
 120         DA_FLAG_TUR_PENDING     = 0x080000
 121 } da_flags;
 122
 123 typedef enum {
 124         DA_Q_NONE               = 0x00,
 125         DA_Q_NO_SYNC_CACHE      = 0x01,
 126         DA_Q_NO_6_BYTE          = 0x02,
 127         DA_Q_NO_PREVENT         = 0x04,
 128         DA_Q_4K                 = 0x08,
 129         DA_Q_NO_RC16            = 0x10,
 130         DA_Q_NO_UNMAP           = 0x20,
 131         DA_Q_RETRY_BUSY         = 0x40,
 132         DA_Q_SMR_DM             = 0x80,
 133         DA_Q_STRICT_UNMAP       = 0x100,
 134         DA_Q_128KB              = 0x200
 135 } da_quirks;
 136
 137 #define DA_Q_BIT_STRING         \
 138         "\020"                  \
 139         "\001NO_SYNC_CACHE"     \
 140         "\002NO_6_BYTE"         \
 141         "\003NO_PREVENT"        \
 142         "\0044K"                \
 143         "\005NO_RC16"           \
 144         "\006NO_UNMAP"          \
 145         "\007RETRY_BUSY"        \
 146         "\010SMR_DM"            \
 147         "\011STRICT_UNMAP"      \
 148         "\012128KB"
 149
 150 typedef enum {
 151         DA_CCB_PROBE_RC         = 0x01,
 152         DA_CCB_PROBE_RC16       = 0x02,
 153         DA_CCB_PROBE_LBP        = 0x03,
 154         DA_CCB_PROBE_BLK_LIMITS = 0x04,
 155         DA_CCB_PROBE_BDC        = 0x05,
 156         DA_CCB_PROBE_ATA        = 0x06,
 157         DA_CCB_BUFFER_IO        = 0x07,
 158         DA_CCB_DUMP             = 0x0A,
 159         DA_CCB_DELETE           = 0x0B,
 160         DA_CCB_TUR              = 0x0C,
 161         DA_CCB_PROBE_ZONE       = 0x0D,
 162         DA_CCB_PROBE_ATA_LOGDIR = 0x0E,
 163         DA_CCB_PROBE_ATA_IDDIR  = 0x0F,
 164         DA_CCB_PROBE_ATA_SUP    = 0x10,
 165         DA_CCB_PROBE_ATA_ZONE   = 0x11,
 166         DA_CCB_PROBE_WP         = 0x12,
 167         DA_CCB_TYPE_MASK        = 0x1F,
 168         DA_CCB_RETRY_UA         = 0x20
 169 } da_ccb_state;
 170
 171 /*
 172  * Order here is important for method choice
 173  *
 174  * We prefer ATA_TRIM as tests run against a Sandforce 2281 SSD attached to
 175  * LSI 2008 (mps) controller (FW: v12, Drv: v14) resulted 20% quicker deletes
 176  * using ATA_TRIM than the corresponding UNMAP results for a real world mysql
 177  * import taking 5mins.
 178  *
 179  */
 180 typedef enum {
 181         DA_DELETE_NONE,
 182         DA_DELETE_DISABLE,
 183         DA_DELETE_ATA_TRIM,
 184         DA_DELETE_UNMAP,
 185         DA_DELETE_WS16,
 186         DA_DELETE_WS10,
 187         DA_DELETE_ZERO,
 188         DA_DELETE_MIN = DA_DELETE_ATA_TRIM,
 189         DA_DELETE_MAX = DA_DELETE_ZERO
 190 } da_delete_methods;
 191
 192 /*
 193  * For SCSI, host managed drives show up as a separate device type.  For
 194  * ATA, host managed drives also have a different device signature.
 195  * XXX KDM figure out the ATA host managed signature.
 196  */
 197 typedef enum {
 198         DA_ZONE_NONE            = 0x00,
 199         DA_ZONE_DRIVE_MANAGED   = 0x01,
 200         DA_ZONE_HOST_AWARE      = 0x02,
 201         DA_ZONE_HOST_MANAGED    = 0x03
 202 } da_zone_mode;
 203
 204 /*
 205  * We distinguish between these interface cases in addition to the drive type:
 206  * o ATA drive behind a SCSI translation layer that knows about ZBC/ZAC
 207  * o ATA drive behind a SCSI translation layer that does not know about
 208  *   ZBC/ZAC, and so needs to be managed via ATA passthrough.  In this
 209  *   case, we would need to share the ATA code with the ada(4) driver.
 210  * o SCSI drive.
 211  */
 212 typedef enum {
 213         DA_ZONE_IF_SCSI,
 214         DA_ZONE_IF_ATA_PASS,
 215         DA_ZONE_IF_ATA_SAT,
 216 } da_zone_interface;
 217
 218 typedef enum {
 219         DA_ZONE_FLAG_RZ_SUP             = 0x0001,
 220         DA_ZONE_FLAG_OPEN_SUP           = 0x0002,
 221         DA_ZONE_FLAG_CLOSE_SUP          = 0x0004,
 222         DA_ZONE_FLAG_FINISH_SUP         = 0x0008,
 223         DA_ZONE_FLAG_RWP_SUP            = 0x0010,
 224         DA_ZONE_FLAG_SUP_MASK           = (DA_ZONE_FLAG_RZ_SUP |
 225                                            DA_ZONE_FLAG_OPEN_SUP |
 226                                            DA_ZONE_FLAG_CLOSE_SUP |
 227                                            DA_ZONE_FLAG_FINISH_SUP |
 228                                            DA_ZONE_FLAG_RWP_SUP),
 229         DA_ZONE_FLAG_URSWRZ             = 0x0020,
 230         DA_ZONE_FLAG_OPT_SEQ_SET        = 0x0040,
 231         DA_ZONE_FLAG_OPT_NONSEQ_SET     = 0x0080,
 232         DA_ZONE_FLAG_MAX_SEQ_SET        = 0x0100,
 233         DA_ZONE_FLAG_SET_MASK           = (DA_ZONE_FLAG_OPT_SEQ_SET |
 234                                            DA_ZONE_FLAG_OPT_NONSEQ_SET |
 235                                            DA_ZONE_FLAG_MAX_SEQ_SET)
 236 } da_zone_flags;
 237
 238 static struct da_zone_desc {
 239         da_zone_flags value;
 240         const char *desc;
 241 } da_zone_desc_table[] = {
 242         {DA_ZONE_FLAG_RZ_SUP, "Report Zones" },
 243         {DA_ZONE_FLAG_OPEN_SUP, "Open" },
 244         {DA_ZONE_FLAG_CLOSE_SUP, "Close" },
 245         {DA_ZONE_FLAG_FINISH_SUP, "Finish" },
 246         {DA_ZONE_FLAG_RWP_SUP, "Reset Write Pointer" },
 247 };
 248
 249 typedef void da_delete_func_t (struct cam_periph *periph, union ccb *ccb,
 250                               struct bio *bp);
 251 static da_delete_func_t da_delete_trim;
 252 static da_delete_func_t da_delete_unmap;
 253 static da_delete_func_t da_delete_ws;
 254
 255 static const void * da_delete_functions[] = {
 256         NULL,
 257         NULL,
 258         da_delete_trim,
 259         da_delete_unmap,
 260         da_delete_ws,
 261         da_delete_ws,
 262         da_delete_ws
 263 };
 264
 265 static const char *da_delete_method_names[] =
 266     { "NONE", "DISABLE", "ATA_TRIM", "UNMAP", "WS16", "WS10", "ZERO" };
 267 static const char *da_delete_method_desc[] =
 268     { "NONE", "DISABLED", "ATA TRIM", "UNMAP", "WRITE SAME(16) with UNMAP",
 269       "WRITE SAME(10) with UNMAP", "ZERO" };
 270
 271 /* Offsets into our private area for storing information */
 272 #define ccb_state       ppriv_field0
 273 #define ccb_bp          ppriv_ptr1
 274
 275 struct disk_params {
 276         u_int8_t  heads;
 277         u_int32_t cylinders;
 278         u_int8_t  secs_per_track;
 279         u_int32_t secsize;      /* Number of bytes/sector */
 280         u_int64_t sectors;      /* total number sectors */
 281         u_int     stripesize;
 282         u_int     stripeoffset;
 283 };
 284
 285 #define UNMAP_RANGE_MAX         0xffffffff
 286 #define UNMAP_HEAD_SIZE         8
 287 #define UNMAP_RANGE_SIZE        16
 288 #define UNMAP_MAX_RANGES        2048 /* Protocol Max is 4095 */
 289 #define UNMAP_BUF_SIZE          ((UNMAP_MAX_RANGES * UNMAP_RANGE_SIZE) + \
 290                                 UNMAP_HEAD_SIZE)
 291
 292 #define WS10_MAX_BLKS           0xffff
 293 #define WS16_MAX_BLKS           0xffffffff
 294 #define ATA_TRIM_MAX_RANGES     ((UNMAP_BUF_SIZE / \
 295         (ATA_DSM_RANGE_SIZE * ATA_DSM_BLK_SIZE)) * ATA_DSM_BLK_SIZE)
 296
 297 #define DA_WORK_TUR             (1 << 16)
 298
 299 typedef enum {
 300         DA_REF_OPEN = 1,
 301         DA_REF_OPEN_HOLD,
 302         DA_REF_CLOSE_HOLD,
 303         DA_REF_PROBE_HOLD,
 304         DA_REF_TUR,
 305         DA_REF_GEOM,
 306         DA_REF_SYSCTL,
 307         DA_REF_REPROBE,
 308         DA_REF_MAX              /* KEEP LAST */
 309 } da_ref_token;
 310
 311 struct da_softc {
 312         struct   cam_iosched_softc *cam_iosched;
 313         struct   bio_queue_head delete_run_queue;
 314         LIST_HEAD(, ccb_hdr) pending_ccbs;
 315         int      refcount;              /* Active xpt_action() calls */
 316         da_state state;
 317         da_flags flags;
 318         da_quirks quirks;
 319         int      minimum_cmd_size;
 320         int      error_inject;
 321         int      trim_max_ranges;
 322         int      delete_available;      /* Delete methods possibly available */
 323         da_zone_mode                    zone_mode;
 324         da_zone_interface               zone_interface;
 325         da_zone_flags                   zone_flags;
 326         struct ata_gp_log_dir           ata_logdir;
 327         int                             valid_logdir_len;
 328         struct ata_identify_log_pages   ata_iddir;
 329         int                             valid_iddir_len;
 330         uint64_t                        optimal_seq_zones;
 331         uint64_t                        optimal_nonseq_zones;
 332         uint64_t                        max_seq_zones;
 333         u_int                   maxio;
 334         uint32_t                unmap_max_ranges;
 335         uint32_t                unmap_max_lba; /* Max LBAs in UNMAP req */
 336         uint32_t                unmap_gran;
 337         uint32_t                unmap_gran_align;
 338         uint64_t                ws_max_blks;
 339         uint64_t                trim_count;
 340         uint64_t                trim_ranges;
 341         uint64_t                trim_lbas;
 342         da_delete_methods       delete_method_pref;
 343         da_delete_methods       delete_method;
 344         da_delete_func_t        *delete_func;
 345         int                     unmappedio;
 346         int                     rotating;
 347         int                     p_type;
 348         struct   disk_params params;
 349         struct   disk *disk;
 350         union    ccb saved_ccb;
 351         struct task             sysctl_task;
 352         struct sysctl_ctx_list  sysctl_ctx;
 353         struct sysctl_oid       *sysctl_tree;
 354         struct callout          sendordered_c;
 355         uint64_t wwpn;
 356         uint8_t  unmap_buf[UNMAP_BUF_SIZE];
 357         struct scsi_read_capacity_data_long rcaplong;
 358         struct callout          mediapoll_c;
 359         int                     ref_flags[DA_REF_MAX];
 360 #ifdef CAM_IO_STATS
 361         struct sysctl_ctx_list  sysctl_stats_ctx;
 362         struct sysctl_oid       *sysctl_stats_tree;
 363         u_int   errors;
 364         u_int   timeouts;
 365         u_int   invalidations;
 366 #endif
 367 #define DA_ANNOUNCETMP_SZ 160
 368         char                    announce_temp[DA_ANNOUNCETMP_SZ];
 369 #define DA_ANNOUNCE_SZ 400
 370         char                    announcebuf[DA_ANNOUNCE_SZ];
 371 };
 372
 373 #define dadeleteflag(softc, delete_method, enable)                      \
 374         if (enable) {                                                   \
 375                 softc->delete_available |= (1 << delete_method);        \
 376         } else {                                                        \
 377                 softc->delete_available &= ~(1 << delete_method);       \
 378         }
 379
 380 struct da_quirk_entry {
 381         struct scsi_inquiry_pattern inq_pat;
 382         da_quirks quirks;
 383 };
 384
 385 static const char quantum[] = "QUANTUM";
 386 static const char microp[] = "MICROP";
 387
 388 static struct da_quirk_entry da_quirk_table[] =
 389 {
 390         /* SPI, FC devices */
 391         {
 392                 /*
 393                  * Fujitsu M2513A MO drives.
 394                  * Tested devices: M2513A2 firmware versions 1200 & 1300.
 395                  * (dip switch selects whether T_DIRECT or T_OPTICAL device)
 396                  * Reported by: W.Scholten <whs@xs4all.nl>
 397                  */
 398                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "FUJITSU", "M2513A", "*"},
 399                 /*quirks*/ DA_Q_NO_SYNC_CACHE
 400         },
 401         {
 402                 /* See above. */
 403                 {T_OPTICAL, SIP_MEDIA_REMOVABLE, "FUJITSU", "M2513A", "*"},
 404                 /*quirks*/ DA_Q_NO_SYNC_CACHE
 405         },
 406         {
 407                 /*
 408                  * This particular Fujitsu drive doesn't like the
 409                  * synchronize cache command.
 410                  * Reported by: Tom Jackson <toj@gorilla.net>
 411                  */
 412                 {T_DIRECT, SIP_MEDIA_FIXED, "FUJITSU", "M2954*", "*"},
 413                 /*quirks*/ DA_Q_NO_SYNC_CACHE
 414         },
 415         {
 416                 /*
 417                  * This drive doesn't like the synchronize cache command
 418                  * either.  Reported by: Matthew Jacob <mjacob@feral.com>
 419                  * in NetBSD PR kern/6027, August 24, 1998.
 420                  */
 421                 {T_DIRECT, SIP_MEDIA_FIXED, microp, "2217*", "*"},
 422                 /*quirks*/ DA_Q_NO_SYNC_CACHE
 423         },
 424         {
 425                 /*
 426                  * This drive doesn't like the synchronize cache command
 427                  * either.  Reported by: Hellmuth Michaelis (hm@kts.org)
 428                  * (PR 8882).
 429                  */
 430                 {T_DIRECT, SIP_MEDIA_FIXED, microp, "2112*", "*"},
 431                 /*quirks*/ DA_Q_NO_SYNC_CACHE
 432         },
 433         {
 434                 /*
 435                  * Doesn't like the synchronize cache command.
 436                  * Reported by: Blaz Zupan <blaz@gold.amis.net>
 437                  */
 438                 {T_DIRECT, SIP_MEDIA_FIXED, "NEC", "D3847*", "*"},
 439                 /*quirks*/ DA_Q_NO_SYNC_CACHE
 440         },
 441         {
 442                 /*
 443                  * Doesn't like the synchronize cache command.
 444                  * Reported by: Blaz Zupan <blaz@gold.amis.net>
 445                  */
 446                 {T_DIRECT, SIP_MEDIA_FIXED, quantum, "MAVERICK 540S", "*"},
 447                 /*quirks*/ DA_Q_NO_SYNC_CACHE
 448         },
 449         {
 450                 /*
 451                  * Doesn't like the synchronize cache command.
 452                  */
 453                 {T_DIRECT, SIP_MEDIA_FIXED, quantum, "LPS525S", "*"},
 454                 /*quirks*/ DA_Q_NO_SYNC_CACHE
 455         },
 456         {
 457                 /*
 458                  * Doesn't like the synchronize cache command.
 459                  * Reported by: walter@pelissero.de
 460                  */
 461                 {T_DIRECT, SIP_MEDIA_FIXED, quantum, "LPS540S", "*"},
 462                 /*quirks*/ DA_Q_NO_SYNC_CACHE
 463         },
 464         {
 465                 /*
 466                  * Doesn't work correctly with 6 byte reads/writes.
 467                  * Returns illegal request, and points to byte 9 of the
 468                  * 6-byte CDB.
 469                  * Reported by:  Adam McDougall <bsdx@spawnet.com>
 470                  */
 471                 {T_DIRECT, SIP_MEDIA_FIXED, quantum, "VIKING 4*", "*"},
 472                 /*quirks*/ DA_Q_NO_6_BYTE
 473         },
 474         {
 475                 /* See above. */
 476                 {T_DIRECT, SIP_MEDIA_FIXED, quantum, "VIKING 2*", "*"},
 477                 /*quirks*/ DA_Q_NO_6_BYTE
 478         },
 479         {
 480                 /*
 481                  * Doesn't like the synchronize cache command.
 482                  * Reported by: walter@pelissero.de
 483                  */
 484                 {T_DIRECT, SIP_MEDIA_FIXED, "CONNER", "CP3500*", "*"},
 485                 /*quirks*/ DA_Q_NO_SYNC_CACHE
 486         },
 487         {
 488                 /*
 489                  * The CISS RAID controllers do not support SYNC_CACHE
 490                  */
 491                 {T_DIRECT, SIP_MEDIA_FIXED, "COMPAQ", "RAID*", "*"},
 492                 /*quirks*/ DA_Q_NO_SYNC_CACHE
 493         },
 494         {
 495                 /*
 496                  * The STEC SSDs sometimes hang on UNMAP.
 497                  */
 498                 {T_DIRECT, SIP_MEDIA_FIXED, "STEC", "*", "*"},
 499                 /*quirks*/ DA_Q_NO_UNMAP
 500         },
 501         {
 502                 /*
 503                  * VMware returns BUSY status when storage has transient
 504                  * connectivity problems, so better wait.
 505                  * Also VMware returns odd errors on misaligned UNMAPs.
 506                  */
 507                 {T_DIRECT, SIP_MEDIA_FIXED, "VMware*", "*", "*"},
 508                 /*quirks*/ DA_Q_RETRY_BUSY | DA_Q_STRICT_UNMAP
 509         },
 510         /* USB mass storage devices supported by umass(4) */
 511         {
 512                 /*
 513                  * EXATELECOM (Sigmatel) i-Bead 100/105 USB Flash MP3 Player
 514                  * PR: kern/51675
 515                  */
 516                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "EXATEL", "i-BEAD10*", "*"},
 517                 /*quirks*/ DA_Q_NO_SYNC_CACHE
 518         },
 519         {
 520                 /*
 521                  * Power Quotient Int. (PQI) USB flash key
 522                  * PR: kern/53067
 523                  */
 524                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "Generic*", "USB Flash Disk*",
 525                 "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
 526         },
 527         {
 528                 /*
 529                  * Creative Nomad MUVO mp3 player (USB)
 530                  * PR: kern/53094
 531                  */
 532                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "CREATIVE", "NOMAD_MUVO", "*"},
 533                 /*quirks*/ DA_Q_NO_SYNC_CACHE|DA_Q_NO_PREVENT
 534         },
 535         {
 536                 /*
 537                  * Jungsoft NEXDISK USB flash key
 538                  * PR: kern/54737
 539                  */
 540                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "JUNGSOFT", "NEXDISK*", "*"},
 541                 /*quirks*/ DA_Q_NO_SYNC_CACHE
 542         },
 543         {
 544                 /*
 545                  * FreeDik USB Mini Data Drive
 546                  * PR: kern/54786
 547                  */
 548                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "FreeDik*", "Mini Data Drive",
 549                 "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
 550         },
 551         {
 552                 /*
 553                  * Sigmatel USB Flash MP3 Player
 554                  * PR: kern/57046
 555                  */
 556                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "SigmaTel", "MSCN", "*"},
 557                 /*quirks*/ DA_Q_NO_SYNC_CACHE|DA_Q_NO_PREVENT
 558         },
 559         {
 560                 /*
 561                  * Neuros USB Digital Audio Computer
 562                  * PR: kern/63645
 563                  */
 564                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "NEUROS", "dig. audio comp.",
 565                 "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
 566         },
 567         {
 568                 /*
 569                  * SEAGRAND NP-900 MP3 Player
 570                  * PR: kern/64563
 571                  */
 572                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "SEAGRAND", "NP-900*", "*"},
 573                 /*quirks*/ DA_Q_NO_SYNC_CACHE|DA_Q_NO_PREVENT
 574         },
 575         {
 576                 /*
 577                  * iRiver iFP MP3 player (with UMS Firmware)
 578                  * PR: kern/54881, i386/63941, kern/66124
 579                  */
 580                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "iRiver", "iFP*", "*"},
 581                 /*quirks*/ DA_Q_NO_SYNC_CACHE
 582         },
 583         {
 584                 /*
 585                  * Frontier Labs NEX IA+ Digital Audio Player, rev 1.10/0.01
 586                  * PR: kern/70158
 587                  */
 588                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "FL" , "Nex*", "*"},
 589                 /*quirks*/ DA_Q_NO_SYNC_CACHE
 590         },
 591         {
 592                 /*
 593                  * ZICPlay USB MP3 Player with FM
 594                  * PR: kern/75057
 595                  */
 596                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "ACTIONS*" , "USB DISK*", "*"},
 597                 /*quirks*/ DA_Q_NO_SYNC_CACHE
 598         },
 599         {
 600                 /*
 601                  * TEAC USB floppy mechanisms
 602                  */
 603                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "TEAC" , "FD-05*", "*"},
 604                 /*quirks*/ DA_Q_NO_SYNC_CACHE
 605         },
 606         {
 607                 /*
 608                  * Kingston DataTraveler II+ USB Pen-Drive.
 609                  * Reported by: Pawel Jakub Dawidek <pjd@FreeBSD.org>
 610                  */
 611                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "Kingston" , "DataTraveler II+",
 612                 "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
 613         },
 614         {
 615                 /*
 616                  * USB DISK Pro PMAP
 617                  * Reported by: jhs
 618                  * PR: usb/96381
 619                  */
 620                 {T_DIRECT, SIP_MEDIA_REMOVABLE, " ", "USB DISK Pro", "PMAP"},
 621                 /*quirks*/ DA_Q_NO_SYNC_CACHE
 622         },
 623         {
 624                 /*
 625                  * Motorola E398 Mobile Phone (TransFlash memory card).
 626                  * Reported by: Wojciech A. Koszek <dunstan@FreeBSD.czest.pl>
 627                  * PR: usb/89889
 628                  */
 629                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "Motorola" , "Motorola Phone",
 630                 "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
 631         },
 632         {
 633                 /*
 634                  * Qware BeatZkey! Pro
 635                  * PR: usb/79164
 636                  */
 637                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "GENERIC", "USB DISK DEVICE",
 638                 "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
 639         },
 640         {
 641                 /*
 642                  * Time DPA20B 1GB MP3 Player
 643                  * PR: usb/81846
 644                  */
 645                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "USB2.0*", "(FS) FLASH DISK*",
 646                 "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
 647         },
 648         {
 649                 /*
 650                  * Samsung USB key 128Mb
 651                  * PR: usb/90081
 652                  */
 653                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "USB-DISK", "FreeDik-FlashUsb",
 654                 "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
 655         },
 656         {
 657                 /*
 658                  * Kingston DataTraveler 2.0 USB Flash memory.
 659                  * PR: usb/89196
 660                  */
 661                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "Kingston", "DataTraveler 2.0",
 662                 "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
 663         },
 664         {
 665                 /*
 666                  * Creative MUVO Slim mp3 player (USB)
 667                  * PR: usb/86131
 668                  */
 669                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "CREATIVE", "MuVo Slim",
 670                 "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE|DA_Q_NO_PREVENT
 671                 },
 672         {
 673                 /*
 674                  * United MP5512 Portable MP3 Player (2-in-1 USB DISK/MP3)
 675                  * PR: usb/80487
 676                  */
 677                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "Generic*", "MUSIC DISK",
 678                 "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
 679         },
 680         {
 681                 /*
 682                  * SanDisk Micro Cruzer 128MB
 683                  * PR: usb/75970
 684                  */
 685                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "SanDisk" , "Micro Cruzer",
 686                 "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
 687         },
 688         {
 689                 /*
 690                  * TOSHIBA TransMemory USB sticks
 691                  * PR: kern/94660
 692                  */
 693                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "TOSHIBA", "TransMemory",
 694                 "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
 695         },
 696         {
 697                 /*
 698                  * PNY USB 3.0 Flash Drives
 699                 */
 700                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "PNY", "USB 3.0 FD*",
 701                 "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE | DA_Q_NO_RC16
 702         },
 703         {
 704                 /*
 705                  * PNY USB Flash keys
 706                  * PR: usb/75578, usb/72344, usb/65436
 707                  */
 708                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "*" , "USB DISK*",
 709                 "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
 710         },
 711         {
 712                 /*
 713                  * Genesys GL3224
 714                  */
 715                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "Generic*", "STORAGE DEVICE*",
 716                 "120?"}, /*quirks*/ DA_Q_NO_SYNC_CACHE | DA_Q_4K | DA_Q_NO_RC16
 717         },
 718         {
 719                 /*
 720                  * Genesys 6-in-1 Card Reader
 721                  * PR: usb/94647
 722                  */
 723                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "Generic*", "STORAGE DEVICE*",
 724                 "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
 725         },
 726         {
 727                 /*
 728                  * Rekam Digital CAMERA
 729                  * PR: usb/98713
 730                  */
 731                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "CAMERA*", "4MP-9J6*",
 732                 "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
 733         },
 734         {
 735                 /*
 736                  * iRiver H10 MP3 player
 737                  * PR: usb/102547
 738                  */
 739                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "iriver", "H10*",
 740                 "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
 741         },
 742         {
 743                 /*
 744                  * iRiver U10 MP3 player
 745                  * PR: usb/92306
 746                  */
 747                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "iriver", "U10*",
 748                 "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
 749         },
 750         {
 751                 /*
 752                  * X-Micro Flash Disk
 753                  * PR: usb/96901
 754                  */
 755                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "X-Micro", "Flash Disk",
 756                 "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
 757         },
 758         {
 759                 /*
 760                  * EasyMP3 EM732X USB 2.0 Flash MP3 Player
 761                  * PR: usb/96546
 762                  */
 763                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "EM732X", "MP3 Player*",
 764                 "1.00"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
 765         },
 766         {
 767                 /*
 768                  * Denver MP3 player
 769                  * PR: usb/107101
 770                  */
 771                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "DENVER", "MP3 PLAYER",
 772                  "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
 773         },
 774         {
 775                 /*
 776                  * Philips USB Key Audio KEY013
 777                  * PR: usb/68412
 778                  */
 779                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "PHILIPS", "Key*", "*"},
 780                 /*quirks*/ DA_Q_NO_SYNC_CACHE | DA_Q_NO_PREVENT
 781         },
 782         {
 783                 /*
 784                  * JNC MP3 Player
 785                  * PR: usb/94439
 786                  */
 787                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "JNC*" , "MP3 Player*",
 788                  "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
 789         },
 790         {
 791                 /*
 792                  * SAMSUNG MP0402H
 793                  * PR: usb/108427
 794                  */
 795                 {T_DIRECT, SIP_MEDIA_FIXED, "SAMSUNG", "MP0402H", "*"},
 796                 /*quirks*/ DA_Q_NO_SYNC_CACHE
 797         },
 798         {
 799                 /*
 800                  * I/O Magic USB flash - Giga Bank
 801                  * PR: usb/108810
 802                  */
 803                 {T_DIRECT, SIP_MEDIA_FIXED, "GS-Magic", "stor*", "*"},
 804                 /*quirks*/ DA_Q_NO_SYNC_CACHE
 805         },
 806         {
 807                 /*
 808                  * JoyFly 128mb USB Flash Drive
 809                  * PR: 96133
 810                  */
 811                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "USB 2.0", "Flash Disk*",
 812                  "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
 813         },
 814         {
 815                 /*
 816                  * ChipsBnk usb stick
 817                  * PR: 103702
 818                  */
 819                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "ChipsBnk", "USB*",
 820                  "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
 821         },
 822         {
 823                 /*
 824                  * Storcase (Kingston) InfoStation IFS FC2/SATA-R 201A
 825                  * PR: 129858
 826                  */
 827                 {T_DIRECT, SIP_MEDIA_FIXED, "IFS", "FC2/SATA-R*",
 828                  "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
 829         },
 830         {
 831                 /*
 832                  * Samsung YP-U3 mp3-player
 833                  * PR: 125398
 834                  */
 835                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "Samsung", "YP-U3",
 836                  "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
 837         },
 838         {
 839                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "Netac", "OnlyDisk*",
 840                  "2000"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
 841         },
 842         {
 843                 /*
 844                  * Sony Cyber-Shot DSC cameras
 845                  * PR: usb/137035
 846                  */
 847                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "Sony", "Sony DSC", "*"},
 848                 /*quirks*/ DA_Q_NO_SYNC_CACHE | DA_Q_NO_PREVENT
 849         },
 850         {
 851                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "Kingston", "DataTraveler G3",
 852                  "1.00"}, /*quirks*/ DA_Q_NO_PREVENT
 853         },
 854         {
 855                 /* At least several Transcent USB sticks lie on RC16. */
 856                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "JetFlash", "Transcend*",
 857                  "*"}, /*quirks*/ DA_Q_NO_RC16
 858         },
 859         {
 860                 /*
 861                  * I-O Data USB Flash Disk
 862                  * PR: usb/211716
 863                  */
 864                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "I-O DATA", "USB Flash Disk*",
 865                  "*"}, /*quirks*/ DA_Q_NO_RC16
 866         },
 867         {
 868                 /*
 869                  * SLC CHIPFANCIER USB drives
 870                  * PR: usb/234503 (RC10 right, RC16 wrong)
 871                  * 16GB, 32GB and 128GB confirmed to have same issue
 872                  */
 873                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "*SLC", "CHIPFANCIER",
 874                  "*"}, /*quirks*/ DA_Q_NO_RC16
 875        },
 876         /* ATA/SATA devices over SAS/USB/... */
 877         {
 878                 /* Sandisk X400 */
 879                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "SanDisk SD8SB8U1*", "*" },
 880                 /*quirks*/DA_Q_128KB
 881         },
 882         {
 883                 /* Hitachi Advanced Format (4k) drives */
 884                 { T_DIRECT, SIP_MEDIA_FIXED, "Hitachi", "H??????????E3*", "*" },
 885                 /*quirks*/DA_Q_4K
 886         },
 887         {
 888                 /* Micron Advanced Format (4k) drives */
 889                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "Micron 5100 MTFDDAK*", "*" },
 890                 /*quirks*/DA_Q_4K
 891         },
 892         {
 893                 /* Samsung Advanced Format (4k) drives */
 894                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "SAMSUNG HD155UI*", "*" },
 895                 /*quirks*/DA_Q_4K
 896         },
 897         {
 898                 /* Samsung Advanced Format (4k) drives */
 899                 { T_DIRECT, SIP_MEDIA_FIXED, "SAMSUNG", "HD155UI*", "*" },
 900                 /*quirks*/DA_Q_4K
 901         },
 902         {
 903                 /* Samsung Advanced Format (4k) drives */
 904                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "SAMSUNG HD204UI*", "*" },
 905                 /*quirks*/DA_Q_4K
 906         },
 907         {
 908                 /* Samsung Advanced Format (4k) drives */
 909                 { T_DIRECT, SIP_MEDIA_FIXED, "SAMSUNG", "HD204UI*", "*" },
 910                 /*quirks*/DA_Q_4K
 911         },
 912         {
 913                 /* Seagate Barracuda Green Advanced Format (4k) drives */
 914                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "ST????DL*", "*" },
 915                 /*quirks*/DA_Q_4K
 916         },
 917         {
 918                 /* Seagate Barracuda Green Advanced Format (4k) drives */
 919                 { T_DIRECT, SIP_MEDIA_FIXED, "ST????DL", "*", "*" },
 920                 /*quirks*/DA_Q_4K
 921         },
 922         {
 923                 /* Seagate Barracuda Green Advanced Format (4k) drives */
 924                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "ST???DM*", "*" },
 925                 /*quirks*/DA_Q_4K
 926         },
 927         {
 928                 /* Seagate Barracuda Green Advanced Format (4k) drives */
 929                 { T_DIRECT, SIP_MEDIA_FIXED, "ST???DM*", "*", "*" },
 930                 /*quirks*/DA_Q_4K
 931         },
 932         {
 933                 /* Seagate Barracuda Green Advanced Format (4k) drives */
 934                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "ST????DM*", "*" },
 935                 /*quirks*/DA_Q_4K
 936         },
 937         {
 938                 /* Seagate Barracuda Green Advanced Format (4k) drives */
 939                 { T_DIRECT, SIP_MEDIA_FIXED, "ST????DM", "*", "*" },
 940                 /*quirks*/DA_Q_4K
 941         },
 942         {
 943                 /* Seagate Momentus Advanced Format (4k) drives */
 944                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "ST9500423AS*", "*" },
 945                 /*quirks*/DA_Q_4K
 946         },
 947         {
 948                 /* Seagate Momentus Advanced Format (4k) drives */
 949                 { T_DIRECT, SIP_MEDIA_FIXED, "ST950042", "3AS*", "*" },
 950                 /*quirks*/DA_Q_4K
 951         },
 952         {
 953                 /* Seagate Momentus Advanced Format (4k) drives */
 954                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "ST9500424AS*", "*" },
 955                 /*quirks*/DA_Q_4K
 956         },
 957         {
 958                 /* Seagate Momentus Advanced Format (4k) drives */
 959                 { T_DIRECT, SIP_MEDIA_FIXED, "ST950042", "4AS*", "*" },
 960                 /*quirks*/DA_Q_4K
 961         },
 962         {
 963                 /* Seagate Momentus Advanced Format (4k) drives */
 964                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "ST9640423AS*", "*" },
 965                 /*quirks*/DA_Q_4K
 966         },
 967         {
 968                 /* Seagate Momentus Advanced Format (4k) drives */
 969                 { T_DIRECT, SIP_MEDIA_FIXED, "ST964042", "3AS*", "*" },
 970                 /*quirks*/DA_Q_4K
 971         },
 972         {
 973                 /* Seagate Momentus Advanced Format (4k) drives */
 974                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "ST9640424AS*", "*" },
 975                 /*quirks*/DA_Q_4K
 976         },
 977         {
 978                 /* Seagate Momentus Advanced Format (4k) drives */
 979                 { T_DIRECT, SIP_MEDIA_FIXED, "ST964042", "4AS*", "*" },
 980                 /*quirks*/DA_Q_4K
 981         },
 982         {
 983                 /* Seagate Momentus Advanced Format (4k) drives */
 984                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "ST9750420AS*", "*" },
 985                 /*quirks*/DA_Q_4K
 986         },
 987         {
 988                 /* Seagate Momentus Advanced Format (4k) drives */
 989                 { T_DIRECT, SIP_MEDIA_FIXED, "ST975042", "0AS*", "*" },
 990                 /*quirks*/DA_Q_4K
 991         },
 992         {
 993                 /* Seagate Momentus Advanced Format (4k) drives */
 994                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "ST9750422AS*", "*" },
 995                 /*quirks*/DA_Q_4K
 996         },
 997         {
 998                 /* Seagate Momentus Advanced Format (4k) drives */
 999                 { T_DIRECT, SIP_MEDIA_FIXED, "ST975042", "2AS*", "*" },
1000                 /*quirks*/DA_Q_4K
1001         },
1002         {
1003                 /* Seagate Momentus Advanced Format (4k) drives */
1004                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "ST9750423AS*", "*" },
1005                 /*quirks*/DA_Q_4K
1006         },
1007         {
1008                 /* Seagate Momentus Advanced Format (4k) drives */
1009                 { T_DIRECT, SIP_MEDIA_FIXED, "ST975042", "3AS*", "*" },
1010                 /*quirks*/DA_Q_4K
1011         },
1012         {
1013                 /* Seagate Momentus Thin Advanced Format (4k) drives */
1014                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "ST???LT*", "*" },
1015                 /*quirks*/DA_Q_4K
1016         },
1017         {
1018                 /* Seagate Momentus Thin Advanced Format (4k) drives */
1019                 { T_DIRECT, SIP_MEDIA_FIXED, "ST???LT*", "*", "*" },
1020                 /*quirks*/DA_Q_4K
1021         },
1022         {
1023                 /* WDC Caviar Green Advanced Format (4k) drives */
1024                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "WDC WD????RS*", "*" },
1025                 /*quirks*/DA_Q_4K
1026         },
1027         {
1028                 /* WDC Caviar Green Advanced Format (4k) drives */
1029                 { T_DIRECT, SIP_MEDIA_FIXED, "WDC WD??", "??RS*", "*" },
1030                 /*quirks*/DA_Q_4K
1031         },
1032         {
1033                 /* WDC Caviar Green Advanced Format (4k) drives */
1034                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "WDC WD????RX*", "*" },
1035                 /*quirks*/DA_Q_4K
1036         },
1037         {
1038                 /* WDC Caviar Green Advanced Format (4k) drives */
1039                 { T_DIRECT, SIP_MEDIA_FIXED, "WDC WD??", "??RX*", "*" },
1040                 /*quirks*/DA_Q_4K
1041         },
1042         {
1043                 /* WDC Caviar Green Advanced Format (4k) drives */
1044                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "WDC WD??????RS*", "*" },
1045                 /*quirks*/DA_Q_4K
1046         },
1047         {
1048                 /* WDC Caviar Green Advanced Format (4k) drives */
1049                 { T_DIRECT, SIP_MEDIA_FIXED, "WDC WD??", "????RS*", "*" },
1050                 /*quirks*/DA_Q_4K
1051         },
1052         {
1053                 /* WDC Caviar Green Advanced Format (4k) drives */
1054                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "WDC WD??????RX*", "*" },
1055                 /*quirks*/DA_Q_4K
1056         },
1057         {
1058                 /* WDC Caviar Green Advanced Format (4k) drives */
1059                 { T_DIRECT, SIP_MEDIA_FIXED, "WDC WD??", "????RX*", "*" },
1060                 /*quirks*/DA_Q_4K
1061         },
1062         {
1063                 /* WDC Scorpio Black Advanced Format (4k) drives */
1064                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "WDC WD???PKT*", "*" },
1065                 /*quirks*/DA_Q_4K
1066         },
1067         {
1068                 /* WDC Scorpio Black Advanced Format (4k) drives */
1069                 { T_DIRECT, SIP_MEDIA_FIXED, "WDC WD??", "?PKT*", "*" },
1070                 /*quirks*/DA_Q_4K
1071         },
1072         {
1073                 /* WDC Scorpio Black Advanced Format (4k) drives */
1074                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "WDC WD?????PKT*", "*" },
1075                 /*quirks*/DA_Q_4K
1076         },
1077         {
1078                 /* WDC Scorpio Black Advanced Format (4k) drives */
1079                 { T_DIRECT, SIP_MEDIA_FIXED, "WDC WD??", "???PKT*", "*" },
1080                 /*quirks*/DA_Q_4K
1081         },
1082         {
1083                 /* WDC Scorpio Blue Advanced Format (4k) drives */
1084                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "WDC WD???PVT*", "*" },
1085                 /*quirks*/DA_Q_4K
1086         },
1087         {
1088                 /* WDC Scorpio Blue Advanced Format (4k) drives */
1089                 { T_DIRECT, SIP_MEDIA_FIXED, "WDC WD??", "?PVT*", "*" },
1090                 /*quirks*/DA_Q_4K
1091         },
1092         {
1093                 /* WDC Scorpio Blue Advanced Format (4k) drives */
1094                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "WDC WD?????PVT*", "*" },
1095                 /*quirks*/DA_Q_4K
1096         },
1097         {
1098                 /* WDC Scorpio Blue Advanced Format (4k) drives */
1099                 { T_DIRECT, SIP_MEDIA_FIXED, "WDC WD??", "???PVT*", "*" },
1100                 /*quirks*/DA_Q_4K
1101         },
1102         {
1103                 /*
1104                  * Olympus digital cameras (C-3040ZOOM, C-2040ZOOM, C-1)
1105                  * PR: usb/97472
1106                  */
1107                 { T_DIRECT, SIP_MEDIA_REMOVABLE, "OLYMPUS", "C*", "*"},
1108                 /*quirks*/ DA_Q_NO_6_BYTE | DA_Q_NO_SYNC_CACHE
1109         },
1110         {
1111                 /*
1112                  * Olympus digital cameras (D-370)
1113                  * PR: usb/97472
1114                  */
1115                 { T_DIRECT, SIP_MEDIA_REMOVABLE, "OLYMPUS", "D*", "*"},
1116                 /*quirks*/ DA_Q_NO_6_BYTE
1117         },
1118         {
1119                 /*
1120                  * Olympus digital cameras (E-100RS, E-10).
1121                  * PR: usb/97472
1122                  */
1123                 { T_DIRECT, SIP_MEDIA_REMOVABLE, "OLYMPUS", "E*", "*"},
1124                 /*quirks*/ DA_Q_NO_6_BYTE | DA_Q_NO_SYNC_CACHE
1125         },
1126         {
1127                 /*
1128                  * Olympus FE-210 camera
1129                  */
1130                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "OLYMPUS", "FE210*",
1131                 "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
1132         },
1133         {
1134                 /*
1135                 * Pentax Digital Camera
1136                 * PR: usb/93389
1137                 */
1138                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "PENTAX", "DIGITAL CAMERA",
1139                 "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
1140         },
1141         {
1142                 /*
1143                  * LG UP3S MP3 player
1144                  */
1145                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "LG", "UP3S",
1146                 "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
1147         },
1148         {
1149                 /*
1150                  * Laser MP3-2GA13 MP3 player
1151                  */
1152                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "USB 2.0", "(HS) Flash Disk",
1153                 "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
1154         },
1155         {
1156                 /*
1157                  * LaCie external 250GB Hard drive des by Porsche
1158                  * Submitted by: Ben Stuyts <ben@altesco.nl>
1159                  * PR: 121474
1160                  */
1161                 {T_DIRECT, SIP_MEDIA_FIXED, "SAMSUNG", "HM250JI", "*"},
1162                 /*quirks*/ DA_Q_NO_SYNC_CACHE
1163         },
1164         /* SATA SSDs */
1165         {
1166                 /*
1167                  * Corsair Force 2 SSDs
1168                  * 4k optimised & trim only works in 4k requests + 4k aligned
1169                  */
1170                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "Corsair CSSD-F*", "*" },
1171                 /*quirks*/DA_Q_4K
1172         },
1173         {
1174                 /*
1175                  * Corsair Force 3 SSDs
1176                  * 4k optimised & trim only works in 4k requests + 4k aligned
1177                  */
1178                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "Corsair Force 3*", "*" },
1179                 /*quirks*/DA_Q_4K
1180         },
1181         {
1182                 /*
1183                  * Corsair Neutron GTX SSDs
1184                  * 4k optimised & trim only works in 4k requests + 4k aligned
1185                  */
1186                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair Neutron GTX*", "*" },
1187                 /*quirks*/DA_Q_4K
1188         },
1189         {
1190                 /*
1191                  * Corsair Force GT & GS SSDs
1192                  * 4k optimised & trim only works in 4k requests + 4k aligned
1193                  */
1194                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "Corsair Force G*", "*" },
1195                 /*quirks*/DA_Q_4K
1196         },
1197         {
1198                 /*
1199                  * Crucial M4 SSDs
1200                  * 4k optimised & trim only works in 4k requests + 4k aligned
1201                  */
1202                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "M4-CT???M4SSD2*", "*" },
1203                 /*quirks*/DA_Q_4K
1204         },
1205         {
1206                 /*
1207                  * Crucial RealSSD C300 SSDs
1208                  * 4k optimised
1209                  */
1210                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "C300-CTFDDAC???MAG*",
1211                 "*" }, /*quirks*/DA_Q_4K
1212         },
1213         {
1214                 /*
1215                  * Intel 320 Series SSDs
1216                  * 4k optimised & trim only works in 4k requests + 4k aligned
1217                  */
1218                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "INTEL SSDSA2CW*", "*" },
1219                 /*quirks*/DA_Q_4K
1220         },
1221         {
1222                 /*
1223                  * Intel 330 Series SSDs
1224                  * 4k optimised & trim only works in 4k requests + 4k aligned
1225                  */
1226                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "INTEL SSDSC2CT*", "*" },
1227                 /*quirks*/DA_Q_4K
1228         },
1229         {
1230                 /*
1231                  * Intel 510 Series SSDs
1232                  * 4k optimised & trim only works in 4k requests + 4k aligned
1233                  */
1234                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "INTEL SSDSC2MH*", "*" },
1235                 /*quirks*/DA_Q_4K
1236         },
1237         {
1238                 /*
1239                  * Intel 520 Series SSDs
1240                  * 4k optimised & trim only works in 4k requests + 4k aligned
1241                  */
1242                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "INTEL SSDSC2BW*", "*" },
1243                 /*quirks*/DA_Q_4K
1244         },
1245         {
1246                 /*
1247                  * Intel S3610 Series SSDs
1248                  * 4k optimised & trim only works in 4k requests + 4k aligned
1249                  */
1250                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "INTEL SSDSC2BX*", "*" },
1251                 /*quirks*/DA_Q_4K
1252         },
1253         {
1254                 /*
1255                  * Intel X25-M Series SSDs
1256                  * 4k optimised & trim only works in 4k requests + 4k aligned
1257                  */
1258                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "INTEL SSDSA2M*", "*" },
1259                 /*quirks*/DA_Q_4K
1260         },
1261         {
1262                 /*
1263                  * Kingston E100 Series SSDs
1264                  * 4k optimised & trim only works in 4k requests + 4k aligned
1265                  */
1266                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "KINGSTON SE100S3*", "*" },
1267                 /*quirks*/DA_Q_4K
1268         },
1269         {
1270                 /*
1271                  * Kingston HyperX 3k SSDs
1272                  * 4k optimised & trim only works in 4k requests + 4k aligned
1273                  */
1274                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "KINGSTON SH103S3*", "*" },
1275                 /*quirks*/DA_Q_4K
1276         },
1277         {
1278                 /*
1279                  * Marvell SSDs (entry taken from OpenSolaris)
1280                  * 4k optimised & trim only works in 4k requests + 4k aligned
1281                  */
1282                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "MARVELL SD88SA02*", "*" },
1283                 /*quirks*/DA_Q_4K
1284         },
1285         {
1286                 /*
1287                  * OCZ Agility 2 SSDs
1288                  * 4k optimised & trim only works in 4k requests + 4k aligned
1289                  */
1290                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-AGILITY2*", "*" },
1291                 /*quirks*/DA_Q_4K
1292         },
1293         {
1294                 /*
1295                  * OCZ Agility 3 SSDs
1296                  * 4k optimised & trim only works in 4k requests + 4k aligned
1297                  */
1298                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "OCZ-AGILITY3*", "*" },
1299                 /*quirks*/DA_Q_4K
1300         },
1301         {
1302                 /*
1303                  * OCZ Deneva R Series SSDs
1304                  * 4k optimised & trim only works in 4k requests + 4k aligned
1305                  */
1306                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "DENRSTE251M45*", "*" },
1307                 /*quirks*/DA_Q_4K
1308         },
1309         {
1310                 /*
1311                  * OCZ Vertex 2 SSDs (inc pro series)
1312                  * 4k optimised & trim only works in 4k requests + 4k aligned
1313                  */
1314                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "OCZ?VERTEX2*", "*" },
1315                 /*quirks*/DA_Q_4K
1316         },
1317         {
1318                 /*
1319                  * OCZ Vertex 3 SSDs
1320                  * 4k optimised & trim only works in 4k requests + 4k aligned
1321                  */
1322                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "OCZ-VERTEX3*", "*" },
1323                 /*quirks*/DA_Q_4K
1324         },
1325         {
1326                 /*
1327                  * OCZ Vertex 4 SSDs
1328                  * 4k optimised & trim only works in 4k requests + 4k aligned
1329                  */
1330                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "OCZ-VERTEX4*", "*" },
1331                 /*quirks*/DA_Q_4K
1332         },
1333         {
1334                 /*
1335                  * Samsung 750 Series SSDs
1336                  * 4k optimised & trim only works in 4k requests + 4k aligned
1337                  */
1338                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "Samsung SSD 750*", "*" },
1339                 /*quirks*/DA_Q_4K
1340         },
1341         {
1342                 /*
1343                  * Samsung 830 Series SSDs
1344                  * 4k optimised & trim only works in 4k requests + 4k aligned
1345                  */
1346                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "SAMSUNG SSD 830 Series*", "*" },
1347                 /*quirks*/DA_Q_4K
1348         },
1349         {
1350                 /*
1351                  * Samsung 840 SSDs
1352                  * 4k optimised & trim only works in 4k requests + 4k aligned
1353                  */
1354                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "Samsung SSD 840*", "*" },
1355                 /*quirks*/DA_Q_4K
1356         },
1357         {
1358                 /*
1359                  * Samsung 845 SSDs
1360                  * 4k optimised & trim only works in 4k requests + 4k aligned
1361                  */
1362                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "Samsung SSD 845*", "*" },
1363                 /*quirks*/DA_Q_4K
1364         },
1365         {
1366                 /*
1367                  * Samsung 850 SSDs
1368                  * 4k optimised & trim only works in 4k requests + 4k aligned
1369                  */
1370                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "Samsung SSD 850*", "*" },
1371                 /*quirks*/DA_Q_4K
1372         },
1373         {
1374                 /*
1375                  * Samsung 843T Series SSDs (MZ7WD*)
1376                  * Samsung PM851 Series SSDs (MZ7TE*)
1377                  * Samsung PM853T Series SSDs (MZ7GE*)
1378                  * Samsung SM863 Series SSDs (MZ7KM*)
1379                  * 4k optimised
1380                  */
1381                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "SAMSUNG MZ7*", "*" },
1382                 /*quirks*/DA_Q_4K
1383         },
1384         {
1385                 /*
1386                  * Same as for SAMSUNG MZ7* but enable the quirks for SSD
1387                  * starting with MZ7* too
1388                  */
1389                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "MZ7*", "*" },
1390                 /*quirks*/DA_Q_4K
1391         },
1392         {
1393                 /*
1394                  * SuperTalent TeraDrive CT SSDs
1395                  * 4k optimised & trim only works in 4k requests + 4k aligned
1396                  */
1397                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "FTM??CT25H*", "*" },
1398                 /*quirks*/DA_Q_4K
1399         },
1400         {
1401                 /*
1402                  * XceedIOPS SATA SSDs
1403                  * 4k optimised
1404                  */
1405                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "SG9XCS2D*", "*" },
1406                 /*quirks*/DA_Q_4K
1407         },
1408         {
1409                 /*
1410                  * Hama Innostor USB-Stick
1411                  */
1412                 { T_DIRECT, SIP_MEDIA_REMOVABLE, "Innostor", "Innostor*", "*" },
1413                 /*quirks*/DA_Q_NO_RC16
1414         },
1415         {
1416                 /*
1417                  * Seagate Lamarr 8TB Shingled Magnetic Recording (SMR)
1418                  * Drive Managed SATA hard drive.  This drive doesn't report
1419                  * in firmware that it is a drive managed SMR drive.
1420                  */
1421                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "ST8000AS000[23]*", "*" },
1422                 /*quirks*/DA_Q_SMR_DM
1423         },
1424         {
1425                 /*
1426                  * MX-ES USB Drive by Mach Xtreme
1427                  */
1428                 { T_DIRECT, SIP_MEDIA_REMOVABLE, "MX", "MXUB3*", "*"},
1429                 /*quirks*/DA_Q_NO_RC16
1430         },
1431 };
1432
1433 static  disk_strategy_t dastrategy;
1434 static  dumper_t        dadump;
1435 static  periph_init_t   dainit;
1436 static  void            daasync(void *callback_arg, u_int32_t code,
1437                                 struct cam_path *path, void *arg);
1438 static  void            dasysctlinit(void *context, int pending);
1439 static  int             dasysctlsofttimeout(SYSCTL_HANDLER_ARGS);
1440 static  int             dacmdsizesysctl(SYSCTL_HANDLER_ARGS);
1441 static  int             dadeletemethodsysctl(SYSCTL_HANDLER_ARGS);
1442 static  int             dazonemodesysctl(SYSCTL_HANDLER_ARGS);
1443 static  int             dazonesupsysctl(SYSCTL_HANDLER_ARGS);
1444 static  int             dadeletemaxsysctl(SYSCTL_HANDLER_ARGS);
1445 static  void            dadeletemethodset(struct da_softc *softc,
1446                                           da_delete_methods delete_method);
1447 static  off_t           dadeletemaxsize(struct da_softc *softc,
1448                                         da_delete_methods delete_method);
1449 static  void            dadeletemethodchoose(struct da_softc *softc,
1450                                              da_delete_methods default_method);
1451 static  void            daprobedone(struct cam_periph *periph, union ccb *ccb);
1452
1453 static  periph_ctor_t   daregister;
1454 static  periph_dtor_t   dacleanup;
1455 static  periph_start_t  dastart;
1456 static  periph_oninv_t  daoninvalidate;
1457 static  void            dazonedone(struct cam_periph *periph, union ccb *ccb);
1458 static  void            dadone(struct cam_periph *periph,
1459                                union ccb *done_ccb);
1460 static void             dadone_probewp(struct cam_periph *periph,
1461                                        union ccb *done_ccb);
1462 static void             dadone_proberc(struct cam_periph *periph,
1463                                        union ccb *done_ccb);
1464 static void             dadone_probelbp(struct cam_periph *periph,
1465                                         union ccb *done_ccb);
1466 static void             dadone_probeblklimits(struct cam_periph *periph,
1467                                               union ccb *done_ccb);
1468 static void             dadone_probebdc(struct cam_periph *periph,
1469                                         union ccb *done_ccb);
1470 static void             dadone_probeata(struct cam_periph *periph,
1471                                         union ccb *done_ccb);
1472 static void             dadone_probeatalogdir(struct cam_periph *periph,
1473                                               union ccb *done_ccb);
1474 static void             dadone_probeataiddir(struct cam_periph *periph,
1475                                              union ccb *done_ccb);
1476 static void             dadone_probeatasup(struct cam_periph *periph,
1477                                            union ccb *done_ccb);
1478 static void             dadone_probeatazone(struct cam_periph *periph,
1479                                             union ccb *done_ccb);
1480 static void             dadone_probezone(struct cam_periph *periph,
1481                                          union ccb *done_ccb);
1482 static void             dadone_tur(struct cam_periph *periph,
1483                                    union ccb *done_ccb);
1484 static  int             daerror(union ccb *ccb, u_int32_t cam_flags,
1485                                 u_int32_t sense_flags);
1486 static void             daprevent(struct cam_periph *periph, int action);
1487 static void             dareprobe(struct cam_periph *periph);
1488 static void             dasetgeom(struct cam_periph *periph, uint32_t block_len,
1489                                   uint64_t maxsector,
1490                                   struct scsi_read_capacity_data_long *rcaplong,
1491                                   size_t rcap_size);
1492 static timeout_t        dasendorderedtag;
1493 static void             dashutdown(void *arg, int howto);
1494 static timeout_t        damediapoll;
1495
1496 #ifndef DA_DEFAULT_POLL_PERIOD
1497 #define DA_DEFAULT_POLL_PERIOD  3
1498 #endif
1499
1500 #ifndef DA_DEFAULT_TIMEOUT
1501 #define DA_DEFAULT_TIMEOUT 60   /* Timeout in seconds */
1502 #endif
1503
1504 #ifndef DA_DEFAULT_SOFTTIMEOUT
1505 #define DA_DEFAULT_SOFTTIMEOUT  0
1506 #endif
1507
1508 #ifndef DA_DEFAULT_RETRY
1509 #define DA_DEFAULT_RETRY        4
1510 #endif
1511
1512 #ifndef DA_DEFAULT_SEND_ORDERED
1513 #define DA_DEFAULT_SEND_ORDERED 1
1514 #endif
1515
1516 static int da_poll_period = DA_DEFAULT_POLL_PERIOD;
1517 static int da_retry_count = DA_DEFAULT_RETRY;
1518 static int da_default_timeout = DA_DEFAULT_TIMEOUT;
1519 static sbintime_t da_default_softtimeout = DA_DEFAULT_SOFTTIMEOUT;
1520 static int da_send_ordered = DA_DEFAULT_SEND_ORDERED;
1521 static int da_disable_wp_detection = 0;
1522
1523 static SYSCTL_NODE(_kern_cam, OID_AUTO, da, CTLFLAG_RD, 0,
1524             "CAM Direct Access Disk driver");
1525 SYSCTL_INT(_kern_cam_da, OID_AUTO, poll_period, CTLFLAG_RWTUN,
1526            &da_poll_period, 0, "Media polling period in seconds");
1527 SYSCTL_INT(_kern_cam_da, OID_AUTO, retry_count, CTLFLAG_RWTUN,
1528            &da_retry_count, 0, "Normal I/O retry count");
1529 SYSCTL_INT(_kern_cam_da, OID_AUTO, default_timeout, CTLFLAG_RWTUN,
1530            &da_default_timeout, 0, "Normal I/O timeout (in seconds)");
1531 SYSCTL_INT(_kern_cam_da, OID_AUTO, send_ordered, CTLFLAG_RWTUN,
1532            &da_send_ordered, 0, "Send Ordered Tags");
1533 SYSCTL_INT(_kern_cam_da, OID_AUTO, disable_wp_detection, CTLFLAG_RWTUN,
1534            &da_disable_wp_detection, 0,
1535            "Disable detection of write-protected disks");
1536
1537 SYSCTL_PROC(_kern_cam_da, OID_AUTO, default_softtimeout,
1538     CTLTYPE_UINT | CTLFLAG_RW, NULL, 0, dasysctlsofttimeout, "I",
1539     "Soft I/O timeout (ms)");
1540 TUNABLE_INT64("kern.cam.da.default_softtimeout", &da_default_softtimeout);
1541
1542 /*
1543  * DA_ORDEREDTAG_INTERVAL determines how often, relative
1544  * to the default timeout, we check to see whether an ordered
1545  * tagged transaction is appropriate to prevent simple tag
1546  * starvation.  Since we'd like to ensure that there is at least
1547  * 1/2 of the timeout length left for a starved transaction to
1548  * complete after we've sent an ordered tag, we must poll at least
1549  * four times in every timeout period.  This takes care of the worst
1550  * case where a starved transaction starts during an interval that
1551  * meets the requirement "don't send an ordered tag" test so it takes
1552  * us two intervals to determine that a tag must be sent.
1553  */
1554 #ifndef DA_ORDEREDTAG_INTERVAL
1555 #define DA_ORDEREDTAG_INTERVAL 4
1556 #endif
1557
1558 static struct periph_driver dadriver =
1559 {
1560         dainit, "da",
1561         TAILQ_HEAD_INITIALIZER(dadriver.units), /* generation */ 0
1562 };
1563
1564 PERIPHDRIVER_DECLARE(da, dadriver);
1565
1566 static MALLOC_DEFINE(M_SCSIDA, "scsi_da", "scsi_da buffers");
1567
1568 /*
1569  * This driver takes out references / holds in well defined pairs, never
1570  * recursively. These macros / inline functions enforce those rules. They
1571  * are only enabled with DA_TRACK_REFS or INVARIANTS. If DA_TRACK_REFS is
1572  * defined to be 2 or larger, the tracking also includes debug printfs.
1573  */
1574 #if defined(DA_TRACK_REFS) || defined(INVARIANTS)
1575
1576 #ifndef DA_TRACK_REFS
1577 #define DA_TRACK_REFS 1
1578 #endif
1579
1580 #if DA_TRACK_REFS > 1
1581 static const char *da_ref_text[] = {
1582         "bogus",
1583         "open",
1584         "open hold",
1585         "close hold",
1586         "reprobe hold",
1587         "Test Unit Ready",
1588         "Geom",
1589         "sysctl",
1590         "reprobe",
1591         "max -- also bogus"
1592 };
1593
1594 #define DA_PERIPH_PRINT(periph, msg, args...)           \
1595         CAM_PERIPH_PRINT(periph, msg, ##args)
1596 #else
1597 #define DA_PERIPH_PRINT(periph, msg, args...)
1598 #endif
1599
1600 static inline void
1601 token_sanity(da_ref_token token)
1602 {
1603         if ((unsigned)token >= DA_REF_MAX)
1604                 panic("Bad token value passed in %d\n", token);
1605 }
1606
1607 static inline int
1608 da_periph_hold(struct cam_periph *periph, int priority, da_ref_token token)
1609 {
1610         int err = cam_periph_hold(periph, priority);
1611
1612         token_sanity(token);
1613         DA_PERIPH_PRINT(periph, "Holding device %s (%d): %d\n",
1614             da_ref_text[token], token, err);
1615         if (err == 0) {
1616                 int cnt;
1617                 struct da_softc *softc = periph->softc;
1618
1619                 cnt = atomic_fetchadd_int(&softc->ref_flags[token], 1);
1620                 if (cnt != 0)
1621                         panic("Re-holding for reason %d, cnt = %d", token, cnt);
1622         }
1623         return (err);
1624 }
1625
1626 static inline void
1627 da_periph_unhold(struct cam_periph *periph, da_ref_token token)
1628 {
1629         int cnt;
1630         struct da_softc *softc = periph->softc;
1631
1632         token_sanity(token);
1633         DA_PERIPH_PRINT(periph, "Unholding device %s (%d)\n",
1634             da_ref_text[token], token);
1635         cnt = atomic_fetchadd_int(&softc->ref_flags[token], -1);
1636         if (cnt != 1)
1637                 panic("Unholding %d with cnt = %d", token, cnt);
1638         cam_periph_unhold(periph);
1639 }
1640
1641 static inline int
1642 da_periph_acquire(struct cam_periph *periph, da_ref_token token)
1643 {
1644         int err = cam_periph_acquire(periph);
1645
1646         token_sanity(token);
1647         DA_PERIPH_PRINT(periph, "acquiring device %s (%d): %d\n",
1648             da_ref_text[token], token, err);
1649         if (err == 0) {
1650                 int cnt;
1651                 struct da_softc *softc = periph->softc;
1652
1653                 cnt = atomic_fetchadd_int(&softc->ref_flags[token], 1);
1654                 if (cnt != 0)
1655                         panic("Re-refing for reason %d, cnt = %d", token, cnt);
1656         }
1657         return (err);
1658 }
1659
1660 static inline void
1661 da_periph_release(struct cam_periph *periph, da_ref_token token)
1662 {
1663         int cnt;
1664         struct da_softc *softc = periph->softc;
1665
1666         token_sanity(token);
1667         DA_PERIPH_PRINT(periph, "releasing device %s (%d)\n",
1668             da_ref_text[token], token);
1669         cnt = atomic_fetchadd_int(&softc->ref_flags[token], -1);
1670         if (cnt != 1)
1671                 panic("Releasing %d with cnt = %d", token, cnt);
1672         cam_periph_release(periph);
1673 }
1674
1675 static inline void
1676 da_periph_release_locked(struct cam_periph *periph, da_ref_token token)
1677 {
1678         int cnt;
1679         struct da_softc *softc = periph->softc;
1680
1681         token_sanity(token);
1682         DA_PERIPH_PRINT(periph, "releasing device (locked) %s (%d)\n",
1683             da_ref_text[token], token);
1684         cnt = atomic_fetchadd_int(&softc->ref_flags[token], -1);
1685         if (cnt != 1)
1686                 panic("Unholding %d with cnt = %d", token, cnt);
1687         cam_periph_release_locked(periph);
1688 }
1689
1690 #define cam_periph_hold POISON
1691 #define cam_periph_unhold POISON
1692 #define cam_periph_acquire POISON
1693 #define cam_periph_release POISON
1694 #define cam_periph_release_locked POISON
1695
1696 #else
1697 #define da_periph_hold(periph, prio, token)     cam_periph_hold((periph), (prio))
1698 #define da_periph_unhold(periph, token)         cam_periph_unhold((periph))
1699 #define da_periph_acquire(periph, token)        cam_periph_acquire((periph))
1700 #define da_periph_release(periph, token)        cam_periph_release((periph))
1701 #define da_periph_release_locked(periph, token) cam_periph_release_locked((periph))
1702 #endif
1703
1704 static int
1705 daopen(struct disk *dp)
1706 {
1707         struct cam_periph *periph;
1708         struct da_softc *softc;
1709         int error;
1710
1711         periph = (struct cam_periph *)dp->d_drv1;
1712         if (da_periph_acquire(periph, DA_REF_OPEN) != 0) {
1713                 return (ENXIO);
1714         }
1715
1716         cam_periph_lock(periph);
1717         if ((error = da_periph_hold(periph, PRIBIO|PCATCH, DA_REF_OPEN_HOLD)) != 0) {
1718                 cam_periph_unlock(periph);
1719                 da_periph_release(periph, DA_REF_OPEN);
1720                 return (error);
1721         }
1722
1723         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE | CAM_DEBUG_PERIPH,
1724             ("daopen\n"));
1725
1726         softc = (struct da_softc *)periph->softc;
1727         dareprobe(periph);
1728
1729         /* Wait for the disk size update.  */
1730         error = cam_periph_sleep(periph, &softc->disk->d_mediasize, PRIBIO,
1731             "dareprobe", 0);
1732         if (error != 0)
1733                 xpt_print(periph->path, "unable to retrieve capacity data\n");
1734
1735         if (periph->flags & CAM_PERIPH_INVALID)
1736                 error = ENXIO;
1737
1738         if (error == 0 && (softc->flags & DA_FLAG_PACK_REMOVABLE) != 0 &&
1739             (softc->quirks & DA_Q_NO_PREVENT) == 0)
1740                 daprevent(periph, PR_PREVENT);
1741
1742         if (error == 0) {
1743                 softc->flags &= ~DA_FLAG_PACK_INVALID;
1744                 softc->flags |= DA_FLAG_OPEN;
1745         }
1746
1747         da_periph_unhold(periph, DA_REF_OPEN_HOLD);
1748         cam_periph_unlock(periph);
1749
1750         if (error != 0)
1751                 da_periph_release(periph, DA_REF_OPEN);
1752
1753         return (error);
1754 }
1755
1756 static int
1757 daclose(struct disk *dp)
1758 {
1759         struct  cam_periph *periph;
1760         struct  da_softc *softc;
1761         union   ccb *ccb;
1762
1763         periph = (struct cam_periph *)dp->d_drv1;
1764         softc = (struct da_softc *)periph->softc;
1765         cam_periph_lock(periph);
1766         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE | CAM_DEBUG_PERIPH,
1767             ("daclose\n"));
1768
1769         if (da_periph_hold(periph, PRIBIO, DA_REF_CLOSE_HOLD) == 0) {
1770
1771                 /* Flush disk cache. */
1772                 if ((softc->flags & DA_FLAG_DIRTY) != 0 &&
1773                     (softc->quirks & DA_Q_NO_SYNC_CACHE) == 0 &&
1774                     (softc->flags & DA_FLAG_PACK_INVALID) == 0) {
1775                         ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL);
1776                         scsi_synchronize_cache(&ccb->csio, /*retries*/1,
1777                             /*cbfcnp*/NULL, MSG_SIMPLE_Q_TAG,
1778                             /*begin_lba*/0, /*lb_count*/0, SSD_FULL_SIZE,
1779                             5 * 60 * 1000);
1780                         cam_periph_runccb(ccb, daerror, /*cam_flags*/0,
1781                             /*sense_flags*/SF_RETRY_UA | SF_QUIET_IR,
1782                             softc->disk->d_devstat);
1783                         softc->flags &= ~DA_FLAG_DIRTY;
1784                         xpt_release_ccb(ccb);
1785                 }
1786
1787                 /* Allow medium removal. */
1788                 if ((softc->flags & DA_FLAG_PACK_REMOVABLE) != 0 &&
1789                     (softc->quirks & DA_Q_NO_PREVENT) == 0)
1790                         daprevent(periph, PR_ALLOW);
1791
1792                 da_periph_unhold(periph, DA_REF_CLOSE_HOLD);
1793         }
1794
1795         /*
1796          * If we've got removeable media, mark the blocksize as
1797          * unavailable, since it could change when new media is
1798          * inserted.
1799          */
1800         if ((softc->flags & DA_FLAG_PACK_REMOVABLE) != 0)
1801                 softc->disk->d_devstat->flags |= DEVSTAT_BS_UNAVAILABLE;
1802
1803         softc->flags &= ~DA_FLAG_OPEN;
1804         while (softc->refcount != 0)
1805                 cam_periph_sleep(periph, &softc->refcount, PRIBIO, "daclose", 1);
1806         cam_periph_unlock(periph);
1807         da_periph_release(periph, DA_REF_OPEN);
1808         return (0);
1809 }
1810
1811 static void
1812 daschedule(struct cam_periph *periph)
1813 {
1814         struct da_softc *softc = (struct da_softc *)periph->softc;
1815
1816         if (softc->state != DA_STATE_NORMAL)
1817                 return;
1818
1819         cam_iosched_schedule(softc->cam_iosched, periph);
1820 }
1821
1822 /*
1823  * Actually translate the requested transfer into one the physical driver
1824  * can understand.  The transfer is described by a buf and will include
1825  * only one physical transfer.
1826  */
1827 static void
1828 dastrategy(struct bio *bp)
1829 {
1830         struct cam_periph *periph;
1831         struct da_softc *softc;
1832
1833         periph = (struct cam_periph *)bp->bio_disk->d_drv1;
1834         softc = (struct da_softc *)periph->softc;
1835
1836         cam_periph_lock(periph);
1837
1838         /*
1839          * If the device has been made invalid, error out
1840          */
1841         if ((softc->flags & DA_FLAG_PACK_INVALID)) {
1842                 cam_periph_unlock(periph);
1843                 biofinish(bp, NULL, ENXIO);
1844                 return;
1845         }
1846
1847         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dastrategy(%p)\n", bp));
1848
1849         /*
1850          * Zone commands must be ordered, because they can depend on the
1851          * effects of previously issued commands, and they may affect
1852          * commands after them.
1853          */
1854         if (bp->bio_cmd == BIO_ZONE)
1855                 bp->bio_flags |= BIO_ORDERED;
1856
1857         /*
1858          * Place it in the queue of disk activities for this disk
1859          */
1860         cam_iosched_queue_work(softc->cam_iosched, bp);
1861
1862         /*
1863          * Schedule ourselves for performing the work.
1864          */
1865         daschedule(periph);
1866         cam_periph_unlock(periph);
1867
1868         return;
1869 }
1870
1871 static int
1872 dadump(void *arg, void *virtual, vm_offset_t physical, off_t offset, size_t length)
1873 {
1874         struct      cam_periph *periph;
1875         struct      da_softc *softc;
1876         u_int       secsize;
1877         struct      ccb_scsiio csio;
1878         struct      disk *dp;
1879         int         error = 0;
1880
1881         dp = arg;
1882         periph = dp->d_drv1;
1883         softc = (struct da_softc *)periph->softc;
1884         secsize = softc->params.secsize;
1885
1886         if ((softc->flags & DA_FLAG_PACK_INVALID) != 0)
1887                 return (ENXIO);
1888
1889         memset(&csio, 0, sizeof(csio));
1890         if (length > 0) {
1891                 xpt_setup_ccb(&csio.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
1892                 csio.ccb_h.ccb_state = DA_CCB_DUMP;
1893                 scsi_read_write(&csio,
1894                                 /*retries*/0,
1895                                 /*cbfcnp*/NULL,
1896                                 MSG_ORDERED_Q_TAG,
1897                                 /*read*/SCSI_RW_WRITE,
1898                                 /*byte2*/0,
1899                                 /*minimum_cmd_size*/ softc->minimum_cmd_size,
1900                                 offset / secsize,
1901                                 length / secsize,
1902                                 /*data_ptr*/(u_int8_t *) virtual,
1903                                 /*dxfer_len*/length,
1904                                 /*sense_len*/SSD_FULL_SIZE,
1905                                 da_default_timeout * 1000);
1906                 error = cam_periph_runccb((union ccb *)&csio, cam_periph_error,
1907                     0, SF_NO_RECOVERY | SF_NO_RETRY, NULL);
1908                 if (error != 0)
1909                         printf("Aborting dump due to I/O error.\n");
1910                 return (error);
1911         }
1912
1913         /*
1914          * Sync the disk cache contents to the physical media.
1915          */
1916         if ((softc->quirks & DA_Q_NO_SYNC_CACHE) == 0) {
1917
1918                 xpt_setup_ccb(&csio.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
1919                 csio.ccb_h.ccb_state = DA_CCB_DUMP;
1920                 scsi_synchronize_cache(&csio,
1921                                        /*retries*/0,
1922                                        /*cbfcnp*/NULL,
1923                                        MSG_SIMPLE_Q_TAG,
1924                                        /*begin_lba*/0,/* Cover the whole disk */
1925                                        /*lb_count*/0,
1926                                        SSD_FULL_SIZE,
1927                                        5 * 1000);
1928                 error = cam_periph_runccb((union ccb *)&csio, cam_periph_error,
1929                     0, SF_NO_RECOVERY | SF_NO_RETRY, NULL);
1930                 if (error != 0)
1931                         xpt_print(periph->path, "Synchronize cache failed\n");
1932         }
1933         return (error);
1934 }
1935
1936 static int
1937 dagetattr(struct bio *bp)
1938 {
1939         int ret;
1940         struct cam_periph *periph;
1941
1942         periph = (struct cam_periph *)bp->bio_disk->d_drv1;
1943         cam_periph_lock(periph);
1944         ret = xpt_getattr(bp->bio_data, bp->bio_length, bp->bio_attribute,
1945             periph->path);
1946         cam_periph_unlock(periph);
1947         if (ret == 0)
1948                 bp->bio_completed = bp->bio_length;
1949         return ret;
1950 }
1951
1952 static void
1953 dainit(void)
1954 {
1955         cam_status status;
1956
1957         /*
1958          * Install a global async callback.  This callback will
1959          * receive async callbacks like "new device found".
1960          */
1961         status = xpt_register_async(AC_FOUND_DEVICE, daasync, NULL, NULL);
1962
1963         if (status != CAM_REQ_CMP) {
1964                 printf("da: Failed to attach master async callback "
1965                        "due to status 0x%x!\n", status);
1966         } else if (da_send_ordered) {
1967
1968                 /* Register our shutdown event handler */
1969                 if ((EVENTHANDLER_REGISTER(shutdown_post_sync, dashutdown,
1970                                            NULL, SHUTDOWN_PRI_DEFAULT)) == NULL)
1971                     printf("dainit: shutdown event registration failed!\n");
1972         }
1973 }
1974
1975 /*
1976  * Callback from GEOM, called when it has finished cleaning up its
1977  * resources.
1978  */
1979 static void
1980 dadiskgonecb(struct disk *dp)
1981 {
1982         struct cam_periph *periph;
1983
1984         periph = (struct cam_periph *)dp->d_drv1;
1985         da_periph_release(periph, DA_REF_GEOM);
1986 }
1987
1988 static void
1989 daoninvalidate(struct cam_periph *periph)
1990 {
1991         struct da_softc *softc;
1992
1993         cam_periph_assert(periph, MA_OWNED);
1994         softc = (struct da_softc *)periph->softc;
1995
1996         /*
1997          * De-register any async callbacks.
1998          */
1999         xpt_register_async(0, daasync, periph, periph->path);
2000
2001         softc->flags |= DA_FLAG_PACK_INVALID;
2002 #ifdef CAM_IO_STATS
2003         softc->invalidations++;
2004 #endif
2005
2006         /*
2007          * Return all queued I/O with ENXIO.
2008          * XXX Handle any transactions queued to the card
2009          *     with XPT_ABORT_CCB.
2010          */
2011         cam_iosched_flush(softc->cam_iosched, NULL, ENXIO);
2012
2013         /*
2014          * Tell GEOM that we've gone away, we'll get a callback when it is
2015          * done cleaning up its resources.
2016          */
2017         disk_gone(softc->disk);
2018 }
2019
2020 static void
2021 dacleanup(struct cam_periph *periph)
2022 {
2023         struct da_softc *softc;
2024
2025         softc = (struct da_softc *)periph->softc;
2026
2027         cam_periph_unlock(periph);
2028
2029         cam_iosched_fini(softc->cam_iosched);
2030
2031         /*
2032          * If we can't free the sysctl tree, oh well...
2033          */
2034         if ((softc->flags & DA_FLAG_SCTX_INIT) != 0) {
2035 #ifdef CAM_IO_STATS
2036                 if (sysctl_ctx_free(&softc->sysctl_stats_ctx) != 0)
2037                         xpt_print(periph->path,
2038                             "can't remove sysctl stats context\n");
2039 #endif
2040                 if (sysctl_ctx_free(&softc->sysctl_ctx) != 0)
2041                         xpt_print(periph->path,
2042                             "can't remove sysctl context\n");
2043         }
2044
2045         callout_drain(&softc->mediapoll_c);
2046         disk_destroy(softc->disk);
2047         callout_drain(&softc->sendordered_c);
2048         free(softc, M_DEVBUF);
2049         cam_periph_lock(periph);
2050 }
2051
2052 static void
2053 daasync(void *callback_arg, u_int32_t code,
2054         struct cam_path *path, void *arg)
2055 {
2056         struct cam_periph *periph;
2057         struct da_softc *softc;
2058
2059         periph = (struct cam_periph *)callback_arg;
2060         switch (code) {
2061         case AC_FOUND_DEVICE:   /* callback to create periph, no locking yet */
2062         {
2063                 struct ccb_getdev *cgd;
2064                 cam_status status;
2065
2066                 cgd = (struct ccb_getdev *)arg;
2067                 if (cgd == NULL)
2068                         break;
2069
2070                 if (cgd->protocol != PROTO_SCSI)
2071                         break;
2072                 if (SID_QUAL(&cgd->inq_data) != SID_QUAL_LU_CONNECTED)
2073                         break;
2074                 if (SID_TYPE(&cgd->inq_data) != T_DIRECT
2075                     && SID_TYPE(&cgd->inq_data) != T_RBC
2076                     && SID_TYPE(&cgd->inq_data) != T_OPTICAL
2077                     && SID_TYPE(&cgd->inq_data) != T_ZBC_HM)
2078                         break;
2079
2080                 /*
2081                  * Allocate a peripheral instance for
2082                  * this device and start the probe
2083                  * process.
2084                  */
2085                 status = cam_periph_alloc(daregister, daoninvalidate,
2086                                           dacleanup, dastart,
2087                                           "da", CAM_PERIPH_BIO,
2088                                           path, daasync,
2089                                           AC_FOUND_DEVICE, cgd);
2090
2091                 if (status != CAM_REQ_CMP
2092                  && status != CAM_REQ_INPROG)
2093                         printf("daasync: Unable to attach to new device "
2094                                 "due to status 0x%x\n", status);
2095                 return;
2096         }
2097         case AC_ADVINFO_CHANGED:        /* Doesn't touch periph */
2098         {
2099                 uintptr_t buftype;
2100
2101                 buftype = (uintptr_t)arg;
2102                 if (buftype == CDAI_TYPE_PHYS_PATH) {
2103                         struct da_softc *softc;
2104
2105                         softc = periph->softc;
2106                         disk_attr_changed(softc->disk, "GEOM::physpath",
2107                                           M_NOWAIT);
2108                 }
2109                 break;
2110         }
2111         case AC_UNIT_ATTENTION:
2112         {
2113                 union ccb *ccb;
2114                 int error_code, sense_key, asc, ascq;
2115
2116                 softc = (struct da_softc *)periph->softc;
2117                 ccb = (union ccb *)arg;
2118
2119                 /*
2120                  * Handle all UNIT ATTENTIONs except our own, as they will be
2121                  * handled by daerror(). Since this comes from a different periph,
2122                  * that periph's lock is held, not ours, so we have to take it ours
2123                  * out to touch softc flags.
2124                  */
2125                 if (xpt_path_periph(ccb->ccb_h.path) != periph &&
2126                     scsi_extract_sense_ccb(ccb,
2127                      &error_code, &sense_key, &asc, &ascq)) {
2128                         if (asc == 0x2A && ascq == 0x09) {
2129                                 xpt_print(ccb->ccb_h.path,
2130                                     "Capacity data has changed\n");
2131                                 cam_periph_lock(periph);
2132                                 softc->flags &= ~DA_FLAG_PROBED;
2133                                 cam_periph_unlock(periph);
2134                                 dareprobe(periph);
2135                         } else if (asc == 0x28 && ascq == 0x00) {
2136                                 cam_periph_lock(periph);
2137                                 softc->flags &= ~DA_FLAG_PROBED;
2138                                 cam_periph_unlock(periph);
2139                                 disk_media_changed(softc->disk, M_NOWAIT);
2140                         } else if (asc == 0x3F && ascq == 0x03) {
2141                                 xpt_print(ccb->ccb_h.path,
2142                                     "INQUIRY data has changed\n");
2143                                 cam_periph_lock(periph);
2144                                 softc->flags &= ~DA_FLAG_PROBED;
2145                                 cam_periph_unlock(periph);
2146                                 dareprobe(periph);
2147                         }
2148                 }
2149                 break;
2150         }
2151         case AC_SCSI_AEN:               /* Called for this path: periph locked */
2152                 /*
2153                  * Appears to be currently unused for SCSI devices, only ata SIMs
2154                  * generate this.
2155                  */
2156                 cam_periph_assert(periph, MA_OWNED);
2157                 softc = (struct da_softc *)periph->softc;
2158                 if (!cam_iosched_has_work_flags(softc->cam_iosched, DA_WORK_TUR) &&
2159                     (softc->flags & DA_FLAG_TUR_PENDING) == 0) {
2160                         if (da_periph_acquire(periph, DA_REF_TUR) == 0) {
2161                                 cam_iosched_set_work_flags(softc->cam_iosched, DA_WORK_TUR);
2162                                 daschedule(periph);
2163                         }
2164                 }
2165                 /* FALLTHROUGH */
2166         case AC_SENT_BDR:               /* Called for this path: periph locked */
2167         case AC_BUS_RESET:              /* Called for this path: periph locked */
2168         {
2169                 struct ccb_hdr *ccbh;
2170
2171                 cam_periph_assert(periph, MA_OWNED);
2172                 softc = (struct da_softc *)periph->softc;
2173                 /*
2174                  * Don't fail on the expected unit attention
2175                  * that will occur.
2176                  */
2177                 softc->flags |= DA_FLAG_RETRY_UA;
2178                 LIST_FOREACH(ccbh, &softc->pending_ccbs, periph_links.le)
2179                         ccbh->ccb_state |= DA_CCB_RETRY_UA;
2180                 break;
2181         }
2182         case AC_INQ_CHANGED:            /* Called for this path: periph locked */
2183                 cam_periph_assert(periph, MA_OWNED);
2184                 softc = (struct da_softc *)periph->softc;
2185                 softc->flags &= ~DA_FLAG_PROBED;
2186                 dareprobe(periph);
2187                 break;
2188         default:
2189                 break;
2190         }
2191         cam_periph_async(periph, code, path, arg);
2192 }
2193
2194 static void
2195 dasysctlinit(void *context, int pending)
2196 {
2197         struct cam_periph *periph;
2198         struct da_softc *softc;
2199         char tmpstr[32], tmpstr2[16];
2200         struct ccb_trans_settings cts;
2201
2202         periph = (struct cam_periph *)context;
2203         /*
2204          * periph was held for us when this task was enqueued
2205          */
2206         if (periph->flags & CAM_PERIPH_INVALID) {
2207                 da_periph_release(periph, DA_REF_SYSCTL);
2208                 return;
2209         }
2210
2211         softc = (struct da_softc *)periph->softc;
2212         snprintf(tmpstr, sizeof(tmpstr), "CAM DA unit %d", periph->unit_number);
2213         snprintf(tmpstr2, sizeof(tmpstr2), "%d", periph->unit_number);
2214
2215         sysctl_ctx_init(&softc->sysctl_ctx);
2216         cam_periph_lock(periph);
2217         softc->flags |= DA_FLAG_SCTX_INIT;
2218         cam_periph_unlock(periph);
2219         softc->sysctl_tree = SYSCTL_ADD_NODE_WITH_LABEL(&softc->sysctl_ctx,
2220                 SYSCTL_STATIC_CHILDREN(_kern_cam_da), OID_AUTO, tmpstr2,
2221                 CTLFLAG_RD, 0, tmpstr, "device_index");
2222         if (softc->sysctl_tree == NULL) {
2223                 printf("dasysctlinit: unable to allocate sysctl tree\n");
2224                 da_periph_release(periph, DA_REF_SYSCTL);
2225                 return;
2226         }
2227
2228         /*
2229          * Now register the sysctl handler, so the user can change the value on
2230          * the fly.
2231          */
2232         SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
2233                 OID_AUTO, "delete_method", CTLTYPE_STRING | CTLFLAG_RWTUN,
2234                 softc, 0, dadeletemethodsysctl, "A",
2235                 "BIO_DELETE execution method");
2236         SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
2237                 OID_AUTO, "delete_max", CTLTYPE_U64 | CTLFLAG_RW,
2238                 softc, 0, dadeletemaxsysctl, "Q",
2239                 "Maximum BIO_DELETE size");
2240         SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
2241                 OID_AUTO, "minimum_cmd_size", CTLTYPE_INT | CTLFLAG_RW,
2242                 &softc->minimum_cmd_size, 0, dacmdsizesysctl, "I",
2243                 "Minimum CDB size");
2244         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
2245                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
2246                 "trim_count", CTLFLAG_RD, &softc->trim_count,
2247                 "Total number of unmap/dsm commands sent");
2248         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
2249                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
2250                 "trim_ranges", CTLFLAG_RD, &softc->trim_ranges,
2251                 "Total number of ranges in unmap/dsm commands");
2252         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
2253                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
2254                 "trim_lbas", CTLFLAG_RD, &softc->trim_lbas,
2255                 "Total lbas in the unmap/dsm commands sent");
2256
2257         SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
2258                 OID_AUTO, "zone_mode", CTLTYPE_STRING | CTLFLAG_RD,
2259                 softc, 0, dazonemodesysctl, "A",
2260                 "Zone Mode");
2261         SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
2262                 OID_AUTO, "zone_support", CTLTYPE_STRING | CTLFLAG_RD,
2263                 softc, 0, dazonesupsysctl, "A",
2264                 "Zone Support");
2265         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
2266                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
2267                 "optimal_seq_zones", CTLFLAG_RD, &softc->optimal_seq_zones,
2268                 "Optimal Number of Open Sequential Write Preferred Zones");
2269         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
2270                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
2271                 "optimal_nonseq_zones", CTLFLAG_RD,
2272                 &softc->optimal_nonseq_zones,
2273                 "Optimal Number of Non-Sequentially Written Sequential Write "
2274                 "Preferred Zones");
2275         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
2276                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
2277                 "max_seq_zones", CTLFLAG_RD, &softc->max_seq_zones,
2278                 "Maximum Number of Open Sequential Write Required Zones");
2279
2280         SYSCTL_ADD_INT(&softc->sysctl_ctx,
2281                        SYSCTL_CHILDREN(softc->sysctl_tree),
2282                        OID_AUTO,
2283                        "error_inject",
2284                        CTLFLAG_RW,
2285                        &softc->error_inject,
2286                        0,
2287                        "error_inject leaf");
2288
2289         SYSCTL_ADD_INT(&softc->sysctl_ctx,
2290                        SYSCTL_CHILDREN(softc->sysctl_tree),
2291                        OID_AUTO,
2292                        "unmapped_io",
2293                        CTLFLAG_RD,
2294                        &softc->unmappedio,
2295                        0,
2296                        "Unmapped I/O support");
2297
2298         SYSCTL_ADD_INT(&softc->sysctl_ctx,
2299                        SYSCTL_CHILDREN(softc->sysctl_tree),
2300                        OID_AUTO,
2301                        "rotating",
2302                        CTLFLAG_RD,
2303                        &softc->rotating,
2304                        0,
2305                        "Rotating media");
2306
2307         SYSCTL_ADD_INT(&softc->sysctl_ctx,
2308                        SYSCTL_CHILDREN(softc->sysctl_tree),
2309                        OID_AUTO,
2310                        "p_type",
2311                        CTLFLAG_RD,
2312                        &softc->p_type,
2313                        0,
2314                        "DIF protection type");
2315
2316 #ifdef CAM_TEST_FAILURE
2317         SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
2318                 OID_AUTO, "invalidate", CTLTYPE_U64 | CTLFLAG_RW | CTLFLAG_MPSAFE,
2319                 periph, 0, cam_periph_invalidate_sysctl, "I",
2320                 "Write 1 to invalidate the drive immediately");
2321 #endif
2322
2323         /*
2324          * Add some addressing info.
2325          */
2326         memset(&cts, 0, sizeof (cts));
2327         xpt_setup_ccb(&cts.ccb_h, periph->path, CAM_PRIORITY_NONE);
2328         cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS;
2329         cts.type = CTS_TYPE_CURRENT_SETTINGS;
2330         cam_periph_lock(periph);
2331         xpt_action((union ccb *)&cts);
2332         cam_periph_unlock(periph);
2333         if (cts.ccb_h.status != CAM_REQ_CMP) {
2334                 da_periph_release(periph, DA_REF_SYSCTL);
2335                 return;
2336         }
2337         if (cts.protocol == PROTO_SCSI && cts.transport == XPORT_FC) {
2338                 struct ccb_trans_settings_fc *fc = &cts.xport_specific.fc;
2339                 if (fc->valid & CTS_FC_VALID_WWPN) {
2340                         softc->wwpn = fc->wwpn;
2341                         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
2342                             SYSCTL_CHILDREN(softc->sysctl_tree),
2343                             OID_AUTO, "wwpn", CTLFLAG_RD,
2344                             &softc->wwpn, "World Wide Port Name");
2345                 }
2346         }
2347
2348 #ifdef CAM_IO_STATS
2349         /*
2350          * Now add some useful stats.
2351          * XXX These should live in cam_periph and be common to all periphs
2352          */
2353         softc->sysctl_stats_tree = SYSCTL_ADD_NODE(&softc->sysctl_stats_ctx,
2354             SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO, "stats",
2355             CTLFLAG_RD, 0, "Statistics");
2356         SYSCTL_ADD_INT(&softc->sysctl_stats_ctx,
2357                        SYSCTL_CHILDREN(softc->sysctl_stats_tree),
2358                        OID_AUTO,
2359                        "errors",
2360                        CTLFLAG_RD,
2361                        &softc->errors,
2362                        0,
2363                        "Transport errors reported by the SIM");
2364         SYSCTL_ADD_INT(&softc->sysctl_stats_ctx,
2365                        SYSCTL_CHILDREN(softc->sysctl_stats_tree),
2366                        OID_AUTO,
2367                        "timeouts",
2368                        CTLFLAG_RD,
2369                        &softc->timeouts,
2370                        0,
2371                        "Device timeouts reported by the SIM");
2372         SYSCTL_ADD_INT(&softc->sysctl_stats_ctx,
2373                        SYSCTL_CHILDREN(softc->sysctl_stats_tree),
2374                        OID_AUTO,
2375                        "pack_invalidations",
2376                        CTLFLAG_RD,
2377                        &softc->invalidations,
2378                        0,
2379                        "Device pack invalidations");
2380 #endif
2381
2382         cam_iosched_sysctl_init(softc->cam_iosched, &softc->sysctl_ctx,
2383             softc->sysctl_tree);
2384
2385         da_periph_release(periph, DA_REF_SYSCTL);
2386 }
2387
2388 static int
2389 dadeletemaxsysctl(SYSCTL_HANDLER_ARGS)
2390 {
2391         int error;
2392         uint64_t value;
2393         struct da_softc *softc;
2394
2395         softc = (struct da_softc *)arg1;
2396
2397         value = softc->disk->d_delmaxsize;
2398         error = sysctl_handle_64(oidp, &value, 0, req);
2399         if ((error != 0) || (req->newptr == NULL))
2400                 return (error);
2401
2402         /* only accept values smaller than the calculated value */
2403         if (value > dadeletemaxsize(softc, softc->delete_method)) {
2404                 return (EINVAL);
2405         }
2406         softc->disk->d_delmaxsize = value;
2407
2408         return (0);
2409 }
2410
2411 static int
2412 dacmdsizesysctl(SYSCTL_HANDLER_ARGS)
2413 {
2414         int error, value;
2415
2416         value = *(int *)arg1;
2417
2418         error = sysctl_handle_int(oidp, &value, 0, req);
2419
2420         if ((error != 0)
2421          || (req->newptr == NULL))
2422                 return (error);
2423
2424         /*
2425          * Acceptable values here are 6, 10, 12 or 16.
2426          */
2427         if (value < 6)
2428                 value = 6;
2429         else if ((value > 6)
2430               && (value <= 10))
2431                 value = 10;
2432         else if ((value > 10)
2433               && (value <= 12))
2434                 value = 12;
2435         else if (value > 12)
2436                 value = 16;
2437
2438         *(int *)arg1 = value;
2439
2440         return (0);
2441 }
2442
2443 static int
2444 dasysctlsofttimeout(SYSCTL_HANDLER_ARGS)
2445 {
2446         sbintime_t value;
2447         int error;
2448
2449         value = da_default_softtimeout / SBT_1MS;
2450
2451         error = sysctl_handle_int(oidp, (int *)&value, 0, req);
2452         if ((error != 0) || (req->newptr == NULL))
2453                 return (error);
2454
2455         /* XXX Should clip this to a reasonable level */
2456         if (value > da_default_timeout * 1000)
2457                 return (EINVAL);
2458
2459         da_default_softtimeout = value * SBT_1MS;
2460         return (0);
2461 }
2462
2463 static void
2464 dadeletemethodset(struct da_softc *softc, da_delete_methods delete_method)
2465 {
2466
2467         softc->delete_method = delete_method;
2468         softc->disk->d_delmaxsize = dadeletemaxsize(softc, delete_method);
2469         softc->delete_func = da_delete_functions[delete_method];
2470
2471         if (softc->delete_method > DA_DELETE_DISABLE)
2472                 softc->disk->d_flags |= DISKFLAG_CANDELETE;
2473         else
2474                 softc->disk->d_flags &= ~DISKFLAG_CANDELETE;
2475 }
2476
2477 static off_t
2478 dadeletemaxsize(struct da_softc *softc, da_delete_methods delete_method)
2479 {
2480         off_t sectors;
2481
2482         switch(delete_method) {
2483         case DA_DELETE_UNMAP:
2484                 sectors = (off_t)softc->unmap_max_lba;
2485                 break;
2486         case DA_DELETE_ATA_TRIM:
2487                 sectors = (off_t)ATA_DSM_RANGE_MAX * softc->trim_max_ranges;
2488                 break;
2489         case DA_DELETE_WS16:
2490                 sectors = omin(softc->ws_max_blks, WS16_MAX_BLKS);
2491                 break;
2492         case DA_DELETE_ZERO:
2493         case DA_DELETE_WS10:
2494                 sectors = omin(softc->ws_max_blks, WS10_MAX_BLKS);
2495                 break;
2496         default:
2497                 return 0;
2498         }
2499
2500         return (off_t)softc->params.secsize *
2501             omin(sectors, softc->params.sectors);
2502 }
2503
2504 static void
2505 daprobedone(struct cam_periph *periph, union ccb *ccb)
2506 {
2507         struct da_softc *softc;
2508
2509         softc = (struct da_softc *)periph->softc;
2510
2511         cam_periph_assert(periph, MA_OWNED);
2512
2513         dadeletemethodchoose(softc, DA_DELETE_NONE);
2514
2515         if (bootverbose && (softc->flags & DA_FLAG_ANNOUNCED) == 0) {
2516                 char buf[80];
2517                 int i, sep;
2518
2519                 snprintf(buf, sizeof(buf), "Delete methods: <");
2520                 sep = 0;
2521                 for (i = 0; i <= DA_DELETE_MAX; i++) {
2522                         if ((softc->delete_available & (1 << i)) == 0 &&
2523                             i != softc->delete_method)
2524                                 continue;
2525                         if (sep)
2526                                 strlcat(buf, ",", sizeof(buf));
2527                         strlcat(buf, da_delete_method_names[i],
2528                             sizeof(buf));
2529                         if (i == softc->delete_method)
2530                                 strlcat(buf, "(*)", sizeof(buf));
2531                         sep = 1;
2532                 }
2533                 strlcat(buf, ">", sizeof(buf));
2534                 printf("%s%d: %s\n", periph->periph_name,
2535                     periph->unit_number, buf);
2536         }
2537         if ((softc->disk->d_flags & DISKFLAG_WRITE_PROTECT) != 0 &&
2538             (softc->flags & DA_FLAG_ANNOUNCED) == 0) {
2539                 printf("%s%d: Write Protected\n", periph->periph_name,
2540                     periph->unit_number);
2541         }
2542
2543         /*
2544          * Since our peripheral may be invalidated by an error
2545          * above or an external event, we must release our CCB
2546          * before releasing the probe lock on the peripheral.
2547          * The peripheral will only go away once the last lock
2548          * is removed, and we need it around for the CCB release
2549          * operation.
2550          */
2551         xpt_release_ccb(ccb);
2552         softc->state = DA_STATE_NORMAL;
2553         softc->flags |= DA_FLAG_PROBED;
2554         daschedule(periph);
2555         wakeup(&softc->disk->d_mediasize);
2556         if ((softc->flags & DA_FLAG_ANNOUNCED) == 0) {
2557                 softc->flags |= DA_FLAG_ANNOUNCED;
2558                 da_periph_unhold(periph, DA_REF_PROBE_HOLD);
2559         } else
2560                 da_periph_release_locked(periph, DA_REF_REPROBE);
2561 }
2562
2563 static void
2564 dadeletemethodchoose(struct da_softc *softc, da_delete_methods default_method)
2565 {
2566         int i, methods;
2567
2568         /* If available, prefer the method requested by user. */
2569         i = softc->delete_method_pref;
2570         methods = softc->delete_available | (1 << DA_DELETE_DISABLE);
2571         if (methods & (1 << i)) {
2572                 dadeletemethodset(softc, i);
2573                 return;
2574         }
2575
2576         /* Use the pre-defined order to choose the best performing delete. */
2577         for (i = DA_DELETE_MIN; i <= DA_DELETE_MAX; i++) {
2578                 if (i == DA_DELETE_ZERO)
2579                         continue;
2580                 if (softc->delete_available & (1 << i)) {
2581                         dadeletemethodset(softc, i);
2582                         return;
2583                 }
2584         }
2585
2586         /* Fallback to default. */
2587         dadeletemethodset(softc, default_method);
2588 }
2589
2590 static int
2591 dadeletemethodsysctl(SYSCTL_HANDLER_ARGS)
2592 {
2593         char buf[16];
2594         const char *p;
2595         struct da_softc *softc;
2596         int i, error, value;
2597
2598         softc = (struct da_softc *)arg1;
2599
2600         value = softc->delete_method;
2601         if (value < 0 || value > DA_DELETE_MAX)
2602                 p = "UNKNOWN";
2603         else
2604                 p = da_delete_method_names[value];
2605         strncpy(buf, p, sizeof(buf));
2606         error = sysctl_handle_string(oidp, buf, sizeof(buf), req);
2607         if (error != 0 || req->newptr == NULL)
2608                 return (error);
2609         for (i = 0; i <= DA_DELETE_MAX; i++) {
2610                 if (strcmp(buf, da_delete_method_names[i]) == 0)
2611                         break;
2612         }
2613         if (i > DA_DELETE_MAX)
2614                 return (EINVAL);
2615         softc->delete_method_pref = i;
2616         dadeletemethodchoose(softc, DA_DELETE_NONE);
2617         return (0);
2618 }
2619
2620 static int
2621 dazonemodesysctl(SYSCTL_HANDLER_ARGS)
2622 {
2623         char tmpbuf[40];
2624         struct da_softc *softc;
2625         int error;
2626
2627         softc = (struct da_softc *)arg1;
2628
2629         switch (softc->zone_mode) {
2630         case DA_ZONE_DRIVE_MANAGED:
2631                 snprintf(tmpbuf, sizeof(tmpbuf), "Drive Managed");
2632                 break;
2633         case DA_ZONE_HOST_AWARE:
2634                 snprintf(tmpbuf, sizeof(tmpbuf), "Host Aware");
2635                 break;
2636         case DA_ZONE_HOST_MANAGED:
2637                 snprintf(tmpbuf, sizeof(tmpbuf), "Host Managed");
2638                 break;
2639         case DA_ZONE_NONE:
2640         default:
2641                 snprintf(tmpbuf, sizeof(tmpbuf), "Not Zoned");
2642                 break;
2643         }
2644
2645         error = sysctl_handle_string(oidp, tmpbuf, sizeof(tmpbuf), req);
2646
2647         return (error);
2648 }
2649
2650 static int
2651 dazonesupsysctl(SYSCTL_HANDLER_ARGS)
2652 {
2653         char tmpbuf[180];
2654         struct da_softc *softc;
2655         struct sbuf sb;
2656         int error, first;
2657         unsigned int i;
2658
2659         softc = (struct da_softc *)arg1;
2660
2661         error = 0;
2662         first = 1;
2663         sbuf_new(&sb, tmpbuf, sizeof(tmpbuf), 0);
2664
2665         for (i = 0; i < sizeof(da_zone_desc_table) /
2666              sizeof(da_zone_desc_table[0]); i++) {
2667                 if (softc->zone_flags & da_zone_desc_table[i].value) {
2668                         if (first == 0)
2669                                 sbuf_printf(&sb, ", ");
2670                         else
2671                                 first = 0;
2672                         sbuf_cat(&sb, da_zone_desc_table[i].desc);
2673                 }
2674         }
2675
2676         if (first == 1)
2677                 sbuf_printf(&sb, "None");
2678
2679         sbuf_finish(&sb);
2680
2681         error = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
2682
2683         return (error);
2684 }
2685
2686 static cam_status
2687 daregister(struct cam_periph *periph, void *arg)
2688 {
2689         struct da_softc *softc;
2690         struct ccb_pathinq cpi;
2691         struct ccb_getdev *cgd;
2692         char tmpstr[80];
2693         caddr_t match;
2694
2695         cgd = (struct ccb_getdev *)arg;
2696         if (cgd == NULL) {
2697                 printf("daregister: no getdev CCB, can't register device\n");
2698                 return(CAM_REQ_CMP_ERR);
2699         }
2700
2701         softc = (struct da_softc *)malloc(sizeof(*softc), M_DEVBUF,
2702             M_NOWAIT|M_ZERO);
2703
2704         if (softc == NULL) {
2705                 printf("daregister: Unable to probe new device. "
2706                        "Unable to allocate softc\n");
2707                 return(CAM_REQ_CMP_ERR);
2708         }
2709
2710         if (cam_iosched_init(&softc->cam_iosched, periph) != 0) {
2711                 printf("daregister: Unable to probe new device. "
2712                        "Unable to allocate iosched memory\n");
2713                 free(softc, M_DEVBUF);
2714                 return(CAM_REQ_CMP_ERR);
2715         }
2716
2717         LIST_INIT(&softc->pending_ccbs);
2718         softc->state = DA_STATE_PROBE_WP;
2719         bioq_init(&softc->delete_run_queue);
2720         if (SID_IS_REMOVABLE(&cgd->inq_data))
2721                 softc->flags |= DA_FLAG_PACK_REMOVABLE;
2722         softc->unmap_max_ranges = UNMAP_MAX_RANGES;
2723         softc->unmap_max_lba = UNMAP_RANGE_MAX;
2724         softc->unmap_gran = 0;
2725         softc->unmap_gran_align = 0;
2726         softc->ws_max_blks = WS16_MAX_BLKS;
2727         softc->trim_max_ranges = ATA_TRIM_MAX_RANGES;
2728         softc->rotating = 1;
2729
2730         periph->softc = softc;
2731
2732         /*
2733          * See if this device has any quirks.
2734          */
2735         match = cam_quirkmatch((caddr_t)&cgd->inq_data,
2736                                (caddr_t)da_quirk_table,
2737                                nitems(da_quirk_table),
2738                                sizeof(*da_quirk_table), scsi_inquiry_match);
2739
2740         if (match != NULL)
2741                 softc->quirks = ((struct da_quirk_entry *)match)->quirks;
2742         else
2743                 softc->quirks = DA_Q_NONE;
2744
2745         /* Check if the SIM does not want 6 byte commands */
2746         xpt_path_inq(&cpi, periph->path);
2747         if (cpi.ccb_h.status == CAM_REQ_CMP && (cpi.hba_misc & PIM_NO_6_BYTE))
2748                 softc->quirks |= DA_Q_NO_6_BYTE;
2749
2750         if (SID_TYPE(&cgd->inq_data) == T_ZBC_HM)
2751                 softc->zone_mode = DA_ZONE_HOST_MANAGED;
2752         else if (softc->quirks & DA_Q_SMR_DM)
2753                 softc->zone_mode = DA_ZONE_DRIVE_MANAGED;
2754         else
2755                 softc->zone_mode = DA_ZONE_NONE;
2756
2757         if (softc->zone_mode != DA_ZONE_NONE) {
2758                 if (scsi_vpd_supported_page(periph, SVPD_ATA_INFORMATION)) {
2759                         if (scsi_vpd_supported_page(periph, SVPD_ZONED_BDC))
2760                                 softc->zone_interface = DA_ZONE_IF_ATA_SAT;
2761                         else
2762                                 softc->zone_interface = DA_ZONE_IF_ATA_PASS;
2763                 } else
2764                         softc->zone_interface = DA_ZONE_IF_SCSI;
2765         }
2766
2767         TASK_INIT(&softc->sysctl_task, 0, dasysctlinit, periph);
2768
2769         /*
2770          * Take an exclusive section lock qon the periph while dastart is called
2771          * to finish the probe.  The lock will be dropped in dadone at the end
2772          * of probe. This locks out daopen and daclose from racing with the
2773          * probe.
2774          *
2775          * XXX if cam_periph_hold returns an error, we don't hold a refcount.
2776          */
2777         (void)da_periph_hold(periph, PRIBIO, DA_REF_PROBE_HOLD);
2778
2779         /*
2780          * Schedule a periodic event to occasionally send an
2781          * ordered tag to a device.
2782          */
2783         callout_init_mtx(&softc->sendordered_c, cam_periph_mtx(periph), 0);
2784         callout_reset(&softc->sendordered_c,
2785             (da_default_timeout * hz) / DA_ORDEREDTAG_INTERVAL,
2786             dasendorderedtag, periph);
2787
2788         cam_periph_unlock(periph);
2789         /*
2790          * RBC devices don't have to support READ(6), only READ(10).
2791          */
2792         if (softc->quirks & DA_Q_NO_6_BYTE || SID_TYPE(&cgd->inq_data) == T_RBC)
2793                 softc->minimum_cmd_size = 10;
2794         else
2795                 softc->minimum_cmd_size = 6;
2796
2797         /*
2798          * Load the user's default, if any.
2799          */
2800         snprintf(tmpstr, sizeof(tmpstr), "kern.cam.da.%d.minimum_cmd_size",
2801                  periph->unit_number);
2802         TUNABLE_INT_FETCH(tmpstr, &softc->minimum_cmd_size);
2803
2804         /*
2805          * 6, 10, 12 and 16 are the currently permissible values.
2806          */
2807         if (softc->minimum_cmd_size > 12)
2808                 softc->minimum_cmd_size = 16;
2809         else if (softc->minimum_cmd_size > 10)
2810                 softc->minimum_cmd_size = 12;
2811         else if (softc->minimum_cmd_size > 6)
2812                 softc->minimum_cmd_size = 10;
2813         else
2814                 softc->minimum_cmd_size = 6;
2815
2816         /* Predict whether device may support READ CAPACITY(16). */
2817         if (SID_ANSI_REV(&cgd->inq_data) >= SCSI_REV_SPC3 &&
2818             (softc->quirks & DA_Q_NO_RC16) == 0) {
2819                 softc->flags |= DA_FLAG_CAN_RC16;
2820         }
2821
2822         /*
2823          * Register this media as a disk.
2824          */
2825         softc->disk = disk_alloc();
2826         softc->disk->d_devstat = devstat_new_entry(periph->periph_name,
2827                           periph->unit_number, 0,
2828                           DEVSTAT_BS_UNAVAILABLE,
2829                           SID_TYPE(&cgd->inq_data) |
2830                           XPORT_DEVSTAT_TYPE(cpi.transport),
2831                           DEVSTAT_PRIORITY_DISK);
2832         softc->disk->d_open = daopen;
2833         softc->disk->d_close = daclose;
2834         softc->disk->d_strategy = dastrategy;
2835         softc->disk->d_dump = dadump;
2836         softc->disk->d_getattr = dagetattr;
2837         softc->disk->d_gone = dadiskgonecb;
2838         softc->disk->d_name = "da";
2839         softc->disk->d_drv1 = periph;
2840         if (cpi.maxio == 0)
2841                 softc->maxio = DFLTPHYS;        /* traditional default */
2842         else if (cpi.maxio > MAXPHYS)
2843                 softc->maxio = MAXPHYS;         /* for safety */
2844         else
2845                 softc->maxio = cpi.maxio;
2846         if (softc->quirks & DA_Q_128KB)
2847                 softc->maxio = min(softc->maxio, 128 * 1024);
2848         softc->disk->d_maxsize = softc->maxio;
2849         softc->disk->d_unit = periph->unit_number;
2850         softc->disk->d_flags = DISKFLAG_DIRECT_COMPLETION | DISKFLAG_CANZONE;
2851         if ((softc->quirks & DA_Q_NO_SYNC_CACHE) == 0)
2852                 softc->disk->d_flags |= DISKFLAG_CANFLUSHCACHE;
2853         if ((cpi.hba_misc & PIM_UNMAPPED) != 0) {
2854                 softc->unmappedio = 1;
2855                 softc->disk->d_flags |= DISKFLAG_UNMAPPED_BIO;
2856         }
2857         cam_strvis(softc->disk->d_descr, cgd->inq_data.vendor,
2858             sizeof(cgd->inq_data.vendor), sizeof(softc->disk->d_descr));
2859         strlcat(softc->disk->d_descr, " ", sizeof(softc->disk->d_descr));
2860         cam_strvis(&softc->disk->d_descr[strlen(softc->disk->d_descr)],
2861             cgd->inq_data.product, sizeof(cgd->inq_data.product),
2862             sizeof(softc->disk->d_descr) - strlen(softc->disk->d_descr));
2863         softc->disk->d_hba_vendor = cpi.hba_vendor;
2864         softc->disk->d_hba_device = cpi.hba_device;
2865         softc->disk->d_hba_subvendor = cpi.hba_subvendor;
2866         softc->disk->d_hba_subdevice = cpi.hba_subdevice;
2867
2868         /*
2869          * Acquire a reference to the periph before we register with GEOM.
2870          * We'll release this reference once GEOM calls us back (via
2871          * dadiskgonecb()) telling us that our provider has been freed.
2872          */
2873         if (da_periph_acquire(periph, DA_REF_GEOM) != 0) {
2874                 xpt_print(periph->path, "%s: lost periph during "
2875                           "registration!\n", __func__);
2876                 cam_periph_lock(periph);
2877                 return (CAM_REQ_CMP_ERR);
2878         }
2879
2880         disk_create(softc->disk, DISK_VERSION);
2881         cam_periph_lock(periph);
2882
2883         /*
2884          * Add async callbacks for events of interest.
2885          * I don't bother checking if this fails as,
2886          * in most cases, the system will function just
2887          * fine without them and the only alternative
2888          * would be to not attach the device on failure.
2889          */
2890         xpt_register_async(AC_SENT_BDR | AC_BUS_RESET | AC_LOST_DEVICE |
2891             AC_ADVINFO_CHANGED | AC_SCSI_AEN | AC_UNIT_ATTENTION |
2892             AC_INQ_CHANGED, daasync, periph, periph->path);
2893
2894         /*
2895          * Emit an attribute changed notification just in case
2896          * physical path information arrived before our async
2897          * event handler was registered, but after anyone attaching
2898          * to our disk device polled it.
2899          */
2900         disk_attr_changed(softc->disk, "GEOM::physpath", M_NOWAIT);
2901
2902         /*
2903          * Schedule a periodic media polling events.
2904          */
2905         callout_init_mtx(&softc->mediapoll_c, cam_periph_mtx(periph), 0);
2906         if ((softc->flags & DA_FLAG_PACK_REMOVABLE) &&
2907             (cgd->inq_flags & SID_AEN) == 0 &&
2908             da_poll_period != 0)
2909                 callout_reset(&softc->mediapoll_c, da_poll_period * hz,
2910                     damediapoll, periph);
2911
2912         xpt_schedule(periph, CAM_PRIORITY_DEV);
2913
2914         return(CAM_REQ_CMP);
2915 }
2916
2917 static int
2918 da_zone_bio_to_scsi(int disk_zone_cmd)
2919 {
2920         switch (disk_zone_cmd) {
2921         case DISK_ZONE_OPEN:
2922                 return ZBC_OUT_SA_OPEN;
2923         case DISK_ZONE_CLOSE:
2924                 return ZBC_OUT_SA_CLOSE;
2925         case DISK_ZONE_FINISH:
2926                 return ZBC_OUT_SA_FINISH;
2927         case DISK_ZONE_RWP:
2928                 return ZBC_OUT_SA_RWP;
2929         }
2930
2931         return -1;
2932 }
2933
2934 static int
2935 da_zone_cmd(struct cam_periph *periph, union ccb *ccb, struct bio *bp,
2936             int *queue_ccb)
2937 {
2938         struct da_softc *softc;
2939         int error;
2940
2941         error = 0;
2942
2943         if (bp->bio_cmd != BIO_ZONE) {
2944                 error = EINVAL;
2945                 goto bailout;
2946         }
2947
2948         softc = periph->softc;
2949
2950         switch (bp->bio_zone.zone_cmd) {
2951         case DISK_ZONE_OPEN:
2952         case DISK_ZONE_CLOSE:
2953         case DISK_ZONE_FINISH:
2954         case DISK_ZONE_RWP: {
2955                 int zone_flags;
2956                 int zone_sa;
2957                 uint64_t lba;
2958
2959                 zone_sa = da_zone_bio_to_scsi(bp->bio_zone.zone_cmd);
2960                 if (zone_sa == -1) {
2961                         xpt_print(periph->path, "Cannot translate zone "
2962                             "cmd %#x to SCSI\n", bp->bio_zone.zone_cmd);
2963                         error = EINVAL;
2964                         goto bailout;
2965                 }
2966
2967                 zone_flags = 0;
2968                 lba = bp->bio_zone.zone_params.rwp.id;
2969
2970                 if (bp->bio_zone.zone_params.rwp.flags &
2971                     DISK_ZONE_RWP_FLAG_ALL)
2972                         zone_flags |= ZBC_OUT_ALL;
2973
2974                 if (softc->zone_interface != DA_ZONE_IF_ATA_PASS) {
2975                         scsi_zbc_out(&ccb->csio,
2976                                      /*retries*/ da_retry_count,
2977                                      /*cbfcnp*/ dadone,
2978                                      /*tag_action*/ MSG_SIMPLE_Q_TAG,
2979                                      /*service_action*/ zone_sa,
2980                                      /*zone_id*/ lba,
2981                                      /*zone_flags*/ zone_flags,
2982                                      /*data_ptr*/ NULL,
2983                                      /*dxfer_len*/ 0,
2984                                      /*sense_len*/ SSD_FULL_SIZE,
2985                                      /*timeout*/ da_default_timeout * 1000);
2986                 } else {
2987                         /*
2988                          * Note that in this case, even though we can
2989                          * technically use NCQ, we don't bother for several
2990                          * reasons:
2991                          * 1. It hasn't been tested on a SAT layer that
2992                          *    supports it.  This is new as of SAT-4.
2993                          * 2. Even when there is a SAT layer that supports
2994                          *    it, that SAT layer will also probably support
2995                          *    ZBC -> ZAC translation, since they are both
2996                          *    in the SAT-4 spec.
2997                          * 3. Translation will likely be preferable to ATA
2998                          *    passthrough.  LSI / Avago at least single
2999                          *    steps ATA passthrough commands in the HBA,
3000                          *    regardless of protocol, so unless that
3001                          *    changes, there is a performance penalty for
3002                          *    doing ATA passthrough no matter whether
3003                          *    you're using NCQ/FPDMA, DMA or PIO.
3004                          * 4. It requires a 32-byte CDB, which at least at
3005                          *    this point in CAM requires a CDB pointer, which
3006                          *    would require us to allocate an additional bit
3007                          *    of storage separate from the CCB.
3008                          */
3009                         error = scsi_ata_zac_mgmt_out(&ccb->csio,
3010                             /*retries*/ da_retry_count,
3011                             /*cbfcnp*/ dadone,
3012                             /*tag_action*/ MSG_SIMPLE_Q_TAG,
3013                             /*use_ncq*/ 0,
3014                             /*zm_action*/ zone_sa,
3015                             /*zone_id*/ lba,
3016                             /*zone_flags*/ zone_flags,
3017                             /*data_ptr*/ NULL,
3018                             /*dxfer_len*/ 0,
3019                             /*cdb_storage*/ NULL,
3020                             /*cdb_storage_len*/ 0,
3021                             /*sense_len*/ SSD_FULL_SIZE,
3022                             /*timeout*/ da_default_timeout * 1000);
3023                         if (error != 0) {
3024                                 error = EINVAL;
3025                                 xpt_print(periph->path,
3026                                     "scsi_ata_zac_mgmt_out() returned an "
3027                                     "error!");
3028                                 goto bailout;
3029                         }
3030                 }
3031                 *queue_ccb = 1;
3032
3033                 break;
3034         }
3035         case DISK_ZONE_REPORT_ZONES: {
3036                 uint8_t *rz_ptr;
3037                 uint32_t num_entries, alloc_size;
3038                 struct disk_zone_report *rep;
3039
3040                 rep = &bp->bio_zone.zone_params.report;
3041
3042                 num_entries = rep->entries_allocated;
3043                 if (num_entries == 0) {
3044                         xpt_print(periph->path, "No entries allocated for "
3045                             "Report Zones request\n");
3046                         error = EINVAL;
3047                         goto bailout;
3048                 }
3049                 alloc_size = sizeof(struct scsi_report_zones_hdr) +
3050                     (sizeof(struct scsi_report_zones_desc) * num_entries);
3051                 alloc_size = min(alloc_size, softc->disk->d_maxsize);
3052                 rz_ptr = malloc(alloc_size, M_SCSIDA, M_NOWAIT | M_ZERO);
3053                 if (rz_ptr == NULL) {
3054                         xpt_print(periph->path, "Unable to allocate memory "
3055                            "for Report Zones request\n");
3056                         error = ENOMEM;
3057                         goto bailout;
3058                 }
3059
3060                 if (softc->zone_interface != DA_ZONE_IF_ATA_PASS) {
3061                         scsi_zbc_in(&ccb->csio,
3062                                     /*retries*/ da_retry_count,
3063                                     /*cbcfnp*/ dadone,
3064                                     /*tag_action*/ MSG_SIMPLE_Q_TAG,
3065                                     /*service_action*/ ZBC_IN_SA_REPORT_ZONES,
3066                                     /*zone_start_lba*/ rep->starting_id,
3067                                     /*zone_options*/ rep->rep_options,
3068                                     /*data_ptr*/ rz_ptr,
3069                                     /*dxfer_len*/ alloc_size,
3070                                     /*sense_len*/ SSD_FULL_SIZE,
3071                                     /*timeout*/ da_default_timeout * 1000);
3072                 } else {
3073                         /*
3074                          * Note that in this case, even though we can
3075                          * technically use NCQ, we don't bother for several
3076                          * reasons:
3077                          * 1. It hasn't been tested on a SAT layer that
3078                          *    supports it.  This is new as of SAT-4.
3079                          * 2. Even when there is a SAT layer that supports
3080                          *    it, that SAT layer will also probably support
3081                          *    ZBC -> ZAC translation, since they are both
3082                          *    in the SAT-4 spec.
3083                          * 3. Translation will likely be preferable to ATA
3084                          *    passthrough.  LSI / Avago at least single
3085                          *    steps ATA passthrough commands in the HBA,
3086                          *    regardless of protocol, so unless that
3087                          *    changes, there is a performance penalty for
3088                          *    doing ATA passthrough no matter whether
3089                          *    you're using NCQ/FPDMA, DMA or PIO.
3090                          * 4. It requires a 32-byte CDB, which at least at
3091                          *    this point in CAM requires a CDB pointer, which
3092                          *    would require us to allocate an additional bit
3093                          *    of storage separate from the CCB.
3094                          */
3095                         error = scsi_ata_zac_mgmt_in(&ccb->csio,
3096                             /*retries*/ da_retry_count,
3097                             /*cbcfnp*/ dadone,
3098                             /*tag_action*/ MSG_SIMPLE_Q_TAG,
3099                             /*use_ncq*/ 0,
3100                             /*zm_action*/ ATA_ZM_REPORT_ZONES,
3101                             /*zone_id*/ rep->starting_id,
3102                             /*zone_flags*/ rep->rep_options,
3103                             /*data_ptr*/ rz_ptr,
3104                             /*dxfer_len*/ alloc_size,
3105                             /*cdb_storage*/ NULL,
3106                             /*cdb_storage_len*/ 0,
3107                             /*sense_len*/ SSD_FULL_SIZE,
3108                             /*timeout*/ da_default_timeout * 1000);
3109                         if (error != 0) {
3110                                 error = EINVAL;
3111                                 xpt_print(periph->path,
3112                                     "scsi_ata_zac_mgmt_in() returned an "
3113                                     "error!");
3114                                 goto bailout;
3115                         }
3116                 }
3117
3118                 /*
3119                  * For BIO_ZONE, this isn't normally needed.  However, it
3120                  * is used by devstat_end_transaction_bio() to determine
3121                  * how much data was transferred.
3122                  */
3123                 /*
3124                  * XXX KDM we have a problem.  But I'm not sure how to fix
3125                  * it.  devstat uses bio_bcount - bio_resid to calculate
3126                  * the amount of data transferred.   The GEOM disk code
3127                  * uses bio_length - bio_resid to calculate the amount of
3128                  * data in bio_completed.  We have different structure
3129                  * sizes above and below the ada(4) driver.  So, if we
3130                  * use the sizes above, the amount transferred won't be
3131                  * quite accurate for devstat.  If we use different sizes
3132                  * for bio_bcount and bio_length (above and below
3133                  * respectively), then the residual needs to match one or
3134                  * the other.  Everything is calculated after the bio
3135                  * leaves the driver, so changing the values around isn't
3136                  * really an option.  For now, just set the count to the
3137                  * passed in length.  This means that the calculations
3138                  * above (e.g. bio_completed) will be correct, but the
3139                  * amount of data reported to devstat will be slightly
3140                  * under or overstated.
3141                  */
3142                 bp->bio_bcount = bp->bio_length;
3143
3144                 *queue_ccb = 1;
3145
3146                 break;
3147         }
3148         case DISK_ZONE_GET_PARAMS: {
3149                 struct disk_zone_disk_params *params;
3150
3151                 params = &bp->bio_zone.zone_params.disk_params;
3152                 bzero(params, sizeof(*params));
3153
3154                 switch (softc->zone_mode) {
3155                 case DA_ZONE_DRIVE_MANAGED:
3156                         params->zone_mode = DISK_ZONE_MODE_DRIVE_MANAGED;
3157                         break;
3158                 case DA_ZONE_HOST_AWARE:
3159                         params->zone_mode = DISK_ZONE_MODE_HOST_AWARE;
3160                         break;
3161                 case DA_ZONE_HOST_MANAGED:
3162                         params->zone_mode = DISK_ZONE_MODE_HOST_MANAGED;
3163                         break;
3164                 default:
3165                 case DA_ZONE_NONE:
3166                         params->zone_mode = DISK_ZONE_MODE_NONE;
3167                         break;
3168                 }
3169
3170                 if (softc->zone_flags & DA_ZONE_FLAG_URSWRZ)
3171                         params->flags |= DISK_ZONE_DISK_URSWRZ;
3172
3173                 if (softc->zone_flags & DA_ZONE_FLAG_OPT_SEQ_SET) {
3174                         params->optimal_seq_zones = softc->optimal_seq_zones;
3175                         params->flags |= DISK_ZONE_OPT_SEQ_SET;
3176                 }
3177
3178                 if (softc->zone_flags & DA_ZONE_FLAG_OPT_NONSEQ_SET) {
3179                         params->optimal_nonseq_zones =
3180                             softc->optimal_nonseq_zones;
3181                         params->flags |= DISK_ZONE_OPT_NONSEQ_SET;
3182                 }
3183
3184                 if (softc->zone_flags & DA_ZONE_FLAG_MAX_SEQ_SET) {
3185                         params->max_seq_zones = softc->max_seq_zones;
3186                         params->flags |= DISK_ZONE_MAX_SEQ_SET;
3187                 }
3188                 if (softc->zone_flags & DA_ZONE_FLAG_RZ_SUP)
3189                         params->flags |= DISK_ZONE_RZ_SUP;
3190
3191                 if (softc->zone_flags & DA_ZONE_FLAG_OPEN_SUP)
3192                         params->flags |= DISK_ZONE_OPEN_SUP;
3193
3194                 if (softc->zone_flags & DA_ZONE_FLAG_CLOSE_SUP)
3195                         params->flags |= DISK_ZONE_CLOSE_SUP;
3196
3197                 if (softc->zone_flags & DA_ZONE_FLAG_FINISH_SUP)
3198                         params->flags |= DISK_ZONE_FINISH_SUP;
3199
3200                 if (softc->zone_flags & DA_ZONE_FLAG_RWP_SUP)
3201                         params->flags |= DISK_ZONE_RWP_SUP;
3202                 break;
3203         }
3204         default:
3205                 break;
3206         }
3207 bailout:
3208         return (error);
3209 }
3210
3211 static void
3212 dastart(struct cam_periph *periph, union ccb *start_ccb)
3213 {
3214         struct da_softc *softc;
3215
3216         cam_periph_assert(periph, MA_OWNED);
3217         softc = (struct da_softc *)periph->softc;
3218
3219         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dastart\n"));
3220
3221 skipstate:
3222         switch (softc->state) {
3223         case DA_STATE_NORMAL:
3224         {
3225                 struct bio *bp;
3226                 uint8_t tag_code;
3227
3228 more:
3229                 bp = cam_iosched_next_bio(softc->cam_iosched);
3230                 if (bp == NULL) {
3231                         if (cam_iosched_has_work_flags(softc->cam_iosched,
3232                             DA_WORK_TUR)) {
3233                                 softc->flags |= DA_FLAG_TUR_PENDING;
3234                                 cam_iosched_clr_work_flags(softc->cam_iosched,
3235                                     DA_WORK_TUR);
3236                                 scsi_test_unit_ready(&start_ccb->csio,
3237                                      /*retries*/ da_retry_count,
3238                                      dadone_tur,
3239                                      MSG_SIMPLE_Q_TAG,
3240                                      SSD_FULL_SIZE,
3241                                      da_default_timeout * 1000);
3242                                 start_ccb->ccb_h.ccb_bp = NULL;
3243                                 start_ccb->ccb_h.ccb_state = DA_CCB_TUR;
3244                                 xpt_action(start_ccb);
3245                         } else
3246                                 xpt_release_ccb(start_ccb);
3247                         break;
3248                 }
3249
3250                 if (bp->bio_cmd == BIO_DELETE) {
3251                         if (softc->delete_func != NULL) {
3252                                 softc->delete_func(periph, start_ccb, bp);
3253                                 goto out;
3254                         } else {
3255                                 /*
3256                                  * Not sure this is possible, but failsafe by
3257                                  * lying and saying "sure, done."
3258                                  */
3259                                 biofinish(bp, NULL, 0);
3260                                 goto more;
3261                         }
3262                 }
3263
3264                 if (cam_iosched_has_work_flags(softc->cam_iosched,
3265                     DA_WORK_TUR)) {
3266                         cam_iosched_clr_work_flags(softc->cam_iosched,
3267                             DA_WORK_TUR);
3268                         da_periph_release_locked(periph, DA_REF_TUR);
3269                 }
3270
3271                 if ((bp->bio_flags & BIO_ORDERED) != 0 ||
3272                     (softc->flags & DA_FLAG_NEED_OTAG) != 0) {
3273                         softc->flags &= ~DA_FLAG_NEED_OTAG;
3274                         softc->flags |= DA_FLAG_WAS_OTAG;
3275                         tag_code = MSG_ORDERED_Q_TAG;
3276                 } else {
3277                         tag_code = MSG_SIMPLE_Q_TAG;
3278                 }
3279
3280                 switch (bp->bio_cmd) {
3281                 case BIO_WRITE:
3282                 case BIO_READ:
3283                 {
3284                         void *data_ptr;
3285                         int rw_op;
3286
3287                         biotrack(bp, __func__);
3288
3289                         if (bp->bio_cmd == BIO_WRITE) {
3290                                 softc->flags |= DA_FLAG_DIRTY;
3291                                 rw_op = SCSI_RW_WRITE;
3292                         } else {
3293                                 rw_op = SCSI_RW_READ;
3294                         }
3295
3296                         data_ptr = bp->bio_data;
3297                         if ((bp->bio_flags & (BIO_UNMAPPED|BIO_VLIST)) != 0) {
3298                                 rw_op |= SCSI_RW_BIO;
3299                                 data_ptr = bp;
3300                         }
3301
3302                         scsi_read_write(&start_ccb->csio,
3303                                         /*retries*/da_retry_count,
3304                                         /*cbfcnp*/dadone,
3305                                         /*tag_action*/tag_code,
3306                                         rw_op,
3307                                         /*byte2*/0,
3308                                         softc->minimum_cmd_size,
3309                                         /*lba*/bp->bio_pblkno,
3310                                         /*block_count*/bp->bio_bcount /
3311                                         softc->params.secsize,
3312                                         data_ptr,
3313                                         /*dxfer_len*/ bp->bio_bcount,
3314                                         /*sense_len*/SSD_FULL_SIZE,
3315                                         da_default_timeout * 1000);
3316 #if defined(BUF_TRACKING) || defined(FULL_BUF_TRACKING)
3317                         start_ccb->csio.bio = bp;
3318 #endif
3319                         break;
3320                 }
3321                 case BIO_FLUSH:
3322                         /*
3323                          * If we don't support sync cache, or the disk
3324                          * isn't dirty, FLUSH is a no-op.  Use the
3325                          * allocated CCB for the next bio if one is
3326                          * available.
3327                          */
3328                         if ((softc->quirks & DA_Q_NO_SYNC_CACHE) != 0 ||
3329                             (softc->flags & DA_FLAG_DIRTY) == 0) {
3330                                 biodone(bp);
3331                                 goto skipstate;
3332                         }
3333
3334                         /*
3335                          * BIO_FLUSH doesn't currently communicate
3336                          * range data, so we synchronize the cache
3337                          * over the whole disk.
3338                          */
3339                         scsi_synchronize_cache(&start_ccb->csio,
3340                                                /*retries*/1,
3341                                                /*cbfcnp*/dadone,
3342                                                /*tag_action*/tag_code,
3343                                                /*begin_lba*/0,
3344                                                /*lb_count*/0,
3345                                                SSD_FULL_SIZE,
3346                                                da_default_timeout*1000);
3347                         /*
3348                          * Clear the dirty flag before sending the command.
3349                          * Either this sync cache will be successful, or it
3350                          * will fail after a retry.  If it fails, it is
3351                          * unlikely to be successful if retried later, so
3352                          * we'll save ourselves time by just marking the
3353                          * device clean.
3354                          */
3355                         softc->flags &= ~DA_FLAG_DIRTY;
3356                         break;
3357                 case BIO_ZONE: {
3358                         int error, queue_ccb;
3359
3360                         queue_ccb = 0;
3361
3362                         error = da_zone_cmd(periph, start_ccb, bp,&queue_ccb);
3363                         if ((error != 0)
3364                          || (queue_ccb == 0)) {
3365                                 biofinish(bp, NULL, error);
3366                                 xpt_release_ccb(start_ccb);
3367                                 return;
3368                         }
3369                         break;
3370                 }
3371                 }
3372                 start_ccb->ccb_h.ccb_state = DA_CCB_BUFFER_IO;
3373                 start_ccb->ccb_h.flags |= CAM_UNLOCKED;
3374                 start_ccb->ccb_h.softtimeout = sbttotv(da_default_softtimeout);
3375
3376 out:
3377                 LIST_INSERT_HEAD(&softc->pending_ccbs,
3378                                  &start_ccb->ccb_h, periph_links.le);
3379
3380                 /* We expect a unit attention from this device */
3381                 if ((softc->flags & DA_FLAG_RETRY_UA) != 0) {
3382                         start_ccb->ccb_h.ccb_state |= DA_CCB_RETRY_UA;
3383                         softc->flags &= ~DA_FLAG_RETRY_UA;
3384                 }
3385
3386                 start_ccb->ccb_h.ccb_bp = bp;
3387                 softc->refcount++;
3388                 cam_periph_unlock(periph);
3389                 xpt_action(start_ccb);
3390                 cam_periph_lock(periph);
3391
3392                 /* May have more work to do, so ensure we stay scheduled */
3393                 daschedule(periph);
3394                 break;
3395         }
3396         case DA_STATE_PROBE_WP:
3397         {
3398                 void  *mode_buf;
3399                 int    mode_buf_len;
3400
3401                 if (da_disable_wp_detection) {
3402                         if ((softc->flags & DA_FLAG_CAN_RC16) != 0)
3403                                 softc->state = DA_STATE_PROBE_RC16;
3404                         else
3405                                 softc->state = DA_STATE_PROBE_RC;
3406                         goto skipstate;
3407                 }
3408                 mode_buf_len = 192;
3409                 mode_buf = malloc(mode_buf_len, M_SCSIDA, M_NOWAIT);
3410                 if (mode_buf == NULL) {
3411                         xpt_print(periph->path, "Unable to send mode sense - "
3412                             "malloc failure\n");
3413                         if ((softc->flags & DA_FLAG_CAN_RC16) != 0)
3414                                 softc->state = DA_STATE_PROBE_RC16;
3415                         else
3416                                 softc->state = DA_STATE_PROBE_RC;
3417                         goto skipstate;
3418                 }
3419                 scsi_mode_sense_len(&start_ccb->csio,
3420                                     /*retries*/ da_retry_count,
3421                                     /*cbfcnp*/ dadone_probewp,
3422                                     /*tag_action*/ MSG_SIMPLE_Q_TAG,
3423                                     /*dbd*/ FALSE,
3424                                     /*pc*/ SMS_PAGE_CTRL_CURRENT,
3425                                     /*page*/ SMS_ALL_PAGES_PAGE,
3426                                     /*param_buf*/ mode_buf,
3427                                     /*param_len*/ mode_buf_len,
3428                                     /*minimum_cmd_size*/ softc->minimum_cmd_size,
3429                                     /*sense_len*/ SSD_FULL_SIZE,
3430                                     /*timeout*/ da_default_timeout * 1000);
3431                 start_ccb->ccb_h.ccb_bp = NULL;
3432                 start_ccb->ccb_h.ccb_state = DA_CCB_PROBE_WP;
3433                 xpt_action(start_ccb);
3434                 break;
3435         }
3436         case DA_STATE_PROBE_RC:
3437         {
3438                 struct scsi_read_capacity_data *rcap;
3439
3440                 rcap = (struct scsi_read_capacity_data *)
3441                     malloc(sizeof(*rcap), M_SCSIDA, M_NOWAIT|M_ZERO);
3442                 if (rcap == NULL) {
3443                         printf("dastart: Couldn't malloc read_capacity data\n");
3444                         /* da_free_periph??? */
3445                         break;
3446                 }
3447                 scsi_read_capacity(&start_ccb->csio,
3448                                    /*retries*/da_retry_count,
3449                                    dadone_proberc,
3450                                    MSG_SIMPLE_Q_TAG,
3451                                    rcap,
3452                                    SSD_FULL_SIZE,
3453                                    /*timeout*/5000);
3454                 start_ccb->ccb_h.ccb_bp = NULL;
3455                 start_ccb->ccb_h.ccb_state = DA_CCB_PROBE_RC;
3456                 xpt_action(start_ccb);
3457                 break;
3458         }
3459         case DA_STATE_PROBE_RC16:
3460         {
3461                 struct scsi_read_capacity_data_long *rcaplong;
3462
3463                 rcaplong = (struct scsi_read_capacity_data_long *)
3464                         malloc(sizeof(*rcaplong), M_SCSIDA, M_NOWAIT|M_ZERO);
3465                 if (rcaplong == NULL) {
3466                         printf("dastart: Couldn't malloc read_capacity data\n");
3467                         /* da_free_periph??? */
3468                         break;
3469                 }
3470                 scsi_read_capacity_16(&start_ccb->csio,
3471                                       /*retries*/ da_retry_count,
3472                                       /*cbfcnp*/ dadone_proberc,
3473                                       /*tag_action*/ MSG_SIMPLE_Q_TAG,
3474                                       /*lba*/ 0,
3475                                       /*reladr*/ 0,
3476                                       /*pmi*/ 0,
3477                                       /*rcap_buf*/ (uint8_t *)rcaplong,
3478                                       /*rcap_buf_len*/ sizeof(*rcaplong),
3479                                       /*sense_len*/ SSD_FULL_SIZE,
3480                                       /*timeout*/ da_default_timeout * 1000);
3481                 start_ccb->ccb_h.ccb_bp = NULL;
3482                 start_ccb->ccb_h.ccb_state = DA_CCB_PROBE_RC16;
3483                 xpt_action(start_ccb);
3484                 break;
3485         }
3486         case DA_STATE_PROBE_LBP:
3487         {
3488                 struct scsi_vpd_logical_block_prov *lbp;
3489
3490                 if (!scsi_vpd_supported_page(periph, SVPD_LBP)) {
3491                         /*
3492                          * If we get here we don't support any SBC-3 delete
3493                          * methods with UNMAP as the Logical Block Provisioning
3494                          * VPD page support is required for devices which
3495                          * support it according to T10/1799-D Revision 31
3496                          * however older revisions of the spec don't mandate
3497                          * this so we currently don't remove these methods
3498                          * from the available set.
3499                          */
3500                         softc->state = DA_STATE_PROBE_BLK_LIMITS;
3501                         goto skipstate;
3502                 }
3503
3504                 lbp = (struct scsi_vpd_logical_block_prov *)
3505                         malloc(sizeof(*lbp), M_SCSIDA, M_NOWAIT|M_ZERO);
3506
3507                 if (lbp == NULL) {
3508                         printf("dastart: Couldn't malloc lbp data\n");
3509                         /* da_free_periph??? */
3510                         break;
3511                 }
3512
3513                 scsi_inquiry(&start_ccb->csio,
3514                              /*retries*/da_retry_count,
3515                              /*cbfcnp*/dadone_probelbp,
3516                              /*tag_action*/MSG_SIMPLE_Q_TAG,
3517                              /*inq_buf*/(u_int8_t *)lbp,
3518                              /*inq_len*/sizeof(*lbp),
3519                              /*evpd*/TRUE,
3520                              /*page_code*/SVPD_LBP,
3521                              /*sense_len*/SSD_MIN_SIZE,
3522                              /*timeout*/da_default_timeout * 1000);
3523                 start_ccb->ccb_h.ccb_bp = NULL;
3524                 start_ccb->ccb_h.ccb_state = DA_CCB_PROBE_LBP;
3525                 xpt_action(start_ccb);
3526                 break;
3527         }
3528         case DA_STATE_PROBE_BLK_LIMITS:
3529         {
3530                 struct scsi_vpd_block_limits *block_limits;
3531
3532                 if (!scsi_vpd_supported_page(periph, SVPD_BLOCK_LIMITS)) {
3533                         /* Not supported skip to next probe */
3534                         softc->state = DA_STATE_PROBE_BDC;
3535                         goto skipstate;
3536                 }
3537
3538                 block_limits = (struct scsi_vpd_block_limits *)
3539                         malloc(sizeof(*block_limits), M_SCSIDA, M_NOWAIT|M_ZERO);
3540
3541                 if (block_limits == NULL) {
3542                         printf("dastart: Couldn't malloc block_limits data\n");
3543                         /* da_free_periph??? */
3544                         break;
3545                 }
3546
3547                 scsi_inquiry(&start_ccb->csio,
3548                              /*retries*/da_retry_count,
3549                              /*cbfcnp*/dadone_probeblklimits,
3550                              /*tag_action*/MSG_SIMPLE_Q_TAG,
3551                              /*inq_buf*/(u_int8_t *)block_limits,
3552                              /*inq_len*/sizeof(*block_limits),
3553                              /*evpd*/TRUE,
3554                              /*page_code*/SVPD_BLOCK_LIMITS,
3555                              /*sense_len*/SSD_MIN_SIZE,
3556                              /*timeout*/da_default_timeout * 1000);
3557                 start_ccb->ccb_h.ccb_bp = NULL;
3558                 start_ccb->ccb_h.ccb_state = DA_CCB_PROBE_BLK_LIMITS;
3559                 xpt_action(start_ccb);
3560                 break;
3561         }
3562         case DA_STATE_PROBE_BDC:
3563         {
3564                 struct scsi_vpd_block_characteristics *bdc;
3565
3566                 if (!scsi_vpd_supported_page(periph, SVPD_BDC)) {
3567                         softc->state = DA_STATE_PROBE_ATA;
3568                         goto skipstate;
3569                 }
3570
3571                 bdc = (struct scsi_vpd_block_characteristics *)
3572                         malloc(sizeof(*bdc), M_SCSIDA, M_NOWAIT|M_ZERO);
3573
3574                 if (bdc == NULL) {
3575                         printf("dastart: Couldn't malloc bdc data\n");
3576                         /* da_free_periph??? */
3577                         break;
3578                 }
3579
3580                 scsi_inquiry(&start_ccb->csio,
3581                              /*retries*/da_retry_count,
3582                              /*cbfcnp*/dadone_probebdc,
3583                              /*tag_action*/MSG_SIMPLE_Q_TAG,
3584                              /*inq_buf*/(u_int8_t *)bdc,
3585                              /*inq_len*/sizeof(*bdc),
3586                              /*evpd*/TRUE,
3587                              /*page_code*/SVPD_BDC,
3588                              /*sense_len*/SSD_MIN_SIZE,
3589                              /*timeout*/da_default_timeout * 1000);
3590                 start_ccb->ccb_h.ccb_bp = NULL;
3591                 start_ccb->ccb_h.ccb_state = DA_CCB_PROBE_BDC;
3592                 xpt_action(start_ccb);
3593                 break;
3594         }
3595         case DA_STATE_PROBE_ATA:
3596         {
3597                 struct ata_params *ata_params;
3598
3599                 if (!scsi_vpd_supported_page(periph, SVPD_ATA_INFORMATION)) {
3600                         if ((softc->zone_mode == DA_ZONE_HOST_AWARE)
3601                          || (softc->zone_mode == DA_ZONE_HOST_MANAGED)) {
3602                                 /*
3603                                  * Note that if the ATA VPD page isn't
3604                                  * supported, we aren't talking to an ATA
3605                                  * device anyway.  Support for that VPD
3606                                  * page is mandatory for SCSI to ATA (SAT)
3607                                  * translation layers.
3608                                  */
3609                                 softc->state = DA_STATE_PROBE_ZONE;
3610                                 goto skipstate;
3611                         }
3612                         daprobedone(periph, start_ccb);
3613                         break;
3614                 }
3615
3616                 ata_params = (struct ata_params*)
3617                         malloc(sizeof(*ata_params), M_SCSIDA,M_NOWAIT|M_ZERO);
3618
3619                 if (ata_params == NULL) {
3620                         xpt_print(periph->path, "Couldn't malloc ata_params "
3621                             "data\n");
3622                         /* da_free_periph??? */
3623                         break;
3624                 }
3625
3626                 scsi_ata_identify(&start_ccb->csio,
3627                                   /*retries*/da_retry_count,
3628                                   /*cbfcnp*/dadone_probeata,
3629                                   /*tag_action*/MSG_SIMPLE_Q_TAG,
3630                                   /*data_ptr*/(u_int8_t *)ata_params,
3631                                   /*dxfer_len*/sizeof(*ata_params),
3632                                   /*sense_len*/SSD_FULL_SIZE,
3633                                   /*timeout*/da_default_timeout * 1000);
3634                 start_ccb->ccb_h.ccb_bp = NULL;
3635                 start_ccb->ccb_h.ccb_state = DA_CCB_PROBE_ATA;
3636                 xpt_action(start_ccb);
3637                 break;
3638         }
3639         case DA_STATE_PROBE_ATA_LOGDIR:
3640         {
3641                 struct ata_gp_log_dir *log_dir;
3642                 int retval;
3643
3644                 retval = 0;
3645
3646                 if ((softc->flags & DA_FLAG_CAN_ATA_LOG) == 0) {
3647                         /*
3648                          * If we don't have log support, not much point in
3649                          * trying to probe zone support.
3650                          */
3651                         daprobedone(periph, start_ccb);
3652                         break;
3653                 }
3654
3655                 /*
3656                  * If we have an ATA device (the SCSI ATA Information VPD
3657                  * page should be present and the ATA identify should have
3658                  * succeeded) and it supports logs, ask for the log directory.
3659                  */
3660
3661                 log_dir = malloc(sizeof(*log_dir), M_SCSIDA, M_NOWAIT|M_ZERO);
3662                 if (log_dir == NULL) {
3663                         xpt_print(periph->path, "Couldn't malloc log_dir "
3664                             "data\n");
3665                         daprobedone(periph, start_ccb);
3666                         break;
3667                 }
3668
3669                 retval = scsi_ata_read_log(&start_ccb->csio,
3670                     /*retries*/ da_retry_count,
3671                     /*cbfcnp*/ dadone_probeatalogdir,
3672                     /*tag_action*/ MSG_SIMPLE_Q_TAG,
3673                     /*log_address*/ ATA_LOG_DIRECTORY,
3674                     /*page_number*/ 0,
3675                     /*block_count*/ 1,
3676                     /*protocol*/ softc->flags & DA_FLAG_CAN_ATA_DMA ?
3677                                  AP_PROTO_DMA : AP_PROTO_PIO_IN,
3678                     /*data_ptr*/ (uint8_t *)log_dir,
3679                     /*dxfer_len*/ sizeof(*log_dir),
3680                     /*sense_len*/ SSD_FULL_SIZE,
3681                     /*timeout*/ da_default_timeout * 1000);
3682
3683                 if (retval != 0) {
3684                         xpt_print(periph->path, "scsi_ata_read_log() failed!");
3685                         free(log_dir, M_SCSIDA);
3686                         daprobedone(periph, start_ccb);
3687                         break;
3688                 }
3689                 start_ccb->ccb_h.ccb_bp = NULL;
3690                 start_ccb->ccb_h.ccb_state = DA_CCB_PROBE_ATA_LOGDIR;
3691                 xpt_action(start_ccb);
3692                 break;
3693         }
3694         case DA_STATE_PROBE_ATA_IDDIR:
3695         {
3696                 struct ata_identify_log_pages *id_dir;
3697                 int retval;
3698
3699                 retval = 0;
3700
3701                 /*
3702                  * Check here to see whether the Identify Device log is
3703                  * supported in the directory of logs.  If so, continue
3704                  * with requesting the log of identify device pages.
3705                  */
3706                 if ((softc->flags & DA_FLAG_CAN_ATA_IDLOG) == 0) {
3707                         daprobedone(periph, start_ccb);
3708                         break;
3709                 }
3710
3711                 id_dir = malloc(sizeof(*id_dir), M_SCSIDA, M_NOWAIT | M_ZERO);
3712                 if (id_dir == NULL) {
3713                         xpt_print(periph->path, "Couldn't malloc id_dir "
3714                             "data\n");
3715                         daprobedone(periph, start_ccb);
3716                         break;
3717                 }
3718
3719                 retval = scsi_ata_read_log(&start_ccb->csio,
3720                     /*retries*/ da_retry_count,
3721                     /*cbfcnp*/ dadone_probeataiddir,
3722                     /*tag_action*/ MSG_SIMPLE_Q_TAG,
3723                     /*log_address*/ ATA_IDENTIFY_DATA_LOG,
3724                     /*page_number*/ ATA_IDL_PAGE_LIST,
3725                     /*block_count*/ 1,
3726                     /*protocol*/ softc->flags & DA_FLAG_CAN_ATA_DMA ?
3727                                  AP_PROTO_DMA : AP_PROTO_PIO_IN,
3728                     /*data_ptr*/ (uint8_t *)id_dir,
3729                     /*dxfer_len*/ sizeof(*id_dir),
3730                     /*sense_len*/ SSD_FULL_SIZE,
3731                     /*timeout*/ da_default_timeout * 1000);
3732
3733                 if (retval != 0) {
3734                         xpt_print(periph->path, "scsi_ata_read_log() failed!");
3735                         free(id_dir, M_SCSIDA);
3736                         daprobedone(periph, start_ccb);
3737                         break;
3738                 }
3739                 start_ccb->ccb_h.ccb_bp = NULL;
3740                 start_ccb->ccb_h.ccb_state = DA_CCB_PROBE_ATA_IDDIR;
3741                 xpt_action(start_ccb);
3742                 break;
3743         }
3744         case DA_STATE_PROBE_ATA_SUP:
3745         {
3746                 struct ata_identify_log_sup_cap *sup_cap;
3747                 int retval;
3748
3749                 retval = 0;
3750
3751                 /*
3752                  * Check here to see whether the Supported Capabilities log
3753                  * is in the list of Identify Device logs.
3754                  */
3755                 if ((softc->flags & DA_FLAG_CAN_ATA_SUPCAP) == 0) {
3756                         daprobedone(periph, start_ccb);
3757                         break;
3758                 }
3759
3760                 sup_cap = malloc(sizeof(*sup_cap), M_SCSIDA, M_NOWAIT|M_ZERO);
3761                 if (sup_cap == NULL) {
3762                         xpt_print(periph->path, "Couldn't malloc sup_cap "
3763                             "data\n");
3764                         daprobedone(periph, start_ccb);
3765                         break;
3766                 }
3767
3768                 retval = scsi_ata_read_log(&start_ccb->csio,
3769                     /*retries*/ da_retry_count,
3770                     /*cbfcnp*/ dadone_probeatasup,
3771                     /*tag_action*/ MSG_SIMPLE_Q_TAG,
3772                     /*log_address*/ ATA_IDENTIFY_DATA_LOG,
3773                     /*page_number*/ ATA_IDL_SUP_CAP,
3774                     /*block_count*/ 1,
3775                     /*protocol*/ softc->flags & DA_FLAG_CAN_ATA_DMA ?
3776                                  AP_PROTO_DMA : AP_PROTO_PIO_IN,
3777                     /*data_ptr*/ (uint8_t *)sup_cap,
3778                     /*dxfer_len*/ sizeof(*sup_cap),
3779                     /*sense_len*/ SSD_FULL_SIZE,
3780                     /*timeout*/ da_default_timeout * 1000);
3781
3782                 if (retval != 0) {
3783                         xpt_print(periph->path, "scsi_ata_read_log() failed!");
3784                         free(sup_cap, M_SCSIDA);
3785                         daprobedone(periph, start_ccb);
3786                         break;
3787
3788                 }
3789
3790                 start_ccb->ccb_h.ccb_bp = NULL;
3791                 start_ccb->ccb_h.ccb_state = DA_CCB_PROBE_ATA_SUP;
3792                 xpt_action(start_ccb);
3793                 break;
3794         }
3795         case DA_STATE_PROBE_ATA_ZONE:
3796         {
3797                 struct ata_zoned_info_log *ata_zone;
3798                 int retval;
3799
3800                 retval = 0;
3801
3802                 /*
3803                  * Check here to see whether the zoned device information
3804                  * page is supported.  If so, continue on to request it.
3805                  * If not, skip to DA_STATE_PROBE_LOG or done.
3806                  */
3807                 if ((softc->flags & DA_FLAG_CAN_ATA_ZONE) == 0) {
3808                         daprobedone(periph, start_ccb);
3809                         break;
3810                 }
3811                 ata_zone = malloc(sizeof(*ata_zone), M_SCSIDA,
3812                                   M_NOWAIT|M_ZERO);
3813                 if (ata_zone == NULL) {
3814                         xpt_print(periph->path, "Couldn't malloc ata_zone "
3815                             "data\n");
3816                         daprobedone(periph, start_ccb);
3817                         break;
3818                 }
3819
3820                 retval = scsi_ata_read_log(&start_ccb->csio,
3821                     /*retries*/ da_retry_count,
3822                     /*cbfcnp*/ dadone_probeatazone,
3823                     /*tag_action*/ MSG_SIMPLE_Q_TAG,
3824                     /*log_address*/ ATA_IDENTIFY_DATA_LOG,
3825                     /*page_number*/ ATA_IDL_ZDI,
3826                     /*block_count*/ 1,
3827                     /*protocol*/ softc->flags & DA_FLAG_CAN_ATA_DMA ?
3828                                  AP_PROTO_DMA : AP_PROTO_PIO_IN,
3829                     /*data_ptr*/ (uint8_t *)ata_zone,
3830                     /*dxfer_len*/ sizeof(*ata_zone),
3831                     /*sense_len*/ SSD_FULL_SIZE,
3832                     /*timeout*/ da_default_timeout * 1000);
3833
3834                 if (retval != 0) {
3835                         xpt_print(periph->path, "scsi_ata_read_log() failed!");
3836                         free(ata_zone, M_SCSIDA);
3837                         daprobedone(periph, start_ccb);
3838                         break;
3839                 }
3840                 start_ccb->ccb_h.ccb_bp = NULL;
3841                 start_ccb->ccb_h.ccb_state = DA_CCB_PROBE_ATA_ZONE;
3842                 xpt_action(start_ccb);
3843
3844                 break;
3845         }
3846         case DA_STATE_PROBE_ZONE:
3847         {
3848                 struct scsi_vpd_zoned_bdc *bdc;
3849
3850                 /*
3851                  * Note that this page will be supported for SCSI protocol
3852                  * devices that support ZBC (SMR devices), as well as ATA
3853                  * protocol devices that are behind a SAT (SCSI to ATA
3854                  * Translation) layer that supports converting ZBC commands
3855                  * to their ZAC equivalents.
3856                  */
3857                 if (!scsi_vpd_supported_page(periph, SVPD_ZONED_BDC)) {
3858                         daprobedone(periph, start_ccb);
3859                         break;
3860                 }
3861                 bdc = (struct scsi_vpd_zoned_bdc *)
3862                         malloc(sizeof(*bdc), M_SCSIDA, M_NOWAIT|M_ZERO);
3863
3864                 if (bdc == NULL) {
3865                         xpt_release_ccb(start_ccb);
3866                         xpt_print(periph->path, "Couldn't malloc zone VPD "
3867                             "data\n");
3868                         break;
3869                 }
3870                 scsi_inquiry(&start_ccb->csio,
3871                              /*retries*/da_retry_count,
3872                              /*cbfcnp*/dadone_probezone,
3873                              /*tag_action*/MSG_SIMPLE_Q_TAG,
3874                              /*inq_buf*/(u_int8_t *)bdc,
3875                              /*inq_len*/sizeof(*bdc),
3876                              /*evpd*/TRUE,
3877                              /*page_code*/SVPD_ZONED_BDC,
3878                              /*sense_len*/SSD_FULL_SIZE,
3879                              /*timeout*/da_default_timeout * 1000);
3880                 start_ccb->ccb_h.ccb_bp = NULL;
3881                 start_ccb->ccb_h.ccb_state = DA_CCB_PROBE_ZONE;
3882                 xpt_action(start_ccb);
3883                 break;
3884         }
3885         }
3886 }
3887
3888 /*
3889  * In each of the methods below, while its the caller's
3890  * responsibility to ensure the request will fit into a
3891  * single device request, we might have changed the delete
3892  * method due to the device incorrectly advertising either
3893  * its supported methods or limits.
3894  *
3895  * To prevent this causing further issues we validate the
3896  * against the methods limits, and warn which would
3897  * otherwise be unnecessary.
3898  */
3899 static void
3900 da_delete_unmap(struct cam_periph *periph, union ccb *ccb, struct bio *bp)
3901 {
3902         struct da_softc *softc = (struct da_softc *)periph->softc;;
3903         struct bio *bp1;
3904         uint8_t *buf = softc->unmap_buf;
3905         struct scsi_unmap_desc *d = (void *)&buf[UNMAP_HEAD_SIZE];
3906         uint64_t lba, lastlba = (uint64_t)-1;
3907         uint64_t totalcount = 0;
3908         uint64_t count;
3909         uint32_t c, lastcount = 0, ranges = 0;
3910
3911         /*
3912          * Currently this doesn't take the UNMAP
3913          * Granularity and Granularity Alignment
3914          * fields into account.
3915          *
3916          * This could result in both unoptimal unmap
3917          * requests as as well as UNMAP calls unmapping
3918          * fewer LBA's than requested.
3919          */
3920
3921         bzero(softc->unmap_buf, sizeof(softc->unmap_buf));
3922         bp1 = bp;
3923         do {
3924                 /*
3925                  * Note: ada and da are different in how they store the
3926                  * pending bp's in a trim. ada stores all of them in the
3927                  * trim_req.bps. da stores all but the first one in the
3928                  * delete_run_queue. ada then completes all the bps in
3929                  * its adadone() loop. da completes all the bps in the
3930                  * delete_run_queue in dadone, and relies on the biodone
3931                  * after to complete. This should be reconciled since there's
3932                  * no real reason to do it differently. XXX
3933                  */
3934                 if (bp1 != bp)
3935                         bioq_insert_tail(&softc->delete_run_queue, bp1);
3936                 lba = bp1->bio_pblkno;
3937                 count = bp1->bio_bcount / softc->params.secsize;
3938
3939                 /* Try to extend the previous range. */
3940                 if (lba == lastlba) {
3941                         c = omin(count, UNMAP_RANGE_MAX - lastcount);
3942                         lastlba += c;
3943                         lastcount += c;
3944                         scsi_ulto4b(lastcount, d[ranges - 1].length);
3945                         count -= c;
3946                         lba += c;
3947                         totalcount += c;
3948                 } else if ((softc->quirks & DA_Q_STRICT_UNMAP) &&
3949                     softc->unmap_gran != 0) {
3950                         /* Align length of the previous range. */
3951                         if ((c = lastcount % softc->unmap_gran) != 0) {
3952                                 if (lastcount <= c) {
3953                                         totalcount -= lastcount;
3954                                         lastlba = (uint64_t)-1;
3955                                         lastcount = 0;
3956                                         ranges--;
3957                                 } else {
3958                                         totalcount -= c;
3959                                         lastlba -= c;
3960                                         lastcount -= c;
3961                                         scsi_ulto4b(lastcount,
3962                                             d[ranges - 1].length);
3963                                 }
3964                         }
3965                         /* Align beginning of the new range. */
3966                         c = (lba - softc->unmap_gran_align) % softc->unmap_gran;
3967                         if (c != 0) {
3968                                 c = softc->unmap_gran - c;
3969                                 if (count <= c) {
3970                                         count = 0;
3971                                 } else {
3972                                         lba += c;
3973                                         count -= c;
3974                                 }
3975                         }
3976                 }
3977
3978                 while (count > 0) {
3979                         c = omin(count, UNMAP_RANGE_MAX);
3980                         if (totalcount + c > softc->unmap_max_lba ||
3981                             ranges >= softc->unmap_max_ranges) {
3982                                 xpt_print(periph->path,
3983                                     "%s issuing short delete %ld > %ld"
3984                                     "|| %d >= %d",
3985                                     da_delete_method_desc[softc->delete_method],
3986                                     totalcount + c, softc->unmap_max_lba,
3987                                     ranges, softc->unmap_max_ranges);
3988                                 break;
3989                         }
3990                         scsi_u64to8b(lba, d[ranges].lba);
3991                         scsi_ulto4b(c, d[ranges].length);
3992                         lba += c;
3993                         totalcount += c;
3994                         ranges++;
3995                         count -= c;
3996                         lastlba = lba;
3997                         lastcount = c;
3998                 }
3999                 bp1 = cam_iosched_next_trim(softc->cam_iosched);
4000                 if (bp1 == NULL)
4001                         break;
4002                 if (ranges >= softc->unmap_max_ranges ||
4003                     totalcount + bp1->bio_bcount /
4004                     softc->params.secsize > softc->unmap_max_lba) {
4005                         cam_iosched_put_back_trim(softc->cam_iosched, bp1);
4006                         break;
4007                 }
4008         } while (1);
4009
4010         /* Align length of the last range. */
4011         if ((softc->quirks & DA_Q_STRICT_UNMAP) && softc->unmap_gran != 0 &&
4012             (c = lastcount % softc->unmap_gran) != 0) {
4013                 if (lastcount <= c)
4014                         ranges--;
4015                 else
4016                         scsi_ulto4b(lastcount - c, d[ranges - 1].length);
4017         }
4018
4019         scsi_ulto2b(ranges * 16 + 6, &buf[0]);
4020         scsi_ulto2b(ranges * 16, &buf[2]);
4021
4022         scsi_unmap(&ccb->csio,
4023                    /*retries*/da_retry_count,
4024                    /*cbfcnp*/dadone,
4025                    /*tag_action*/MSG_SIMPLE_Q_TAG,
4026                    /*byte2*/0,
4027                    /*data_ptr*/ buf,
4028                    /*dxfer_len*/ ranges * 16 + 8,
4029                    /*sense_len*/SSD_FULL_SIZE,
4030                    da_default_timeout * 1000);
4031         ccb->ccb_h.ccb_state = DA_CCB_DELETE;
4032         ccb->ccb_h.flags |= CAM_UNLOCKED;
4033         softc->trim_count++;
4034         softc->trim_ranges += ranges;
4035         softc->trim_lbas += totalcount;
4036         cam_iosched_submit_trim(softc->cam_iosched);
4037 }
4038
4039 static void
4040 da_delete_trim(struct cam_periph *periph, union ccb *ccb, struct bio *bp)
4041 {
4042         struct da_softc *softc = (struct da_softc *)periph->softc;
4043         struct bio *bp1;
4044         uint8_t *buf = softc->unmap_buf;
4045         uint64_t lastlba = (uint64_t)-1;
4046         uint64_t count;
4047         uint64_t lba;
4048         uint32_t lastcount = 0, c, requestcount;
4049         int ranges = 0, off, block_count;
4050
4051         bzero(softc->unmap_buf, sizeof(softc->unmap_buf));
4052         bp1 = bp;
4053         do {
4054                 if (bp1 != bp)//XXX imp XXX
4055                         bioq_insert_tail(&softc->delete_run_queue, bp1);
4056                 lba = bp1->bio_pblkno;
4057                 count = bp1->bio_bcount / softc->params.secsize;
4058                 requestcount = count;
4059
4060                 /* Try to extend the previous range. */
4061                 if (lba == lastlba) {
4062                         c = omin(count, ATA_DSM_RANGE_MAX - lastcount);
4063                         lastcount += c;
4064                         off = (ranges - 1) * 8;
4065                         buf[off + 6] = lastcount & 0xff;
4066                         buf[off + 7] = (lastcount >> 8) & 0xff;
4067                         count -= c;
4068                         lba += c;
4069                 }
4070
4071                 while (count > 0) {
4072                         c = omin(count, ATA_DSM_RANGE_MAX);
4073                         off = ranges * 8;
4074
4075                         buf[off + 0] = lba & 0xff;
4076                         buf[off + 1] = (lba >> 8) & 0xff;
4077                         buf[off + 2] = (lba >> 16) & 0xff;
4078                         buf[off + 3] = (lba >> 24) & 0xff;
4079                         buf[off + 4] = (lba >> 32) & 0xff;
4080                         buf[off + 5] = (lba >> 40) & 0xff;
4081                         buf[off + 6] = c & 0xff;
4082                         buf[off + 7] = (c >> 8) & 0xff;
4083                         lba += c;
4084                         ranges++;
4085                         count -= c;
4086                         lastcount = c;
4087                         if (count != 0 && ranges == softc->trim_max_ranges) {
4088                                 xpt_print(periph->path,
4089                                     "%s issuing short delete %ld > %ld\n",
4090                                     da_delete_method_desc[softc->delete_method],
4091                                     requestcount,
4092                                     (softc->trim_max_ranges - ranges) *
4093                                     ATA_DSM_RANGE_MAX);
4094                                 break;
4095                         }
4096                 }
4097                 lastlba = lba;
4098                 bp1 = cam_iosched_next_trim(softc->cam_iosched);
4099                 if (bp1 == NULL)
4100                         break;
4101                 if (bp1->bio_bcount / softc->params.secsize >
4102                     (softc->trim_max_ranges - ranges) * ATA_DSM_RANGE_MAX) {
4103                         cam_iosched_put_back_trim(softc->cam_iosched, bp1);
4104                         break;
4105                 }
4106         } while (1);
4107
4108         block_count = howmany(ranges, ATA_DSM_BLK_RANGES);
4109         scsi_ata_trim(&ccb->csio,
4110                       /*retries*/da_retry_count,
4111                       /*cbfcnp*/dadone,
4112                       /*tag_action*/MSG_SIMPLE_Q_TAG,
4113                       block_count,
4114                       /*data_ptr*/buf,
4115                       /*dxfer_len*/block_count * ATA_DSM_BLK_SIZE,
4116                       /*sense_len*/SSD_FULL_SIZE,
4117                       da_default_timeout * 1000);
4118         ccb->ccb_h.ccb_state = DA_CCB_DELETE;
4119         ccb->ccb_h.flags |= CAM_UNLOCKED;
4120         cam_iosched_submit_trim(softc->cam_iosched);
4121 }
4122
4123 /*
4124  * We calculate ws_max_blks here based off d_delmaxsize instead
4125  * of using softc->ws_max_blks as it is absolute max for the
4126  * device not the protocol max which may well be lower.
4127  */
4128 static void
4129 da_delete_ws(struct cam_periph *periph, union ccb *ccb, struct bio *bp)
4130 {
4131         struct da_softc *softc;
4132         struct bio *bp1;
4133         uint64_t ws_max_blks;
4134         uint64_t lba;
4135         uint64_t count; /* forward compat with WS32 */
4136
4137         softc = (struct da_softc *)periph->softc;
4138         ws_max_blks = softc->disk->d_delmaxsize / softc->params.secsize;
4139         lba = bp->bio_pblkno;
4140         count = 0;
4141         bp1 = bp;
4142         do {
4143                 if (bp1 != bp)//XXX imp XXX
4144                         bioq_insert_tail(&softc->delete_run_queue, bp1);
4145                 count += bp1->bio_bcount / softc->params.secsize;
4146                 if (count > ws_max_blks) {
4147                         xpt_print(periph->path,
4148                             "%s issuing short delete %ld > %ld\n",
4149                             da_delete_method_desc[softc->delete_method],
4150                             count, ws_max_blks);
4151                         count = omin(count, ws_max_blks);
4152                         break;
4153                 }
4154                 bp1 = cam_iosched_next_trim(softc->cam_iosched);
4155                 if (bp1 == NULL)
4156                         break;
4157                 if (lba + count != bp1->bio_pblkno ||
4158                     count + bp1->bio_bcount /
4159                     softc->params.secsize > ws_max_blks) {
4160                         cam_iosched_put_back_trim(softc->cam_iosched, bp1);
4161                         break;
4162                 }
4163         } while (1);
4164
4165         scsi_write_same(&ccb->csio,
4166                         /*retries*/da_retry_count,
4167                         /*cbfcnp*/dadone,
4168                         /*tag_action*/MSG_SIMPLE_Q_TAG,
4169                         /*byte2*/softc->delete_method ==
4170                             DA_DELETE_ZERO ? 0 : SWS_UNMAP,
4171                         softc->delete_method == DA_DELETE_WS16 ? 16 : 10,
4172                         /*lba*/lba,
4173                         /*block_count*/count,
4174                         /*data_ptr*/ __DECONST(void *, zero_region),
4175                         /*dxfer_len*/ softc->params.secsize,
4176                         /*sense_len*/SSD_FULL_SIZE,
4177                         da_default_timeout * 1000);
4178         ccb->ccb_h.ccb_state = DA_CCB_DELETE;
4179         ccb->ccb_h.flags |= CAM_UNLOCKED;
4180         cam_iosched_submit_trim(softc->cam_iosched);
4181 }
4182
4183 static int
4184 cmd6workaround(union ccb *ccb)
4185 {
4186         struct scsi_rw_6 cmd6;
4187         struct scsi_rw_10 *cmd10;
4188         struct da_softc *softc;
4189         u_int8_t *cdb;
4190         struct bio *bp;
4191         int frozen;
4192
4193         cdb = ccb->csio.cdb_io.cdb_bytes;
4194         softc = (struct da_softc *)xpt_path_periph(ccb->ccb_h.path)->softc;
4195
4196         if (ccb->ccb_h.ccb_state == DA_CCB_DELETE) {
4197                 da_delete_methods old_method = softc->delete_method;
4198
4199                 /*
4200                  * Typically there are two reasons for failure here
4201                  * 1. Delete method was detected as supported but isn't
4202                  * 2. Delete failed due to invalid params e.g. too big
4203                  *
4204                  * While we will attempt to choose an alternative delete method
4205                  * this may result in short deletes if the existing delete
4206                  * requests from geom are big for the new method chosen.
4207                  *
4208                  * This method assumes that the error which triggered this
4209                  * will not retry the io otherwise a panic will occur
4210                  */
4211                 dadeleteflag(softc, old_method, 0);
4212                 dadeletemethodchoose(softc, DA_DELETE_DISABLE);
4213                 if (softc->delete_method == DA_DELETE_DISABLE)
4214                         xpt_print(ccb->ccb_h.path,
4215                                   "%s failed, disabling BIO_DELETE\n",
4216                                   da_delete_method_desc[old_method]);
4217                 else
4218                         xpt_print(ccb->ccb_h.path,
4219                                   "%s failed, switching to %s BIO_DELETE\n",
4220                                   da_delete_method_desc[old_method],
4221                                   da_delete_method_desc[softc->delete_method]);
4222
4223                 while ((bp = bioq_takefirst(&softc->delete_run_queue)) != NULL)
4224                         cam_iosched_queue_work(softc->cam_iosched, bp);
4225                 cam_iosched_queue_work(softc->cam_iosched,
4226                     (struct bio *)ccb->ccb_h.ccb_bp);
4227                 ccb->ccb_h.ccb_bp = NULL;
4228                 return (0);
4229         }
4230
4231         /* Detect unsupported PREVENT ALLOW MEDIUM REMOVAL. */
4232         if ((ccb->ccb_h.flags & CAM_CDB_POINTER) == 0 &&
4233             (*cdb == PREVENT_ALLOW) &&
4234             (softc->quirks & DA_Q_NO_PREVENT) == 0) {
4235                 if (bootverbose)
4236                         xpt_print(ccb->ccb_h.path,
4237                             "PREVENT ALLOW MEDIUM REMOVAL not supported.\n");
4238                 softc->quirks |= DA_Q_NO_PREVENT;
4239                 return (0);
4240         }
4241
4242         /* Detect unsupported SYNCHRONIZE CACHE(10). */
4243         if ((ccb->ccb_h.flags & CAM_CDB_POINTER) == 0 &&
4244             (*cdb == SYNCHRONIZE_CACHE) &&
4245             (softc->quirks & DA_Q_NO_SYNC_CACHE) == 0) {
4246                 if (bootverbose)
4247                         xpt_print(ccb->ccb_h.path,
4248                             "SYNCHRONIZE CACHE(10) not supported.\n");
4249                 softc->quirks |= DA_Q_NO_SYNC_CACHE;
4250                 softc->disk->d_flags &= ~DISKFLAG_CANFLUSHCACHE;
4251                 return (0);
4252         }
4253
4254         /* Translation only possible if CDB is an array and cmd is R/W6 */
4255         if ((ccb->ccb_h.flags & CAM_CDB_POINTER) != 0 ||
4256             (*cdb != READ_6 && *cdb != WRITE_6))
4257                 return 0;
4258
4259         xpt_print(ccb->ccb_h.path, "READ(6)/WRITE(6) not supported, "
4260             "increasing minimum_cmd_size to 10.\n");
4261         softc->minimum_cmd_size = 10;
4262
4263         bcopy(cdb, &cmd6, sizeof(struct scsi_rw_6));
4264         cmd10 = (struct scsi_rw_10 *)cdb;
4265         cmd10->opcode = (cmd6.opcode == READ_6) ? READ_10 : WRITE_10;
4266         cmd10->byte2 = 0;
4267         scsi_ulto4b(scsi_3btoul(cmd6.addr), cmd10->addr);
4268         cmd10->reserved = 0;
4269         scsi_ulto2b(cmd6.length, cmd10->length);
4270         cmd10->control = cmd6.control;
4271         ccb->csio.cdb_len = sizeof(*cmd10);
4272
4273         /* Requeue request, unfreezing queue if necessary */
4274         frozen = (ccb->ccb_h.status & CAM_DEV_QFRZN) != 0;
4275         ccb->ccb_h.status = CAM_REQUEUE_REQ;
4276         xpt_action(ccb);
4277         if (frozen) {
4278                 cam_release_devq(ccb->ccb_h.path,
4279                                  /*relsim_flags*/0,
4280                                  /*reduction*/0,
4281                                  /*timeout*/0,
4282                                  /*getcount_only*/0);
4283         }
4284         return (ERESTART);
4285 }
4286
4287 static void
4288 dazonedone(struct cam_periph *periph, union ccb *ccb)
4289 {
4290         struct da_softc *softc;
4291         struct bio *bp;
4292
4293         softc = periph->softc;
4294         bp = (struct bio *)ccb->ccb_h.ccb_bp;
4295
4296         switch (bp->bio_zone.zone_cmd) {
4297         case DISK_ZONE_OPEN:
4298         case DISK_ZONE_CLOSE:
4299         case DISK_ZONE_FINISH:
4300         case DISK_ZONE_RWP:
4301                 break;
4302         case DISK_ZONE_REPORT_ZONES: {
4303                 uint32_t avail_len;
4304                 struct disk_zone_report *rep;
4305                 struct scsi_report_zones_hdr *hdr;
4306                 struct scsi_report_zones_desc *desc;
4307                 struct disk_zone_rep_entry *entry;
4308                 uint32_t hdr_len, num_avail;
4309                 uint32_t num_to_fill, i;
4310                 int ata;
4311
4312                 rep = &bp->bio_zone.zone_params.report;
4313                 avail_len = ccb->csio.dxfer_len - ccb->csio.resid;
4314                 /*
4315                  * Note that bio_resid isn't normally used for zone
4316                  * commands, but it is used by devstat_end_transaction_bio()
4317                  * to determine how much data was transferred.  Because
4318                  * the size of the SCSI/ATA data structures is different
4319                  * than the size of the BIO interface structures, the
4320                  * amount of data actually transferred from the drive will
4321                  * be different than the amount of data transferred to
4322                  * the user.
4323                  */
4324                 bp->bio_resid = ccb->csio.resid;
4325                 hdr = (struct scsi_report_zones_hdr *)ccb->csio.data_ptr;
4326                 if (avail_len < sizeof(*hdr)) {
4327                         /*
4328                          * Is there a better error than EIO here?  We asked
4329                          * for at least the header, and we got less than
4330                          * that.
4331                          */
4332                         bp->bio_error = EIO;
4333                         bp->bio_flags |= BIO_ERROR;
4334                         bp->bio_resid = bp->bio_bcount;
4335                         break;
4336                 }
4337
4338                 if (softc->zone_interface == DA_ZONE_IF_ATA_PASS)
4339                         ata = 1;
4340                 else
4341                         ata = 0;
4342
4343                 hdr_len = ata ? le32dec(hdr->length) :
4344                                 scsi_4btoul(hdr->length);
4345                 if (hdr_len > 0)
4346                         rep->entries_available = hdr_len / sizeof(*desc);
4347                 else
4348                         rep->entries_available = 0;
4349                 /*
4350                  * NOTE: using the same values for the BIO version of the
4351                  * same field as the SCSI/ATA values.  This means we could
4352                  * get some additional values that aren't defined in bio.h
4353                  * if more values of the same field are defined later.
4354                  */
4355                 rep->header.same = hdr->byte4 & SRZ_SAME_MASK;
4356                 rep->header.maximum_lba = ata ?  le64dec(hdr->maximum_lba) :
4357                                           scsi_8btou64(hdr->maximum_lba);
4358                 /*
4359                  * If the drive reports no entries that match the query,
4360                  * we're done.
4361                  */
4362                 if (hdr_len == 0) {
4363                         rep->entries_filled = 0;
4364                         break;
4365                 }
4366
4367                 num_avail = min((avail_len - sizeof(*hdr)) / sizeof(*desc),
4368                                 hdr_len / sizeof(*desc));
4369                 /*
4370                  * If the drive didn't return any data, then we're done.
4371                  */
4372                 if (num_avail == 0) {
4373                         rep->entries_filled = 0;
4374                         break;
4375                 }
4376
4377                 num_to_fill = min(num_avail, rep->entries_allocated);
4378                 /*
4379                  * If the user didn't allocate any entries for us to fill,
4380                  * we're done.
4381                  */
4382                 if (num_to_fill == 0) {
4383                         rep->entries_filled = 0;
4384                         break;
4385                 }
4386
4387                 for (i = 0, desc = &hdr->desc_list[0], entry=&rep->entries[0];
4388                      i < num_to_fill; i++, desc++, entry++) {
4389                         /*
4390                          * NOTE: we're mapping the values here directly
4391                          * from the SCSI/ATA bit definitions to the bio.h
4392                          * definitons.  There is also a warning in
4393                          * disk_zone.h, but the impact is that if
4394                          * additional values are added in the SCSI/ATA
4395                          * specs these will be visible to consumers of
4396                          * this interface.
4397                          */
4398                         entry->zone_type = desc->zone_type & SRZ_TYPE_MASK;
4399                         entry->zone_condition =
4400                             (desc->zone_flags & SRZ_ZONE_COND_MASK) >>
4401                             SRZ_ZONE_COND_SHIFT;
4402                         entry->zone_flags |= desc->zone_flags &
4403                             (SRZ_ZONE_NON_SEQ|SRZ_ZONE_RESET);
4404                         entry->zone_length =
4405                             ata ? le64dec(desc->zone_length) :
4406                                   scsi_8btou64(desc->zone_length);
4407                         entry->zone_start_lba =
4408                             ata ? le64dec(desc->zone_start_lba) :
4409                                   scsi_8btou64(desc->zone_start_lba);
4410                         entry->write_pointer_lba =
4411                             ata ? le64dec(desc->write_pointer_lba) :
4412                                   scsi_8btou64(desc->write_pointer_lba);
4413                 }
4414                 rep->entries_filled = num_to_fill;
4415                 break;
4416         }
4417         case DISK_ZONE_GET_PARAMS:
4418         default:
4419                 /*
4420                  * In theory we should not get a GET_PARAMS bio, since it
4421                  * should be handled without queueing the command to the
4422                  * drive.
4423                  */
4424                 panic("%s: Invalid zone command %d", __func__,
4425                     bp->bio_zone.zone_cmd);
4426                 break;
4427         }
4428
4429         if (bp->bio_zone.zone_cmd == DISK_ZONE_REPORT_ZONES)
4430                 free(ccb->csio.data_ptr, M_SCSIDA);
4431 }
4432
4433 static void
4434 dadone(struct cam_periph *periph, union ccb *done_ccb)
4435 {
4436         struct bio *bp, *bp1;
4437         struct da_softc *softc;
4438         struct ccb_scsiio *csio;
4439         u_int32_t  priority;
4440         da_ccb_state state;
4441
4442         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dadone\n"));
4443
4444         softc = (struct da_softc *)periph->softc;
4445         priority = done_ccb->ccb_h.pinfo.priority;
4446         csio = &done_ccb->csio;
4447
4448 #if defined(BUF_TRACKING) || defined(FULL_BUF_TRACKING)
4449         if (csio->bio != NULL)
4450                 biotrack(csio->bio, __func__);
4451 #endif
4452         state = csio->ccb_h.ccb_state & DA_CCB_TYPE_MASK;
4453
4454         cam_periph_lock(periph);
4455         bp = (struct bio *)done_ccb->ccb_h.ccb_bp;
4456         if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
4457                 int error;
4458                 int sf;
4459
4460                 if ((csio->ccb_h.ccb_state & DA_CCB_RETRY_UA) != 0)
4461                         sf = SF_RETRY_UA;
4462                 else
4463                         sf = 0;
4464
4465                 error = daerror(done_ccb, CAM_RETRY_SELTO, sf);
4466                 if (error == ERESTART) {
4467                         /* A retry was scheduled, so just return. */
4468                         cam_periph_unlock(periph);
4469                         return;
4470                 }
4471                 bp = (struct bio *)done_ccb->ccb_h.ccb_bp;
4472                 if (error != 0) {
4473                         int queued_error;
4474
4475                         /*
4476                          * return all queued I/O with EIO, so that
4477                          * the client can retry these I/Os in the
4478                          * proper order should it attempt to recover.
4479                          */
4480                         queued_error = EIO;
4481
4482                         if (error == ENXIO
4483                          && (softc->flags & DA_FLAG_PACK_INVALID)== 0) {
4484                                 /*
4485                                  * Catastrophic error.  Mark our pack as
4486                                  * invalid.
4487                                  *
4488                                  * XXX See if this is really a media
4489                                  * XXX change first?
4490                                  */
4491                                 xpt_print(periph->path, "Invalidating pack\n");
4492                                 softc->flags |= DA_FLAG_PACK_INVALID;
4493 #ifdef CAM_IO_STATS
4494                                 softc->invalidations++;
4495 #endif
4496                                 queued_error = ENXIO;
4497                         }
4498                         cam_iosched_flush(softc->cam_iosched, NULL,
4499                            queued_error);
4500                         if (bp != NULL) {
4501                                 bp->bio_error = error;
4502                                 bp->bio_resid = bp->bio_bcount;
4503                                 bp->bio_flags |= BIO_ERROR;
4504                         }
4505                 } else if (bp != NULL) {
4506                         if (state == DA_CCB_DELETE)
4507                                 bp->bio_resid = 0;
4508                         else
4509                                 bp->bio_resid = csio->resid;
4510                         bp->bio_error = 0;
4511                         if (bp->bio_resid != 0)
4512                                 bp->bio_flags |= BIO_ERROR;
4513                 }
4514                 if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0)
4515                         cam_release_devq(done_ccb->ccb_h.path,
4516                                          /*relsim_flags*/0,
4517                                          /*reduction*/0,
4518                                          /*timeout*/0,
4519                                          /*getcount_only*/0);
4520         } else if (bp != NULL) {
4521                 if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0)
4522                         panic("REQ_CMP with QFRZN");
4523                 if (bp->bio_cmd == BIO_ZONE)
4524                         dazonedone(periph, done_ccb);
4525                 else if (state == DA_CCB_DELETE)
4526                         bp->bio_resid = 0;
4527                 else
4528                         bp->bio_resid = csio->resid;
4529                 if ((csio->resid > 0) && (bp->bio_cmd != BIO_ZONE))
4530                         bp->bio_flags |= BIO_ERROR;
4531                 if (softc->error_inject != 0) {
4532                         bp->bio_error = softc->error_inject;
4533                         bp->bio_resid = bp->bio_bcount;
4534                         bp->bio_flags |= BIO_ERROR;
4535                         softc->error_inject = 0;
4536                 }
4537         }
4538
4539         if (bp != NULL)
4540                 biotrack(bp, __func__);
4541         LIST_REMOVE(&done_ccb->ccb_h, periph_links.le);
4542         if (LIST_EMPTY(&softc->pending_ccbs))
4543                 softc->flags |= DA_FLAG_WAS_OTAG;
4544
4545         /*
4546          * We need to call cam_iosched before we call biodone so that we don't
4547          * measure any activity that happens in the completion routine, which in
4548          * the case of sendfile can be quite extensive. Release the periph
4549          * refcount taken in dastart() for each CCB.
4550          */
4551         cam_iosched_bio_complete(softc->cam_iosched, bp, done_ccb);
4552         xpt_release_ccb(done_ccb);
4553         KASSERT(softc->refcount >= 1, ("dadone softc %p refcount %d", softc, softc->refcount));
4554         softc->refcount--;
4555         if (state == DA_CCB_DELETE) {
4556                 TAILQ_HEAD(, bio) queue;
4557
4558                 TAILQ_INIT(&queue);
4559                 TAILQ_CONCAT(&queue, &softc->delete_run_queue.queue, bio_queue);
4560                 softc->delete_run_queue.insert_point = NULL;
4561                 /*
4562                  * Normally, the xpt_release_ccb() above would make sure
4563                  * that when we have more work to do, that work would
4564                  * get kicked off. However, we specifically keep
4565                  * delete_running set to 0 before the call above to
4566                  * allow other I/O to progress when many BIO_DELETE
4567                  * requests are pushed down. We set delete_running to 0
4568                  * and call daschedule again so that we don't stall if
4569                  * there are no other I/Os pending apart from BIO_DELETEs.
4570                  */
4571                 cam_iosched_trim_done(softc->cam_iosched);
4572                 daschedule(periph);
4573                 cam_periph_unlock(periph);
4574                 while ((bp1 = TAILQ_FIRST(&queue)) != NULL) {
4575                         TAILQ_REMOVE(&queue, bp1, bio_queue);
4576                         bp1->bio_error = bp->bio_error;
4577                         if (bp->bio_flags & BIO_ERROR) {
4578                                 bp1->bio_flags |= BIO_ERROR;
4579                                 bp1->bio_resid = bp1->bio_bcount;
4580                         } else
4581                                 bp1->bio_resid = 0;
4582                         biodone(bp1);
4583                 }
4584         } else {
4585                 daschedule(periph);
4586                 cam_periph_unlock(periph);
4587         }
4588         if (bp != NULL)
4589                 biodone(bp);
4590         return;
4591 }
4592
4593 static void
4594 dadone_probewp(struct cam_periph *periph, union ccb *done_ccb)
4595 {
4596         struct scsi_mode_header_6 *mode_hdr6;
4597         struct scsi_mode_header_10 *mode_hdr10;
4598         struct da_softc *softc;
4599         struct ccb_scsiio *csio;
4600         u_int32_t  priority;
4601         uint8_t dev_spec;
4602
4603         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dadone_probewp\n"));
4604
4605         softc = (struct da_softc *)periph->softc;
4606         priority = done_ccb->ccb_h.pinfo.priority;
4607         csio = &done_ccb->csio;
4608
4609         cam_periph_assert(periph, MA_OWNED);
4610
4611         if (softc->minimum_cmd_size > 6) {
4612                 mode_hdr10 = (struct scsi_mode_header_10 *)csio->data_ptr;
4613                 dev_spec = mode_hdr10->dev_spec;
4614         } else {
4615                 mode_hdr6 = (struct scsi_mode_header_6 *)csio->data_ptr;
4616                 dev_spec = mode_hdr6->dev_spec;
4617         }
4618         if (cam_ccb_status(done_ccb) == CAM_REQ_CMP) {
4619                 if ((dev_spec & 0x80) != 0)
4620                         softc->disk->d_flags |= DISKFLAG_WRITE_PROTECT;
4621                 else
4622                         softc->disk->d_flags &= ~DISKFLAG_WRITE_PROTECT;
4623         } else {
4624                 int error;
4625
4626                 error = daerror(done_ccb, CAM_RETRY_SELTO,
4627                                 SF_RETRY_UA|SF_NO_PRINT);
4628                 if (error == ERESTART)
4629                         return;
4630                 else if (error != 0) {
4631                         if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
4632                                 /* Don't wedge this device's queue */
4633                                 cam_release_devq(done_ccb->ccb_h.path,
4634                                                  /*relsim_flags*/0,
4635                                                  /*reduction*/0,
4636                                                  /*timeout*/0,
4637                                                  /*getcount_only*/0);
4638                         }
4639                 }
4640         }
4641
4642         free(csio->data_ptr, M_SCSIDA);
4643         xpt_release_ccb(done_ccb);
4644         if ((softc->flags & DA_FLAG_CAN_RC16) != 0)
4645                 softc->state = DA_STATE_PROBE_RC16;
4646         else
4647                 softc->state = DA_STATE_PROBE_RC;
4648         xpt_schedule(periph, priority);
4649         return;
4650 }
4651
4652 static void
4653 dadone_proberc(struct cam_periph *periph, union ccb *done_ccb)
4654 {
4655         struct scsi_read_capacity_data *rdcap;
4656         struct scsi_read_capacity_data_long *rcaplong;
4657         struct da_softc *softc;
4658         struct ccb_scsiio *csio;
4659         da_ccb_state state;
4660         char *announce_buf;
4661         u_int32_t  priority;
4662         int lbp, n;
4663
4664         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dadone_proberc\n"));
4665
4666         softc = (struct da_softc *)periph->softc;
4667         priority = done_ccb->ccb_h.pinfo.priority;
4668         csio = &done_ccb->csio;
4669         state = csio->ccb_h.ccb_state & DA_CCB_TYPE_MASK;
4670
4671         lbp = 0;
4672         rdcap = NULL;
4673         rcaplong = NULL;
4674         /* XXX TODO: can this be a malloc? */
4675         announce_buf = softc->announce_temp;
4676         bzero(announce_buf, DA_ANNOUNCETMP_SZ);
4677
4678         if (state == DA_CCB_PROBE_RC)
4679                 rdcap =(struct scsi_read_capacity_data *)csio->data_ptr;
4680         else
4681                 rcaplong = (struct scsi_read_capacity_data_long *)
4682                         csio->data_ptr;
4683
4684         cam_periph_assert(periph, MA_OWNED);
4685
4686         if ((csio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
4687                 struct disk_params *dp;
4688                 uint32_t block_size;
4689                 uint64_t maxsector;
4690                 u_int lalba;    /* Lowest aligned LBA. */
4691
4692                 if (state == DA_CCB_PROBE_RC) {
4693                         block_size = scsi_4btoul(rdcap->length);
4694                         maxsector = scsi_4btoul(rdcap->addr);
4695                         lalba = 0;
4696
4697                         /*
4698                          * According to SBC-2, if the standard 10
4699                          * byte READ CAPACITY command returns 2^32,
4700                          * we should issue the 16 byte version of
4701                          * the command, since the device in question
4702                          * has more sectors than can be represented
4703                          * with the short version of the command.
4704                          */
4705                         if (maxsector == 0xffffffff) {
4706                                 free(rdcap, M_SCSIDA);
4707                                 xpt_release_ccb(done_ccb);
4708                                 softc->state = DA_STATE_PROBE_RC16;
4709                                 xpt_schedule(periph, priority);
4710                                 return;
4711                         }
4712                 } else {
4713                         block_size = scsi_4btoul(rcaplong->length);
4714                         maxsector = scsi_8btou64(rcaplong->addr);
4715                         lalba = scsi_2btoul(rcaplong->lalba_lbp);
4716                 }
4717
4718                 /*
4719                  * Because GEOM code just will panic us if we
4720                  * give them an 'illegal' value we'll avoid that
4721                  * here.
4722                  */
4723                 if (block_size == 0) {
4724                         block_size = 512;
4725                         if (maxsector == 0)
4726                                 maxsector = -1;
4727                 }
4728                 if (block_size >= MAXPHYS) {
4729                         xpt_print(periph->path,
4730                             "unsupportable block size %ju\n",
4731                             (uintmax_t) block_size);
4732                         announce_buf = NULL;
4733                         cam_periph_invalidate(periph);
4734                 } else {
4735                         /*
4736                          * We pass rcaplong into dasetgeom(),
4737                          * because it will only use it if it is
4738                          * non-NULL.
4739                          */
4740                         dasetgeom(periph, block_size, maxsector,
4741                                   rcaplong, sizeof(*rcaplong));
4742                         lbp = (lalba & SRC16_LBPME_A);
4743                         dp = &softc->params;
4744                         n = snprintf(announce_buf, DA_ANNOUNCETMP_SZ,
4745                             "%juMB (%ju %u byte sectors",
4746                             ((uintmax_t)dp->secsize * dp->sectors) /
4747                              (1024 * 1024),
4748                             (uintmax_t)dp->sectors, dp->secsize);
4749                         if (softc->p_type != 0) {
4750                                 n += snprintf(announce_buf + n,
4751                                     DA_ANNOUNCETMP_SZ - n,
4752                                     ", DIF type %d", softc->p_type);
4753                         }
4754                         snprintf(announce_buf + n, DA_ANNOUNCETMP_SZ - n, ")");
4755                 }
4756         } else {
4757                 int error;
4758
4759                 /*
4760                  * Retry any UNIT ATTENTION type errors.  They
4761                  * are expected at boot.
4762                  */
4763                 error = daerror(done_ccb, CAM_RETRY_SELTO,
4764                                 SF_RETRY_UA|SF_NO_PRINT);
4765                 if (error == ERESTART) {
4766                         /*
4767                          * A retry was scheuled, so
4768                          * just return.
4769                          */
4770                         return;
4771                 } else if (error != 0) {
4772                         int asc, ascq;
4773                         int sense_key, error_code;
4774                         int have_sense;
4775                         cam_status status;
4776                         struct ccb_getdev cgd;
4777
4778                         /* Don't wedge this device's queue */
4779                         status = done_ccb->ccb_h.status;
4780                         if ((status & CAM_DEV_QFRZN) != 0)
4781                                 cam_release_devq(done_ccb->ccb_h.path,
4782                                                  /*relsim_flags*/0,
4783                                                  /*reduction*/0,
4784                                                  /*timeout*/0,
4785                                                  /*getcount_only*/0);
4786
4787
4788                         xpt_setup_ccb(&cgd.ccb_h, done_ccb->ccb_h.path,
4789                                       CAM_PRIORITY_NORMAL);
4790                         cgd.ccb_h.func_code = XPT_GDEV_TYPE;
4791                         xpt_action((union ccb *)&cgd);
4792
4793                         if (scsi_extract_sense_ccb(done_ccb,
4794                             &error_code, &sense_key, &asc, &ascq))
4795                                 have_sense = TRUE;
4796                         else
4797                                 have_sense = FALSE;
4798
4799                         /*
4800                          * If we tried READ CAPACITY(16) and failed,
4801                          * fallback to READ CAPACITY(10).
4802                          */
4803                         if ((state == DA_CCB_PROBE_RC16) &&
4804                             (softc->flags & DA_FLAG_CAN_RC16) &&
4805                             (((csio->ccb_h.status & CAM_STATUS_MASK) ==
4806                                 CAM_REQ_INVALID) ||
4807                              ((have_sense) &&
4808                               (error_code == SSD_CURRENT_ERROR ||
4809                                error_code == SSD_DESC_CURRENT_ERROR) &&
4810                               (sense_key == SSD_KEY_ILLEGAL_REQUEST)))) {
4811                                 cam_periph_assert(periph, MA_OWNED);
4812                                 softc->flags &= ~DA_FLAG_CAN_RC16;
4813                                 free(rdcap, M_SCSIDA);
4814                                 xpt_release_ccb(done_ccb);
4815                                 softc->state = DA_STATE_PROBE_RC;
4816                                 xpt_schedule(periph, priority);
4817                                 return;
4818                         }
4819
4820                         /*
4821                          * Attach to anything that claims to be a
4822                          * direct access or optical disk device,
4823                          * as long as it doesn't return a "Logical
4824                          * unit not supported" (0x25) error.
4825                          * "Internal Target Failure" (0x44) is also
4826                          * special and typically means that the
4827                          * device is a SATA drive behind a SATL
4828                          * translation that's fallen into a
4829                          * terminally fatal state.
4830                          */
4831                         if ((have_sense)
4832                          && (asc != 0x25) && (asc != 0x44)
4833                          && (error_code == SSD_CURRENT_ERROR
4834                           || error_code == SSD_DESC_CURRENT_ERROR)) {
4835                                 const char *sense_key_desc;
4836                                 const char *asc_desc;
4837
4838                                 dasetgeom(periph, 512, -1, NULL, 0);
4839                                 scsi_sense_desc(sense_key, asc, ascq,
4840                                                 &cgd.inq_data, &sense_key_desc,
4841                                                 &asc_desc);
4842                                 snprintf(announce_buf, DA_ANNOUNCETMP_SZ,
4843                                     "Attempt to query device "
4844                                     "size failed: %s, %s",
4845                                     sense_key_desc, asc_desc);
4846                         } else {
4847                                 if (have_sense)
4848                                         scsi_sense_print(&done_ccb->csio);
4849                                 else {
4850                                         xpt_print(periph->path,
4851                                             "got CAM status %#x\n",
4852                                             done_ccb->ccb_h.status);
4853                                 }
4854
4855                                 xpt_print(periph->path, "fatal error, "
4856                                     "failed to attach to device\n");
4857
4858                                 announce_buf = NULL;
4859
4860                                 /*
4861                                  * Free up resources.
4862                                  */
4863                                 cam_periph_invalidate(periph);
4864                         }
4865                 }
4866         }
4867         free(csio->data_ptr, M_SCSIDA);
4868         if (announce_buf != NULL &&
4869             ((softc->flags & DA_FLAG_ANNOUNCED) == 0)) {
4870                 struct sbuf sb;
4871
4872                 sbuf_new(&sb, softc->announcebuf, DA_ANNOUNCE_SZ,
4873                     SBUF_FIXEDLEN);
4874                 xpt_announce_periph_sbuf(periph, &sb, announce_buf);
4875                 xpt_announce_quirks_sbuf(periph, &sb, softc->quirks,
4876                     DA_Q_BIT_STRING);
4877                 sbuf_finish(&sb);
4878                 sbuf_putbuf(&sb);
4879
4880                 /*
4881                  * Create our sysctl variables, now that we know
4882                  * we have successfully attached.
4883                  */
4884                 /* increase the refcount */
4885                 if (da_periph_acquire(periph, DA_REF_SYSCTL) == 0) {
4886                         taskqueue_enqueue(taskqueue_thread,
4887                                           &softc->sysctl_task);
4888                 } else {
4889                         /* XXX This message is useless! */
4890                         xpt_print(periph->path, "fatal error, "
4891                             "could not acquire reference count\n");
4892                 }
4893         }
4894
4895         /* We already probed the device. */
4896         if (softc->flags & DA_FLAG_PROBED) {
4897                 daprobedone(periph, done_ccb);
4898                 return;
4899         }
4900
4901         /* Ensure re-probe doesn't see old delete. */
4902         softc->delete_available = 0;
4903         dadeleteflag(softc, DA_DELETE_ZERO, 1);
4904         if (lbp && (softc->quirks & DA_Q_NO_UNMAP) == 0) {
4905                 /*
4906                  * Based on older SBC-3 spec revisions
4907                  * any of the UNMAP methods "may" be
4908                  * available via LBP given this flag so
4909                  * we flag all of them as available and
4910                  * then remove those which further
4911                  * probes confirm aren't available
4912                  * later.
4913                  *
4914                  * We could also check readcap(16) p_type
4915                  * flag to exclude one or more invalid
4916                  * write same (X) types here
4917                  */
4918                 dadeleteflag(softc, DA_DELETE_WS16, 1);
4919                 dadeleteflag(softc, DA_DELETE_WS10, 1);
4920                 dadeleteflag(softc, DA_DELETE_UNMAP, 1);
4921
4922                 xpt_release_ccb(done_ccb);
4923                 softc->state = DA_STATE_PROBE_LBP;
4924                 xpt_schedule(periph, priority);
4925                 return;
4926         }
4927
4928         xpt_release_ccb(done_ccb);
4929         softc->state = DA_STATE_PROBE_BDC;
4930         xpt_schedule(periph, priority);
4931         return;
4932 }
4933
4934 static void
4935 dadone_probelbp(struct cam_periph *periph, union ccb *done_ccb)
4936 {
4937         struct scsi_vpd_logical_block_prov *lbp;
4938         struct da_softc *softc;
4939         struct ccb_scsiio *csio;
4940         u_int32_t  priority;
4941
4942         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dadone_probelbp\n"));
4943
4944         softc = (struct da_softc *)periph->softc;
4945         priority = done_ccb->ccb_h.pinfo.priority;
4946         csio = &done_ccb->csio;
4947         lbp = (struct scsi_vpd_logical_block_prov *)csio->data_ptr;
4948
4949         cam_periph_assert(periph, MA_OWNED);
4950
4951         if ((csio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
4952                 /*
4953                  * T10/1799-D Revision 31 states at least one of these
4954                  * must be supported but we don't currently enforce this.
4955                  */
4956                 dadeleteflag(softc, DA_DELETE_WS16,
4957                      (lbp->flags & SVPD_LBP_WS16));
4958                 dadeleteflag(softc, DA_DELETE_WS10,
4959                              (lbp->flags & SVPD_LBP_WS10));
4960                 dadeleteflag(softc, DA_DELETE_UNMAP,
4961                              (lbp->flags & SVPD_LBP_UNMAP));
4962         } else {
4963                 int error;
4964                 error = daerror(done_ccb, CAM_RETRY_SELTO,
4965                                 SF_RETRY_UA|SF_NO_PRINT);
4966                 if (error == ERESTART)
4967                         return;
4968                 else if (error != 0) {
4969                         if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
4970                                 /* Don't wedge this device's queue */
4971                                 cam_release_devq(done_ccb->ccb_h.path,
4972                                                  /*relsim_flags*/0,
4973                                                  /*reduction*/0,
4974                                                  /*timeout*/0,
4975                                                  /*getcount_only*/0);
4976                         }
4977
4978                         /*
4979                          * Failure indicates we don't support any SBC-3
4980                          * delete methods with UNMAP
4981                          */
4982                 }
4983         }
4984
4985         free(lbp, M_SCSIDA);
4986         xpt_release_ccb(done_ccb);
4987         softc->state = DA_STATE_PROBE_BLK_LIMITS;
4988         xpt_schedule(periph, priority);
4989         return;
4990 }
4991
4992 static void
4993 dadone_probeblklimits(struct cam_periph *periph, union ccb *done_ccb)
4994 {
4995         struct scsi_vpd_block_limits *block_limits;
4996         struct da_softc *softc;
4997         struct ccb_scsiio *csio;
4998         u_int32_t  priority;
4999
5000         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dadone_probeblklimits\n"));
5001
5002         softc = (struct da_softc *)periph->softc;
5003         priority = done_ccb->ccb_h.pinfo.priority;
5004         csio = &done_ccb->csio;
5005         block_limits = (struct scsi_vpd_block_limits *)csio->data_ptr;
5006
5007         cam_periph_assert(periph, MA_OWNED);
5008
5009         if ((csio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
5010                 uint32_t max_txfer_len = scsi_4btoul(
5011                         block_limits->max_txfer_len);
5012                 uint32_t max_unmap_lba_cnt = scsi_4btoul(
5013                         block_limits->max_unmap_lba_cnt);
5014                 uint32_t max_unmap_blk_cnt = scsi_4btoul(
5015                         block_limits->max_unmap_blk_cnt);
5016                 uint32_t unmap_gran = scsi_4btoul(
5017                         block_limits->opt_unmap_grain);
5018                 uint32_t unmap_gran_align = scsi_4btoul(
5019                         block_limits->unmap_grain_align);
5020                 uint64_t ws_max_blks = scsi_8btou64(
5021                         block_limits->max_write_same_length);
5022
5023                 if (max_txfer_len != 0) {
5024                         softc->disk->d_maxsize = MIN(softc->maxio,
5025                             (off_t)max_txfer_len * softc->params.secsize);
5026                 }
5027
5028                 /*
5029                  * We should already support UNMAP but we check lba
5030                  * and block count to be sure
5031                  */
5032                 if (max_unmap_lba_cnt != 0x00L &&
5033                     max_unmap_blk_cnt != 0x00L) {
5034                         softc->unmap_max_lba = max_unmap_lba_cnt;
5035                         softc->unmap_max_ranges = min(max_unmap_blk_cnt,
5036                                 UNMAP_MAX_RANGES);
5037                         if (unmap_gran > 1) {
5038                                 softc->unmap_gran = unmap_gran;
5039                                 if (unmap_gran_align & 0x80000000) {
5040                                         softc->unmap_gran_align =
5041                                             unmap_gran_align & 0x7fffffff;
5042                                 }
5043                         }
5044                 } else {
5045                         /*
5046                          * Unexpected UNMAP limits which means the
5047                          * device doesn't actually support UNMAP
5048                          */
5049                         dadeleteflag(softc, DA_DELETE_UNMAP, 0);
5050                 }
5051
5052                 if (ws_max_blks != 0x00L)
5053                         softc->ws_max_blks = ws_max_blks;
5054         } else {
5055                 int error;
5056                 error = daerror(done_ccb, CAM_RETRY_SELTO,
5057                                 SF_RETRY_UA|SF_NO_PRINT);
5058                 if (error == ERESTART)
5059                         return;
5060                 else if (error != 0) {
5061                         if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
5062                                 /* Don't wedge this device's queue */
5063                                 cam_release_devq(done_ccb->ccb_h.path,
5064                                                  /*relsim_flags*/0,
5065                                                  /*reduction*/0,
5066                                                  /*timeout*/0,
5067                                                  /*getcount_only*/0);
5068                         }
5069
5070                         /*
5071                          * Failure here doesn't mean UNMAP is not
5072                          * supported as this is an optional page.
5073                          */
5074                         softc->unmap_max_lba = 1;
5075                         softc->unmap_max_ranges = 1;
5076                 }
5077         }
5078
5079         free(block_limits, M_SCSIDA);
5080         xpt_release_ccb(done_ccb);
5081         softc->state = DA_STATE_PROBE_BDC;
5082         xpt_schedule(periph, priority);
5083         return;
5084 }
5085
5086 static void
5087 dadone_probebdc(struct cam_periph *periph, union ccb *done_ccb)
5088 {
5089         struct scsi_vpd_block_device_characteristics *bdc;
5090         struct da_softc *softc;
5091         struct ccb_scsiio *csio;
5092         u_int32_t  priority;
5093
5094         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dadone_probebdc\n"));
5095
5096         softc = (struct da_softc *)periph->softc;
5097         priority = done_ccb->ccb_h.pinfo.priority;
5098         csio = &done_ccb->csio;
5099         bdc = (struct scsi_vpd_block_device_characteristics *)csio->data_ptr;
5100
5101         cam_periph_assert(periph, MA_OWNED);
5102
5103         if ((csio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
5104                 uint32_t valid_len;
5105
5106                 /*
5107                  * Disable queue sorting for non-rotational media
5108                  * by default.
5109                  */
5110                 u_int16_t old_rate = softc->disk->d_rotation_rate;
5111
5112                 valid_len = csio->dxfer_len - csio->resid;
5113                 if (SBDC_IS_PRESENT(bdc, valid_len,
5114                     medium_rotation_rate)) {
5115                         softc->disk->d_rotation_rate =
5116                                 scsi_2btoul(bdc->medium_rotation_rate);
5117                         if (softc->disk->d_rotation_rate ==
5118                             SVPD_BDC_RATE_NON_ROTATING) {
5119                                 cam_iosched_set_sort_queue(
5120                                     softc->cam_iosched, 0);
5121                                 softc->rotating = 0;
5122                         }
5123                         if (softc->disk->d_rotation_rate != old_rate) {
5124                                 disk_attr_changed(softc->disk,
5125                                     "GEOM::rotation_rate", M_NOWAIT);
5126                         }
5127                 }
5128                 if ((SBDC_IS_PRESENT(bdc, valid_len, flags))
5129                  && (softc->zone_mode == DA_ZONE_NONE)) {
5130                         int ata_proto;
5131
5132                         if (scsi_vpd_supported_page(periph,
5133                             SVPD_ATA_INFORMATION))
5134                                 ata_proto = 1;
5135                         else
5136                                 ata_proto = 0;
5137
5138                         /*
5139                          * The Zoned field will only be set for
5140                          * Drive Managed and Host Aware drives.  If
5141                          * they are Host Managed, the device type
5142                          * in the standard INQUIRY data should be
5143                          * set to T_ZBC_HM (0x14).
5144                          */
5145                         if ((bdc->flags & SVPD_ZBC_MASK) ==
5146                              SVPD_HAW_ZBC) {
5147                                 softc->zone_mode = DA_ZONE_HOST_AWARE;
5148                                 softc->zone_interface = (ata_proto) ?
5149                                    DA_ZONE_IF_ATA_SAT : DA_ZONE_IF_SCSI;
5150                         } else if ((bdc->flags & SVPD_ZBC_MASK) ==
5151                              SVPD_DM_ZBC) {
5152                                 softc->zone_mode =DA_ZONE_DRIVE_MANAGED;
5153                                 softc->zone_interface = (ata_proto) ?
5154                                    DA_ZONE_IF_ATA_SAT : DA_ZONE_IF_SCSI;
5155                         } else if ((bdc->flags & SVPD_ZBC_MASK) !=
5156                                   SVPD_ZBC_NR) {
5157                                 xpt_print(periph->path, "Unknown zoned "
5158                                     "type %#x",
5159                                     bdc->flags & SVPD_ZBC_MASK);
5160                         }
5161                 }
5162         } else {
5163                 int error;
5164                 error = daerror(done_ccb, CAM_RETRY_SELTO,
5165                                 SF_RETRY_UA|SF_NO_PRINT);
5166                 if (error == ERESTART)
5167                         return;
5168                 else if (error != 0) {
5169                         if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
5170                                 /* Don't wedge this device's queue */
5171                                 cam_release_devq(done_ccb->ccb_h.path,
5172                                                  /*relsim_flags*/0,
5173                                                  /*reduction*/0,
5174                                                  /*timeout*/0,
5175                                                  /*getcount_only*/0);
5176                         }
5177                 }
5178         }
5179
5180         free(bdc, M_SCSIDA);
5181         xpt_release_ccb(done_ccb);
5182         softc->state = DA_STATE_PROBE_ATA;
5183         xpt_schedule(periph, priority);
5184         return;
5185 }
5186
5187 static void
5188 dadone_probeata(struct cam_periph *periph, union ccb *done_ccb)
5189 {
5190         struct ata_params *ata_params;
5191         struct ccb_scsiio *csio;
5192         struct da_softc *softc;
5193         u_int32_t  priority;
5194         int continue_probe;
5195         int error, i;
5196         int16_t *ptr;
5197
5198         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dadone_probeata\n"));
5199
5200         softc = (struct da_softc *)periph->softc;
5201         priority = done_ccb->ccb_h.pinfo.priority;
5202         csio = &done_ccb->csio;
5203         ata_params = (struct ata_params *)csio->data_ptr;
5204         ptr = (uint16_t *)ata_params;
5205         continue_probe = 0;
5206         error = 0;
5207
5208         cam_periph_assert(periph, MA_OWNED);
5209
5210         if ((csio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
5211                 uint16_t old_rate;
5212
5213                 for (i = 0; i < sizeof(*ata_params) / 2; i++)
5214                         ptr[i] = le16toh(ptr[i]);
5215                 if (ata_params->support_dsm & ATA_SUPPORT_DSM_TRIM &&
5216                     (softc->quirks & DA_Q_NO_UNMAP) == 0) {
5217                         dadeleteflag(softc, DA_DELETE_ATA_TRIM, 1);
5218                         if (ata_params->max_dsm_blocks != 0)
5219                                 softc->trim_max_ranges = min(
5220                                   softc->trim_max_ranges,
5221                                   ata_params->max_dsm_blocks *
5222                                   ATA_DSM_BLK_RANGES);
5223                 }
5224                 /*
5225                  * Disable queue sorting for non-rotational media
5226                  * by default.
5227                  */
5228                 old_rate = softc->disk->d_rotation_rate;
5229                 softc->disk->d_rotation_rate = ata_params->media_rotation_rate;
5230                 if (softc->disk->d_rotation_rate == ATA_RATE_NON_ROTATING) {
5231                         cam_iosched_set_sort_queue(softc->cam_iosched, 0);
5232                         softc->rotating = 0;
5233                 }
5234                 if (softc->disk->d_rotation_rate != old_rate) {
5235                         disk_attr_changed(softc->disk,
5236                             "GEOM::rotation_rate", M_NOWAIT);
5237                 }
5238
5239                 cam_periph_assert(periph, MA_OWNED);
5240                 if (ata_params->capabilities1 & ATA_SUPPORT_DMA)
5241                         softc->flags |= DA_FLAG_CAN_ATA_DMA;
5242
5243                 if (ata_params->support.extension & ATA_SUPPORT_GENLOG)
5244                         softc->flags |= DA_FLAG_CAN_ATA_LOG;
5245
5246                 /*
5247                  * At this point, if we have a SATA host aware drive,
5248                  * we communicate via ATA passthrough unless the
5249                  * SAT layer supports ZBC -> ZAC translation.  In
5250                  * that case,
5251                  *
5252                  * XXX KDM figure out how to detect a host managed
5253                  * SATA drive.
5254                  */
5255                 if (softc->zone_mode == DA_ZONE_NONE) {
5256                         /*
5257                          * Note that we don't override the zone
5258                          * mode or interface if it has already been
5259                          * set.  This is because it has either been
5260                          * set as a quirk, or when we probed the
5261                          * SCSI Block Device Characteristics page,
5262                          * the zoned field was set.  The latter
5263                          * means that the SAT layer supports ZBC to
5264                          * ZAC translation, and we would prefer to
5265                          * use that if it is available.
5266                          */
5267                         if ((ata_params->support3 &
5268                             ATA_SUPPORT_ZONE_MASK) ==
5269                             ATA_SUPPORT_ZONE_HOST_AWARE) {
5270                                 softc->zone_mode = DA_ZONE_HOST_AWARE;
5271                                 softc->zone_interface =
5272                                     DA_ZONE_IF_ATA_PASS;
5273                         } else if ((ata_params->support3 &
5274                                     ATA_SUPPORT_ZONE_MASK) ==
5275                                     ATA_SUPPORT_ZONE_DEV_MANAGED) {
5276                                 softc->zone_mode =DA_ZONE_DRIVE_MANAGED;
5277                                 softc->zone_interface = DA_ZONE_IF_ATA_PASS;
5278                         }
5279                 }
5280
5281         } else {
5282                 error = daerror(done_ccb, CAM_RETRY_SELTO,
5283                                 SF_RETRY_UA|SF_NO_PRINT);
5284                 if (error == ERESTART)
5285                         return;
5286                 else if (error != 0) {
5287                         if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
5288                                 /* Don't wedge this device's queue */
5289                                 cam_release_devq(done_ccb->ccb_h.path,
5290                                                  /*relsim_flags*/0,
5291                                                  /*reduction*/0,
5292                                                  /*timeout*/0,
5293                                                  /*getcount_only*/0);
5294                         }
5295                 }
5296         }
5297
5298         free(ata_params, M_SCSIDA);
5299         if ((softc->zone_mode == DA_ZONE_HOST_AWARE)
5300          || (softc->zone_mode == DA_ZONE_HOST_MANAGED)) {
5301                 /*
5302                  * If the ATA IDENTIFY failed, we could be talking
5303                  * to a SCSI drive, although that seems unlikely,
5304                  * since the drive did report that it supported the
5305                  * ATA Information VPD page.  If the ATA IDENTIFY
5306                  * succeeded, and the SAT layer doesn't support
5307                  * ZBC -> ZAC translation, continue on to get the
5308                  * directory of ATA logs, and complete the rest of
5309                  * the ZAC probe.  If the SAT layer does support
5310                  * ZBC -> ZAC translation, we want to use that,
5311                  * and we'll probe the SCSI Zoned Block Device
5312                  * Characteristics VPD page next.
5313                  */
5314                 if ((error == 0)
5315                  && (softc->flags & DA_FLAG_CAN_ATA_LOG)
5316                  && (softc->zone_interface == DA_ZONE_IF_ATA_PASS))
5317                         softc->state = DA_STATE_PROBE_ATA_LOGDIR;
5318                 else
5319                         softc->state = DA_STATE_PROBE_ZONE;
5320                 continue_probe = 1;
5321         }
5322         if (continue_probe != 0) {
5323                 xpt_release_ccb(done_ccb);
5324                 xpt_schedule(periph, priority);
5325                 return;
5326         } else
5327                 daprobedone(periph, done_ccb);
5328         return;
5329 }
5330
5331 static void
5332 dadone_probeatalogdir(struct cam_periph *periph, union ccb *done_ccb)
5333 {
5334         struct da_softc *softc;
5335         struct ccb_scsiio *csio;
5336         u_int32_t  priority;
5337         int error;
5338
5339         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dadone_probeatalogdir\n"));
5340
5341         softc = (struct da_softc *)periph->softc;
5342         priority = done_ccb->ccb_h.pinfo.priority;
5343         csio = &done_ccb->csio;
5344
5345         cam_periph_assert(periph, MA_OWNED);
5346         if ((csio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
5347                 error = 0;
5348                 softc->valid_logdir_len = 0;
5349                 bzero(&softc->ata_logdir, sizeof(softc->ata_logdir));
5350                 softc->valid_logdir_len = csio->dxfer_len - csio->resid;
5351                 if (softc->valid_logdir_len > 0)
5352                         bcopy(csio->data_ptr, &softc->ata_logdir,
5353                             min(softc->valid_logdir_len,
5354                                 sizeof(softc->ata_logdir)));
5355                 /*
5356                  * Figure out whether the Identify Device log is
5357                  * supported.  The General Purpose log directory
5358                  * has a header, and lists the number of pages
5359                  * available for each GP log identified by the
5360                  * offset into the list.
5361                  */
5362                 if ((softc->valid_logdir_len >=
5363                     ((ATA_IDENTIFY_DATA_LOG + 1) * sizeof(uint16_t)))
5364                  && (le16dec(softc->ata_logdir.header) ==
5365                      ATA_GP_LOG_DIR_VERSION)
5366                  && (le16dec(&softc->ata_logdir.num_pages[
5367                      (ATA_IDENTIFY_DATA_LOG *
5368                      sizeof(uint16_t)) - sizeof(uint16_t)]) > 0)){
5369                         softc->flags |= DA_FLAG_CAN_ATA_IDLOG;
5370                 } else {
5371                         softc->flags &= ~DA_FLAG_CAN_ATA_IDLOG;
5372                 }
5373         } else {
5374                 error = daerror(done_ccb, CAM_RETRY_SELTO,
5375                                 SF_RETRY_UA|SF_NO_PRINT);
5376                 if (error == ERESTART)
5377                         return;
5378                 else if (error != 0) {
5379                         /*
5380                          * If we can't get the ATA log directory,
5381                          * then ATA logs are effectively not
5382                          * supported even if the bit is set in the
5383                          * identify data.
5384                          */
5385                         softc->flags &= ~(DA_FLAG_CAN_ATA_LOG |
5386                                           DA_FLAG_CAN_ATA_IDLOG);
5387                         if ((done_ccb->ccb_h.status &
5388                              CAM_DEV_QFRZN) != 0) {
5389                                 /* Don't wedge this device's queue */
5390                                 cam_release_devq(done_ccb->ccb_h.path,
5391                                                  /*relsim_flags*/0,
5392                                                  /*reduction*/0,
5393                                                  /*timeout*/0,
5394                                                  /*getcount_only*/0);
5395                         }
5396                 }
5397         }
5398
5399         free(csio->data_ptr, M_SCSIDA);
5400
5401         if ((error == 0)
5402          && (softc->flags & DA_FLAG_CAN_ATA_IDLOG)) {
5403                 softc->state = DA_STATE_PROBE_ATA_IDDIR;
5404                 xpt_release_ccb(done_ccb);
5405                 xpt_schedule(periph, priority);
5406                 return;
5407         }
5408         daprobedone(periph, done_ccb);
5409         return;
5410 }
5411
5412 static void
5413 dadone_probeataiddir(struct cam_periph *periph, union ccb *done_ccb)
5414 {
5415         struct da_softc *softc;
5416         struct ccb_scsiio *csio;
5417         u_int32_t  priority;
5418         int error;
5419
5420         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dadone_probeataiddir\n"));
5421
5422         softc = (struct da_softc *)periph->softc;
5423         priority = done_ccb->ccb_h.pinfo.priority;
5424         csio = &done_ccb->csio;
5425
5426         cam_periph_assert(periph, MA_OWNED);
5427
5428         if ((csio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
5429                 off_t entries_offset, max_entries;
5430                 error = 0;
5431
5432                 softc->valid_iddir_len = 0;
5433                 bzero(&softc->ata_iddir, sizeof(softc->ata_iddir));
5434                 softc->flags &= ~(DA_FLAG_CAN_ATA_SUPCAP |
5435                                   DA_FLAG_CAN_ATA_ZONE);
5436                 softc->valid_iddir_len = csio->dxfer_len - csio->resid;
5437                 if (softc->valid_iddir_len > 0)
5438                         bcopy(csio->data_ptr, &softc->ata_iddir,
5439                             min(softc->valid_iddir_len,
5440                                 sizeof(softc->ata_iddir)));
5441
5442                 entries_offset =
5443                     __offsetof(struct ata_identify_log_pages,entries);
5444                 max_entries = softc->valid_iddir_len - entries_offset;
5445                 if ((softc->valid_iddir_len > (entries_offset + 1))
5446                  && (le64dec(softc->ata_iddir.header) == ATA_IDLOG_REVISION)
5447                  && (softc->ata_iddir.entry_count > 0)) {
5448                         int num_entries, i;
5449
5450                         num_entries = softc->ata_iddir.entry_count;
5451                         num_entries = min(num_entries,
5452                            softc->valid_iddir_len - entries_offset);
5453                         for (i = 0; i < num_entries && i < max_entries; i++) {
5454                                 if (softc->ata_iddir.entries[i] ==
5455                                     ATA_IDL_SUP_CAP)
5456                                         softc->flags |= DA_FLAG_CAN_ATA_SUPCAP;
5457                                 else if (softc->ata_iddir.entries[i] ==
5458                                          ATA_IDL_ZDI)
5459                                         softc->flags |= DA_FLAG_CAN_ATA_ZONE;
5460
5461                                 if ((softc->flags & DA_FLAG_CAN_ATA_SUPCAP)
5462                                  && (softc->flags & DA_FLAG_CAN_ATA_ZONE))
5463                                         break;
5464                         }
5465                 }
5466         } else {
5467                 error = daerror(done_ccb, CAM_RETRY_SELTO,
5468                                 SF_RETRY_UA|SF_NO_PRINT);
5469                 if (error == ERESTART)
5470                         return;
5471                 else if (error != 0) {
5472                         /*
5473                          * If we can't get the ATA Identify Data log
5474                          * directory, then it effectively isn't
5475                          * supported even if the ATA Log directory
5476                          * a non-zero number of pages present for
5477                          * this log.
5478                          */
5479                         softc->flags &= ~DA_FLAG_CAN_ATA_IDLOG;
5480                         if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
5481                                 /* Don't wedge this device's queue */
5482                                 cam_release_devq(done_ccb->ccb_h.path,
5483                                                  /*relsim_flags*/0,
5484                                                  /*reduction*/0,
5485                                                  /*timeout*/0,
5486                                                  /*getcount_only*/0);
5487                         }
5488                 }
5489         }
5490
5491         free(csio->data_ptr, M_SCSIDA);
5492
5493         if ((error == 0) && (softc->flags & DA_FLAG_CAN_ATA_SUPCAP)) {
5494                 softc->state = DA_STATE_PROBE_ATA_SUP;
5495                 xpt_release_ccb(done_ccb);
5496                 xpt_schedule(periph, priority);
5497                 return;
5498         }
5499         daprobedone(periph, done_ccb);
5500         return;
5501 }
5502
5503 static void
5504 dadone_probeatasup(struct cam_periph *periph, union ccb *done_ccb)
5505 {
5506         struct da_softc *softc;
5507         struct ccb_scsiio *csio;
5508         u_int32_t  priority;
5509         int error;
5510
5511         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dadone_probeatasup\n"));
5512
5513         softc = (struct da_softc *)periph->softc;
5514         priority = done_ccb->ccb_h.pinfo.priority;
5515         csio = &done_ccb->csio;
5516
5517         cam_periph_assert(periph, MA_OWNED);
5518
5519         if ((csio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
5520                 uint32_t valid_len;
5521                 size_t needed_size;
5522                 struct ata_identify_log_sup_cap *sup_cap;
5523                 error = 0;
5524
5525                 sup_cap = (struct ata_identify_log_sup_cap *)csio->data_ptr;
5526                 valid_len = csio->dxfer_len - csio->resid;
5527                 needed_size = __offsetof(struct ata_identify_log_sup_cap,
5528                     sup_zac_cap) + 1 + sizeof(sup_cap->sup_zac_cap);
5529                 if (valid_len >= needed_size) {
5530                         uint64_t zoned, zac_cap;
5531
5532                         zoned = le64dec(sup_cap->zoned_cap);
5533                         if (zoned & ATA_ZONED_VALID) {
5534                                 /*
5535                                  * This should have already been
5536                                  * set, because this is also in the
5537                                  * ATA identify data.
5538                                  */
5539                                 if ((zoned & ATA_ZONED_MASK) ==
5540                                     ATA_SUPPORT_ZONE_HOST_AWARE)
5541                                         softc->zone_mode = DA_ZONE_HOST_AWARE;
5542                                 else if ((zoned & ATA_ZONED_MASK) ==
5543                                     ATA_SUPPORT_ZONE_DEV_MANAGED)
5544                                         softc->zone_mode =
5545                                             DA_ZONE_DRIVE_MANAGED;
5546                         }
5547
5548                         zac_cap = le64dec(sup_cap->sup_zac_cap);
5549                         if (zac_cap & ATA_SUP_ZAC_CAP_VALID) {
5550                                 if (zac_cap & ATA_REPORT_ZONES_SUP)
5551                                         softc->zone_flags |=
5552                                             DA_ZONE_FLAG_RZ_SUP;
5553                                 if (zac_cap & ATA_ND_OPEN_ZONE_SUP)
5554                                         softc->zone_flags |=
5555                                             DA_ZONE_FLAG_OPEN_SUP;
5556                                 if (zac_cap & ATA_ND_CLOSE_ZONE_SUP)
5557                                         softc->zone_flags |=
5558                                             DA_ZONE_FLAG_CLOSE_SUP;
5559                                 if (zac_cap & ATA_ND_FINISH_ZONE_SUP)
5560                                         softc->zone_flags |=
5561                                             DA_ZONE_FLAG_FINISH_SUP;
5562                                 if (zac_cap & ATA_ND_RWP_SUP)
5563                                         softc->zone_flags |=
5564                                             DA_ZONE_FLAG_RWP_SUP;
5565                         } else {
5566                                 /*
5567                                  * This field was introduced in
5568                                  * ACS-4, r08 on April 28th, 2015.
5569                                  * If the drive firmware was written
5570                                  * to an earlier spec, it won't have
5571                                  * the field.  So, assume all
5572                                  * commands are supported.
5573                                  */
5574                                 softc->zone_flags |= DA_ZONE_FLAG_SUP_MASK;
5575                         }
5576                 }
5577         } else {
5578                 error = daerror(done_ccb, CAM_RETRY_SELTO,
5579                                 SF_RETRY_UA|SF_NO_PRINT);
5580                 if (error == ERESTART)
5581                         return;
5582                 else if (error != 0) {
5583                         /*
5584                          * If we can't get the ATA Identify Data
5585                          * Supported Capabilities page, clear the
5586                          * flag...
5587                          */
5588                         softc->flags &= ~DA_FLAG_CAN_ATA_SUPCAP;
5589                         /*
5590                          * And clear zone capabilities.
5591                          */
5592                         softc->zone_flags &= ~DA_ZONE_FLAG_SUP_MASK;
5593                         if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
5594                                 /* Don't wedge this device's queue */
5595                                 cam_release_devq(done_ccb->ccb_h.path,
5596                                                  /*relsim_flags*/0,
5597                                                  /*reduction*/0,
5598                                                  /*timeout*/0,
5599                                                  /*getcount_only*/0);
5600                         }
5601                 }
5602         }
5603
5604         free(csio->data_ptr, M_SCSIDA);
5605
5606         if ((error == 0) && (softc->flags & DA_FLAG_CAN_ATA_ZONE)) {
5607                 softc->state = DA_STATE_PROBE_ATA_ZONE;
5608                 xpt_release_ccb(done_ccb);
5609                 xpt_schedule(periph, priority);
5610                 return;
5611         }
5612         daprobedone(periph, done_ccb);
5613         return;
5614 }
5615
5616 static void
5617 dadone_probeatazone(struct cam_periph *periph, union ccb *done_ccb)
5618 {
5619         struct da_softc *softc;
5620         struct ccb_scsiio *csio;
5621         int error;
5622
5623         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dadone_probeatazone\n"));
5624
5625         softc = (struct da_softc *)periph->softc;
5626         csio = &done_ccb->csio;
5627
5628         cam_periph_assert(periph, MA_OWNED);
5629
5630         if ((csio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
5631                 struct ata_zoned_info_log *zi_log;
5632                 uint32_t valid_len;
5633                 size_t needed_size;
5634
5635                 zi_log = (struct ata_zoned_info_log *)csio->data_ptr;
5636
5637                 valid_len = csio->dxfer_len - csio->resid;
5638                 needed_size = __offsetof(struct ata_zoned_info_log,
5639                     version_info) + 1 + sizeof(zi_log->version_info);
5640                 if (valid_len >= needed_size) {
5641                         uint64_t tmpvar;
5642
5643                         tmpvar = le64dec(zi_log->zoned_cap);
5644                         if (tmpvar & ATA_ZDI_CAP_VALID) {
5645                                 if (tmpvar & ATA_ZDI_CAP_URSWRZ)
5646                                         softc->zone_flags |=
5647                                             DA_ZONE_FLAG_URSWRZ;
5648                                 else
5649                                         softc->zone_flags &=
5650                                             ~DA_ZONE_FLAG_URSWRZ;
5651                         }
5652                         tmpvar = le64dec(zi_log->optimal_seq_zones);
5653                         if (tmpvar & ATA_ZDI_OPT_SEQ_VALID) {
5654                                 softc->zone_flags |= DA_ZONE_FLAG_OPT_SEQ_SET;
5655                                 softc->optimal_seq_zones = (tmpvar &
5656                                     ATA_ZDI_OPT_SEQ_MASK);
5657                         } else {
5658                                 softc->zone_flags &= ~DA_ZONE_FLAG_OPT_SEQ_SET;
5659                                 softc->optimal_seq_zones = 0;
5660                         }
5661
5662                         tmpvar =le64dec(zi_log->optimal_nonseq_zones);
5663                         if (tmpvar & ATA_ZDI_OPT_NS_VALID) {
5664                                 softc->zone_flags |=
5665                                     DA_ZONE_FLAG_OPT_NONSEQ_SET;
5666                                 softc->optimal_nonseq_zones =
5667                                     (tmpvar & ATA_ZDI_OPT_NS_MASK);
5668                         } else {
5669                                 softc->zone_flags &=
5670                                     ~DA_ZONE_FLAG_OPT_NONSEQ_SET;
5671                                 softc->optimal_nonseq_zones = 0;
5672                         }
5673
5674                         tmpvar = le64dec(zi_log->max_seq_req_zones);
5675                         if (tmpvar & ATA_ZDI_MAX_SEQ_VALID) {
5676                                 softc->zone_flags |= DA_ZONE_FLAG_MAX_SEQ_SET;
5677                                 softc->max_seq_zones =
5678                                     (tmpvar & ATA_ZDI_MAX_SEQ_MASK);
5679                         } else {
5680                                 softc->zone_flags &= ~DA_ZONE_FLAG_MAX_SEQ_SET;
5681                                 softc->max_seq_zones = 0;
5682                         }
5683                 }
5684         } else {
5685                 error = daerror(done_ccb, CAM_RETRY_SELTO,
5686                                 SF_RETRY_UA|SF_NO_PRINT);
5687                 if (error == ERESTART)
5688                         return;
5689                 else if (error != 0) {
5690                         softc->flags &= ~DA_FLAG_CAN_ATA_ZONE;
5691                         softc->flags &= ~DA_ZONE_FLAG_SET_MASK;
5692
5693                         if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
5694                                 /* Don't wedge this device's queue */
5695                                 cam_release_devq(done_ccb->ccb_h.path,
5696                                                  /*relsim_flags*/0,
5697                                                  /*reduction*/0,
5698                                                  /*timeout*/0,
5699                                                  /*getcount_only*/0);
5700                         }
5701                 }
5702
5703         }
5704
5705         free(csio->data_ptr, M_SCSIDA);
5706
5707         daprobedone(periph, done_ccb);
5708         return;
5709 }
5710
5711 static void
5712 dadone_probezone(struct cam_periph *periph, union ccb *done_ccb)
5713 {
5714         struct da_softc *softc;
5715         struct ccb_scsiio *csio;
5716         int error;
5717
5718         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dadone_probezone\n"));
5719
5720         softc = (struct da_softc *)periph->softc;
5721         csio = &done_ccb->csio;
5722
5723         cam_periph_assert(periph, MA_OWNED);
5724
5725         if ((csio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
5726                 uint32_t valid_len;
5727                 size_t needed_len;
5728                 struct scsi_vpd_zoned_bdc *zoned_bdc;
5729
5730                 error = 0;
5731                 zoned_bdc = (struct scsi_vpd_zoned_bdc *)csio->data_ptr;
5732                 valid_len = csio->dxfer_len - csio->resid;
5733                 needed_len = __offsetof(struct scsi_vpd_zoned_bdc,
5734                     max_seq_req_zones) + 1 +
5735                     sizeof(zoned_bdc->max_seq_req_zones);
5736                 if ((valid_len >= needed_len)
5737                  && (scsi_2btoul(zoned_bdc->page_length) >= SVPD_ZBDC_PL)) {
5738                         if (zoned_bdc->flags & SVPD_ZBDC_URSWRZ)
5739                                 softc->zone_flags |= DA_ZONE_FLAG_URSWRZ;
5740                         else
5741                                 softc->zone_flags &= ~DA_ZONE_FLAG_URSWRZ;
5742                         softc->optimal_seq_zones =
5743                             scsi_4btoul(zoned_bdc->optimal_seq_zones);
5744                         softc->zone_flags |= DA_ZONE_FLAG_OPT_SEQ_SET;
5745                         softc->optimal_nonseq_zones = scsi_4btoul(
5746                             zoned_bdc->optimal_nonseq_zones);
5747                         softc->zone_flags |= DA_ZONE_FLAG_OPT_NONSEQ_SET;
5748                         softc->max_seq_zones =
5749                             scsi_4btoul(zoned_bdc->max_seq_req_zones);
5750                         softc->zone_flags |= DA_ZONE_FLAG_MAX_SEQ_SET;
5751                 }
5752                 /*
5753                  * All of the zone commands are mandatory for SCSI
5754                  * devices.
5755                  *
5756                  * XXX KDM this is valid as of September 2015.
5757                  * Re-check this assumption once the SAT spec is
5758                  * updated to support SCSI ZBC to ATA ZAC mapping.
5759                  * Since ATA allows zone commands to be reported
5760                  * as supported or not, this may not necessarily
5761                  * be true for an ATA device behind a SAT (SCSI to
5762                  * ATA Translation) layer.
5763                  */
5764                 softc->zone_flags |= DA_ZONE_FLAG_SUP_MASK;
5765         } else {
5766                 error = daerror(done_ccb, CAM_RETRY_SELTO,
5767                                 SF_RETRY_UA|SF_NO_PRINT);
5768                 if (error == ERESTART)
5769                         return;
5770                 else if (error != 0) {
5771                         if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
5772                                 /* Don't wedge this device's queue */
5773                                 cam_release_devq(done_ccb->ccb_h.path,
5774                                                  /*relsim_flags*/0,
5775                                                  /*reduction*/0,
5776                                                  /*timeout*/0,
5777                                                  /*getcount_only*/0);
5778                         }
5779                 }
5780         }
5781
5782         free(csio->data_ptr, M_SCSIDA);
5783
5784         daprobedone(periph, done_ccb);
5785         return;
5786 }
5787
5788 static void
5789 dadone_tur(struct cam_periph *periph, union ccb *done_ccb)
5790 {
5791         struct da_softc *softc;
5792         struct ccb_scsiio *csio;
5793
5794         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dadone_tur\n"));
5795
5796         softc = (struct da_softc *)periph->softc;
5797         csio = &done_ccb->csio;
5798
5799         cam_periph_assert(periph, MA_OWNED);
5800
5801         if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
5802
5803                 if (daerror(done_ccb, CAM_RETRY_SELTO,
5804                     SF_RETRY_UA | SF_NO_RECOVERY | SF_NO_PRINT) == ERESTART)
5805                         return; /* Will complete again, keep reference */
5806                 if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0)
5807                         cam_release_devq(done_ccb->ccb_h.path,
5808                                          /*relsim_flags*/0,
5809                                          /*reduction*/0,
5810                                          /*timeout*/0,
5811                                          /*getcount_only*/0);
5812         }
5813         xpt_release_ccb(done_ccb);
5814         softc->flags &= ~DA_FLAG_TUR_PENDING;
5815         da_periph_release_locked(periph, DA_REF_TUR);
5816         return;
5817 }
5818
5819 static void
5820 dareprobe(struct cam_periph *periph)
5821 {
5822         struct da_softc   *softc;
5823         int status;
5824
5825         softc = (struct da_softc *)periph->softc;
5826
5827         /* Probe in progress; don't interfere. */
5828         if (softc->state != DA_STATE_NORMAL)
5829                 return;
5830
5831         status = da_periph_acquire(periph, DA_REF_REPROBE);
5832         KASSERT(status == 0, ("dareprobe: cam_periph_acquire failed"));
5833
5834         softc->state = DA_STATE_PROBE_WP;
5835         xpt_schedule(periph, CAM_PRIORITY_DEV);
5836 }
5837
5838 static int
5839 daerror(union ccb *ccb, u_int32_t cam_flags, u_int32_t sense_flags)
5840 {
5841         struct da_softc   *softc;
5842         struct cam_periph *periph;
5843         int error, error_code, sense_key, asc, ascq;
5844
5845 #if defined(BUF_TRACKING) || defined(FULL_BUF_TRACKING)
5846         if (ccb->csio.bio != NULL)
5847                 biotrack(ccb->csio.bio, __func__);
5848 #endif
5849
5850         periph = xpt_path_periph(ccb->ccb_h.path);
5851         softc = (struct da_softc *)periph->softc;
5852
5853         cam_periph_assert(periph, MA_OWNED);
5854
5855         /*
5856          * Automatically detect devices that do not support
5857          * READ(6)/WRITE(6) and upgrade to using 10 byte cdbs.
5858          */
5859         error = 0;
5860         if ((ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_INVALID) {
5861                 error = cmd6workaround(ccb);
5862         } else if (scsi_extract_sense_ccb(ccb,
5863             &error_code, &sense_key, &asc, &ascq)) {
5864                 if (sense_key == SSD_KEY_ILLEGAL_REQUEST)
5865                         error = cmd6workaround(ccb);
5866                 /*
5867                  * If the target replied with CAPACITY DATA HAS CHANGED UA,
5868                  * query the capacity and notify upper layers.
5869                  */
5870                 else if (sense_key == SSD_KEY_UNIT_ATTENTION &&
5871                     asc == 0x2A && ascq == 0x09) {
5872                         xpt_print(periph->path, "Capacity data has changed\n");
5873                         softc->flags &= ~DA_FLAG_PROBED;
5874                         dareprobe(periph);
5875                         sense_flags |= SF_NO_PRINT;
5876                 } else if (sense_key == SSD_KEY_UNIT_ATTENTION &&
5877                     asc == 0x28 && ascq == 0x00) {
5878                         softc->flags &= ~DA_FLAG_PROBED;
5879                         disk_media_changed(softc->disk, M_NOWAIT);
5880                 } else if (sense_key == SSD_KEY_UNIT_ATTENTION &&
5881                     asc == 0x3F && ascq == 0x03) {
5882                         xpt_print(periph->path, "INQUIRY data has changed\n");
5883                         softc->flags &= ~DA_FLAG_PROBED;
5884                         dareprobe(periph);
5885                         sense_flags |= SF_NO_PRINT;
5886                 } else if (sense_key == SSD_KEY_NOT_READY &&
5887                     asc == 0x3a && (softc->flags & DA_FLAG_PACK_INVALID) == 0) {
5888                         softc->flags |= DA_FLAG_PACK_INVALID;
5889                         disk_media_gone(softc->disk, M_NOWAIT);
5890                 }
5891         }
5892         if (error == ERESTART)
5893                 return (ERESTART);
5894
5895 #ifdef CAM_IO_STATS
5896         switch (ccb->ccb_h.status & CAM_STATUS_MASK) {
5897         case CAM_CMD_TIMEOUT:
5898                 softc->timeouts++;
5899                 break;
5900         case CAM_REQ_ABORTED:
5901         case CAM_REQ_CMP_ERR:
5902         case CAM_REQ_TERMIO:
5903         case CAM_UNREC_HBA_ERROR:
5904         case CAM_DATA_RUN_ERR:
5905                 softc->errors++;
5906                 break;
5907         default:
5908                 break;
5909         }
5910 #endif
5911
5912         /*
5913          * XXX
5914          * Until we have a better way of doing pack validation,
5915          * don't treat UAs as errors.
5916          */
5917         sense_flags |= SF_RETRY_UA;
5918
5919         if (softc->quirks & DA_Q_RETRY_BUSY)
5920                 sense_flags |= SF_RETRY_BUSY;
5921         return(cam_periph_error(ccb, cam_flags, sense_flags));
5922 }
5923
5924 static void
5925 damediapoll(void *arg)
5926 {
5927         struct cam_periph *periph = arg;
5928         struct da_softc *softc = periph->softc;
5929
5930         if (!cam_iosched_has_work_flags(softc->cam_iosched, DA_WORK_TUR) &&
5931             (softc->flags & DA_FLAG_TUR_PENDING) == 0 &&
5932             LIST_EMPTY(&softc->pending_ccbs)) {
5933                 if (da_periph_acquire(periph, DA_REF_TUR) == 0) {
5934                         cam_iosched_set_work_flags(softc->cam_iosched, DA_WORK_TUR);
5935                         daschedule(periph);
5936                 }
5937         }
5938         /* Queue us up again */
5939         if (da_poll_period != 0)
5940                 callout_schedule(&softc->mediapoll_c, da_poll_period * hz);
5941 }
5942
5943 static void
5944 daprevent(struct cam_periph *periph, int action)
5945 {
5946         struct  da_softc *softc;
5947         union   ccb *ccb;
5948         int     error;
5949
5950         cam_periph_assert(periph, MA_OWNED);
5951         softc = (struct da_softc *)periph->softc;
5952
5953         if (((action == PR_ALLOW)
5954           && (softc->flags & DA_FLAG_PACK_LOCKED) == 0)
5955          || ((action == PR_PREVENT)
5956           && (softc->flags & DA_FLAG_PACK_LOCKED) != 0)) {
5957                 return;
5958         }
5959
5960         ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL);
5961
5962         scsi_prevent(&ccb->csio,
5963                      /*retries*/1,
5964                      /*cbcfp*/NULL,
5965                      MSG_SIMPLE_Q_TAG,
5966                      action,
5967                      SSD_FULL_SIZE,
5968                      5000);
5969
5970         error = cam_periph_runccb(ccb, daerror, CAM_RETRY_SELTO,
5971             SF_RETRY_UA | SF_NO_PRINT, softc->disk->d_devstat);
5972
5973         if (error == 0) {
5974                 if (action == PR_ALLOW)
5975                         softc->flags &= ~DA_FLAG_PACK_LOCKED;
5976                 else
5977                         softc->flags |= DA_FLAG_PACK_LOCKED;
5978         }
5979
5980         xpt_release_ccb(ccb);
5981 }
5982
5983 static void
5984 dasetgeom(struct cam_periph *periph, uint32_t block_len, uint64_t maxsector,
5985           struct scsi_read_capacity_data_long *rcaplong, size_t rcap_len)
5986 {
5987         struct ccb_calc_geometry ccg;
5988         struct da_softc *softc;
5989         struct disk_params *dp;
5990         u_int lbppbe, lalba;
5991         int error;
5992
5993         softc = (struct da_softc *)periph->softc;
5994
5995         dp = &softc->params;
5996         dp->secsize = block_len;
5997         dp->sectors = maxsector + 1;
5998         if (rcaplong != NULL) {
5999                 lbppbe = rcaplong->prot_lbppbe & SRC16_LBPPBE;
6000                 lalba = scsi_2btoul(rcaplong->lalba_lbp);
6001                 lalba &= SRC16_LALBA_A;
6002                 if (rcaplong->prot & SRC16_PROT_EN)
6003                         softc->p_type = ((rcaplong->prot & SRC16_P_TYPE) >>
6004                             SRC16_P_TYPE_SHIFT) + 1;
6005                 else
6006                         softc->p_type = 0;
6007         } else {
6008                 lbppbe = 0;
6009                 lalba = 0;
6010                 softc->p_type = 0;
6011         }
6012
6013         if (lbppbe > 0) {
6014                 dp->stripesize = block_len << lbppbe;
6015                 dp->stripeoffset = (dp->stripesize - block_len * lalba) %
6016                     dp->stripesize;
6017         } else if (softc->quirks & DA_Q_4K) {
6018                 dp->stripesize = 4096;
6019                 dp->stripeoffset = 0;
6020         } else if (softc->unmap_gran != 0) {
6021                 dp->stripesize = block_len * softc->unmap_gran;
6022                 dp->stripeoffset = (dp->stripesize - block_len *
6023                     softc->unmap_gran_align) % dp->stripesize;
6024         } else {
6025                 dp->stripesize = 0;
6026                 dp->stripeoffset = 0;
6027         }
6028         /*
6029          * Have the controller provide us with a geometry
6030          * for this disk.  The only time the geometry
6031          * matters is when we boot and the controller
6032          * is the only one knowledgeable enough to come
6033          * up with something that will make this a bootable
6034          * device.
6035          */
6036         xpt_setup_ccb(&ccg.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
6037         ccg.ccb_h.func_code = XPT_CALC_GEOMETRY;
6038         ccg.block_size = dp->secsize;
6039         ccg.volume_size = dp->sectors;
6040         ccg.heads = 0;
6041         ccg.secs_per_track = 0;
6042         ccg.cylinders = 0;
6043         xpt_action((union ccb*)&ccg);
6044         if ((ccg.ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
6045                 /*
6046                  * We don't know what went wrong here- but just pick
6047                  * a geometry so we don't have nasty things like divide
6048                  * by zero.
6049                  */
6050                 dp->heads = 255;
6051                 dp->secs_per_track = 255;
6052                 dp->cylinders = dp->sectors / (255 * 255);
6053                 if (dp->cylinders == 0) {
6054                         dp->cylinders = 1;
6055                 }
6056         } else {
6057                 dp->heads = ccg.heads;
6058                 dp->secs_per_track = ccg.secs_per_track;
6059                 dp->cylinders = ccg.cylinders;
6060         }
6061
6062         /*
6063          * If the user supplied a read capacity buffer, and if it is
6064          * different than the previous buffer, update the data in the EDT.
6065          * If it's the same, we don't bother.  This avoids sending an
6066          * update every time someone opens this device.
6067          */
6068         if ((rcaplong != NULL)
6069          && (bcmp(rcaplong, &softc->rcaplong,
6070                   min(sizeof(softc->rcaplong), rcap_len)) != 0)) {
6071                 struct ccb_dev_advinfo cdai;
6072
6073                 xpt_setup_ccb(&cdai.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
6074                 cdai.ccb_h.func_code = XPT_DEV_ADVINFO;
6075                 cdai.buftype = CDAI_TYPE_RCAPLONG;
6076                 cdai.flags = CDAI_FLAG_STORE;
6077                 cdai.bufsiz = rcap_len;
6078                 cdai.buf = (uint8_t *)rcaplong;
6079                 xpt_action((union ccb *)&cdai);
6080                 if ((cdai.ccb_h.status & CAM_DEV_QFRZN) != 0)
6081                         cam_release_devq(cdai.ccb_h.path, 0, 0, 0, FALSE);
6082                 if (cdai.ccb_h.status != CAM_REQ_CMP) {
6083                         xpt_print(periph->path, "%s: failed to set read "
6084                                   "capacity advinfo\n", __func__);
6085                         /* Use cam_error_print() to decode the status */
6086                         cam_error_print((union ccb *)&cdai, CAM_ESF_CAM_STATUS,
6087                                         CAM_EPF_ALL);
6088                 } else {
6089                         bcopy(rcaplong, &softc->rcaplong,
6090                               min(sizeof(softc->rcaplong), rcap_len));
6091                 }
6092         }
6093
6094         softc->disk->d_sectorsize = softc->params.secsize;
6095         softc->disk->d_mediasize = softc->params.secsize * (off_t)softc->params.sectors;
6096         softc->disk->d_stripesize = softc->params.stripesize;
6097         softc->disk->d_stripeoffset = softc->params.stripeoffset;
6098         /* XXX: these are not actually "firmware" values, so they may be wrong */
6099         softc->disk->d_fwsectors = softc->params.secs_per_track;
6100         softc->disk->d_fwheads = softc->params.heads;
6101         softc->disk->d_devstat->block_size = softc->params.secsize;
6102         softc->disk->d_devstat->flags &= ~DEVSTAT_BS_UNAVAILABLE;
6103
6104         error = disk_resize(softc->disk, M_NOWAIT);
6105         if (error != 0)
6106                 xpt_print(periph->path, "disk_resize(9) failed, error = %d\n", error);
6107 }
6108
6109 static void
6110 dasendorderedtag(void *arg)
6111 {
6112         struct cam_periph *periph = arg;
6113         struct da_softc *softc = periph->softc;
6114
6115         cam_periph_assert(periph, MA_OWNED);
6116         if (da_send_ordered) {
6117                 if (!LIST_EMPTY(&softc->pending_ccbs)) {
6118                         if ((softc->flags & DA_FLAG_WAS_OTAG) == 0)
6119                                 softc->flags |= DA_FLAG_NEED_OTAG;
6120                         softc->flags &= ~DA_FLAG_WAS_OTAG;
6121                 }
6122         }
6123
6124         /* Queue us up again */
6125         callout_reset(&softc->sendordered_c,
6126             (da_default_timeout * hz) / DA_ORDEREDTAG_INTERVAL,
6127             dasendorderedtag, periph);
6128 }
6129
6130 /*
6131  * Step through all DA peripheral drivers, and if the device is still open,
6132  * sync the disk cache to physical media.
6133  */
6134 static void
6135 dashutdown(void * arg, int howto)
6136 {
6137         struct cam_periph *periph;
6138         struct da_softc *softc;
6139         union ccb *ccb;
6140         int error;
6141
6142         CAM_PERIPH_FOREACH(periph, &dadriver) {
6143                 softc = (struct da_softc *)periph->softc;
6144                 if (SCHEDULER_STOPPED()) {
6145                         /* If we paniced with the lock held, do not recurse. */
6146                         if (!cam_periph_owned(periph) &&
6147                             (softc->flags & DA_FLAG_OPEN)) {
6148                                 dadump(softc->disk, NULL, 0, 0, 0);
6149                         }
6150                         continue;
6151                 }
6152                 cam_periph_lock(periph);
6153
6154                 /*
6155                  * We only sync the cache if the drive is still open, and
6156                  * if the drive is capable of it..
6157                  */
6158                 if (((softc->flags & DA_FLAG_OPEN) == 0)
6159                  || (softc->quirks & DA_Q_NO_SYNC_CACHE)) {
6160                         cam_periph_unlock(periph);
6161                         continue;
6162                 }
6163
6164                 ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL);
6165                 scsi_synchronize_cache(&ccb->csio,
6166                                        /*retries*/0,
6167                                        /*cbfcnp*/NULL,
6168                                        MSG_SIMPLE_Q_TAG,
6169                                        /*begin_lba*/0, /* whole disk */
6170                                        /*lb_count*/0,
6171                                        SSD_FULL_SIZE,
6172                                        60 * 60 * 1000);
6173
6174                 error = cam_periph_runccb(ccb, daerror, /*cam_flags*/0,
6175                     /*sense_flags*/ SF_NO_RECOVERY | SF_NO_RETRY | SF_QUIET_IR,
6176                     softc->disk->d_devstat);
6177                 if (error != 0)
6178                         xpt_print(periph->path, "Synchronize cache failed\n");
6179                 xpt_release_ccb(ccb);
6180                 cam_periph_unlock(periph);
6181         }
6182 }
6183
6184 #else /* !_KERNEL */
6185
6186 /*
6187  * XXX These are only left out of the kernel build to silence warnings.  If,
6188  * for some reason these functions are used in the kernel, the ifdefs should
6189  * be moved so they are included both in the kernel and userland.
6190  */
6191 void
6192 scsi_format_unit(struct ccb_scsiio *csio, u_int32_t retries,
6193                  void (*cbfcnp)(struct cam_periph *, union ccb *),
6194                  u_int8_t tag_action, u_int8_t byte2, u_int16_t ileave,
6195                  u_int8_t *data_ptr, u_int32_t dxfer_len, u_int8_t sense_len,
6196                  u_int32_t timeout)
6197 {
6198         struct scsi_format_unit *scsi_cmd;
6199
6200         scsi_cmd = (struct scsi_format_unit *)&csio->cdb_io.cdb_bytes;
6201         scsi_cmd->opcode = FORMAT_UNIT;
6202         scsi_cmd->byte2 = byte2;
6203         scsi_ulto2b(ileave, scsi_cmd->interleave);
6204
6205         cam_fill_csio(csio,
6206                       retries,
6207                       cbfcnp,
6208                       /*flags*/ (dxfer_len > 0) ? CAM_DIR_OUT : CAM_DIR_NONE,
6209                       tag_action,
6210                       data_ptr,
6211                       dxfer_len,
6212                       sense_len,
6213                       sizeof(*scsi_cmd),
6214                       timeout);
6215 }
6216
6217 void
6218 scsi_read_defects(struct ccb_scsiio *csio, uint32_t retries,
6219                   void (*cbfcnp)(struct cam_periph *, union ccb *),
6220                   uint8_t tag_action, uint8_t list_format,
6221                   uint32_t addr_desc_index, uint8_t *data_ptr,
6222                   uint32_t dxfer_len, int minimum_cmd_size,
6223                   uint8_t sense_len, uint32_t timeout)
6224 {
6225         uint8_t cdb_len;
6226
6227         /*
6228          * These conditions allow using the 10 byte command.  Otherwise we
6229          * need to use the 12 byte command.
6230          */
6231         if ((minimum_cmd_size <= 10)
6232          && (addr_desc_index == 0)
6233          && (dxfer_len <= SRDD10_MAX_LENGTH)) {
6234                 struct scsi_read_defect_data_10 *cdb10;
6235
6236                 cdb10 = (struct scsi_read_defect_data_10 *)
6237                         &csio->cdb_io.cdb_bytes;
6238
6239                 cdb_len = sizeof(*cdb10);
6240                 bzero(cdb10, cdb_len);
6241                 cdb10->opcode = READ_DEFECT_DATA_10;
6242                 cdb10->format = list_format;
6243                 scsi_ulto2b(dxfer_len, cdb10->alloc_length);
6244         } else {
6245                 struct scsi_read_defect_data_12 *cdb12;
6246
6247                 cdb12 = (struct scsi_read_defect_data_12 *)
6248                         &csio->cdb_io.cdb_bytes;
6249
6250                 cdb_len = sizeof(*cdb12);
6251                 bzero(cdb12, cdb_len);
6252                 cdb12->opcode = READ_DEFECT_DATA_12;
6253                 cdb12->format = list_format;
6254                 scsi_ulto4b(dxfer_len, cdb12->alloc_length);
6255                 scsi_ulto4b(addr_desc_index, cdb12->address_descriptor_index);
6256         }
6257
6258         cam_fill_csio(csio,
6259                       retries,
6260                       cbfcnp,
6261                       /*flags*/ CAM_DIR_IN,
6262                       tag_action,
6263                       data_ptr,
6264                       dxfer_len,
6265                       sense_len,
6266                       cdb_len,
6267                       timeout);
6268 }
6269
6270 void
6271 scsi_sanitize(struct ccb_scsiio *csio, u_int32_t retries,
6272               void (*cbfcnp)(struct cam_periph *, union ccb *),
6273               u_int8_t tag_action, u_int8_t byte2, u_int16_t control,
6274               u_int8_t *data_ptr, u_int32_t dxfer_len, u_int8_t sense_len,
6275               u_int32_t timeout)
6276 {
6277         struct scsi_sanitize *scsi_cmd;
6278
6279         scsi_cmd = (struct scsi_sanitize *)&csio->cdb_io.cdb_bytes;
6280         scsi_cmd->opcode = SANITIZE;
6281         scsi_cmd->byte2 = byte2;
6282         scsi_cmd->control = control;
6283         scsi_ulto2b(dxfer_len, scsi_cmd->length);
6284
6285         cam_fill_csio(csio,
6286                       retries,
6287                       cbfcnp,
6288                       /*flags*/ (dxfer_len > 0) ? CAM_DIR_OUT : CAM_DIR_NONE,
6289                       tag_action,
6290                       data_ptr,
6291                       dxfer_len,
6292                       sense_len,
6293                       sizeof(*scsi_cmd),
6294                       timeout);
6295 }
6296
6297 #endif /* _KERNEL */
6298
6299 void
6300 scsi_zbc_out(struct ccb_scsiio *csio, uint32_t retries,
6301              void (*cbfcnp)(struct cam_periph *, union ccb *),
6302              uint8_t tag_action, uint8_t service_action, uint64_t zone_id,
6303              uint8_t zone_flags, uint8_t *data_ptr, uint32_t dxfer_len,
6304              uint8_t sense_len, uint32_t timeout)
6305 {
6306         struct scsi_zbc_out *scsi_cmd;
6307
6308         scsi_cmd = (struct scsi_zbc_out *)&csio->cdb_io.cdb_bytes;
6309         scsi_cmd->opcode = ZBC_OUT;
6310         scsi_cmd->service_action = service_action;
6311         scsi_u64to8b(zone_id, scsi_cmd->zone_id);
6312         scsi_cmd->zone_flags = zone_flags;
6313
6314         cam_fill_csio(csio,
6315                       retries,
6316                       cbfcnp,
6317                       /*flags*/ (dxfer_len > 0) ? CAM_DIR_OUT : CAM_DIR_NONE,
6318                       tag_action,
6319                       data_ptr,
6320                       dxfer_len,
6321                       sense_len,
6322                       sizeof(*scsi_cmd),
6323                       timeout);
6324 }
6325
6326 void
6327 scsi_zbc_in(struct ccb_scsiio *csio, uint32_t retries,
6328             void (*cbfcnp)(struct cam_periph *, union ccb *),
6329             uint8_t tag_action, uint8_t service_action, uint64_t zone_start_lba,
6330             uint8_t zone_options, uint8_t *data_ptr, uint32_t dxfer_len,
6331             uint8_t sense_len, uint32_t timeout)
6332 {
6333         struct scsi_zbc_in *scsi_cmd;
6334
6335         scsi_cmd = (struct scsi_zbc_in *)&csio->cdb_io.cdb_bytes;
6336         scsi_cmd->opcode = ZBC_IN;
6337         scsi_cmd->service_action = service_action;
6338         scsi_ulto4b(dxfer_len, scsi_cmd->length);
6339         scsi_u64to8b(zone_start_lba, scsi_cmd->zone_start_lba);
6340         scsi_cmd->zone_options = zone_options;
6341
6342         cam_fill_csio(csio,
6343                       retries,
6344                       cbfcnp,
6345                       /*flags*/ (dxfer_len > 0) ? CAM_DIR_IN : CAM_DIR_NONE,
6346                       tag_action,
6347                       data_ptr,
6348                       dxfer_len,
6349                       sense_len,
6350                       sizeof(*scsi_cmd),
6351                       timeout);
6352
6353 }
6354
6355 int
6356 scsi_ata_zac_mgmt_out(struct ccb_scsiio *csio, uint32_t retries,
6357                       void (*cbfcnp)(struct cam_periph *, union ccb *),
6358                       uint8_t tag_action, int use_ncq,
6359                       uint8_t zm_action, uint64_t zone_id, uint8_t zone_flags,
6360                       uint8_t *data_ptr, uint32_t dxfer_len,
6361                       uint8_t *cdb_storage, size_t cdb_storage_len,
6362                       uint8_t sense_len, uint32_t timeout)
6363 {
6364         uint8_t command_out, protocol, ata_flags;
6365         uint16_t features_out;
6366         uint32_t sectors_out, auxiliary;
6367         int retval;
6368
6369         retval = 0;
6370
6371         if (use_ncq == 0) {
6372                 command_out = ATA_ZAC_MANAGEMENT_OUT;
6373                 features_out = (zm_action & 0xf) | (zone_flags << 8);
6374                 ata_flags = AP_FLAG_BYT_BLOK_BLOCKS;
6375                 if (dxfer_len == 0) {
6376                         protocol = AP_PROTO_NON_DATA;
6377                         ata_flags |= AP_FLAG_TLEN_NO_DATA;
6378                         sectors_out = 0;
6379                 } else {
6380                         protocol = AP_PROTO_DMA;
6381                         ata_flags |= AP_FLAG_TLEN_SECT_CNT |
6382                                      AP_FLAG_TDIR_TO_DEV;
6383                         sectors_out = ((dxfer_len >> 9) & 0xffff);
6384                 }
6385                 auxiliary = 0;
6386         } else {
6387                 ata_flags = AP_FLAG_BYT_BLOK_BLOCKS;
6388                 if (dxfer_len == 0) {
6389                         command_out = ATA_NCQ_NON_DATA;
6390                         features_out = ATA_NCQ_ZAC_MGMT_OUT;
6391                         /*
6392                          * We're assuming the SCSI to ATA translation layer
6393                          * will set the NCQ tag number in the tag field.
6394                          * That isn't clear from the SAT-4 spec (as of rev 05).
6395                          */
6396                         sectors_out = 0;
6397                         ata_flags |= AP_FLAG_TLEN_NO_DATA;
6398                 } else {
6399                         command_out = ATA_SEND_FPDMA_QUEUED;
6400                         /*
6401                          * Note that we're defaulting to normal priority,
6402                          * and assuming that the SCSI to ATA translation
6403                          * layer will insert the NCQ tag number in the tag
6404                          * field.  That isn't clear in the SAT-4 spec (as
6405                          * of rev 05).
6406                          */
6407                         sectors_out = ATA_SFPDMA_ZAC_MGMT_OUT << 8;
6408
6409                         ata_flags |= AP_FLAG_TLEN_FEAT |
6410                                      AP_FLAG_TDIR_TO_DEV;
6411
6412                         /*
6413                          * For SEND FPDMA QUEUED, the transfer length is
6414                          * encoded in the FEATURE register, and 0 means
6415                          * that 65536 512 byte blocks are to be tranferred.
6416                          * In practice, it seems unlikely that we'll see
6417                          * a transfer that large, and it may confuse the
6418                          * the SAT layer, because generally that means that
6419                          * 0 bytes should be transferred.
6420                          */
6421                         if (dxfer_len == (65536 * 512)) {
6422                                 features_out = 0;
6423                         } else if (dxfer_len <= (65535 * 512)) {
6424                                 features_out = ((dxfer_len >> 9) & 0xffff);
6425                         } else {
6426                                 /* The transfer is too big. */
6427                                 retval = 1;
6428                                 goto bailout;
6429                         }
6430
6431                 }
6432
6433                 auxiliary = (zm_action & 0xf) | (zone_flags << 8);
6434                 protocol = AP_PROTO_FPDMA;
6435         }
6436
6437         protocol |= AP_EXTEND;
6438
6439         retval = scsi_ata_pass(csio,
6440             retries,
6441             cbfcnp,
6442             /*flags*/ (dxfer_len > 0) ? CAM_DIR_OUT : CAM_DIR_NONE,
6443             tag_action,
6444             /*protocol*/ protocol,
6445             /*ata_flags*/ ata_flags,
6446             /*features*/ features_out,
6447             /*sector_count*/ sectors_out,
6448             /*lba*/ zone_id,
6449             /*command*/ command_out,
6450             /*device*/ 0,
6451             /*icc*/ 0,
6452             /*auxiliary*/ auxiliary,
6453             /*control*/ 0,
6454             /*data_ptr*/ data_ptr,
6455             /*dxfer_len*/ dxfer_len,
6456             /*cdb_storage*/ cdb_storage,
6457             /*cdb_storage_len*/ cdb_storage_len,
6458             /*minimum_cmd_size*/ 0,
6459             /*sense_len*/ SSD_FULL_SIZE,
6460             /*timeout*/ timeout);
6461
6462 bailout:
6463
6464         return (retval);
6465 }
6466
6467 int
6468 scsi_ata_zac_mgmt_in(struct ccb_scsiio *csio, uint32_t retries,
6469                      void (*cbfcnp)(struct cam_periph *, union ccb *),
6470                      uint8_t tag_action, int use_ncq,
6471                      uint8_t zm_action, uint64_t zone_id, uint8_t zone_flags,
6472                      uint8_t *data_ptr, uint32_t dxfer_len,
6473                      uint8_t *cdb_storage, size_t cdb_storage_len,
6474                      uint8_t sense_len, uint32_t timeout)
6475 {
6476         uint8_t command_out, protocol;
6477         uint16_t features_out, sectors_out;
6478         uint32_t auxiliary;
6479         int ata_flags;
6480         int retval;
6481
6482         retval = 0;
6483         ata_flags = AP_FLAG_TDIR_FROM_DEV | AP_FLAG_BYT_BLOK_BLOCKS;
6484
6485         if (use_ncq == 0) {
6486                 command_out = ATA_ZAC_MANAGEMENT_IN;
6487                 /* XXX KDM put a macro here */
6488                 features_out = (zm_action & 0xf) | (zone_flags << 8);
6489                 sectors_out = dxfer_len >> 9; /* XXX KDM macro */
6490                 protocol = AP_PROTO_DMA;
6491                 ata_flags |= AP_FLAG_TLEN_SECT_CNT;
6492                 auxiliary = 0;
6493         } else {
6494                 ata_flags |= AP_FLAG_TLEN_FEAT;
6495
6496                 command_out = ATA_RECV_FPDMA_QUEUED;
6497                 sectors_out = ATA_RFPDMA_ZAC_MGMT_IN << 8;
6498
6499                 /*
6500                  * For RECEIVE FPDMA QUEUED, the transfer length is
6501                  * encoded in the FEATURE register, and 0 means
6502                  * that 65536 512 byte blocks are to be tranferred.
6503                  * In practice, it seems unlikely that we'll see
6504                  * a transfer that large, and it may confuse the
6505                  * the SAT layer, because generally that means that
6506                  * 0 bytes should be transferred.
6507                  */
6508                 if (dxfer_len == (65536 * 512)) {
6509                         features_out = 0;
6510                 } else if (dxfer_len <= (65535 * 512)) {
6511                         features_out = ((dxfer_len >> 9) & 0xffff);
6512                 } else {
6513                         /* The transfer is too big. */
6514                         retval = 1;
6515                         goto bailout;
6516                 }
6517                 auxiliary = (zm_action & 0xf) | (zone_flags << 8),
6518                 protocol = AP_PROTO_FPDMA;
6519         }
6520
6521         protocol |= AP_EXTEND;
6522
6523         retval = scsi_ata_pass(csio,
6524             retries,
6525             cbfcnp,
6526             /*flags*/ CAM_DIR_IN,
6527             tag_action,
6528             /*protocol*/ protocol,
6529             /*ata_flags*/ ata_flags,
6530             /*features*/ features_out,
6531             /*sector_count*/ sectors_out,
6532             /*lba*/ zone_id,
6533             /*command*/ command_out,
6534             /*device*/ 0,
6535             /*icc*/ 0,
6536             /*auxiliary*/ auxiliary,
6537             /*control*/ 0,
6538             /*data_ptr*/ data_ptr,
6539             /*dxfer_len*/ (dxfer_len >> 9) * 512, /* XXX KDM */
6540             /*cdb_storage*/ cdb_storage,
6541             /*cdb_storage_len*/ cdb_storage_len,
6542             /*minimum_cmd_size*/ 0,
6543             /*sense_len*/ SSD_FULL_SIZE,
6544             /*timeout*/ timeout);
6545
6546 bailout:
6547         return (retval);
6548 }