sys/cam/scsi/scsi_da.c

   1 /*-
   2  * Implementation of SCSI Direct Access Peripheral driver for CAM.
   3  *
   4  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   5  *
   6  * Copyright (c) 1997 Justin T. Gibbs.
   7  * All rights reserved.
   8  *
   9  * Redistribution and use in source and binary forms, with or without
  10  * modification, are permitted provided that the following conditions
  11  * are met:
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions, and the following disclaimer,
  14  *    without modification, immediately at the beginning of the file.
  15  * 2. The name of the author may not be used to endorse or promote products
  16  *    derived from this software without specific prior written permission.
  17  *
  18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  21  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
  22  * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  28  * SUCH DAMAGE.
  29  */
  30
  31 #include <sys/cdefs.h>
  32 __FBSDID("$FreeBSD$");
  33
  34 #include <sys/param.h>
  35
  36 #ifdef _KERNEL
  37 #include "opt_da.h"
  38 #include <sys/systm.h>
  39 #include <sys/kernel.h>
  40 #include <sys/bio.h>
  41 #include <sys/sysctl.h>
  42 #include <sys/taskqueue.h>
  43 #include <sys/lock.h>
  44 #include <sys/mutex.h>
  45 #include <sys/conf.h>
  46 #include <sys/devicestat.h>
  47 #include <sys/eventhandler.h>
  48 #include <sys/malloc.h>
  49 #include <sys/cons.h>
  50 #include <sys/endian.h>
  51 #include <sys/proc.h>
  52 #include <sys/sbuf.h>
  53 #include <geom/geom.h>
  54 #include <geom/geom_disk.h>
  55 #include <machine/atomic.h>
  56 #endif /* _KERNEL */
  57
  58 #ifndef _KERNEL
  59 #include <stdio.h>
  60 #include <string.h>
  61 #endif /* _KERNEL */
  62
  63 #include <cam/cam.h>
  64 #include <cam/cam_ccb.h>
  65 #include <cam/cam_periph.h>
  66 #include <cam/cam_xpt_periph.h>
  67 #include <cam/cam_sim.h>
  68 #include <cam/cam_iosched.h>
  69
  70 #include <cam/scsi/scsi_message.h>
  71 #include <cam/scsi/scsi_da.h>
  72
  73 #ifdef _KERNEL
  74 /*
  75  * Note that there are probe ordering dependencies here.  The order isn't
  76  * controlled by this enumeration, but by explicit state transitions in
  77  * dastart() and dadone().  Here are some of the dependencies:
  78  *
  79  * 1. RC should come first, before RC16, unless there is evidence that RC16
  80  *    is supported.
  81  * 2. BDC needs to come before any of the ATA probes, or the ZONE probe.
  82  * 3. The ATA probes should go in this order:
  83  *    ATA -> LOGDIR -> IDDIR -> SUP -> ATA_ZONE
  84  */
  85 typedef enum {
  86         DA_STATE_PROBE_WP,
  87         DA_STATE_PROBE_RC,
  88         DA_STATE_PROBE_RC16,
  89         DA_STATE_PROBE_LBP,
  90         DA_STATE_PROBE_BLK_LIMITS,
  91         DA_STATE_PROBE_BDC,
  92         DA_STATE_PROBE_ATA,
  93         DA_STATE_PROBE_ATA_LOGDIR,
  94         DA_STATE_PROBE_ATA_IDDIR,
  95         DA_STATE_PROBE_ATA_SUP,
  96         DA_STATE_PROBE_ATA_ZONE,
  97         DA_STATE_PROBE_ZONE,
  98         DA_STATE_NORMAL
  99 } da_state;
 100
 101 typedef enum {
 102         DA_FLAG_PACK_INVALID    = 0x000001,
 103         DA_FLAG_NEW_PACK        = 0x000002,
 104         DA_FLAG_PACK_LOCKED     = 0x000004,
 105         DA_FLAG_PACK_REMOVABLE  = 0x000008,
 106         DA_FLAG_NEED_OTAG       = 0x000020,
 107         DA_FLAG_WAS_OTAG        = 0x000040,
 108         DA_FLAG_RETRY_UA        = 0x000080,
 109         DA_FLAG_OPEN            = 0x000100,
 110         DA_FLAG_SCTX_INIT       = 0x000200,
 111         DA_FLAG_CAN_RC16        = 0x000400,
 112         DA_FLAG_PROBED          = 0x000800,
 113         DA_FLAG_DIRTY           = 0x001000,
 114         DA_FLAG_ANNOUNCED       = 0x002000,
 115         DA_FLAG_CAN_ATA_DMA     = 0x004000,
 116         DA_FLAG_CAN_ATA_LOG     = 0x008000,
 117         DA_FLAG_CAN_ATA_IDLOG   = 0x010000,
 118         DA_FLAG_CAN_ATA_SUPCAP  = 0x020000,
 119         DA_FLAG_CAN_ATA_ZONE    = 0x040000,
 120         DA_FLAG_TUR_PENDING     = 0x080000
 121 } da_flags;
 122
 123 typedef enum {
 124         DA_Q_NONE               = 0x00,
 125         DA_Q_NO_SYNC_CACHE      = 0x01,
 126         DA_Q_NO_6_BYTE          = 0x02,
 127         DA_Q_NO_PREVENT         = 0x04,
 128         DA_Q_4K                 = 0x08,
 129         DA_Q_NO_RC16            = 0x10,
 130         DA_Q_NO_UNMAP           = 0x20,
 131         DA_Q_RETRY_BUSY         = 0x40,
 132         DA_Q_SMR_DM             = 0x80,
 133         DA_Q_STRICT_UNMAP       = 0x100
 134 } da_quirks;
 135
 136 #define DA_Q_BIT_STRING         \
 137         "\020"                  \
 138         "\001NO_SYNC_CACHE"     \
 139         "\002NO_6_BYTE"         \
 140         "\003NO_PREVENT"        \
 141         "\0044K"                \
 142         "\005NO_RC16"           \
 143         "\006NO_UNMAP"          \
 144         "\007RETRY_BUSY"        \
 145         "\010SMR_DM"            \
 146         "\011STRICT_UNMAP"
 147
 148 typedef enum {
 149         DA_CCB_PROBE_RC         = 0x01,
 150         DA_CCB_PROBE_RC16       = 0x02,
 151         DA_CCB_PROBE_LBP        = 0x03,
 152         DA_CCB_PROBE_BLK_LIMITS = 0x04,
 153         DA_CCB_PROBE_BDC        = 0x05,
 154         DA_CCB_PROBE_ATA        = 0x06,
 155         DA_CCB_BUFFER_IO        = 0x07,
 156         DA_CCB_DUMP             = 0x0A,
 157         DA_CCB_DELETE           = 0x0B,
 158         DA_CCB_TUR              = 0x0C,
 159         DA_CCB_PROBE_ZONE       = 0x0D,
 160         DA_CCB_PROBE_ATA_LOGDIR = 0x0E,
 161         DA_CCB_PROBE_ATA_IDDIR  = 0x0F,
 162         DA_CCB_PROBE_ATA_SUP    = 0x10,
 163         DA_CCB_PROBE_ATA_ZONE   = 0x11,
 164         DA_CCB_PROBE_WP         = 0x12,
 165         DA_CCB_TYPE_MASK        = 0x1F,
 166         DA_CCB_RETRY_UA         = 0x20
 167 } da_ccb_state;
 168
 169 /*
 170  * Order here is important for method choice
 171  *
 172  * We prefer ATA_TRIM as tests run against a Sandforce 2281 SSD attached to
 173  * LSI 2008 (mps) controller (FW: v12, Drv: v14) resulted 20% quicker deletes
 174  * using ATA_TRIM than the corresponding UNMAP results for a real world mysql
 175  * import taking 5mins.
 176  *
 177  */
 178 typedef enum {
 179         DA_DELETE_NONE,
 180         DA_DELETE_DISABLE,
 181         DA_DELETE_ATA_TRIM,
 182         DA_DELETE_UNMAP,
 183         DA_DELETE_WS16,
 184         DA_DELETE_WS10,
 185         DA_DELETE_ZERO,
 186         DA_DELETE_MIN = DA_DELETE_ATA_TRIM,
 187         DA_DELETE_MAX = DA_DELETE_ZERO
 188 } da_delete_methods;
 189
 190 /*
 191  * For SCSI, host managed drives show up as a separate device type.  For
 192  * ATA, host managed drives also have a different device signature.
 193  * XXX KDM figure out the ATA host managed signature.
 194  */
 195 typedef enum {
 196         DA_ZONE_NONE            = 0x00,
 197         DA_ZONE_DRIVE_MANAGED   = 0x01,
 198         DA_ZONE_HOST_AWARE      = 0x02,
 199         DA_ZONE_HOST_MANAGED    = 0x03
 200 } da_zone_mode;
 201
 202 /*
 203  * We distinguish between these interface cases in addition to the drive type:
 204  * o ATA drive behind a SCSI translation layer that knows about ZBC/ZAC
 205  * o ATA drive behind a SCSI translation layer that does not know about
 206  *   ZBC/ZAC, and so needs to be managed via ATA passthrough.  In this
 207  *   case, we would need to share the ATA code with the ada(4) driver.
 208  * o SCSI drive.
 209  */
 210 typedef enum {
 211         DA_ZONE_IF_SCSI,
 212         DA_ZONE_IF_ATA_PASS,
 213         DA_ZONE_IF_ATA_SAT,
 214 } da_zone_interface;
 215
 216 typedef enum {
 217         DA_ZONE_FLAG_RZ_SUP             = 0x0001,
 218         DA_ZONE_FLAG_OPEN_SUP           = 0x0002,
 219         DA_ZONE_FLAG_CLOSE_SUP          = 0x0004,
 220         DA_ZONE_FLAG_FINISH_SUP         = 0x0008,
 221         DA_ZONE_FLAG_RWP_SUP            = 0x0010,
 222         DA_ZONE_FLAG_SUP_MASK           = (DA_ZONE_FLAG_RZ_SUP |
 223                                            DA_ZONE_FLAG_OPEN_SUP |
 224                                            DA_ZONE_FLAG_CLOSE_SUP |
 225                                            DA_ZONE_FLAG_FINISH_SUP |
 226                                            DA_ZONE_FLAG_RWP_SUP),
 227         DA_ZONE_FLAG_URSWRZ             = 0x0020,
 228         DA_ZONE_FLAG_OPT_SEQ_SET        = 0x0040,
 229         DA_ZONE_FLAG_OPT_NONSEQ_SET     = 0x0080,
 230         DA_ZONE_FLAG_MAX_SEQ_SET        = 0x0100,
 231         DA_ZONE_FLAG_SET_MASK           = (DA_ZONE_FLAG_OPT_SEQ_SET |
 232                                            DA_ZONE_FLAG_OPT_NONSEQ_SET |
 233                                            DA_ZONE_FLAG_MAX_SEQ_SET)
 234 } da_zone_flags;
 235
 236 static struct da_zone_desc {
 237         da_zone_flags value;
 238         const char *desc;
 239 } da_zone_desc_table[] = {
 240         {DA_ZONE_FLAG_RZ_SUP, "Report Zones" },
 241         {DA_ZONE_FLAG_OPEN_SUP, "Open" },
 242         {DA_ZONE_FLAG_CLOSE_SUP, "Close" },
 243         {DA_ZONE_FLAG_FINISH_SUP, "Finish" },
 244         {DA_ZONE_FLAG_RWP_SUP, "Reset Write Pointer" },
 245 };
 246
 247 typedef void da_delete_func_t (struct cam_periph *periph, union ccb *ccb,
 248                               struct bio *bp);
 249 static da_delete_func_t da_delete_trim;
 250 static da_delete_func_t da_delete_unmap;
 251 static da_delete_func_t da_delete_ws;
 252
 253 static const void * da_delete_functions[] = {
 254         NULL,
 255         NULL,
 256         da_delete_trim,
 257         da_delete_unmap,
 258         da_delete_ws,
 259         da_delete_ws,
 260         da_delete_ws
 261 };
 262
 263 static const char *da_delete_method_names[] =
 264     { "NONE", "DISABLE", "ATA_TRIM", "UNMAP", "WS16", "WS10", "ZERO" };
 265 static const char *da_delete_method_desc[] =
 266     { "NONE", "DISABLED", "ATA TRIM", "UNMAP", "WRITE SAME(16) with UNMAP",
 267       "WRITE SAME(10) with UNMAP", "ZERO" };
 268
 269 /* Offsets into our private area for storing information */
 270 #define ccb_state       ppriv_field0
 271 #define ccb_bp          ppriv_ptr1
 272
 273 struct disk_params {
 274         u_int8_t  heads;
 275         u_int32_t cylinders;
 276         u_int8_t  secs_per_track;
 277         u_int32_t secsize;      /* Number of bytes/sector */
 278         u_int64_t sectors;      /* total number sectors */
 279         u_int     stripesize;
 280         u_int     stripeoffset;
 281 };
 282
 283 #define UNMAP_RANGE_MAX         0xffffffff
 284 #define UNMAP_HEAD_SIZE         8
 285 #define UNMAP_RANGE_SIZE        16
 286 #define UNMAP_MAX_RANGES        2048 /* Protocol Max is 4095 */
 287 #define UNMAP_BUF_SIZE          ((UNMAP_MAX_RANGES * UNMAP_RANGE_SIZE) + \
 288                                 UNMAP_HEAD_SIZE)
 289
 290 #define WS10_MAX_BLKS           0xffff
 291 #define WS16_MAX_BLKS           0xffffffff
 292 #define ATA_TRIM_MAX_RANGES     ((UNMAP_BUF_SIZE / \
 293         (ATA_DSM_RANGE_SIZE * ATA_DSM_BLK_SIZE)) * ATA_DSM_BLK_SIZE)
 294
 295 #define DA_WORK_TUR             (1 << 16)
 296
 297 typedef enum {
 298         DA_REF_OPEN = 1,
 299         DA_REF_OPEN_HOLD,
 300         DA_REF_CLOSE_HOLD,
 301         DA_REF_PROBE_HOLD,
 302         DA_REF_TUR,
 303         DA_REF_GEOM,
 304         DA_REF_SYSCTL,
 305         DA_REF_REPROBE,
 306         DA_REF_MAX              /* KEEP LAST */
 307 } da_ref_token;
 308
 309 struct da_softc {
 310         struct   cam_iosched_softc *cam_iosched;
 311         struct   bio_queue_head delete_run_queue;
 312         LIST_HEAD(, ccb_hdr) pending_ccbs;
 313         int      refcount;              /* Active xpt_action() calls */
 314         da_state state;
 315         da_flags flags;
 316         da_quirks quirks;
 317         int      minimum_cmd_size;
 318         int      error_inject;
 319         int      trim_max_ranges;
 320         int      delete_available;      /* Delete methods possibly available */
 321         da_zone_mode                    zone_mode;
 322         da_zone_interface               zone_interface;
 323         da_zone_flags                   zone_flags;
 324         struct ata_gp_log_dir           ata_logdir;
 325         int                             valid_logdir_len;
 326         struct ata_identify_log_pages   ata_iddir;
 327         int                             valid_iddir_len;
 328         uint64_t                        optimal_seq_zones;
 329         uint64_t                        optimal_nonseq_zones;
 330         uint64_t                        max_seq_zones;
 331         u_int                   maxio;
 332         uint32_t                unmap_max_ranges;
 333         uint32_t                unmap_max_lba; /* Max LBAs in UNMAP req */
 334         uint32_t                unmap_gran;
 335         uint32_t                unmap_gran_align;
 336         uint64_t                ws_max_blks;
 337         uint64_t                trim_count;
 338         uint64_t                trim_ranges;
 339         uint64_t                trim_lbas;
 340         da_delete_methods       delete_method_pref;
 341         da_delete_methods       delete_method;
 342         da_delete_func_t        *delete_func;
 343         int                     unmappedio;
 344         int                     rotating;
 345         struct   disk_params params;
 346         struct   disk *disk;
 347         union    ccb saved_ccb;
 348         struct task             sysctl_task;
 349         struct sysctl_ctx_list  sysctl_ctx;
 350         struct sysctl_oid       *sysctl_tree;
 351         struct callout          sendordered_c;
 352         uint64_t wwpn;
 353         uint8_t  unmap_buf[UNMAP_BUF_SIZE];
 354         struct scsi_read_capacity_data_long rcaplong;
 355         struct callout          mediapoll_c;
 356         int                     ref_flags[DA_REF_MAX];
 357 #ifdef CAM_IO_STATS
 358         struct sysctl_ctx_list  sysctl_stats_ctx;
 359         struct sysctl_oid       *sysctl_stats_tree;
 360         u_int   errors;
 361         u_int   timeouts;
 362         u_int   invalidations;
 363 #endif
 364 #define DA_ANNOUNCETMP_SZ 160
 365         char                    announce_temp[DA_ANNOUNCETMP_SZ];
 366 #define DA_ANNOUNCE_SZ 400
 367         char                    announcebuf[DA_ANNOUNCE_SZ];
 368 };
 369
 370 #define dadeleteflag(softc, delete_method, enable)                      \
 371         if (enable) {                                                   \
 372                 softc->delete_available |= (1 << delete_method);        \
 373         } else {                                                        \
 374                 softc->delete_available &= ~(1 << delete_method);       \
 375         }
 376
 377 struct da_quirk_entry {
 378         struct scsi_inquiry_pattern inq_pat;
 379         da_quirks quirks;
 380 };
 381
 382 static const char quantum[] = "QUANTUM";
 383 static const char microp[] = "MICROP";
 384
 385 static struct da_quirk_entry da_quirk_table[] =
 386 {
 387         /* SPI, FC devices */
 388         {
 389                 /*
 390                  * Fujitsu M2513A MO drives.
 391                  * Tested devices: M2513A2 firmware versions 1200 & 1300.
 392                  * (dip switch selects whether T_DIRECT or T_OPTICAL device)
 393                  * Reported by: W.Scholten <whs@xs4all.nl>
 394                  */
 395                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "FUJITSU", "M2513A", "*"},
 396                 /*quirks*/ DA_Q_NO_SYNC_CACHE
 397         },
 398         {
 399                 /* See above. */
 400                 {T_OPTICAL, SIP_MEDIA_REMOVABLE, "FUJITSU", "M2513A", "*"},
 401                 /*quirks*/ DA_Q_NO_SYNC_CACHE
 402         },
 403         {
 404                 /*
 405                  * This particular Fujitsu drive doesn't like the
 406                  * synchronize cache command.
 407                  * Reported by: Tom Jackson <toj@gorilla.net>
 408                  */
 409                 {T_DIRECT, SIP_MEDIA_FIXED, "FUJITSU", "M2954*", "*"},
 410                 /*quirks*/ DA_Q_NO_SYNC_CACHE
 411         },
 412         {
 413                 /*
 414                  * This drive doesn't like the synchronize cache command
 415                  * either.  Reported by: Matthew Jacob <mjacob@feral.com>
 416                  * in NetBSD PR kern/6027, August 24, 1998.
 417                  */
 418                 {T_DIRECT, SIP_MEDIA_FIXED, microp, "2217*", "*"},
 419                 /*quirks*/ DA_Q_NO_SYNC_CACHE
 420         },
 421         {
 422                 /*
 423                  * This drive doesn't like the synchronize cache command
 424                  * either.  Reported by: Hellmuth Michaelis (hm@kts.org)
 425                  * (PR 8882).
 426                  */
 427                 {T_DIRECT, SIP_MEDIA_FIXED, microp, "2112*", "*"},
 428                 /*quirks*/ DA_Q_NO_SYNC_CACHE
 429         },
 430         {
 431                 /*
 432                  * Doesn't like the synchronize cache command.
 433                  * Reported by: Blaz Zupan <blaz@gold.amis.net>
 434                  */
 435                 {T_DIRECT, SIP_MEDIA_FIXED, "NEC", "D3847*", "*"},
 436                 /*quirks*/ DA_Q_NO_SYNC_CACHE
 437         },
 438         {
 439                 /*
 440                  * Doesn't like the synchronize cache command.
 441                  * Reported by: Blaz Zupan <blaz@gold.amis.net>
 442                  */
 443                 {T_DIRECT, SIP_MEDIA_FIXED, quantum, "MAVERICK 540S", "*"},
 444                 /*quirks*/ DA_Q_NO_SYNC_CACHE
 445         },
 446         {
 447                 /*
 448                  * Doesn't like the synchronize cache command.
 449                  */
 450                 {T_DIRECT, SIP_MEDIA_FIXED, quantum, "LPS525S", "*"},
 451                 /*quirks*/ DA_Q_NO_SYNC_CACHE
 452         },
 453         {
 454                 /*
 455                  * Doesn't like the synchronize cache command.
 456                  * Reported by: walter@pelissero.de
 457                  */
 458                 {T_DIRECT, SIP_MEDIA_FIXED, quantum, "LPS540S", "*"},
 459                 /*quirks*/ DA_Q_NO_SYNC_CACHE
 460         },
 461         {
 462                 /*
 463                  * Doesn't work correctly with 6 byte reads/writes.
 464                  * Returns illegal request, and points to byte 9 of the
 465                  * 6-byte CDB.
 466                  * Reported by:  Adam McDougall <bsdx@spawnet.com>
 467                  */
 468                 {T_DIRECT, SIP_MEDIA_FIXED, quantum, "VIKING 4*", "*"},
 469                 /*quirks*/ DA_Q_NO_6_BYTE
 470         },
 471         {
 472                 /* See above. */
 473                 {T_DIRECT, SIP_MEDIA_FIXED, quantum, "VIKING 2*", "*"},
 474                 /*quirks*/ DA_Q_NO_6_BYTE
 475         },
 476         {
 477                 /*
 478                  * Doesn't like the synchronize cache command.
 479                  * Reported by: walter@pelissero.de
 480                  */
 481                 {T_DIRECT, SIP_MEDIA_FIXED, "CONNER", "CP3500*", "*"},
 482                 /*quirks*/ DA_Q_NO_SYNC_CACHE
 483         },
 484         {
 485                 /*
 486                  * The CISS RAID controllers do not support SYNC_CACHE
 487                  */
 488                 {T_DIRECT, SIP_MEDIA_FIXED, "COMPAQ", "RAID*", "*"},
 489                 /*quirks*/ DA_Q_NO_SYNC_CACHE
 490         },
 491         {
 492                 /*
 493                  * The STEC SSDs sometimes hang on UNMAP.
 494                  */
 495                 {T_DIRECT, SIP_MEDIA_FIXED, "STEC", "*", "*"},
 496                 /*quirks*/ DA_Q_NO_UNMAP
 497         },
 498         {
 499                 /*
 500                  * VMware returns BUSY status when storage has transient
 501                  * connectivity problems, so better wait.
 502                  * Also VMware returns odd errors on misaligned UNMAPs.
 503                  */
 504                 {T_DIRECT, SIP_MEDIA_FIXED, "VMware*", "*", "*"},
 505                 /*quirks*/ DA_Q_RETRY_BUSY | DA_Q_STRICT_UNMAP
 506         },
 507         /* USB mass storage devices supported by umass(4) */
 508         {
 509                 /*
 510                  * EXATELECOM (Sigmatel) i-Bead 100/105 USB Flash MP3 Player
 511                  * PR: kern/51675
 512                  */
 513                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "EXATEL", "i-BEAD10*", "*"},
 514                 /*quirks*/ DA_Q_NO_SYNC_CACHE
 515         },
 516         {
 517                 /*
 518                  * Power Quotient Int. (PQI) USB flash key
 519                  * PR: kern/53067
 520                  */
 521                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "Generic*", "USB Flash Disk*",
 522                 "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
 523         },
 524         {
 525                 /*
 526                  * Creative Nomad MUVO mp3 player (USB)
 527                  * PR: kern/53094
 528                  */
 529                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "CREATIVE", "NOMAD_MUVO", "*"},
 530                 /*quirks*/ DA_Q_NO_SYNC_CACHE|DA_Q_NO_PREVENT
 531         },
 532         {
 533                 /*
 534                  * Jungsoft NEXDISK USB flash key
 535                  * PR: kern/54737
 536                  */
 537                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "JUNGSOFT", "NEXDISK*", "*"},
 538                 /*quirks*/ DA_Q_NO_SYNC_CACHE
 539         },
 540         {
 541                 /*
 542                  * FreeDik USB Mini Data Drive
 543                  * PR: kern/54786
 544                  */
 545                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "FreeDik*", "Mini Data Drive",
 546                 "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
 547         },
 548         {
 549                 /*
 550                  * Sigmatel USB Flash MP3 Player
 551                  * PR: kern/57046
 552                  */
 553                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "SigmaTel", "MSCN", "*"},
 554                 /*quirks*/ DA_Q_NO_SYNC_CACHE|DA_Q_NO_PREVENT
 555         },
 556         {
 557                 /*
 558                  * Neuros USB Digital Audio Computer
 559                  * PR: kern/63645
 560                  */
 561                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "NEUROS", "dig. audio comp.",
 562                 "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
 563         },
 564         {
 565                 /*
 566                  * SEAGRAND NP-900 MP3 Player
 567                  * PR: kern/64563
 568                  */
 569                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "SEAGRAND", "NP-900*", "*"},
 570                 /*quirks*/ DA_Q_NO_SYNC_CACHE|DA_Q_NO_PREVENT
 571         },
 572         {
 573                 /*
 574                  * iRiver iFP MP3 player (with UMS Firmware)
 575                  * PR: kern/54881, i386/63941, kern/66124
 576                  */
 577                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "iRiver", "iFP*", "*"},
 578                 /*quirks*/ DA_Q_NO_SYNC_CACHE
 579         },
 580         {
 581                 /*
 582                  * Frontier Labs NEX IA+ Digital Audio Player, rev 1.10/0.01
 583                  * PR: kern/70158
 584                  */
 585                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "FL" , "Nex*", "*"},
 586                 /*quirks*/ DA_Q_NO_SYNC_CACHE
 587         },
 588         {
 589                 /*
 590                  * ZICPlay USB MP3 Player with FM
 591                  * PR: kern/75057
 592                  */
 593                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "ACTIONS*" , "USB DISK*", "*"},
 594                 /*quirks*/ DA_Q_NO_SYNC_CACHE
 595         },
 596         {
 597                 /*
 598                  * TEAC USB floppy mechanisms
 599                  */
 600                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "TEAC" , "FD-05*", "*"},
 601                 /*quirks*/ DA_Q_NO_SYNC_CACHE
 602         },
 603         {
 604                 /*
 605                  * Kingston DataTraveler II+ USB Pen-Drive.
 606                  * Reported by: Pawel Jakub Dawidek <pjd@FreeBSD.org>
 607                  */
 608                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "Kingston" , "DataTraveler II+",
 609                 "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
 610         },
 611         {
 612                 /*
 613                  * USB DISK Pro PMAP
 614                  * Reported by: jhs
 615                  * PR: usb/96381
 616                  */
 617                 {T_DIRECT, SIP_MEDIA_REMOVABLE, " ", "USB DISK Pro", "PMAP"},
 618                 /*quirks*/ DA_Q_NO_SYNC_CACHE
 619         },
 620         {
 621                 /*
 622                  * Motorola E398 Mobile Phone (TransFlash memory card).
 623                  * Reported by: Wojciech A. Koszek <dunstan@FreeBSD.czest.pl>
 624                  * PR: usb/89889
 625                  */
 626                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "Motorola" , "Motorola Phone",
 627                 "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
 628         },
 629         {
 630                 /*
 631                  * Qware BeatZkey! Pro
 632                  * PR: usb/79164
 633                  */
 634                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "GENERIC", "USB DISK DEVICE",
 635                 "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
 636         },
 637         {
 638                 /*
 639                  * Time DPA20B 1GB MP3 Player
 640                  * PR: usb/81846
 641                  */
 642                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "USB2.0*", "(FS) FLASH DISK*",
 643                 "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
 644         },
 645         {
 646                 /*
 647                  * Samsung USB key 128Mb
 648                  * PR: usb/90081
 649                  */
 650                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "USB-DISK", "FreeDik-FlashUsb",
 651                 "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
 652         },
 653         {
 654                 /*
 655                  * Kingston DataTraveler 2.0 USB Flash memory.
 656                  * PR: usb/89196
 657                  */
 658                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "Kingston", "DataTraveler 2.0",
 659                 "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
 660         },
 661         {
 662                 /*
 663                  * Creative MUVO Slim mp3 player (USB)
 664                  * PR: usb/86131
 665                  */
 666                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "CREATIVE", "MuVo Slim",
 667                 "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE|DA_Q_NO_PREVENT
 668                 },
 669         {
 670                 /*
 671                  * United MP5512 Portable MP3 Player (2-in-1 USB DISK/MP3)
 672                  * PR: usb/80487
 673                  */
 674                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "Generic*", "MUSIC DISK",
 675                 "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
 676         },
 677         {
 678                 /*
 679                  * SanDisk Micro Cruzer 128MB
 680                  * PR: usb/75970
 681                  */
 682                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "SanDisk" , "Micro Cruzer",
 683                 "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
 684         },
 685         {
 686                 /*
 687                  * TOSHIBA TransMemory USB sticks
 688                  * PR: kern/94660
 689                  */
 690                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "TOSHIBA", "TransMemory",
 691                 "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
 692         },
 693         {
 694                 /*
 695                  * PNY USB 3.0 Flash Drives
 696                 */
 697                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "PNY", "USB 3.0 FD*",
 698                 "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE | DA_Q_NO_RC16
 699         },
 700         {
 701                 /*
 702                  * PNY USB Flash keys
 703                  * PR: usb/75578, usb/72344, usb/65436
 704                  */
 705                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "*" , "USB DISK*",
 706                 "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
 707         },
 708         {
 709                 /*
 710                  * Genesys GL3224
 711                  */
 712                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "Generic*", "STORAGE DEVICE*",
 713                 "120?"}, /*quirks*/ DA_Q_NO_SYNC_CACHE | DA_Q_4K | DA_Q_NO_RC16
 714         },
 715         {
 716                 /*
 717                  * Genesys 6-in-1 Card Reader
 718                  * PR: usb/94647
 719                  */
 720                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "Generic*", "STORAGE DEVICE*",
 721                 "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
 722         },
 723         {
 724                 /*
 725                  * Rekam Digital CAMERA
 726                  * PR: usb/98713
 727                  */
 728                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "CAMERA*", "4MP-9J6*",
 729                 "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
 730         },
 731         {
 732                 /*
 733                  * iRiver H10 MP3 player
 734                  * PR: usb/102547
 735                  */
 736                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "iriver", "H10*",
 737                 "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
 738         },
 739         {
 740                 /*
 741                  * iRiver U10 MP3 player
 742                  * PR: usb/92306
 743                  */
 744                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "iriver", "U10*",
 745                 "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
 746         },
 747         {
 748                 /*
 749                  * X-Micro Flash Disk
 750                  * PR: usb/96901
 751                  */
 752                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "X-Micro", "Flash Disk",
 753                 "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
 754         },
 755         {
 756                 /*
 757                  * EasyMP3 EM732X USB 2.0 Flash MP3 Player
 758                  * PR: usb/96546
 759                  */
 760                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "EM732X", "MP3 Player*",
 761                 "1.00"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
 762         },
 763         {
 764                 /*
 765                  * Denver MP3 player
 766                  * PR: usb/107101
 767                  */
 768                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "DENVER", "MP3 PLAYER",
 769                  "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
 770         },
 771         {
 772                 /*
 773                  * Philips USB Key Audio KEY013
 774                  * PR: usb/68412
 775                  */
 776                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "PHILIPS", "Key*", "*"},
 777                 /*quirks*/ DA_Q_NO_SYNC_CACHE | DA_Q_NO_PREVENT
 778         },
 779         {
 780                 /*
 781                  * JNC MP3 Player
 782                  * PR: usb/94439
 783                  */
 784                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "JNC*" , "MP3 Player*",
 785                  "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
 786         },
 787         {
 788                 /*
 789                  * SAMSUNG MP0402H
 790                  * PR: usb/108427
 791                  */
 792                 {T_DIRECT, SIP_MEDIA_FIXED, "SAMSUNG", "MP0402H", "*"},
 793                 /*quirks*/ DA_Q_NO_SYNC_CACHE
 794         },
 795         {
 796                 /*
 797                  * I/O Magic USB flash - Giga Bank
 798                  * PR: usb/108810
 799                  */
 800                 {T_DIRECT, SIP_MEDIA_FIXED, "GS-Magic", "stor*", "*"},
 801                 /*quirks*/ DA_Q_NO_SYNC_CACHE
 802         },
 803         {
 804                 /*
 805                  * JoyFly 128mb USB Flash Drive
 806                  * PR: 96133
 807                  */
 808                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "USB 2.0", "Flash Disk*",
 809                  "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
 810         },
 811         {
 812                 /*
 813                  * ChipsBnk usb stick
 814                  * PR: 103702
 815                  */
 816                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "ChipsBnk", "USB*",
 817                  "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
 818         },
 819         {
 820                 /*
 821                  * Storcase (Kingston) InfoStation IFS FC2/SATA-R 201A
 822                  * PR: 129858
 823                  */
 824                 {T_DIRECT, SIP_MEDIA_FIXED, "IFS", "FC2/SATA-R*",
 825                  "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
 826         },
 827         {
 828                 /*
 829                  * Samsung YP-U3 mp3-player
 830                  * PR: 125398
 831                  */
 832                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "Samsung", "YP-U3",
 833                  "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
 834         },
 835         {
 836                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "Netac", "OnlyDisk*",
 837                  "2000"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
 838         },
 839         {
 840                 /*
 841                  * Sony Cyber-Shot DSC cameras
 842                  * PR: usb/137035
 843                  */
 844                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "Sony", "Sony DSC", "*"},
 845                 /*quirks*/ DA_Q_NO_SYNC_CACHE | DA_Q_NO_PREVENT
 846         },
 847         {
 848                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "Kingston", "DataTraveler G3",
 849                  "1.00"}, /*quirks*/ DA_Q_NO_PREVENT
 850         },
 851         {
 852                 /* At least several Transcent USB sticks lie on RC16. */
 853                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "JetFlash", "Transcend*",
 854                  "*"}, /*quirks*/ DA_Q_NO_RC16
 855         },
 856         {
 857                 /*
 858                  * I-O Data USB Flash Disk
 859                  * PR: usb/211716
 860                  */
 861                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "I-O DATA", "USB Flash Disk*",
 862                  "*"}, /*quirks*/ DA_Q_NO_RC16
 863         },
 864         {
 865                 /*
 866                  * 16GB SLC CHIPFANCIER
 867                  * PR: usb/234503
 868                  */
 869                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "16G SLC", "CHIPFANCIER",
 870                  "1.00"}, /*quirks*/ DA_Q_NO_RC16
 871        },
 872         /* ATA/SATA devices over SAS/USB/... */
 873         {
 874                 /* Hitachi Advanced Format (4k) drives */
 875                 { T_DIRECT, SIP_MEDIA_FIXED, "Hitachi", "H??????????E3*", "*" },
 876                 /*quirks*/DA_Q_4K
 877         },
 878         {
 879                 /* Micron Advanced Format (4k) drives */
 880                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "Micron 5100 MTFDDAK*", "*" },
 881                 /*quirks*/DA_Q_4K
 882         },
 883         {
 884                 /* Samsung Advanced Format (4k) drives */
 885                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "SAMSUNG HD155UI*", "*" },
 886                 /*quirks*/DA_Q_4K
 887         },
 888         {
 889                 /* Samsung Advanced Format (4k) drives */
 890                 { T_DIRECT, SIP_MEDIA_FIXED, "SAMSUNG", "HD155UI*", "*" },
 891                 /*quirks*/DA_Q_4K
 892         },
 893         {
 894                 /* Samsung Advanced Format (4k) drives */
 895                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "SAMSUNG HD204UI*", "*" },
 896                 /*quirks*/DA_Q_4K
 897         },
 898         {
 899                 /* Samsung Advanced Format (4k) drives */
 900                 { T_DIRECT, SIP_MEDIA_FIXED, "SAMSUNG", "HD204UI*", "*" },
 901                 /*quirks*/DA_Q_4K
 902         },
 903         {
 904                 /* Seagate Barracuda Green Advanced Format (4k) drives */
 905                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "ST????DL*", "*" },
 906                 /*quirks*/DA_Q_4K
 907         },
 908         {
 909                 /* Seagate Barracuda Green Advanced Format (4k) drives */
 910                 { T_DIRECT, SIP_MEDIA_FIXED, "ST????DL", "*", "*" },
 911                 /*quirks*/DA_Q_4K
 912         },
 913         {
 914                 /* Seagate Barracuda Green Advanced Format (4k) drives */
 915                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "ST???DM*", "*" },
 916                 /*quirks*/DA_Q_4K
 917         },
 918         {
 919                 /* Seagate Barracuda Green Advanced Format (4k) drives */
 920                 { T_DIRECT, SIP_MEDIA_FIXED, "ST???DM*", "*", "*" },
 921                 /*quirks*/DA_Q_4K
 922         },
 923         {
 924                 /* Seagate Barracuda Green Advanced Format (4k) drives */
 925                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "ST????DM*", "*" },
 926                 /*quirks*/DA_Q_4K
 927         },
 928         {
 929                 /* Seagate Barracuda Green Advanced Format (4k) drives */
 930                 { T_DIRECT, SIP_MEDIA_FIXED, "ST????DM", "*", "*" },
 931                 /*quirks*/DA_Q_4K
 932         },
 933         {
 934                 /* Seagate Momentus Advanced Format (4k) drives */
 935                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "ST9500423AS*", "*" },
 936                 /*quirks*/DA_Q_4K
 937         },
 938         {
 939                 /* Seagate Momentus Advanced Format (4k) drives */
 940                 { T_DIRECT, SIP_MEDIA_FIXED, "ST950042", "3AS*", "*" },
 941                 /*quirks*/DA_Q_4K
 942         },
 943         {
 944                 /* Seagate Momentus Advanced Format (4k) drives */
 945                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "ST9500424AS*", "*" },
 946                 /*quirks*/DA_Q_4K
 947         },
 948         {
 949                 /* Seagate Momentus Advanced Format (4k) drives */
 950                 { T_DIRECT, SIP_MEDIA_FIXED, "ST950042", "4AS*", "*" },
 951                 /*quirks*/DA_Q_4K
 952         },
 953         {
 954                 /* Seagate Momentus Advanced Format (4k) drives */
 955                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "ST9640423AS*", "*" },
 956                 /*quirks*/DA_Q_4K
 957         },
 958         {
 959                 /* Seagate Momentus Advanced Format (4k) drives */
 960                 { T_DIRECT, SIP_MEDIA_FIXED, "ST964042", "3AS*", "*" },
 961                 /*quirks*/DA_Q_4K
 962         },
 963         {
 964                 /* Seagate Momentus Advanced Format (4k) drives */
 965                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "ST9640424AS*", "*" },
 966                 /*quirks*/DA_Q_4K
 967         },
 968         {
 969                 /* Seagate Momentus Advanced Format (4k) drives */
 970                 { T_DIRECT, SIP_MEDIA_FIXED, "ST964042", "4AS*", "*" },
 971                 /*quirks*/DA_Q_4K
 972         },
 973         {
 974                 /* Seagate Momentus Advanced Format (4k) drives */
 975                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "ST9750420AS*", "*" },
 976                 /*quirks*/DA_Q_4K
 977         },
 978         {
 979                 /* Seagate Momentus Advanced Format (4k) drives */
 980                 { T_DIRECT, SIP_MEDIA_FIXED, "ST975042", "0AS*", "*" },
 981                 /*quirks*/DA_Q_4K
 982         },
 983         {
 984                 /* Seagate Momentus Advanced Format (4k) drives */
 985                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "ST9750422AS*", "*" },
 986                 /*quirks*/DA_Q_4K
 987         },
 988         {
 989                 /* Seagate Momentus Advanced Format (4k) drives */
 990                 { T_DIRECT, SIP_MEDIA_FIXED, "ST975042", "2AS*", "*" },
 991                 /*quirks*/DA_Q_4K
 992         },
 993         {
 994                 /* Seagate Momentus Advanced Format (4k) drives */
 995                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "ST9750423AS*", "*" },
 996                 /*quirks*/DA_Q_4K
 997         },
 998         {
 999                 /* Seagate Momentus Advanced Format (4k) drives */
1000                 { T_DIRECT, SIP_MEDIA_FIXED, "ST975042", "3AS*", "*" },
1001                 /*quirks*/DA_Q_4K
1002         },
1003         {
1004                 /* Seagate Momentus Thin Advanced Format (4k) drives */
1005                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "ST???LT*", "*" },
1006                 /*quirks*/DA_Q_4K
1007         },
1008         {
1009                 /* Seagate Momentus Thin Advanced Format (4k) drives */
1010                 { T_DIRECT, SIP_MEDIA_FIXED, "ST???LT*", "*", "*" },
1011                 /*quirks*/DA_Q_4K
1012         },
1013         {
1014                 /* WDC Caviar Green Advanced Format (4k) drives */
1015                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "WDC WD????RS*", "*" },
1016                 /*quirks*/DA_Q_4K
1017         },
1018         {
1019                 /* WDC Caviar Green Advanced Format (4k) drives */
1020                 { T_DIRECT, SIP_MEDIA_FIXED, "WDC WD??", "??RS*", "*" },
1021                 /*quirks*/DA_Q_4K
1022         },
1023         {
1024                 /* WDC Caviar Green Advanced Format (4k) drives */
1025                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "WDC WD????RX*", "*" },
1026                 /*quirks*/DA_Q_4K
1027         },
1028         {
1029                 /* WDC Caviar Green Advanced Format (4k) drives */
1030                 { T_DIRECT, SIP_MEDIA_FIXED, "WDC WD??", "??RX*", "*" },
1031                 /*quirks*/DA_Q_4K
1032         },
1033         {
1034                 /* WDC Caviar Green Advanced Format (4k) drives */
1035                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "WDC WD??????RS*", "*" },
1036                 /*quirks*/DA_Q_4K
1037         },
1038         {
1039                 /* WDC Caviar Green Advanced Format (4k) drives */
1040                 { T_DIRECT, SIP_MEDIA_FIXED, "WDC WD??", "????RS*", "*" },
1041                 /*quirks*/DA_Q_4K
1042         },
1043         {
1044                 /* WDC Caviar Green Advanced Format (4k) drives */
1045                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "WDC WD??????RX*", "*" },
1046                 /*quirks*/DA_Q_4K
1047         },
1048         {
1049                 /* WDC Caviar Green Advanced Format (4k) drives */
1050                 { T_DIRECT, SIP_MEDIA_FIXED, "WDC WD??", "????RX*", "*" },
1051                 /*quirks*/DA_Q_4K
1052         },
1053         {
1054                 /* WDC Scorpio Black Advanced Format (4k) drives */
1055                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "WDC WD???PKT*", "*" },
1056                 /*quirks*/DA_Q_4K
1057         },
1058         {
1059                 /* WDC Scorpio Black Advanced Format (4k) drives */
1060                 { T_DIRECT, SIP_MEDIA_FIXED, "WDC WD??", "?PKT*", "*" },
1061                 /*quirks*/DA_Q_4K
1062         },
1063         {
1064                 /* WDC Scorpio Black Advanced Format (4k) drives */
1065                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "WDC WD?????PKT*", "*" },
1066                 /*quirks*/DA_Q_4K
1067         },
1068         {
1069                 /* WDC Scorpio Black Advanced Format (4k) drives */
1070                 { T_DIRECT, SIP_MEDIA_FIXED, "WDC WD??", "???PKT*", "*" },
1071                 /*quirks*/DA_Q_4K
1072         },
1073         {
1074                 /* WDC Scorpio Blue Advanced Format (4k) drives */
1075                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "WDC WD???PVT*", "*" },
1076                 /*quirks*/DA_Q_4K
1077         },
1078         {
1079                 /* WDC Scorpio Blue Advanced Format (4k) drives */
1080                 { T_DIRECT, SIP_MEDIA_FIXED, "WDC WD??", "?PVT*", "*" },
1081                 /*quirks*/DA_Q_4K
1082         },
1083         {
1084                 /* WDC Scorpio Blue Advanced Format (4k) drives */
1085                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "WDC WD?????PVT*", "*" },
1086                 /*quirks*/DA_Q_4K
1087         },
1088         {
1089                 /* WDC Scorpio Blue Advanced Format (4k) drives */
1090                 { T_DIRECT, SIP_MEDIA_FIXED, "WDC WD??", "???PVT*", "*" },
1091                 /*quirks*/DA_Q_4K
1092         },
1093         {
1094                 /*
1095                  * Olympus FE-210 camera
1096                  */
1097                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "OLYMPUS", "FE210*",
1098                 "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
1099         },
1100         {
1101                 /*
1102                 * Pentax Digital Camera
1103                 * PR: usb/93389
1104                 */
1105                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "PENTAX", "DIGITAL CAMERA",
1106                 "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
1107         },
1108         {
1109                 /*
1110                  * LG UP3S MP3 player
1111                  */
1112                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "LG", "UP3S",
1113                 "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
1114         },
1115         {
1116                 /*
1117                  * Laser MP3-2GA13 MP3 player
1118                  */
1119                 {T_DIRECT, SIP_MEDIA_REMOVABLE, "USB 2.0", "(HS) Flash Disk",
1120                 "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
1121         },
1122         {
1123                 /*
1124                  * LaCie external 250GB Hard drive des by Porsche
1125                  * Submitted by: Ben Stuyts <ben@altesco.nl>
1126                  * PR: 121474
1127                  */
1128                 {T_DIRECT, SIP_MEDIA_FIXED, "SAMSUNG", "HM250JI", "*"},
1129                 /*quirks*/ DA_Q_NO_SYNC_CACHE
1130         },
1131         /* SATA SSDs */
1132         {
1133                 /*
1134                  * Corsair Force 2 SSDs
1135                  * 4k optimised & trim only works in 4k requests + 4k aligned
1136                  */
1137                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "Corsair CSSD-F*", "*" },
1138                 /*quirks*/DA_Q_4K
1139         },
1140         {
1141                 /*
1142                  * Corsair Force 3 SSDs
1143                  * 4k optimised & trim only works in 4k requests + 4k aligned
1144                  */
1145                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "Corsair Force 3*", "*" },
1146                 /*quirks*/DA_Q_4K
1147         },
1148         {
1149                 /*
1150                  * Corsair Neutron GTX SSDs
1151                  * 4k optimised & trim only works in 4k requests + 4k aligned
1152                  */
1153                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair Neutron GTX*", "*" },
1154                 /*quirks*/DA_Q_4K
1155         },
1156         {
1157                 /*
1158                  * Corsair Force GT & GS SSDs
1159                  * 4k optimised & trim only works in 4k requests + 4k aligned
1160                  */
1161                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "Corsair Force G*", "*" },
1162                 /*quirks*/DA_Q_4K
1163         },
1164         {
1165                 /*
1166                  * Crucial M4 SSDs
1167                  * 4k optimised & trim only works in 4k requests + 4k aligned
1168                  */
1169                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "M4-CT???M4SSD2*", "*" },
1170                 /*quirks*/DA_Q_4K
1171         },
1172         {
1173                 /*
1174                  * Crucial RealSSD C300 SSDs
1175                  * 4k optimised
1176                  */
1177                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "C300-CTFDDAC???MAG*",
1178                 "*" }, /*quirks*/DA_Q_4K
1179         },
1180         {
1181                 /*
1182                  * Intel 320 Series SSDs
1183                  * 4k optimised & trim only works in 4k requests + 4k aligned
1184                  */
1185                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "INTEL SSDSA2CW*", "*" },
1186                 /*quirks*/DA_Q_4K
1187         },
1188         {
1189                 /*
1190                  * Intel 330 Series SSDs
1191                  * 4k optimised & trim only works in 4k requests + 4k aligned
1192                  */
1193                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "INTEL SSDSC2CT*", "*" },
1194                 /*quirks*/DA_Q_4K
1195         },
1196         {
1197                 /*
1198                  * Intel 510 Series SSDs
1199                  * 4k optimised & trim only works in 4k requests + 4k aligned
1200                  */
1201                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "INTEL SSDSC2MH*", "*" },
1202                 /*quirks*/DA_Q_4K
1203         },
1204         {
1205                 /*
1206                  * Intel 520 Series SSDs
1207                  * 4k optimised & trim only works in 4k requests + 4k aligned
1208                  */
1209                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "INTEL SSDSC2BW*", "*" },
1210                 /*quirks*/DA_Q_4K
1211         },
1212         {
1213                 /*
1214                  * Intel S3610 Series SSDs
1215                  * 4k optimised & trim only works in 4k requests + 4k aligned
1216                  */
1217                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "INTEL SSDSC2BX*", "*" },
1218                 /*quirks*/DA_Q_4K
1219         },
1220         {
1221                 /*
1222                  * Intel X25-M Series SSDs
1223                  * 4k optimised & trim only works in 4k requests + 4k aligned
1224                  */
1225                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "INTEL SSDSA2M*", "*" },
1226                 /*quirks*/DA_Q_4K
1227         },
1228         {
1229                 /*
1230                  * Kingston E100 Series SSDs
1231                  * 4k optimised & trim only works in 4k requests + 4k aligned
1232                  */
1233                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "KINGSTON SE100S3*", "*" },
1234                 /*quirks*/DA_Q_4K
1235         },
1236         {
1237                 /*
1238                  * Kingston HyperX 3k SSDs
1239                  * 4k optimised & trim only works in 4k requests + 4k aligned
1240                  */
1241                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "KINGSTON SH103S3*", "*" },
1242                 /*quirks*/DA_Q_4K
1243         },
1244         {
1245                 /*
1246                  * Marvell SSDs (entry taken from OpenSolaris)
1247                  * 4k optimised & trim only works in 4k requests + 4k aligned
1248                  */
1249                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "MARVELL SD88SA02*", "*" },
1250                 /*quirks*/DA_Q_4K
1251         },
1252         {
1253                 /*
1254                  * OCZ Agility 2 SSDs
1255                  * 4k optimised & trim only works in 4k requests + 4k aligned
1256                  */
1257                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-AGILITY2*", "*" },
1258                 /*quirks*/DA_Q_4K
1259         },
1260         {
1261                 /*
1262                  * OCZ Agility 3 SSDs
1263                  * 4k optimised & trim only works in 4k requests + 4k aligned
1264                  */
1265                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "OCZ-AGILITY3*", "*" },
1266                 /*quirks*/DA_Q_4K
1267         },
1268         {
1269                 /*
1270                  * OCZ Deneva R Series SSDs
1271                  * 4k optimised & trim only works in 4k requests + 4k aligned
1272                  */
1273                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "DENRSTE251M45*", "*" },
1274                 /*quirks*/DA_Q_4K
1275         },
1276         {
1277                 /*
1278                  * OCZ Vertex 2 SSDs (inc pro series)
1279                  * 4k optimised & trim only works in 4k requests + 4k aligned
1280                  */
1281                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "OCZ?VERTEX2*", "*" },
1282                 /*quirks*/DA_Q_4K
1283         },
1284         {
1285                 /*
1286                  * OCZ Vertex 3 SSDs
1287                  * 4k optimised & trim only works in 4k requests + 4k aligned
1288                  */
1289                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "OCZ-VERTEX3*", "*" },
1290                 /*quirks*/DA_Q_4K
1291         },
1292         {
1293                 /*
1294                  * OCZ Vertex 4 SSDs
1295                  * 4k optimised & trim only works in 4k requests + 4k aligned
1296                  */
1297                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "OCZ-VERTEX4*", "*" },
1298                 /*quirks*/DA_Q_4K
1299         },
1300         {
1301                 /*
1302                  * Samsung 750 Series SSDs
1303                  * 4k optimised & trim only works in 4k requests + 4k aligned
1304                  */
1305                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "Samsung SSD 750*", "*" },
1306                 /*quirks*/DA_Q_4K
1307         },
1308         {
1309                 /*
1310                  * Samsung 830 Series SSDs
1311                  * 4k optimised & trim only works in 4k requests + 4k aligned
1312                  */
1313                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "SAMSUNG SSD 830 Series*", "*" },
1314                 /*quirks*/DA_Q_4K
1315         },
1316         {
1317                 /*
1318                  * Samsung 840 SSDs
1319                  * 4k optimised & trim only works in 4k requests + 4k aligned
1320                  */
1321                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "Samsung SSD 840*", "*" },
1322                 /*quirks*/DA_Q_4K
1323         },
1324         {
1325                 /*
1326                  * Samsung 845 SSDs
1327                  * 4k optimised & trim only works in 4k requests + 4k aligned
1328                  */
1329                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "Samsung SSD 845*", "*" },
1330                 /*quirks*/DA_Q_4K
1331         },
1332         {
1333                 /*
1334                  * Samsung 850 SSDs
1335                  * 4k optimised & trim only works in 4k requests + 4k aligned
1336                  */
1337                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "Samsung SSD 850*", "*" },
1338                 /*quirks*/DA_Q_4K
1339         },
1340         {
1341                 /*
1342                  * Samsung 843T Series SSDs (MZ7WD*)
1343                  * Samsung PM851 Series SSDs (MZ7TE*)
1344                  * Samsung PM853T Series SSDs (MZ7GE*)
1345                  * Samsung SM863 Series SSDs (MZ7KM*)
1346                  * 4k optimised
1347                  */
1348                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "SAMSUNG MZ7*", "*" },
1349                 /*quirks*/DA_Q_4K
1350         },
1351         {
1352                 /*
1353                  * Same as for SAMSUNG MZ7* but enable the quirks for SSD
1354                  * starting with MZ7* too
1355                  */
1356                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "MZ7*", "*" },
1357                 /*quirks*/DA_Q_4K
1358         },
1359         {
1360                 /*
1361                  * SuperTalent TeraDrive CT SSDs
1362                  * 4k optimised & trim only works in 4k requests + 4k aligned
1363                  */
1364                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "FTM??CT25H*", "*" },
1365                 /*quirks*/DA_Q_4K
1366         },
1367         {
1368                 /*
1369                  * XceedIOPS SATA SSDs
1370                  * 4k optimised
1371                  */
1372                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "SG9XCS2D*", "*" },
1373                 /*quirks*/DA_Q_4K
1374         },
1375         {
1376                 /*
1377                  * Hama Innostor USB-Stick
1378                  */
1379                 { T_DIRECT, SIP_MEDIA_REMOVABLE, "Innostor", "Innostor*", "*" },
1380                 /*quirks*/DA_Q_NO_RC16
1381         },
1382         {
1383                 /*
1384                  * Seagate Lamarr 8TB Shingled Magnetic Recording (SMR)
1385                  * Drive Managed SATA hard drive.  This drive doesn't report
1386                  * in firmware that it is a drive managed SMR drive.
1387                  */
1388                 { T_DIRECT, SIP_MEDIA_FIXED, "ATA", "ST8000AS000[23]*", "*" },
1389                 /*quirks*/DA_Q_SMR_DM
1390         },
1391         {
1392                 /*
1393                  * MX-ES USB Drive by Mach Xtreme
1394                  */
1395                 { T_DIRECT, SIP_MEDIA_REMOVABLE, "MX", "MXUB3*", "*"},
1396                 /*quirks*/DA_Q_NO_RC16
1397         },
1398 };
1399
1400 static  disk_strategy_t dastrategy;
1401 static  dumper_t        dadump;
1402 static  periph_init_t   dainit;
1403 static  void            daasync(void *callback_arg, u_int32_t code,
1404                                 struct cam_path *path, void *arg);
1405 static  void            dasysctlinit(void *context, int pending);
1406 static  int             dasysctlsofttimeout(SYSCTL_HANDLER_ARGS);
1407 static  int             dacmdsizesysctl(SYSCTL_HANDLER_ARGS);
1408 static  int             dadeletemethodsysctl(SYSCTL_HANDLER_ARGS);
1409 static  int             dazonemodesysctl(SYSCTL_HANDLER_ARGS);
1410 static  int             dazonesupsysctl(SYSCTL_HANDLER_ARGS);
1411 static  int             dadeletemaxsysctl(SYSCTL_HANDLER_ARGS);
1412 static  void            dadeletemethodset(struct da_softc *softc,
1413                                           da_delete_methods delete_method);
1414 static  off_t           dadeletemaxsize(struct da_softc *softc,
1415                                         da_delete_methods delete_method);
1416 static  void            dadeletemethodchoose(struct da_softc *softc,
1417                                              da_delete_methods default_method);
1418 static  void            daprobedone(struct cam_periph *periph, union ccb *ccb);
1419
1420 static  periph_ctor_t   daregister;
1421 static  periph_dtor_t   dacleanup;
1422 static  periph_start_t  dastart;
1423 static  periph_oninv_t  daoninvalidate;
1424 static  void            dazonedone(struct cam_periph *periph, union ccb *ccb);
1425 static  void            dadone(struct cam_periph *periph,
1426                                union ccb *done_ccb);
1427 static void             dadone_probewp(struct cam_periph *periph,
1428                                        union ccb *done_ccb);
1429 static void             dadone_proberc(struct cam_periph *periph,
1430                                        union ccb *done_ccb);
1431 static void             dadone_probelbp(struct cam_periph *periph,
1432                                         union ccb *done_ccb);
1433 static void             dadone_probeblklimits(struct cam_periph *periph,
1434                                               union ccb *done_ccb);
1435 static void             dadone_probebdc(struct cam_periph *periph,
1436                                         union ccb *done_ccb);
1437 static void             dadone_probeata(struct cam_periph *periph,
1438                                         union ccb *done_ccb);
1439 static void             dadone_probeatalogdir(struct cam_periph *periph,
1440                                               union ccb *done_ccb);
1441 static void             dadone_probeataiddir(struct cam_periph *periph,
1442                                              union ccb *done_ccb);
1443 static void             dadone_probeatasup(struct cam_periph *periph,
1444                                            union ccb *done_ccb);
1445 static void             dadone_probeatazone(struct cam_periph *periph,
1446                                             union ccb *done_ccb);
1447 static void             dadone_probezone(struct cam_periph *periph,
1448                                          union ccb *done_ccb);
1449 static void             dadone_tur(struct cam_periph *periph,
1450                                    union ccb *done_ccb);
1451 static  int             daerror(union ccb *ccb, u_int32_t cam_flags,
1452                                 u_int32_t sense_flags);
1453 static void             daprevent(struct cam_periph *periph, int action);
1454 static void             dareprobe(struct cam_periph *periph);
1455 static void             dasetgeom(struct cam_periph *periph, uint32_t block_len,
1456                                   uint64_t maxsector,
1457                                   struct scsi_read_capacity_data_long *rcaplong,
1458                                   size_t rcap_size);
1459 static timeout_t        dasendorderedtag;
1460 static void             dashutdown(void *arg, int howto);
1461 static timeout_t        damediapoll;
1462
1463 #ifndef DA_DEFAULT_POLL_PERIOD
1464 #define DA_DEFAULT_POLL_PERIOD  3
1465 #endif
1466
1467 #ifndef DA_DEFAULT_TIMEOUT
1468 #define DA_DEFAULT_TIMEOUT 60   /* Timeout in seconds */
1469 #endif
1470
1471 #ifndef DA_DEFAULT_SOFTTIMEOUT
1472 #define DA_DEFAULT_SOFTTIMEOUT  0
1473 #endif
1474
1475 #ifndef DA_DEFAULT_RETRY
1476 #define DA_DEFAULT_RETRY        4
1477 #endif
1478
1479 #ifndef DA_DEFAULT_SEND_ORDERED
1480 #define DA_DEFAULT_SEND_ORDERED 1
1481 #endif
1482
1483 static int da_poll_period = DA_DEFAULT_POLL_PERIOD;
1484 static int da_retry_count = DA_DEFAULT_RETRY;
1485 static int da_default_timeout = DA_DEFAULT_TIMEOUT;
1486 static sbintime_t da_default_softtimeout = DA_DEFAULT_SOFTTIMEOUT;
1487 static int da_send_ordered = DA_DEFAULT_SEND_ORDERED;
1488 static int da_disable_wp_detection = 0;
1489
1490 static SYSCTL_NODE(_kern_cam, OID_AUTO, da, CTLFLAG_RD, 0,
1491             "CAM Direct Access Disk driver");
1492 SYSCTL_INT(_kern_cam_da, OID_AUTO, poll_period, CTLFLAG_RWTUN,
1493            &da_poll_period, 0, "Media polling period in seconds");
1494 SYSCTL_INT(_kern_cam_da, OID_AUTO, retry_count, CTLFLAG_RWTUN,
1495            &da_retry_count, 0, "Normal I/O retry count");
1496 SYSCTL_INT(_kern_cam_da, OID_AUTO, default_timeout, CTLFLAG_RWTUN,
1497            &da_default_timeout, 0, "Normal I/O timeout (in seconds)");
1498 SYSCTL_INT(_kern_cam_da, OID_AUTO, send_ordered, CTLFLAG_RWTUN,
1499            &da_send_ordered, 0, "Send Ordered Tags");
1500 SYSCTL_INT(_kern_cam_da, OID_AUTO, disable_wp_detection, CTLFLAG_RWTUN,
1501            &da_disable_wp_detection, 0,
1502            "Disable detection of write-protected disks");
1503
1504 SYSCTL_PROC(_kern_cam_da, OID_AUTO, default_softtimeout,
1505     CTLTYPE_UINT | CTLFLAG_RW, NULL, 0, dasysctlsofttimeout, "I",
1506     "Soft I/O timeout (ms)");
1507 TUNABLE_INT64("kern.cam.da.default_softtimeout", &da_default_softtimeout);
1508
1509 /*
1510  * DA_ORDEREDTAG_INTERVAL determines how often, relative
1511  * to the default timeout, we check to see whether an ordered
1512  * tagged transaction is appropriate to prevent simple tag
1513  * starvation.  Since we'd like to ensure that there is at least
1514  * 1/2 of the timeout length left for a starved transaction to
1515  * complete after we've sent an ordered tag, we must poll at least
1516  * four times in every timeout period.  This takes care of the worst
1517  * case where a starved transaction starts during an interval that
1518  * meets the requirement "don't send an ordered tag" test so it takes
1519  * us two intervals to determine that a tag must be sent.
1520  */
1521 #ifndef DA_ORDEREDTAG_INTERVAL
1522 #define DA_ORDEREDTAG_INTERVAL 4
1523 #endif
1524
1525 static struct periph_driver dadriver =
1526 {
1527         dainit, "da",
1528         TAILQ_HEAD_INITIALIZER(dadriver.units), /* generation */ 0
1529 };
1530
1531 PERIPHDRIVER_DECLARE(da, dadriver);
1532
1533 static MALLOC_DEFINE(M_SCSIDA, "scsi_da", "scsi_da buffers");
1534
1535 /*
1536  * This driver takes out references / holds in well defined pairs, never
1537  * recursively. These macros / inline functions enforce those rules. They
1538  * are only enabled with DA_TRACK_REFS or INVARIANTS. If DA_TRACK_REFS is
1539  * defined to be 2 or larger, the tracking also includes debug printfs.
1540  */
1541 #if defined(DA_TRACK_REFS) || defined(INVARIANTS)
1542
1543 #ifndef DA_TRACK_REFS
1544 #define DA_TRACK_REFS 1
1545 #endif
1546
1547 #if DA_TRACK_REFS > 1
1548 static const char *da_ref_text[] = {
1549         "bogus",
1550         "open",
1551         "open hold",
1552         "close hold",
1553         "reprobe hold",
1554         "Test Unit Ready",
1555         "Geom",
1556         "sysctl",
1557         "reprobe",
1558         "max -- also bogus"
1559 };
1560
1561 #define DA_PERIPH_PRINT(periph, msg, args...)           \
1562         CAM_PERIPH_PRINT(periph, msg, ##args)
1563 #else
1564 #define DA_PERIPH_PRINT(periph, msg, args...)
1565 #endif
1566
1567 static inline void
1568 token_sanity(da_ref_token token)
1569 {
1570         if ((unsigned)token >= DA_REF_MAX)
1571                 panic("Bad token value passed in %d\n", token);
1572 }
1573
1574 static inline int
1575 da_periph_hold(struct cam_periph *periph, int priority, da_ref_token token)
1576 {
1577         int err = cam_periph_hold(periph, priority);
1578
1579         token_sanity(token);
1580         DA_PERIPH_PRINT(periph, "Holding device %s (%d): %d\n",
1581             da_ref_text[token], token, err);
1582         if (err == 0) {
1583                 int cnt;
1584                 struct da_softc *softc = periph->softc;
1585
1586                 cnt = atomic_fetchadd_int(&softc->ref_flags[token], 1);
1587                 if (cnt != 0)
1588                         panic("Re-holding for reason %d, cnt = %d", token, cnt);
1589         }
1590         return (err);
1591 }
1592
1593 static inline void
1594 da_periph_unhold(struct cam_periph *periph, da_ref_token token)
1595 {
1596         int cnt;
1597         struct da_softc *softc = periph->softc;
1598
1599         token_sanity(token);
1600         DA_PERIPH_PRINT(periph, "Unholding device %s (%d)\n",
1601             da_ref_text[token], token);
1602         cnt = atomic_fetchadd_int(&softc->ref_flags[token], -1);
1603         if (cnt != 1)
1604                 panic("Unholding %d with cnt = %d", token, cnt);
1605         cam_periph_unhold(periph);
1606 }
1607
1608 static inline int
1609 da_periph_acquire(struct cam_periph *periph, da_ref_token token)
1610 {
1611         int err = cam_periph_acquire(periph);
1612
1613         token_sanity(token);
1614         DA_PERIPH_PRINT(periph, "acquiring device %s (%d): %d\n",
1615             da_ref_text[token], token, err);
1616         if (err == 0) {
1617                 int cnt;
1618                 struct da_softc *softc = periph->softc;
1619
1620                 cnt = atomic_fetchadd_int(&softc->ref_flags[token], 1);
1621                 if (cnt != 0)
1622                         panic("Re-refing for reason %d, cnt = %d", token, cnt);
1623         }
1624         return (err);
1625 }
1626
1627 static inline void
1628 da_periph_release(struct cam_periph *periph, da_ref_token token)
1629 {
1630         int cnt;
1631         struct da_softc *softc = periph->softc;
1632
1633         token_sanity(token);
1634         DA_PERIPH_PRINT(periph, "releasing device %s (%d)\n",
1635             da_ref_text[token], token);
1636         cnt = atomic_fetchadd_int(&softc->ref_flags[token], -1);
1637         if (cnt != 1)
1638                 panic("Releasing %d with cnt = %d", token, cnt);
1639         cam_periph_release(periph);
1640 }
1641
1642 static inline void
1643 da_periph_release_locked(struct cam_periph *periph, da_ref_token token)
1644 {
1645         int cnt;
1646         struct da_softc *softc = periph->softc;
1647
1648         token_sanity(token);
1649         DA_PERIPH_PRINT(periph, "releasing device (locked) %s (%d)\n",
1650             da_ref_text[token], token);
1651         cnt = atomic_fetchadd_int(&softc->ref_flags[token], -1);
1652         if (cnt != 1)
1653                 panic("Unholding %d with cnt = %d", token, cnt);
1654         cam_periph_release_locked(periph);
1655 }
1656
1657 #define cam_periph_hold POISON
1658 #define cam_periph_unhold POISON
1659 #define cam_periph_acquire POISON
1660 #define cam_periph_release POISON
1661 #define cam_periph_release_locked POISON
1662
1663 #else
1664 #define da_periph_hold(periph, prio, token)     cam_periph_hold((periph), (prio))
1665 #define da_periph_unhold(periph, token)         cam_periph_unhold((periph))
1666 #define da_periph_acquire(periph, token)        cam_periph_acquire((periph))
1667 #define da_periph_release(periph, token)        cam_periph_release((periph))
1668 #define da_periph_release_locked(periph, token) cam_periph_release_locked((periph))
1669 #endif
1670
1671 static int
1672 daopen(struct disk *dp)
1673 {
1674         struct cam_periph *periph;
1675         struct da_softc *softc;
1676         int error;
1677
1678         periph = (struct cam_periph *)dp->d_drv1;
1679         if (da_periph_acquire(periph, DA_REF_OPEN) != 0) {
1680                 return (ENXIO);
1681         }
1682
1683         cam_periph_lock(periph);
1684         if ((error = da_periph_hold(periph, PRIBIO|PCATCH, DA_REF_OPEN_HOLD)) != 0) {
1685                 cam_periph_unlock(periph);
1686                 da_periph_release(periph, DA_REF_OPEN);
1687                 return (error);
1688         }
1689
1690         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE | CAM_DEBUG_PERIPH,
1691             ("daopen\n"));
1692
1693         softc = (struct da_softc *)periph->softc;
1694         dareprobe(periph);
1695
1696         /* Wait for the disk size update.  */
1697         error = cam_periph_sleep(periph, &softc->disk->d_mediasize, PRIBIO,
1698             "dareprobe", 0);
1699         if (error != 0)
1700                 xpt_print(periph->path, "unable to retrieve capacity data\n");
1701
1702         if (periph->flags & CAM_PERIPH_INVALID)
1703                 error = ENXIO;
1704
1705         if (error == 0 && (softc->flags & DA_FLAG_PACK_REMOVABLE) != 0 &&
1706             (softc->quirks & DA_Q_NO_PREVENT) == 0)
1707                 daprevent(periph, PR_PREVENT);
1708
1709         if (error == 0) {
1710                 softc->flags &= ~DA_FLAG_PACK_INVALID;
1711                 softc->flags |= DA_FLAG_OPEN;
1712         }
1713
1714         da_periph_unhold(periph, DA_REF_OPEN_HOLD);
1715         cam_periph_unlock(periph);
1716
1717         if (error != 0)
1718                 da_periph_release(periph, DA_REF_OPEN);
1719
1720         return (error);
1721 }
1722
1723 static int
1724 daclose(struct disk *dp)
1725 {
1726         struct  cam_periph *periph;
1727         struct  da_softc *softc;
1728         union   ccb *ccb;
1729
1730         periph = (struct cam_periph *)dp->d_drv1;
1731         softc = (struct da_softc *)periph->softc;
1732         cam_periph_lock(periph);
1733         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE | CAM_DEBUG_PERIPH,
1734             ("daclose\n"));
1735
1736         if (da_periph_hold(periph, PRIBIO, DA_REF_CLOSE_HOLD) == 0) {
1737
1738                 /* Flush disk cache. */
1739                 if ((softc->flags & DA_FLAG_DIRTY) != 0 &&
1740                     (softc->quirks & DA_Q_NO_SYNC_CACHE) == 0 &&
1741                     (softc->flags & DA_FLAG_PACK_INVALID) == 0) {
1742                         ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL);
1743                         scsi_synchronize_cache(&ccb->csio, /*retries*/1,
1744                             /*cbfcnp*/NULL, MSG_SIMPLE_Q_TAG,
1745                             /*begin_lba*/0, /*lb_count*/0, SSD_FULL_SIZE,
1746                             5 * 60 * 1000);
1747                         cam_periph_runccb(ccb, daerror, /*cam_flags*/0,
1748                             /*sense_flags*/SF_RETRY_UA | SF_QUIET_IR,
1749                             softc->disk->d_devstat);
1750                         softc->flags &= ~DA_FLAG_DIRTY;
1751                         xpt_release_ccb(ccb);
1752                 }
1753
1754                 /* Allow medium removal. */
1755                 if ((softc->flags & DA_FLAG_PACK_REMOVABLE) != 0 &&
1756                     (softc->quirks & DA_Q_NO_PREVENT) == 0)
1757                         daprevent(periph, PR_ALLOW);
1758
1759                 da_periph_unhold(periph, DA_REF_CLOSE_HOLD);
1760         }
1761
1762         /*
1763          * If we've got removeable media, mark the blocksize as
1764          * unavailable, since it could change when new media is
1765          * inserted.
1766          */
1767         if ((softc->flags & DA_FLAG_PACK_REMOVABLE) != 0)
1768                 softc->disk->d_devstat->flags |= DEVSTAT_BS_UNAVAILABLE;
1769
1770         softc->flags &= ~DA_FLAG_OPEN;
1771         while (softc->refcount != 0)
1772                 cam_periph_sleep(periph, &softc->refcount, PRIBIO, "daclose", 1);
1773         cam_periph_unlock(periph);
1774         da_periph_release(periph, DA_REF_OPEN);
1775         return (0);
1776 }
1777
1778 static void
1779 daschedule(struct cam_periph *periph)
1780 {
1781         struct da_softc *softc = (struct da_softc *)periph->softc;
1782
1783         if (softc->state != DA_STATE_NORMAL)
1784                 return;
1785
1786         cam_iosched_schedule(softc->cam_iosched, periph);
1787 }
1788
1789 /*
1790  * Actually translate the requested transfer into one the physical driver
1791  * can understand.  The transfer is described by a buf and will include
1792  * only one physical transfer.
1793  */
1794 static void
1795 dastrategy(struct bio *bp)
1796 {
1797         struct cam_periph *periph;
1798         struct da_softc *softc;
1799
1800         periph = (struct cam_periph *)bp->bio_disk->d_drv1;
1801         softc = (struct da_softc *)periph->softc;
1802
1803         cam_periph_lock(periph);
1804
1805         /*
1806          * If the device has been made invalid, error out
1807          */
1808         if ((softc->flags & DA_FLAG_PACK_INVALID)) {
1809                 cam_periph_unlock(periph);
1810                 biofinish(bp, NULL, ENXIO);
1811                 return;
1812         }
1813
1814         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dastrategy(%p)\n", bp));
1815
1816         /*
1817          * Zone commands must be ordered, because they can depend on the
1818          * effects of previously issued commands, and they may affect
1819          * commands after them.
1820          */
1821         if (bp->bio_cmd == BIO_ZONE)
1822                 bp->bio_flags |= BIO_ORDERED;
1823
1824         /*
1825          * Place it in the queue of disk activities for this disk
1826          */
1827         cam_iosched_queue_work(softc->cam_iosched, bp);
1828
1829         /*
1830          * Schedule ourselves for performing the work.
1831          */
1832         daschedule(periph);
1833         cam_periph_unlock(periph);
1834
1835         return;
1836 }
1837
1838 static int
1839 dadump(void *arg, void *virtual, vm_offset_t physical, off_t offset, size_t length)
1840 {
1841         struct      cam_periph *periph;
1842         struct      da_softc *softc;
1843         u_int       secsize;
1844         struct      ccb_scsiio csio;
1845         struct      disk *dp;
1846         int         error = 0;
1847
1848         dp = arg;
1849         periph = dp->d_drv1;
1850         softc = (struct da_softc *)periph->softc;
1851         secsize = softc->params.secsize;
1852
1853         if ((softc->flags & DA_FLAG_PACK_INVALID) != 0)
1854                 return (ENXIO);
1855
1856         memset(&csio, 0, sizeof(csio));
1857         if (length > 0) {
1858                 xpt_setup_ccb(&csio.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
1859                 csio.ccb_h.ccb_state = DA_CCB_DUMP;
1860                 scsi_read_write(&csio,
1861                                 /*retries*/0,
1862                                 /*cbfcnp*/NULL,
1863                                 MSG_ORDERED_Q_TAG,
1864                                 /*read*/SCSI_RW_WRITE,
1865                                 /*byte2*/0,
1866                                 /*minimum_cmd_size*/ softc->minimum_cmd_size,
1867                                 offset / secsize,
1868                                 length / secsize,
1869                                 /*data_ptr*/(u_int8_t *) virtual,
1870                                 /*dxfer_len*/length,
1871                                 /*sense_len*/SSD_FULL_SIZE,
1872                                 da_default_timeout * 1000);
1873                 error = cam_periph_runccb((union ccb *)&csio, cam_periph_error,
1874                     0, SF_NO_RECOVERY | SF_NO_RETRY, NULL);
1875                 if (error != 0)
1876                         printf("Aborting dump due to I/O error.\n");
1877                 return (error);
1878         }
1879
1880         /*
1881          * Sync the disk cache contents to the physical media.
1882          */
1883         if ((softc->quirks & DA_Q_NO_SYNC_CACHE) == 0) {
1884
1885                 xpt_setup_ccb(&csio.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
1886                 csio.ccb_h.ccb_state = DA_CCB_DUMP;
1887                 scsi_synchronize_cache(&csio,
1888                                        /*retries*/0,
1889                                        /*cbfcnp*/NULL,
1890                                        MSG_SIMPLE_Q_TAG,
1891                                        /*begin_lba*/0,/* Cover the whole disk */
1892                                        /*lb_count*/0,
1893                                        SSD_FULL_SIZE,
1894                                        5 * 1000);
1895                 error = cam_periph_runccb((union ccb *)&csio, cam_periph_error,
1896                     0, SF_NO_RECOVERY | SF_NO_RETRY, NULL);
1897                 if (error != 0)
1898                         xpt_print(periph->path, "Synchronize cache failed\n");
1899         }
1900         return (error);
1901 }
1902
1903 static int
1904 dagetattr(struct bio *bp)
1905 {
1906         int ret;
1907         struct cam_periph *periph;
1908
1909         periph = (struct cam_periph *)bp->bio_disk->d_drv1;
1910         cam_periph_lock(periph);
1911         ret = xpt_getattr(bp->bio_data, bp->bio_length, bp->bio_attribute,
1912             periph->path);
1913         cam_periph_unlock(periph);
1914         if (ret == 0)
1915                 bp->bio_completed = bp->bio_length;
1916         return ret;
1917 }
1918
1919 static void
1920 dainit(void)
1921 {
1922         cam_status status;
1923
1924         /*
1925          * Install a global async callback.  This callback will
1926          * receive async callbacks like "new device found".
1927          */
1928         status = xpt_register_async(AC_FOUND_DEVICE, daasync, NULL, NULL);
1929
1930         if (status != CAM_REQ_CMP) {
1931                 printf("da: Failed to attach master async callback "
1932                        "due to status 0x%x!\n", status);
1933         } else if (da_send_ordered) {
1934
1935                 /* Register our shutdown event handler */
1936                 if ((EVENTHANDLER_REGISTER(shutdown_post_sync, dashutdown,
1937                                            NULL, SHUTDOWN_PRI_DEFAULT)) == NULL)
1938                     printf("dainit: shutdown event registration failed!\n");
1939         }
1940 }
1941
1942 /*
1943  * Callback from GEOM, called when it has finished cleaning up its
1944  * resources.
1945  */
1946 static void
1947 dadiskgonecb(struct disk *dp)
1948 {
1949         struct cam_periph *periph;
1950
1951         periph = (struct cam_periph *)dp->d_drv1;
1952         da_periph_release(periph, DA_REF_GEOM);
1953 }
1954
1955 static void
1956 daoninvalidate(struct cam_periph *periph)
1957 {
1958         struct da_softc *softc;
1959
1960         cam_periph_assert(periph, MA_OWNED);
1961         softc = (struct da_softc *)periph->softc;
1962
1963         /*
1964          * De-register any async callbacks.
1965          */
1966         xpt_register_async(0, daasync, periph, periph->path);
1967
1968         softc->flags |= DA_FLAG_PACK_INVALID;
1969 #ifdef CAM_IO_STATS
1970         softc->invalidations++;
1971 #endif
1972
1973         /*
1974          * Return all queued I/O with ENXIO.
1975          * XXX Handle any transactions queued to the card
1976          *     with XPT_ABORT_CCB.
1977          */
1978         cam_iosched_flush(softc->cam_iosched, NULL, ENXIO);
1979
1980         /*
1981          * Tell GEOM that we've gone away, we'll get a callback when it is
1982          * done cleaning up its resources.
1983          */
1984         disk_gone(softc->disk);
1985 }
1986
1987 static void
1988 dacleanup(struct cam_periph *periph)
1989 {
1990         struct da_softc *softc;
1991
1992         softc = (struct da_softc *)periph->softc;
1993
1994         cam_periph_unlock(periph);
1995
1996         cam_iosched_fini(softc->cam_iosched);
1997
1998         /*
1999          * If we can't free the sysctl tree, oh well...
2000          */
2001         if ((softc->flags & DA_FLAG_SCTX_INIT) != 0) {
2002 #ifdef CAM_IO_STATS
2003                 if (sysctl_ctx_free(&softc->sysctl_stats_ctx) != 0)
2004                         xpt_print(periph->path,
2005                             "can't remove sysctl stats context\n");
2006 #endif
2007                 if (sysctl_ctx_free(&softc->sysctl_ctx) != 0)
2008                         xpt_print(periph->path,
2009                             "can't remove sysctl context\n");
2010         }
2011
2012         callout_drain(&softc->mediapoll_c);
2013         disk_destroy(softc->disk);
2014         callout_drain(&softc->sendordered_c);
2015         free(softc, M_DEVBUF);
2016         cam_periph_lock(periph);
2017 }
2018
2019 static void
2020 daasync(void *callback_arg, u_int32_t code,
2021         struct cam_path *path, void *arg)
2022 {
2023         struct cam_periph *periph;
2024         struct da_softc *softc;
2025
2026         periph = (struct cam_periph *)callback_arg;
2027         switch (code) {
2028         case AC_FOUND_DEVICE:   /* callback to create periph, no locking yet */
2029         {
2030                 struct ccb_getdev *cgd;
2031                 cam_status status;
2032
2033                 cgd = (struct ccb_getdev *)arg;
2034                 if (cgd == NULL)
2035                         break;
2036
2037                 if (cgd->protocol != PROTO_SCSI)
2038                         break;
2039                 if (SID_QUAL(&cgd->inq_data) != SID_QUAL_LU_CONNECTED)
2040                         break;
2041                 if (SID_TYPE(&cgd->inq_data) != T_DIRECT
2042                     && SID_TYPE(&cgd->inq_data) != T_RBC
2043                     && SID_TYPE(&cgd->inq_data) != T_OPTICAL
2044                     && SID_TYPE(&cgd->inq_data) != T_ZBC_HM)
2045                         break;
2046
2047                 /*
2048                  * Allocate a peripheral instance for
2049                  * this device and start the probe
2050                  * process.
2051                  */
2052                 status = cam_periph_alloc(daregister, daoninvalidate,
2053                                           dacleanup, dastart,
2054                                           "da", CAM_PERIPH_BIO,
2055                                           path, daasync,
2056                                           AC_FOUND_DEVICE, cgd);
2057
2058                 if (status != CAM_REQ_CMP
2059                  && status != CAM_REQ_INPROG)
2060                         printf("daasync: Unable to attach to new device "
2061                                 "due to status 0x%x\n", status);
2062                 return;
2063         }
2064         case AC_ADVINFO_CHANGED:        /* Doesn't touch periph */
2065         {
2066                 uintptr_t buftype;
2067
2068                 buftype = (uintptr_t)arg;
2069                 if (buftype == CDAI_TYPE_PHYS_PATH) {
2070                         struct da_softc *softc;
2071
2072                         softc = periph->softc;
2073                         disk_attr_changed(softc->disk, "GEOM::physpath",
2074                                           M_NOWAIT);
2075                 }
2076                 break;
2077         }
2078         case AC_UNIT_ATTENTION:
2079         {
2080                 union ccb *ccb;
2081                 int error_code, sense_key, asc, ascq;
2082
2083                 softc = (struct da_softc *)periph->softc;
2084                 ccb = (union ccb *)arg;
2085
2086                 /*
2087                  * Handle all UNIT ATTENTIONs except our own, as they will be
2088                  * handled by daerror(). Since this comes from a different periph,
2089                  * that periph's lock is held, not ours, so we have to take it ours
2090                  * out to touch softc flags.
2091                  */
2092                 if (xpt_path_periph(ccb->ccb_h.path) != periph &&
2093                     scsi_extract_sense_ccb(ccb,
2094                      &error_code, &sense_key, &asc, &ascq)) {
2095                         if (asc == 0x2A && ascq == 0x09) {
2096                                 xpt_print(ccb->ccb_h.path,
2097                                     "Capacity data has changed\n");
2098                                 cam_periph_lock(periph);
2099                                 softc->flags &= ~DA_FLAG_PROBED;
2100                                 cam_periph_unlock(periph);
2101                                 dareprobe(periph);
2102                         } else if (asc == 0x28 && ascq == 0x00) {
2103                                 cam_periph_lock(periph);
2104                                 softc->flags &= ~DA_FLAG_PROBED;
2105                                 cam_periph_unlock(periph);
2106                                 disk_media_changed(softc->disk, M_NOWAIT);
2107                         } else if (asc == 0x3F && ascq == 0x03) {
2108                                 xpt_print(ccb->ccb_h.path,
2109                                     "INQUIRY data has changed\n");
2110                                 cam_periph_lock(periph);
2111                                 softc->flags &= ~DA_FLAG_PROBED;
2112                                 cam_periph_unlock(periph);
2113                                 dareprobe(periph);
2114                         }
2115                 }
2116                 break;
2117         }
2118         case AC_SCSI_AEN:               /* Called for this path: periph locked */
2119                 /*
2120                  * Appears to be currently unused for SCSI devices, only ata SIMs
2121                  * generate this.
2122                  */
2123                 cam_periph_assert(periph, MA_OWNED);
2124                 softc = (struct da_softc *)periph->softc;
2125                 if (!cam_iosched_has_work_flags(softc->cam_iosched, DA_WORK_TUR) &&
2126                     (softc->flags & DA_FLAG_TUR_PENDING) == 0) {
2127                         if (da_periph_acquire(periph, DA_REF_TUR) == 0) {
2128                                 cam_iosched_set_work_flags(softc->cam_iosched, DA_WORK_TUR);
2129                                 daschedule(periph);
2130                         }
2131                 }
2132                 /* FALLTHROUGH */
2133         case AC_SENT_BDR:               /* Called for this path: periph locked */
2134         case AC_BUS_RESET:              /* Called for this path: periph locked */
2135         {
2136                 struct ccb_hdr *ccbh;
2137
2138                 cam_periph_assert(periph, MA_OWNED);
2139                 softc = (struct da_softc *)periph->softc;
2140                 /*
2141                  * Don't fail on the expected unit attention
2142                  * that will occur.
2143                  */
2144                 softc->flags |= DA_FLAG_RETRY_UA;
2145                 LIST_FOREACH(ccbh, &softc->pending_ccbs, periph_links.le)
2146                         ccbh->ccb_state |= DA_CCB_RETRY_UA;
2147                 break;
2148         }
2149         case AC_INQ_CHANGED:            /* Called for this path: periph locked */
2150                 cam_periph_assert(periph, MA_OWNED);
2151                 softc = (struct da_softc *)periph->softc;
2152                 softc->flags &= ~DA_FLAG_PROBED;
2153                 dareprobe(periph);
2154                 break;
2155         default:
2156                 break;
2157         }
2158         cam_periph_async(periph, code, path, arg);
2159 }
2160
2161 static void
2162 dasysctlinit(void *context, int pending)
2163 {
2164         struct cam_periph *periph;
2165         struct da_softc *softc;
2166         char tmpstr[32], tmpstr2[16];
2167         struct ccb_trans_settings cts;
2168
2169         periph = (struct cam_periph *)context;
2170         /*
2171          * periph was held for us when this task was enqueued
2172          */
2173         if (periph->flags & CAM_PERIPH_INVALID) {
2174                 da_periph_release(periph, DA_REF_SYSCTL);
2175                 return;
2176         }
2177
2178         softc = (struct da_softc *)periph->softc;
2179         snprintf(tmpstr, sizeof(tmpstr), "CAM DA unit %d", periph->unit_number);
2180         snprintf(tmpstr2, sizeof(tmpstr2), "%d", periph->unit_number);
2181
2182         sysctl_ctx_init(&softc->sysctl_ctx);
2183         cam_periph_lock(periph);
2184         softc->flags |= DA_FLAG_SCTX_INIT;
2185         cam_periph_unlock(periph);
2186         softc->sysctl_tree = SYSCTL_ADD_NODE_WITH_LABEL(&softc->sysctl_ctx,
2187                 SYSCTL_STATIC_CHILDREN(_kern_cam_da), OID_AUTO, tmpstr2,
2188                 CTLFLAG_RD, 0, tmpstr, "device_index");
2189         if (softc->sysctl_tree == NULL) {
2190                 printf("dasysctlinit: unable to allocate sysctl tree\n");
2191                 da_periph_release(periph, DA_REF_SYSCTL);
2192                 return;
2193         }
2194
2195         /*
2196          * Now register the sysctl handler, so the user can change the value on
2197          * the fly.
2198          */
2199         SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
2200                 OID_AUTO, "delete_method", CTLTYPE_STRING | CTLFLAG_RWTUN,
2201                 softc, 0, dadeletemethodsysctl, "A",
2202                 "BIO_DELETE execution method");
2203         SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
2204                 OID_AUTO, "delete_max", CTLTYPE_U64 | CTLFLAG_RW,
2205                 softc, 0, dadeletemaxsysctl, "Q",
2206                 "Maximum BIO_DELETE size");
2207         SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
2208                 OID_AUTO, "minimum_cmd_size", CTLTYPE_INT | CTLFLAG_RW,
2209                 &softc->minimum_cmd_size, 0, dacmdsizesysctl, "I",
2210                 "Minimum CDB size");
2211         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
2212                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
2213                 "trim_count", CTLFLAG_RD, &softc->trim_count,
2214                 "Total number of unmap/dsm commands sent");
2215         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
2216                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
2217                 "trim_ranges", CTLFLAG_RD, &softc->trim_ranges,
2218                 "Total number of ranges in unmap/dsm commands");
2219         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
2220                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
2221                 "trim_lbas", CTLFLAG_RD, &softc->trim_lbas,
2222                 "Total lbas in the unmap/dsm commands sent");
2223
2224         SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
2225                 OID_AUTO, "zone_mode", CTLTYPE_STRING | CTLFLAG_RD,
2226                 softc, 0, dazonemodesysctl, "A",
2227                 "Zone Mode");
2228         SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
2229                 OID_AUTO, "zone_support", CTLTYPE_STRING | CTLFLAG_RD,
2230                 softc, 0, dazonesupsysctl, "A",
2231                 "Zone Support");
2232         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
2233                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
2234                 "optimal_seq_zones", CTLFLAG_RD, &softc->optimal_seq_zones,
2235                 "Optimal Number of Open Sequential Write Preferred Zones");
2236         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
2237                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
2238                 "optimal_nonseq_zones", CTLFLAG_RD,
2239                 &softc->optimal_nonseq_zones,
2240                 "Optimal Number of Non-Sequentially Written Sequential Write "
2241                 "Preferred Zones");
2242         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
2243                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
2244                 "max_seq_zones", CTLFLAG_RD, &softc->max_seq_zones,
2245                 "Maximum Number of Open Sequential Write Required Zones");
2246
2247         SYSCTL_ADD_INT(&softc->sysctl_ctx,
2248                        SYSCTL_CHILDREN(softc->sysctl_tree),
2249                        OID_AUTO,
2250                        "error_inject",
2251                        CTLFLAG_RW,
2252                        &softc->error_inject,
2253                        0,
2254                        "error_inject leaf");
2255
2256         SYSCTL_ADD_INT(&softc->sysctl_ctx,
2257                        SYSCTL_CHILDREN(softc->sysctl_tree),
2258                        OID_AUTO,
2259                        "unmapped_io",
2260                        CTLFLAG_RD,
2261                        &softc->unmappedio,
2262                        0,
2263                        "Unmapped I/O leaf");
2264
2265         SYSCTL_ADD_INT(&softc->sysctl_ctx,
2266                        SYSCTL_CHILDREN(softc->sysctl_tree),
2267                        OID_AUTO,
2268                        "rotating",
2269                        CTLFLAG_RD,
2270                        &softc->rotating,
2271                        0,
2272                        "Rotating media");
2273
2274 #ifdef CAM_TEST_FAILURE
2275         SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
2276                 OID_AUTO, "invalidate", CTLTYPE_U64 | CTLFLAG_RW | CTLFLAG_MPSAFE,
2277                 periph, 0, cam_periph_invalidate_sysctl, "I",
2278                 "Write 1 to invalidate the drive immediately");
2279 #endif
2280
2281         /*
2282          * Add some addressing info.
2283          */
2284         memset(&cts, 0, sizeof (cts));
2285         xpt_setup_ccb(&cts.ccb_h, periph->path, CAM_PRIORITY_NONE);
2286         cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS;
2287         cts.type = CTS_TYPE_CURRENT_SETTINGS;
2288         cam_periph_lock(periph);
2289         xpt_action((union ccb *)&cts);
2290         cam_periph_unlock(periph);
2291         if (cts.ccb_h.status != CAM_REQ_CMP) {
2292                 da_periph_release(periph, DA_REF_SYSCTL);
2293                 return;
2294         }
2295         if (cts.protocol == PROTO_SCSI && cts.transport == XPORT_FC) {
2296                 struct ccb_trans_settings_fc *fc = &cts.xport_specific.fc;
2297                 if (fc->valid & CTS_FC_VALID_WWPN) {
2298                         softc->wwpn = fc->wwpn;
2299                         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
2300                             SYSCTL_CHILDREN(softc->sysctl_tree),
2301                             OID_AUTO, "wwpn", CTLFLAG_RD,
2302                             &softc->wwpn, "World Wide Port Name");
2303                 }
2304         }
2305
2306 #ifdef CAM_IO_STATS
2307         /*
2308          * Now add some useful stats.
2309          * XXX These should live in cam_periph and be common to all periphs
2310          */
2311         softc->sysctl_stats_tree = SYSCTL_ADD_NODE(&softc->sysctl_stats_ctx,
2312             SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO, "stats",
2313             CTLFLAG_RD, 0, "Statistics");
2314         SYSCTL_ADD_INT(&softc->sysctl_stats_ctx,
2315                        SYSCTL_CHILDREN(softc->sysctl_stats_tree),
2316                        OID_AUTO,
2317                        "errors",
2318                        CTLFLAG_RD,
2319                        &softc->errors,
2320                        0,
2321                        "Transport errors reported by the SIM");
2322         SYSCTL_ADD_INT(&softc->sysctl_stats_ctx,
2323                        SYSCTL_CHILDREN(softc->sysctl_stats_tree),
2324                        OID_AUTO,
2325                        "timeouts",
2326                        CTLFLAG_RD,
2327                        &softc->timeouts,
2328                        0,
2329                        "Device timeouts reported by the SIM");
2330         SYSCTL_ADD_INT(&softc->sysctl_stats_ctx,
2331                        SYSCTL_CHILDREN(softc->sysctl_stats_tree),
2332                        OID_AUTO,
2333                        "pack_invalidations",
2334                        CTLFLAG_RD,
2335                        &softc->invalidations,
2336                        0,
2337                        "Device pack invalidations");
2338 #endif
2339
2340         cam_iosched_sysctl_init(softc->cam_iosched, &softc->sysctl_ctx,
2341             softc->sysctl_tree);
2342
2343         da_periph_release(periph, DA_REF_SYSCTL);
2344 }
2345
2346 static int
2347 dadeletemaxsysctl(SYSCTL_HANDLER_ARGS)
2348 {
2349         int error;
2350         uint64_t value;
2351         struct da_softc *softc;
2352
2353         softc = (struct da_softc *)arg1;
2354
2355         value = softc->disk->d_delmaxsize;
2356         error = sysctl_handle_64(oidp, &value, 0, req);
2357         if ((error != 0) || (req->newptr == NULL))
2358                 return (error);
2359
2360         /* only accept values smaller than the calculated value */
2361         if (value > dadeletemaxsize(softc, softc->delete_method)) {
2362                 return (EINVAL);
2363         }
2364         softc->disk->d_delmaxsize = value;
2365
2366         return (0);
2367 }
2368
2369 static int
2370 dacmdsizesysctl(SYSCTL_HANDLER_ARGS)
2371 {
2372         int error, value;
2373
2374         value = *(int *)arg1;
2375
2376         error = sysctl_handle_int(oidp, &value, 0, req);
2377
2378         if ((error != 0)
2379          || (req->newptr == NULL))
2380                 return (error);
2381
2382         /*
2383          * Acceptable values here are 6, 10, 12 or 16.
2384          */
2385         if (value < 6)
2386                 value = 6;
2387         else if ((value > 6)
2388               && (value <= 10))
2389                 value = 10;
2390         else if ((value > 10)
2391               && (value <= 12))
2392                 value = 12;
2393         else if (value > 12)
2394                 value = 16;
2395
2396         *(int *)arg1 = value;
2397
2398         return (0);
2399 }
2400
2401 static int
2402 dasysctlsofttimeout(SYSCTL_HANDLER_ARGS)
2403 {
2404         sbintime_t value;
2405         int error;
2406
2407         value = da_default_softtimeout / SBT_1MS;
2408
2409         error = sysctl_handle_int(oidp, (int *)&value, 0, req);
2410         if ((error != 0) || (req->newptr == NULL))
2411                 return (error);
2412
2413         /* XXX Should clip this to a reasonable level */
2414         if (value > da_default_timeout * 1000)
2415                 return (EINVAL);
2416
2417         da_default_softtimeout = value * SBT_1MS;
2418         return (0);
2419 }
2420
2421 static void
2422 dadeletemethodset(struct da_softc *softc, da_delete_methods delete_method)
2423 {
2424
2425         softc->delete_method = delete_method;
2426         softc->disk->d_delmaxsize = dadeletemaxsize(softc, delete_method);
2427         softc->delete_func = da_delete_functions[delete_method];
2428
2429         if (softc->delete_method > DA_DELETE_DISABLE)
2430                 softc->disk->d_flags |= DISKFLAG_CANDELETE;
2431         else
2432                 softc->disk->d_flags &= ~DISKFLAG_CANDELETE;
2433 }
2434
2435 static off_t
2436 dadeletemaxsize(struct da_softc *softc, da_delete_methods delete_method)
2437 {
2438         off_t sectors;
2439
2440         switch(delete_method) {
2441         case DA_DELETE_UNMAP:
2442                 sectors = (off_t)softc->unmap_max_lba;
2443                 break;
2444         case DA_DELETE_ATA_TRIM:
2445                 sectors = (off_t)ATA_DSM_RANGE_MAX * softc->trim_max_ranges;
2446                 break;
2447         case DA_DELETE_WS16:
2448                 sectors = omin(softc->ws_max_blks, WS16_MAX_BLKS);
2449                 break;
2450         case DA_DELETE_ZERO:
2451         case DA_DELETE_WS10:
2452                 sectors = omin(softc->ws_max_blks, WS10_MAX_BLKS);
2453                 break;
2454         default:
2455                 return 0;
2456         }
2457
2458         return (off_t)softc->params.secsize *
2459             omin(sectors, softc->params.sectors);
2460 }
2461
2462 static void
2463 daprobedone(struct cam_periph *periph, union ccb *ccb)
2464 {
2465         struct da_softc *softc;
2466
2467         softc = (struct da_softc *)periph->softc;
2468
2469         cam_periph_assert(periph, MA_OWNED);
2470
2471         dadeletemethodchoose(softc, DA_DELETE_NONE);
2472
2473         if (bootverbose && (softc->flags & DA_FLAG_ANNOUNCED) == 0) {
2474                 char buf[80];
2475                 int i, sep;
2476
2477                 snprintf(buf, sizeof(buf), "Delete methods: <");
2478                 sep = 0;
2479                 for (i = 0; i <= DA_DELETE_MAX; i++) {
2480                         if ((softc->delete_available & (1 << i)) == 0 &&
2481                             i != softc->delete_method)
2482                                 continue;
2483                         if (sep)
2484                                 strlcat(buf, ",", sizeof(buf));
2485                         strlcat(buf, da_delete_method_names[i],
2486                             sizeof(buf));
2487                         if (i == softc->delete_method)
2488                                 strlcat(buf, "(*)", sizeof(buf));
2489                         sep = 1;
2490                 }
2491                 strlcat(buf, ">", sizeof(buf));
2492                 printf("%s%d: %s\n", periph->periph_name,
2493                     periph->unit_number, buf);
2494         }
2495         if ((softc->disk->d_flags & DISKFLAG_WRITE_PROTECT) != 0 &&
2496             (softc->flags & DA_FLAG_ANNOUNCED) == 0) {
2497                 printf("%s%d: Write Protected\n", periph->periph_name,
2498                     periph->unit_number);
2499         }
2500
2501         /*
2502          * Since our peripheral may be invalidated by an error
2503          * above or an external event, we must release our CCB
2504          * before releasing the probe lock on the peripheral.
2505          * The peripheral will only go away once the last lock
2506          * is removed, and we need it around for the CCB release
2507          * operation.
2508          */
2509         xpt_release_ccb(ccb);
2510         softc->state = DA_STATE_NORMAL;
2511         softc->flags |= DA_FLAG_PROBED;
2512         daschedule(periph);
2513         wakeup(&softc->disk->d_mediasize);
2514         if ((softc->flags & DA_FLAG_ANNOUNCED) == 0) {
2515                 softc->flags |= DA_FLAG_ANNOUNCED;
2516                 da_periph_unhold(periph, DA_REF_PROBE_HOLD);
2517         } else
2518                 da_periph_release_locked(periph, DA_REF_REPROBE);
2519 }
2520
2521 static void
2522 dadeletemethodchoose(struct da_softc *softc, da_delete_methods default_method)
2523 {
2524         int i, methods;
2525
2526         /* If available, prefer the method requested by user. */
2527         i = softc->delete_method_pref;
2528         methods = softc->delete_available | (1 << DA_DELETE_DISABLE);
2529         if (methods & (1 << i)) {
2530                 dadeletemethodset(softc, i);
2531                 return;
2532         }
2533
2534         /* Use the pre-defined order to choose the best performing delete. */
2535         for (i = DA_DELETE_MIN; i <= DA_DELETE_MAX; i++) {
2536                 if (i == DA_DELETE_ZERO)
2537                         continue;
2538                 if (softc->delete_available & (1 << i)) {
2539                         dadeletemethodset(softc, i);
2540                         return;
2541                 }
2542         }
2543
2544         /* Fallback to default. */
2545         dadeletemethodset(softc, default_method);
2546 }
2547
2548 static int
2549 dadeletemethodsysctl(SYSCTL_HANDLER_ARGS)
2550 {
2551         char buf[16];
2552         const char *p;
2553         struct da_softc *softc;
2554         int i, error, value;
2555
2556         softc = (struct da_softc *)arg1;
2557
2558         value = softc->delete_method;
2559         if (value < 0 || value > DA_DELETE_MAX)
2560                 p = "UNKNOWN";
2561         else
2562                 p = da_delete_method_names[value];
2563         strncpy(buf, p, sizeof(buf));
2564         error = sysctl_handle_string(oidp, buf, sizeof(buf), req);
2565         if (error != 0 || req->newptr == NULL)
2566                 return (error);
2567         for (i = 0; i <= DA_DELETE_MAX; i++) {
2568                 if (strcmp(buf, da_delete_method_names[i]) == 0)
2569                         break;
2570         }
2571         if (i > DA_DELETE_MAX)
2572                 return (EINVAL);
2573         softc->delete_method_pref = i;
2574         dadeletemethodchoose(softc, DA_DELETE_NONE);
2575         return (0);
2576 }
2577
2578 static int
2579 dazonemodesysctl(SYSCTL_HANDLER_ARGS)
2580 {
2581         char tmpbuf[40];
2582         struct da_softc *softc;
2583         int error;
2584
2585         softc = (struct da_softc *)arg1;
2586
2587         switch (softc->zone_mode) {
2588         case DA_ZONE_DRIVE_MANAGED:
2589                 snprintf(tmpbuf, sizeof(tmpbuf), "Drive Managed");
2590                 break;
2591         case DA_ZONE_HOST_AWARE:
2592                 snprintf(tmpbuf, sizeof(tmpbuf), "Host Aware");
2593                 break;
2594         case DA_ZONE_HOST_MANAGED:
2595                 snprintf(tmpbuf, sizeof(tmpbuf), "Host Managed");
2596                 break;
2597         case DA_ZONE_NONE:
2598         default:
2599                 snprintf(tmpbuf, sizeof(tmpbuf), "Not Zoned");
2600                 break;
2601         }
2602
2603         error = sysctl_handle_string(oidp, tmpbuf, sizeof(tmpbuf), req);
2604
2605         return (error);
2606 }
2607
2608 static int
2609 dazonesupsysctl(SYSCTL_HANDLER_ARGS)
2610 {
2611         char tmpbuf[180];
2612         struct da_softc *softc;
2613         struct sbuf sb;
2614         int error, first;
2615         unsigned int i;
2616
2617         softc = (struct da_softc *)arg1;
2618
2619         error = 0;
2620         first = 1;
2621         sbuf_new(&sb, tmpbuf, sizeof(tmpbuf), 0);
2622
2623         for (i = 0; i < sizeof(da_zone_desc_table) /
2624              sizeof(da_zone_desc_table[0]); i++) {
2625                 if (softc->zone_flags & da_zone_desc_table[i].value) {
2626                         if (first == 0)
2627                                 sbuf_printf(&sb, ", ");
2628                         else
2629                                 first = 0;
2630                         sbuf_cat(&sb, da_zone_desc_table[i].desc);
2631                 }
2632         }
2633
2634         if (first == 1)
2635                 sbuf_printf(&sb, "None");
2636
2637         sbuf_finish(&sb);
2638
2639         error = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
2640
2641         return (error);
2642 }
2643
2644 static cam_status
2645 daregister(struct cam_periph *periph, void *arg)
2646 {
2647         struct da_softc *softc;
2648         struct ccb_pathinq cpi;
2649         struct ccb_getdev *cgd;
2650         char tmpstr[80];
2651         caddr_t match;
2652
2653         cgd = (struct ccb_getdev *)arg;
2654         if (cgd == NULL) {
2655                 printf("daregister: no getdev CCB, can't register device\n");
2656                 return(CAM_REQ_CMP_ERR);
2657         }
2658
2659         softc = (struct da_softc *)malloc(sizeof(*softc), M_DEVBUF,
2660             M_NOWAIT|M_ZERO);
2661
2662         if (softc == NULL) {
2663                 printf("daregister: Unable to probe new device. "
2664                        "Unable to allocate softc\n");
2665                 return(CAM_REQ_CMP_ERR);
2666         }
2667
2668         if (cam_iosched_init(&softc->cam_iosched, periph) != 0) {
2669                 printf("daregister: Unable to probe new device. "
2670                        "Unable to allocate iosched memory\n");
2671                 free(softc, M_DEVBUF);
2672                 return(CAM_REQ_CMP_ERR);
2673         }
2674
2675         LIST_INIT(&softc->pending_ccbs);
2676         softc->state = DA_STATE_PROBE_WP;
2677         bioq_init(&softc->delete_run_queue);
2678         if (SID_IS_REMOVABLE(&cgd->inq_data))
2679                 softc->flags |= DA_FLAG_PACK_REMOVABLE;
2680         softc->unmap_max_ranges = UNMAP_MAX_RANGES;
2681         softc->unmap_max_lba = UNMAP_RANGE_MAX;
2682         softc->unmap_gran = 0;
2683         softc->unmap_gran_align = 0;
2684         softc->ws_max_blks = WS16_MAX_BLKS;
2685         softc->trim_max_ranges = ATA_TRIM_MAX_RANGES;
2686         softc->rotating = 1;
2687
2688         periph->softc = softc;
2689
2690         /*
2691          * See if this device has any quirks.
2692          */
2693         match = cam_quirkmatch((caddr_t)&cgd->inq_data,
2694                                (caddr_t)da_quirk_table,
2695                                nitems(da_quirk_table),
2696                                sizeof(*da_quirk_table), scsi_inquiry_match);
2697
2698         if (match != NULL)
2699                 softc->quirks = ((struct da_quirk_entry *)match)->quirks;
2700         else
2701                 softc->quirks = DA_Q_NONE;
2702
2703         /* Check if the SIM does not want 6 byte commands */
2704         xpt_path_inq(&cpi, periph->path);
2705         if (cpi.ccb_h.status == CAM_REQ_CMP && (cpi.hba_misc & PIM_NO_6_BYTE))
2706                 softc->quirks |= DA_Q_NO_6_BYTE;
2707
2708         if (SID_TYPE(&cgd->inq_data) == T_ZBC_HM)
2709                 softc->zone_mode = DA_ZONE_HOST_MANAGED;
2710         else if (softc->quirks & DA_Q_SMR_DM)
2711                 softc->zone_mode = DA_ZONE_DRIVE_MANAGED;
2712         else
2713                 softc->zone_mode = DA_ZONE_NONE;
2714
2715         if (softc->zone_mode != DA_ZONE_NONE) {
2716                 if (scsi_vpd_supported_page(periph, SVPD_ATA_INFORMATION)) {
2717                         if (scsi_vpd_supported_page(periph, SVPD_ZONED_BDC))
2718                                 softc->zone_interface = DA_ZONE_IF_ATA_SAT;
2719                         else
2720                                 softc->zone_interface = DA_ZONE_IF_ATA_PASS;
2721                 } else
2722                         softc->zone_interface = DA_ZONE_IF_SCSI;
2723         }
2724
2725         TASK_INIT(&softc->sysctl_task, 0, dasysctlinit, periph);
2726
2727         /*
2728          * Take an exclusive section lock qon the periph while dastart is called
2729          * to finish the probe.  The lock will be dropped in dadone at the end
2730          * of probe. This locks out daopen and daclose from racing with the
2731          * probe.
2732          *
2733          * XXX if cam_periph_hold returns an error, we don't hold a refcount.
2734          */
2735         (void)da_periph_hold(periph, PRIBIO, DA_REF_PROBE_HOLD);
2736
2737         /*
2738          * Schedule a periodic event to occasionally send an
2739          * ordered tag to a device.
2740          */
2741         callout_init_mtx(&softc->sendordered_c, cam_periph_mtx(periph), 0);
2742         callout_reset(&softc->sendordered_c,
2743             (da_default_timeout * hz) / DA_ORDEREDTAG_INTERVAL,
2744             dasendorderedtag, periph);
2745
2746         cam_periph_unlock(periph);
2747         /*
2748          * RBC devices don't have to support READ(6), only READ(10).
2749          */
2750         if (softc->quirks & DA_Q_NO_6_BYTE || SID_TYPE(&cgd->inq_data) == T_RBC)
2751                 softc->minimum_cmd_size = 10;
2752         else
2753                 softc->minimum_cmd_size = 6;
2754
2755         /*
2756          * Load the user's default, if any.
2757          */
2758         snprintf(tmpstr, sizeof(tmpstr), "kern.cam.da.%d.minimum_cmd_size",
2759                  periph->unit_number);
2760         TUNABLE_INT_FETCH(tmpstr, &softc->minimum_cmd_size);
2761
2762         /*
2763          * 6, 10, 12 and 16 are the currently permissible values.
2764          */
2765         if (softc->minimum_cmd_size > 12)
2766                 softc->minimum_cmd_size = 16;
2767         else if (softc->minimum_cmd_size > 10)
2768                 softc->minimum_cmd_size = 12;
2769         else if (softc->minimum_cmd_size > 6)
2770                 softc->minimum_cmd_size = 10;
2771         else
2772                 softc->minimum_cmd_size = 6;
2773
2774         /* Predict whether device may support READ CAPACITY(16). */
2775         if (SID_ANSI_REV(&cgd->inq_data) >= SCSI_REV_SPC3 &&
2776             (softc->quirks & DA_Q_NO_RC16) == 0) {
2777                 softc->flags |= DA_FLAG_CAN_RC16;
2778         }
2779
2780         /*
2781          * Register this media as a disk.
2782          */
2783         softc->disk = disk_alloc();
2784         softc->disk->d_devstat = devstat_new_entry(periph->periph_name,
2785                           periph->unit_number, 0,
2786                           DEVSTAT_BS_UNAVAILABLE,
2787                           SID_TYPE(&cgd->inq_data) |
2788                           XPORT_DEVSTAT_TYPE(cpi.transport),
2789                           DEVSTAT_PRIORITY_DISK);
2790         softc->disk->d_open = daopen;
2791         softc->disk->d_close = daclose;
2792         softc->disk->d_strategy = dastrategy;
2793         softc->disk->d_dump = dadump;
2794         softc->disk->d_getattr = dagetattr;
2795         softc->disk->d_gone = dadiskgonecb;
2796         softc->disk->d_name = "da";
2797         softc->disk->d_drv1 = periph;
2798         if (cpi.maxio == 0)
2799                 softc->maxio = DFLTPHYS;        /* traditional default */
2800         else if (cpi.maxio > MAXPHYS)
2801                 softc->maxio = MAXPHYS;         /* for safety */
2802         else
2803                 softc->maxio = cpi.maxio;
2804         softc->disk->d_maxsize = softc->maxio;
2805         softc->disk->d_unit = periph->unit_number;
2806         softc->disk->d_flags = DISKFLAG_DIRECT_COMPLETION | DISKFLAG_CANZONE;
2807         if ((softc->quirks & DA_Q_NO_SYNC_CACHE) == 0)
2808                 softc->disk->d_flags |= DISKFLAG_CANFLUSHCACHE;
2809         if ((cpi.hba_misc & PIM_UNMAPPED) != 0) {
2810                 softc->unmappedio = 1;
2811                 softc->disk->d_flags |= DISKFLAG_UNMAPPED_BIO;
2812         }
2813         cam_strvis(softc->disk->d_descr, cgd->inq_data.vendor,
2814             sizeof(cgd->inq_data.vendor), sizeof(softc->disk->d_descr));
2815         strlcat(softc->disk->d_descr, " ", sizeof(softc->disk->d_descr));
2816         cam_strvis(&softc->disk->d_descr[strlen(softc->disk->d_descr)],
2817             cgd->inq_data.product, sizeof(cgd->inq_data.product),
2818             sizeof(softc->disk->d_descr) - strlen(softc->disk->d_descr));
2819         softc->disk->d_hba_vendor = cpi.hba_vendor;
2820         softc->disk->d_hba_device = cpi.hba_device;
2821         softc->disk->d_hba_subvendor = cpi.hba_subvendor;
2822         softc->disk->d_hba_subdevice = cpi.hba_subdevice;
2823
2824         /*
2825          * Acquire a reference to the periph before we register with GEOM.
2826          * We'll release this reference once GEOM calls us back (via
2827          * dadiskgonecb()) telling us that our provider has been freed.
2828          */
2829         if (da_periph_acquire(periph, DA_REF_GEOM) != 0) {
2830                 xpt_print(periph->path, "%s: lost periph during "
2831                           "registration!\n", __func__);
2832                 cam_periph_lock(periph);
2833                 return (CAM_REQ_CMP_ERR);
2834         }
2835
2836         disk_create(softc->disk, DISK_VERSION);
2837         cam_periph_lock(periph);
2838
2839         /*
2840          * Add async callbacks for events of interest.
2841          * I don't bother checking if this fails as,
2842          * in most cases, the system will function just
2843          * fine without them and the only alternative
2844          * would be to not attach the device on failure.
2845          */
2846         xpt_register_async(AC_SENT_BDR | AC_BUS_RESET | AC_LOST_DEVICE |
2847             AC_ADVINFO_CHANGED | AC_SCSI_AEN | AC_UNIT_ATTENTION |
2848             AC_INQ_CHANGED, daasync, periph, periph->path);
2849
2850         /*
2851          * Emit an attribute changed notification just in case
2852          * physical path information arrived before our async
2853          * event handler was registered, but after anyone attaching
2854          * to our disk device polled it.
2855          */
2856         disk_attr_changed(softc->disk, "GEOM::physpath", M_NOWAIT);
2857
2858         /*
2859          * Schedule a periodic media polling events.
2860          */
2861         callout_init_mtx(&softc->mediapoll_c, cam_periph_mtx(periph), 0);
2862         if ((softc->flags & DA_FLAG_PACK_REMOVABLE) &&
2863             (cgd->inq_flags & SID_AEN) == 0 &&
2864             da_poll_period != 0)
2865                 callout_reset(&softc->mediapoll_c, da_poll_period * hz,
2866                     damediapoll, periph);
2867
2868         xpt_schedule(periph, CAM_PRIORITY_DEV);
2869
2870         return(CAM_REQ_CMP);
2871 }
2872
2873 static int
2874 da_zone_bio_to_scsi(int disk_zone_cmd)
2875 {
2876         switch (disk_zone_cmd) {
2877         case DISK_ZONE_OPEN:
2878                 return ZBC_OUT_SA_OPEN;
2879         case DISK_ZONE_CLOSE:
2880                 return ZBC_OUT_SA_CLOSE;
2881         case DISK_ZONE_FINISH:
2882                 return ZBC_OUT_SA_FINISH;
2883         case DISK_ZONE_RWP:
2884                 return ZBC_OUT_SA_RWP;
2885         }
2886
2887         return -1;
2888 }
2889
2890 static int
2891 da_zone_cmd(struct cam_periph *periph, union ccb *ccb, struct bio *bp,
2892             int *queue_ccb)
2893 {
2894         struct da_softc *softc;
2895         int error;
2896
2897         error = 0;
2898
2899         if (bp->bio_cmd != BIO_ZONE) {
2900                 error = EINVAL;
2901                 goto bailout;
2902         }
2903
2904         softc = periph->softc;
2905
2906         switch (bp->bio_zone.zone_cmd) {
2907         case DISK_ZONE_OPEN:
2908         case DISK_ZONE_CLOSE:
2909         case DISK_ZONE_FINISH:
2910         case DISK_ZONE_RWP: {
2911                 int zone_flags;
2912                 int zone_sa;
2913                 uint64_t lba;
2914
2915                 zone_sa = da_zone_bio_to_scsi(bp->bio_zone.zone_cmd);
2916                 if (zone_sa == -1) {
2917                         xpt_print(periph->path, "Cannot translate zone "
2918                             "cmd %#x to SCSI\n", bp->bio_zone.zone_cmd);
2919                         error = EINVAL;
2920                         goto bailout;
2921                 }
2922
2923                 zone_flags = 0;
2924                 lba = bp->bio_zone.zone_params.rwp.id;
2925
2926                 if (bp->bio_zone.zone_params.rwp.flags &
2927                     DISK_ZONE_RWP_FLAG_ALL)
2928                         zone_flags |= ZBC_OUT_ALL;
2929
2930                 if (softc->zone_interface != DA_ZONE_IF_ATA_PASS) {
2931                         scsi_zbc_out(&ccb->csio,
2932                                      /*retries*/ da_retry_count,
2933                                      /*cbfcnp*/ dadone,
2934                                      /*tag_action*/ MSG_SIMPLE_Q_TAG,
2935                                      /*service_action*/ zone_sa,
2936                                      /*zone_id*/ lba,
2937                                      /*zone_flags*/ zone_flags,
2938                                      /*data_ptr*/ NULL,
2939                                      /*dxfer_len*/ 0,
2940                                      /*sense_len*/ SSD_FULL_SIZE,
2941                                      /*timeout*/ da_default_timeout * 1000);
2942                 } else {
2943                         /*
2944                          * Note that in this case, even though we can
2945                          * technically use NCQ, we don't bother for several
2946                          * reasons:
2947                          * 1. It hasn't been tested on a SAT layer that
2948                          *    supports it.  This is new as of SAT-4.
2949                          * 2. Even when there is a SAT layer that supports
2950                          *    it, that SAT layer will also probably support
2951                          *    ZBC -> ZAC translation, since they are both
2952                          *    in the SAT-4 spec.
2953                          * 3. Translation will likely be preferable to ATA
2954                          *    passthrough.  LSI / Avago at least single
2955                          *    steps ATA passthrough commands in the HBA,
2956                          *    regardless of protocol, so unless that
2957                          *    changes, there is a performance penalty for
2958                          *    doing ATA passthrough no matter whether
2959                          *    you're using NCQ/FPDMA, DMA or PIO.
2960                          * 4. It requires a 32-byte CDB, which at least at
2961                          *    this point in CAM requires a CDB pointer, which
2962                          *    would require us to allocate an additional bit
2963                          *    of storage separate from the CCB.
2964                          */
2965                         error = scsi_ata_zac_mgmt_out(&ccb->csio,
2966                             /*retries*/ da_retry_count,
2967                             /*cbfcnp*/ dadone,
2968                             /*tag_action*/ MSG_SIMPLE_Q_TAG,
2969                             /*use_ncq*/ 0,
2970                             /*zm_action*/ zone_sa,
2971                             /*zone_id*/ lba,
2972                             /*zone_flags*/ zone_flags,
2973                             /*data_ptr*/ NULL,
2974                             /*dxfer_len*/ 0,
2975                             /*cdb_storage*/ NULL,
2976                             /*cdb_storage_len*/ 0,
2977                             /*sense_len*/ SSD_FULL_SIZE,
2978                             /*timeout*/ da_default_timeout * 1000);
2979                         if (error != 0) {
2980                                 error = EINVAL;
2981                                 xpt_print(periph->path,
2982                                     "scsi_ata_zac_mgmt_out() returned an "
2983                                     "error!");
2984                                 goto bailout;
2985                         }
2986                 }
2987                 *queue_ccb = 1;
2988
2989                 break;
2990         }
2991         case DISK_ZONE_REPORT_ZONES: {
2992                 uint8_t *rz_ptr;
2993                 uint32_t num_entries, alloc_size;
2994                 struct disk_zone_report *rep;
2995
2996                 rep = &bp->bio_zone.zone_params.report;
2997
2998                 num_entries = rep->entries_allocated;
2999                 if (num_entries == 0) {
3000                         xpt_print(periph->path, "No entries allocated for "
3001                             "Report Zones request\n");
3002                         error = EINVAL;
3003                         goto bailout;
3004                 }
3005                 alloc_size = sizeof(struct scsi_report_zones_hdr) +
3006                     (sizeof(struct scsi_report_zones_desc) * num_entries);
3007                 alloc_size = min(alloc_size, softc->disk->d_maxsize);
3008                 rz_ptr = malloc(alloc_size, M_SCSIDA, M_NOWAIT | M_ZERO);
3009                 if (rz_ptr == NULL) {
3010                         xpt_print(periph->path, "Unable to allocate memory "
3011                            "for Report Zones request\n");
3012                         error = ENOMEM;
3013                         goto bailout;
3014                 }
3015
3016                 if (softc->zone_interface != DA_ZONE_IF_ATA_PASS) {
3017                         scsi_zbc_in(&ccb->csio,
3018                                     /*retries*/ da_retry_count,
3019                                     /*cbcfnp*/ dadone,
3020                                     /*tag_action*/ MSG_SIMPLE_Q_TAG,
3021                                     /*service_action*/ ZBC_IN_SA_REPORT_ZONES,
3022                                     /*zone_start_lba*/ rep->starting_id,
3023                                     /*zone_options*/ rep->rep_options,
3024                                     /*data_ptr*/ rz_ptr,
3025                                     /*dxfer_len*/ alloc_size,
3026                                     /*sense_len*/ SSD_FULL_SIZE,
3027                                     /*timeout*/ da_default_timeout * 1000);
3028                 } else {
3029                         /*
3030                          * Note that in this case, even though we can
3031                          * technically use NCQ, we don't bother for several
3032                          * reasons:
3033                          * 1. It hasn't been tested on a SAT layer that
3034                          *    supports it.  This is new as of SAT-4.
3035                          * 2. Even when there is a SAT layer that supports
3036                          *    it, that SAT layer will also probably support
3037                          *    ZBC -> ZAC translation, since they are both
3038                          *    in the SAT-4 spec.
3039                          * 3. Translation will likely be preferable to ATA
3040                          *    passthrough.  LSI / Avago at least single
3041                          *    steps ATA passthrough commands in the HBA,
3042                          *    regardless of protocol, so unless that
3043                          *    changes, there is a performance penalty for
3044                          *    doing ATA passthrough no matter whether
3045                          *    you're using NCQ/FPDMA, DMA or PIO.
3046                          * 4. It requires a 32-byte CDB, which at least at
3047                          *    this point in CAM requires a CDB pointer, which
3048                          *    would require us to allocate an additional bit
3049                          *    of storage separate from the CCB.
3050                          */
3051                         error = scsi_ata_zac_mgmt_in(&ccb->csio,
3052                             /*retries*/ da_retry_count,
3053                             /*cbcfnp*/ dadone,
3054                             /*tag_action*/ MSG_SIMPLE_Q_TAG,
3055                             /*use_ncq*/ 0,
3056                             /*zm_action*/ ATA_ZM_REPORT_ZONES,
3057                             /*zone_id*/ rep->starting_id,
3058                             /*zone_flags*/ rep->rep_options,
3059                             /*data_ptr*/ rz_ptr,
3060                             /*dxfer_len*/ alloc_size,
3061                             /*cdb_storage*/ NULL,
3062                             /*cdb_storage_len*/ 0,
3063                             /*sense_len*/ SSD_FULL_SIZE,
3064                             /*timeout*/ da_default_timeout * 1000);
3065                         if (error != 0) {
3066                                 error = EINVAL;
3067                                 xpt_print(periph->path,
3068                                     "scsi_ata_zac_mgmt_in() returned an "
3069                                     "error!");
3070                                 goto bailout;
3071                         }
3072                 }
3073
3074                 /*
3075                  * For BIO_ZONE, this isn't normally needed.  However, it
3076                  * is used by devstat_end_transaction_bio() to determine
3077                  * how much data was transferred.
3078                  */
3079                 /*
3080                  * XXX KDM we have a problem.  But I'm not sure how to fix
3081                  * it.  devstat uses bio_bcount - bio_resid to calculate
3082                  * the amount of data transferred.   The GEOM disk code
3083                  * uses bio_length - bio_resid to calculate the amount of
3084                  * data in bio_completed.  We have different structure
3085                  * sizes above and below the ada(4) driver.  So, if we
3086                  * use the sizes above, the amount transferred won't be
3087                  * quite accurate for devstat.  If we use different sizes
3088                  * for bio_bcount and bio_length (above and below
3089                  * respectively), then the residual needs to match one or
3090                  * the other.  Everything is calculated after the bio
3091                  * leaves the driver, so changing the values around isn't
3092                  * really an option.  For now, just set the count to the
3093                  * passed in length.  This means that the calculations
3094                  * above (e.g. bio_completed) will be correct, but the
3095                  * amount of data reported to devstat will be slightly
3096                  * under or overstated.
3097                  */
3098                 bp->bio_bcount = bp->bio_length;
3099
3100                 *queue_ccb = 1;
3101
3102                 break;
3103         }
3104         case DISK_ZONE_GET_PARAMS: {
3105                 struct disk_zone_disk_params *params;
3106
3107                 params = &bp->bio_zone.zone_params.disk_params;
3108                 bzero(params, sizeof(*params));
3109
3110                 switch (softc->zone_mode) {
3111                 case DA_ZONE_DRIVE_MANAGED:
3112                         params->zone_mode = DISK_ZONE_MODE_DRIVE_MANAGED;
3113                         break;
3114                 case DA_ZONE_HOST_AWARE:
3115                         params->zone_mode = DISK_ZONE_MODE_HOST_AWARE;
3116                         break;
3117                 case DA_ZONE_HOST_MANAGED:
3118                         params->zone_mode = DISK_ZONE_MODE_HOST_MANAGED;
3119                         break;
3120                 default:
3121                 case DA_ZONE_NONE:
3122                         params->zone_mode = DISK_ZONE_MODE_NONE;
3123                         break;
3124                 }
3125
3126                 if (softc->zone_flags & DA_ZONE_FLAG_URSWRZ)
3127                         params->flags |= DISK_ZONE_DISK_URSWRZ;
3128
3129                 if (softc->zone_flags & DA_ZONE_FLAG_OPT_SEQ_SET) {
3130                         params->optimal_seq_zones = softc->optimal_seq_zones;
3131                         params->flags |= DISK_ZONE_OPT_SEQ_SET;
3132                 }
3133
3134                 if (softc->zone_flags & DA_ZONE_FLAG_OPT_NONSEQ_SET) {
3135                         params->optimal_nonseq_zones =
3136                             softc->optimal_nonseq_zones;
3137                         params->flags |= DISK_ZONE_OPT_NONSEQ_SET;
3138                 }
3139
3140                 if (softc->zone_flags & DA_ZONE_FLAG_MAX_SEQ_SET) {
3141                         params->max_seq_zones = softc->max_seq_zones;
3142                         params->flags |= DISK_ZONE_MAX_SEQ_SET;
3143                 }
3144                 if (softc->zone_flags & DA_ZONE_FLAG_RZ_SUP)
3145                         params->flags |= DISK_ZONE_RZ_SUP;
3146
3147                 if (softc->zone_flags & DA_ZONE_FLAG_OPEN_SUP)
3148                         params->flags |= DISK_ZONE_OPEN_SUP;
3149
3150                 if (softc->zone_flags & DA_ZONE_FLAG_CLOSE_SUP)
3151                         params->flags |= DISK_ZONE_CLOSE_SUP;
3152
3153                 if (softc->zone_flags & DA_ZONE_FLAG_FINISH_SUP)
3154                         params->flags |= DISK_ZONE_FINISH_SUP;
3155
3156                 if (softc->zone_flags & DA_ZONE_FLAG_RWP_SUP)
3157                         params->flags |= DISK_ZONE_RWP_SUP;
3158                 break;
3159         }
3160         default:
3161                 break;
3162         }
3163 bailout:
3164         return (error);
3165 }
3166
3167 static void
3168 dastart(struct cam_periph *periph, union ccb *start_ccb)
3169 {
3170         struct da_softc *softc;
3171
3172         cam_periph_assert(periph, MA_OWNED);
3173         softc = (struct da_softc *)periph->softc;
3174
3175         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dastart\n"));
3176
3177 skipstate:
3178         switch (softc->state) {
3179         case DA_STATE_NORMAL:
3180         {
3181                 struct bio *bp;
3182                 uint8_t tag_code;
3183
3184 more:
3185                 bp = cam_iosched_next_bio(softc->cam_iosched);
3186                 if (bp == NULL) {
3187                         if (cam_iosched_has_work_flags(softc->cam_iosched,
3188                             DA_WORK_TUR)) {
3189                                 softc->flags |= DA_FLAG_TUR_PENDING;
3190                                 cam_iosched_clr_work_flags(softc->cam_iosched,
3191                                     DA_WORK_TUR);
3192                                 scsi_test_unit_ready(&start_ccb->csio,
3193                                      /*retries*/ da_retry_count,
3194                                      dadone_tur,
3195                                      MSG_SIMPLE_Q_TAG,
3196                                      SSD_FULL_SIZE,
3197                                      da_default_timeout * 1000);
3198                                 start_ccb->ccb_h.ccb_bp = NULL;
3199                                 start_ccb->ccb_h.ccb_state = DA_CCB_TUR;
3200                                 xpt_action(start_ccb);
3201                         } else
3202                                 xpt_release_ccb(start_ccb);
3203                         break;
3204                 }
3205
3206                 if (bp->bio_cmd == BIO_DELETE) {
3207                         if (softc->delete_func != NULL) {
3208                                 softc->delete_func(periph, start_ccb, bp);
3209                                 goto out;
3210                         } else {
3211                                 /*
3212                                  * Not sure this is possible, but failsafe by
3213                                  * lying and saying "sure, done."
3214                                  */
3215                                 biofinish(bp, NULL, 0);
3216                                 goto more;
3217                         }
3218                 }
3219
3220                 if (cam_iosched_has_work_flags(softc->cam_iosched,
3221                     DA_WORK_TUR)) {
3222                         cam_iosched_clr_work_flags(softc->cam_iosched,
3223                             DA_WORK_TUR);
3224                         da_periph_release_locked(periph, DA_REF_TUR);
3225                 }
3226
3227                 if ((bp->bio_flags & BIO_ORDERED) != 0 ||
3228                     (softc->flags & DA_FLAG_NEED_OTAG) != 0) {
3229                         softc->flags &= ~DA_FLAG_NEED_OTAG;
3230                         softc->flags |= DA_FLAG_WAS_OTAG;
3231                         tag_code = MSG_ORDERED_Q_TAG;
3232                 } else {
3233                         tag_code = MSG_SIMPLE_Q_TAG;
3234                 }
3235
3236                 switch (bp->bio_cmd) {
3237                 case BIO_WRITE:
3238                 case BIO_READ:
3239                 {
3240                         void *data_ptr;
3241                         int rw_op;
3242
3243                         biotrack(bp, __func__);
3244
3245                         if (bp->bio_cmd == BIO_WRITE) {
3246                                 softc->flags |= DA_FLAG_DIRTY;
3247                                 rw_op = SCSI_RW_WRITE;
3248                         } else {
3249                                 rw_op = SCSI_RW_READ;
3250                         }
3251
3252                         data_ptr = bp->bio_data;
3253                         if ((bp->bio_flags & (BIO_UNMAPPED|BIO_VLIST)) != 0) {
3254                                 rw_op |= SCSI_RW_BIO;
3255                                 data_ptr = bp;
3256                         }
3257
3258                         scsi_read_write(&start_ccb->csio,
3259                                         /*retries*/da_retry_count,
3260                                         /*cbfcnp*/dadone,
3261                                         /*tag_action*/tag_code,
3262                                         rw_op,
3263                                         /*byte2*/0,
3264                                         softc->minimum_cmd_size,
3265                                         /*lba*/bp->bio_pblkno,
3266                                         /*block_count*/bp->bio_bcount /
3267                                         softc->params.secsize,
3268                                         data_ptr,
3269                                         /*dxfer_len*/ bp->bio_bcount,
3270                                         /*sense_len*/SSD_FULL_SIZE,
3271                                         da_default_timeout * 1000);
3272 #if defined(BUF_TRACKING) || defined(FULL_BUF_TRACKING)
3273                         start_ccb->csio.bio = bp;
3274 #endif
3275                         break;
3276                 }
3277                 case BIO_FLUSH:
3278                         /*
3279                          * If we don't support sync cache, or the disk
3280                          * isn't dirty, FLUSH is a no-op.  Use the
3281                          * allocated CCB for the next bio if one is
3282                          * available.
3283                          */
3284                         if ((softc->quirks & DA_Q_NO_SYNC_CACHE) != 0 ||
3285                             (softc->flags & DA_FLAG_DIRTY) == 0) {
3286                                 biodone(bp);
3287                                 goto skipstate;
3288                         }
3289
3290                         /*
3291                          * BIO_FLUSH doesn't currently communicate
3292                          * range data, so we synchronize the cache
3293                          * over the whole disk.  We also force
3294                          * ordered tag semantics the flush applies
3295                          * to all previously queued I/O.
3296                          */
3297                         scsi_synchronize_cache(&start_ccb->csio,
3298                                                /*retries*/1,
3299                                                /*cbfcnp*/dadone,
3300                                                MSG_ORDERED_Q_TAG,
3301                                                /*begin_lba*/0,
3302                                                /*lb_count*/0,
3303                                                SSD_FULL_SIZE,
3304                                                da_default_timeout*1000);
3305                         /*
3306                          * Clear the dirty flag before sending the command.
3307                          * Either this sync cache will be successful, or it
3308                          * will fail after a retry.  If it fails, it is
3309                          * unlikely to be successful if retried later, so
3310                          * we'll save ourselves time by just marking the
3311                          * device clean.
3312                          */
3313                         softc->flags &= ~DA_FLAG_DIRTY;
3314                         break;
3315                 case BIO_ZONE: {
3316                         int error, queue_ccb;
3317
3318                         queue_ccb = 0;
3319
3320                         error = da_zone_cmd(periph, start_ccb, bp,&queue_ccb);
3321                         if ((error != 0)
3322                          || (queue_ccb == 0)) {
3323                                 biofinish(bp, NULL, error);
3324                                 xpt_release_ccb(start_ccb);
3325                                 return;
3326                         }
3327                         break;
3328                 }
3329                 }
3330                 start_ccb->ccb_h.ccb_state = DA_CCB_BUFFER_IO;
3331                 start_ccb->ccb_h.flags |= CAM_UNLOCKED;
3332                 start_ccb->ccb_h.softtimeout = sbttotv(da_default_softtimeout);
3333
3334 out:
3335                 LIST_INSERT_HEAD(&softc->pending_ccbs,
3336                                  &start_ccb->ccb_h, periph_links.le);
3337
3338                 /* We expect a unit attention from this device */
3339                 if ((softc->flags & DA_FLAG_RETRY_UA) != 0) {
3340                         start_ccb->ccb_h.ccb_state |= DA_CCB_RETRY_UA;
3341                         softc->flags &= ~DA_FLAG_RETRY_UA;
3342                 }
3343
3344                 start_ccb->ccb_h.ccb_bp = bp;
3345                 softc->refcount++;
3346                 cam_periph_unlock(periph);
3347                 xpt_action(start_ccb);
3348                 cam_periph_lock(periph);
3349
3350                 /* May have more work to do, so ensure we stay scheduled */
3351                 daschedule(periph);
3352                 break;
3353         }
3354         case DA_STATE_PROBE_WP:
3355         {
3356                 void  *mode_buf;
3357                 int    mode_buf_len;
3358
3359                 if (da_disable_wp_detection) {
3360                         if ((softc->flags & DA_FLAG_CAN_RC16) != 0)
3361                                 softc->state = DA_STATE_PROBE_RC16;
3362                         else
3363                                 softc->state = DA_STATE_PROBE_RC;
3364                         goto skipstate;
3365                 }
3366                 mode_buf_len = 192;
3367                 mode_buf = malloc(mode_buf_len, M_SCSIDA, M_NOWAIT);
3368                 if (mode_buf == NULL) {
3369                         xpt_print(periph->path, "Unable to send mode sense - "
3370                             "malloc failure\n");
3371                         if ((softc->flags & DA_FLAG_CAN_RC16) != 0)
3372                                 softc->state = DA_STATE_PROBE_RC16;
3373                         else
3374                                 softc->state = DA_STATE_PROBE_RC;
3375                         goto skipstate;
3376                 }
3377                 scsi_mode_sense_len(&start_ccb->csio,
3378                                     /*retries*/ da_retry_count,
3379                                     /*cbfcnp*/ dadone_probewp,
3380                                     /*tag_action*/ MSG_SIMPLE_Q_TAG,
3381                                     /*dbd*/ FALSE,
3382                                     /*pc*/ SMS_PAGE_CTRL_CURRENT,
3383                                     /*page*/ SMS_ALL_PAGES_PAGE,
3384                                     /*param_buf*/ mode_buf,
3385                                     /*param_len*/ mode_buf_len,
3386                                     /*minimum_cmd_size*/ softc->minimum_cmd_size,
3387                                     /*sense_len*/ SSD_FULL_SIZE,
3388                                     /*timeout*/ da_default_timeout * 1000);
3389                 start_ccb->ccb_h.ccb_bp = NULL;
3390                 start_ccb->ccb_h.ccb_state = DA_CCB_PROBE_WP;
3391                 xpt_action(start_ccb);
3392                 break;
3393         }
3394         case DA_STATE_PROBE_RC:
3395         {
3396                 struct scsi_read_capacity_data *rcap;
3397
3398                 rcap = (struct scsi_read_capacity_data *)
3399                     malloc(sizeof(*rcap), M_SCSIDA, M_NOWAIT|M_ZERO);
3400                 if (rcap == NULL) {
3401                         printf("dastart: Couldn't malloc read_capacity data\n");
3402                         /* da_free_periph??? */
3403                         break;
3404                 }
3405                 scsi_read_capacity(&start_ccb->csio,
3406                                    /*retries*/da_retry_count,
3407                                    dadone_proberc,
3408                                    MSG_SIMPLE_Q_TAG,
3409                                    rcap,
3410                                    SSD_FULL_SIZE,
3411                                    /*timeout*/5000);
3412                 start_ccb->ccb_h.ccb_bp = NULL;
3413                 start_ccb->ccb_h.ccb_state = DA_CCB_PROBE_RC;
3414                 xpt_action(start_ccb);
3415                 break;
3416         }
3417         case DA_STATE_PROBE_RC16:
3418         {
3419                 struct scsi_read_capacity_data_long *rcaplong;
3420
3421                 rcaplong = (struct scsi_read_capacity_data_long *)
3422                         malloc(sizeof(*rcaplong), M_SCSIDA, M_NOWAIT|M_ZERO);
3423                 if (rcaplong == NULL) {
3424                         printf("dastart: Couldn't malloc read_capacity data\n");
3425                         /* da_free_periph??? */
3426                         break;
3427                 }
3428                 scsi_read_capacity_16(&start_ccb->csio,
3429                                       /*retries*/ da_retry_count,
3430                                       /*cbfcnp*/ dadone_proberc,
3431                                       /*tag_action*/ MSG_SIMPLE_Q_TAG,
3432                                       /*lba*/ 0,
3433                                       /*reladr*/ 0,
3434                                       /*pmi*/ 0,
3435                                       /*rcap_buf*/ (uint8_t *)rcaplong,
3436                                       /*rcap_buf_len*/ sizeof(*rcaplong),
3437                                       /*sense_len*/ SSD_FULL_SIZE,
3438                                       /*timeout*/ da_default_timeout * 1000);
3439                 start_ccb->ccb_h.ccb_bp = NULL;
3440                 start_ccb->ccb_h.ccb_state = DA_CCB_PROBE_RC16;
3441                 xpt_action(start_ccb);
3442                 break;
3443         }
3444         case DA_STATE_PROBE_LBP:
3445         {
3446                 struct scsi_vpd_logical_block_prov *lbp;
3447
3448                 if (!scsi_vpd_supported_page(periph, SVPD_LBP)) {
3449                         /*
3450                          * If we get here we don't support any SBC-3 delete
3451                          * methods with UNMAP as the Logical Block Provisioning
3452                          * VPD page support is required for devices which
3453                          * support it according to T10/1799-D Revision 31
3454                          * however older revisions of the spec don't mandate
3455                          * this so we currently don't remove these methods
3456                          * from the available set.
3457                          */
3458                         softc->state = DA_STATE_PROBE_BLK_LIMITS;
3459                         goto skipstate;
3460                 }
3461
3462                 lbp = (struct scsi_vpd_logical_block_prov *)
3463                         malloc(sizeof(*lbp), M_SCSIDA, M_NOWAIT|M_ZERO);
3464
3465                 if (lbp == NULL) {
3466                         printf("dastart: Couldn't malloc lbp data\n");
3467                         /* da_free_periph??? */
3468                         break;
3469                 }
3470
3471                 scsi_inquiry(&start_ccb->csio,
3472                              /*retries*/da_retry_count,
3473                              /*cbfcnp*/dadone_probelbp,
3474                              /*tag_action*/MSG_SIMPLE_Q_TAG,
3475                              /*inq_buf*/(u_int8_t *)lbp,
3476                              /*inq_len*/sizeof(*lbp),
3477                              /*evpd*/TRUE,
3478                              /*page_code*/SVPD_LBP,
3479                              /*sense_len*/SSD_MIN_SIZE,
3480                              /*timeout*/da_default_timeout * 1000);
3481                 start_ccb->ccb_h.ccb_bp = NULL;
3482                 start_ccb->ccb_h.ccb_state = DA_CCB_PROBE_LBP;
3483                 xpt_action(start_ccb);
3484                 break;
3485         }
3486         case DA_STATE_PROBE_BLK_LIMITS:
3487         {
3488                 struct scsi_vpd_block_limits *block_limits;
3489
3490                 if (!scsi_vpd_supported_page(periph, SVPD_BLOCK_LIMITS)) {
3491                         /* Not supported skip to next probe */
3492                         softc->state = DA_STATE_PROBE_BDC;
3493                         goto skipstate;
3494                 }
3495
3496                 block_limits = (struct scsi_vpd_block_limits *)
3497                         malloc(sizeof(*block_limits), M_SCSIDA, M_NOWAIT|M_ZERO);
3498
3499                 if (block_limits == NULL) {
3500                         printf("dastart: Couldn't malloc block_limits data\n");
3501                         /* da_free_periph??? */
3502                         break;
3503                 }
3504
3505                 scsi_inquiry(&start_ccb->csio,
3506                              /*retries*/da_retry_count,
3507                              /*cbfcnp*/dadone_probeblklimits,
3508                              /*tag_action*/MSG_SIMPLE_Q_TAG,
3509                              /*inq_buf*/(u_int8_t *)block_limits,
3510                              /*inq_len*/sizeof(*block_limits),
3511                              /*evpd*/TRUE,
3512                              /*page_code*/SVPD_BLOCK_LIMITS,
3513                              /*sense_len*/SSD_MIN_SIZE,
3514                              /*timeout*/da_default_timeout * 1000);
3515                 start_ccb->ccb_h.ccb_bp = NULL;
3516                 start_ccb->ccb_h.ccb_state = DA_CCB_PROBE_BLK_LIMITS;
3517                 xpt_action(start_ccb);
3518                 break;
3519         }
3520         case DA_STATE_PROBE_BDC:
3521         {
3522                 struct scsi_vpd_block_characteristics *bdc;
3523
3524                 if (!scsi_vpd_supported_page(periph, SVPD_BDC)) {
3525                         softc->state = DA_STATE_PROBE_ATA;
3526                         goto skipstate;
3527                 }
3528
3529                 bdc = (struct scsi_vpd_block_characteristics *)
3530                         malloc(sizeof(*bdc), M_SCSIDA, M_NOWAIT|M_ZERO);
3531
3532                 if (bdc == NULL) {
3533                         printf("dastart: Couldn't malloc bdc data\n");
3534                         /* da_free_periph??? */
3535                         break;
3536                 }
3537
3538                 scsi_inquiry(&start_ccb->csio,
3539                              /*retries*/da_retry_count,
3540                              /*cbfcnp*/dadone_probebdc,
3541                              /*tag_action*/MSG_SIMPLE_Q_TAG,
3542                              /*inq_buf*/(u_int8_t *)bdc,
3543                              /*inq_len*/sizeof(*bdc),
3544                              /*evpd*/TRUE,
3545                              /*page_code*/SVPD_BDC,
3546                              /*sense_len*/SSD_MIN_SIZE,
3547                              /*timeout*/da_default_timeout * 1000);
3548                 start_ccb->ccb_h.ccb_bp = NULL;
3549                 start_ccb->ccb_h.ccb_state = DA_CCB_PROBE_BDC;
3550                 xpt_action(start_ccb);
3551                 break;
3552         }
3553         case DA_STATE_PROBE_ATA:
3554         {
3555                 struct ata_params *ata_params;
3556
3557                 if (!scsi_vpd_supported_page(periph, SVPD_ATA_INFORMATION)) {
3558                         if ((softc->zone_mode == DA_ZONE_HOST_AWARE)
3559                          || (softc->zone_mode == DA_ZONE_HOST_MANAGED)) {
3560                                 /*
3561                                  * Note that if the ATA VPD page isn't
3562                                  * supported, we aren't talking to an ATA
3563                                  * device anyway.  Support for that VPD
3564                                  * page is mandatory for SCSI to ATA (SAT)
3565                                  * translation layers.
3566                                  */
3567                                 softc->state = DA_STATE_PROBE_ZONE;
3568                                 goto skipstate;
3569                         }
3570                         daprobedone(periph, start_ccb);
3571                         break;
3572                 }
3573
3574                 ata_params = (struct ata_params*)
3575                         malloc(sizeof(*ata_params), M_SCSIDA,M_NOWAIT|M_ZERO);
3576
3577                 if (ata_params == NULL) {
3578                         xpt_print(periph->path, "Couldn't malloc ata_params "
3579                             "data\n");
3580                         /* da_free_periph??? */
3581                         break;
3582                 }
3583
3584                 scsi_ata_identify(&start_ccb->csio,
3585                                   /*retries*/da_retry_count,
3586                                   /*cbfcnp*/dadone_probeata,
3587                                   /*tag_action*/MSG_SIMPLE_Q_TAG,
3588                                   /*data_ptr*/(u_int8_t *)ata_params,
3589                                   /*dxfer_len*/sizeof(*ata_params),
3590                                   /*sense_len*/SSD_FULL_SIZE,
3591                                   /*timeout*/da_default_timeout * 1000);
3592                 start_ccb->ccb_h.ccb_bp = NULL;
3593                 start_ccb->ccb_h.ccb_state = DA_CCB_PROBE_ATA;
3594                 xpt_action(start_ccb);
3595                 break;
3596         }
3597         case DA_STATE_PROBE_ATA_LOGDIR:
3598         {
3599                 struct ata_gp_log_dir *log_dir;
3600                 int retval;
3601
3602                 retval = 0;
3603
3604                 if ((softc->flags & DA_FLAG_CAN_ATA_LOG) == 0) {
3605                         /*
3606                          * If we don't have log support, not much point in
3607                          * trying to probe zone support.
3608                          */
3609                         daprobedone(periph, start_ccb);
3610                         break;
3611                 }
3612
3613                 /*
3614                  * If we have an ATA device (the SCSI ATA Information VPD
3615                  * page should be present and the ATA identify should have
3616                  * succeeded) and it supports logs, ask for the log directory.
3617                  */
3618
3619                 log_dir = malloc(sizeof(*log_dir), M_SCSIDA, M_NOWAIT|M_ZERO);
3620                 if (log_dir == NULL) {
3621                         xpt_print(periph->path, "Couldn't malloc log_dir "
3622                             "data\n");
3623                         daprobedone(periph, start_ccb);
3624                         break;
3625                 }
3626
3627                 retval = scsi_ata_read_log(&start_ccb->csio,
3628                     /*retries*/ da_retry_count,
3629                     /*cbfcnp*/ dadone_probeatalogdir,
3630                     /*tag_action*/ MSG_SIMPLE_Q_TAG,
3631                     /*log_address*/ ATA_LOG_DIRECTORY,
3632                     /*page_number*/ 0,
3633                     /*block_count*/ 1,
3634                     /*protocol*/ softc->flags & DA_FLAG_CAN_ATA_DMA ?
3635                                  AP_PROTO_DMA : AP_PROTO_PIO_IN,
3636                     /*data_ptr*/ (uint8_t *)log_dir,
3637                     /*dxfer_len*/ sizeof(*log_dir),
3638                     /*sense_len*/ SSD_FULL_SIZE,
3639                     /*timeout*/ da_default_timeout * 1000);
3640
3641                 if (retval != 0) {
3642                         xpt_print(periph->path, "scsi_ata_read_log() failed!");
3643                         free(log_dir, M_SCSIDA);
3644                         daprobedone(periph, start_ccb);
3645                         break;
3646                 }
3647                 start_ccb->ccb_h.ccb_bp = NULL;
3648                 start_ccb->ccb_h.ccb_state = DA_CCB_PROBE_ATA_LOGDIR;
3649                 xpt_action(start_ccb);
3650                 break;
3651         }
3652         case DA_STATE_PROBE_ATA_IDDIR:
3653         {
3654                 struct ata_identify_log_pages *id_dir;
3655                 int retval;
3656
3657                 retval = 0;
3658
3659                 /*
3660                  * Check here to see whether the Identify Device log is
3661                  * supported in the directory of logs.  If so, continue
3662                  * with requesting the log of identify device pages.
3663                  */
3664                 if ((softc->flags & DA_FLAG_CAN_ATA_IDLOG) == 0) {
3665                         daprobedone(periph, start_ccb);
3666                         break;
3667                 }
3668
3669                 id_dir = malloc(sizeof(*id_dir), M_SCSIDA, M_NOWAIT | M_ZERO);
3670                 if (id_dir == NULL) {
3671                         xpt_print(periph->path, "Couldn't malloc id_dir "
3672                             "data\n");
3673                         daprobedone(periph, start_ccb);
3674                         break;
3675                 }
3676
3677                 retval = scsi_ata_read_log(&start_ccb->csio,
3678                     /*retries*/ da_retry_count,
3679                     /*cbfcnp*/ dadone_probeataiddir,
3680                     /*tag_action*/ MSG_SIMPLE_Q_TAG,
3681                     /*log_address*/ ATA_IDENTIFY_DATA_LOG,
3682                     /*page_number*/ ATA_IDL_PAGE_LIST,
3683                     /*block_count*/ 1,
3684                     /*protocol*/ softc->flags & DA_FLAG_CAN_ATA_DMA ?
3685                                  AP_PROTO_DMA : AP_PROTO_PIO_IN,
3686                     /*data_ptr*/ (uint8_t *)id_dir,
3687                     /*dxfer_len*/ sizeof(*id_dir),
3688                     /*sense_len*/ SSD_FULL_SIZE,
3689                     /*timeout*/ da_default_timeout * 1000);
3690
3691                 if (retval != 0) {
3692                         xpt_print(periph->path, "scsi_ata_read_log() failed!");
3693                         free(id_dir, M_SCSIDA);
3694                         daprobedone(periph, start_ccb);
3695                         break;
3696                 }
3697                 start_ccb->ccb_h.ccb_bp = NULL;
3698                 start_ccb->ccb_h.ccb_state = DA_CCB_PROBE_ATA_IDDIR;
3699                 xpt_action(start_ccb);
3700                 break;
3701         }
3702         case DA_STATE_PROBE_ATA_SUP:
3703         {
3704                 struct ata_identify_log_sup_cap *sup_cap;
3705                 int retval;
3706
3707                 retval = 0;
3708
3709                 /*
3710                  * Check here to see whether the Supported Capabilities log
3711                  * is in the list of Identify Device logs.
3712                  */
3713                 if ((softc->flags & DA_FLAG_CAN_ATA_SUPCAP) == 0) {
3714                         daprobedone(periph, start_ccb);
3715                         break;
3716                 }
3717
3718                 sup_cap = malloc(sizeof(*sup_cap), M_SCSIDA, M_NOWAIT|M_ZERO);
3719                 if (sup_cap == NULL) {
3720                         xpt_print(periph->path, "Couldn't malloc sup_cap "
3721                             "data\n");
3722                         daprobedone(periph, start_ccb);
3723                         break;
3724                 }
3725
3726                 retval = scsi_ata_read_log(&start_ccb->csio,
3727                     /*retries*/ da_retry_count,
3728                     /*cbfcnp*/ dadone_probeatasup,
3729                     /*tag_action*/ MSG_SIMPLE_Q_TAG,
3730                     /*log_address*/ ATA_IDENTIFY_DATA_LOG,
3731                     /*page_number*/ ATA_IDL_SUP_CAP,
3732                     /*block_count*/ 1,
3733                     /*protocol*/ softc->flags & DA_FLAG_CAN_ATA_DMA ?
3734                                  AP_PROTO_DMA : AP_PROTO_PIO_IN,
3735                     /*data_ptr*/ (uint8_t *)sup_cap,
3736                     /*dxfer_len*/ sizeof(*sup_cap),
3737                     /*sense_len*/ SSD_FULL_SIZE,
3738                     /*timeout*/ da_default_timeout * 1000);
3739
3740                 if (retval != 0) {
3741                         xpt_print(periph->path, "scsi_ata_read_log() failed!");
3742                         free(sup_cap, M_SCSIDA);
3743                         daprobedone(periph, start_ccb);
3744                         break;
3745
3746                 }
3747
3748                 start_ccb->ccb_h.ccb_bp = NULL;
3749                 start_ccb->ccb_h.ccb_state = DA_CCB_PROBE_ATA_SUP;
3750                 xpt_action(start_ccb);
3751                 break;
3752         }
3753         case DA_STATE_PROBE_ATA_ZONE:
3754         {
3755                 struct ata_zoned_info_log *ata_zone;
3756                 int retval;
3757
3758                 retval = 0;
3759
3760                 /*
3761                  * Check here to see whether the zoned device information
3762                  * page is supported.  If so, continue on to request it.
3763                  * If not, skip to DA_STATE_PROBE_LOG or done.
3764                  */
3765                 if ((softc->flags & DA_FLAG_CAN_ATA_ZONE) == 0) {
3766                         daprobedone(periph, start_ccb);
3767                         break;
3768                 }
3769                 ata_zone = malloc(sizeof(*ata_zone), M_SCSIDA,
3770                                   M_NOWAIT|M_ZERO);
3771                 if (ata_zone == NULL) {
3772                         xpt_print(periph->path, "Couldn't malloc ata_zone "
3773                             "data\n");
3774                         daprobedone(periph, start_ccb);
3775                         break;
3776                 }
3777
3778                 retval = scsi_ata_read_log(&start_ccb->csio,
3779                     /*retries*/ da_retry_count,
3780                     /*cbfcnp*/ dadone_probeatazone,
3781                     /*tag_action*/ MSG_SIMPLE_Q_TAG,
3782                     /*log_address*/ ATA_IDENTIFY_DATA_LOG,
3783                     /*page_number*/ ATA_IDL_ZDI,
3784                     /*block_count*/ 1,
3785                     /*protocol*/ softc->flags & DA_FLAG_CAN_ATA_DMA ?
3786                                  AP_PROTO_DMA : AP_PROTO_PIO_IN,
3787                     /*data_ptr*/ (uint8_t *)ata_zone,
3788                     /*dxfer_len*/ sizeof(*ata_zone),
3789                     /*sense_len*/ SSD_FULL_SIZE,
3790                     /*timeout*/ da_default_timeout * 1000);
3791
3792                 if (retval != 0) {
3793                         xpt_print(periph->path, "scsi_ata_read_log() failed!");
3794                         free(ata_zone, M_SCSIDA);
3795                         daprobedone(periph, start_ccb);
3796                         break;
3797                 }
3798                 start_ccb->ccb_h.ccb_bp = NULL;
3799                 start_ccb->ccb_h.ccb_state = DA_CCB_PROBE_ATA_ZONE;
3800                 xpt_action(start_ccb);
3801
3802                 break;
3803         }
3804         case DA_STATE_PROBE_ZONE:
3805         {
3806                 struct scsi_vpd_zoned_bdc *bdc;
3807
3808                 /*
3809                  * Note that this page will be supported for SCSI protocol
3810                  * devices that support ZBC (SMR devices), as well as ATA
3811                  * protocol devices that are behind a SAT (SCSI to ATA
3812                  * Translation) layer that supports converting ZBC commands
3813                  * to their ZAC equivalents.
3814                  */
3815                 if (!scsi_vpd_supported_page(periph, SVPD_ZONED_BDC)) {
3816                         daprobedone(periph, start_ccb);
3817                         break;
3818                 }
3819                 bdc = (struct scsi_vpd_zoned_bdc *)
3820                         malloc(sizeof(*bdc), M_SCSIDA, M_NOWAIT|M_ZERO);
3821
3822                 if (bdc == NULL) {
3823                         xpt_release_ccb(start_ccb);
3824                         xpt_print(periph->path, "Couldn't malloc zone VPD "
3825                             "data\n");
3826                         break;
3827                 }
3828                 scsi_inquiry(&start_ccb->csio,
3829                              /*retries*/da_retry_count,
3830                              /*cbfcnp*/dadone_probezone,
3831                              /*tag_action*/MSG_SIMPLE_Q_TAG,
3832                              /*inq_buf*/(u_int8_t *)bdc,
3833                              /*inq_len*/sizeof(*bdc),
3834                              /*evpd*/TRUE,
3835                              /*page_code*/SVPD_ZONED_BDC,
3836                              /*sense_len*/SSD_FULL_SIZE,
3837                              /*timeout*/da_default_timeout * 1000);
3838                 start_ccb->ccb_h.ccb_bp = NULL;
3839                 start_ccb->ccb_h.ccb_state = DA_CCB_PROBE_ZONE;
3840                 xpt_action(start_ccb);
3841                 break;
3842         }
3843         }
3844 }
3845
3846 /*
3847  * In each of the methods below, while its the caller's
3848  * responsibility to ensure the request will fit into a
3849  * single device request, we might have changed the delete
3850  * method due to the device incorrectly advertising either
3851  * its supported methods or limits.
3852  *
3853  * To prevent this causing further issues we validate the
3854  * against the methods limits, and warn which would
3855  * otherwise be unnecessary.
3856  */
3857 static void
3858 da_delete_unmap(struct cam_periph *periph, union ccb *ccb, struct bio *bp)
3859 {
3860         struct da_softc *softc = (struct da_softc *)periph->softc;;
3861         struct bio *bp1;
3862         uint8_t *buf = softc->unmap_buf;
3863         struct scsi_unmap_desc *d = (void *)&buf[UNMAP_HEAD_SIZE];
3864         uint64_t lba, lastlba = (uint64_t)-1;
3865         uint64_t totalcount = 0;
3866         uint64_t count;
3867         uint32_t c, lastcount = 0, ranges = 0;
3868
3869         /*
3870          * Currently this doesn't take the UNMAP
3871          * Granularity and Granularity Alignment
3872          * fields into account.
3873          *
3874          * This could result in both unoptimal unmap
3875          * requests as as well as UNMAP calls unmapping
3876          * fewer LBA's than requested.
3877          */
3878
3879         bzero(softc->unmap_buf, sizeof(softc->unmap_buf));
3880         bp1 = bp;
3881         do {
3882                 /*
3883                  * Note: ada and da are different in how they store the
3884                  * pending bp's in a trim. ada stores all of them in the
3885                  * trim_req.bps. da stores all but the first one in the
3886                  * delete_run_queue. ada then completes all the bps in
3887                  * its adadone() loop. da completes all the bps in the
3888                  * delete_run_queue in dadone, and relies on the biodone
3889                  * after to complete. This should be reconciled since there's
3890                  * no real reason to do it differently. XXX
3891                  */
3892                 if (bp1 != bp)
3893                         bioq_insert_tail(&softc->delete_run_queue, bp1);
3894                 lba = bp1->bio_pblkno;
3895                 count = bp1->bio_bcount / softc->params.secsize;
3896
3897                 /* Try to extend the previous range. */
3898                 if (lba == lastlba) {
3899                         c = omin(count, UNMAP_RANGE_MAX - lastcount);
3900                         lastlba += c;
3901                         lastcount += c;
3902                         scsi_ulto4b(lastcount, d[ranges - 1].length);
3903                         count -= c;
3904                         lba += c;
3905                         totalcount += c;
3906                 } else if ((softc->quirks & DA_Q_STRICT_UNMAP) &&
3907                     softc->unmap_gran != 0) {
3908                         /* Align length of the previous range. */
3909                         if ((c = lastcount % softc->unmap_gran) != 0) {
3910                                 if (lastcount <= c) {
3911                                         totalcount -= lastcount;
3912                                         lastlba = (uint64_t)-1;
3913                                         lastcount = 0;
3914                                         ranges--;
3915                                 } else {
3916                                         totalcount -= c;
3917                                         lastlba -= c;
3918                                         lastcount -= c;
3919                                         scsi_ulto4b(lastcount,
3920                                             d[ranges - 1].length);
3921                                 }
3922                         }
3923                         /* Align beginning of the new range. */
3924                         c = (lba - softc->unmap_gran_align) % softc->unmap_gran;
3925                         if (c != 0) {
3926                                 c = softc->unmap_gran - c;
3927                                 if (count <= c) {
3928                                         count = 0;
3929                                 } else {
3930                                         lba += c;
3931                                         count -= c;
3932                                 }
3933                         }
3934                 }
3935
3936                 while (count > 0) {
3937                         c = omin(count, UNMAP_RANGE_MAX);
3938                         if (totalcount + c > softc->unmap_max_lba ||
3939                             ranges >= softc->unmap_max_ranges) {
3940                                 xpt_print(periph->path,
3941                                     "%s issuing short delete %ld > %ld"
3942                                     "|| %d >= %d",
3943                                     da_delete_method_desc[softc->delete_method],
3944                                     totalcount + c, softc->unmap_max_lba,
3945                                     ranges, softc->unmap_max_ranges);
3946                                 break;
3947                         }
3948                         scsi_u64to8b(lba, d[ranges].lba);
3949                         scsi_ulto4b(c, d[ranges].length);
3950                         lba += c;
3951                         totalcount += c;
3952                         ranges++;
3953                         count -= c;
3954                         lastlba = lba;
3955                         lastcount = c;
3956                 }
3957                 bp1 = cam_iosched_next_trim(softc->cam_iosched);
3958                 if (bp1 == NULL)
3959                         break;
3960                 if (ranges >= softc->unmap_max_ranges ||
3961                     totalcount + bp1->bio_bcount /
3962                     softc->params.secsize > softc->unmap_max_lba) {
3963                         cam_iosched_put_back_trim(softc->cam_iosched, bp1);
3964                         break;
3965                 }
3966         } while (1);
3967
3968         /* Align length of the last range. */
3969         if ((softc->quirks & DA_Q_STRICT_UNMAP) && softc->unmap_gran != 0 &&
3970             (c = lastcount % softc->unmap_gran) != 0) {
3971                 if (lastcount <= c)
3972                         ranges--;
3973                 else
3974                         scsi_ulto4b(lastcount - c, d[ranges - 1].length);
3975         }
3976
3977         scsi_ulto2b(ranges * 16 + 6, &buf[0]);
3978         scsi_ulto2b(ranges * 16, &buf[2]);
3979
3980         scsi_unmap(&ccb->csio,
3981                    /*retries*/da_retry_count,
3982                    /*cbfcnp*/dadone,
3983                    /*tag_action*/MSG_SIMPLE_Q_TAG,
3984                    /*byte2*/0,
3985                    /*data_ptr*/ buf,
3986                    /*dxfer_len*/ ranges * 16 + 8,
3987                    /*sense_len*/SSD_FULL_SIZE,
3988                    da_default_timeout * 1000);
3989         ccb->ccb_h.ccb_state = DA_CCB_DELETE;
3990         ccb->ccb_h.flags |= CAM_UNLOCKED;
3991         softc->trim_count++;
3992         softc->trim_ranges += ranges;
3993         softc->trim_lbas += totalcount;
3994         cam_iosched_submit_trim(softc->cam_iosched);
3995 }
3996
3997 static void
3998 da_delete_trim(struct cam_periph *periph, union ccb *ccb, struct bio *bp)
3999 {
4000         struct da_softc *softc = (struct da_softc *)periph->softc;
4001         struct bio *bp1;
4002         uint8_t *buf = softc->unmap_buf;
4003         uint64_t lastlba = (uint64_t)-1;
4004         uint64_t count;
4005         uint64_t lba;
4006         uint32_t lastcount = 0, c, requestcount;
4007         int ranges = 0, off, block_count;
4008
4009         bzero(softc->unmap_buf, sizeof(softc->unmap_buf));
4010         bp1 = bp;
4011         do {
4012                 if (bp1 != bp)//XXX imp XXX
4013                         bioq_insert_tail(&softc->delete_run_queue, bp1);
4014                 lba = bp1->bio_pblkno;
4015                 count = bp1->bio_bcount / softc->params.secsize;
4016                 requestcount = count;
4017
4018                 /* Try to extend the previous range. */
4019                 if (lba == lastlba) {
4020                         c = omin(count, ATA_DSM_RANGE_MAX - lastcount);
4021                         lastcount += c;
4022                         off = (ranges - 1) * 8;
4023                         buf[off + 6] = lastcount & 0xff;
4024                         buf[off + 7] = (lastcount >> 8) & 0xff;
4025                         count -= c;
4026                         lba += c;
4027                 }
4028
4029                 while (count > 0) {
4030                         c = omin(count, ATA_DSM_RANGE_MAX);
4031                         off = ranges * 8;
4032
4033                         buf[off + 0] = lba & 0xff;
4034                         buf[off + 1] = (lba >> 8) & 0xff;
4035                         buf[off + 2] = (lba >> 16) & 0xff;
4036                         buf[off + 3] = (lba >> 24) & 0xff;
4037                         buf[off + 4] = (lba >> 32) & 0xff;
4038                         buf[off + 5] = (lba >> 40) & 0xff;
4039                         buf[off + 6] = c & 0xff;
4040                         buf[off + 7] = (c >> 8) & 0xff;
4041                         lba += c;
4042                         ranges++;
4043                         count -= c;
4044                         lastcount = c;
4045                         if (count != 0 && ranges == softc->trim_max_ranges) {
4046                                 xpt_print(periph->path,
4047                                     "%s issuing short delete %ld > %ld\n",
4048                                     da_delete_method_desc[softc->delete_method],
4049                                     requestcount,
4050                                     (softc->trim_max_ranges - ranges) *
4051                                     ATA_DSM_RANGE_MAX);
4052                                 break;
4053                         }
4054                 }
4055                 lastlba = lba;
4056                 bp1 = cam_iosched_next_trim(softc->cam_iosched);
4057                 if (bp1 == NULL)
4058                         break;
4059                 if (bp1->bio_bcount / softc->params.secsize >
4060                     (softc->trim_max_ranges - ranges) * ATA_DSM_RANGE_MAX) {
4061                         cam_iosched_put_back_trim(softc->cam_iosched, bp1);
4062                         break;
4063                 }
4064         } while (1);
4065
4066         block_count = howmany(ranges, ATA_DSM_BLK_RANGES);
4067         scsi_ata_trim(&ccb->csio,
4068                       /*retries*/da_retry_count,
4069                       /*cbfcnp*/dadone,
4070                       /*tag_action*/MSG_SIMPLE_Q_TAG,
4071                       block_count,
4072                       /*data_ptr*/buf,
4073                       /*dxfer_len*/block_count * ATA_DSM_BLK_SIZE,
4074                       /*sense_len*/SSD_FULL_SIZE,
4075                       da_default_timeout * 1000);
4076         ccb->ccb_h.ccb_state = DA_CCB_DELETE;
4077         ccb->ccb_h.flags |= CAM_UNLOCKED;
4078         cam_iosched_submit_trim(softc->cam_iosched);
4079 }
4080
4081 /*
4082  * We calculate ws_max_blks here based off d_delmaxsize instead
4083  * of using softc->ws_max_blks as it is absolute max for the
4084  * device not the protocol max which may well be lower.
4085  */
4086 static void
4087 da_delete_ws(struct cam_periph *periph, union ccb *ccb, struct bio *bp)
4088 {
4089         struct da_softc *softc;
4090         struct bio *bp1;
4091         uint64_t ws_max_blks;
4092         uint64_t lba;
4093         uint64_t count; /* forward compat with WS32 */
4094
4095         softc = (struct da_softc *)periph->softc;
4096         ws_max_blks = softc->disk->d_delmaxsize / softc->params.secsize;
4097         lba = bp->bio_pblkno;
4098         count = 0;
4099         bp1 = bp;
4100         do {
4101                 if (bp1 != bp)//XXX imp XXX
4102                         bioq_insert_tail(&softc->delete_run_queue, bp1);
4103                 count += bp1->bio_bcount / softc->params.secsize;
4104                 if (count > ws_max_blks) {
4105                         xpt_print(periph->path,
4106                             "%s issuing short delete %ld > %ld\n",
4107                             da_delete_method_desc[softc->delete_method],
4108                             count, ws_max_blks);
4109                         count = omin(count, ws_max_blks);
4110                         break;
4111                 }
4112                 bp1 = cam_iosched_next_trim(softc->cam_iosched);
4113                 if (bp1 == NULL)
4114                         break;
4115                 if (lba + count != bp1->bio_pblkno ||
4116                     count + bp1->bio_bcount /
4117                     softc->params.secsize > ws_max_blks) {
4118                         cam_iosched_put_back_trim(softc->cam_iosched, bp1);
4119                         break;
4120                 }
4121         } while (1);
4122
4123         scsi_write_same(&ccb->csio,
4124                         /*retries*/da_retry_count,
4125                         /*cbfcnp*/dadone,
4126                         /*tag_action*/MSG_SIMPLE_Q_TAG,
4127                         /*byte2*/softc->delete_method ==
4128                             DA_DELETE_ZERO ? 0 : SWS_UNMAP,
4129                         softc->delete_method == DA_DELETE_WS16 ? 16 : 10,
4130                         /*lba*/lba,
4131                         /*block_count*/count,
4132                         /*data_ptr*/ __DECONST(void *, zero_region),
4133                         /*dxfer_len*/ softc->params.secsize,
4134                         /*sense_len*/SSD_FULL_SIZE,
4135                         da_default_timeout * 1000);
4136         ccb->ccb_h.ccb_state = DA_CCB_DELETE;
4137         ccb->ccb_h.flags |= CAM_UNLOCKED;
4138         cam_iosched_submit_trim(softc->cam_iosched);
4139 }
4140
4141 static int
4142 cmd6workaround(union ccb *ccb)
4143 {
4144         struct scsi_rw_6 cmd6;
4145         struct scsi_rw_10 *cmd10;
4146         struct da_softc *softc;
4147         u_int8_t *cdb;
4148         struct bio *bp;
4149         int frozen;
4150
4151         cdb = ccb->csio.cdb_io.cdb_bytes;
4152         softc = (struct da_softc *)xpt_path_periph(ccb->ccb_h.path)->softc;
4153
4154         if (ccb->ccb_h.ccb_state == DA_CCB_DELETE) {
4155                 da_delete_methods old_method = softc->delete_method;
4156
4157                 /*
4158                  * Typically there are two reasons for failure here
4159                  * 1. Delete method was detected as supported but isn't
4160                  * 2. Delete failed due to invalid params e.g. too big
4161                  *
4162                  * While we will attempt to choose an alternative delete method
4163                  * this may result in short deletes if the existing delete
4164                  * requests from geom are big for the new method chosen.
4165                  *
4166                  * This method assumes that the error which triggered this
4167                  * will not retry the io otherwise a panic will occur
4168                  */
4169                 dadeleteflag(softc, old_method, 0);
4170                 dadeletemethodchoose(softc, DA_DELETE_DISABLE);
4171                 if (softc->delete_method == DA_DELETE_DISABLE)
4172                         xpt_print(ccb->ccb_h.path,
4173                                   "%s failed, disabling BIO_DELETE\n",
4174                                   da_delete_method_desc[old_method]);
4175                 else
4176                         xpt_print(ccb->ccb_h.path,
4177                                   "%s failed, switching to %s BIO_DELETE\n",
4178                                   da_delete_method_desc[old_method],
4179                                   da_delete_method_desc[softc->delete_method]);
4180
4181                 while ((bp = bioq_takefirst(&softc->delete_run_queue)) != NULL)
4182                         cam_iosched_queue_work(softc->cam_iosched, bp);
4183                 cam_iosched_queue_work(softc->cam_iosched,
4184                     (struct bio *)ccb->ccb_h.ccb_bp);
4185                 ccb->ccb_h.ccb_bp = NULL;
4186                 return (0);
4187         }
4188
4189         /* Detect unsupported PREVENT ALLOW MEDIUM REMOVAL. */
4190         if ((ccb->ccb_h.flags & CAM_CDB_POINTER) == 0 &&
4191             (*cdb == PREVENT_ALLOW) &&
4192             (softc->quirks & DA_Q_NO_PREVENT) == 0) {
4193                 if (bootverbose)
4194                         xpt_print(ccb->ccb_h.path,
4195                             "PREVENT ALLOW MEDIUM REMOVAL not supported.\n");
4196                 softc->quirks |= DA_Q_NO_PREVENT;
4197                 return (0);
4198         }
4199
4200         /* Detect unsupported SYNCHRONIZE CACHE(10). */
4201         if ((ccb->ccb_h.flags & CAM_CDB_POINTER) == 0 &&
4202             (*cdb == SYNCHRONIZE_CACHE) &&
4203             (softc->quirks & DA_Q_NO_SYNC_CACHE) == 0) {
4204                 if (bootverbose)
4205                         xpt_print(ccb->ccb_h.path,
4206                             "SYNCHRONIZE CACHE(10) not supported.\n");
4207                 softc->quirks |= DA_Q_NO_SYNC_CACHE;
4208                 softc->disk->d_flags &= ~DISKFLAG_CANFLUSHCACHE;
4209                 return (0);
4210         }
4211
4212         /* Translation only possible if CDB is an array and cmd is R/W6 */
4213         if ((ccb->ccb_h.flags & CAM_CDB_POINTER) != 0 ||
4214             (*cdb != READ_6 && *cdb != WRITE_6))
4215                 return 0;
4216
4217         xpt_print(ccb->ccb_h.path, "READ(6)/WRITE(6) not supported, "
4218             "increasing minimum_cmd_size to 10.\n");
4219         softc->minimum_cmd_size = 10;
4220
4221         bcopy(cdb, &cmd6, sizeof(struct scsi_rw_6));
4222         cmd10 = (struct scsi_rw_10 *)cdb;
4223         cmd10->opcode = (cmd6.opcode == READ_6) ? READ_10 : WRITE_10;
4224         cmd10->byte2 = 0;
4225         scsi_ulto4b(scsi_3btoul(cmd6.addr), cmd10->addr);
4226         cmd10->reserved = 0;
4227         scsi_ulto2b(cmd6.length, cmd10->length);
4228         cmd10->control = cmd6.control;
4229         ccb->csio.cdb_len = sizeof(*cmd10);
4230
4231         /* Requeue request, unfreezing queue if necessary */
4232         frozen = (ccb->ccb_h.status & CAM_DEV_QFRZN) != 0;
4233         ccb->ccb_h.status = CAM_REQUEUE_REQ;
4234         xpt_action(ccb);
4235         if (frozen) {
4236                 cam_release_devq(ccb->ccb_h.path,
4237                                  /*relsim_flags*/0,
4238                                  /*reduction*/0,
4239                                  /*timeout*/0,
4240                                  /*getcount_only*/0);
4241         }
4242         return (ERESTART);
4243 }
4244
4245 static void
4246 dazonedone(struct cam_periph *periph, union ccb *ccb)
4247 {
4248         struct da_softc *softc;
4249         struct bio *bp;
4250
4251         softc = periph->softc;
4252         bp = (struct bio *)ccb->ccb_h.ccb_bp;
4253
4254         switch (bp->bio_zone.zone_cmd) {
4255         case DISK_ZONE_OPEN:
4256         case DISK_ZONE_CLOSE:
4257         case DISK_ZONE_FINISH:
4258         case DISK_ZONE_RWP:
4259                 break;
4260         case DISK_ZONE_REPORT_ZONES: {
4261                 uint32_t avail_len;
4262                 struct disk_zone_report *rep;
4263                 struct scsi_report_zones_hdr *hdr;
4264                 struct scsi_report_zones_desc *desc;
4265                 struct disk_zone_rep_entry *entry;
4266                 uint32_t hdr_len, num_avail;
4267                 uint32_t num_to_fill, i;
4268                 int ata;
4269
4270                 rep = &bp->bio_zone.zone_params.report;
4271                 avail_len = ccb->csio.dxfer_len - ccb->csio.resid;
4272                 /*
4273                  * Note that bio_resid isn't normally used for zone
4274                  * commands, but it is used by devstat_end_transaction_bio()
4275                  * to determine how much data was transferred.  Because
4276                  * the size of the SCSI/ATA data structures is different
4277                  * than the size of the BIO interface structures, the
4278                  * amount of data actually transferred from the drive will
4279                  * be different than the amount of data transferred to
4280                  * the user.
4281                  */
4282                 bp->bio_resid = ccb->csio.resid;
4283                 hdr = (struct scsi_report_zones_hdr *)ccb->csio.data_ptr;
4284                 if (avail_len < sizeof(*hdr)) {
4285                         /*
4286                          * Is there a better error than EIO here?  We asked
4287                          * for at least the header, and we got less than
4288                          * that.
4289                          */
4290                         bp->bio_error = EIO;
4291                         bp->bio_flags |= BIO_ERROR;
4292                         bp->bio_resid = bp->bio_bcount;
4293                         break;
4294                 }
4295
4296                 if (softc->zone_interface == DA_ZONE_IF_ATA_PASS)
4297                         ata = 1;
4298                 else
4299                         ata = 0;
4300
4301                 hdr_len = ata ? le32dec(hdr->length) :
4302                                 scsi_4btoul(hdr->length);
4303                 if (hdr_len > 0)
4304                         rep->entries_available = hdr_len / sizeof(*desc);
4305                 else
4306                         rep->entries_available = 0;
4307                 /*
4308                  * NOTE: using the same values for the BIO version of the
4309                  * same field as the SCSI/ATA values.  This means we could
4310                  * get some additional values that aren't defined in bio.h
4311                  * if more values of the same field are defined later.
4312                  */
4313                 rep->header.same = hdr->byte4 & SRZ_SAME_MASK;
4314                 rep->header.maximum_lba = ata ?  le64dec(hdr->maximum_lba) :
4315                                           scsi_8btou64(hdr->maximum_lba);
4316                 /*
4317                  * If the drive reports no entries that match the query,
4318                  * we're done.
4319                  */
4320                 if (hdr_len == 0) {
4321                         rep->entries_filled = 0;
4322                         break;
4323                 }
4324
4325                 num_avail = min((avail_len - sizeof(*hdr)) / sizeof(*desc),
4326                                 hdr_len / sizeof(*desc));
4327                 /*
4328                  * If the drive didn't return any data, then we're done.
4329                  */
4330                 if (num_avail == 0) {
4331                         rep->entries_filled = 0;
4332                         break;
4333                 }
4334
4335                 num_to_fill = min(num_avail, rep->entries_allocated);
4336                 /*
4337                  * If the user didn't allocate any entries for us to fill,
4338                  * we're done.
4339                  */
4340                 if (num_to_fill == 0) {
4341                         rep->entries_filled = 0;
4342                         break;
4343                 }
4344
4345                 for (i = 0, desc = &hdr->desc_list[0], entry=&rep->entries[0];
4346                      i < num_to_fill; i++, desc++, entry++) {
4347                         /*
4348                          * NOTE: we're mapping the values here directly
4349                          * from the SCSI/ATA bit definitions to the bio.h
4350                          * definitons.  There is also a warning in
4351                          * disk_zone.h, but the impact is that if
4352                          * additional values are added in the SCSI/ATA
4353                          * specs these will be visible to consumers of
4354                          * this interface.
4355                          */
4356                         entry->zone_type = desc->zone_type & SRZ_TYPE_MASK;
4357                         entry->zone_condition =
4358                             (desc->zone_flags & SRZ_ZONE_COND_MASK) >>
4359                             SRZ_ZONE_COND_SHIFT;
4360                         entry->zone_flags |= desc->zone_flags &
4361                             (SRZ_ZONE_NON_SEQ|SRZ_ZONE_RESET);
4362                         entry->zone_length =
4363                             ata ? le64dec(desc->zone_length) :
4364                                   scsi_8btou64(desc->zone_length);
4365                         entry->zone_start_lba =
4366                             ata ? le64dec(desc->zone_start_lba) :
4367                                   scsi_8btou64(desc->zone_start_lba);
4368                         entry->write_pointer_lba =
4369                             ata ? le64dec(desc->write_pointer_lba) :
4370                                   scsi_8btou64(desc->write_pointer_lba);
4371                 }
4372                 rep->entries_filled = num_to_fill;
4373                 break;
4374         }
4375         case DISK_ZONE_GET_PARAMS:
4376         default:
4377                 /*
4378                  * In theory we should not get a GET_PARAMS bio, since it
4379                  * should be handled without queueing the command to the
4380                  * drive.
4381                  */
4382                 panic("%s: Invalid zone command %d", __func__,
4383                     bp->bio_zone.zone_cmd);
4384                 break;
4385         }
4386
4387         if (bp->bio_zone.zone_cmd == DISK_ZONE_REPORT_ZONES)
4388                 free(ccb->csio.data_ptr, M_SCSIDA);
4389 }
4390
4391 static void
4392 dadone(struct cam_periph *periph, union ccb *done_ccb)
4393 {
4394         struct bio *bp, *bp1;
4395         struct da_softc *softc;
4396         struct ccb_scsiio *csio;
4397         u_int32_t  priority;
4398         da_ccb_state state;
4399
4400         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dadone\n"));
4401
4402         softc = (struct da_softc *)periph->softc;
4403         priority = done_ccb->ccb_h.pinfo.priority;
4404         csio = &done_ccb->csio;
4405
4406 #if defined(BUF_TRACKING) || defined(FULL_BUF_TRACKING)
4407         if (csio->bio != NULL)
4408                 biotrack(csio->bio, __func__);
4409 #endif
4410         state = csio->ccb_h.ccb_state & DA_CCB_TYPE_MASK;
4411
4412         cam_periph_lock(periph);
4413         bp = (struct bio *)done_ccb->ccb_h.ccb_bp;
4414         if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
4415                 int error;
4416                 int sf;
4417
4418                 if ((csio->ccb_h.ccb_state & DA_CCB_RETRY_UA) != 0)
4419                         sf = SF_RETRY_UA;
4420                 else
4421                         sf = 0;
4422
4423                 error = daerror(done_ccb, CAM_RETRY_SELTO, sf);
4424                 if (error == ERESTART) {
4425                         /* A retry was scheduled, so just return. */
4426                         cam_periph_unlock(periph);
4427                         return;
4428                 }
4429                 bp = (struct bio *)done_ccb->ccb_h.ccb_bp;
4430                 if (error != 0) {
4431                         int queued_error;
4432
4433                         /*
4434                          * return all queued I/O with EIO, so that
4435                          * the client can retry these I/Os in the
4436                          * proper order should it attempt to recover.
4437                          */
4438                         queued_error = EIO;
4439
4440                         if (error == ENXIO
4441                          && (softc->flags & DA_FLAG_PACK_INVALID)== 0) {
4442                                 /*
4443                                  * Catastrophic error.  Mark our pack as
4444                                  * invalid.
4445                                  *
4446                                  * XXX See if this is really a media
4447                                  * XXX change first?
4448                                  */
4449                                 xpt_print(periph->path, "Invalidating pack\n");
4450                                 softc->flags |= DA_FLAG_PACK_INVALID;
4451 #ifdef CAM_IO_STATS
4452                                 softc->invalidations++;
4453 #endif
4454                                 queued_error = ENXIO;
4455                         }
4456                         cam_iosched_flush(softc->cam_iosched, NULL,
4457                            queued_error);
4458                         if (bp != NULL) {
4459                                 bp->bio_error = error;
4460                                 bp->bio_resid = bp->bio_bcount;
4461                                 bp->bio_flags |= BIO_ERROR;
4462                         }
4463                 } else if (bp != NULL) {
4464                         if (state == DA_CCB_DELETE)
4465                                 bp->bio_resid = 0;
4466                         else
4467                                 bp->bio_resid = csio->resid;
4468                         bp->bio_error = 0;
4469                         if (bp->bio_resid != 0)
4470                                 bp->bio_flags |= BIO_ERROR;
4471                 }
4472                 if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0)
4473                         cam_release_devq(done_ccb->ccb_h.path,
4474                                          /*relsim_flags*/0,
4475                                          /*reduction*/0,
4476                                          /*timeout*/0,
4477                                          /*getcount_only*/0);
4478         } else if (bp != NULL) {
4479                 if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0)
4480                         panic("REQ_CMP with QFRZN");
4481                 if (bp->bio_cmd == BIO_ZONE)
4482                         dazonedone(periph, done_ccb);
4483                 else if (state == DA_CCB_DELETE)
4484                         bp->bio_resid = 0;
4485                 else
4486                         bp->bio_resid = csio->resid;
4487                 if ((csio->resid > 0) && (bp->bio_cmd != BIO_ZONE))
4488                         bp->bio_flags |= BIO_ERROR;
4489                 if (softc->error_inject != 0) {
4490                         bp->bio_error = softc->error_inject;
4491                         bp->bio_resid = bp->bio_bcount;
4492                         bp->bio_flags |= BIO_ERROR;
4493                         softc->error_inject = 0;
4494                 }
4495         }
4496
4497         if (bp != NULL)
4498                 biotrack(bp, __func__);
4499         LIST_REMOVE(&done_ccb->ccb_h, periph_links.le);
4500         if (LIST_EMPTY(&softc->pending_ccbs))
4501                 softc->flags |= DA_FLAG_WAS_OTAG;
4502
4503         /*
4504          * We need to call cam_iosched before we call biodone so that we don't
4505          * measure any activity that happens in the completion routine, which in
4506          * the case of sendfile can be quite extensive. Release the periph
4507          * refcount taken in dastart() for each CCB.
4508          */
4509         cam_iosched_bio_complete(softc->cam_iosched, bp, done_ccb);
4510         xpt_release_ccb(done_ccb);
4511         KASSERT(softc->refcount >= 1, ("dadone softc %p refcount %d", softc, softc->refcount));
4512         softc->refcount--;
4513         if (state == DA_CCB_DELETE) {
4514                 TAILQ_HEAD(, bio) queue;
4515
4516                 TAILQ_INIT(&queue);
4517                 TAILQ_CONCAT(&queue, &softc->delete_run_queue.queue, bio_queue);
4518                 softc->delete_run_queue.insert_point = NULL;
4519                 /*
4520                  * Normally, the xpt_release_ccb() above would make sure
4521                  * that when we have more work to do, that work would
4522                  * get kicked off. However, we specifically keep
4523                  * delete_running set to 0 before the call above to
4524                  * allow other I/O to progress when many BIO_DELETE
4525                  * requests are pushed down. We set delete_running to 0
4526                  * and call daschedule again so that we don't stall if
4527                  * there are no other I/Os pending apart from BIO_DELETEs.
4528                  */
4529                 cam_iosched_trim_done(softc->cam_iosched);
4530                 daschedule(periph);
4531                 cam_periph_unlock(periph);
4532                 while ((bp1 = TAILQ_FIRST(&queue)) != NULL) {
4533                         TAILQ_REMOVE(&queue, bp1, bio_queue);
4534                         bp1->bio_error = bp->bio_error;
4535                         if (bp->bio_flags & BIO_ERROR) {
4536                                 bp1->bio_flags |= BIO_ERROR;
4537                                 bp1->bio_resid = bp1->bio_bcount;
4538                         } else
4539                                 bp1->bio_resid = 0;
4540                         biodone(bp1);
4541                 }
4542         } else {
4543                 daschedule(periph);
4544                 cam_periph_unlock(periph);
4545         }
4546         if (bp != NULL)
4547                 biodone(bp);
4548         return;
4549 }
4550
4551 static void
4552 dadone_probewp(struct cam_periph *periph, union ccb *done_ccb)
4553 {
4554         struct scsi_mode_header_6 *mode_hdr6;
4555         struct scsi_mode_header_10 *mode_hdr10;
4556         struct da_softc *softc;
4557         struct ccb_scsiio *csio;
4558         u_int32_t  priority;
4559         uint8_t dev_spec;
4560
4561         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dadone_probewp\n"));
4562
4563         softc = (struct da_softc *)periph->softc;
4564         priority = done_ccb->ccb_h.pinfo.priority;
4565         csio = &done_ccb->csio;
4566
4567         cam_periph_assert(periph, MA_OWNED);
4568
4569         if (softc->minimum_cmd_size > 6) {
4570                 mode_hdr10 = (struct scsi_mode_header_10 *)csio->data_ptr;
4571                 dev_spec = mode_hdr10->dev_spec;
4572         } else {
4573                 mode_hdr6 = (struct scsi_mode_header_6 *)csio->data_ptr;
4574                 dev_spec = mode_hdr6->dev_spec;
4575         }
4576         if (cam_ccb_status(done_ccb) == CAM_REQ_CMP) {
4577                 if ((dev_spec & 0x80) != 0)
4578                         softc->disk->d_flags |= DISKFLAG_WRITE_PROTECT;
4579                 else
4580                         softc->disk->d_flags &= ~DISKFLAG_WRITE_PROTECT;
4581         } else {
4582                 int error;
4583
4584                 error = daerror(done_ccb, CAM_RETRY_SELTO,
4585                                 SF_RETRY_UA|SF_NO_PRINT);
4586                 if (error == ERESTART)
4587                         return;
4588                 else if (error != 0) {
4589                         if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
4590                                 /* Don't wedge this device's queue */
4591                                 cam_release_devq(done_ccb->ccb_h.path,
4592                                                  /*relsim_flags*/0,
4593                                                  /*reduction*/0,
4594                                                  /*timeout*/0,
4595                                                  /*getcount_only*/0);
4596                         }
4597                 }
4598         }
4599
4600         free(csio->data_ptr, M_SCSIDA);
4601         xpt_release_ccb(done_ccb);
4602         if ((softc->flags & DA_FLAG_CAN_RC16) != 0)
4603                 softc->state = DA_STATE_PROBE_RC16;
4604         else
4605                 softc->state = DA_STATE_PROBE_RC;
4606         xpt_schedule(periph, priority);
4607         return;
4608 }
4609
4610 static void
4611 dadone_proberc(struct cam_periph *periph, union ccb *done_ccb)
4612 {
4613         struct scsi_read_capacity_data *rdcap;
4614         struct scsi_read_capacity_data_long *rcaplong;
4615         struct da_softc *softc;
4616         struct ccb_scsiio *csio;
4617         da_ccb_state state;
4618         char *announce_buf;
4619         u_int32_t  priority;
4620         int lbp;
4621
4622         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dadone_proberc\n"));
4623
4624         softc = (struct da_softc *)periph->softc;
4625         priority = done_ccb->ccb_h.pinfo.priority;
4626         csio = &done_ccb->csio;
4627         state = csio->ccb_h.ccb_state & DA_CCB_TYPE_MASK;
4628
4629         lbp = 0;
4630         rdcap = NULL;
4631         rcaplong = NULL;
4632         /* XXX TODO: can this be a malloc? */
4633         announce_buf = softc->announce_temp;
4634         bzero(announce_buf, DA_ANNOUNCETMP_SZ);
4635
4636         if (state == DA_CCB_PROBE_RC)
4637                 rdcap =(struct scsi_read_capacity_data *)csio->data_ptr;
4638         else
4639                 rcaplong = (struct scsi_read_capacity_data_long *)
4640                         csio->data_ptr;
4641
4642         cam_periph_assert(periph, MA_OWNED);
4643
4644         if ((csio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
4645                 struct disk_params *dp;
4646                 uint32_t block_size;
4647                 uint64_t maxsector;
4648                 u_int lalba;    /* Lowest aligned LBA. */
4649
4650                 if (state == DA_CCB_PROBE_RC) {
4651                         block_size = scsi_4btoul(rdcap->length);
4652                         maxsector = scsi_4btoul(rdcap->addr);
4653                         lalba = 0;
4654
4655                         /*
4656                          * According to SBC-2, if the standard 10
4657                          * byte READ CAPACITY command returns 2^32,
4658                          * we should issue the 16 byte version of
4659                          * the command, since the device in question
4660                          * has more sectors than can be represented
4661                          * with the short version of the command.
4662                          */
4663                         if (maxsector == 0xffffffff) {
4664                                 free(rdcap, M_SCSIDA);
4665                                 xpt_release_ccb(done_ccb);
4666                                 softc->state = DA_STATE_PROBE_RC16;
4667                                 xpt_schedule(periph, priority);
4668                                 return;
4669                         }
4670                 } else {
4671                         block_size = scsi_4btoul(rcaplong->length);
4672                         maxsector = scsi_8btou64(rcaplong->addr);
4673                         lalba = scsi_2btoul(rcaplong->lalba_lbp);
4674                 }
4675
4676                 /*
4677                  * Because GEOM code just will panic us if we
4678                  * give them an 'illegal' value we'll avoid that
4679                  * here.
4680                  */
4681                 if (block_size == 0) {
4682                         block_size = 512;
4683                         if (maxsector == 0)
4684                                 maxsector = -1;
4685                 }
4686                 if (block_size >= MAXPHYS) {
4687                         xpt_print(periph->path,
4688                             "unsupportable block size %ju\n",
4689                             (uintmax_t) block_size);
4690                         announce_buf = NULL;
4691                         cam_periph_invalidate(periph);
4692                 } else {
4693                         /*
4694                          * We pass rcaplong into dasetgeom(),
4695                          * because it will only use it if it is
4696                          * non-NULL.
4697                          */
4698                         dasetgeom(periph, block_size, maxsector,
4699                                   rcaplong, sizeof(*rcaplong));
4700                         lbp = (lalba & SRC16_LBPME_A);
4701                         dp = &softc->params;
4702                         snprintf(announce_buf, DA_ANNOUNCETMP_SZ,
4703                             "%juMB (%ju %u byte sectors)",
4704                             ((uintmax_t)dp->secsize * dp->sectors) /
4705                              (1024 * 1024),
4706                             (uintmax_t)dp->sectors, dp->secsize);
4707                 }
4708         } else {
4709                 int error;
4710
4711                 /*
4712                  * Retry any UNIT ATTENTION type errors.  They
4713                  * are expected at boot.
4714                  */
4715                 error = daerror(done_ccb, CAM_RETRY_SELTO,
4716                                 SF_RETRY_UA|SF_NO_PRINT);
4717                 if (error == ERESTART) {
4718                         /*
4719                          * A retry was scheuled, so
4720                          * just return.
4721                          */
4722                         return;
4723                 } else if (error != 0) {
4724                         int asc, ascq;
4725                         int sense_key, error_code;
4726                         int have_sense;
4727                         cam_status status;
4728                         struct ccb_getdev cgd;
4729
4730                         /* Don't wedge this device's queue */
4731                         status = done_ccb->ccb_h.status;
4732                         if ((status & CAM_DEV_QFRZN) != 0)
4733                                 cam_release_devq(done_ccb->ccb_h.path,
4734                                                  /*relsim_flags*/0,
4735                                                  /*reduction*/0,
4736                                                  /*timeout*/0,
4737                                                  /*getcount_only*/0);
4738
4739
4740                         xpt_setup_ccb(&cgd.ccb_h, done_ccb->ccb_h.path,
4741                                       CAM_PRIORITY_NORMAL);
4742                         cgd.ccb_h.func_code = XPT_GDEV_TYPE;
4743                         xpt_action((union ccb *)&cgd);
4744
4745                         if (scsi_extract_sense_ccb(done_ccb,
4746                             &error_code, &sense_key, &asc, &ascq))
4747                                 have_sense = TRUE;
4748                         else
4749                                 have_sense = FALSE;
4750
4751                         /*
4752                          * If we tried READ CAPACITY(16) and failed,
4753                          * fallback to READ CAPACITY(10).
4754                          */
4755                         if ((state == DA_CCB_PROBE_RC16) &&
4756                             (softc->flags & DA_FLAG_CAN_RC16) &&
4757                             (((csio->ccb_h.status & CAM_STATUS_MASK) ==
4758                                 CAM_REQ_INVALID) ||
4759                              ((have_sense) &&
4760                               (error_code == SSD_CURRENT_ERROR ||
4761                                error_code == SSD_DESC_CURRENT_ERROR) &&
4762                               (sense_key == SSD_KEY_ILLEGAL_REQUEST)))) {
4763                                 cam_periph_assert(periph, MA_OWNED);
4764                                 softc->flags &= ~DA_FLAG_CAN_RC16;
4765                                 free(rdcap, M_SCSIDA);
4766                                 xpt_release_ccb(done_ccb);
4767                                 softc->state = DA_STATE_PROBE_RC;
4768                                 xpt_schedule(periph, priority);
4769                                 return;
4770                         }
4771
4772                         /*
4773                          * Attach to anything that claims to be a
4774                          * direct access or optical disk device,
4775                          * as long as it doesn't return a "Logical
4776                          * unit not supported" (0x25) error.
4777                          * "Internal Target Failure" (0x44) is also
4778                          * special and typically means that the
4779                          * device is a SATA drive behind a SATL
4780                          * translation that's fallen into a
4781                          * terminally fatal state.
4782                          */
4783                         if ((have_sense)
4784                          && (asc != 0x25) && (asc != 0x44)
4785                          && (error_code == SSD_CURRENT_ERROR
4786                           || error_code == SSD_DESC_CURRENT_ERROR)) {
4787                                 const char *sense_key_desc;
4788                                 const char *asc_desc;
4789
4790                                 dasetgeom(periph, 512, -1, NULL, 0);
4791                                 scsi_sense_desc(sense_key, asc, ascq,
4792                                                 &cgd.inq_data, &sense_key_desc,
4793                                                 &asc_desc);
4794                                 snprintf(announce_buf, DA_ANNOUNCETMP_SZ,
4795                                     "Attempt to query device "
4796                                     "size failed: %s, %s",
4797                                     sense_key_desc, asc_desc);
4798                         } else {
4799                                 if (have_sense)
4800                                         scsi_sense_print(&done_ccb->csio);
4801                                 else {
4802                                         xpt_print(periph->path,
4803                                             "got CAM status %#x\n",
4804                                             done_ccb->ccb_h.status);
4805                                 }
4806
4807                                 xpt_print(periph->path, "fatal error, "
4808                                     "failed to attach to device\n");
4809
4810                                 announce_buf = NULL;
4811
4812                                 /*
4813                                  * Free up resources.
4814                                  */
4815                                 cam_periph_invalidate(periph);
4816                         }
4817                 }
4818         }
4819         free(csio->data_ptr, M_SCSIDA);
4820         if (announce_buf != NULL &&
4821             ((softc->flags & DA_FLAG_ANNOUNCED) == 0)) {
4822                 struct sbuf sb;
4823
4824                 sbuf_new(&sb, softc->announcebuf, DA_ANNOUNCE_SZ,
4825                     SBUF_FIXEDLEN);
4826                 xpt_announce_periph_sbuf(periph, &sb, announce_buf);
4827                 xpt_announce_quirks_sbuf(periph, &sb, softc->quirks,
4828                     DA_Q_BIT_STRING);
4829                 sbuf_finish(&sb);
4830                 sbuf_putbuf(&sb);
4831
4832                 /*
4833                  * Create our sysctl variables, now that we know
4834                  * we have successfully attached.
4835                  */
4836                 /* increase the refcount */
4837                 if (da_periph_acquire(periph, DA_REF_SYSCTL) == 0) {
4838                         taskqueue_enqueue(taskqueue_thread,
4839                                           &softc->sysctl_task);
4840                 } else {
4841                         /* XXX This message is useless! */
4842                         xpt_print(periph->path, "fatal error, "
4843                             "could not acquire reference count\n");
4844                 }
4845         }
4846
4847         /* We already probed the device. */
4848         if (softc->flags & DA_FLAG_PROBED) {
4849                 daprobedone(periph, done_ccb);
4850                 return;
4851         }
4852
4853         /* Ensure re-probe doesn't see old delete. */
4854         softc->delete_available = 0;
4855         dadeleteflag(softc, DA_DELETE_ZERO, 1);
4856         if (lbp && (softc->quirks & DA_Q_NO_UNMAP) == 0) {
4857                 /*
4858                  * Based on older SBC-3 spec revisions
4859                  * any of the UNMAP methods "may" be
4860                  * available via LBP given this flag so
4861                  * we flag all of them as available and
4862                  * then remove those which further
4863                  * probes confirm aren't available
4864                  * later.
4865                  *
4866                  * We could also check readcap(16) p_type
4867                  * flag to exclude one or more invalid
4868                  * write same (X) types here
4869                  */
4870                 dadeleteflag(softc, DA_DELETE_WS16, 1);
4871                 dadeleteflag(softc, DA_DELETE_WS10, 1);
4872                 dadeleteflag(softc, DA_DELETE_UNMAP, 1);
4873
4874                 xpt_release_ccb(done_ccb);
4875                 softc->state = DA_STATE_PROBE_LBP;
4876                 xpt_schedule(periph, priority);
4877                 return;
4878         }
4879
4880         xpt_release_ccb(done_ccb);
4881         softc->state = DA_STATE_PROBE_BDC;
4882         xpt_schedule(periph, priority);
4883         return;
4884 }
4885
4886 static void
4887 dadone_probelbp(struct cam_periph *periph, union ccb *done_ccb)
4888 {
4889         struct scsi_vpd_logical_block_prov *lbp;
4890         struct da_softc *softc;
4891         struct ccb_scsiio *csio;
4892         u_int32_t  priority;
4893
4894         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dadone_probelbp\n"));
4895
4896         softc = (struct da_softc *)periph->softc;
4897         priority = done_ccb->ccb_h.pinfo.priority;
4898         csio = &done_ccb->csio;
4899         lbp = (struct scsi_vpd_logical_block_prov *)csio->data_ptr;
4900
4901         cam_periph_assert(periph, MA_OWNED);
4902
4903         if ((csio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
4904                 /*
4905                  * T10/1799-D Revision 31 states at least one of these
4906                  * must be supported but we don't currently enforce this.
4907                  */
4908                 dadeleteflag(softc, DA_DELETE_WS16,
4909                      (lbp->flags & SVPD_LBP_WS16));
4910                 dadeleteflag(softc, DA_DELETE_WS10,
4911                              (lbp->flags & SVPD_LBP_WS10));
4912                 dadeleteflag(softc, DA_DELETE_UNMAP,
4913                              (lbp->flags & SVPD_LBP_UNMAP));
4914         } else {
4915                 int error;
4916                 error = daerror(done_ccb, CAM_RETRY_SELTO,
4917                                 SF_RETRY_UA|SF_NO_PRINT);
4918                 if (error == ERESTART)
4919                         return;
4920                 else if (error != 0) {
4921                         if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
4922                                 /* Don't wedge this device's queue */
4923                                 cam_release_devq(done_ccb->ccb_h.path,
4924                                                  /*relsim_flags*/0,
4925                                                  /*reduction*/0,
4926                                                  /*timeout*/0,
4927                                                  /*getcount_only*/0);
4928                         }
4929
4930                         /*
4931                          * Failure indicates we don't support any SBC-3
4932                          * delete methods with UNMAP
4933                          */
4934                 }
4935         }
4936
4937         free(lbp, M_SCSIDA);
4938         xpt_release_ccb(done_ccb);
4939         softc->state = DA_STATE_PROBE_BLK_LIMITS;
4940         xpt_schedule(periph, priority);
4941         return;
4942 }
4943
4944 static void
4945 dadone_probeblklimits(struct cam_periph *periph, union ccb *done_ccb)
4946 {
4947         struct scsi_vpd_block_limits *block_limits;
4948         struct da_softc *softc;
4949         struct ccb_scsiio *csio;
4950         u_int32_t  priority;
4951
4952         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dadone_probeblklimits\n"));
4953
4954         softc = (struct da_softc *)periph->softc;
4955         priority = done_ccb->ccb_h.pinfo.priority;
4956         csio = &done_ccb->csio;
4957         block_limits = (struct scsi_vpd_block_limits *)csio->data_ptr;
4958
4959         cam_periph_assert(periph, MA_OWNED);
4960
4961         if ((csio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
4962                 uint32_t max_txfer_len = scsi_4btoul(
4963                         block_limits->max_txfer_len);
4964                 uint32_t max_unmap_lba_cnt = scsi_4btoul(
4965                         block_limits->max_unmap_lba_cnt);
4966                 uint32_t max_unmap_blk_cnt = scsi_4btoul(
4967                         block_limits->max_unmap_blk_cnt);
4968                 uint32_t unmap_gran = scsi_4btoul(
4969                         block_limits->opt_unmap_grain);
4970                 uint32_t unmap_gran_align = scsi_4btoul(
4971                         block_limits->unmap_grain_align);
4972                 uint64_t ws_max_blks = scsi_8btou64(
4973                         block_limits->max_write_same_length);
4974
4975                 if (max_txfer_len != 0) {
4976                         softc->disk->d_maxsize = MIN(softc->maxio,
4977                             (off_t)max_txfer_len * softc->params.secsize);
4978                 }
4979
4980                 /*
4981                  * We should already support UNMAP but we check lba
4982                  * and block count to be sure
4983                  */
4984                 if (max_unmap_lba_cnt != 0x00L &&
4985                     max_unmap_blk_cnt != 0x00L) {
4986                         softc->unmap_max_lba = max_unmap_lba_cnt;
4987                         softc->unmap_max_ranges = min(max_unmap_blk_cnt,
4988                                 UNMAP_MAX_RANGES);
4989                         if (unmap_gran > 1) {
4990                                 softc->unmap_gran = unmap_gran;
4991                                 if (unmap_gran_align & 0x80000000) {
4992                                         softc->unmap_gran_align =
4993                                             unmap_gran_align & 0x7fffffff;
4994                                 }
4995                         }
4996                 } else {
4997                         /*
4998                          * Unexpected UNMAP limits which means the
4999                          * device doesn't actually support UNMAP
5000                          */
5001                         dadeleteflag(softc, DA_DELETE_UNMAP, 0);
5002                 }
5003
5004                 if (ws_max_blks != 0x00L)
5005                         softc->ws_max_blks = ws_max_blks;
5006         } else {
5007                 int error;
5008                 error = daerror(done_ccb, CAM_RETRY_SELTO,
5009                                 SF_RETRY_UA|SF_NO_PRINT);
5010                 if (error == ERESTART)
5011                         return;
5012                 else if (error != 0) {
5013                         if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
5014                                 /* Don't wedge this device's queue */
5015                                 cam_release_devq(done_ccb->ccb_h.path,
5016                                                  /*relsim_flags*/0,
5017                                                  /*reduction*/0,
5018                                                  /*timeout*/0,
5019                                                  /*getcount_only*/0);
5020                         }
5021
5022                         /*
5023                          * Failure here doesn't mean UNMAP is not
5024                          * supported as this is an optional page.
5025                          */
5026                         softc->unmap_max_lba = 1;
5027                         softc->unmap_max_ranges = 1;
5028                 }
5029         }
5030
5031         free(block_limits, M_SCSIDA);
5032         xpt_release_ccb(done_ccb);
5033         softc->state = DA_STATE_PROBE_BDC;
5034         xpt_schedule(periph, priority);
5035         return;
5036 }
5037
5038 static void
5039 dadone_probebdc(struct cam_periph *periph, union ccb *done_ccb)
5040 {
5041         struct scsi_vpd_block_device_characteristics *bdc;
5042         struct da_softc *softc;
5043         struct ccb_scsiio *csio;
5044         u_int32_t  priority;
5045
5046         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dadone_probebdc\n"));
5047
5048         softc = (struct da_softc *)periph->softc;
5049         priority = done_ccb->ccb_h.pinfo.priority;
5050         csio = &done_ccb->csio;
5051         bdc = (struct scsi_vpd_block_device_characteristics *)csio->data_ptr;
5052
5053         cam_periph_assert(periph, MA_OWNED);
5054
5055         if ((csio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
5056                 uint32_t valid_len;
5057
5058                 /*
5059                  * Disable queue sorting for non-rotational media
5060                  * by default.
5061                  */
5062                 u_int16_t old_rate = softc->disk->d_rotation_rate;
5063
5064                 valid_len = csio->dxfer_len - csio->resid;
5065                 if (SBDC_IS_PRESENT(bdc, valid_len,
5066                     medium_rotation_rate)) {
5067                         softc->disk->d_rotation_rate =
5068                                 scsi_2btoul(bdc->medium_rotation_rate);
5069                         if (softc->disk->d_rotation_rate ==
5070                             SVPD_BDC_RATE_NON_ROTATING) {
5071                                 cam_iosched_set_sort_queue(
5072                                     softc->cam_iosched, 0);
5073                                 softc->rotating = 0;
5074                         }
5075                         if (softc->disk->d_rotation_rate != old_rate) {
5076                                 disk_attr_changed(softc->disk,
5077                                     "GEOM::rotation_rate", M_NOWAIT);
5078                         }
5079                 }
5080                 if ((SBDC_IS_PRESENT(bdc, valid_len, flags))
5081                  && (softc->zone_mode == DA_ZONE_NONE)) {
5082                         int ata_proto;
5083
5084                         if (scsi_vpd_supported_page(periph,
5085                             SVPD_ATA_INFORMATION))
5086                                 ata_proto = 1;
5087                         else
5088                                 ata_proto = 0;
5089
5090                         /*
5091                          * The Zoned field will only be set for
5092                          * Drive Managed and Host Aware drives.  If
5093                          * they are Host Managed, the device type
5094                          * in the standard INQUIRY data should be
5095                          * set to T_ZBC_HM (0x14).
5096                          */
5097                         if ((bdc->flags & SVPD_ZBC_MASK) ==
5098                              SVPD_HAW_ZBC) {
5099                                 softc->zone_mode = DA_ZONE_HOST_AWARE;
5100                                 softc->zone_interface = (ata_proto) ?
5101                                    DA_ZONE_IF_ATA_SAT : DA_ZONE_IF_SCSI;
5102                         } else if ((bdc->flags & SVPD_ZBC_MASK) ==
5103                              SVPD_DM_ZBC) {
5104                                 softc->zone_mode =DA_ZONE_DRIVE_MANAGED;
5105                                 softc->zone_interface = (ata_proto) ?
5106                                    DA_ZONE_IF_ATA_SAT : DA_ZONE_IF_SCSI;
5107                         } else if ((bdc->flags & SVPD_ZBC_MASK) !=
5108                                   SVPD_ZBC_NR) {
5109                                 xpt_print(periph->path, "Unknown zoned "
5110                                     "type %#x",
5111                                     bdc->flags & SVPD_ZBC_MASK);
5112                         }
5113                 }
5114         } else {
5115                 int error;
5116                 error = daerror(done_ccb, CAM_RETRY_SELTO,
5117                                 SF_RETRY_UA|SF_NO_PRINT);
5118                 if (error == ERESTART)
5119                         return;
5120                 else if (error != 0) {
5121                         if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
5122                                 /* Don't wedge this device's queue */
5123                                 cam_release_devq(done_ccb->ccb_h.path,
5124                                                  /*relsim_flags*/0,
5125                                                  /*reduction*/0,
5126                                                  /*timeout*/0,
5127                                                  /*getcount_only*/0);
5128                         }
5129                 }
5130         }
5131
5132         free(bdc, M_SCSIDA);
5133         xpt_release_ccb(done_ccb);
5134         softc->state = DA_STATE_PROBE_ATA;
5135         xpt_schedule(periph, priority);
5136         return;
5137 }
5138
5139 static void
5140 dadone_probeata(struct cam_periph *periph, union ccb *done_ccb)
5141 {
5142         struct ata_params *ata_params;
5143         struct ccb_scsiio *csio;
5144         struct da_softc *softc;
5145         u_int32_t  priority;
5146         int continue_probe;
5147         int error, i;
5148         int16_t *ptr;
5149
5150         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dadone_probeata\n"));
5151
5152         softc = (struct da_softc *)periph->softc;
5153         priority = done_ccb->ccb_h.pinfo.priority;
5154         csio = &done_ccb->csio;
5155         ata_params = (struct ata_params *)csio->data_ptr;
5156         ptr = (uint16_t *)ata_params;
5157         continue_probe = 0;
5158         error = 0;
5159
5160         cam_periph_assert(periph, MA_OWNED);
5161
5162         if ((csio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
5163                 uint16_t old_rate;
5164
5165                 for (i = 0; i < sizeof(*ata_params) / 2; i++)
5166                         ptr[i] = le16toh(ptr[i]);
5167                 if (ata_params->support_dsm & ATA_SUPPORT_DSM_TRIM &&
5168                     (softc->quirks & DA_Q_NO_UNMAP) == 0) {
5169                         dadeleteflag(softc, DA_DELETE_ATA_TRIM, 1);
5170                         if (ata_params->max_dsm_blocks != 0)
5171                                 softc->trim_max_ranges = min(
5172                                   softc->trim_max_ranges,
5173                                   ata_params->max_dsm_blocks *
5174                                   ATA_DSM_BLK_RANGES);
5175                 }
5176                 /*
5177                  * Disable queue sorting for non-rotational media
5178                  * by default.
5179                  */
5180                 old_rate = softc->disk->d_rotation_rate;
5181                 softc->disk->d_rotation_rate = ata_params->media_rotation_rate;
5182                 if (softc->disk->d_rotation_rate == ATA_RATE_NON_ROTATING) {
5183                         cam_iosched_set_sort_queue(softc->cam_iosched, 0);
5184                         softc->rotating = 0;
5185                 }
5186                 if (softc->disk->d_rotation_rate != old_rate) {
5187                         disk_attr_changed(softc->disk,
5188                             "GEOM::rotation_rate", M_NOWAIT);
5189                 }
5190
5191                 cam_periph_assert(periph, MA_OWNED);
5192                 if (ata_params->capabilities1 & ATA_SUPPORT_DMA)
5193                         softc->flags |= DA_FLAG_CAN_ATA_DMA;
5194
5195                 if (ata_params->support.extension & ATA_SUPPORT_GENLOG)
5196                         softc->flags |= DA_FLAG_CAN_ATA_LOG;
5197
5198                 /*
5199                  * At this point, if we have a SATA host aware drive,
5200                  * we communicate via ATA passthrough unless the
5201                  * SAT layer supports ZBC -> ZAC translation.  In
5202                  * that case,
5203                  *
5204                  * XXX KDM figure out how to detect a host managed
5205                  * SATA drive.
5206                  */
5207                 if (softc->zone_mode == DA_ZONE_NONE) {
5208                         /*
5209                          * Note that we don't override the zone
5210                          * mode or interface if it has already been
5211                          * set.  This is because it has either been
5212                          * set as a quirk, or when we probed the
5213                          * SCSI Block Device Characteristics page,
5214                          * the zoned field was set.  The latter
5215                          * means that the SAT layer supports ZBC to
5216                          * ZAC translation, and we would prefer to
5217                          * use that if it is available.
5218                          */
5219                         if ((ata_params->support3 &
5220                             ATA_SUPPORT_ZONE_MASK) ==
5221                             ATA_SUPPORT_ZONE_HOST_AWARE) {
5222                                 softc->zone_mode = DA_ZONE_HOST_AWARE;
5223                                 softc->zone_interface =
5224                                     DA_ZONE_IF_ATA_PASS;
5225                         } else if ((ata_params->support3 &
5226                                     ATA_SUPPORT_ZONE_MASK) ==
5227                                     ATA_SUPPORT_ZONE_DEV_MANAGED) {
5228                                 softc->zone_mode =DA_ZONE_DRIVE_MANAGED;
5229                                 softc->zone_interface = DA_ZONE_IF_ATA_PASS;
5230                         }
5231                 }
5232
5233         } else {
5234                 error = daerror(done_ccb, CAM_RETRY_SELTO,
5235                                 SF_RETRY_UA|SF_NO_PRINT);
5236                 if (error == ERESTART)
5237                         return;
5238                 else if (error != 0) {
5239                         if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
5240                                 /* Don't wedge this device's queue */
5241                                 cam_release_devq(done_ccb->ccb_h.path,
5242                                                  /*relsim_flags*/0,
5243                                                  /*reduction*/0,
5244                                                  /*timeout*/0,
5245                                                  /*getcount_only*/0);
5246                         }
5247                 }
5248         }
5249
5250         free(ata_params, M_SCSIDA);
5251         if ((softc->zone_mode == DA_ZONE_HOST_AWARE)
5252          || (softc->zone_mode == DA_ZONE_HOST_MANAGED)) {
5253                 /*
5254                  * If the ATA IDENTIFY failed, we could be talking
5255                  * to a SCSI drive, although that seems unlikely,
5256                  * since the drive did report that it supported the
5257                  * ATA Information VPD page.  If the ATA IDENTIFY
5258                  * succeeded, and the SAT layer doesn't support
5259                  * ZBC -> ZAC translation, continue on to get the
5260                  * directory of ATA logs, and complete the rest of
5261                  * the ZAC probe.  If the SAT layer does support
5262                  * ZBC -> ZAC translation, we want to use that,
5263                  * and we'll probe the SCSI Zoned Block Device
5264                  * Characteristics VPD page next.
5265                  */
5266                 if ((error == 0)
5267                  && (softc->flags & DA_FLAG_CAN_ATA_LOG)
5268                  && (softc->zone_interface == DA_ZONE_IF_ATA_PASS))
5269                         softc->state = DA_STATE_PROBE_ATA_LOGDIR;
5270                 else
5271                         softc->state = DA_STATE_PROBE_ZONE;
5272                 continue_probe = 1;
5273         }
5274         if (continue_probe != 0) {
5275                 xpt_release_ccb(done_ccb);
5276                 xpt_schedule(periph, priority);
5277                 return;
5278         } else
5279                 daprobedone(periph, done_ccb);
5280         return;
5281 }
5282
5283 static void
5284 dadone_probeatalogdir(struct cam_periph *periph, union ccb *done_ccb)
5285 {
5286         struct da_softc *softc;
5287         struct ccb_scsiio *csio;
5288         u_int32_t  priority;
5289         int error;
5290
5291         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dadone_probeatalogdir\n"));
5292
5293         softc = (struct da_softc *)periph->softc;
5294         priority = done_ccb->ccb_h.pinfo.priority;
5295         csio = &done_ccb->csio;
5296
5297         cam_periph_assert(periph, MA_OWNED);
5298         if ((csio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
5299                 error = 0;
5300                 softc->valid_logdir_len = 0;
5301                 bzero(&softc->ata_logdir, sizeof(softc->ata_logdir));
5302                 softc->valid_logdir_len = csio->dxfer_len - csio->resid;
5303                 if (softc->valid_logdir_len > 0)
5304                         bcopy(csio->data_ptr, &softc->ata_logdir,
5305                             min(softc->valid_logdir_len,
5306                                 sizeof(softc->ata_logdir)));
5307                 /*
5308                  * Figure out whether the Identify Device log is
5309                  * supported.  The General Purpose log directory
5310                  * has a header, and lists the number of pages
5311                  * available for each GP log identified by the
5312                  * offset into the list.
5313                  */
5314                 if ((softc->valid_logdir_len >=
5315                     ((ATA_IDENTIFY_DATA_LOG + 1) * sizeof(uint16_t)))
5316                  && (le16dec(softc->ata_logdir.header) ==
5317                      ATA_GP_LOG_DIR_VERSION)
5318                  && (le16dec(&softc->ata_logdir.num_pages[
5319                      (ATA_IDENTIFY_DATA_LOG *
5320                      sizeof(uint16_t)) - sizeof(uint16_t)]) > 0)){
5321                         softc->flags |= DA_FLAG_CAN_ATA_IDLOG;
5322                 } else {
5323                         softc->flags &= ~DA_FLAG_CAN_ATA_IDLOG;
5324                 }
5325         } else {
5326                 error = daerror(done_ccb, CAM_RETRY_SELTO,
5327                                 SF_RETRY_UA|SF_NO_PRINT);
5328                 if (error == ERESTART)
5329                         return;
5330                 else if (error != 0) {
5331                         /*
5332                          * If we can't get the ATA log directory,
5333                          * then ATA logs are effectively not
5334                          * supported even if the bit is set in the
5335                          * identify data.
5336                          */
5337                         softc->flags &= ~(DA_FLAG_CAN_ATA_LOG |
5338                                           DA_FLAG_CAN_ATA_IDLOG);
5339                         if ((done_ccb->ccb_h.status &
5340                              CAM_DEV_QFRZN) != 0) {
5341                                 /* Don't wedge this device's queue */
5342                                 cam_release_devq(done_ccb->ccb_h.path,
5343                                                  /*relsim_flags*/0,
5344                                                  /*reduction*/0,
5345                                                  /*timeout*/0,
5346                                                  /*getcount_only*/0);
5347                         }
5348                 }
5349         }
5350
5351         free(csio->data_ptr, M_SCSIDA);
5352
5353         if ((error == 0)
5354          && (softc->flags & DA_FLAG_CAN_ATA_IDLOG)) {
5355                 softc->state = DA_STATE_PROBE_ATA_IDDIR;
5356                 xpt_release_ccb(done_ccb);
5357                 xpt_schedule(periph, priority);
5358                 return;
5359         }
5360         daprobedone(periph, done_ccb);
5361         return;
5362 }
5363
5364 static void
5365 dadone_probeataiddir(struct cam_periph *periph, union ccb *done_ccb)
5366 {
5367         struct da_softc *softc;
5368         struct ccb_scsiio *csio;
5369         u_int32_t  priority;
5370         int error;
5371
5372         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dadone_probeataiddir\n"));
5373
5374         softc = (struct da_softc *)periph->softc;
5375         priority = done_ccb->ccb_h.pinfo.priority;
5376         csio = &done_ccb->csio;
5377
5378         cam_periph_assert(periph, MA_OWNED);
5379
5380         if ((csio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
5381                 off_t entries_offset, max_entries;
5382                 error = 0;
5383
5384                 softc->valid_iddir_len = 0;
5385                 bzero(&softc->ata_iddir, sizeof(softc->ata_iddir));
5386                 softc->flags &= ~(DA_FLAG_CAN_ATA_SUPCAP |
5387                                   DA_FLAG_CAN_ATA_ZONE);
5388                 softc->valid_iddir_len = csio->dxfer_len - csio->resid;
5389                 if (softc->valid_iddir_len > 0)
5390                         bcopy(csio->data_ptr, &softc->ata_iddir,
5391                             min(softc->valid_iddir_len,
5392                                 sizeof(softc->ata_iddir)));
5393
5394                 entries_offset =
5395                     __offsetof(struct ata_identify_log_pages,entries);
5396                 max_entries = softc->valid_iddir_len - entries_offset;
5397                 if ((softc->valid_iddir_len > (entries_offset + 1))
5398                  && (le64dec(softc->ata_iddir.header) == ATA_IDLOG_REVISION)
5399                  && (softc->ata_iddir.entry_count > 0)) {
5400                         int num_entries, i;
5401
5402                         num_entries = softc->ata_iddir.entry_count;
5403                         num_entries = min(num_entries,
5404                            softc->valid_iddir_len - entries_offset);
5405                         for (i = 0; i < num_entries && i < max_entries; i++) {
5406                                 if (softc->ata_iddir.entries[i] ==
5407                                     ATA_IDL_SUP_CAP)
5408                                         softc->flags |= DA_FLAG_CAN_ATA_SUPCAP;
5409                                 else if (softc->ata_iddir.entries[i] ==
5410                                          ATA_IDL_ZDI)
5411                                         softc->flags |= DA_FLAG_CAN_ATA_ZONE;
5412
5413                                 if ((softc->flags & DA_FLAG_CAN_ATA_SUPCAP)
5414                                  && (softc->flags & DA_FLAG_CAN_ATA_ZONE))
5415                                         break;
5416                         }
5417                 }
5418         } else {
5419                 error = daerror(done_ccb, CAM_RETRY_SELTO,
5420                                 SF_RETRY_UA|SF_NO_PRINT);
5421                 if (error == ERESTART)
5422                         return;
5423                 else if (error != 0) {
5424                         /*
5425                          * If we can't get the ATA Identify Data log
5426                          * directory, then it effectively isn't
5427                          * supported even if the ATA Log directory
5428                          * a non-zero number of pages present for
5429                          * this log.
5430                          */
5431                         softc->flags &= ~DA_FLAG_CAN_ATA_IDLOG;
5432                         if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
5433                                 /* Don't wedge this device's queue */
5434                                 cam_release_devq(done_ccb->ccb_h.path,
5435                                                  /*relsim_flags*/0,
5436                                                  /*reduction*/0,
5437                                                  /*timeout*/0,
5438                                                  /*getcount_only*/0);
5439                         }
5440                 }
5441         }
5442
5443         free(csio->data_ptr, M_SCSIDA);
5444
5445         if ((error == 0) && (softc->flags & DA_FLAG_CAN_ATA_SUPCAP)) {
5446                 softc->state = DA_STATE_PROBE_ATA_SUP;
5447                 xpt_release_ccb(done_ccb);
5448                 xpt_schedule(periph, priority);
5449                 return;
5450         }
5451         daprobedone(periph, done_ccb);
5452         return;
5453 }
5454
5455 static void
5456 dadone_probeatasup(struct cam_periph *periph, union ccb *done_ccb)
5457 {
5458         struct da_softc *softc;
5459         struct ccb_scsiio *csio;
5460         u_int32_t  priority;
5461         int error;
5462
5463         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dadone_probeatasup\n"));
5464
5465         softc = (struct da_softc *)periph->softc;
5466         priority = done_ccb->ccb_h.pinfo.priority;
5467         csio = &done_ccb->csio;
5468
5469         cam_periph_assert(periph, MA_OWNED);
5470
5471         if ((csio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
5472                 uint32_t valid_len;
5473                 size_t needed_size;
5474                 struct ata_identify_log_sup_cap *sup_cap;
5475                 error = 0;
5476
5477                 sup_cap = (struct ata_identify_log_sup_cap *)csio->data_ptr;
5478                 valid_len = csio->dxfer_len - csio->resid;
5479                 needed_size = __offsetof(struct ata_identify_log_sup_cap,
5480                     sup_zac_cap) + 1 + sizeof(sup_cap->sup_zac_cap);
5481                 if (valid_len >= needed_size) {
5482                         uint64_t zoned, zac_cap;
5483
5484                         zoned = le64dec(sup_cap->zoned_cap);
5485                         if (zoned & ATA_ZONED_VALID) {
5486                                 /*
5487                                  * This should have already been
5488                                  * set, because this is also in the
5489                                  * ATA identify data.
5490                                  */
5491                                 if ((zoned & ATA_ZONED_MASK) ==
5492                                     ATA_SUPPORT_ZONE_HOST_AWARE)
5493                                         softc->zone_mode = DA_ZONE_HOST_AWARE;
5494                                 else if ((zoned & ATA_ZONED_MASK) ==
5495                                     ATA_SUPPORT_ZONE_DEV_MANAGED)
5496                                         softc->zone_mode =
5497                                             DA_ZONE_DRIVE_MANAGED;
5498                         }
5499
5500                         zac_cap = le64dec(sup_cap->sup_zac_cap);
5501                         if (zac_cap & ATA_SUP_ZAC_CAP_VALID) {
5502                                 if (zac_cap & ATA_REPORT_ZONES_SUP)
5503                                         softc->zone_flags |=
5504                                             DA_ZONE_FLAG_RZ_SUP;
5505                                 if (zac_cap & ATA_ND_OPEN_ZONE_SUP)
5506                                         softc->zone_flags |=
5507                                             DA_ZONE_FLAG_OPEN_SUP;
5508                                 if (zac_cap & ATA_ND_CLOSE_ZONE_SUP)
5509                                         softc->zone_flags |=
5510                                             DA_ZONE_FLAG_CLOSE_SUP;
5511                                 if (zac_cap & ATA_ND_FINISH_ZONE_SUP)
5512                                         softc->zone_flags |=
5513                                             DA_ZONE_FLAG_FINISH_SUP;
5514                                 if (zac_cap & ATA_ND_RWP_SUP)
5515                                         softc->zone_flags |=
5516                                             DA_ZONE_FLAG_RWP_SUP;
5517                         } else {
5518                                 /*
5519                                  * This field was introduced in
5520                                  * ACS-4, r08 on April 28th, 2015.
5521                                  * If the drive firmware was written
5522                                  * to an earlier spec, it won't have
5523                                  * the field.  So, assume all
5524                                  * commands are supported.
5525                                  */
5526                                 softc->zone_flags |= DA_ZONE_FLAG_SUP_MASK;
5527                         }
5528                 }
5529         } else {
5530                 error = daerror(done_ccb, CAM_RETRY_SELTO,
5531                                 SF_RETRY_UA|SF_NO_PRINT);
5532                 if (error == ERESTART)
5533                         return;
5534                 else if (error != 0) {
5535                         /*
5536                          * If we can't get the ATA Identify Data
5537                          * Supported Capabilities page, clear the
5538                          * flag...
5539                          */
5540                         softc->flags &= ~DA_FLAG_CAN_ATA_SUPCAP;
5541                         /*
5542                          * And clear zone capabilities.
5543                          */
5544                         softc->zone_flags &= ~DA_ZONE_FLAG_SUP_MASK;
5545                         if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
5546                                 /* Don't wedge this device's queue */
5547                                 cam_release_devq(done_ccb->ccb_h.path,
5548                                                  /*relsim_flags*/0,
5549                                                  /*reduction*/0,
5550                                                  /*timeout*/0,
5551                                                  /*getcount_only*/0);
5552                         }
5553                 }
5554         }
5555
5556         free(csio->data_ptr, M_SCSIDA);
5557
5558         if ((error == 0) && (softc->flags & DA_FLAG_CAN_ATA_ZONE)) {
5559                 softc->state = DA_STATE_PROBE_ATA_ZONE;
5560                 xpt_release_ccb(done_ccb);
5561                 xpt_schedule(periph, priority);
5562                 return;
5563         }
5564         daprobedone(periph, done_ccb);
5565         return;
5566 }
5567
5568 static void
5569 dadone_probeatazone(struct cam_periph *periph, union ccb *done_ccb)
5570 {
5571         struct da_softc *softc;
5572         struct ccb_scsiio *csio;
5573         int error;
5574
5575         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dadone_probeatazone\n"));
5576
5577         softc = (struct da_softc *)periph->softc;
5578         csio = &done_ccb->csio;
5579
5580         cam_periph_assert(periph, MA_OWNED);
5581
5582         if ((csio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
5583                 struct ata_zoned_info_log *zi_log;
5584                 uint32_t valid_len;
5585                 size_t needed_size;
5586
5587                 zi_log = (struct ata_zoned_info_log *)csio->data_ptr;
5588
5589                 valid_len = csio->dxfer_len - csio->resid;
5590                 needed_size = __offsetof(struct ata_zoned_info_log,
5591                     version_info) + 1 + sizeof(zi_log->version_info);
5592                 if (valid_len >= needed_size) {
5593                         uint64_t tmpvar;
5594
5595                         tmpvar = le64dec(zi_log->zoned_cap);
5596                         if (tmpvar & ATA_ZDI_CAP_VALID) {
5597                                 if (tmpvar & ATA_ZDI_CAP_URSWRZ)
5598                                         softc->zone_flags |=
5599                                             DA_ZONE_FLAG_URSWRZ;
5600                                 else
5601                                         softc->zone_flags &=
5602                                             ~DA_ZONE_FLAG_URSWRZ;
5603                         }
5604                         tmpvar = le64dec(zi_log->optimal_seq_zones);
5605                         if (tmpvar & ATA_ZDI_OPT_SEQ_VALID) {
5606                                 softc->zone_flags |= DA_ZONE_FLAG_OPT_SEQ_SET;
5607                                 softc->optimal_seq_zones = (tmpvar &
5608                                     ATA_ZDI_OPT_SEQ_MASK);
5609                         } else {
5610                                 softc->zone_flags &= ~DA_ZONE_FLAG_OPT_SEQ_SET;
5611                                 softc->optimal_seq_zones = 0;
5612                         }
5613
5614                         tmpvar =le64dec(zi_log->optimal_nonseq_zones);
5615                         if (tmpvar & ATA_ZDI_OPT_NS_VALID) {
5616                                 softc->zone_flags |=
5617                                     DA_ZONE_FLAG_OPT_NONSEQ_SET;
5618                                 softc->optimal_nonseq_zones =
5619                                     (tmpvar & ATA_ZDI_OPT_NS_MASK);
5620                         } else {
5621                                 softc->zone_flags &=
5622                                     ~DA_ZONE_FLAG_OPT_NONSEQ_SET;
5623                                 softc->optimal_nonseq_zones = 0;
5624                         }
5625
5626                         tmpvar = le64dec(zi_log->max_seq_req_zones);
5627                         if (tmpvar & ATA_ZDI_MAX_SEQ_VALID) {
5628                                 softc->zone_flags |= DA_ZONE_FLAG_MAX_SEQ_SET;
5629                                 softc->max_seq_zones =
5630                                     (tmpvar & ATA_ZDI_MAX_SEQ_MASK);
5631                         } else {
5632                                 softc->zone_flags &= ~DA_ZONE_FLAG_MAX_SEQ_SET;
5633                                 softc->max_seq_zones = 0;
5634                         }
5635                 }
5636         } else {
5637                 error = daerror(done_ccb, CAM_RETRY_SELTO,
5638                                 SF_RETRY_UA|SF_NO_PRINT);
5639                 if (error == ERESTART)
5640                         return;
5641                 else if (error != 0) {
5642                         softc->flags &= ~DA_FLAG_CAN_ATA_ZONE;
5643                         softc->flags &= ~DA_ZONE_FLAG_SET_MASK;
5644
5645                         if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
5646                                 /* Don't wedge this device's queue */
5647                                 cam_release_devq(done_ccb->ccb_h.path,
5648                                                  /*relsim_flags*/0,
5649                                                  /*reduction*/0,
5650                                                  /*timeout*/0,
5651                                                  /*getcount_only*/0);
5652                         }
5653                 }
5654
5655         }
5656
5657         free(csio->data_ptr, M_SCSIDA);
5658
5659         daprobedone(periph, done_ccb);
5660         return;
5661 }
5662
5663 static void
5664 dadone_probezone(struct cam_periph *periph, union ccb *done_ccb)
5665 {
5666         struct da_softc *softc;
5667         struct ccb_scsiio *csio;
5668         int error;
5669
5670         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dadone_probezone\n"));
5671
5672         softc = (struct da_softc *)periph->softc;
5673         csio = &done_ccb->csio;
5674
5675         cam_periph_assert(periph, MA_OWNED);
5676
5677         if ((csio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
5678                 uint32_t valid_len;
5679                 size_t needed_len;
5680                 struct scsi_vpd_zoned_bdc *zoned_bdc;
5681
5682                 error = 0;
5683                 zoned_bdc = (struct scsi_vpd_zoned_bdc *)csio->data_ptr;
5684                 valid_len = csio->dxfer_len - csio->resid;
5685                 needed_len = __offsetof(struct scsi_vpd_zoned_bdc,
5686                     max_seq_req_zones) + 1 +
5687                     sizeof(zoned_bdc->max_seq_req_zones);
5688                 if ((valid_len >= needed_len)
5689                  && (scsi_2btoul(zoned_bdc->page_length) >= SVPD_ZBDC_PL)) {
5690                         if (zoned_bdc->flags & SVPD_ZBDC_URSWRZ)
5691                                 softc->zone_flags |= DA_ZONE_FLAG_URSWRZ;
5692                         else
5693                                 softc->zone_flags &= ~DA_ZONE_FLAG_URSWRZ;
5694                         softc->optimal_seq_zones =
5695                             scsi_4btoul(zoned_bdc->optimal_seq_zones);
5696                         softc->zone_flags |= DA_ZONE_FLAG_OPT_SEQ_SET;
5697                         softc->optimal_nonseq_zones = scsi_4btoul(
5698                             zoned_bdc->optimal_nonseq_zones);
5699                         softc->zone_flags |= DA_ZONE_FLAG_OPT_NONSEQ_SET;
5700                         softc->max_seq_zones =
5701                             scsi_4btoul(zoned_bdc->max_seq_req_zones);
5702                         softc->zone_flags |= DA_ZONE_FLAG_MAX_SEQ_SET;
5703                 }
5704                 /*
5705                  * All of the zone commands are mandatory for SCSI
5706                  * devices.
5707                  *
5708                  * XXX KDM this is valid as of September 2015.
5709                  * Re-check this assumption once the SAT spec is
5710                  * updated to support SCSI ZBC to ATA ZAC mapping.
5711                  * Since ATA allows zone commands to be reported
5712                  * as supported or not, this may not necessarily
5713                  * be true for an ATA device behind a SAT (SCSI to
5714                  * ATA Translation) layer.
5715                  */
5716                 softc->zone_flags |= DA_ZONE_FLAG_SUP_MASK;
5717         } else {
5718                 error = daerror(done_ccb, CAM_RETRY_SELTO,
5719                                 SF_RETRY_UA|SF_NO_PRINT);
5720                 if (error == ERESTART)
5721                         return;
5722                 else if (error != 0) {
5723                         if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
5724                                 /* Don't wedge this device's queue */
5725                                 cam_release_devq(done_ccb->ccb_h.path,
5726                                                  /*relsim_flags*/0,
5727                                                  /*reduction*/0,
5728                                                  /*timeout*/0,
5729                                                  /*getcount_only*/0);
5730                         }
5731                 }
5732         }
5733
5734         free(csio->data_ptr, M_SCSIDA);
5735
5736         daprobedone(periph, done_ccb);
5737         return;
5738 }
5739
5740 static void
5741 dadone_tur(struct cam_periph *periph, union ccb *done_ccb)
5742 {
5743         struct da_softc *softc;
5744         struct ccb_scsiio *csio;
5745
5746         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dadone_tur\n"));
5747
5748         softc = (struct da_softc *)periph->softc;
5749         csio = &done_ccb->csio;
5750
5751         cam_periph_assert(periph, MA_OWNED);
5752
5753         if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
5754
5755                 if (daerror(done_ccb, CAM_RETRY_SELTO,
5756                     SF_RETRY_UA | SF_NO_RECOVERY | SF_NO_PRINT) == ERESTART)
5757                         return; /* Will complete again, keep reference */
5758                 if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0)
5759                         cam_release_devq(done_ccb->ccb_h.path,
5760                                          /*relsim_flags*/0,
5761                                          /*reduction*/0,
5762                                          /*timeout*/0,
5763                                          /*getcount_only*/0);
5764         }
5765         xpt_release_ccb(done_ccb);
5766         softc->flags &= ~DA_FLAG_TUR_PENDING;
5767         da_periph_release_locked(periph, DA_REF_TUR);
5768         return;
5769 }
5770
5771 static void
5772 dareprobe(struct cam_periph *periph)
5773 {
5774         struct da_softc   *softc;
5775         int status;
5776
5777         softc = (struct da_softc *)periph->softc;
5778
5779         /* Probe in progress; don't interfere. */
5780         if (softc->state != DA_STATE_NORMAL)
5781                 return;
5782
5783         status = da_periph_acquire(periph, DA_REF_REPROBE);
5784         KASSERT(status == 0, ("dareprobe: cam_periph_acquire failed"));
5785
5786         softc->state = DA_STATE_PROBE_WP;
5787         xpt_schedule(periph, CAM_PRIORITY_DEV);
5788 }
5789
5790 static int
5791 daerror(union ccb *ccb, u_int32_t cam_flags, u_int32_t sense_flags)
5792 {
5793         struct da_softc   *softc;
5794         struct cam_periph *periph;
5795         int error, error_code, sense_key, asc, ascq;
5796
5797 #if defined(BUF_TRACKING) || defined(FULL_BUF_TRACKING)
5798         if (ccb->csio.bio != NULL)
5799                 biotrack(ccb->csio.bio, __func__);
5800 #endif
5801
5802         periph = xpt_path_periph(ccb->ccb_h.path);
5803         softc = (struct da_softc *)periph->softc;
5804
5805         cam_periph_assert(periph, MA_OWNED);
5806
5807         /*
5808          * Automatically detect devices that do not support
5809          * READ(6)/WRITE(6) and upgrade to using 10 byte cdbs.
5810          */
5811         error = 0;
5812         if ((ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_INVALID) {
5813                 error = cmd6workaround(ccb);
5814         } else if (scsi_extract_sense_ccb(ccb,
5815             &error_code, &sense_key, &asc, &ascq)) {
5816                 if (sense_key == SSD_KEY_ILLEGAL_REQUEST)
5817                         error = cmd6workaround(ccb);
5818                 /*
5819                  * If the target replied with CAPACITY DATA HAS CHANGED UA,
5820                  * query the capacity and notify upper layers.
5821                  */
5822                 else if (sense_key == SSD_KEY_UNIT_ATTENTION &&
5823                     asc == 0x2A && ascq == 0x09) {
5824                         xpt_print(periph->path, "Capacity data has changed\n");
5825                         softc->flags &= ~DA_FLAG_PROBED;
5826                         dareprobe(periph);
5827                         sense_flags |= SF_NO_PRINT;
5828                 } else if (sense_key == SSD_KEY_UNIT_ATTENTION &&
5829                     asc == 0x28 && ascq == 0x00) {
5830                         softc->flags &= ~DA_FLAG_PROBED;
5831                         disk_media_changed(softc->disk, M_NOWAIT);
5832                 } else if (sense_key == SSD_KEY_UNIT_ATTENTION &&
5833                     asc == 0x3F && ascq == 0x03) {
5834                         xpt_print(periph->path, "INQUIRY data has changed\n");
5835                         softc->flags &= ~DA_FLAG_PROBED;
5836                         dareprobe(periph);
5837                         sense_flags |= SF_NO_PRINT;
5838                 } else if (sense_key == SSD_KEY_NOT_READY &&
5839                     asc == 0x3a && (softc->flags & DA_FLAG_PACK_INVALID) == 0) {
5840                         softc->flags |= DA_FLAG_PACK_INVALID;
5841                         disk_media_gone(softc->disk, M_NOWAIT);
5842                 }
5843         }
5844         if (error == ERESTART)
5845                 return (ERESTART);
5846
5847 #ifdef CAM_IO_STATS
5848         switch (ccb->ccb_h.status & CAM_STATUS_MASK) {
5849         case CAM_CMD_TIMEOUT:
5850                 softc->timeouts++;
5851                 break;
5852         case CAM_REQ_ABORTED:
5853         case CAM_REQ_CMP_ERR:
5854         case CAM_REQ_TERMIO:
5855         case CAM_UNREC_HBA_ERROR:
5856         case CAM_DATA_RUN_ERR:
5857                 softc->errors++;
5858                 break;
5859         default:
5860                 break;
5861         }
5862 #endif
5863
5864         /*
5865          * XXX
5866          * Until we have a better way of doing pack validation,
5867          * don't treat UAs as errors.
5868          */
5869         sense_flags |= SF_RETRY_UA;
5870
5871         if (softc->quirks & DA_Q_RETRY_BUSY)
5872                 sense_flags |= SF_RETRY_BUSY;
5873         return(cam_periph_error(ccb, cam_flags, sense_flags));
5874 }
5875
5876 static void
5877 damediapoll(void *arg)
5878 {
5879         struct cam_periph *periph = arg;
5880         struct da_softc *softc = periph->softc;
5881
5882         if (!cam_iosched_has_work_flags(softc->cam_iosched, DA_WORK_TUR) &&
5883             (softc->flags & DA_FLAG_TUR_PENDING) == 0 &&
5884             LIST_EMPTY(&softc->pending_ccbs)) {
5885                 if (da_periph_acquire(periph, DA_REF_TUR) == 0) {
5886                         cam_iosched_set_work_flags(softc->cam_iosched, DA_WORK_TUR);
5887                         daschedule(periph);
5888                 }
5889         }
5890         /* Queue us up again */
5891         if (da_poll_period != 0)
5892                 callout_schedule(&softc->mediapoll_c, da_poll_period * hz);
5893 }
5894
5895 static void
5896 daprevent(struct cam_periph *periph, int action)
5897 {
5898         struct  da_softc *softc;
5899         union   ccb *ccb;
5900         int     error;
5901
5902         cam_periph_assert(periph, MA_OWNED);
5903         softc = (struct da_softc *)periph->softc;
5904
5905         if (((action == PR_ALLOW)
5906           && (softc->flags & DA_FLAG_PACK_LOCKED) == 0)
5907          || ((action == PR_PREVENT)
5908           && (softc->flags & DA_FLAG_PACK_LOCKED) != 0)) {
5909                 return;
5910         }
5911
5912         ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL);
5913
5914         scsi_prevent(&ccb->csio,
5915                      /*retries*/1,
5916                      /*cbcfp*/NULL,
5917                      MSG_SIMPLE_Q_TAG,
5918                      action,
5919                      SSD_FULL_SIZE,
5920                      5000);
5921
5922         error = cam_periph_runccb(ccb, daerror, CAM_RETRY_SELTO,
5923             SF_RETRY_UA | SF_NO_PRINT, softc->disk->d_devstat);
5924
5925         if (error == 0) {
5926                 if (action == PR_ALLOW)
5927                         softc->flags &= ~DA_FLAG_PACK_LOCKED;
5928                 else
5929                         softc->flags |= DA_FLAG_PACK_LOCKED;
5930         }
5931
5932         xpt_release_ccb(ccb);
5933 }
5934
5935 static void
5936 dasetgeom(struct cam_periph *periph, uint32_t block_len, uint64_t maxsector,
5937           struct scsi_read_capacity_data_long *rcaplong, size_t rcap_len)
5938 {
5939         struct ccb_calc_geometry ccg;
5940         struct da_softc *softc;
5941         struct disk_params *dp;
5942         u_int lbppbe, lalba;
5943         int error;
5944
5945         softc = (struct da_softc *)periph->softc;
5946
5947         dp = &softc->params;
5948         dp->secsize = block_len;
5949         dp->sectors = maxsector + 1;
5950         if (rcaplong != NULL) {
5951                 lbppbe = rcaplong->prot_lbppbe & SRC16_LBPPBE;
5952                 lalba = scsi_2btoul(rcaplong->lalba_lbp);
5953                 lalba &= SRC16_LALBA_A;
5954         } else {
5955                 lbppbe = 0;
5956                 lalba = 0;
5957         }
5958
5959         if (lbppbe > 0) {
5960                 dp->stripesize = block_len << lbppbe;
5961                 dp->stripeoffset = (dp->stripesize - block_len * lalba) %
5962                     dp->stripesize;
5963         } else if (softc->quirks & DA_Q_4K) {
5964                 dp->stripesize = 4096;
5965                 dp->stripeoffset = 0;
5966         } else if (softc->unmap_gran != 0) {
5967                 dp->stripesize = block_len * softc->unmap_gran;
5968                 dp->stripeoffset = (dp->stripesize - block_len *
5969                     softc->unmap_gran_align) % dp->stripesize;
5970         } else {
5971                 dp->stripesize = 0;
5972                 dp->stripeoffset = 0;
5973         }
5974         /*
5975          * Have the controller provide us with a geometry
5976          * for this disk.  The only time the geometry
5977          * matters is when we boot and the controller
5978          * is the only one knowledgeable enough to come
5979          * up with something that will make this a bootable
5980          * device.
5981          */
5982         xpt_setup_ccb(&ccg.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
5983         ccg.ccb_h.func_code = XPT_CALC_GEOMETRY;
5984         ccg.block_size = dp->secsize;
5985         ccg.volume_size = dp->sectors;
5986         ccg.heads = 0;
5987         ccg.secs_per_track = 0;
5988         ccg.cylinders = 0;
5989         xpt_action((union ccb*)&ccg);
5990         if ((ccg.ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
5991                 /*
5992                  * We don't know what went wrong here- but just pick
5993                  * a geometry so we don't have nasty things like divide
5994                  * by zero.
5995                  */
5996                 dp->heads = 255;
5997                 dp->secs_per_track = 255;
5998                 dp->cylinders = dp->sectors / (255 * 255);
5999                 if (dp->cylinders == 0) {
6000                         dp->cylinders = 1;
6001                 }
6002         } else {
6003                 dp->heads = ccg.heads;
6004                 dp->secs_per_track = ccg.secs_per_track;
6005                 dp->cylinders = ccg.cylinders;
6006         }
6007
6008         /*
6009          * If the user supplied a read capacity buffer, and if it is
6010          * different than the previous buffer, update the data in the EDT.
6011          * If it's the same, we don't bother.  This avoids sending an
6012          * update every time someone opens this device.
6013          */
6014         if ((rcaplong != NULL)
6015          && (bcmp(rcaplong, &softc->rcaplong,
6016                   min(sizeof(softc->rcaplong), rcap_len)) != 0)) {
6017                 struct ccb_dev_advinfo cdai;
6018
6019                 xpt_setup_ccb(&cdai.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
6020                 cdai.ccb_h.func_code = XPT_DEV_ADVINFO;
6021                 cdai.buftype = CDAI_TYPE_RCAPLONG;
6022                 cdai.flags = CDAI_FLAG_STORE;
6023                 cdai.bufsiz = rcap_len;
6024                 cdai.buf = (uint8_t *)rcaplong;
6025                 xpt_action((union ccb *)&cdai);
6026                 if ((cdai.ccb_h.status & CAM_DEV_QFRZN) != 0)
6027                         cam_release_devq(cdai.ccb_h.path, 0, 0, 0, FALSE);
6028                 if (cdai.ccb_h.status != CAM_REQ_CMP) {
6029                         xpt_print(periph->path, "%s: failed to set read "
6030                                   "capacity advinfo\n", __func__);
6031                         /* Use cam_error_print() to decode the status */
6032                         cam_error_print((union ccb *)&cdai, CAM_ESF_CAM_STATUS,
6033                                         CAM_EPF_ALL);
6034                 } else {
6035                         bcopy(rcaplong, &softc->rcaplong,
6036                               min(sizeof(softc->rcaplong), rcap_len));
6037                 }
6038         }
6039
6040         softc->disk->d_sectorsize = softc->params.secsize;
6041         softc->disk->d_mediasize = softc->params.secsize * (off_t)softc->params.sectors;
6042         softc->disk->d_stripesize = softc->params.stripesize;
6043         softc->disk->d_stripeoffset = softc->params.stripeoffset;
6044         /* XXX: these are not actually "firmware" values, so they may be wrong */
6045         softc->disk->d_fwsectors = softc->params.secs_per_track;
6046         softc->disk->d_fwheads = softc->params.heads;
6047         softc->disk->d_devstat->block_size = softc->params.secsize;
6048         softc->disk->d_devstat->flags &= ~DEVSTAT_BS_UNAVAILABLE;
6049
6050         error = disk_resize(softc->disk, M_NOWAIT);
6051         if (error != 0)
6052                 xpt_print(periph->path, "disk_resize(9) failed, error = %d\n", error);
6053 }
6054
6055 static void
6056 dasendorderedtag(void *arg)
6057 {
6058         struct cam_periph *periph = arg;
6059         struct da_softc *softc = periph->softc;
6060
6061         cam_periph_assert(periph, MA_OWNED);
6062         if (da_send_ordered) {
6063                 if (!LIST_EMPTY(&softc->pending_ccbs)) {
6064                         if ((softc->flags & DA_FLAG_WAS_OTAG) == 0)
6065                                 softc->flags |= DA_FLAG_NEED_OTAG;
6066                         softc->flags &= ~DA_FLAG_WAS_OTAG;
6067                 }
6068         }
6069
6070         /* Queue us up again */
6071         callout_reset(&softc->sendordered_c,
6072             (da_default_timeout * hz) / DA_ORDEREDTAG_INTERVAL,
6073             dasendorderedtag, periph);
6074 }
6075
6076 /*
6077  * Step through all DA peripheral drivers, and if the device is still open,
6078  * sync the disk cache to physical media.
6079  */
6080 static void
6081 dashutdown(void * arg, int howto)
6082 {
6083         struct cam_periph *periph;
6084         struct da_softc *softc;
6085         union ccb *ccb;
6086         int error;
6087
6088         CAM_PERIPH_FOREACH(periph, &dadriver) {
6089                 softc = (struct da_softc *)periph->softc;
6090                 if (SCHEDULER_STOPPED()) {
6091                         /* If we paniced with the lock held, do not recurse. */
6092                         if (!cam_periph_owned(periph) &&
6093                             (softc->flags & DA_FLAG_OPEN)) {
6094                                 dadump(softc->disk, NULL, 0, 0, 0);
6095                         }
6096                         continue;
6097                 }
6098                 cam_periph_lock(periph);
6099
6100                 /*
6101                  * We only sync the cache if the drive is still open, and
6102                  * if the drive is capable of it..
6103                  */
6104                 if (((softc->flags & DA_FLAG_OPEN) == 0)
6105                  || (softc->quirks & DA_Q_NO_SYNC_CACHE)) {
6106                         cam_periph_unlock(periph);
6107                         continue;
6108                 }
6109
6110                 ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL);
6111                 scsi_synchronize_cache(&ccb->csio,
6112                                        /*retries*/0,
6113                                        /*cbfcnp*/NULL,
6114                                        MSG_SIMPLE_Q_TAG,
6115                                        /*begin_lba*/0, /* whole disk */
6116                                        /*lb_count*/0,
6117                                        SSD_FULL_SIZE,
6118                                        60 * 60 * 1000);
6119
6120                 error = cam_periph_runccb(ccb, daerror, /*cam_flags*/0,
6121                     /*sense_flags*/ SF_NO_RECOVERY | SF_NO_RETRY | SF_QUIET_IR,
6122                     softc->disk->d_devstat);
6123                 if (error != 0)
6124                         xpt_print(periph->path, "Synchronize cache failed\n");
6125                 xpt_release_ccb(ccb);
6126                 cam_periph_unlock(periph);
6127         }
6128 }
6129
6130 #else /* !_KERNEL */
6131
6132 /*
6133  * XXX These are only left out of the kernel build to silence warnings.  If,
6134  * for some reason these functions are used in the kernel, the ifdefs should
6135  * be moved so they are included both in the kernel and userland.
6136  */
6137 void
6138 scsi_format_unit(struct ccb_scsiio *csio, u_int32_t retries,
6139                  void (*cbfcnp)(struct cam_periph *, union ccb *),
6140                  u_int8_t tag_action, u_int8_t byte2, u_int16_t ileave,
6141                  u_int8_t *data_ptr, u_int32_t dxfer_len, u_int8_t sense_len,
6142                  u_int32_t timeout)
6143 {
6144         struct scsi_format_unit *scsi_cmd;
6145
6146         scsi_cmd = (struct scsi_format_unit *)&csio->cdb_io.cdb_bytes;
6147         scsi_cmd->opcode = FORMAT_UNIT;
6148         scsi_cmd->byte2 = byte2;
6149         scsi_ulto2b(ileave, scsi_cmd->interleave);
6150
6151         cam_fill_csio(csio,
6152                       retries,
6153                       cbfcnp,
6154                       /*flags*/ (dxfer_len > 0) ? CAM_DIR_OUT : CAM_DIR_NONE,
6155                       tag_action,
6156                       data_ptr,
6157                       dxfer_len,
6158                       sense_len,
6159                       sizeof(*scsi_cmd),
6160                       timeout);
6161 }
6162
6163 void
6164 scsi_read_defects(struct ccb_scsiio *csio, uint32_t retries,
6165                   void (*cbfcnp)(struct cam_periph *, union ccb *),
6166                   uint8_t tag_action, uint8_t list_format,
6167                   uint32_t addr_desc_index, uint8_t *data_ptr,
6168                   uint32_t dxfer_len, int minimum_cmd_size,
6169                   uint8_t sense_len, uint32_t timeout)
6170 {
6171         uint8_t cdb_len;
6172
6173         /*
6174          * These conditions allow using the 10 byte command.  Otherwise we
6175          * need to use the 12 byte command.
6176          */
6177         if ((minimum_cmd_size <= 10)
6178          && (addr_desc_index == 0)
6179          && (dxfer_len <= SRDD10_MAX_LENGTH)) {
6180                 struct scsi_read_defect_data_10 *cdb10;
6181
6182                 cdb10 = (struct scsi_read_defect_data_10 *)
6183                         &csio->cdb_io.cdb_bytes;
6184
6185                 cdb_len = sizeof(*cdb10);
6186                 bzero(cdb10, cdb_len);
6187                 cdb10->opcode = READ_DEFECT_DATA_10;
6188                 cdb10->format = list_format;
6189                 scsi_ulto2b(dxfer_len, cdb10->alloc_length);
6190         } else {
6191                 struct scsi_read_defect_data_12 *cdb12;
6192
6193                 cdb12 = (struct scsi_read_defect_data_12 *)
6194                         &csio->cdb_io.cdb_bytes;
6195
6196                 cdb_len = sizeof(*cdb12);
6197                 bzero(cdb12, cdb_len);
6198                 cdb12->opcode = READ_DEFECT_DATA_12;
6199                 cdb12->format = list_format;
6200                 scsi_ulto4b(dxfer_len, cdb12->alloc_length);
6201                 scsi_ulto4b(addr_desc_index, cdb12->address_descriptor_index);
6202         }
6203
6204         cam_fill_csio(csio,
6205                       retries,
6206                       cbfcnp,
6207                       /*flags*/ CAM_DIR_IN,
6208                       tag_action,
6209                       data_ptr,
6210                       dxfer_len,
6211                       sense_len,
6212                       cdb_len,
6213                       timeout);
6214 }
6215
6216 void
6217 scsi_sanitize(struct ccb_scsiio *csio, u_int32_t retries,
6218               void (*cbfcnp)(struct cam_periph *, union ccb *),
6219               u_int8_t tag_action, u_int8_t byte2, u_int16_t control,
6220               u_int8_t *data_ptr, u_int32_t dxfer_len, u_int8_t sense_len,
6221               u_int32_t timeout)
6222 {
6223         struct scsi_sanitize *scsi_cmd;
6224
6225         scsi_cmd = (struct scsi_sanitize *)&csio->cdb_io.cdb_bytes;
6226         scsi_cmd->opcode = SANITIZE;
6227         scsi_cmd->byte2 = byte2;
6228         scsi_cmd->control = control;
6229         scsi_ulto2b(dxfer_len, scsi_cmd->length);
6230
6231         cam_fill_csio(csio,
6232                       retries,
6233                       cbfcnp,
6234                       /*flags*/ (dxfer_len > 0) ? CAM_DIR_OUT : CAM_DIR_NONE,
6235                       tag_action,
6236                       data_ptr,
6237                       dxfer_len,
6238                       sense_len,
6239                       sizeof(*scsi_cmd),
6240                       timeout);
6241 }
6242
6243 #endif /* _KERNEL */
6244
6245 void
6246 scsi_zbc_out(struct ccb_scsiio *csio, uint32_t retries,
6247              void (*cbfcnp)(struct cam_periph *, union ccb *),
6248              uint8_t tag_action, uint8_t service_action, uint64_t zone_id,
6249              uint8_t zone_flags, uint8_t *data_ptr, uint32_t dxfer_len,
6250              uint8_t sense_len, uint32_t timeout)
6251 {
6252         struct scsi_zbc_out *scsi_cmd;
6253
6254         scsi_cmd = (struct scsi_zbc_out *)&csio->cdb_io.cdb_bytes;
6255         scsi_cmd->opcode = ZBC_OUT;
6256         scsi_cmd->service_action = service_action;
6257         scsi_u64to8b(zone_id, scsi_cmd->zone_id);
6258         scsi_cmd->zone_flags = zone_flags;
6259
6260         cam_fill_csio(csio,
6261                       retries,
6262                       cbfcnp,
6263                       /*flags*/ (dxfer_len > 0) ? CAM_DIR_OUT : CAM_DIR_NONE,
6264                       tag_action,
6265                       data_ptr,
6266                       dxfer_len,
6267                       sense_len,
6268                       sizeof(*scsi_cmd),
6269                       timeout);
6270 }
6271
6272 void
6273 scsi_zbc_in(struct ccb_scsiio *csio, uint32_t retries,
6274             void (*cbfcnp)(struct cam_periph *, union ccb *),
6275             uint8_t tag_action, uint8_t service_action, uint64_t zone_start_lba,
6276             uint8_t zone_options, uint8_t *data_ptr, uint32_t dxfer_len,
6277             uint8_t sense_len, uint32_t timeout)
6278 {
6279         struct scsi_zbc_in *scsi_cmd;
6280
6281         scsi_cmd = (struct scsi_zbc_in *)&csio->cdb_io.cdb_bytes;
6282         scsi_cmd->opcode = ZBC_IN;
6283         scsi_cmd->service_action = service_action;
6284         scsi_ulto4b(dxfer_len, scsi_cmd->length);
6285         scsi_u64to8b(zone_start_lba, scsi_cmd->zone_start_lba);
6286         scsi_cmd->zone_options = zone_options;
6287
6288         cam_fill_csio(csio,
6289                       retries,
6290                       cbfcnp,
6291                       /*flags*/ (dxfer_len > 0) ? CAM_DIR_IN : CAM_DIR_NONE,
6292                       tag_action,
6293                       data_ptr,
6294                       dxfer_len,
6295                       sense_len,
6296                       sizeof(*scsi_cmd),
6297                       timeout);
6298
6299 }
6300
6301 int
6302 scsi_ata_zac_mgmt_out(struct ccb_scsiio *csio, uint32_t retries,
6303                       void (*cbfcnp)(struct cam_periph *, union ccb *),
6304                       uint8_t tag_action, int use_ncq,
6305                       uint8_t zm_action, uint64_t zone_id, uint8_t zone_flags,
6306                       uint8_t *data_ptr, uint32_t dxfer_len,
6307                       uint8_t *cdb_storage, size_t cdb_storage_len,
6308                       uint8_t sense_len, uint32_t timeout)
6309 {
6310         uint8_t command_out, protocol, ata_flags;
6311         uint16_t features_out;
6312         uint32_t sectors_out, auxiliary;
6313         int retval;
6314
6315         retval = 0;
6316
6317         if (use_ncq == 0) {
6318                 command_out = ATA_ZAC_MANAGEMENT_OUT;
6319                 features_out = (zm_action & 0xf) | (zone_flags << 8);
6320                 ata_flags = AP_FLAG_BYT_BLOK_BLOCKS;
6321                 if (dxfer_len == 0) {
6322                         protocol = AP_PROTO_NON_DATA;
6323                         ata_flags |= AP_FLAG_TLEN_NO_DATA;
6324                         sectors_out = 0;
6325                 } else {
6326                         protocol = AP_PROTO_DMA;
6327                         ata_flags |= AP_FLAG_TLEN_SECT_CNT |
6328                                      AP_FLAG_TDIR_TO_DEV;
6329                         sectors_out = ((dxfer_len >> 9) & 0xffff);
6330                 }
6331                 auxiliary = 0;
6332         } else {
6333                 ata_flags = AP_FLAG_BYT_BLOK_BLOCKS;
6334                 if (dxfer_len == 0) {
6335                         command_out = ATA_NCQ_NON_DATA;
6336                         features_out = ATA_NCQ_ZAC_MGMT_OUT;
6337                         /*
6338                          * We're assuming the SCSI to ATA translation layer
6339                          * will set the NCQ tag number in the tag field.
6340                          * That isn't clear from the SAT-4 spec (as of rev 05).
6341                          */
6342                         sectors_out = 0;
6343                         ata_flags |= AP_FLAG_TLEN_NO_DATA;
6344                 } else {
6345                         command_out = ATA_SEND_FPDMA_QUEUED;
6346                         /*
6347                          * Note that we're defaulting to normal priority,
6348                          * and assuming that the SCSI to ATA translation
6349                          * layer will insert the NCQ tag number in the tag
6350                          * field.  That isn't clear in the SAT-4 spec (as
6351                          * of rev 05).
6352                          */
6353                         sectors_out = ATA_SFPDMA_ZAC_MGMT_OUT << 8;
6354
6355                         ata_flags |= AP_FLAG_TLEN_FEAT |
6356                                      AP_FLAG_TDIR_TO_DEV;
6357
6358                         /*
6359                          * For SEND FPDMA QUEUED, the transfer length is
6360                          * encoded in the FEATURE register, and 0 means
6361                          * that 65536 512 byte blocks are to be tranferred.
6362                          * In practice, it seems unlikely that we'll see
6363                          * a transfer that large, and it may confuse the
6364                          * the SAT layer, because generally that means that
6365                          * 0 bytes should be transferred.
6366                          */
6367                         if (dxfer_len == (65536 * 512)) {
6368                                 features_out = 0;
6369                         } else if (dxfer_len <= (65535 * 512)) {
6370                                 features_out = ((dxfer_len >> 9) & 0xffff);
6371                         } else {
6372                                 /* The transfer is too big. */
6373                                 retval = 1;
6374                                 goto bailout;
6375                         }
6376
6377                 }
6378
6379                 auxiliary = (zm_action & 0xf) | (zone_flags << 8);
6380                 protocol = AP_PROTO_FPDMA;
6381         }
6382
6383         protocol |= AP_EXTEND;
6384
6385         retval = scsi_ata_pass(csio,
6386             retries,
6387             cbfcnp,
6388             /*flags*/ (dxfer_len > 0) ? CAM_DIR_OUT : CAM_DIR_NONE,
6389             tag_action,
6390             /*protocol*/ protocol,
6391             /*ata_flags*/ ata_flags,
6392             /*features*/ features_out,
6393             /*sector_count*/ sectors_out,
6394             /*lba*/ zone_id,
6395             /*command*/ command_out,
6396             /*device*/ 0,
6397             /*icc*/ 0,
6398             /*auxiliary*/ auxiliary,
6399             /*control*/ 0,
6400             /*data_ptr*/ data_ptr,
6401             /*dxfer_len*/ dxfer_len,
6402             /*cdb_storage*/ cdb_storage,
6403             /*cdb_storage_len*/ cdb_storage_len,
6404             /*minimum_cmd_size*/ 0,
6405             /*sense_len*/ SSD_FULL_SIZE,
6406             /*timeout*/ timeout);
6407
6408 bailout:
6409
6410         return (retval);
6411 }
6412
6413 int
6414 scsi_ata_zac_mgmt_in(struct ccb_scsiio *csio, uint32_t retries,
6415                      void (*cbfcnp)(struct cam_periph *, union ccb *),
6416                      uint8_t tag_action, int use_ncq,
6417                      uint8_t zm_action, uint64_t zone_id, uint8_t zone_flags,
6418                      uint8_t *data_ptr, uint32_t dxfer_len,
6419                      uint8_t *cdb_storage, size_t cdb_storage_len,
6420                      uint8_t sense_len, uint32_t timeout)
6421 {
6422         uint8_t command_out, protocol;
6423         uint16_t features_out, sectors_out;
6424         uint32_t auxiliary;
6425         int ata_flags;
6426         int retval;
6427
6428         retval = 0;
6429         ata_flags = AP_FLAG_TDIR_FROM_DEV | AP_FLAG_BYT_BLOK_BLOCKS;
6430
6431         if (use_ncq == 0) {
6432                 command_out = ATA_ZAC_MANAGEMENT_IN;
6433                 /* XXX KDM put a macro here */
6434                 features_out = (zm_action & 0xf) | (zone_flags << 8);
6435                 sectors_out = dxfer_len >> 9; /* XXX KDM macro */
6436                 protocol = AP_PROTO_DMA;
6437                 ata_flags |= AP_FLAG_TLEN_SECT_CNT;
6438                 auxiliary = 0;
6439         } else {
6440                 ata_flags |= AP_FLAG_TLEN_FEAT;
6441
6442                 command_out = ATA_RECV_FPDMA_QUEUED;
6443                 sectors_out = ATA_RFPDMA_ZAC_MGMT_IN << 8;
6444
6445                 /*
6446                  * For RECEIVE FPDMA QUEUED, the transfer length is
6447                  * encoded in the FEATURE register, and 0 means
6448                  * that 65536 512 byte blocks are to be tranferred.
6449                  * In practice, it seems unlikely that we'll see
6450                  * a transfer that large, and it may confuse the
6451                  * the SAT layer, because generally that means that
6452                  * 0 bytes should be transferred.
6453                  */
6454                 if (dxfer_len == (65536 * 512)) {
6455                         features_out = 0;
6456                 } else if (dxfer_len <= (65535 * 512)) {
6457                         features_out = ((dxfer_len >> 9) & 0xffff);
6458                 } else {
6459                         /* The transfer is too big. */
6460                         retval = 1;
6461                         goto bailout;
6462                 }
6463                 auxiliary = (zm_action & 0xf) | (zone_flags << 8),
6464                 protocol = AP_PROTO_FPDMA;
6465         }
6466
6467         protocol |= AP_EXTEND;
6468
6469         retval = scsi_ata_pass(csio,
6470             retries,
6471             cbfcnp,
6472             /*flags*/ CAM_DIR_IN,
6473             tag_action,
6474             /*protocol*/ protocol,
6475             /*ata_flags*/ ata_flags,
6476             /*features*/ features_out,
6477             /*sector_count*/ sectors_out,
6478             /*lba*/ zone_id,
6479             /*command*/ command_out,
6480             /*device*/ 0,
6481             /*icc*/ 0,
6482             /*auxiliary*/ auxiliary,
6483             /*control*/ 0,
6484             /*data_ptr*/ data_ptr,
6485             /*dxfer_len*/ (dxfer_len >> 9) * 512, /* XXX KDM */
6486             /*cdb_storage*/ cdb_storage,
6487             /*cdb_storage_len*/ cdb_storage_len,
6488             /*minimum_cmd_size*/ 0,
6489             /*sense_len*/ SSD_FULL_SIZE,
6490             /*timeout*/ timeout);
6491
6492 bailout:
6493         return (retval);
6494 }