sys/cam/ata/ata_da.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2009 Alexander Motin <mav@FreeBSD.org>
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer,
  12  *    without modification, immediately at the beginning of the file.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  18  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  19  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  20  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  21  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  22  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  23  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  24  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  25  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  26  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27  */
  28
  29 #include <sys/cdefs.h>
  30 __FBSDID("$FreeBSD$");
  31
  32 #include "opt_ada.h"
  33
  34 #include <sys/param.h>
  35
  36 #ifdef _KERNEL
  37 #include <sys/systm.h>
  38 #include <sys/kernel.h>
  39 #include <sys/bio.h>
  40 #include <sys/sysctl.h>
  41 #include <sys/taskqueue.h>
  42 #include <sys/lock.h>
  43 #include <sys/mutex.h>
  44 #include <sys/conf.h>
  45 #include <sys/devicestat.h>
  46 #include <sys/eventhandler.h>
  47 #include <sys/malloc.h>
  48 #include <sys/endian.h>
  49 #include <sys/cons.h>
  50 #include <sys/proc.h>
  51 #include <sys/reboot.h>
  52 #include <sys/sbuf.h>
  53 #include <geom/geom_disk.h>
  54 #endif /* _KERNEL */
  55
  56 #ifndef _KERNEL
  57 #include <stdio.h>
  58 #include <string.h>
  59 #endif /* _KERNEL */
  60
  61 #include <cam/cam.h>
  62 #include <cam/cam_ccb.h>
  63 #include <cam/cam_periph.h>
  64 #include <cam/cam_xpt_periph.h>
  65 #include <cam/scsi/scsi_all.h>
  66 #include <cam/scsi/scsi_da.h>
  67 #include <cam/cam_sim.h>
  68 #include <cam/cam_iosched.h>
  69
  70 #include <cam/ata/ata_all.h>
  71
  72 #include <machine/md_var.h>     /* geometry translation */
  73
  74 #ifdef _KERNEL
  75
  76 #define ATA_MAX_28BIT_LBA               268435455UL
  77
  78 extern int iosched_debug;
  79
  80 typedef enum {
  81         ADA_STATE_RAHEAD,
  82         ADA_STATE_WCACHE,
  83         ADA_STATE_LOGDIR,
  84         ADA_STATE_IDDIR,
  85         ADA_STATE_SUP_CAP,
  86         ADA_STATE_ZONE,
  87         ADA_STATE_NORMAL
  88 } ada_state;
  89
  90 typedef enum {
  91         ADA_FLAG_CAN_48BIT      = 0x00000002,
  92         ADA_FLAG_CAN_FLUSHCACHE = 0x00000004,
  93         ADA_FLAG_CAN_NCQ        = 0x00000008,
  94         ADA_FLAG_CAN_DMA        = 0x00000010,
  95         ADA_FLAG_NEED_OTAG      = 0x00000020,
  96         ADA_FLAG_WAS_OTAG       = 0x00000040,
  97         ADA_FLAG_CAN_TRIM       = 0x00000080,
  98         ADA_FLAG_OPEN           = 0x00000100,
  99         ADA_FLAG_SCTX_INIT      = 0x00000200,
 100         ADA_FLAG_CAN_CFA        = 0x00000400,
 101         ADA_FLAG_CAN_POWERMGT   = 0x00000800,
 102         ADA_FLAG_CAN_DMA48      = 0x00001000,
 103         ADA_FLAG_CAN_LOG        = 0x00002000,
 104         ADA_FLAG_CAN_IDLOG      = 0x00004000,
 105         ADA_FLAG_CAN_SUPCAP     = 0x00008000,
 106         ADA_FLAG_CAN_ZONE       = 0x00010000,
 107         ADA_FLAG_CAN_WCACHE     = 0x00020000,
 108         ADA_FLAG_CAN_RAHEAD     = 0x00040000,
 109         ADA_FLAG_PROBED         = 0x00080000,
 110         ADA_FLAG_ANNOUNCED      = 0x00100000,
 111         ADA_FLAG_DIRTY          = 0x00200000,
 112         ADA_FLAG_CAN_NCQ_TRIM   = 0x00400000,   /* CAN_TRIM also set */
 113         ADA_FLAG_PIM_ATA_EXT    = 0x00800000
 114 } ada_flags;
 115
 116 typedef enum {
 117         ADA_Q_NONE              = 0x00,
 118         ADA_Q_4K                = 0x01,
 119         ADA_Q_NCQ_TRIM_BROKEN   = 0x02,
 120         ADA_Q_LOG_BROKEN        = 0x04,
 121         ADA_Q_SMR_DM            = 0x08
 122 } ada_quirks;
 123
 124 #define ADA_Q_BIT_STRING        \
 125         "\020"                  \
 126         "\0014K"                \
 127         "\002NCQ_TRIM_BROKEN"   \
 128         "\003LOG_BROKEN"        \
 129         "\004SMR_DM"
 130
 131 typedef enum {
 132         ADA_CCB_RAHEAD          = 0x01,
 133         ADA_CCB_WCACHE          = 0x02,
 134         ADA_CCB_BUFFER_IO       = 0x03,
 135         ADA_CCB_DUMP            = 0x05,
 136         ADA_CCB_TRIM            = 0x06,
 137         ADA_CCB_LOGDIR          = 0x07,
 138         ADA_CCB_IDDIR           = 0x08,
 139         ADA_CCB_SUP_CAP         = 0x09,
 140         ADA_CCB_ZONE            = 0x0a,
 141         ADA_CCB_TYPE_MASK       = 0x0F,
 142 } ada_ccb_state;
 143
 144 typedef enum {
 145         ADA_ZONE_NONE           = 0x00,
 146         ADA_ZONE_DRIVE_MANAGED  = 0x01,
 147         ADA_ZONE_HOST_AWARE     = 0x02,
 148         ADA_ZONE_HOST_MANAGED   = 0x03
 149 } ada_zone_mode;
 150
 151 typedef enum {
 152         ADA_ZONE_FLAG_RZ_SUP            = 0x0001,
 153         ADA_ZONE_FLAG_OPEN_SUP          = 0x0002,
 154         ADA_ZONE_FLAG_CLOSE_SUP         = 0x0004,
 155         ADA_ZONE_FLAG_FINISH_SUP        = 0x0008,
 156         ADA_ZONE_FLAG_RWP_SUP           = 0x0010,
 157         ADA_ZONE_FLAG_SUP_MASK          = (ADA_ZONE_FLAG_RZ_SUP |
 158                                            ADA_ZONE_FLAG_OPEN_SUP |
 159                                            ADA_ZONE_FLAG_CLOSE_SUP |
 160                                            ADA_ZONE_FLAG_FINISH_SUP |
 161                                            ADA_ZONE_FLAG_RWP_SUP),
 162         ADA_ZONE_FLAG_URSWRZ            = 0x0020,
 163         ADA_ZONE_FLAG_OPT_SEQ_SET       = 0x0040,
 164         ADA_ZONE_FLAG_OPT_NONSEQ_SET    = 0x0080,
 165         ADA_ZONE_FLAG_MAX_SEQ_SET       = 0x0100,
 166         ADA_ZONE_FLAG_SET_MASK          = (ADA_ZONE_FLAG_OPT_SEQ_SET |
 167                                            ADA_ZONE_FLAG_OPT_NONSEQ_SET |
 168                                            ADA_ZONE_FLAG_MAX_SEQ_SET)
 169 } ada_zone_flags;
 170
 171 static struct ada_zone_desc {
 172         ada_zone_flags value;
 173         const char *desc;
 174 } ada_zone_desc_table[] = {
 175         {ADA_ZONE_FLAG_RZ_SUP, "Report Zones" },
 176         {ADA_ZONE_FLAG_OPEN_SUP, "Open" },
 177         {ADA_ZONE_FLAG_CLOSE_SUP, "Close" },
 178         {ADA_ZONE_FLAG_FINISH_SUP, "Finish" },
 179         {ADA_ZONE_FLAG_RWP_SUP, "Reset Write Pointer" },
 180 };
 181
 182
 183 /* Offsets into our private area for storing information */
 184 #define ccb_state       ppriv_field0
 185 #define ccb_bp          ppriv_ptr1
 186
 187 typedef enum {
 188         ADA_DELETE_NONE,
 189         ADA_DELETE_DISABLE,
 190         ADA_DELETE_CFA_ERASE,
 191         ADA_DELETE_DSM_TRIM,
 192         ADA_DELETE_NCQ_DSM_TRIM,
 193         ADA_DELETE_MIN = ADA_DELETE_CFA_ERASE,
 194         ADA_DELETE_MAX = ADA_DELETE_NCQ_DSM_TRIM,
 195 } ada_delete_methods;
 196
 197 static const char *ada_delete_method_names[] =
 198     { "NONE", "DISABLE", "CFA_ERASE", "DSM_TRIM", "NCQ_DSM_TRIM" };
 199 #if 0
 200 static const char *ada_delete_method_desc[] =
 201     { "NONE", "DISABLED", "CFA Erase", "DSM Trim", "DSM Trim via NCQ" };
 202 #endif
 203
 204 struct disk_params {
 205         u_int8_t  heads;
 206         u_int8_t  secs_per_track;
 207         u_int32_t cylinders;
 208         u_int32_t secsize;      /* Number of bytes/logical sector */
 209         u_int64_t sectors;      /* Total number sectors */
 210 };
 211
 212 #define TRIM_MAX_BLOCKS 8
 213 #define TRIM_MAX_RANGES (TRIM_MAX_BLOCKS * ATA_DSM_BLK_RANGES)
 214 struct trim_request {
 215         uint8_t         data[TRIM_MAX_RANGES * ATA_DSM_RANGE_SIZE];
 216         TAILQ_HEAD(, bio) bps;
 217 };
 218
 219 struct ada_softc {
 220         struct   cam_iosched_softc *cam_iosched;
 221         int      outstanding_cmds;      /* Number of active commands */
 222         int      refcount;              /* Active xpt_action() calls */
 223         ada_state state;
 224         ada_flags flags;
 225         ada_zone_mode zone_mode;
 226         ada_zone_flags zone_flags;
 227         struct ata_gp_log_dir ata_logdir;
 228         int valid_logdir_len;
 229         struct ata_identify_log_pages ata_iddir;
 230         int valid_iddir_len;
 231         uint64_t optimal_seq_zones;
 232         uint64_t optimal_nonseq_zones;
 233         uint64_t max_seq_zones;
 234         ada_quirks quirks;
 235         ada_delete_methods delete_method;
 236         int      trim_max_ranges;
 237         int      read_ahead;
 238         int      write_cache;
 239         int      unmappedio;
 240         int      rotating;
 241 #ifdef CAM_TEST_FAILURE
 242         int      force_read_error;
 243         int      force_write_error;
 244         int      periodic_read_error;
 245         int      periodic_read_count;
 246 #endif
 247         struct   disk_params params;
 248         struct   disk *disk;
 249         struct task             sysctl_task;
 250         struct sysctl_ctx_list  sysctl_ctx;
 251         struct sysctl_oid       *sysctl_tree;
 252         struct callout          sendordered_c;
 253         struct trim_request     trim_req;
 254         uint64_t                trim_count;
 255         uint64_t                trim_ranges;
 256         uint64_t                trim_lbas;
 257 #ifdef CAM_IO_STATS
 258         struct sysctl_ctx_list  sysctl_stats_ctx;
 259         struct sysctl_oid       *sysctl_stats_tree;
 260         u_int   timeouts;
 261         u_int   errors;
 262         u_int   invalidations;
 263 #endif
 264 #define ADA_ANNOUNCETMP_SZ 80
 265         char    announce_temp[ADA_ANNOUNCETMP_SZ];
 266 #define ADA_ANNOUNCE_SZ 400
 267         char    announce_buffer[ADA_ANNOUNCE_SZ];
 268 };
 269
 270 struct ada_quirk_entry {
 271         struct scsi_inquiry_pattern inq_pat;
 272         ada_quirks quirks;
 273 };
 274
 275 static struct ada_quirk_entry ada_quirk_table[] =
 276 {
 277         {
 278                 /* Hitachi Advanced Format (4k) drives */
 279                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Hitachi H??????????E3*", "*" },
 280                 /*quirks*/ADA_Q_4K
 281         },
 282         {
 283                 /* Samsung Advanced Format (4k) drives */
 284                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG HD155UI*", "*" },
 285                 /*quirks*/ADA_Q_4K
 286         },
 287         {
 288                 /* Samsung Advanced Format (4k) drives */
 289                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG HD204UI*", "*" },
 290                 /*quirks*/ADA_Q_4K
 291         },
 292         {
 293                 /* Seagate Barracuda Green Advanced Format (4k) drives */
 294                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST????DL*", "*" },
 295                 /*quirks*/ADA_Q_4K
 296         },
 297         {
 298                 /* Seagate Barracuda Advanced Format (4k) drives */
 299                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST???DM*", "*" },
 300                 /*quirks*/ADA_Q_4K
 301         },
 302         {
 303                 /* Seagate Barracuda Advanced Format (4k) drives */
 304                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST????DM*", "*" },
 305                 /*quirks*/ADA_Q_4K
 306         },
 307         {
 308                 /* Seagate Momentus Advanced Format (4k) drives */
 309                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9500423AS*", "*" },
 310                 /*quirks*/ADA_Q_4K
 311         },
 312         {
 313                 /* Seagate Momentus Advanced Format (4k) drives */
 314                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9500424AS*", "*" },
 315                 /*quirks*/ADA_Q_4K
 316         },
 317         {
 318                 /* Seagate Momentus Advanced Format (4k) drives */
 319                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9640423AS*", "*" },
 320                 /*quirks*/ADA_Q_4K
 321         },
 322         {
 323                 /* Seagate Momentus Advanced Format (4k) drives */
 324                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9640424AS*", "*" },
 325                 /*quirks*/ADA_Q_4K
 326         },
 327         {
 328                 /* Seagate Momentus Advanced Format (4k) drives */
 329                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9750420AS*", "*" },
 330                 /*quirks*/ADA_Q_4K
 331         },
 332         {
 333                 /* Seagate Momentus Advanced Format (4k) drives */
 334                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9750422AS*", "*" },
 335                 /*quirks*/ADA_Q_4K
 336         },
 337         {
 338                 /* Seagate Momentus Advanced Format (4k) drives */
 339                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9750423AS*", "*" },
 340                 /*quirks*/ADA_Q_4K
 341         },
 342         {
 343                 /* Seagate Momentus Thin Advanced Format (4k) drives */
 344                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST???LT*", "*" },
 345                 /*quirks*/ADA_Q_4K
 346         },
 347         {
 348                 /* WDC Caviar Red Advanced Format (4k) drives */
 349                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD????CX*", "*" },
 350                 /*quirks*/ADA_Q_4K
 351         },
 352         {
 353                 /* WDC Caviar Green Advanced Format (4k) drives */
 354                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD????RS*", "*" },
 355                 /*quirks*/ADA_Q_4K
 356         },
 357         {
 358                 /* WDC Caviar Green/Red Advanced Format (4k) drives */
 359                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD????RX*", "*" },
 360                 /*quirks*/ADA_Q_4K
 361         },
 362         {
 363                 /* WDC Caviar Red Advanced Format (4k) drives */
 364                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD??????CX*", "*" },
 365                 /*quirks*/ADA_Q_4K
 366         },
 367         {
 368                 /* WDC Caviar Black Advanced Format (4k) drives */
 369                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD????AZEX*", "*" },
 370                 /*quirks*/ADA_Q_4K
 371         },
 372         {
 373                 /* WDC Caviar Black Advanced Format (4k) drives */
 374                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD????FZEX*", "*" },
 375                 /*quirks*/ADA_Q_4K
 376         },
 377         {
 378                 /* WDC Caviar Green Advanced Format (4k) drives */
 379                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD??????RS*", "*" },
 380                 /*quirks*/ADA_Q_4K
 381         },
 382         {
 383                 /* WDC Caviar Green Advanced Format (4k) drives */
 384                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD??????RX*", "*" },
 385                 /*quirks*/ADA_Q_4K
 386         },
 387         {
 388                 /* WDC Scorpio Black Advanced Format (4k) drives */
 389                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD???PKT*", "*" },
 390                 /*quirks*/ADA_Q_4K
 391         },
 392         {
 393                 /* WDC Scorpio Black Advanced Format (4k) drives */
 394                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD?????PKT*", "*" },
 395                 /*quirks*/ADA_Q_4K
 396         },
 397         {
 398                 /* WDC Scorpio Blue Advanced Format (4k) drives */
 399                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD???PVT*", "*" },
 400                 /*quirks*/ADA_Q_4K
 401         },
 402         {
 403                 /* WDC Scorpio Blue Advanced Format (4k) drives */
 404                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD?????PVT*", "*" },
 405                 /*quirks*/ADA_Q_4K
 406         },
 407         /* SSDs */
 408         {
 409                 /*
 410                  * Corsair Force 2 SSDs
 411                  * 4k optimised & trim only works in 4k requests + 4k aligned
 412                  */
 413                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair CSSD-F*", "*" },
 414                 /*quirks*/ADA_Q_4K
 415         },
 416         {
 417                 /*
 418                  * Corsair Force 3 SSDs
 419                  * 4k optimised & trim only works in 4k requests + 4k aligned
 420                  */
 421                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair Force 3*", "*" },
 422                 /*quirks*/ADA_Q_4K
 423         },
 424         {
 425                 /*
 426                  * Corsair Neutron GTX SSDs
 427                  * 4k optimised & trim only works in 4k requests + 4k aligned
 428                  */
 429                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair Neutron GTX*", "*" },
 430                 /*quirks*/ADA_Q_4K
 431         },
 432         {
 433                 /*
 434                  * Corsair Force GT & GS SSDs
 435                  * 4k optimised & trim only works in 4k requests + 4k aligned
 436                  */
 437                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair Force G*", "*" },
 438                 /*quirks*/ADA_Q_4K
 439         },
 440         {
 441                 /*
 442                  * Crucial M4 SSDs
 443                  * 4k optimised & trim only works in 4k requests + 4k aligned
 444                  */
 445                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "M4-CT???M4SSD2*", "*" },
 446                 /*quirks*/ADA_Q_4K
 447         },
 448         {
 449                 /*
 450                  * Crucial M500 SSDs MU07 firmware
 451                  * NCQ Trim works
 452                  */
 453                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Crucial CT*M500*", "MU07" },
 454                 /*quirks*/0
 455         },
 456         {
 457                 /*
 458                  * Crucial M500 SSDs all other firmware
 459                  * NCQ Trim doesn't work
 460                  */
 461                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Crucial CT*M500*", "*" },
 462                 /*quirks*/ADA_Q_NCQ_TRIM_BROKEN
 463         },
 464         {
 465                 /*
 466                  * Crucial M550 SSDs
 467                  * NCQ Trim doesn't work, but only on MU01 firmware
 468                  */
 469                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Crucial CT*M550*", "MU01" },
 470                 /*quirks*/ADA_Q_NCQ_TRIM_BROKEN
 471         },
 472         {
 473                 /*
 474                  * Crucial MX100 SSDs
 475                  * NCQ Trim doesn't work, but only on MU01 firmware
 476                  */
 477                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Crucial CT*MX100*", "MU01" },
 478                 /*quirks*/ADA_Q_NCQ_TRIM_BROKEN
 479         },
 480         {
 481                 /*
 482                  * Crucial RealSSD C300 SSDs
 483                  * 4k optimised
 484                  */
 485                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "C300-CTFDDAC???MAG*",
 486                 "*" }, /*quirks*/ADA_Q_4K
 487         },
 488         {
 489                 /*
 490                  * FCCT M500 SSDs
 491                  * NCQ Trim doesn't work
 492                  */
 493                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "FCCT*M500*", "*" },
 494                 /*quirks*/ADA_Q_NCQ_TRIM_BROKEN
 495         },
 496         {
 497                 /*
 498                  * Intel 320 Series SSDs
 499                  * 4k optimised & trim only works in 4k requests + 4k aligned
 500                  */
 501                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSA2CW*", "*" },
 502                 /*quirks*/ADA_Q_4K
 503         },
 504         {
 505                 /*
 506                  * Intel 330 Series SSDs
 507                  * 4k optimised & trim only works in 4k requests + 4k aligned
 508                  */
 509                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSC2CT*", "*" },
 510                 /*quirks*/ADA_Q_4K
 511         },
 512         {
 513                 /*
 514                  * Intel 510 Series SSDs
 515                  * 4k optimised & trim only works in 4k requests + 4k aligned
 516                  */
 517                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSC2MH*", "*" },
 518                 /*quirks*/ADA_Q_4K
 519         },
 520         {
 521                 /*
 522                  * Intel 520 Series SSDs
 523                  * 4k optimised & trim only works in 4k requests + 4k aligned
 524                  */
 525                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSC2BW*", "*" },
 526                 /*quirks*/ADA_Q_4K
 527         },
 528         {
 529                 /*
 530                  * Intel S3610 Series SSDs
 531                  * 4k optimised & trim only works in 4k requests + 4k aligned
 532                  */
 533                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSC2BX*", "*" },
 534                 /*quirks*/ADA_Q_4K
 535         },
 536         {
 537                 /*
 538                  * Intel X25-M Series SSDs
 539                  * 4k optimised & trim only works in 4k requests + 4k aligned
 540                  */
 541                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSA2M*", "*" },
 542                 /*quirks*/ADA_Q_4K
 543         },
 544         {
 545                 /*
 546                  * Kingston E100 Series SSDs
 547                  * 4k optimised & trim only works in 4k requests + 4k aligned
 548                  */
 549                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "KINGSTON SE100S3*", "*" },
 550                 /*quirks*/ADA_Q_4K
 551         },
 552         {
 553                 /*
 554                  * Kingston HyperX 3k SSDs
 555                  * 4k optimised & trim only works in 4k requests + 4k aligned
 556                  */
 557                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "KINGSTON SH103S3*", "*" },
 558                 /*quirks*/ADA_Q_4K
 559         },
 560         {
 561                 /*
 562                  * Marvell SSDs (entry taken from OpenSolaris)
 563                  * 4k optimised & trim only works in 4k requests + 4k aligned
 564                  */
 565                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "MARVELL SD88SA02*", "*" },
 566                 /*quirks*/ADA_Q_4K
 567         },
 568         {
 569                 /*
 570                  * Micron M500 SSDs firmware MU07
 571                  * NCQ Trim works?
 572                  */
 573                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Micron M500*", "MU07" },
 574                 /*quirks*/0
 575         },
 576         {
 577                 /*
 578                  * Micron M500 SSDs all other firmware
 579                  * NCQ Trim doesn't work
 580                  */
 581                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Micron M500*", "*" },
 582                 /*quirks*/ADA_Q_NCQ_TRIM_BROKEN
 583         },
 584         {
 585                 /*
 586                  * Micron M5[15]0 SSDs
 587                  * NCQ Trim doesn't work, but only MU01 firmware
 588                  */
 589                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Micron M5[15]0*", "MU01" },
 590                 /*quirks*/ADA_Q_NCQ_TRIM_BROKEN
 591         },
 592         {
 593                 /*
 594                  * Micron 5100 SSDs
 595                  * 4k optimised & trim only works in 4k requests + 4k aligned
 596                  */
 597                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Micron 5100 MTFDDAK*", "*" },
 598                 /*quirks*/ADA_Q_4K
 599         },
 600         {
 601                 /*
 602                  * OCZ Agility 2 SSDs
 603                  * 4k optimised & trim only works in 4k requests + 4k aligned
 604                  */
 605                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-AGILITY2*", "*" },
 606                 /*quirks*/ADA_Q_4K
 607         },
 608         {
 609                 /*
 610                  * OCZ Agility 3 SSDs
 611                  * 4k optimised & trim only works in 4k requests + 4k aligned
 612                  */
 613                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-AGILITY3*", "*" },
 614                 /*quirks*/ADA_Q_4K
 615         },
 616         {
 617                 /*
 618                  * OCZ Deneva R Series SSDs
 619                  * 4k optimised & trim only works in 4k requests + 4k aligned
 620                  */
 621                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "DENRSTE251M45*", "*" },
 622                 /*quirks*/ADA_Q_4K
 623         },
 624         {
 625                 /*
 626                  * OCZ Vertex 2 SSDs (inc pro series)
 627                  * 4k optimised & trim only works in 4k requests + 4k aligned
 628                  */
 629                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ?VERTEX2*", "*" },
 630                 /*quirks*/ADA_Q_4K
 631         },
 632         {
 633                 /*
 634                  * OCZ Vertex 3 SSDs
 635                  * 4k optimised & trim only works in 4k requests + 4k aligned
 636                  */
 637                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-VERTEX3*", "*" },
 638                 /*quirks*/ADA_Q_4K
 639         },
 640         {
 641                 /*
 642                  * OCZ Vertex 4 SSDs
 643                  * 4k optimised & trim only works in 4k requests + 4k aligned
 644                  */
 645                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-VERTEX4*", "*" },
 646                 /*quirks*/ADA_Q_4K
 647         },
 648         {
 649                 /*
 650                  * Samsung 750 SSDs
 651                  * 4k optimised, NCQ TRIM seems to work
 652                  */
 653                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Samsung SSD 750*", "*" },
 654                 /*quirks*/ADA_Q_4K
 655         },
 656         {
 657                 /*
 658                  * Samsung 830 Series SSDs
 659                  * 4k optimised, NCQ TRIM Broken (normal TRIM is fine)
 660                  */
 661                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG SSD 830 Series*", "*" },
 662                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 663         },
 664         {
 665                 /*
 666                  * Samsung 840 SSDs
 667                  * 4k optimised, NCQ TRIM Broken (normal TRIM is fine)
 668                  */
 669                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Samsung SSD 840*", "*" },
 670                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 671         },
 672         {
 673                 /*
 674                  * Samsung 845 SSDs
 675                  * 4k optimised, NCQ TRIM Broken (normal TRIM is fine)
 676                  */
 677                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Samsung SSD 845*", "*" },
 678                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 679         },
 680         {
 681                 /*
 682                  * Samsung 850 SSDs
 683                  * 4k optimised, NCQ TRIM broken (normal TRIM fine)
 684                  */
 685                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Samsung SSD 850*", "*" },
 686                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 687         },
 688         {
 689                 /*
 690                  * Samsung SM863 Series SSDs (MZ7KM*)
 691                  * 4k optimised, NCQ believed to be working
 692                  */
 693                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG MZ7KM*", "*" },
 694                 /*quirks*/ADA_Q_4K
 695         },
 696         {
 697                 /*
 698                  * Samsung 843T Series SSDs (MZ7WD*)
 699                  * Samsung PM851 Series SSDs (MZ7TE*)
 700                  * Samsung PM853T Series SSDs (MZ7GE*)
 701                  * 4k optimised, NCQ believed to be broken since these are
 702                  * appear to be built with the same controllers as the 840/850.
 703                  */
 704                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG MZ7*", "*" },
 705                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 706         },
 707         {
 708                 /*
 709                  * Same as for SAMSUNG MZ7* but enable the quirks for SSD
 710                  * starting with MZ7* too
 711                  */
 712                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "MZ7*", "*" },
 713                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 714         },
 715         {
 716                 /*
 717                  * Samsung PM851 Series SSDs Dell OEM
 718                  * device model          "SAMSUNG SSD PM851 mSATA 256GB"
 719                  * 4k optimised, NCQ broken
 720                  */
 721                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG SSD PM851*", "*" },
 722                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 723         },
 724         {
 725                 /*
 726                  * SuperTalent TeraDrive CT SSDs
 727                  * 4k optimised & trim only works in 4k requests + 4k aligned
 728                  */
 729                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "FTM??CT25H*", "*" },
 730                 /*quirks*/ADA_Q_4K
 731         },
 732         {
 733                 /*
 734                  * XceedIOPS SATA SSDs
 735                  * 4k optimised
 736                  */
 737                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SG9XCS2D*", "*" },
 738                 /*quirks*/ADA_Q_4K
 739         },
 740         {
 741                 /*
 742                  * Samsung drive that doesn't support READ LOG EXT or
 743                  * READ LOG DMA EXT, despite reporting that it does in
 744                  * ATA identify data:
 745                  * SAMSUNG HD200HJ KF100-06
 746                  */
 747                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG HD200*", "*" },
 748                 /*quirks*/ADA_Q_LOG_BROKEN
 749         },
 750         {
 751                 /*
 752                  * Samsung drive that doesn't support READ LOG EXT or
 753                  * READ LOG DMA EXT, despite reporting that it does in
 754                  * ATA identify data:
 755                  * SAMSUNG HD501LJ CR100-10
 756                  */
 757                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG HD501*", "*" },
 758                 /*quirks*/ADA_Q_LOG_BROKEN
 759         },
 760         {
 761                 /*
 762                  * Seagate Lamarr 8TB Shingled Magnetic Recording (SMR)
 763                  * Drive Managed SATA hard drive.  This drive doesn't report
 764                  * in firmware that it is a drive managed SMR drive.
 765                  */
 766                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST8000AS000[23]*", "*" },
 767                 /*quirks*/ADA_Q_SMR_DM
 768         },
 769         {
 770                 /* Default */
 771                 {
 772                   T_ANY, SIP_MEDIA_REMOVABLE|SIP_MEDIA_FIXED,
 773                   /*vendor*/"*", /*product*/"*", /*revision*/"*"
 774                 },
 775                 /*quirks*/0
 776         },
 777 };
 778
 779 static  disk_strategy_t adastrategy;
 780 static  dumper_t        adadump;
 781 static  periph_init_t   adainit;
 782 static  void            adadiskgonecb(struct disk *dp);
 783 static  periph_oninv_t  adaoninvalidate;
 784 static  periph_dtor_t   adacleanup;
 785 static  void            adaasync(void *callback_arg, u_int32_t code,
 786                                 struct cam_path *path, void *arg);
 787 static  int             adazonemodesysctl(SYSCTL_HANDLER_ARGS);
 788 static  int             adazonesupsysctl(SYSCTL_HANDLER_ARGS);
 789 static  void            adasysctlinit(void *context, int pending);
 790 static  int             adagetattr(struct bio *bp);
 791 static  void            adasetflags(struct ada_softc *softc,
 792                                     struct ccb_getdev *cgd);
 793 static  periph_ctor_t   adaregister;
 794 static  void            ada_dsmtrim(struct ada_softc *softc, struct bio *bp,
 795                                     struct ccb_ataio *ataio);
 796 static  void            ada_cfaerase(struct ada_softc *softc, struct bio *bp,
 797                                      struct ccb_ataio *ataio);
 798 static  int             ada_zone_bio_to_ata(int disk_zone_cmd);
 799 static  int             ada_zone_cmd(struct cam_periph *periph, union ccb *ccb,
 800                                      struct bio *bp, int *queue_ccb);
 801 static  periph_start_t  adastart;
 802 static  void            adaprobedone(struct cam_periph *periph, union ccb *ccb);
 803 static  void            adazonedone(struct cam_periph *periph, union ccb *ccb);
 804 static  void            adadone(struct cam_periph *periph,
 805                                union ccb *done_ccb);
 806 static  int             adaerror(union ccb *ccb, u_int32_t cam_flags,
 807                                 u_int32_t sense_flags);
 808 static void             adagetparams(struct cam_periph *periph,
 809                                 struct ccb_getdev *cgd);
 810 static timeout_t        adasendorderedtag;
 811 static void             adashutdown(void *arg, int howto);
 812 static void             adasuspend(void *arg);
 813 static void             adaresume(void *arg);
 814
 815 #ifndef ADA_DEFAULT_TIMEOUT
 816 #define ADA_DEFAULT_TIMEOUT 30  /* Timeout in seconds */
 817 #endif
 818
 819 #ifndef ADA_DEFAULT_RETRY
 820 #define ADA_DEFAULT_RETRY       4
 821 #endif
 822
 823 #ifndef ADA_DEFAULT_SEND_ORDERED
 824 #define ADA_DEFAULT_SEND_ORDERED        1
 825 #endif
 826
 827 #ifndef ADA_DEFAULT_SPINDOWN_SHUTDOWN
 828 #define ADA_DEFAULT_SPINDOWN_SHUTDOWN   1
 829 #endif
 830
 831 #ifndef ADA_DEFAULT_SPINDOWN_SUSPEND
 832 #define ADA_DEFAULT_SPINDOWN_SUSPEND    1
 833 #endif
 834
 835 #ifndef ADA_DEFAULT_READ_AHEAD
 836 #define ADA_DEFAULT_READ_AHEAD  1
 837 #endif
 838
 839 #ifndef ADA_DEFAULT_WRITE_CACHE
 840 #define ADA_DEFAULT_WRITE_CACHE 1
 841 #endif
 842
 843 #define ADA_RA  (softc->read_ahead >= 0 ? \
 844                  softc->read_ahead : ada_read_ahead)
 845 #define ADA_WC  (softc->write_cache >= 0 ? \
 846                  softc->write_cache : ada_write_cache)
 847
 848 /*
 849  * Most platforms map firmware geometry to actual, but some don't.  If
 850  * not overridden, default to nothing.
 851  */
 852 #ifndef ata_disk_firmware_geom_adjust
 853 #define ata_disk_firmware_geom_adjust(disk)
 854 #endif
 855
 856 static int ada_retry_count = ADA_DEFAULT_RETRY;
 857 static int ada_default_timeout = ADA_DEFAULT_TIMEOUT;
 858 static int ada_send_ordered = ADA_DEFAULT_SEND_ORDERED;
 859 static int ada_spindown_shutdown = ADA_DEFAULT_SPINDOWN_SHUTDOWN;
 860 static int ada_spindown_suspend = ADA_DEFAULT_SPINDOWN_SUSPEND;
 861 static int ada_read_ahead = ADA_DEFAULT_READ_AHEAD;
 862 static int ada_write_cache = ADA_DEFAULT_WRITE_CACHE;
 863
 864 static SYSCTL_NODE(_kern_cam, OID_AUTO, ada, CTLFLAG_RD, 0,
 865             "CAM Direct Access Disk driver");
 866 SYSCTL_INT(_kern_cam_ada, OID_AUTO, retry_count, CTLFLAG_RWTUN,
 867            &ada_retry_count, 0, "Normal I/O retry count");
 868 SYSCTL_INT(_kern_cam_ada, OID_AUTO, default_timeout, CTLFLAG_RWTUN,
 869            &ada_default_timeout, 0, "Normal I/O timeout (in seconds)");
 870 SYSCTL_INT(_kern_cam_ada, OID_AUTO, send_ordered, CTLFLAG_RWTUN,
 871            &ada_send_ordered, 0, "Send Ordered Tags");
 872 SYSCTL_INT(_kern_cam_ada, OID_AUTO, spindown_shutdown, CTLFLAG_RWTUN,
 873            &ada_spindown_shutdown, 0, "Spin down upon shutdown");
 874 SYSCTL_INT(_kern_cam_ada, OID_AUTO, spindown_suspend, CTLFLAG_RWTUN,
 875            &ada_spindown_suspend, 0, "Spin down upon suspend");
 876 SYSCTL_INT(_kern_cam_ada, OID_AUTO, read_ahead, CTLFLAG_RWTUN,
 877            &ada_read_ahead, 0, "Enable disk read-ahead");
 878 SYSCTL_INT(_kern_cam_ada, OID_AUTO, write_cache, CTLFLAG_RWTUN,
 879            &ada_write_cache, 0, "Enable disk write cache");
 880
 881 /*
 882  * ADA_ORDEREDTAG_INTERVAL determines how often, relative
 883  * to the default timeout, we check to see whether an ordered
 884  * tagged transaction is appropriate to prevent simple tag
 885  * starvation.  Since we'd like to ensure that there is at least
 886  * 1/2 of the timeout length left for a starved transaction to
 887  * complete after we've sent an ordered tag, we must poll at least
 888  * four times in every timeout period.  This takes care of the worst
 889  * case where a starved transaction starts during an interval that
 890  * meets the requirement "don't send an ordered tag" test so it takes
 891  * us two intervals to determine that a tag must be sent.
 892  */
 893 #ifndef ADA_ORDEREDTAG_INTERVAL
 894 #define ADA_ORDEREDTAG_INTERVAL 4
 895 #endif
 896
 897 static struct periph_driver adadriver =
 898 {
 899         adainit, "ada",
 900         TAILQ_HEAD_INITIALIZER(adadriver.units), /* generation */ 0
 901 };
 902
 903 static int adadeletemethodsysctl(SYSCTL_HANDLER_ARGS);
 904
 905 PERIPHDRIVER_DECLARE(ada, adadriver);
 906
 907 static MALLOC_DEFINE(M_ATADA, "ata_da", "ata_da buffers");
 908
 909 static int
 910 adaopen(struct disk *dp)
 911 {
 912         struct cam_periph *periph;
 913         struct ada_softc *softc;
 914         int error;
 915
 916         periph = (struct cam_periph *)dp->d_drv1;
 917         if (cam_periph_acquire(periph) != 0) {
 918                 return(ENXIO);
 919         }
 920
 921         cam_periph_lock(periph);
 922         if ((error = cam_periph_hold(periph, PRIBIO|PCATCH)) != 0) {
 923                 cam_periph_unlock(periph);
 924                 cam_periph_release(periph);
 925                 return (error);
 926         }
 927
 928         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE | CAM_DEBUG_PERIPH,
 929             ("adaopen\n"));
 930
 931         softc = (struct ada_softc *)periph->softc;
 932         softc->flags |= ADA_FLAG_OPEN;
 933
 934         cam_periph_unhold(periph);
 935         cam_periph_unlock(periph);
 936         return (0);
 937 }
 938
 939 static int
 940 adaclose(struct disk *dp)
 941 {
 942         struct  cam_periph *periph;
 943         struct  ada_softc *softc;
 944         union ccb *ccb;
 945         int error;
 946
 947         periph = (struct cam_periph *)dp->d_drv1;
 948         softc = (struct ada_softc *)periph->softc;
 949         cam_periph_lock(periph);
 950
 951         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE | CAM_DEBUG_PERIPH,
 952             ("adaclose\n"));
 953
 954         /* We only sync the cache if the drive is capable of it. */
 955         if ((softc->flags & ADA_FLAG_DIRTY) != 0 &&
 956             (softc->flags & ADA_FLAG_CAN_FLUSHCACHE) != 0 &&
 957             (periph->flags & CAM_PERIPH_INVALID) == 0 &&
 958             cam_periph_hold(periph, PRIBIO) == 0) {
 959
 960                 ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL);
 961                 cam_fill_ataio(&ccb->ataio,
 962                                     1,
 963                                     NULL,
 964                                     CAM_DIR_NONE,
 965                                     0,
 966                                     NULL,
 967                                     0,
 968                                     ada_default_timeout*1000);
 969
 970                 if (softc->flags & ADA_FLAG_CAN_48BIT)
 971                         ata_48bit_cmd(&ccb->ataio, ATA_FLUSHCACHE48, 0, 0, 0);
 972                 else
 973                         ata_28bit_cmd(&ccb->ataio, ATA_FLUSHCACHE, 0, 0, 0);
 974                 error = cam_periph_runccb(ccb, adaerror, /*cam_flags*/0,
 975                     /*sense_flags*/0, softc->disk->d_devstat);
 976
 977                 if (error != 0)
 978                         xpt_print(periph->path, "Synchronize cache failed\n");
 979                 softc->flags &= ~ADA_FLAG_DIRTY;
 980                 xpt_release_ccb(ccb);
 981                 cam_periph_unhold(periph);
 982         }
 983
 984         softc->flags &= ~ADA_FLAG_OPEN;
 985
 986         while (softc->refcount != 0)
 987                 cam_periph_sleep(periph, &softc->refcount, PRIBIO, "adaclose", 1);
 988         cam_periph_unlock(periph);
 989         cam_periph_release(periph);
 990         return (0);
 991 }
 992
 993 static void
 994 adaschedule(struct cam_periph *periph)
 995 {
 996         struct ada_softc *softc = (struct ada_softc *)periph->softc;
 997
 998         if (softc->state != ADA_STATE_NORMAL)
 999                 return;
1000
1001         cam_iosched_schedule(softc->cam_iosched, periph);
1002 }
1003
1004 /*
1005  * Actually translate the requested transfer into one the physical driver
1006  * can understand.  The transfer is described by a buf and will include
1007  * only one physical transfer.
1008  */
1009 static void
1010 adastrategy(struct bio *bp)
1011 {
1012         struct cam_periph *periph;
1013         struct ada_softc *softc;
1014
1015         periph = (struct cam_periph *)bp->bio_disk->d_drv1;
1016         softc = (struct ada_softc *)periph->softc;
1017
1018         cam_periph_lock(periph);
1019
1020         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("adastrategy(%p)\n", bp));
1021
1022         /*
1023          * If the device has been made invalid, error out
1024          */
1025         if ((periph->flags & CAM_PERIPH_INVALID) != 0) {
1026                 cam_periph_unlock(periph);
1027                 biofinish(bp, NULL, ENXIO);
1028                 return;
1029         }
1030
1031         /*
1032          * Zone commands must be ordered, because they can depend on the
1033          * effects of previously issued commands, and they may affect
1034          * commands after them.
1035          */
1036         if (bp->bio_cmd == BIO_ZONE)
1037                 bp->bio_flags |= BIO_ORDERED;
1038
1039         /*
1040          * Place it in the queue of disk activities for this disk
1041          */
1042         cam_iosched_queue_work(softc->cam_iosched, bp);
1043
1044         /*
1045          * Schedule ourselves for performing the work.
1046          */
1047         adaschedule(periph);
1048         cam_periph_unlock(periph);
1049
1050         return;
1051 }
1052
1053 static int
1054 adadump(void *arg, void *virtual, vm_offset_t physical, off_t offset, size_t length)
1055 {
1056         struct      cam_periph *periph;
1057         struct      ada_softc *softc;
1058         u_int       secsize;
1059         struct      ccb_ataio ataio;
1060         struct      disk *dp;
1061         uint64_t    lba;
1062         uint16_t    count;
1063         int         error = 0;
1064
1065         dp = arg;
1066         periph = dp->d_drv1;
1067         softc = (struct ada_softc *)periph->softc;
1068         secsize = softc->params.secsize;
1069         lba = offset / secsize;
1070         count = length / secsize;
1071
1072         if ((periph->flags & CAM_PERIPH_INVALID) != 0)
1073                 return (ENXIO);
1074
1075         memset(&ataio, 0, sizeof(ataio));
1076         if (length > 0) {
1077                 xpt_setup_ccb(&ataio.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
1078                 ataio.ccb_h.ccb_state = ADA_CCB_DUMP;
1079                 cam_fill_ataio(&ataio,
1080                     0,
1081                     NULL,
1082                     CAM_DIR_OUT,
1083                     0,
1084                     (u_int8_t *) virtual,
1085                     length,
1086                     ada_default_timeout*1000);
1087                 if ((softc->flags & ADA_FLAG_CAN_48BIT) &&
1088                     (lba + count >= ATA_MAX_28BIT_LBA ||
1089                     count >= 256)) {
1090                         ata_48bit_cmd(&ataio, ATA_WRITE_DMA48,
1091                             0, lba, count);
1092                 } else {
1093                         ata_28bit_cmd(&ataio, ATA_WRITE_DMA,
1094                             0, lba, count);
1095                 }
1096                 error = cam_periph_runccb((union ccb *)&ataio, adaerror,
1097                     0, SF_NO_RECOVERY | SF_NO_RETRY, NULL);
1098                 if (error != 0)
1099                         printf("Aborting dump due to I/O error.\n");
1100
1101                 return (error);
1102         }
1103
1104         if (softc->flags & ADA_FLAG_CAN_FLUSHCACHE) {
1105                 xpt_setup_ccb(&ataio.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
1106
1107                 /*
1108                  * Tell the drive to flush its internal cache. if we
1109                  * can't flush in 5s we have big problems. No need to
1110                  * wait the default 60s to detect problems.
1111                  */
1112                 ataio.ccb_h.ccb_state = ADA_CCB_DUMP;
1113                 cam_fill_ataio(&ataio,
1114                                     0,
1115                                     NULL,
1116                                     CAM_DIR_NONE,
1117                                     0,
1118                                     NULL,
1119                                     0,
1120                                     5*1000);
1121
1122                 if (softc->flags & ADA_FLAG_CAN_48BIT)
1123                         ata_48bit_cmd(&ataio, ATA_FLUSHCACHE48, 0, 0, 0);
1124                 else
1125                         ata_28bit_cmd(&ataio, ATA_FLUSHCACHE, 0, 0, 0);
1126                 error = cam_periph_runccb((union ccb *)&ataio, adaerror,
1127                     0, SF_NO_RECOVERY | SF_NO_RETRY, NULL);
1128                 if (error != 0)
1129                         xpt_print(periph->path, "Synchronize cache failed\n");
1130         }
1131         return (error);
1132 }
1133
1134 static void
1135 adainit(void)
1136 {
1137         cam_status status;
1138
1139         /*
1140          * Install a global async callback.  This callback will
1141          * receive async callbacks like "new device found".
1142          */
1143         status = xpt_register_async(AC_FOUND_DEVICE, adaasync, NULL, NULL);
1144
1145         if (status != CAM_REQ_CMP) {
1146                 printf("ada: Failed to attach master async callback "
1147                        "due to status 0x%x!\n", status);
1148         } else if (ada_send_ordered) {
1149
1150                 /* Register our event handlers */
1151                 if ((EVENTHANDLER_REGISTER(power_suspend, adasuspend,
1152                                            NULL, EVENTHANDLER_PRI_LAST)) == NULL)
1153                     printf("adainit: power event registration failed!\n");
1154                 if ((EVENTHANDLER_REGISTER(power_resume, adaresume,
1155                                            NULL, EVENTHANDLER_PRI_LAST)) == NULL)
1156                     printf("adainit: power event registration failed!\n");
1157                 if ((EVENTHANDLER_REGISTER(shutdown_post_sync, adashutdown,
1158                                            NULL, SHUTDOWN_PRI_DEFAULT)) == NULL)
1159                     printf("adainit: shutdown event registration failed!\n");
1160         }
1161 }
1162
1163 /*
1164  * Callback from GEOM, called when it has finished cleaning up its
1165  * resources.
1166  */
1167 static void
1168 adadiskgonecb(struct disk *dp)
1169 {
1170         struct cam_periph *periph;
1171
1172         periph = (struct cam_periph *)dp->d_drv1;
1173
1174         cam_periph_release(periph);
1175 }
1176
1177 static void
1178 adaoninvalidate(struct cam_periph *periph)
1179 {
1180         struct ada_softc *softc;
1181
1182         softc = (struct ada_softc *)periph->softc;
1183
1184         /*
1185          * De-register any async callbacks.
1186          */
1187         xpt_register_async(0, adaasync, periph, periph->path);
1188 #ifdef CAM_IO_STATS
1189         softc->invalidations++;
1190 #endif
1191
1192         /*
1193          * Return all queued I/O with ENXIO.
1194          * XXX Handle any transactions queued to the card
1195          *     with XPT_ABORT_CCB.
1196          */
1197         cam_iosched_flush(softc->cam_iosched, NULL, ENXIO);
1198
1199         disk_gone(softc->disk);
1200 }
1201
1202 static void
1203 adacleanup(struct cam_periph *periph)
1204 {
1205         struct ada_softc *softc;
1206
1207         softc = (struct ada_softc *)periph->softc;
1208
1209         cam_periph_unlock(periph);
1210
1211         cam_iosched_fini(softc->cam_iosched);
1212
1213         /*
1214          * If we can't free the sysctl tree, oh well...
1215          */
1216         if ((softc->flags & ADA_FLAG_SCTX_INIT) != 0) {
1217 #ifdef CAM_IO_STATS
1218                 if (sysctl_ctx_free(&softc->sysctl_stats_ctx) != 0)
1219                         xpt_print(periph->path,
1220                             "can't remove sysctl stats context\n");
1221 #endif
1222                 if (sysctl_ctx_free(&softc->sysctl_ctx) != 0)
1223                         xpt_print(periph->path,
1224                             "can't remove sysctl context\n");
1225         }
1226
1227         disk_destroy(softc->disk);
1228         callout_drain(&softc->sendordered_c);
1229         free(softc, M_DEVBUF);
1230         cam_periph_lock(periph);
1231 }
1232
1233 static void
1234 adasetdeletemethod(struct ada_softc *softc)
1235 {
1236
1237         if (softc->flags & ADA_FLAG_CAN_NCQ_TRIM)
1238                 softc->delete_method = ADA_DELETE_NCQ_DSM_TRIM;
1239         else if (softc->flags & ADA_FLAG_CAN_TRIM)
1240                 softc->delete_method = ADA_DELETE_DSM_TRIM;
1241         else if ((softc->flags & ADA_FLAG_CAN_CFA) && !(softc->flags & ADA_FLAG_CAN_48BIT))
1242                 softc->delete_method = ADA_DELETE_CFA_ERASE;
1243         else
1244                 softc->delete_method = ADA_DELETE_NONE;
1245 }
1246
1247 static void
1248 adaasync(void *callback_arg, u_int32_t code,
1249         struct cam_path *path, void *arg)
1250 {
1251         struct ccb_getdev cgd;
1252         struct cam_periph *periph;
1253         struct ada_softc *softc;
1254
1255         periph = (struct cam_periph *)callback_arg;
1256         switch (code) {
1257         case AC_FOUND_DEVICE:
1258         {
1259                 struct ccb_getdev *cgd;
1260                 cam_status status;
1261
1262                 cgd = (struct ccb_getdev *)arg;
1263                 if (cgd == NULL)
1264                         break;
1265
1266                 if (cgd->protocol != PROTO_ATA)
1267                         break;
1268
1269                 /*
1270                  * Allocate a peripheral instance for
1271                  * this device and start the probe
1272                  * process.
1273                  */
1274                 status = cam_periph_alloc(adaregister, adaoninvalidate,
1275                                           adacleanup, adastart,
1276                                           "ada", CAM_PERIPH_BIO,
1277                                           path, adaasync,
1278                                           AC_FOUND_DEVICE, cgd);
1279
1280                 if (status != CAM_REQ_CMP
1281                  && status != CAM_REQ_INPROG)
1282                         printf("adaasync: Unable to attach to new device "
1283                                 "due to status 0x%x\n", status);
1284                 break;
1285         }
1286         case AC_GETDEV_CHANGED:
1287         {
1288                 softc = (struct ada_softc *)periph->softc;
1289                 xpt_setup_ccb(&cgd.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
1290                 cgd.ccb_h.func_code = XPT_GDEV_TYPE;
1291                 xpt_action((union ccb *)&cgd);
1292
1293                 /*
1294                  * Set/clear support flags based on the new Identify data.
1295                  */
1296                 adasetflags(softc, &cgd);
1297
1298                 cam_periph_async(periph, code, path, arg);
1299                 break;
1300         }
1301         case AC_ADVINFO_CHANGED:
1302         {
1303                 uintptr_t buftype;
1304
1305                 buftype = (uintptr_t)arg;
1306                 if (buftype == CDAI_TYPE_PHYS_PATH) {
1307                         struct ada_softc *softc;
1308
1309                         softc = periph->softc;
1310                         disk_attr_changed(softc->disk, "GEOM::physpath",
1311                                           M_NOWAIT);
1312                 }
1313                 break;
1314         }
1315         case AC_SENT_BDR:
1316         case AC_BUS_RESET:
1317         {
1318                 softc = (struct ada_softc *)periph->softc;
1319                 cam_periph_async(periph, code, path, arg);
1320                 if (softc->state != ADA_STATE_NORMAL)
1321                         break;
1322                 xpt_setup_ccb(&cgd.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
1323                 cgd.ccb_h.func_code = XPT_GDEV_TYPE;
1324                 xpt_action((union ccb *)&cgd);
1325                 if (ADA_RA >= 0 && softc->flags & ADA_FLAG_CAN_RAHEAD)
1326                         softc->state = ADA_STATE_RAHEAD;
1327                 else if (ADA_WC >= 0 && softc->flags & ADA_FLAG_CAN_WCACHE)
1328                         softc->state = ADA_STATE_WCACHE;
1329                 else if ((softc->flags & ADA_FLAG_CAN_LOG)
1330                       && (softc->zone_mode != ADA_ZONE_NONE))
1331                         softc->state = ADA_STATE_LOGDIR;
1332                 else
1333                     break;
1334                 if (cam_periph_acquire(periph) != 0)
1335                         softc->state = ADA_STATE_NORMAL;
1336                 else
1337                         xpt_schedule(periph, CAM_PRIORITY_DEV);
1338         }
1339         default:
1340                 cam_periph_async(periph, code, path, arg);
1341                 break;
1342         }
1343 }
1344
1345 static int
1346 adazonemodesysctl(SYSCTL_HANDLER_ARGS)
1347 {
1348         char tmpbuf[40];
1349         struct ada_softc *softc;
1350         int error;
1351
1352         softc = (struct ada_softc *)arg1;
1353
1354         switch (softc->zone_mode) {
1355         case ADA_ZONE_DRIVE_MANAGED:
1356                 snprintf(tmpbuf, sizeof(tmpbuf), "Drive Managed");
1357                 break;
1358         case ADA_ZONE_HOST_AWARE:
1359                 snprintf(tmpbuf, sizeof(tmpbuf), "Host Aware");
1360                 break;
1361         case ADA_ZONE_HOST_MANAGED:
1362                 snprintf(tmpbuf, sizeof(tmpbuf), "Host Managed");
1363                 break;
1364         case ADA_ZONE_NONE:
1365         default:
1366                 snprintf(tmpbuf, sizeof(tmpbuf), "Not Zoned");
1367                 break;
1368         }
1369
1370         error = sysctl_handle_string(oidp, tmpbuf, sizeof(tmpbuf), req);
1371
1372         return (error);
1373 }
1374
1375 static int
1376 adazonesupsysctl(SYSCTL_HANDLER_ARGS)
1377 {
1378         char tmpbuf[180];
1379         struct ada_softc *softc;
1380         struct sbuf sb;
1381         int error, first;
1382         unsigned int i;
1383
1384         softc = (struct ada_softc *)arg1;
1385
1386         error = 0;
1387         first = 1;
1388         sbuf_new(&sb, tmpbuf, sizeof(tmpbuf), 0);
1389
1390         for (i = 0; i < sizeof(ada_zone_desc_table) /
1391              sizeof(ada_zone_desc_table[0]); i++) {
1392                 if (softc->zone_flags & ada_zone_desc_table[i].value) {
1393                         if (first == 0)
1394                                 sbuf_printf(&sb, ", ");
1395                         else
1396                                 first = 0;
1397                         sbuf_cat(&sb, ada_zone_desc_table[i].desc);
1398                 }
1399         }
1400
1401         if (first == 1)
1402                 sbuf_printf(&sb, "None");
1403
1404         sbuf_finish(&sb);
1405
1406         error = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
1407
1408         return (error);
1409 }
1410
1411
1412 static void
1413 adasysctlinit(void *context, int pending)
1414 {
1415         struct cam_periph *periph;
1416         struct ada_softc *softc;
1417         char tmpstr[32], tmpstr2[16];
1418
1419         periph = (struct cam_periph *)context;
1420
1421         /* periph was held for us when this task was enqueued */
1422         if ((periph->flags & CAM_PERIPH_INVALID) != 0) {
1423                 cam_periph_release(periph);
1424                 return;
1425         }
1426
1427         softc = (struct ada_softc *)periph->softc;
1428         snprintf(tmpstr, sizeof(tmpstr), "CAM ADA unit %d",periph->unit_number);
1429         snprintf(tmpstr2, sizeof(tmpstr2), "%d", periph->unit_number);
1430
1431         sysctl_ctx_init(&softc->sysctl_ctx);
1432         softc->flags |= ADA_FLAG_SCTX_INIT;
1433         softc->sysctl_tree = SYSCTL_ADD_NODE_WITH_LABEL(&softc->sysctl_ctx,
1434                 SYSCTL_STATIC_CHILDREN(_kern_cam_ada), OID_AUTO, tmpstr2,
1435                 CTLFLAG_RD, 0, tmpstr, "device_index");
1436         if (softc->sysctl_tree == NULL) {
1437                 printf("adasysctlinit: unable to allocate sysctl tree\n");
1438                 cam_periph_release(periph);
1439                 return;
1440         }
1441
1442         SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1443                 OID_AUTO, "delete_method", CTLTYPE_STRING | CTLFLAG_RW,
1444                 softc, 0, adadeletemethodsysctl, "A",
1445                 "BIO_DELETE execution method");
1446         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
1447                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
1448                 "trim_count", CTLFLAG_RD, &softc->trim_count,
1449                 "Total number of dsm commands sent");
1450         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
1451                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
1452                 "trim_ranges", CTLFLAG_RD, &softc->trim_ranges,
1453                 "Total number of ranges in dsm commands");
1454         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
1455                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
1456                 "trim_lbas", CTLFLAG_RD, &softc->trim_lbas,
1457                 "Total lbas in the dsm commands sent");
1458         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1459                 OID_AUTO, "read_ahead", CTLFLAG_RW | CTLFLAG_MPSAFE,
1460                 &softc->read_ahead, 0, "Enable disk read ahead.");
1461         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1462                 OID_AUTO, "write_cache", CTLFLAG_RW | CTLFLAG_MPSAFE,
1463                 &softc->write_cache, 0, "Enable disk write cache.");
1464         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1465                 OID_AUTO, "unmapped_io", CTLFLAG_RD | CTLFLAG_MPSAFE,
1466                 &softc->unmappedio, 0, "Unmapped I/O leaf");
1467         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1468                 OID_AUTO, "rotating", CTLFLAG_RD | CTLFLAG_MPSAFE,
1469                 &softc->rotating, 0, "Rotating media");
1470         SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1471                 OID_AUTO, "zone_mode", CTLTYPE_STRING | CTLFLAG_RD,
1472                 softc, 0, adazonemodesysctl, "A",
1473                 "Zone Mode");
1474         SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1475                 OID_AUTO, "zone_support", CTLTYPE_STRING | CTLFLAG_RD,
1476                 softc, 0, adazonesupsysctl, "A",
1477                 "Zone Support");
1478         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
1479                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
1480                 "optimal_seq_zones", CTLFLAG_RD, &softc->optimal_seq_zones,
1481                 "Optimal Number of Open Sequential Write Preferred Zones");
1482         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
1483                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
1484                 "optimal_nonseq_zones", CTLFLAG_RD,
1485                 &softc->optimal_nonseq_zones,
1486                 "Optimal Number of Non-Sequentially Written Sequential Write "
1487                 "Preferred Zones");
1488         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
1489                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
1490                 "max_seq_zones", CTLFLAG_RD, &softc->max_seq_zones,
1491                 "Maximum Number of Open Sequential Write Required Zones");
1492
1493 #ifdef CAM_TEST_FAILURE
1494         /*
1495          * Add a 'door bell' sysctl which allows one to set it from userland
1496          * and cause something bad to happen.  For the moment, we only allow
1497          * whacking the next read or write.
1498          */
1499         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1500                 OID_AUTO, "force_read_error", CTLFLAG_RW | CTLFLAG_MPSAFE,
1501                 &softc->force_read_error, 0,
1502                 "Force a read error for the next N reads.");
1503         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1504                 OID_AUTO, "force_write_error", CTLFLAG_RW | CTLFLAG_MPSAFE,
1505                 &softc->force_write_error, 0,
1506                 "Force a write error for the next N writes.");
1507         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1508                 OID_AUTO, "periodic_read_error", CTLFLAG_RW | CTLFLAG_MPSAFE,
1509                 &softc->periodic_read_error, 0,
1510                 "Force a read error every N reads (don't set too low).");
1511         SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1512                 OID_AUTO, "invalidate", CTLTYPE_U64 | CTLFLAG_RW | CTLFLAG_MPSAFE,
1513                 periph, 0, cam_periph_invalidate_sysctl, "I",
1514                 "Write 1 to invalidate the drive immediately");
1515 #endif
1516
1517 #ifdef CAM_IO_STATS
1518         softc->sysctl_stats_tree = SYSCTL_ADD_NODE(&softc->sysctl_stats_ctx,
1519                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO, "stats",
1520                 CTLFLAG_RD, 0, "Statistics");
1521         SYSCTL_ADD_INT(&softc->sysctl_stats_ctx,
1522                 SYSCTL_CHILDREN(softc->sysctl_stats_tree),
1523                 OID_AUTO, "timeouts", CTLFLAG_RD | CTLFLAG_MPSAFE,
1524                 &softc->timeouts, 0,
1525                 "Device timeouts reported by the SIM");
1526         SYSCTL_ADD_INT(&softc->sysctl_stats_ctx,
1527                 SYSCTL_CHILDREN(softc->sysctl_stats_tree),
1528                 OID_AUTO, "errors", CTLFLAG_RD | CTLFLAG_MPSAFE,
1529                 &softc->errors, 0,
1530                 "Transport errors reported by the SIM.");
1531         SYSCTL_ADD_INT(&softc->sysctl_stats_ctx,
1532                 SYSCTL_CHILDREN(softc->sysctl_stats_tree),
1533                 OID_AUTO, "pack_invalidations", CTLFLAG_RD | CTLFLAG_MPSAFE,
1534                 &softc->invalidations, 0,
1535                 "Device pack invalidations.");
1536 #endif
1537
1538         cam_iosched_sysctl_init(softc->cam_iosched, &softc->sysctl_ctx,
1539             softc->sysctl_tree);
1540
1541         cam_periph_release(periph);
1542 }
1543
1544 static int
1545 adagetattr(struct bio *bp)
1546 {
1547         int ret;
1548         struct cam_periph *periph;
1549
1550         periph = (struct cam_periph *)bp->bio_disk->d_drv1;
1551         cam_periph_lock(periph);
1552         ret = xpt_getattr(bp->bio_data, bp->bio_length, bp->bio_attribute,
1553             periph->path);
1554         cam_periph_unlock(periph);
1555         if (ret == 0)
1556                 bp->bio_completed = bp->bio_length;
1557         return ret;
1558 }
1559
1560 static int
1561 adadeletemethodsysctl(SYSCTL_HANDLER_ARGS)
1562 {
1563         char buf[16];
1564         const char *p;
1565         struct ada_softc *softc;
1566         int i, error, value, methods;
1567
1568         softc = (struct ada_softc *)arg1;
1569
1570         value = softc->delete_method;
1571         if (value < 0 || value > ADA_DELETE_MAX)
1572                 p = "UNKNOWN";
1573         else
1574                 p = ada_delete_method_names[value];
1575         strncpy(buf, p, sizeof(buf));
1576         error = sysctl_handle_string(oidp, buf, sizeof(buf), req);
1577         if (error != 0 || req->newptr == NULL)
1578                 return (error);
1579         methods = 1 << ADA_DELETE_DISABLE;
1580         if ((softc->flags & ADA_FLAG_CAN_CFA) &&
1581             !(softc->flags & ADA_FLAG_CAN_48BIT))
1582                 methods |= 1 << ADA_DELETE_CFA_ERASE;
1583         if (softc->flags & ADA_FLAG_CAN_TRIM)
1584                 methods |= 1 << ADA_DELETE_DSM_TRIM;
1585         if (softc->flags & ADA_FLAG_CAN_NCQ_TRIM)
1586                 methods |= 1 << ADA_DELETE_NCQ_DSM_TRIM;
1587         for (i = 0; i <= ADA_DELETE_MAX; i++) {
1588                 if (!(methods & (1 << i)) ||
1589                     strcmp(buf, ada_delete_method_names[i]) != 0)
1590                         continue;
1591                 softc->delete_method = i;
1592                 return (0);
1593         }
1594         return (EINVAL);
1595 }
1596
1597 static void
1598 adasetflags(struct ada_softc *softc, struct ccb_getdev *cgd)
1599 {
1600         if ((cgd->ident_data.capabilities1 & ATA_SUPPORT_DMA) &&
1601             (cgd->inq_flags & SID_DMA))
1602                 softc->flags |= ADA_FLAG_CAN_DMA;
1603         else
1604                 softc->flags &= ~ADA_FLAG_CAN_DMA;
1605
1606         if (cgd->ident_data.support.command2 & ATA_SUPPORT_ADDRESS48) {
1607                 softc->flags |= ADA_FLAG_CAN_48BIT;
1608                 if (cgd->inq_flags & SID_DMA48)
1609                         softc->flags |= ADA_FLAG_CAN_DMA48;
1610                 else
1611                         softc->flags &= ~ADA_FLAG_CAN_DMA48;
1612         } else
1613                 softc->flags &= ~(ADA_FLAG_CAN_48BIT | ADA_FLAG_CAN_DMA48);
1614
1615         if (cgd->ident_data.support.command2 & ATA_SUPPORT_FLUSHCACHE)
1616                 softc->flags |= ADA_FLAG_CAN_FLUSHCACHE;
1617         else
1618                 softc->flags &= ~ADA_FLAG_CAN_FLUSHCACHE;
1619
1620         if (cgd->ident_data.support.command1 & ATA_SUPPORT_POWERMGT)
1621                 softc->flags |= ADA_FLAG_CAN_POWERMGT;
1622         else
1623                 softc->flags &= ~ADA_FLAG_CAN_POWERMGT;
1624
1625         if ((cgd->ident_data.satacapabilities & ATA_SUPPORT_NCQ) &&
1626             (cgd->inq_flags & SID_DMA) && (cgd->inq_flags & SID_CmdQue))
1627                 softc->flags |= ADA_FLAG_CAN_NCQ;
1628         else
1629                 softc->flags &= ~ADA_FLAG_CAN_NCQ;
1630
1631         if ((cgd->ident_data.support_dsm & ATA_SUPPORT_DSM_TRIM) &&
1632             (cgd->inq_flags & SID_DMA)) {
1633                 softc->flags |= ADA_FLAG_CAN_TRIM;
1634                 softc->trim_max_ranges = TRIM_MAX_RANGES;
1635                 if (cgd->ident_data.max_dsm_blocks != 0) {
1636                         softc->trim_max_ranges =
1637                             min(cgd->ident_data.max_dsm_blocks *
1638                                 ATA_DSM_BLK_RANGES, softc->trim_max_ranges);
1639                 }
1640                 /*
1641                  * If we can do RCVSND_FPDMA_QUEUED commands, we may be able
1642                  * to do NCQ trims, if we support trims at all. We also need
1643                  * support from the SIM to do things properly. Perhaps we
1644                  * should look at log 13 dword 0 bit 0 and dword 1 bit 0 are
1645                  * set too...
1646                  */
1647                 if ((softc->quirks & ADA_Q_NCQ_TRIM_BROKEN) == 0 &&
1648                     (softc->flags & ADA_FLAG_PIM_ATA_EXT) != 0 &&
1649                     (cgd->ident_data.satacapabilities2 &
1650                      ATA_SUPPORT_RCVSND_FPDMA_QUEUED) != 0 &&
1651                     (softc->flags & ADA_FLAG_CAN_TRIM) != 0)
1652                         softc->flags |= ADA_FLAG_CAN_NCQ_TRIM;
1653                 else
1654                         softc->flags &= ~ADA_FLAG_CAN_NCQ_TRIM;
1655         } else
1656                 softc->flags &= ~(ADA_FLAG_CAN_TRIM | ADA_FLAG_CAN_NCQ_TRIM);
1657
1658         if (cgd->ident_data.support.command2 & ATA_SUPPORT_CFA)
1659                 softc->flags |= ADA_FLAG_CAN_CFA;
1660         else
1661                 softc->flags &= ~ADA_FLAG_CAN_CFA;
1662
1663         /*
1664          * Now that we've set the appropriate flags, setup the delete
1665          * method.
1666          */
1667         adasetdeletemethod(softc);
1668
1669         if ((cgd->ident_data.support.extension & ATA_SUPPORT_GENLOG)
1670          && ((softc->quirks & ADA_Q_LOG_BROKEN) == 0))
1671                 softc->flags |= ADA_FLAG_CAN_LOG;
1672         else
1673                 softc->flags &= ~ADA_FLAG_CAN_LOG;
1674
1675         if ((cgd->ident_data.support3 & ATA_SUPPORT_ZONE_MASK) ==
1676              ATA_SUPPORT_ZONE_HOST_AWARE)
1677                 softc->zone_mode = ADA_ZONE_HOST_AWARE;
1678         else if (((cgd->ident_data.support3 & ATA_SUPPORT_ZONE_MASK) ==
1679                    ATA_SUPPORT_ZONE_DEV_MANAGED)
1680               || (softc->quirks & ADA_Q_SMR_DM))
1681                 softc->zone_mode = ADA_ZONE_DRIVE_MANAGED;
1682         else
1683                 softc->zone_mode = ADA_ZONE_NONE;
1684
1685         if (cgd->ident_data.support.command1 & ATA_SUPPORT_LOOKAHEAD)
1686                 softc->flags |= ADA_FLAG_CAN_RAHEAD;
1687         else
1688                 softc->flags &= ~ADA_FLAG_CAN_RAHEAD;
1689
1690         if (cgd->ident_data.support.command1 & ATA_SUPPORT_WRITECACHE)
1691                 softc->flags |= ADA_FLAG_CAN_WCACHE;
1692         else
1693                 softc->flags &= ~ADA_FLAG_CAN_WCACHE;
1694 }
1695
1696 static cam_status
1697 adaregister(struct cam_periph *periph, void *arg)
1698 {
1699         struct ada_softc *softc;
1700         struct ccb_pathinq cpi;
1701         struct ccb_getdev *cgd;
1702         struct disk_params *dp;
1703         struct sbuf sb;
1704         char   *announce_buf;
1705         caddr_t match;
1706         u_int maxio;
1707         int quirks;
1708
1709         cgd = (struct ccb_getdev *)arg;
1710         if (cgd == NULL) {
1711                 printf("adaregister: no getdev CCB, can't register device\n");
1712                 return(CAM_REQ_CMP_ERR);
1713         }
1714
1715         softc = (struct ada_softc *)malloc(sizeof(*softc), M_DEVBUF,
1716             M_NOWAIT|M_ZERO);
1717
1718         if (softc == NULL) {
1719                 printf("adaregister: Unable to probe new device. "
1720                     "Unable to allocate softc\n");
1721                 return(CAM_REQ_CMP_ERR);
1722         }
1723
1724         announce_buf = softc->announce_temp;
1725         bzero(announce_buf, ADA_ANNOUNCETMP_SZ);
1726
1727         if (cam_iosched_init(&softc->cam_iosched, periph) != 0) {
1728                 printf("adaregister: Unable to probe new device. "
1729                        "Unable to allocate iosched memory\n");
1730                 free(softc, M_DEVBUF);
1731                 return(CAM_REQ_CMP_ERR);
1732         }
1733
1734         periph->softc = softc;
1735
1736         /*
1737          * See if this device has any quirks.
1738          */
1739         match = cam_quirkmatch((caddr_t)&cgd->ident_data,
1740                                (caddr_t)ada_quirk_table,
1741                                nitems(ada_quirk_table),
1742                                sizeof(*ada_quirk_table), ata_identify_match);
1743         if (match != NULL)
1744                 softc->quirks = ((struct ada_quirk_entry *)match)->quirks;
1745         else
1746                 softc->quirks = ADA_Q_NONE;
1747
1748         xpt_path_inq(&cpi, periph->path);
1749
1750         TASK_INIT(&softc->sysctl_task, 0, adasysctlinit, periph);
1751
1752         /*
1753          * Register this media as a disk
1754          */
1755         (void)cam_periph_hold(periph, PRIBIO);
1756         cam_periph_unlock(periph);
1757         snprintf(announce_buf, ADA_ANNOUNCETMP_SZ,
1758             "kern.cam.ada.%d.quirks", periph->unit_number);
1759         quirks = softc->quirks;
1760         TUNABLE_INT_FETCH(announce_buf, &quirks);
1761         softc->quirks = quirks;
1762         softc->read_ahead = -1;
1763         snprintf(announce_buf, ADA_ANNOUNCETMP_SZ,
1764             "kern.cam.ada.%d.read_ahead", periph->unit_number);
1765         TUNABLE_INT_FETCH(announce_buf, &softc->read_ahead);
1766         softc->write_cache = -1;
1767         snprintf(announce_buf, ADA_ANNOUNCETMP_SZ,
1768             "kern.cam.ada.%d.write_cache", periph->unit_number);
1769         TUNABLE_INT_FETCH(announce_buf, &softc->write_cache);
1770
1771         /*
1772          * Set support flags based on the Identify data and quirks.
1773          */
1774         adasetflags(softc, cgd);
1775
1776         /* Disable queue sorting for non-rotational media by default. */
1777         if (cgd->ident_data.media_rotation_rate == ATA_RATE_NON_ROTATING) {
1778                 softc->rotating = 0;
1779         } else {
1780                 softc->rotating = 1;
1781         }
1782         cam_iosched_set_sort_queue(softc->cam_iosched,  softc->rotating ? -1 : 0);
1783         adagetparams(periph, cgd);
1784         softc->disk = disk_alloc();
1785         softc->disk->d_rotation_rate = cgd->ident_data.media_rotation_rate;
1786         softc->disk->d_devstat = devstat_new_entry(periph->periph_name,
1787                           periph->unit_number, softc->params.secsize,
1788                           DEVSTAT_ALL_SUPPORTED,
1789                           DEVSTAT_TYPE_DIRECT |
1790                           XPORT_DEVSTAT_TYPE(cpi.transport),
1791                           DEVSTAT_PRIORITY_DISK);
1792         softc->disk->d_open = adaopen;
1793         softc->disk->d_close = adaclose;
1794         softc->disk->d_strategy = adastrategy;
1795         softc->disk->d_getattr = adagetattr;
1796         softc->disk->d_dump = adadump;
1797         softc->disk->d_gone = adadiskgonecb;
1798         softc->disk->d_name = "ada";
1799         softc->disk->d_drv1 = periph;
1800         maxio = cpi.maxio;              /* Honor max I/O size of SIM */
1801         if (maxio == 0)
1802                 maxio = DFLTPHYS;       /* traditional default */
1803         else if (maxio > MAXPHYS)
1804                 maxio = MAXPHYS;        /* for safety */
1805         if (softc->flags & ADA_FLAG_CAN_48BIT)
1806                 maxio = min(maxio, 65536 * softc->params.secsize);
1807         else                                    /* 28bit ATA command limit */
1808                 maxio = min(maxio, 256 * softc->params.secsize);
1809         softc->disk->d_maxsize = maxio;
1810         softc->disk->d_unit = periph->unit_number;
1811         softc->disk->d_flags = DISKFLAG_DIRECT_COMPLETION | DISKFLAG_CANZONE;
1812         if (softc->flags & ADA_FLAG_CAN_FLUSHCACHE)
1813                 softc->disk->d_flags |= DISKFLAG_CANFLUSHCACHE;
1814         if (softc->flags & ADA_FLAG_CAN_TRIM) {
1815                 softc->disk->d_flags |= DISKFLAG_CANDELETE;
1816                 softc->disk->d_delmaxsize = softc->params.secsize *
1817                                             ATA_DSM_RANGE_MAX *
1818                                             softc->trim_max_ranges;
1819         } else if ((softc->flags & ADA_FLAG_CAN_CFA) &&
1820             !(softc->flags & ADA_FLAG_CAN_48BIT)) {
1821                 softc->disk->d_flags |= DISKFLAG_CANDELETE;
1822                 softc->disk->d_delmaxsize = 256 * softc->params.secsize;
1823         } else
1824                 softc->disk->d_delmaxsize = maxio;
1825         if ((cpi.hba_misc & PIM_UNMAPPED) != 0) {
1826                 softc->disk->d_flags |= DISKFLAG_UNMAPPED_BIO;
1827                 softc->unmappedio = 1;
1828         }
1829         if (cpi.hba_misc & PIM_ATA_EXT)
1830                 softc->flags |= ADA_FLAG_PIM_ATA_EXT;
1831         strlcpy(softc->disk->d_descr, cgd->ident_data.model,
1832             MIN(sizeof(softc->disk->d_descr), sizeof(cgd->ident_data.model)));
1833         strlcpy(softc->disk->d_ident, cgd->ident_data.serial,
1834             MIN(sizeof(softc->disk->d_ident), sizeof(cgd->ident_data.serial)));
1835         softc->disk->d_hba_vendor = cpi.hba_vendor;
1836         softc->disk->d_hba_device = cpi.hba_device;
1837         softc->disk->d_hba_subvendor = cpi.hba_subvendor;
1838         softc->disk->d_hba_subdevice = cpi.hba_subdevice;
1839
1840         softc->disk->d_sectorsize = softc->params.secsize;
1841         softc->disk->d_mediasize = (off_t)softc->params.sectors *
1842             softc->params.secsize;
1843         if (ata_physical_sector_size(&cgd->ident_data) !=
1844             softc->params.secsize) {
1845                 softc->disk->d_stripesize =
1846                     ata_physical_sector_size(&cgd->ident_data);
1847                 softc->disk->d_stripeoffset = (softc->disk->d_stripesize -
1848                     ata_logical_sector_offset(&cgd->ident_data)) %
1849                     softc->disk->d_stripesize;
1850         } else if (softc->quirks & ADA_Q_4K) {
1851                 softc->disk->d_stripesize = 4096;
1852                 softc->disk->d_stripeoffset = 0;
1853         }
1854         softc->disk->d_fwsectors = softc->params.secs_per_track;
1855         softc->disk->d_fwheads = softc->params.heads;
1856         ata_disk_firmware_geom_adjust(softc->disk);
1857
1858         /*
1859          * Acquire a reference to the periph before we register with GEOM.
1860          * We'll release this reference once GEOM calls us back (via
1861          * adadiskgonecb()) telling us that our provider has been freed.
1862          */
1863         if (cam_periph_acquire(periph) != 0) {
1864                 xpt_print(periph->path, "%s: lost periph during "
1865                           "registration!\n", __func__);
1866                 cam_periph_lock(periph);
1867                 return (CAM_REQ_CMP_ERR);
1868         }
1869         disk_create(softc->disk, DISK_VERSION);
1870         cam_periph_lock(periph);
1871
1872         dp = &softc->params;
1873         snprintf(announce_buf, ADA_ANNOUNCETMP_SZ,
1874             "%juMB (%ju %u byte sectors)",
1875             ((uintmax_t)dp->secsize * dp->sectors) / (1024 * 1024),
1876             (uintmax_t)dp->sectors, dp->secsize);
1877
1878         sbuf_new(&sb, softc->announce_buffer, ADA_ANNOUNCE_SZ, SBUF_FIXEDLEN);
1879         xpt_announce_periph_sbuf(periph, &sb, announce_buf);
1880         xpt_announce_quirks_sbuf(periph, &sb, softc->quirks, ADA_Q_BIT_STRING);
1881         sbuf_finish(&sb);
1882         sbuf_putbuf(&sb);
1883
1884         /*
1885          * Create our sysctl variables, now that we know
1886          * we have successfully attached.
1887          */
1888         if (cam_periph_acquire(periph) == 0)
1889                 taskqueue_enqueue(taskqueue_thread, &softc->sysctl_task);
1890
1891         /*
1892          * Add async callbacks for bus reset and
1893          * bus device reset calls.  I don't bother
1894          * checking if this fails as, in most cases,
1895          * the system will function just fine without
1896          * them and the only alternative would be to
1897          * not attach the device on failure.
1898          */
1899         xpt_register_async(AC_SENT_BDR | AC_BUS_RESET | AC_LOST_DEVICE |
1900             AC_GETDEV_CHANGED | AC_ADVINFO_CHANGED,
1901             adaasync, periph, periph->path);
1902
1903         /*
1904          * Schedule a periodic event to occasionally send an
1905          * ordered tag to a device.
1906          */
1907         callout_init_mtx(&softc->sendordered_c, cam_periph_mtx(periph), 0);
1908         callout_reset(&softc->sendordered_c,
1909             (ada_default_timeout * hz) / ADA_ORDEREDTAG_INTERVAL,
1910             adasendorderedtag, softc);
1911
1912         if (ADA_RA >= 0 && softc->flags & ADA_FLAG_CAN_RAHEAD) {
1913                 softc->state = ADA_STATE_RAHEAD;
1914         } else if (ADA_WC >= 0 && softc->flags & ADA_FLAG_CAN_WCACHE) {
1915                 softc->state = ADA_STATE_WCACHE;
1916         } else if ((softc->flags & ADA_FLAG_CAN_LOG)
1917                 && (softc->zone_mode != ADA_ZONE_NONE)) {
1918                 softc->state = ADA_STATE_LOGDIR;
1919         } else {
1920                 /*
1921                  * Nothing to probe, so we can just transition to the
1922                  * normal state.
1923                  */
1924                 adaprobedone(periph, NULL);
1925                 return(CAM_REQ_CMP);
1926         }
1927
1928         xpt_schedule(periph, CAM_PRIORITY_DEV);
1929
1930         return(CAM_REQ_CMP);
1931 }
1932
1933 static int
1934 ada_dsmtrim_req_create(struct ada_softc *softc, struct bio *bp, struct trim_request *req)
1935 {
1936         uint64_t lastlba = (uint64_t)-1, lbas = 0;
1937         int c, lastcount = 0, off, ranges = 0;
1938
1939         bzero(req, sizeof(*req));
1940         TAILQ_INIT(&req->bps);
1941         do {
1942                 uint64_t lba = bp->bio_pblkno;
1943                 int count = bp->bio_bcount / softc->params.secsize;
1944
1945                 /* Try to extend the previous range. */
1946                 if (lba == lastlba) {
1947                         c = min(count, ATA_DSM_RANGE_MAX - lastcount);
1948                         lastcount += c;
1949                         off = (ranges - 1) * ATA_DSM_RANGE_SIZE;
1950                         req->data[off + 6] = lastcount & 0xff;
1951                         req->data[off + 7] =
1952                                 (lastcount >> 8) & 0xff;
1953                         count -= c;
1954                         lba += c;
1955                         lbas += c;
1956                 }
1957
1958                 while (count > 0) {
1959                         c = min(count, ATA_DSM_RANGE_MAX);
1960                         off = ranges * ATA_DSM_RANGE_SIZE;
1961                         req->data[off + 0] = lba & 0xff;
1962                         req->data[off + 1] = (lba >> 8) & 0xff;
1963                         req->data[off + 2] = (lba >> 16) & 0xff;
1964                         req->data[off + 3] = (lba >> 24) & 0xff;
1965                         req->data[off + 4] = (lba >> 32) & 0xff;
1966                         req->data[off + 5] = (lba >> 40) & 0xff;
1967                         req->data[off + 6] = c & 0xff;
1968                         req->data[off + 7] = (c >> 8) & 0xff;
1969                         lba += c;
1970                         lbas += c;
1971                         count -= c;
1972                         lastcount = c;
1973                         ranges++;
1974                         /*
1975                          * Its the caller's responsibility to ensure the
1976                          * request will fit so we don't need to check for
1977                          * overrun here
1978                          */
1979                 }
1980                 lastlba = lba;
1981                 TAILQ_INSERT_TAIL(&req->bps, bp, bio_queue);
1982
1983                 bp = cam_iosched_next_trim(softc->cam_iosched);
1984                 if (bp == NULL)
1985                         break;
1986                 if (bp->bio_bcount / softc->params.secsize >
1987                     (softc->trim_max_ranges - ranges) * ATA_DSM_RANGE_MAX) {
1988                         cam_iosched_put_back_trim(softc->cam_iosched, bp);
1989                         break;
1990                 }
1991         } while (1);
1992         softc->trim_count++;
1993         softc->trim_ranges += ranges;
1994         softc->trim_lbas += lbas;
1995
1996         return (ranges);
1997 }
1998
1999 static void
2000 ada_dsmtrim(struct ada_softc *softc, struct bio *bp, struct ccb_ataio *ataio)
2001 {
2002         struct trim_request *req = &softc->trim_req;
2003         int ranges;
2004
2005         ranges = ada_dsmtrim_req_create(softc, bp, req);
2006         cam_fill_ataio(ataio,
2007             ada_retry_count,
2008             adadone,
2009             CAM_DIR_OUT,
2010             0,
2011             req->data,
2012             howmany(ranges, ATA_DSM_BLK_RANGES) * ATA_DSM_BLK_SIZE,
2013             ada_default_timeout * 1000);
2014         ata_48bit_cmd(ataio, ATA_DATA_SET_MANAGEMENT,
2015             ATA_DSM_TRIM, 0, howmany(ranges, ATA_DSM_BLK_RANGES));
2016 }
2017
2018 static void
2019 ada_ncq_dsmtrim(struct ada_softc *softc, struct bio *bp, struct ccb_ataio *ataio)
2020 {
2021         struct trim_request *req = &softc->trim_req;
2022         int ranges;
2023
2024         ranges = ada_dsmtrim_req_create(softc, bp, req);
2025         cam_fill_ataio(ataio,
2026             ada_retry_count,
2027             adadone,
2028             CAM_DIR_OUT,
2029             0,
2030             req->data,
2031             howmany(ranges, ATA_DSM_BLK_RANGES) * ATA_DSM_BLK_SIZE,
2032             ada_default_timeout * 1000);
2033         ata_ncq_cmd(ataio,
2034             ATA_SEND_FPDMA_QUEUED,
2035             0,
2036             howmany(ranges, ATA_DSM_BLK_RANGES));
2037         ataio->cmd.sector_count_exp = ATA_SFPDMA_DSM;
2038         ataio->ata_flags |= ATA_FLAG_AUX;
2039         ataio->aux = 1;
2040 }
2041
2042 static void
2043 ada_cfaerase(struct ada_softc *softc, struct bio *bp, struct ccb_ataio *ataio)
2044 {
2045         struct trim_request *req = &softc->trim_req;
2046         uint64_t lba = bp->bio_pblkno;
2047         uint16_t count = bp->bio_bcount / softc->params.secsize;
2048
2049         bzero(req, sizeof(*req));
2050         TAILQ_INIT(&req->bps);
2051         TAILQ_INSERT_TAIL(&req->bps, bp, bio_queue);
2052
2053         cam_fill_ataio(ataio,
2054             ada_retry_count,
2055             adadone,
2056             CAM_DIR_NONE,
2057             0,
2058             NULL,
2059             0,
2060             ada_default_timeout*1000);
2061
2062         if (count >= 256)
2063                 count = 0;
2064         ata_28bit_cmd(ataio, ATA_CFA_ERASE, 0, lba, count);
2065 }
2066
2067 static int
2068 ada_zone_bio_to_ata(int disk_zone_cmd)
2069 {
2070         switch (disk_zone_cmd) {
2071         case DISK_ZONE_OPEN:
2072                 return ATA_ZM_OPEN_ZONE;
2073         case DISK_ZONE_CLOSE:
2074                 return ATA_ZM_CLOSE_ZONE;
2075         case DISK_ZONE_FINISH:
2076                 return ATA_ZM_FINISH_ZONE;
2077         case DISK_ZONE_RWP:
2078                 return ATA_ZM_RWP;
2079         }
2080
2081         return -1;
2082 }
2083
2084 static int
2085 ada_zone_cmd(struct cam_periph *periph, union ccb *ccb, struct bio *bp,
2086              int *queue_ccb)
2087 {
2088         struct ada_softc *softc;
2089         int error;
2090
2091         error = 0;
2092
2093         if (bp->bio_cmd != BIO_ZONE) {
2094                 error = EINVAL;
2095                 goto bailout;
2096         }
2097
2098         softc = periph->softc;
2099
2100         switch (bp->bio_zone.zone_cmd) {
2101         case DISK_ZONE_OPEN:
2102         case DISK_ZONE_CLOSE:
2103         case DISK_ZONE_FINISH:
2104         case DISK_ZONE_RWP: {
2105                 int zone_flags;
2106                 int zone_sa;
2107                 uint64_t lba;
2108
2109                 zone_sa = ada_zone_bio_to_ata(bp->bio_zone.zone_cmd);
2110                 if (zone_sa == -1) {
2111                         xpt_print(periph->path, "Cannot translate zone "
2112                             "cmd %#x to ATA\n", bp->bio_zone.zone_cmd);
2113                         error = EINVAL;
2114                         goto bailout;
2115                 }
2116
2117                 zone_flags = 0;
2118                 lba = bp->bio_zone.zone_params.rwp.id;
2119
2120                 if (bp->bio_zone.zone_params.rwp.flags &
2121                     DISK_ZONE_RWP_FLAG_ALL)
2122                         zone_flags |= ZBC_OUT_ALL;
2123
2124                 ata_zac_mgmt_out(&ccb->ataio,
2125                                  /*retries*/ ada_retry_count,
2126                                  /*cbfcnp*/ adadone,
2127                                  /*use_ncq*/ (softc->flags &
2128                                               ADA_FLAG_PIM_ATA_EXT) ? 1 : 0,
2129                                  /*zm_action*/ zone_sa,
2130                                  /*zone_id*/ lba,
2131                                  /*zone_flags*/ zone_flags,
2132                                  /*sector_count*/ 0,
2133                                  /*data_ptr*/ NULL,
2134                                  /*dxfer_len*/ 0,
2135                                  /*timeout*/ ada_default_timeout * 1000);
2136                 *queue_ccb = 1;
2137
2138                 break;
2139         }
2140         case DISK_ZONE_REPORT_ZONES: {
2141                 uint8_t *rz_ptr;
2142                 uint32_t num_entries, alloc_size;
2143                 struct disk_zone_report *rep;
2144
2145                 rep = &bp->bio_zone.zone_params.report;
2146
2147                 num_entries = rep->entries_allocated;
2148                 if (num_entries == 0) {
2149                         xpt_print(periph->path, "No entries allocated for "
2150                             "Report Zones request\n");
2151                         error = EINVAL;
2152                         goto bailout;
2153                 }
2154                 alloc_size = sizeof(struct scsi_report_zones_hdr) +
2155                     (sizeof(struct scsi_report_zones_desc) * num_entries);
2156                 alloc_size = min(alloc_size, softc->disk->d_maxsize);
2157                 rz_ptr = malloc(alloc_size, M_ATADA, M_NOWAIT | M_ZERO);
2158                 if (rz_ptr == NULL) {
2159                         xpt_print(periph->path, "Unable to allocate memory "
2160                            "for Report Zones request\n");
2161                         error = ENOMEM;
2162                         goto bailout;
2163                 }
2164
2165                 ata_zac_mgmt_in(&ccb->ataio,
2166                                 /*retries*/ ada_retry_count,
2167                                 /*cbcfnp*/ adadone,
2168                                 /*use_ncq*/ (softc->flags &
2169                                              ADA_FLAG_PIM_ATA_EXT) ? 1 : 0,
2170                                 /*zm_action*/ ATA_ZM_REPORT_ZONES,
2171                                 /*zone_id*/ rep->starting_id,
2172                                 /*zone_flags*/ rep->rep_options,
2173                                 /*data_ptr*/ rz_ptr,
2174                                 /*dxfer_len*/ alloc_size,
2175                                 /*timeout*/ ada_default_timeout * 1000);
2176
2177                 /*
2178                  * For BIO_ZONE, this isn't normally needed.  However, it
2179                  * is used by devstat_end_transaction_bio() to determine
2180                  * how much data was transferred.
2181                  */
2182                 /*
2183                  * XXX KDM we have a problem.  But I'm not sure how to fix
2184                  * it.  devstat uses bio_bcount - bio_resid to calculate
2185                  * the amount of data transferred.   The GEOM disk code
2186                  * uses bio_length - bio_resid to calculate the amount of
2187                  * data in bio_completed.  We have different structure
2188                  * sizes above and below the ada(4) driver.  So, if we
2189                  * use the sizes above, the amount transferred won't be
2190                  * quite accurate for devstat.  If we use different sizes
2191                  * for bio_bcount and bio_length (above and below
2192                  * respectively), then the residual needs to match one or
2193                  * the other.  Everything is calculated after the bio
2194                  * leaves the driver, so changing the values around isn't
2195                  * really an option.  For now, just set the count to the
2196                  * passed in length.  This means that the calculations
2197                  * above (e.g. bio_completed) will be correct, but the
2198                  * amount of data reported to devstat will be slightly
2199                  * under or overstated.
2200                  */
2201                 bp->bio_bcount = bp->bio_length;
2202
2203                 *queue_ccb = 1;
2204
2205                 break;
2206         }
2207         case DISK_ZONE_GET_PARAMS: {
2208                 struct disk_zone_disk_params *params;
2209
2210                 params = &bp->bio_zone.zone_params.disk_params;
2211                 bzero(params, sizeof(*params));
2212
2213                 switch (softc->zone_mode) {
2214                 case ADA_ZONE_DRIVE_MANAGED:
2215                         params->zone_mode = DISK_ZONE_MODE_DRIVE_MANAGED;
2216                         break;
2217                 case ADA_ZONE_HOST_AWARE:
2218                         params->zone_mode = DISK_ZONE_MODE_HOST_AWARE;
2219                         break;
2220                 case ADA_ZONE_HOST_MANAGED:
2221                         params->zone_mode = DISK_ZONE_MODE_HOST_MANAGED;
2222                         break;
2223                 default:
2224                 case ADA_ZONE_NONE:
2225                         params->zone_mode = DISK_ZONE_MODE_NONE;
2226                         break;
2227                 }
2228
2229                 if (softc->zone_flags & ADA_ZONE_FLAG_URSWRZ)
2230                         params->flags |= DISK_ZONE_DISK_URSWRZ;
2231
2232                 if (softc->zone_flags & ADA_ZONE_FLAG_OPT_SEQ_SET) {
2233                         params->optimal_seq_zones = softc->optimal_seq_zones;
2234                         params->flags |= DISK_ZONE_OPT_SEQ_SET;
2235                 }
2236
2237                 if (softc->zone_flags & ADA_ZONE_FLAG_OPT_NONSEQ_SET) {
2238                         params->optimal_nonseq_zones =
2239                             softc->optimal_nonseq_zones;
2240                         params->flags |= DISK_ZONE_OPT_NONSEQ_SET;
2241                 }
2242
2243                 if (softc->zone_flags & ADA_ZONE_FLAG_MAX_SEQ_SET) {
2244                         params->max_seq_zones = softc->max_seq_zones;
2245                         params->flags |= DISK_ZONE_MAX_SEQ_SET;
2246                 }
2247                 if (softc->zone_flags & ADA_ZONE_FLAG_RZ_SUP)
2248                         params->flags |= DISK_ZONE_RZ_SUP;
2249
2250                 if (softc->zone_flags & ADA_ZONE_FLAG_OPEN_SUP)
2251                         params->flags |= DISK_ZONE_OPEN_SUP;
2252
2253                 if (softc->zone_flags & ADA_ZONE_FLAG_CLOSE_SUP)
2254                         params->flags |= DISK_ZONE_CLOSE_SUP;
2255
2256                 if (softc->zone_flags & ADA_ZONE_FLAG_FINISH_SUP)
2257                         params->flags |= DISK_ZONE_FINISH_SUP;
2258
2259                 if (softc->zone_flags & ADA_ZONE_FLAG_RWP_SUP)
2260                         params->flags |= DISK_ZONE_RWP_SUP;
2261                 break;
2262         }
2263         default:
2264                 break;
2265         }
2266 bailout:
2267         return (error);
2268 }
2269
2270 static void
2271 adastart(struct cam_periph *periph, union ccb *start_ccb)
2272 {
2273         struct ada_softc *softc = (struct ada_softc *)periph->softc;
2274         struct ccb_ataio *ataio = &start_ccb->ataio;
2275
2276         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("adastart\n"));
2277
2278         switch (softc->state) {
2279         case ADA_STATE_NORMAL:
2280         {
2281                 struct bio *bp;
2282                 u_int8_t tag_code;
2283
2284                 bp = cam_iosched_next_bio(softc->cam_iosched);
2285                 if (bp == NULL) {
2286                         xpt_release_ccb(start_ccb);
2287                         break;
2288                 }
2289
2290                 if ((bp->bio_flags & BIO_ORDERED) != 0 ||
2291                     (bp->bio_cmd != BIO_DELETE && (softc->flags & ADA_FLAG_NEED_OTAG) != 0)) {
2292                         softc->flags &= ~ADA_FLAG_NEED_OTAG;
2293                         softc->flags |= ADA_FLAG_WAS_OTAG;
2294                         tag_code = 0;
2295                 } else {
2296                         tag_code = 1;
2297                 }
2298                 switch (bp->bio_cmd) {
2299                 case BIO_WRITE:
2300                 case BIO_READ:
2301                 {
2302                         uint64_t lba = bp->bio_pblkno;
2303                         uint16_t count = bp->bio_bcount / softc->params.secsize;
2304                         void *data_ptr;
2305                         int rw_op;
2306
2307                         if (bp->bio_cmd == BIO_WRITE) {
2308                                 softc->flags |= ADA_FLAG_DIRTY;
2309                                 rw_op = CAM_DIR_OUT;
2310                         } else {
2311                                 rw_op = CAM_DIR_IN;
2312                         }
2313
2314                         data_ptr = bp->bio_data;
2315                         if ((bp->bio_flags & (BIO_UNMAPPED|BIO_VLIST)) != 0) {
2316                                 rw_op |= CAM_DATA_BIO;
2317                                 data_ptr = bp;
2318                         }
2319
2320 #ifdef CAM_TEST_FAILURE
2321                         int fail = 0;
2322
2323                         /*
2324                          * Support the failure ioctls.  If the command is a
2325                          * read, and there are pending forced read errors, or
2326                          * if a write and pending write errors, then fail this
2327                          * operation with EIO.  This is useful for testing
2328                          * purposes.  Also, support having every Nth read fail.
2329                          *
2330                          * This is a rather blunt tool.
2331                          */
2332                         if (bp->bio_cmd == BIO_READ) {
2333                                 if (softc->force_read_error) {
2334                                         softc->force_read_error--;
2335                                         fail = 1;
2336                                 }
2337                                 if (softc->periodic_read_error > 0) {
2338                                         if (++softc->periodic_read_count >=
2339                                             softc->periodic_read_error) {
2340                                                 softc->periodic_read_count = 0;
2341                                                 fail = 1;
2342                                         }
2343                                 }
2344                         } else {
2345                                 if (softc->force_write_error) {
2346                                         softc->force_write_error--;
2347                                         fail = 1;
2348                                 }
2349                         }
2350                         if (fail) {
2351                                 biofinish(bp, NULL, EIO);
2352                                 xpt_release_ccb(start_ccb);
2353                                 adaschedule(periph);
2354                                 return;
2355                         }
2356 #endif
2357                         KASSERT((bp->bio_flags & BIO_UNMAPPED) == 0 ||
2358                             round_page(bp->bio_bcount + bp->bio_ma_offset) /
2359                             PAGE_SIZE == bp->bio_ma_n,
2360                             ("Short bio %p", bp));
2361                         cam_fill_ataio(ataio,
2362                             ada_retry_count,
2363                             adadone,
2364                             rw_op,
2365                             0,
2366                             data_ptr,
2367                             bp->bio_bcount,
2368                             ada_default_timeout*1000);
2369
2370                         if ((softc->flags & ADA_FLAG_CAN_NCQ) && tag_code) {
2371                                 if (bp->bio_cmd == BIO_READ) {
2372                                         ata_ncq_cmd(ataio, ATA_READ_FPDMA_QUEUED,
2373                                             lba, count);
2374                                 } else {
2375                                         ata_ncq_cmd(ataio, ATA_WRITE_FPDMA_QUEUED,
2376                                             lba, count);
2377                                 }
2378                         } else if ((softc->flags & ADA_FLAG_CAN_48BIT) &&
2379                             (lba + count >= ATA_MAX_28BIT_LBA ||
2380                             count > 256)) {
2381                                 if (softc->flags & ADA_FLAG_CAN_DMA48) {
2382                                         if (bp->bio_cmd == BIO_READ) {
2383                                                 ata_48bit_cmd(ataio, ATA_READ_DMA48,
2384                                                     0, lba, count);
2385                                         } else {
2386                                                 ata_48bit_cmd(ataio, ATA_WRITE_DMA48,
2387                                                     0, lba, count);
2388                                         }
2389                                 } else {
2390                                         if (bp->bio_cmd == BIO_READ) {
2391                                                 ata_48bit_cmd(ataio, ATA_READ_MUL48,
2392                                                     0, lba, count);
2393                                         } else {
2394                                                 ata_48bit_cmd(ataio, ATA_WRITE_MUL48,
2395                                                     0, lba, count);
2396                                         }
2397                                 }
2398                         } else {
2399                                 if (count == 256)
2400                                         count = 0;
2401                                 if (softc->flags & ADA_FLAG_CAN_DMA) {
2402                                         if (bp->bio_cmd == BIO_READ) {
2403                                                 ata_28bit_cmd(ataio, ATA_READ_DMA,
2404                                                     0, lba, count);
2405                                         } else {
2406                                                 ata_28bit_cmd(ataio, ATA_WRITE_DMA,
2407                                                     0, lba, count);
2408                                         }
2409                                 } else {
2410                                         if (bp->bio_cmd == BIO_READ) {
2411                                                 ata_28bit_cmd(ataio, ATA_READ_MUL,
2412                                                     0, lba, count);
2413                                         } else {
2414                                                 ata_28bit_cmd(ataio, ATA_WRITE_MUL,
2415                                                     0, lba, count);
2416                                         }
2417                                 }
2418                         }
2419                         break;
2420                 }
2421                 case BIO_DELETE:
2422                         switch (softc->delete_method) {
2423                         case ADA_DELETE_NCQ_DSM_TRIM:
2424                                 ada_ncq_dsmtrim(softc, bp, ataio);
2425                                 break;
2426                         case ADA_DELETE_DSM_TRIM:
2427                                 ada_dsmtrim(softc, bp, ataio);
2428                                 break;
2429                         case ADA_DELETE_CFA_ERASE:
2430                                 ada_cfaerase(softc, bp, ataio);
2431                                 break;
2432                         default:
2433                                 biofinish(bp, NULL, EOPNOTSUPP);
2434                                 xpt_release_ccb(start_ccb);
2435                                 adaschedule(periph);
2436                                 return;
2437                         }
2438                         start_ccb->ccb_h.ccb_state = ADA_CCB_TRIM;
2439                         start_ccb->ccb_h.flags |= CAM_UNLOCKED;
2440                         cam_iosched_submit_trim(softc->cam_iosched);
2441                         goto out;
2442                 case BIO_FLUSH:
2443                         cam_fill_ataio(ataio,
2444                             1,
2445                             adadone,
2446                             CAM_DIR_NONE,
2447                             0,
2448                             NULL,
2449                             0,
2450                             ada_default_timeout*1000);
2451
2452                         if (softc->flags & ADA_FLAG_CAN_48BIT)
2453                                 ata_48bit_cmd(ataio, ATA_FLUSHCACHE48, 0, 0, 0);
2454                         else
2455                                 ata_28bit_cmd(ataio, ATA_FLUSHCACHE, 0, 0, 0);
2456                         break;
2457                 case BIO_ZONE: {
2458                         int error, queue_ccb;
2459
2460                         queue_ccb = 0;
2461
2462                         error = ada_zone_cmd(periph, start_ccb, bp, &queue_ccb);
2463                         if ((error != 0)
2464                          || (queue_ccb == 0)) {
2465                                 biofinish(bp, NULL, error);
2466                                 xpt_release_ccb(start_ccb);
2467                                 return;
2468                         }
2469                         break;
2470                 }
2471                 }
2472                 start_ccb->ccb_h.ccb_state = ADA_CCB_BUFFER_IO;
2473                 start_ccb->ccb_h.flags |= CAM_UNLOCKED;
2474 out:
2475                 start_ccb->ccb_h.ccb_bp = bp;
2476                 softc->outstanding_cmds++;
2477                 softc->refcount++;
2478                 cam_periph_unlock(periph);
2479                 xpt_action(start_ccb);
2480                 cam_periph_lock(periph);
2481
2482                 /* May have more work to do, so ensure we stay scheduled */
2483                 adaschedule(periph);
2484                 break;
2485         }
2486         case ADA_STATE_RAHEAD:
2487         case ADA_STATE_WCACHE:
2488         {
2489                 cam_fill_ataio(ataio,
2490                     1,
2491                     adadone,
2492                     CAM_DIR_NONE,
2493                     0,
2494                     NULL,
2495                     0,
2496                     ada_default_timeout*1000);
2497
2498                 if (softc->state == ADA_STATE_RAHEAD) {
2499                         ata_28bit_cmd(ataio, ATA_SETFEATURES, ADA_RA ?
2500                             ATA_SF_ENAB_RCACHE : ATA_SF_DIS_RCACHE, 0, 0);
2501                         start_ccb->ccb_h.ccb_state = ADA_CCB_RAHEAD;
2502                 } else {
2503                         ata_28bit_cmd(ataio, ATA_SETFEATURES, ADA_WC ?
2504                             ATA_SF_ENAB_WCACHE : ATA_SF_DIS_WCACHE, 0, 0);
2505                         start_ccb->ccb_h.ccb_state = ADA_CCB_WCACHE;
2506                 }
2507                 start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
2508                 xpt_action(start_ccb);
2509                 break;
2510         }
2511         case ADA_STATE_LOGDIR:
2512         {
2513                 struct ata_gp_log_dir *log_dir;
2514
2515                 if ((softc->flags & ADA_FLAG_CAN_LOG) == 0) {
2516                         adaprobedone(periph, start_ccb);
2517                         break;
2518                 }
2519
2520                 log_dir = malloc(sizeof(*log_dir), M_ATADA, M_NOWAIT|M_ZERO);
2521                 if (log_dir == NULL) {
2522                         xpt_print(periph->path, "Couldn't malloc log_dir "
2523                             "data\n");
2524                         softc->state = ADA_STATE_NORMAL;
2525                         xpt_release_ccb(start_ccb);
2526                         break;
2527                 }
2528
2529
2530                 ata_read_log(ataio,
2531                     /*retries*/1,
2532                     /*cbfcnp*/adadone,
2533                     /*log_address*/ ATA_LOG_DIRECTORY,
2534                     /*page_number*/ 0,
2535                     /*block_count*/ 1,
2536                     /*protocol*/ softc->flags & ADA_FLAG_CAN_DMA ?
2537                                  CAM_ATAIO_DMA : 0,
2538                     /*data_ptr*/ (uint8_t *)log_dir,
2539                     /*dxfer_len*/sizeof(*log_dir),
2540                     /*timeout*/ada_default_timeout*1000);
2541
2542                 start_ccb->ccb_h.ccb_state = ADA_CCB_LOGDIR;
2543                 xpt_action(start_ccb);
2544                 break;
2545         }
2546         case ADA_STATE_IDDIR:
2547         {
2548                 struct ata_identify_log_pages *id_dir;
2549
2550                 id_dir = malloc(sizeof(*id_dir), M_ATADA, M_NOWAIT | M_ZERO);
2551                 if (id_dir == NULL) {
2552                         xpt_print(periph->path, "Couldn't malloc id_dir "
2553                             "data\n");
2554                         adaprobedone(periph, start_ccb);
2555                         break;
2556                 }
2557
2558                 ata_read_log(ataio,
2559                     /*retries*/1,
2560                     /*cbfcnp*/adadone,
2561                     /*log_address*/ ATA_IDENTIFY_DATA_LOG,
2562                     /*page_number*/ ATA_IDL_PAGE_LIST,
2563                     /*block_count*/ 1,
2564                     /*protocol*/ softc->flags & ADA_FLAG_CAN_DMA ?
2565                                  CAM_ATAIO_DMA : 0,
2566                     /*data_ptr*/ (uint8_t *)id_dir,
2567                     /*dxfer_len*/ sizeof(*id_dir),
2568                     /*timeout*/ada_default_timeout*1000);
2569
2570                 start_ccb->ccb_h.ccb_state = ADA_CCB_IDDIR;
2571                 xpt_action(start_ccb);
2572                 break;
2573         }
2574         case ADA_STATE_SUP_CAP:
2575         {
2576                 struct ata_identify_log_sup_cap *sup_cap;
2577
2578                 sup_cap = malloc(sizeof(*sup_cap), M_ATADA, M_NOWAIT|M_ZERO);
2579                 if (sup_cap == NULL) {
2580                         xpt_print(periph->path, "Couldn't malloc sup_cap "
2581                             "data\n");
2582                         adaprobedone(periph, start_ccb);
2583                         break;
2584                 }
2585
2586                 ata_read_log(ataio,
2587                     /*retries*/1,
2588                     /*cbfcnp*/adadone,
2589                     /*log_address*/ ATA_IDENTIFY_DATA_LOG,
2590                     /*page_number*/ ATA_IDL_SUP_CAP,
2591                     /*block_count*/ 1,
2592                     /*protocol*/ softc->flags & ADA_FLAG_CAN_DMA ?
2593                                  CAM_ATAIO_DMA : 0,
2594                     /*data_ptr*/ (uint8_t *)sup_cap,
2595                     /*dxfer_len*/ sizeof(*sup_cap),
2596                     /*timeout*/ada_default_timeout*1000);
2597
2598                 start_ccb->ccb_h.ccb_state = ADA_CCB_SUP_CAP;
2599                 xpt_action(start_ccb);
2600                 break;
2601         }
2602         case ADA_STATE_ZONE:
2603         {
2604                 struct ata_zoned_info_log *ata_zone;
2605
2606                 ata_zone = malloc(sizeof(*ata_zone), M_ATADA, M_NOWAIT|M_ZERO);
2607                 if (ata_zone == NULL) {
2608                         xpt_print(periph->path, "Couldn't malloc ata_zone "
2609                             "data\n");
2610                         adaprobedone(periph, start_ccb);
2611                         break;
2612                 }
2613
2614                 ata_read_log(ataio,
2615                     /*retries*/1,
2616                     /*cbfcnp*/adadone,
2617                     /*log_address*/ ATA_IDENTIFY_DATA_LOG,
2618                     /*page_number*/ ATA_IDL_ZDI,
2619                     /*block_count*/ 1,
2620                     /*protocol*/ softc->flags & ADA_FLAG_CAN_DMA ?
2621                                  CAM_ATAIO_DMA : 0,
2622                     /*data_ptr*/ (uint8_t *)ata_zone,
2623                     /*dxfer_len*/ sizeof(*ata_zone),
2624                     /*timeout*/ada_default_timeout*1000);
2625
2626                 start_ccb->ccb_h.ccb_state = ADA_CCB_ZONE;
2627                 xpt_action(start_ccb);
2628                 break;
2629         }
2630         }
2631 }
2632
2633 static void
2634 adaprobedone(struct cam_periph *periph, union ccb *ccb)
2635 {
2636         struct ada_softc *softc;
2637
2638         softc = (struct ada_softc *)periph->softc;
2639
2640         if (ccb != NULL)
2641                 xpt_release_ccb(ccb);
2642
2643         softc->state = ADA_STATE_NORMAL;
2644         softc->flags |= ADA_FLAG_PROBED;
2645         adaschedule(periph);
2646         if ((softc->flags & ADA_FLAG_ANNOUNCED) == 0) {
2647                 softc->flags |= ADA_FLAG_ANNOUNCED;
2648                 cam_periph_unhold(periph);
2649         } else {
2650                 cam_periph_release_locked(periph);
2651         }
2652 }
2653
2654 static void
2655 adazonedone(struct cam_periph *periph, union ccb *ccb)
2656 {
2657         struct bio *bp;
2658
2659         bp = (struct bio *)ccb->ccb_h.ccb_bp;
2660
2661         switch (bp->bio_zone.zone_cmd) {
2662         case DISK_ZONE_OPEN:
2663         case DISK_ZONE_CLOSE:
2664         case DISK_ZONE_FINISH:
2665         case DISK_ZONE_RWP:
2666                 break;
2667         case DISK_ZONE_REPORT_ZONES: {
2668                 uint32_t avail_len;
2669                 struct disk_zone_report *rep;
2670                 struct scsi_report_zones_hdr *hdr;
2671                 struct scsi_report_zones_desc *desc;
2672                 struct disk_zone_rep_entry *entry;
2673                 uint32_t hdr_len, num_avail;
2674                 uint32_t num_to_fill, i;
2675
2676                 rep = &bp->bio_zone.zone_params.report;
2677                 avail_len = ccb->ataio.dxfer_len - ccb->ataio.resid;
2678                 /*
2679                  * Note that bio_resid isn't normally used for zone
2680                  * commands, but it is used by devstat_end_transaction_bio()
2681                  * to determine how much data was transferred.  Because
2682                  * the size of the SCSI/ATA data structures is different
2683                  * than the size of the BIO interface structures, the
2684                  * amount of data actually transferred from the drive will
2685                  * be different than the amount of data transferred to
2686                  * the user.
2687                  */
2688                 hdr = (struct scsi_report_zones_hdr *)ccb->ataio.data_ptr;
2689                 if (avail_len < sizeof(*hdr)) {
2690                         /*
2691                          * Is there a better error than EIO here?  We asked
2692                          * for at least the header, and we got less than
2693                          * that.
2694                          */
2695                         bp->bio_error = EIO;
2696                         bp->bio_flags |= BIO_ERROR;
2697                         bp->bio_resid = bp->bio_bcount;
2698                         break;
2699                 }
2700
2701                 hdr_len = le32dec(hdr->length);
2702                 if (hdr_len > 0)
2703                         rep->entries_available = hdr_len / sizeof(*desc);
2704                 else
2705                         rep->entries_available = 0;
2706                 /*
2707                  * NOTE: using the same values for the BIO version of the
2708                  * same field as the SCSI/ATA values.  This means we could
2709                  * get some additional values that aren't defined in bio.h
2710                  * if more values of the same field are defined later.
2711                  */
2712                 rep->header.same = hdr->byte4 & SRZ_SAME_MASK;
2713                 rep->header.maximum_lba = le64dec(hdr->maximum_lba);
2714                 /*
2715                  * If the drive reports no entries that match the query,
2716                  * we're done.
2717                  */
2718                 if (hdr_len == 0) {
2719                         rep->entries_filled = 0;
2720                         bp->bio_resid = bp->bio_bcount;
2721                         break;
2722                 }
2723
2724                 num_avail = min((avail_len - sizeof(*hdr)) / sizeof(*desc),
2725                                 hdr_len / sizeof(*desc));
2726                 /*
2727                  * If the drive didn't return any data, then we're done.
2728                  */
2729                 if (num_avail == 0) {
2730                         rep->entries_filled = 0;
2731                         bp->bio_resid = bp->bio_bcount;
2732                         break;
2733                 }
2734
2735                 num_to_fill = min(num_avail, rep->entries_allocated);
2736                 /*
2737                  * If the user didn't allocate any entries for us to fill,
2738                  * we're done.
2739                  */
2740                 if (num_to_fill == 0) {
2741                         rep->entries_filled = 0;
2742                         bp->bio_resid = bp->bio_bcount;
2743                         break;
2744                 }
2745
2746                 for (i = 0, desc = &hdr->desc_list[0], entry=&rep->entries[0];
2747                      i < num_to_fill; i++, desc++, entry++) {
2748                         /*
2749                          * NOTE: we're mapping the values here directly
2750                          * from the SCSI/ATA bit definitions to the bio.h
2751                          * definitions.  There is also a warning in
2752                          * disk_zone.h, but the impact is that if
2753                          * additional values are added in the SCSI/ATA
2754                          * specs these will be visible to consumers of
2755                          * this interface.
2756                          */
2757                         entry->zone_type = desc->zone_type & SRZ_TYPE_MASK;
2758                         entry->zone_condition =
2759                             (desc->zone_flags & SRZ_ZONE_COND_MASK) >>
2760                             SRZ_ZONE_COND_SHIFT;
2761                         entry->zone_flags |= desc->zone_flags &
2762                             (SRZ_ZONE_NON_SEQ|SRZ_ZONE_RESET);
2763                         entry->zone_length = le64dec(desc->zone_length);
2764                         entry->zone_start_lba = le64dec(desc->zone_start_lba);
2765                         entry->write_pointer_lba =
2766                             le64dec(desc->write_pointer_lba);
2767                 }
2768                 rep->entries_filled = num_to_fill;
2769                 /*
2770                  * Note that this residual is accurate from the user's
2771                  * standpoint, but the amount transferred isn't accurate
2772                  * from the standpoint of what actually came back from the
2773                  * drive.
2774                  */
2775                 bp->bio_resid = bp->bio_bcount - (num_to_fill * sizeof(*entry));
2776                 break;
2777         }
2778         case DISK_ZONE_GET_PARAMS:
2779         default:
2780                 /*
2781                  * In theory we should not get a GET_PARAMS bio, since it
2782                  * should be handled without queueing the command to the
2783                  * drive.
2784                  */
2785                 panic("%s: Invalid zone command %d", __func__,
2786                     bp->bio_zone.zone_cmd);
2787                 break;
2788         }
2789
2790         if (bp->bio_zone.zone_cmd == DISK_ZONE_REPORT_ZONES)
2791                 free(ccb->ataio.data_ptr, M_ATADA);
2792 }
2793
2794
2795 static void
2796 adadone(struct cam_periph *periph, union ccb *done_ccb)
2797 {
2798         struct ada_softc *softc;
2799         struct ccb_ataio *ataio;
2800         struct cam_path *path;
2801         uint32_t priority;
2802         int state;
2803
2804         softc = (struct ada_softc *)periph->softc;
2805         ataio = &done_ccb->ataio;
2806         path = done_ccb->ccb_h.path;
2807         priority = done_ccb->ccb_h.pinfo.priority;
2808
2809         CAM_DEBUG(path, CAM_DEBUG_TRACE, ("adadone\n"));
2810
2811         state = ataio->ccb_h.ccb_state & ADA_CCB_TYPE_MASK;
2812         switch (state) {
2813         case ADA_CCB_BUFFER_IO:
2814         case ADA_CCB_TRIM:
2815         {
2816                 struct bio *bp;
2817                 int error;
2818
2819                 cam_periph_lock(periph);
2820                 bp = (struct bio *)done_ccb->ccb_h.ccb_bp;
2821                 if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
2822                         error = adaerror(done_ccb, 0, 0);
2823                         if (error == ERESTART) {
2824                                 /* A retry was scheduled, so just return. */
2825                                 cam_periph_unlock(periph);
2826                                 return;
2827                         }
2828                         if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0)
2829                                 cam_release_devq(path,
2830                                                  /*relsim_flags*/0,
2831                                                  /*reduction*/0,
2832                                                  /*timeout*/0,
2833                                                  /*getcount_only*/0);
2834                         /*
2835                          * If we get an error on an NCQ DSM TRIM, fall back
2836                          * to a non-NCQ DSM TRIM forever. Please note that if
2837                          * CAN_NCQ_TRIM is set, CAN_TRIM is necessarily set too.
2838                          * However, for this one trim, we treat it as advisory
2839                          * and return success up the stack.
2840                          */
2841                         if (state == ADA_CCB_TRIM &&
2842                             error != 0 &&
2843                             (softc->flags & ADA_FLAG_CAN_NCQ_TRIM) != 0) {
2844                                 softc->flags &= ~ADA_FLAG_CAN_NCQ_TRIM;
2845                                 error = 0;
2846                                 adasetdeletemethod(softc);
2847                         }
2848                 } else {
2849                         if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0)
2850                                 panic("REQ_CMP with QFRZN");
2851
2852                         error = 0;
2853                 }
2854                 bp->bio_error = error;
2855                 if (error != 0) {
2856                         bp->bio_resid = bp->bio_bcount;
2857                         bp->bio_flags |= BIO_ERROR;
2858                 } else {
2859                         if (bp->bio_cmd == BIO_ZONE)
2860                                 adazonedone(periph, done_ccb);
2861                         else if (state == ADA_CCB_TRIM)
2862                                 bp->bio_resid = 0;
2863                         else
2864                                 bp->bio_resid = ataio->resid;
2865
2866                         if ((bp->bio_resid > 0)
2867                          && (bp->bio_cmd != BIO_ZONE))
2868                                 bp->bio_flags |= BIO_ERROR;
2869                 }
2870                 softc->outstanding_cmds--;
2871                 if (softc->outstanding_cmds == 0)
2872                         softc->flags |= ADA_FLAG_WAS_OTAG;
2873
2874                 /*
2875                  * We need to call cam_iosched before we call biodone so that we
2876                  * don't measure any activity that happens in the completion
2877                  * routine, which in the case of sendfile can be quite
2878                  * extensive.  Release the periph refcount taken in adastart()
2879                  * for each CCB.
2880                  */
2881                 cam_iosched_bio_complete(softc->cam_iosched, bp, done_ccb);
2882                 xpt_release_ccb(done_ccb);
2883                 KASSERT(softc->refcount >= 1, ("adadone softc %p refcount %d", softc, softc->refcount));
2884                 softc->refcount--;
2885                 if (state == ADA_CCB_TRIM) {
2886                         TAILQ_HEAD(, bio) queue;
2887                         struct bio *bp1;
2888
2889                         TAILQ_INIT(&queue);
2890                         TAILQ_CONCAT(&queue, &softc->trim_req.bps, bio_queue);
2891                         /*
2892                          * Normally, the xpt_release_ccb() above would make sure
2893                          * that when we have more work to do, that work would
2894                          * get kicked off. However, we specifically keep
2895                          * trim_running set to 0 before the call above to allow
2896                          * other I/O to progress when many BIO_DELETE requests
2897                          * are pushed down. We set trim_running to 0 and call
2898                          * daschedule again so that we don't stall if there are
2899                          * no other I/Os pending apart from BIO_DELETEs.
2900                          */
2901                         cam_iosched_trim_done(softc->cam_iosched);
2902                         adaschedule(periph);
2903                         cam_periph_unlock(periph);
2904                         while ((bp1 = TAILQ_FIRST(&queue)) != NULL) {
2905                                 TAILQ_REMOVE(&queue, bp1, bio_queue);
2906                                 bp1->bio_error = error;
2907                                 if (error != 0) {
2908                                         bp1->bio_flags |= BIO_ERROR;
2909                                         bp1->bio_resid = bp1->bio_bcount;
2910                                 } else
2911                                         bp1->bio_resid = 0;
2912                                 biodone(bp1);
2913                         }
2914                 } else {
2915                         adaschedule(periph);
2916                         cam_periph_unlock(periph);
2917                         biodone(bp);
2918                 }
2919                 return;
2920         }
2921         case ADA_CCB_RAHEAD:
2922         {
2923                 if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
2924                         if (adaerror(done_ccb, 0, 0) == ERESTART) {
2925                                 /* Drop freeze taken due to CAM_DEV_QFREEZE */
2926                                 cam_release_devq(path, 0, 0, 0, FALSE);
2927                                 return;
2928                         } else if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
2929                                 cam_release_devq(path,
2930                                     /*relsim_flags*/0,
2931                                     /*reduction*/0,
2932                                     /*timeout*/0,
2933                                     /*getcount_only*/0);
2934                         }
2935                 }
2936
2937                 /*
2938                  * Since our peripheral may be invalidated by an error
2939                  * above or an external event, we must release our CCB
2940                  * before releasing the reference on the peripheral.
2941                  * The peripheral will only go away once the last reference
2942                  * is removed, and we need it around for the CCB release
2943                  * operation.
2944                  */
2945
2946                 xpt_release_ccb(done_ccb);
2947                 softc->state = ADA_STATE_WCACHE;
2948                 xpt_schedule(periph, priority);
2949                 /* Drop freeze taken due to CAM_DEV_QFREEZE */
2950                 cam_release_devq(path, 0, 0, 0, FALSE);
2951                 return;
2952         }
2953         case ADA_CCB_WCACHE:
2954         {
2955                 if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
2956                         if (adaerror(done_ccb, 0, 0) == ERESTART) {
2957                                 /* Drop freeze taken due to CAM_DEV_QFREEZE */
2958                                 cam_release_devq(path, 0, 0, 0, FALSE);
2959                                 return;
2960                         } else if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
2961                                 cam_release_devq(path,
2962                                     /*relsim_flags*/0,
2963                                     /*reduction*/0,
2964                                     /*timeout*/0,
2965                                     /*getcount_only*/0);
2966                         }
2967                 }
2968
2969                 /* Drop freeze taken due to CAM_DEV_QFREEZE */
2970                 cam_release_devq(path, 0, 0, 0, FALSE);
2971
2972                 if ((softc->flags & ADA_FLAG_CAN_LOG)
2973                  && (softc->zone_mode != ADA_ZONE_NONE)) {
2974                         xpt_release_ccb(done_ccb);
2975                         softc->state = ADA_STATE_LOGDIR;
2976                         xpt_schedule(periph, priority);
2977                 } else {
2978                         adaprobedone(periph, done_ccb);
2979                 }
2980                 return;
2981         }
2982         case ADA_CCB_LOGDIR:
2983         {
2984                 int error;
2985
2986                 if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
2987                         error = 0;
2988                         softc->valid_logdir_len = 0;
2989                         bzero(&softc->ata_logdir, sizeof(softc->ata_logdir));
2990                         softc->valid_logdir_len =
2991                                 ataio->dxfer_len - ataio->resid;
2992                         if (softc->valid_logdir_len > 0)
2993                                 bcopy(ataio->data_ptr, &softc->ata_logdir,
2994                                     min(softc->valid_logdir_len,
2995                                         sizeof(softc->ata_logdir)));
2996                         /*
2997                          * Figure out whether the Identify Device log is
2998                          * supported.  The General Purpose log directory
2999                          * has a header, and lists the number of pages
3000                          * available for each GP log identified by the
3001                          * offset into the list.
3002                          */
3003                         if ((softc->valid_logdir_len >=
3004                             ((ATA_IDENTIFY_DATA_LOG + 1) * sizeof(uint16_t)))
3005                          && (le16dec(softc->ata_logdir.header) ==
3006                              ATA_GP_LOG_DIR_VERSION)
3007                          && (le16dec(&softc->ata_logdir.num_pages[
3008                              (ATA_IDENTIFY_DATA_LOG *
3009                              sizeof(uint16_t)) - sizeof(uint16_t)]) > 0)){
3010                                 softc->flags |= ADA_FLAG_CAN_IDLOG;
3011                         } else {
3012                                 softc->flags &= ~ADA_FLAG_CAN_IDLOG;
3013                         }
3014                 } else {
3015                         error = adaerror(done_ccb, CAM_RETRY_SELTO,
3016                                          SF_RETRY_UA|SF_NO_PRINT);
3017                         if (error == ERESTART)
3018                                 return;
3019                         else if (error != 0) {
3020                                 /*
3021                                  * If we can't get the ATA log directory,
3022                                  * then ATA logs are effectively not
3023                                  * supported even if the bit is set in the
3024                                  * identify data.
3025                                  */
3026                                 softc->flags &= ~(ADA_FLAG_CAN_LOG |
3027                                                   ADA_FLAG_CAN_IDLOG);
3028                                 if ((done_ccb->ccb_h.status &
3029                                      CAM_DEV_QFRZN) != 0) {
3030                                         /* Don't wedge this device's queue */
3031                                         cam_release_devq(done_ccb->ccb_h.path,
3032                                                          /*relsim_flags*/0,
3033                                                          /*reduction*/0,
3034                                                          /*timeout*/0,
3035                                                          /*getcount_only*/0);
3036                                 }
3037                         }
3038
3039
3040                 }
3041
3042                 free(ataio->data_ptr, M_ATADA);
3043
3044                 if ((error == 0)
3045                  && (softc->flags & ADA_FLAG_CAN_IDLOG)) {
3046                         softc->state = ADA_STATE_IDDIR;
3047                         xpt_release_ccb(done_ccb);
3048                         xpt_schedule(periph, priority);
3049                 } else
3050                         adaprobedone(periph, done_ccb);
3051
3052                 return;
3053         }
3054         case ADA_CCB_IDDIR: {
3055                 int error;
3056
3057                 if ((ataio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
3058                         off_t entries_offset, max_entries;
3059                         error = 0;
3060
3061                         softc->valid_iddir_len = 0;
3062                         bzero(&softc->ata_iddir, sizeof(softc->ata_iddir));
3063                         softc->flags &= ~(ADA_FLAG_CAN_SUPCAP |
3064                                           ADA_FLAG_CAN_ZONE);
3065                         softc->valid_iddir_len =
3066                                 ataio->dxfer_len - ataio->resid;
3067                         if (softc->valid_iddir_len > 0)
3068                                 bcopy(ataio->data_ptr, &softc->ata_iddir,
3069                                     min(softc->valid_iddir_len,
3070                                         sizeof(softc->ata_iddir)));
3071
3072                         entries_offset =
3073                             __offsetof(struct ata_identify_log_pages,entries);
3074                         max_entries = softc->valid_iddir_len - entries_offset;
3075                         if ((softc->valid_iddir_len > (entries_offset + 1))
3076                          && (le64dec(softc->ata_iddir.header) ==
3077                              ATA_IDLOG_REVISION)
3078                          && (softc->ata_iddir.entry_count > 0)) {
3079                                 int num_entries, i;
3080
3081                                 num_entries = softc->ata_iddir.entry_count;
3082                                 num_entries = min(num_entries,
3083                                    softc->valid_iddir_len - entries_offset);
3084                                 for (i = 0; i < num_entries &&
3085                                      i < max_entries; i++) {
3086                                         if (softc->ata_iddir.entries[i] ==
3087                                             ATA_IDL_SUP_CAP)
3088                                                 softc->flags |=
3089                                                     ADA_FLAG_CAN_SUPCAP;
3090                                         else if (softc->ata_iddir.entries[i]==
3091                                                  ATA_IDL_ZDI)
3092                                                 softc->flags |=
3093                                                     ADA_FLAG_CAN_ZONE;
3094
3095                                         if ((softc->flags &
3096                                              ADA_FLAG_CAN_SUPCAP)
3097                                          && (softc->flags &
3098                                              ADA_FLAG_CAN_ZONE))
3099                                                 break;
3100                                 }
3101                         }
3102                 } else {
3103                         error = adaerror(done_ccb, CAM_RETRY_SELTO,
3104                                          SF_RETRY_UA|SF_NO_PRINT);
3105                         if (error == ERESTART)
3106                                 return;
3107                         else if (error != 0) {
3108                                 /*
3109                                  * If we can't get the ATA Identify Data log
3110                                  * directory, then it effectively isn't
3111                                  * supported even if the ATA Log directory
3112                                  * a non-zero number of pages present for
3113                                  * this log.
3114                                  */
3115                                 softc->flags &= ~ADA_FLAG_CAN_IDLOG;
3116                                 if ((done_ccb->ccb_h.status &
3117                                      CAM_DEV_QFRZN) != 0) {
3118                                         /* Don't wedge this device's queue */
3119                                         cam_release_devq(done_ccb->ccb_h.path,
3120                                                          /*relsim_flags*/0,
3121                                                          /*reduction*/0,
3122                                                          /*timeout*/0,
3123                                                          /*getcount_only*/0);
3124                                 }
3125                         }
3126                 }
3127
3128                 free(ataio->data_ptr, M_ATADA);
3129
3130                 if ((error == 0)
3131                  && (softc->flags & ADA_FLAG_CAN_SUPCAP)) {
3132                         softc->state = ADA_STATE_SUP_CAP;
3133                         xpt_release_ccb(done_ccb);
3134                         xpt_schedule(periph, priority);
3135                 } else
3136                         adaprobedone(periph, done_ccb);
3137                 return;
3138         }
3139         case ADA_CCB_SUP_CAP: {
3140                 int error;
3141
3142                 if ((ataio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
3143                         uint32_t valid_len;
3144                         size_t needed_size;
3145                         struct ata_identify_log_sup_cap *sup_cap;
3146                         error = 0;
3147
3148                         sup_cap = (struct ata_identify_log_sup_cap *)
3149                             ataio->data_ptr;
3150                         valid_len = ataio->dxfer_len - ataio->resid;
3151                         needed_size =
3152                             __offsetof(struct ata_identify_log_sup_cap,
3153                             sup_zac_cap) + 1 + sizeof(sup_cap->sup_zac_cap);
3154                         if (valid_len >= needed_size) {
3155                                 uint64_t zoned, zac_cap;
3156
3157                                 zoned = le64dec(sup_cap->zoned_cap);
3158                                 if (zoned & ATA_ZONED_VALID) {
3159                                         /*
3160                                          * This should have already been
3161                                          * set, because this is also in the
3162                                          * ATA identify data.
3163                                          */
3164                                         if ((zoned & ATA_ZONED_MASK) ==
3165                                             ATA_SUPPORT_ZONE_HOST_AWARE)
3166                                                 softc->zone_mode =
3167                                                     ADA_ZONE_HOST_AWARE;
3168                                         else if ((zoned & ATA_ZONED_MASK) ==
3169                                             ATA_SUPPORT_ZONE_DEV_MANAGED)
3170                                                 softc->zone_mode =
3171                                                     ADA_ZONE_DRIVE_MANAGED;
3172                                 }
3173
3174                                 zac_cap = le64dec(sup_cap->sup_zac_cap);
3175                                 if (zac_cap & ATA_SUP_ZAC_CAP_VALID) {
3176                                         if (zac_cap & ATA_REPORT_ZONES_SUP)
3177                                                 softc->zone_flags |=
3178                                                     ADA_ZONE_FLAG_RZ_SUP;
3179                                         if (zac_cap & ATA_ND_OPEN_ZONE_SUP)
3180                                                 softc->zone_flags |=
3181                                                     ADA_ZONE_FLAG_OPEN_SUP;
3182                                         if (zac_cap & ATA_ND_CLOSE_ZONE_SUP)
3183                                                 softc->zone_flags |=
3184                                                     ADA_ZONE_FLAG_CLOSE_SUP;
3185                                         if (zac_cap & ATA_ND_FINISH_ZONE_SUP)
3186                                                 softc->zone_flags |=
3187                                                     ADA_ZONE_FLAG_FINISH_SUP;
3188                                         if (zac_cap & ATA_ND_RWP_SUP)
3189                                                 softc->zone_flags |=
3190                                                     ADA_ZONE_FLAG_RWP_SUP;
3191                                 } else {
3192                                         /*
3193                                          * This field was introduced in
3194                                          * ACS-4, r08 on April 28th, 2015.
3195                                          * If the drive firmware was written
3196                                          * to an earlier spec, it won't have
3197                                          * the field.  So, assume all
3198                                          * commands are supported.
3199                                          */
3200                                         softc->zone_flags |=
3201                                             ADA_ZONE_FLAG_SUP_MASK;
3202                                 }
3203
3204                         }
3205                 } else {
3206                         error = adaerror(done_ccb, CAM_RETRY_SELTO,
3207                                          SF_RETRY_UA|SF_NO_PRINT);
3208                         if (error == ERESTART)
3209                                 return;
3210                         else if (error != 0) {
3211                                 /*
3212                                  * If we can't get the ATA Identify Data
3213                                  * Supported Capabilities page, clear the
3214                                  * flag...
3215                                  */
3216                                 softc->flags &= ~ADA_FLAG_CAN_SUPCAP;
3217                                 /*
3218                                  * And clear zone capabilities.
3219                                  */
3220                                 softc->zone_flags &= ~ADA_ZONE_FLAG_SUP_MASK;
3221                                 if ((done_ccb->ccb_h.status &
3222                                      CAM_DEV_QFRZN) != 0) {
3223                                         /* Don't wedge this device's queue */
3224                                         cam_release_devq(done_ccb->ccb_h.path,
3225                                                          /*relsim_flags*/0,
3226                                                          /*reduction*/0,
3227                                                          /*timeout*/0,
3228                                                          /*getcount_only*/0);
3229                                 }
3230                         }
3231                 }
3232
3233                 free(ataio->data_ptr, M_ATADA);
3234
3235                 if ((error == 0)
3236                  && (softc->flags & ADA_FLAG_CAN_ZONE)) {
3237                         softc->state = ADA_STATE_ZONE;
3238                         xpt_release_ccb(done_ccb);
3239                         xpt_schedule(periph, priority);
3240                 } else
3241                         adaprobedone(periph, done_ccb);
3242                 return;
3243         }
3244         case ADA_CCB_ZONE: {
3245                 int error;
3246
3247                 if ((ataio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
3248                         struct ata_zoned_info_log *zi_log;
3249                         uint32_t valid_len;
3250                         size_t needed_size;
3251
3252                         zi_log = (struct ata_zoned_info_log *)ataio->data_ptr;
3253
3254                         valid_len = ataio->dxfer_len - ataio->resid;
3255                         needed_size = __offsetof(struct ata_zoned_info_log,
3256                             version_info) + 1 + sizeof(zi_log->version_info);
3257                         if (valid_len >= needed_size) {
3258                                 uint64_t tmpvar;
3259
3260                                 tmpvar = le64dec(zi_log->zoned_cap);
3261                                 if (tmpvar & ATA_ZDI_CAP_VALID) {
3262                                         if (tmpvar & ATA_ZDI_CAP_URSWRZ)
3263                                                 softc->zone_flags |=
3264                                                     ADA_ZONE_FLAG_URSWRZ;
3265                                         else
3266                                                 softc->zone_flags &=
3267                                                     ~ADA_ZONE_FLAG_URSWRZ;
3268                                 }
3269                                 tmpvar = le64dec(zi_log->optimal_seq_zones);
3270                                 if (tmpvar & ATA_ZDI_OPT_SEQ_VALID) {
3271                                         softc->zone_flags |=
3272                                             ADA_ZONE_FLAG_OPT_SEQ_SET;
3273                                         softc->optimal_seq_zones = (tmpvar &
3274                                             ATA_ZDI_OPT_SEQ_MASK);
3275                                 } else {
3276                                         softc->zone_flags &=
3277                                             ~ADA_ZONE_FLAG_OPT_SEQ_SET;
3278                                         softc->optimal_seq_zones = 0;
3279                                 }
3280
3281                                 tmpvar =le64dec(zi_log->optimal_nonseq_zones);
3282                                 if (tmpvar & ATA_ZDI_OPT_NS_VALID) {
3283                                         softc->zone_flags |=
3284                                             ADA_ZONE_FLAG_OPT_NONSEQ_SET;
3285                                         softc->optimal_nonseq_zones =
3286                                             (tmpvar & ATA_ZDI_OPT_NS_MASK);
3287                                 } else {
3288                                         softc->zone_flags &=
3289                                             ~ADA_ZONE_FLAG_OPT_NONSEQ_SET;
3290                                         softc->optimal_nonseq_zones = 0;
3291                                 }
3292
3293                                 tmpvar = le64dec(zi_log->max_seq_req_zones);
3294                                 if (tmpvar & ATA_ZDI_MAX_SEQ_VALID) {
3295                                         softc->zone_flags |=
3296                                             ADA_ZONE_FLAG_MAX_SEQ_SET;
3297                                         softc->max_seq_zones =
3298                                             (tmpvar & ATA_ZDI_MAX_SEQ_MASK);
3299                                 } else {
3300                                         softc->zone_flags &=
3301                                             ~ADA_ZONE_FLAG_MAX_SEQ_SET;
3302                                         softc->max_seq_zones = 0;
3303                                 }
3304                         }
3305                 } else {
3306                         error = adaerror(done_ccb, CAM_RETRY_SELTO,
3307                                          SF_RETRY_UA|SF_NO_PRINT);
3308                         if (error == ERESTART)
3309                                 return;
3310                         else if (error != 0) {
3311                                 softc->flags &= ~ADA_FLAG_CAN_ZONE;
3312                                 softc->flags &= ~ADA_ZONE_FLAG_SET_MASK;
3313
3314                                 if ((done_ccb->ccb_h.status &
3315                                      CAM_DEV_QFRZN) != 0) {
3316                                         /* Don't wedge this device's queue */
3317                                         cam_release_devq(done_ccb->ccb_h.path,
3318                                                          /*relsim_flags*/0,
3319                                                          /*reduction*/0,
3320                                                          /*timeout*/0,
3321                                                          /*getcount_only*/0);
3322                                 }
3323                         }
3324
3325                 }
3326                 free(ataio->data_ptr, M_ATADA);
3327
3328                 adaprobedone(periph, done_ccb);
3329                 return;
3330         }
3331         case ADA_CCB_DUMP:
3332                 /* No-op.  We're polling */
3333                 return;
3334         default:
3335                 break;
3336         }
3337         xpt_release_ccb(done_ccb);
3338 }
3339
3340 static int
3341 adaerror(union ccb *ccb, u_int32_t cam_flags, u_int32_t sense_flags)
3342 {
3343 #ifdef CAM_IO_STATS
3344         struct ada_softc *softc;
3345         struct cam_periph *periph;
3346
3347         periph = xpt_path_periph(ccb->ccb_h.path);
3348         softc = (struct ada_softc *)periph->softc;
3349
3350         switch (ccb->ccb_h.status & CAM_STATUS_MASK) {
3351         case CAM_CMD_TIMEOUT:
3352                 softc->timeouts++;
3353                 break;
3354         case CAM_REQ_ABORTED:
3355         case CAM_REQ_CMP_ERR:
3356         case CAM_REQ_TERMIO:
3357         case CAM_UNREC_HBA_ERROR:
3358         case CAM_DATA_RUN_ERR:
3359         case CAM_ATA_STATUS_ERROR:
3360                 softc->errors++;
3361                 break;
3362         default:
3363                 break;
3364         }
3365 #endif
3366
3367         return(cam_periph_error(ccb, cam_flags, sense_flags));
3368 }
3369
3370 static void
3371 adagetparams(struct cam_periph *periph, struct ccb_getdev *cgd)
3372 {
3373         struct ada_softc *softc = (struct ada_softc *)periph->softc;
3374         struct disk_params *dp = &softc->params;
3375         u_int64_t lbasize48;
3376         u_int32_t lbasize;
3377
3378         dp->secsize = ata_logical_sector_size(&cgd->ident_data);
3379         if ((cgd->ident_data.atavalid & ATA_FLAG_54_58) &&
3380                 cgd->ident_data.current_heads && cgd->ident_data.current_sectors) {
3381                 dp->heads = cgd->ident_data.current_heads;
3382                 dp->secs_per_track = cgd->ident_data.current_sectors;
3383                 dp->cylinders = cgd->ident_data.cylinders;
3384                 dp->sectors = (u_int32_t)cgd->ident_data.current_size_1 |
3385                           ((u_int32_t)cgd->ident_data.current_size_2 << 16);
3386         } else {
3387                 dp->heads = cgd->ident_data.heads;
3388                 dp->secs_per_track = cgd->ident_data.sectors;
3389                 dp->cylinders = cgd->ident_data.cylinders;
3390                 dp->sectors = cgd->ident_data.cylinders *
3391                               (u_int32_t)(dp->heads * dp->secs_per_track);
3392         }
3393         lbasize = (u_int32_t)cgd->ident_data.lba_size_1 |
3394                   ((u_int32_t)cgd->ident_data.lba_size_2 << 16);
3395
3396         /* use the 28bit LBA size if valid or bigger than the CHS mapping */
3397         if (cgd->ident_data.cylinders == 16383 || dp->sectors < lbasize)
3398                 dp->sectors = lbasize;
3399
3400         /* use the 48bit LBA size if valid */
3401         lbasize48 = ((u_int64_t)cgd->ident_data.lba_size48_1) |
3402                     ((u_int64_t)cgd->ident_data.lba_size48_2 << 16) |
3403                     ((u_int64_t)cgd->ident_data.lba_size48_3 << 32) |
3404                     ((u_int64_t)cgd->ident_data.lba_size48_4 << 48);
3405         if ((cgd->ident_data.support.command2 & ATA_SUPPORT_ADDRESS48) &&
3406             lbasize48 > ATA_MAX_28BIT_LBA)
3407                 dp->sectors = lbasize48;
3408 }
3409
3410 static void
3411 adasendorderedtag(void *arg)
3412 {
3413         struct ada_softc *softc = arg;
3414
3415         if (ada_send_ordered) {
3416                 if (softc->outstanding_cmds > 0) {
3417                         if ((softc->flags & ADA_FLAG_WAS_OTAG) == 0)
3418                                 softc->flags |= ADA_FLAG_NEED_OTAG;
3419                         softc->flags &= ~ADA_FLAG_WAS_OTAG;
3420                 }
3421         }
3422         /* Queue us up again */
3423         callout_reset(&softc->sendordered_c,
3424             (ada_default_timeout * hz) / ADA_ORDEREDTAG_INTERVAL,
3425             adasendorderedtag, softc);
3426 }
3427
3428 /*
3429  * Step through all ADA peripheral drivers, and if the device is still open,
3430  * sync the disk cache to physical media.
3431  */
3432 static void
3433 adaflush(void)
3434 {
3435         struct cam_periph *periph;
3436         struct ada_softc *softc;
3437         union ccb *ccb;
3438         int error;
3439
3440         CAM_PERIPH_FOREACH(periph, &adadriver) {
3441                 softc = (struct ada_softc *)periph->softc;
3442                 if (SCHEDULER_STOPPED()) {
3443                         /* If we paniced with the lock held, do not recurse. */
3444                         if (!cam_periph_owned(periph) &&
3445                             (softc->flags & ADA_FLAG_OPEN)) {
3446                                 adadump(softc->disk, NULL, 0, 0, 0);
3447                         }
3448                         continue;
3449                 }
3450                 cam_periph_lock(periph);
3451                 /*
3452                  * We only sync the cache if the drive is still open, and
3453                  * if the drive is capable of it..
3454                  */
3455                 if (((softc->flags & ADA_FLAG_OPEN) == 0) ||
3456                     (softc->flags & ADA_FLAG_CAN_FLUSHCACHE) == 0) {
3457                         cam_periph_unlock(periph);
3458                         continue;
3459                 }
3460
3461                 ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL);
3462                 cam_fill_ataio(&ccb->ataio,
3463                                     0,
3464                                     NULL,
3465                                     CAM_DIR_NONE,
3466                                     0,
3467                                     NULL,
3468                                     0,
3469                                     ada_default_timeout*1000);
3470                 if (softc->flags & ADA_FLAG_CAN_48BIT)
3471                         ata_48bit_cmd(&ccb->ataio, ATA_FLUSHCACHE48, 0, 0, 0);
3472                 else
3473                         ata_28bit_cmd(&ccb->ataio, ATA_FLUSHCACHE, 0, 0, 0);
3474
3475                 error = cam_periph_runccb(ccb, adaerror, /*cam_flags*/0,
3476                     /*sense_flags*/ SF_NO_RECOVERY | SF_NO_RETRY,
3477                     softc->disk->d_devstat);
3478                 if (error != 0)
3479                         xpt_print(periph->path, "Synchronize cache failed\n");
3480                 xpt_release_ccb(ccb);
3481                 cam_periph_unlock(periph);
3482         }
3483 }
3484
3485 static void
3486 adaspindown(uint8_t cmd, int flags)
3487 {
3488         struct cam_periph *periph;
3489         struct ada_softc *softc;
3490         struct ccb_ataio local_ccb;
3491         int error;
3492
3493         CAM_PERIPH_FOREACH(periph, &adadriver) {
3494                 /* If we paniced with lock held - not recurse here. */
3495                 if (cam_periph_owned(periph))
3496                         continue;
3497                 cam_periph_lock(periph);
3498                 softc = (struct ada_softc *)periph->softc;
3499                 /*
3500                  * We only spin-down the drive if it is capable of it..
3501                  */
3502                 if ((softc->flags & ADA_FLAG_CAN_POWERMGT) == 0) {
3503                         cam_periph_unlock(periph);
3504                         continue;
3505                 }
3506
3507                 if (bootverbose)
3508                         xpt_print(periph->path, "spin-down\n");
3509
3510                 memset(&local_ccb, 0, sizeof(local_ccb));
3511                 xpt_setup_ccb(&local_ccb.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
3512                 local_ccb.ccb_h.ccb_state = ADA_CCB_DUMP;
3513
3514                 cam_fill_ataio(&local_ccb,
3515                                     0,
3516                                     NULL,
3517                                     CAM_DIR_NONE | flags,
3518                                     0,
3519                                     NULL,
3520                                     0,
3521                                     ada_default_timeout*1000);
3522                 ata_28bit_cmd(&local_ccb, cmd, 0, 0, 0);
3523                 error = cam_periph_runccb((union ccb *)&local_ccb, adaerror,
3524                     /*cam_flags*/0, /*sense_flags*/ SF_NO_RECOVERY | SF_NO_RETRY,
3525                     softc->disk->d_devstat);
3526                 if (error != 0)
3527                         xpt_print(periph->path, "Spin-down disk failed\n");
3528                 cam_periph_unlock(periph);
3529         }
3530 }
3531
3532 static void
3533 adashutdown(void *arg, int howto)
3534 {
3535         int how;
3536
3537         adaflush();
3538
3539         /*
3540          * STANDBY IMMEDIATE saves any volatile data to the drive. It also spins
3541          * down hard drives. IDLE IMMEDIATE also saves the volatile data without
3542          * a spindown. We send the former when we expect to lose power soon. For
3543          * a warm boot, we send the latter to avoid a thundering herd of spinups
3544          * just after the kernel loads while probing. We have to do something to
3545          * flush the data because the BIOS in many systems resets the HBA
3546          * causing a COMINIT/COMRESET negotiation, which some drives interpret
3547          * as license to toss the volatile data, and others count as unclean
3548          * shutdown when in the Active PM state in SMART attributes.
3549          *
3550          * adaspindown will ensure that we don't send this to a drive that
3551          * doesn't support it.
3552          */
3553         if (ada_spindown_shutdown != 0) {
3554                 how = (howto & (RB_HALT | RB_POWEROFF | RB_POWERCYCLE)) ?
3555                     ATA_STANDBY_IMMEDIATE : ATA_IDLE_IMMEDIATE;
3556                 adaspindown(how, 0);
3557         }
3558 }
3559
3560 static void
3561 adasuspend(void *arg)
3562 {
3563
3564         adaflush();
3565         /*
3566          * SLEEP also fushes any volatile data, like STANDBY IMEDIATE,
3567          * so we don't need to send it as well.
3568          */
3569         if (ada_spindown_suspend != 0)
3570                 adaspindown(ATA_SLEEP, CAM_DEV_QFREEZE);
3571 }
3572
3573 static void
3574 adaresume(void *arg)
3575 {
3576         struct cam_periph *periph;
3577         struct ada_softc *softc;
3578
3579         if (ada_spindown_suspend == 0)
3580                 return;
3581
3582         CAM_PERIPH_FOREACH(periph, &adadriver) {
3583                 cam_periph_lock(periph);
3584                 softc = (struct ada_softc *)periph->softc;
3585                 /*
3586                  * We only spin-down the drive if it is capable of it..
3587                  */
3588                 if ((softc->flags & ADA_FLAG_CAN_POWERMGT) == 0) {
3589                         cam_periph_unlock(periph);
3590                         continue;
3591                 }
3592
3593                 if (bootverbose)
3594                         xpt_print(periph->path, "resume\n");
3595
3596                 /*
3597                  * Drop freeze taken due to CAM_DEV_QFREEZE flag set on
3598                  * sleep request.
3599                  */
3600                 cam_release_devq(periph->path,
3601                          /*relsim_flags*/0,
3602                          /*openings*/0,
3603                          /*timeout*/0,
3604                          /*getcount_only*/0);
3605
3606                 cam_periph_unlock(periph);
3607         }
3608 }
3609
3610 #endif /* _KERNEL */