sys/cam/ata/ata_da.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2009 Alexander Motin <mav@FreeBSD.org>
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer,
  12  *    without modification, immediately at the beginning of the file.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  18  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  19  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  20  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  21  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  22  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  23  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  24  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  25  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  26  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27  */
  28
  29 #include <sys/cdefs.h>
  30 __FBSDID("$FreeBSD$");
  31
  32 #include "opt_ada.h"
  33
  34 #include <sys/param.h>
  35
  36 #ifdef _KERNEL
  37 #include <sys/systm.h>
  38 #include <sys/kernel.h>
  39 #include <sys/bio.h>
  40 #include <sys/sysctl.h>
  41 #include <sys/taskqueue.h>
  42 #include <sys/lock.h>
  43 #include <sys/mutex.h>
  44 #include <sys/conf.h>
  45 #include <sys/devicestat.h>
  46 #include <sys/eventhandler.h>
  47 #include <sys/malloc.h>
  48 #include <sys/endian.h>
  49 #include <sys/cons.h>
  50 #include <sys/proc.h>
  51 #include <sys/reboot.h>
  52 #include <sys/sbuf.h>
  53 #include <geom/geom_disk.h>
  54 #endif /* _KERNEL */
  55
  56 #ifndef _KERNEL
  57 #include <stdio.h>
  58 #include <string.h>
  59 #endif /* _KERNEL */
  60
  61 #include <cam/cam.h>
  62 #include <cam/cam_ccb.h>
  63 #include <cam/cam_periph.h>
  64 #include <cam/cam_xpt_periph.h>
  65 #include <cam/scsi/scsi_all.h>
  66 #include <cam/scsi/scsi_da.h>
  67 #include <cam/cam_sim.h>
  68 #include <cam/cam_iosched.h>
  69
  70 #include <cam/ata/ata_all.h>
  71
  72 #include <machine/md_var.h>     /* geometry translation */
  73
  74 #ifdef _KERNEL
  75
  76 #define ATA_MAX_28BIT_LBA               268435455UL
  77
  78 extern int iosched_debug;
  79
  80 typedef enum {
  81         ADA_STATE_RAHEAD,
  82         ADA_STATE_WCACHE,
  83         ADA_STATE_LOGDIR,
  84         ADA_STATE_IDDIR,
  85         ADA_STATE_SUP_CAP,
  86         ADA_STATE_ZONE,
  87         ADA_STATE_NORMAL
  88 } ada_state;
  89
  90 typedef enum {
  91         ADA_FLAG_CAN_48BIT      = 0x00000002,
  92         ADA_FLAG_CAN_FLUSHCACHE = 0x00000004,
  93         ADA_FLAG_CAN_NCQ        = 0x00000008,
  94         ADA_FLAG_CAN_DMA        = 0x00000010,
  95         ADA_FLAG_NEED_OTAG      = 0x00000020,
  96         ADA_FLAG_WAS_OTAG       = 0x00000040,
  97         ADA_FLAG_CAN_TRIM       = 0x00000080,
  98         ADA_FLAG_OPEN           = 0x00000100,
  99         ADA_FLAG_SCTX_INIT      = 0x00000200,
 100         ADA_FLAG_CAN_CFA        = 0x00000400,
 101         ADA_FLAG_CAN_POWERMGT   = 0x00000800,
 102         ADA_FLAG_CAN_DMA48      = 0x00001000,
 103         ADA_FLAG_CAN_LOG        = 0x00002000,
 104         ADA_FLAG_CAN_IDLOG      = 0x00004000,
 105         ADA_FLAG_CAN_SUPCAP     = 0x00008000,
 106         ADA_FLAG_CAN_ZONE       = 0x00010000,
 107         ADA_FLAG_CAN_WCACHE     = 0x00020000,
 108         ADA_FLAG_CAN_RAHEAD     = 0x00040000,
 109         ADA_FLAG_PROBED         = 0x00080000,
 110         ADA_FLAG_ANNOUNCED      = 0x00100000,
 111         ADA_FLAG_DIRTY          = 0x00200000,
 112         ADA_FLAG_CAN_NCQ_TRIM   = 0x00400000,   /* CAN_TRIM also set */
 113         ADA_FLAG_PIM_ATA_EXT    = 0x00800000
 114 } ada_flags;
 115
 116 typedef enum {
 117         ADA_Q_NONE              = 0x00,
 118         ADA_Q_4K                = 0x01,
 119         ADA_Q_NCQ_TRIM_BROKEN   = 0x02,
 120         ADA_Q_LOG_BROKEN        = 0x04,
 121         ADA_Q_SMR_DM            = 0x08,
 122         ADA_Q_NO_TRIM           = 0x10
 123 } ada_quirks;
 124
 125 #define ADA_Q_BIT_STRING        \
 126         "\020"                  \
 127         "\0014K"                \
 128         "\002NCQ_TRIM_BROKEN"   \
 129         "\003LOG_BROKEN"        \
 130         "\004SMR_DM"            \
 131         "\005NO_TRIM"
 132
 133 typedef enum {
 134         ADA_CCB_RAHEAD          = 0x01,
 135         ADA_CCB_WCACHE          = 0x02,
 136         ADA_CCB_BUFFER_IO       = 0x03,
 137         ADA_CCB_DUMP            = 0x05,
 138         ADA_CCB_TRIM            = 0x06,
 139         ADA_CCB_LOGDIR          = 0x07,
 140         ADA_CCB_IDDIR           = 0x08,
 141         ADA_CCB_SUP_CAP         = 0x09,
 142         ADA_CCB_ZONE            = 0x0a,
 143         ADA_CCB_TYPE_MASK       = 0x0F,
 144 } ada_ccb_state;
 145
 146 typedef enum {
 147         ADA_ZONE_NONE           = 0x00,
 148         ADA_ZONE_DRIVE_MANAGED  = 0x01,
 149         ADA_ZONE_HOST_AWARE     = 0x02,
 150         ADA_ZONE_HOST_MANAGED   = 0x03
 151 } ada_zone_mode;
 152
 153 typedef enum {
 154         ADA_ZONE_FLAG_RZ_SUP            = 0x0001,
 155         ADA_ZONE_FLAG_OPEN_SUP          = 0x0002,
 156         ADA_ZONE_FLAG_CLOSE_SUP         = 0x0004,
 157         ADA_ZONE_FLAG_FINISH_SUP        = 0x0008,
 158         ADA_ZONE_FLAG_RWP_SUP           = 0x0010,
 159         ADA_ZONE_FLAG_SUP_MASK          = (ADA_ZONE_FLAG_RZ_SUP |
 160                                            ADA_ZONE_FLAG_OPEN_SUP |
 161                                            ADA_ZONE_FLAG_CLOSE_SUP |
 162                                            ADA_ZONE_FLAG_FINISH_SUP |
 163                                            ADA_ZONE_FLAG_RWP_SUP),
 164         ADA_ZONE_FLAG_URSWRZ            = 0x0020,
 165         ADA_ZONE_FLAG_OPT_SEQ_SET       = 0x0040,
 166         ADA_ZONE_FLAG_OPT_NONSEQ_SET    = 0x0080,
 167         ADA_ZONE_FLAG_MAX_SEQ_SET       = 0x0100,
 168         ADA_ZONE_FLAG_SET_MASK          = (ADA_ZONE_FLAG_OPT_SEQ_SET |
 169                                            ADA_ZONE_FLAG_OPT_NONSEQ_SET |
 170                                            ADA_ZONE_FLAG_MAX_SEQ_SET)
 171 } ada_zone_flags;
 172
 173 static struct ada_zone_desc {
 174         ada_zone_flags value;
 175         const char *desc;
 176 } ada_zone_desc_table[] = {
 177         {ADA_ZONE_FLAG_RZ_SUP, "Report Zones" },
 178         {ADA_ZONE_FLAG_OPEN_SUP, "Open" },
 179         {ADA_ZONE_FLAG_CLOSE_SUP, "Close" },
 180         {ADA_ZONE_FLAG_FINISH_SUP, "Finish" },
 181         {ADA_ZONE_FLAG_RWP_SUP, "Reset Write Pointer" },
 182 };
 183
 184
 185 /* Offsets into our private area for storing information */
 186 #define ccb_state       ppriv_field0
 187 #define ccb_bp          ppriv_ptr1
 188
 189 typedef enum {
 190         ADA_DELETE_NONE,
 191         ADA_DELETE_DISABLE,
 192         ADA_DELETE_CFA_ERASE,
 193         ADA_DELETE_DSM_TRIM,
 194         ADA_DELETE_NCQ_DSM_TRIM,
 195         ADA_DELETE_MIN = ADA_DELETE_CFA_ERASE,
 196         ADA_DELETE_MAX = ADA_DELETE_NCQ_DSM_TRIM,
 197 } ada_delete_methods;
 198
 199 static const char *ada_delete_method_names[] =
 200     { "NONE", "DISABLE", "CFA_ERASE", "DSM_TRIM", "NCQ_DSM_TRIM" };
 201 #if 0
 202 static const char *ada_delete_method_desc[] =
 203     { "NONE", "DISABLED", "CFA Erase", "DSM Trim", "DSM Trim via NCQ" };
 204 #endif
 205
 206 struct disk_params {
 207         u_int8_t  heads;
 208         u_int8_t  secs_per_track;
 209         u_int32_t cylinders;
 210         u_int32_t secsize;      /* Number of bytes/logical sector */
 211         u_int64_t sectors;      /* Total number sectors */
 212 };
 213
 214 #define TRIM_MAX_BLOCKS 8
 215 #define TRIM_MAX_RANGES (TRIM_MAX_BLOCKS * ATA_DSM_BLK_RANGES)
 216 struct trim_request {
 217         uint8_t         data[TRIM_MAX_RANGES * ATA_DSM_RANGE_SIZE];
 218         TAILQ_HEAD(, bio) bps;
 219 };
 220
 221 struct ada_softc {
 222         struct   cam_iosched_softc *cam_iosched;
 223         int      outstanding_cmds;      /* Number of active commands */
 224         int      refcount;              /* Active xpt_action() calls */
 225         ada_state state;
 226         ada_flags flags;
 227         ada_zone_mode zone_mode;
 228         ada_zone_flags zone_flags;
 229         struct ata_gp_log_dir ata_logdir;
 230         int valid_logdir_len;
 231         struct ata_identify_log_pages ata_iddir;
 232         int valid_iddir_len;
 233         uint64_t optimal_seq_zones;
 234         uint64_t optimal_nonseq_zones;
 235         uint64_t max_seq_zones;
 236         ada_quirks quirks;
 237         ada_delete_methods delete_method;
 238         int      trim_max_ranges;
 239         int      read_ahead;
 240         int      write_cache;
 241         int      unmappedio;
 242         int      rotating;
 243 #ifdef CAM_TEST_FAILURE
 244         int      force_read_error;
 245         int      force_write_error;
 246         int      periodic_read_error;
 247         int      periodic_read_count;
 248 #endif
 249         struct   disk_params params;
 250         struct   disk *disk;
 251         struct task             sysctl_task;
 252         struct sysctl_ctx_list  sysctl_ctx;
 253         struct sysctl_oid       *sysctl_tree;
 254         struct callout          sendordered_c;
 255         struct trim_request     trim_req;
 256         uint64_t                trim_count;
 257         uint64_t                trim_ranges;
 258         uint64_t                trim_lbas;
 259 #ifdef CAM_IO_STATS
 260         struct sysctl_ctx_list  sysctl_stats_ctx;
 261         struct sysctl_oid       *sysctl_stats_tree;
 262         u_int   timeouts;
 263         u_int   errors;
 264         u_int   invalidations;
 265 #endif
 266 #define ADA_ANNOUNCETMP_SZ 80
 267         char    announce_temp[ADA_ANNOUNCETMP_SZ];
 268 #define ADA_ANNOUNCE_SZ 400
 269         char    announce_buffer[ADA_ANNOUNCE_SZ];
 270 };
 271
 272 struct ada_quirk_entry {
 273         struct scsi_inquiry_pattern inq_pat;
 274         ada_quirks quirks;
 275 };
 276
 277 static struct ada_quirk_entry ada_quirk_table[] =
 278 {
 279         {
 280                 /* Hitachi Advanced Format (4k) drives */
 281                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Hitachi H??????????E3*", "*" },
 282                 /*quirks*/ADA_Q_4K
 283         },
 284         {
 285                 /* Samsung Advanced Format (4k) drives */
 286                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG HD155UI*", "*" },
 287                 /*quirks*/ADA_Q_4K
 288         },
 289         {
 290                 /* Samsung Advanced Format (4k) drives */
 291                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG HD204UI*", "*" },
 292                 /*quirks*/ADA_Q_4K
 293         },
 294         {
 295                 /* Seagate Barracuda Green Advanced Format (4k) drives */
 296                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST????DL*", "*" },
 297                 /*quirks*/ADA_Q_4K
 298         },
 299         {
 300                 /* Seagate Barracuda Advanced Format (4k) drives */
 301                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST???DM*", "*" },
 302                 /*quirks*/ADA_Q_4K
 303         },
 304         {
 305                 /* Seagate Barracuda Advanced Format (4k) drives */
 306                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST????DM*", "*" },
 307                 /*quirks*/ADA_Q_4K
 308         },
 309         {
 310                 /* Seagate Momentus Advanced Format (4k) drives */
 311                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9500423AS*", "*" },
 312                 /*quirks*/ADA_Q_4K
 313         },
 314         {
 315                 /* Seagate Momentus Advanced Format (4k) drives */
 316                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9500424AS*", "*" },
 317                 /*quirks*/ADA_Q_4K
 318         },
 319         {
 320                 /* Seagate Momentus Advanced Format (4k) drives */
 321                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9640423AS*", "*" },
 322                 /*quirks*/ADA_Q_4K
 323         },
 324         {
 325                 /* Seagate Momentus Advanced Format (4k) drives */
 326                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9640424AS*", "*" },
 327                 /*quirks*/ADA_Q_4K
 328         },
 329         {
 330                 /* Seagate Momentus Advanced Format (4k) drives */
 331                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9750420AS*", "*" },
 332                 /*quirks*/ADA_Q_4K
 333         },
 334         {
 335                 /* Seagate Momentus Advanced Format (4k) drives */
 336                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9750422AS*", "*" },
 337                 /*quirks*/ADA_Q_4K
 338         },
 339         {
 340                 /* Seagate Momentus Advanced Format (4k) drives */
 341                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9750423AS*", "*" },
 342                 /*quirks*/ADA_Q_4K
 343         },
 344         {
 345                 /* Seagate Momentus Thin Advanced Format (4k) drives */
 346                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST???LT*", "*" },
 347                 /*quirks*/ADA_Q_4K
 348         },
 349         {
 350                 /* WDC Caviar Red Advanced Format (4k) drives */
 351                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD????CX*", "*" },
 352                 /*quirks*/ADA_Q_4K
 353         },
 354         {
 355                 /* WDC Caviar Green Advanced Format (4k) drives */
 356                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD????RS*", "*" },
 357                 /*quirks*/ADA_Q_4K
 358         },
 359         {
 360                 /* WDC Caviar Green/Red Advanced Format (4k) drives */
 361                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD????RX*", "*" },
 362                 /*quirks*/ADA_Q_4K
 363         },
 364         {
 365                 /* WDC Caviar Red Advanced Format (4k) drives */
 366                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD??????CX*", "*" },
 367                 /*quirks*/ADA_Q_4K
 368         },
 369         {
 370                 /* WDC Caviar Black Advanced Format (4k) drives */
 371                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD????AZEX*", "*" },
 372                 /*quirks*/ADA_Q_4K
 373         },
 374         {
 375                 /* WDC Caviar Black Advanced Format (4k) drives */
 376                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD????FZEX*", "*" },
 377                 /*quirks*/ADA_Q_4K
 378         },
 379         {
 380                 /* WDC Caviar Green Advanced Format (4k) drives */
 381                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD??????RS*", "*" },
 382                 /*quirks*/ADA_Q_4K
 383         },
 384         {
 385                 /* WDC Caviar Green Advanced Format (4k) drives */
 386                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD??????RX*", "*" },
 387                 /*quirks*/ADA_Q_4K
 388         },
 389         {
 390                 /* WDC Scorpio Black Advanced Format (4k) drives */
 391                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD???PKT*", "*" },
 392                 /*quirks*/ADA_Q_4K
 393         },
 394         {
 395                 /* WDC Scorpio Black Advanced Format (4k) drives */
 396                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD?????PKT*", "*" },
 397                 /*quirks*/ADA_Q_4K
 398         },
 399         {
 400                 /* WDC Scorpio Blue Advanced Format (4k) drives */
 401                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD???PVT*", "*" },
 402                 /*quirks*/ADA_Q_4K
 403         },
 404         {
 405                 /* WDC Scorpio Blue Advanced Format (4k) drives */
 406                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD?????PVT*", "*" },
 407                 /*quirks*/ADA_Q_4K
 408         },
 409         /* SSDs */
 410         {
 411                 /*
 412                  * Corsair Force 2 SSDs
 413                  * 4k optimised & trim only works in 4k requests + 4k aligned
 414                  */
 415                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair CSSD-F*", "*" },
 416                 /*quirks*/ADA_Q_4K
 417         },
 418         {
 419                 /*
 420                  * Corsair Force 3 SSDs
 421                  * 4k optimised & trim only works in 4k requests + 4k aligned
 422                  */
 423                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair Force 3*", "*" },
 424                 /*quirks*/ADA_Q_4K
 425         },
 426         {
 427                 /*
 428                  * Corsair Neutron GTX SSDs
 429                  * 4k optimised & trim only works in 4k requests + 4k aligned
 430                  */
 431                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair Neutron GTX*", "*" },
 432                 /*quirks*/ADA_Q_4K
 433         },
 434         {
 435                 /*
 436                  * Corsair Force GT & GS SSDs
 437                  * 4k optimised & trim only works in 4k requests + 4k aligned
 438                  */
 439                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair Force G*", "*" },
 440                 /*quirks*/ADA_Q_4K
 441         },
 442         {
 443                 /*
 444                  * Crucial M4 SSDs
 445                  * 4k optimised & trim only works in 4k requests + 4k aligned
 446                  */
 447                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "M4-CT???M4SSD2*", "*" },
 448                 /*quirks*/ADA_Q_4K
 449         },
 450         {
 451                 /*
 452                  * Crucial M500 SSDs MU07 firmware
 453                  * NCQ Trim works
 454                  */
 455                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Crucial CT*M500*", "MU07" },
 456                 /*quirks*/0
 457         },
 458         {
 459                 /*
 460                  * Crucial M500 SSDs all other firmware
 461                  * NCQ Trim doesn't work
 462                  */
 463                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Crucial CT*M500*", "*" },
 464                 /*quirks*/ADA_Q_NCQ_TRIM_BROKEN
 465         },
 466         {
 467                 /*
 468                  * Crucial M550 SSDs
 469                  * NCQ Trim doesn't work, but only on MU01 firmware
 470                  */
 471                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Crucial CT*M550*", "MU01" },
 472                 /*quirks*/ADA_Q_NCQ_TRIM_BROKEN
 473         },
 474         {
 475                 /*
 476                  * Crucial MX100 SSDs
 477                  * NCQ Trim doesn't work, but only on MU01 firmware
 478                  */
 479                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Crucial CT*MX100*", "MU01" },
 480                 /*quirks*/ADA_Q_NCQ_TRIM_BROKEN
 481         },
 482         {
 483                 /*
 484                  * Crucial RealSSD C300 SSDs
 485                  * 4k optimised
 486                  */
 487                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "C300-CTFDDAC???MAG*",
 488                 "*" }, /*quirks*/ADA_Q_4K
 489         },
 490         {
 491                 /*
 492                  * FCCT M500 SSDs
 493                  * NCQ Trim doesn't work
 494                  */
 495                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "FCCT*M500*", "*" },
 496                 /*quirks*/ADA_Q_NCQ_TRIM_BROKEN
 497         },
 498         {
 499                 /*
 500                  * Intel 320 Series SSDs
 501                  * 4k optimised & trim only works in 4k requests + 4k aligned
 502                  */
 503                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSA2CW*", "*" },
 504                 /*quirks*/ADA_Q_4K
 505         },
 506         {
 507                 /*
 508                  * Intel 330 Series SSDs
 509                  * 4k optimised & trim only works in 4k requests + 4k aligned
 510                  */
 511                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSC2CT*", "*" },
 512                 /*quirks*/ADA_Q_4K
 513         },
 514         {
 515                 /*
 516                  * Intel 510 Series SSDs
 517                  * 4k optimised & trim only works in 4k requests + 4k aligned
 518                  */
 519                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSC2MH*", "*" },
 520                 /*quirks*/ADA_Q_4K
 521         },
 522         {
 523                 /*
 524                  * Intel 520 Series SSDs
 525                  * 4k optimised & trim only works in 4k requests + 4k aligned
 526                  */
 527                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSC2BW*", "*" },
 528                 /*quirks*/ADA_Q_4K
 529         },
 530         {
 531                 /*
 532                  * Intel S3610 Series SSDs
 533                  * 4k optimised & trim only works in 4k requests + 4k aligned
 534                  */
 535                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSC2BX*", "*" },
 536                 /*quirks*/ADA_Q_4K
 537         },
 538         {
 539                 /*
 540                  * Intel X25-M Series SSDs
 541                  * 4k optimised & trim only works in 4k requests + 4k aligned
 542                  */
 543                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSA2M*", "*" },
 544                 /*quirks*/ADA_Q_4K
 545         },
 546         {
 547                 /*
 548                  * KingDian S200 60GB P0921B
 549                  * Trimming crash the SSD
 550                  */
 551                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "KingDian S200 *", "*" },
 552                 /*quirks*/ADA_Q_NO_TRIM
 553         },
 554         {
 555                 /*
 556                  * Kingston E100 Series SSDs
 557                  * 4k optimised & trim only works in 4k requests + 4k aligned
 558                  */
 559                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "KINGSTON SE100S3*", "*" },
 560                 /*quirks*/ADA_Q_4K
 561         },
 562         {
 563                 /*
 564                  * Kingston HyperX 3k SSDs
 565                  * 4k optimised & trim only works in 4k requests + 4k aligned
 566                  */
 567                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "KINGSTON SH103S3*", "*" },
 568                 /*quirks*/ADA_Q_4K
 569         },
 570         {
 571                 /*
 572                  * Marvell SSDs (entry taken from OpenSolaris)
 573                  * 4k optimised & trim only works in 4k requests + 4k aligned
 574                  */
 575                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "MARVELL SD88SA02*", "*" },
 576                 /*quirks*/ADA_Q_4K
 577         },
 578         {
 579                 /*
 580                  * Micron M500 SSDs firmware MU07
 581                  * NCQ Trim works?
 582                  */
 583                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Micron M500*", "MU07" },
 584                 /*quirks*/0
 585         },
 586         {
 587                 /*
 588                  * Micron M500 SSDs all other firmware
 589                  * NCQ Trim doesn't work
 590                  */
 591                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Micron M500*", "*" },
 592                 /*quirks*/ADA_Q_NCQ_TRIM_BROKEN
 593         },
 594         {
 595                 /*
 596                  * Micron M5[15]0 SSDs
 597                  * NCQ Trim doesn't work, but only MU01 firmware
 598                  */
 599                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Micron M5[15]0*", "MU01" },
 600                 /*quirks*/ADA_Q_NCQ_TRIM_BROKEN
 601         },
 602         {
 603                 /*
 604                  * Micron 5100 SSDs
 605                  * 4k optimised & trim only works in 4k requests + 4k aligned
 606                  */
 607                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Micron 5100 MTFDDAK*", "*" },
 608                 /*quirks*/ADA_Q_4K
 609         },
 610         {
 611                 /*
 612                  * OCZ Agility 2 SSDs
 613                  * 4k optimised & trim only works in 4k requests + 4k aligned
 614                  */
 615                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-AGILITY2*", "*" },
 616                 /*quirks*/ADA_Q_4K
 617         },
 618         {
 619                 /*
 620                  * OCZ Agility 3 SSDs
 621                  * 4k optimised & trim only works in 4k requests + 4k aligned
 622                  */
 623                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-AGILITY3*", "*" },
 624                 /*quirks*/ADA_Q_4K
 625         },
 626         {
 627                 /*
 628                  * OCZ Deneva R Series SSDs
 629                  * 4k optimised & trim only works in 4k requests + 4k aligned
 630                  */
 631                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "DENRSTE251M45*", "*" },
 632                 /*quirks*/ADA_Q_4K
 633         },
 634         {
 635                 /*
 636                  * OCZ Vertex 2 SSDs (inc pro series)
 637                  * 4k optimised & trim only works in 4k requests + 4k aligned
 638                  */
 639                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ?VERTEX2*", "*" },
 640                 /*quirks*/ADA_Q_4K
 641         },
 642         {
 643                 /*
 644                  * OCZ Vertex 3 SSDs
 645                  * 4k optimised & trim only works in 4k requests + 4k aligned
 646                  */
 647                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-VERTEX3*", "*" },
 648                 /*quirks*/ADA_Q_4K
 649         },
 650         {
 651                 /*
 652                  * OCZ Vertex 4 SSDs
 653                  * 4k optimised & trim only works in 4k requests + 4k aligned
 654                  */
 655                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-VERTEX4*", "*" },
 656                 /*quirks*/ADA_Q_4K
 657         },
 658         {
 659                 /*
 660                  * Samsung 750 SSDs
 661                  * 4k optimised, NCQ TRIM seems to work
 662                  */
 663                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Samsung SSD 750*", "*" },
 664                 /*quirks*/ADA_Q_4K
 665         },
 666         {
 667                 /*
 668                  * Samsung 830 Series SSDs
 669                  * 4k optimised, NCQ TRIM Broken (normal TRIM is fine)
 670                  */
 671                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG SSD 830 Series*", "*" },
 672                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 673         },
 674         {
 675                 /*
 676                  * Samsung 840 SSDs
 677                  * 4k optimised, NCQ TRIM Broken (normal TRIM is fine)
 678                  */
 679                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Samsung SSD 840*", "*" },
 680                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 681         },
 682         {
 683                 /*
 684                  * Samsung 845 SSDs
 685                  * 4k optimised, NCQ TRIM Broken (normal TRIM is fine)
 686                  */
 687                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Samsung SSD 845*", "*" },
 688                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 689         },
 690         {
 691                 /*
 692                  * Samsung 850 SSDs
 693                  * 4k optimised, NCQ TRIM broken (normal TRIM fine)
 694                  */
 695                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Samsung SSD 850*", "*" },
 696                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 697         },
 698         {
 699                 /*
 700                  * Samsung SM863 Series SSDs (MZ7KM*)
 701                  * 4k optimised, NCQ believed to be working
 702                  */
 703                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG MZ7KM*", "*" },
 704                 /*quirks*/ADA_Q_4K
 705         },
 706         {
 707                 /*
 708                  * Samsung 843T Series SSDs (MZ7WD*)
 709                  * Samsung PM851 Series SSDs (MZ7TE*)
 710                  * Samsung PM853T Series SSDs (MZ7GE*)
 711                  * 4k optimised, NCQ believed to be broken since these are
 712                  * appear to be built with the same controllers as the 840/850.
 713                  */
 714                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG MZ7*", "*" },
 715                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 716         },
 717         {
 718                 /*
 719                  * Same as for SAMSUNG MZ7* but enable the quirks for SSD
 720                  * starting with MZ7* too
 721                  */
 722                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "MZ7*", "*" },
 723                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 724         },
 725         {
 726                 /*
 727                  * Samsung PM851 Series SSDs Dell OEM
 728                  * device model          "SAMSUNG SSD PM851 mSATA 256GB"
 729                  * 4k optimised, NCQ broken
 730                  */
 731                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG SSD PM851*", "*" },
 732                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 733         },
 734         {
 735                 /*
 736                  * SuperTalent TeraDrive CT SSDs
 737                  * 4k optimised & trim only works in 4k requests + 4k aligned
 738                  */
 739                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "FTM??CT25H*", "*" },
 740                 /*quirks*/ADA_Q_4K
 741         },
 742         {
 743                 /*
 744                  * XceedIOPS SATA SSDs
 745                  * 4k optimised
 746                  */
 747                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SG9XCS2D*", "*" },
 748                 /*quirks*/ADA_Q_4K
 749         },
 750         {
 751                 /*
 752                  * Samsung drive that doesn't support READ LOG EXT or
 753                  * READ LOG DMA EXT, despite reporting that it does in
 754                  * ATA identify data:
 755                  * SAMSUNG HD200HJ KF100-06
 756                  */
 757                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG HD200*", "*" },
 758                 /*quirks*/ADA_Q_LOG_BROKEN
 759         },
 760         {
 761                 /*
 762                  * Samsung drive that doesn't support READ LOG EXT or
 763                  * READ LOG DMA EXT, despite reporting that it does in
 764                  * ATA identify data:
 765                  * SAMSUNG HD501LJ CR100-10
 766                  */
 767                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG HD501*", "*" },
 768                 /*quirks*/ADA_Q_LOG_BROKEN
 769         },
 770         {
 771                 /*
 772                  * Seagate Lamarr 8TB Shingled Magnetic Recording (SMR)
 773                  * Drive Managed SATA hard drive.  This drive doesn't report
 774                  * in firmware that it is a drive managed SMR drive.
 775                  */
 776                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST8000AS000[23]*", "*" },
 777                 /*quirks*/ADA_Q_SMR_DM
 778         },
 779         {
 780                 /* Default */
 781                 {
 782                   T_ANY, SIP_MEDIA_REMOVABLE|SIP_MEDIA_FIXED,
 783                   /*vendor*/"*", /*product*/"*", /*revision*/"*"
 784                 },
 785                 /*quirks*/0
 786         },
 787 };
 788
 789 static  disk_strategy_t adastrategy;
 790 static  dumper_t        adadump;
 791 static  periph_init_t   adainit;
 792 static  void            adadiskgonecb(struct disk *dp);
 793 static  periph_oninv_t  adaoninvalidate;
 794 static  periph_dtor_t   adacleanup;
 795 static  void            adaasync(void *callback_arg, u_int32_t code,
 796                                 struct cam_path *path, void *arg);
 797 static  int             adazonemodesysctl(SYSCTL_HANDLER_ARGS);
 798 static  int             adazonesupsysctl(SYSCTL_HANDLER_ARGS);
 799 static  void            adasysctlinit(void *context, int pending);
 800 static  int             adagetattr(struct bio *bp);
 801 static  void            adasetflags(struct ada_softc *softc,
 802                                     struct ccb_getdev *cgd);
 803 static  periph_ctor_t   adaregister;
 804 static  void            ada_dsmtrim(struct ada_softc *softc, struct bio *bp,
 805                                     struct ccb_ataio *ataio);
 806 static  void            ada_cfaerase(struct ada_softc *softc, struct bio *bp,
 807                                      struct ccb_ataio *ataio);
 808 static  int             ada_zone_bio_to_ata(int disk_zone_cmd);
 809 static  int             ada_zone_cmd(struct cam_periph *periph, union ccb *ccb,
 810                                      struct bio *bp, int *queue_ccb);
 811 static  periph_start_t  adastart;
 812 static  void            adaprobedone(struct cam_periph *periph, union ccb *ccb);
 813 static  void            adazonedone(struct cam_periph *periph, union ccb *ccb);
 814 static  void            adadone(struct cam_periph *periph,
 815                                union ccb *done_ccb);
 816 static  int             adaerror(union ccb *ccb, u_int32_t cam_flags,
 817                                 u_int32_t sense_flags);
 818 static void             adagetparams(struct cam_periph *periph,
 819                                 struct ccb_getdev *cgd);
 820 static timeout_t        adasendorderedtag;
 821 static void             adashutdown(void *arg, int howto);
 822 static void             adasuspend(void *arg);
 823 static void             adaresume(void *arg);
 824
 825 #ifndef ADA_DEFAULT_TIMEOUT
 826 #define ADA_DEFAULT_TIMEOUT 30  /* Timeout in seconds */
 827 #endif
 828
 829 #ifndef ADA_DEFAULT_RETRY
 830 #define ADA_DEFAULT_RETRY       4
 831 #endif
 832
 833 #ifndef ADA_DEFAULT_SEND_ORDERED
 834 #define ADA_DEFAULT_SEND_ORDERED        1
 835 #endif
 836
 837 #ifndef ADA_DEFAULT_SPINDOWN_SHUTDOWN
 838 #define ADA_DEFAULT_SPINDOWN_SHUTDOWN   1
 839 #endif
 840
 841 #ifndef ADA_DEFAULT_SPINDOWN_SUSPEND
 842 #define ADA_DEFAULT_SPINDOWN_SUSPEND    1
 843 #endif
 844
 845 #ifndef ADA_DEFAULT_READ_AHEAD
 846 #define ADA_DEFAULT_READ_AHEAD  1
 847 #endif
 848
 849 #ifndef ADA_DEFAULT_WRITE_CACHE
 850 #define ADA_DEFAULT_WRITE_CACHE 1
 851 #endif
 852
 853 #define ADA_RA  (softc->read_ahead >= 0 ? \
 854                  softc->read_ahead : ada_read_ahead)
 855 #define ADA_WC  (softc->write_cache >= 0 ? \
 856                  softc->write_cache : ada_write_cache)
 857
 858 /*
 859  * Most platforms map firmware geometry to actual, but some don't.  If
 860  * not overridden, default to nothing.
 861  */
 862 #ifndef ata_disk_firmware_geom_adjust
 863 #define ata_disk_firmware_geom_adjust(disk)
 864 #endif
 865
 866 static int ada_retry_count = ADA_DEFAULT_RETRY;
 867 static int ada_default_timeout = ADA_DEFAULT_TIMEOUT;
 868 static int ada_send_ordered = ADA_DEFAULT_SEND_ORDERED;
 869 static int ada_spindown_shutdown = ADA_DEFAULT_SPINDOWN_SHUTDOWN;
 870 static int ada_spindown_suspend = ADA_DEFAULT_SPINDOWN_SUSPEND;
 871 static int ada_read_ahead = ADA_DEFAULT_READ_AHEAD;
 872 static int ada_write_cache = ADA_DEFAULT_WRITE_CACHE;
 873
 874 static SYSCTL_NODE(_kern_cam, OID_AUTO, ada, CTLFLAG_RD, 0,
 875             "CAM Direct Access Disk driver");
 876 SYSCTL_INT(_kern_cam_ada, OID_AUTO, retry_count, CTLFLAG_RWTUN,
 877            &ada_retry_count, 0, "Normal I/O retry count");
 878 SYSCTL_INT(_kern_cam_ada, OID_AUTO, default_timeout, CTLFLAG_RWTUN,
 879            &ada_default_timeout, 0, "Normal I/O timeout (in seconds)");
 880 SYSCTL_INT(_kern_cam_ada, OID_AUTO, send_ordered, CTLFLAG_RWTUN,
 881            &ada_send_ordered, 0, "Send Ordered Tags");
 882 SYSCTL_INT(_kern_cam_ada, OID_AUTO, spindown_shutdown, CTLFLAG_RWTUN,
 883            &ada_spindown_shutdown, 0, "Spin down upon shutdown");
 884 SYSCTL_INT(_kern_cam_ada, OID_AUTO, spindown_suspend, CTLFLAG_RWTUN,
 885            &ada_spindown_suspend, 0, "Spin down upon suspend");
 886 SYSCTL_INT(_kern_cam_ada, OID_AUTO, read_ahead, CTLFLAG_RWTUN,
 887            &ada_read_ahead, 0, "Enable disk read-ahead");
 888 SYSCTL_INT(_kern_cam_ada, OID_AUTO, write_cache, CTLFLAG_RWTUN,
 889            &ada_write_cache, 0, "Enable disk write cache");
 890
 891 /*
 892  * ADA_ORDEREDTAG_INTERVAL determines how often, relative
 893  * to the default timeout, we check to see whether an ordered
 894  * tagged transaction is appropriate to prevent simple tag
 895  * starvation.  Since we'd like to ensure that there is at least
 896  * 1/2 of the timeout length left for a starved transaction to
 897  * complete after we've sent an ordered tag, we must poll at least
 898  * four times in every timeout period.  This takes care of the worst
 899  * case where a starved transaction starts during an interval that
 900  * meets the requirement "don't send an ordered tag" test so it takes
 901  * us two intervals to determine that a tag must be sent.
 902  */
 903 #ifndef ADA_ORDEREDTAG_INTERVAL
 904 #define ADA_ORDEREDTAG_INTERVAL 4
 905 #endif
 906
 907 static struct periph_driver adadriver =
 908 {
 909         adainit, "ada",
 910         TAILQ_HEAD_INITIALIZER(adadriver.units), /* generation */ 0
 911 };
 912
 913 static int adadeletemethodsysctl(SYSCTL_HANDLER_ARGS);
 914
 915 PERIPHDRIVER_DECLARE(ada, adadriver);
 916
 917 static MALLOC_DEFINE(M_ATADA, "ata_da", "ata_da buffers");
 918
 919 static int
 920 adaopen(struct disk *dp)
 921 {
 922         struct cam_periph *periph;
 923         struct ada_softc *softc;
 924         int error;
 925
 926         periph = (struct cam_periph *)dp->d_drv1;
 927         if (cam_periph_acquire(periph) != 0) {
 928                 return(ENXIO);
 929         }
 930
 931         cam_periph_lock(periph);
 932         if ((error = cam_periph_hold(periph, PRIBIO|PCATCH)) != 0) {
 933                 cam_periph_unlock(periph);
 934                 cam_periph_release(periph);
 935                 return (error);
 936         }
 937
 938         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE | CAM_DEBUG_PERIPH,
 939             ("adaopen\n"));
 940
 941         softc = (struct ada_softc *)periph->softc;
 942         softc->flags |= ADA_FLAG_OPEN;
 943
 944         cam_periph_unhold(periph);
 945         cam_periph_unlock(periph);
 946         return (0);
 947 }
 948
 949 static int
 950 adaclose(struct disk *dp)
 951 {
 952         struct  cam_periph *periph;
 953         struct  ada_softc *softc;
 954         union ccb *ccb;
 955         int error;
 956
 957         periph = (struct cam_periph *)dp->d_drv1;
 958         softc = (struct ada_softc *)periph->softc;
 959         cam_periph_lock(periph);
 960
 961         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE | CAM_DEBUG_PERIPH,
 962             ("adaclose\n"));
 963
 964         /* We only sync the cache if the drive is capable of it. */
 965         if ((softc->flags & ADA_FLAG_DIRTY) != 0 &&
 966             (softc->flags & ADA_FLAG_CAN_FLUSHCACHE) != 0 &&
 967             (periph->flags & CAM_PERIPH_INVALID) == 0 &&
 968             cam_periph_hold(periph, PRIBIO) == 0) {
 969
 970                 ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL);
 971                 cam_fill_ataio(&ccb->ataio,
 972                                     1,
 973                                     NULL,
 974                                     CAM_DIR_NONE,
 975                                     0,
 976                                     NULL,
 977                                     0,
 978                                     ada_default_timeout*1000);
 979
 980                 if (softc->flags & ADA_FLAG_CAN_48BIT)
 981                         ata_48bit_cmd(&ccb->ataio, ATA_FLUSHCACHE48, 0, 0, 0);
 982                 else
 983                         ata_28bit_cmd(&ccb->ataio, ATA_FLUSHCACHE, 0, 0, 0);
 984                 error = cam_periph_runccb(ccb, adaerror, /*cam_flags*/0,
 985                     /*sense_flags*/0, softc->disk->d_devstat);
 986
 987                 if (error != 0)
 988                         xpt_print(periph->path, "Synchronize cache failed\n");
 989                 softc->flags &= ~ADA_FLAG_DIRTY;
 990                 xpt_release_ccb(ccb);
 991                 cam_periph_unhold(periph);
 992         }
 993
 994         softc->flags &= ~ADA_FLAG_OPEN;
 995
 996         while (softc->refcount != 0)
 997                 cam_periph_sleep(periph, &softc->refcount, PRIBIO, "adaclose", 1);
 998         cam_periph_unlock(periph);
 999         cam_periph_release(periph);
1000         return (0);
1001 }
1002
1003 static void
1004 adaschedule(struct cam_periph *periph)
1005 {
1006         struct ada_softc *softc = (struct ada_softc *)periph->softc;
1007
1008         if (softc->state != ADA_STATE_NORMAL)
1009                 return;
1010
1011         cam_iosched_schedule(softc->cam_iosched, periph);
1012 }
1013
1014 /*
1015  * Actually translate the requested transfer into one the physical driver
1016  * can understand.  The transfer is described by a buf and will include
1017  * only one physical transfer.
1018  */
1019 static void
1020 adastrategy(struct bio *bp)
1021 {
1022         struct cam_periph *periph;
1023         struct ada_softc *softc;
1024
1025         periph = (struct cam_periph *)bp->bio_disk->d_drv1;
1026         softc = (struct ada_softc *)periph->softc;
1027
1028         cam_periph_lock(periph);
1029
1030         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("adastrategy(%p)\n", bp));
1031
1032         /*
1033          * If the device has been made invalid, error out
1034          */
1035         if ((periph->flags & CAM_PERIPH_INVALID) != 0) {
1036                 cam_periph_unlock(periph);
1037                 biofinish(bp, NULL, ENXIO);
1038                 return;
1039         }
1040
1041         /*
1042          * Zone commands must be ordered, because they can depend on the
1043          * effects of previously issued commands, and they may affect
1044          * commands after them.
1045          */
1046         if (bp->bio_cmd == BIO_ZONE)
1047                 bp->bio_flags |= BIO_ORDERED;
1048
1049         /*
1050          * Place it in the queue of disk activities for this disk
1051          */
1052         cam_iosched_queue_work(softc->cam_iosched, bp);
1053
1054         /*
1055          * Schedule ourselves for performing the work.
1056          */
1057         adaschedule(periph);
1058         cam_periph_unlock(periph);
1059
1060         return;
1061 }
1062
1063 static int
1064 adadump(void *arg, void *virtual, vm_offset_t physical, off_t offset, size_t length)
1065 {
1066         struct      cam_periph *periph;
1067         struct      ada_softc *softc;
1068         u_int       secsize;
1069         struct      ccb_ataio ataio;
1070         struct      disk *dp;
1071         uint64_t    lba;
1072         uint16_t    count;
1073         int         error = 0;
1074
1075         dp = arg;
1076         periph = dp->d_drv1;
1077         softc = (struct ada_softc *)periph->softc;
1078         secsize = softc->params.secsize;
1079         lba = offset / secsize;
1080         count = length / secsize;
1081         if ((periph->flags & CAM_PERIPH_INVALID) != 0)
1082                 return (ENXIO);
1083
1084         memset(&ataio, 0, sizeof(ataio));
1085         if (length > 0) {
1086                 xpt_setup_ccb(&ataio.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
1087                 ataio.ccb_h.ccb_state = ADA_CCB_DUMP;
1088                 cam_fill_ataio(&ataio,
1089                     0,
1090                     NULL,
1091                     CAM_DIR_OUT,
1092                     0,
1093                     (u_int8_t *) virtual,
1094                     length,
1095                     ada_default_timeout*1000);
1096                 if ((softc->flags & ADA_FLAG_CAN_48BIT) &&
1097                     (lba + count >= ATA_MAX_28BIT_LBA ||
1098                     count >= 256)) {
1099                         ata_48bit_cmd(&ataio, ATA_WRITE_DMA48,
1100                             0, lba, count);
1101                 } else {
1102                         ata_28bit_cmd(&ataio, ATA_WRITE_DMA,
1103                             0, lba, count);
1104                 }
1105                 error = cam_periph_runccb((union ccb *)&ataio, adaerror,
1106                     0, SF_NO_RECOVERY | SF_NO_RETRY, NULL);
1107                 if (error != 0)
1108                         printf("Aborting dump due to I/O error.\n");
1109
1110                 return (error);
1111         }
1112
1113         if (softc->flags & ADA_FLAG_CAN_FLUSHCACHE) {
1114                 xpt_setup_ccb(&ataio.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
1115
1116                 /*
1117                  * Tell the drive to flush its internal cache. if we
1118                  * can't flush in 5s we have big problems. No need to
1119                  * wait the default 60s to detect problems.
1120                  */
1121                 ataio.ccb_h.ccb_state = ADA_CCB_DUMP;
1122                 cam_fill_ataio(&ataio,
1123                                     0,
1124                                     NULL,
1125                                     CAM_DIR_NONE,
1126                                     0,
1127                                     NULL,
1128                                     0,
1129                                     5*1000);
1130
1131                 if (softc->flags & ADA_FLAG_CAN_48BIT)
1132                         ata_48bit_cmd(&ataio, ATA_FLUSHCACHE48, 0, 0, 0);
1133                 else
1134                         ata_28bit_cmd(&ataio, ATA_FLUSHCACHE, 0, 0, 0);
1135                 error = cam_periph_runccb((union ccb *)&ataio, adaerror,
1136                     0, SF_NO_RECOVERY | SF_NO_RETRY, NULL);
1137                 if (error != 0)
1138                         xpt_print(periph->path, "Synchronize cache failed\n");
1139         }
1140         return (error);
1141 }
1142
1143 static void
1144 adainit(void)
1145 {
1146         cam_status status;
1147
1148         /*
1149          * Install a global async callback.  This callback will
1150          * receive async callbacks like "new device found".
1151          */
1152         status = xpt_register_async(AC_FOUND_DEVICE, adaasync, NULL, NULL);
1153
1154         if (status != CAM_REQ_CMP) {
1155                 printf("ada: Failed to attach master async callback "
1156                        "due to status 0x%x!\n", status);
1157         } else if (ada_send_ordered) {
1158
1159                 /* Register our event handlers */
1160                 if ((EVENTHANDLER_REGISTER(power_suspend, adasuspend,
1161                                            NULL, EVENTHANDLER_PRI_LAST)) == NULL)
1162                     printf("adainit: power event registration failed!\n");
1163                 if ((EVENTHANDLER_REGISTER(power_resume, adaresume,
1164                                            NULL, EVENTHANDLER_PRI_LAST)) == NULL)
1165                     printf("adainit: power event registration failed!\n");
1166                 if ((EVENTHANDLER_REGISTER(shutdown_post_sync, adashutdown,
1167                                            NULL, SHUTDOWN_PRI_DEFAULT)) == NULL)
1168                     printf("adainit: shutdown event registration failed!\n");
1169         }
1170 }
1171
1172 /*
1173  * Callback from GEOM, called when it has finished cleaning up its
1174  * resources.
1175  */
1176 static void
1177 adadiskgonecb(struct disk *dp)
1178 {
1179         struct cam_periph *periph;
1180
1181         periph = (struct cam_periph *)dp->d_drv1;
1182
1183         cam_periph_release(periph);
1184 }
1185
1186 static void
1187 adaoninvalidate(struct cam_periph *periph)
1188 {
1189         struct ada_softc *softc;
1190
1191         softc = (struct ada_softc *)periph->softc;
1192
1193         /*
1194          * De-register any async callbacks.
1195          */
1196         xpt_register_async(0, adaasync, periph, periph->path);
1197 #ifdef CAM_IO_STATS
1198         softc->invalidations++;
1199 #endif
1200
1201         /*
1202          * Return all queued I/O with ENXIO.
1203          * XXX Handle any transactions queued to the card
1204          *     with XPT_ABORT_CCB.
1205          */
1206         cam_iosched_flush(softc->cam_iosched, NULL, ENXIO);
1207
1208         disk_gone(softc->disk);
1209 }
1210
1211 static void
1212 adacleanup(struct cam_periph *periph)
1213 {
1214         struct ada_softc *softc;
1215
1216         softc = (struct ada_softc *)periph->softc;
1217
1218         cam_periph_unlock(periph);
1219
1220         cam_iosched_fini(softc->cam_iosched);
1221
1222         /*
1223          * If we can't free the sysctl tree, oh well...
1224          */
1225         if ((softc->flags & ADA_FLAG_SCTX_INIT) != 0) {
1226 #ifdef CAM_IO_STATS
1227                 if (sysctl_ctx_free(&softc->sysctl_stats_ctx) != 0)
1228                         xpt_print(periph->path,
1229                             "can't remove sysctl stats context\n");
1230 #endif
1231                 if (sysctl_ctx_free(&softc->sysctl_ctx) != 0)
1232                         xpt_print(periph->path,
1233                             "can't remove sysctl context\n");
1234         }
1235
1236         disk_destroy(softc->disk);
1237         callout_drain(&softc->sendordered_c);
1238         free(softc, M_DEVBUF);
1239         cam_periph_lock(periph);
1240 }
1241
1242 static void
1243 adasetdeletemethod(struct ada_softc *softc)
1244 {
1245
1246         if (softc->flags & ADA_FLAG_CAN_NCQ_TRIM)
1247                 softc->delete_method = ADA_DELETE_NCQ_DSM_TRIM;
1248         else if (softc->flags & ADA_FLAG_CAN_TRIM)
1249                 softc->delete_method = ADA_DELETE_DSM_TRIM;
1250         else if ((softc->flags & ADA_FLAG_CAN_CFA) && !(softc->flags & ADA_FLAG_CAN_48BIT))
1251                 softc->delete_method = ADA_DELETE_CFA_ERASE;
1252         else
1253                 softc->delete_method = ADA_DELETE_NONE;
1254 }
1255
1256 static void
1257 adaasync(void *callback_arg, u_int32_t code,
1258         struct cam_path *path, void *arg)
1259 {
1260         struct ccb_getdev cgd;
1261         struct cam_periph *periph;
1262         struct ada_softc *softc;
1263
1264         periph = (struct cam_periph *)callback_arg;
1265         switch (code) {
1266         case AC_FOUND_DEVICE:
1267         {
1268                 struct ccb_getdev *cgd;
1269                 cam_status status;
1270
1271                 cgd = (struct ccb_getdev *)arg;
1272                 if (cgd == NULL)
1273                         break;
1274
1275                 if (cgd->protocol != PROTO_ATA)
1276                         break;
1277
1278                 /*
1279                  * Allocate a peripheral instance for
1280                  * this device and start the probe
1281                  * process.
1282                  */
1283                 status = cam_periph_alloc(adaregister, adaoninvalidate,
1284                                           adacleanup, adastart,
1285                                           "ada", CAM_PERIPH_BIO,
1286                                           path, adaasync,
1287                                           AC_FOUND_DEVICE, cgd);
1288
1289                 if (status != CAM_REQ_CMP
1290                  && status != CAM_REQ_INPROG)
1291                         printf("adaasync: Unable to attach to new device "
1292                                 "due to status 0x%x\n", status);
1293                 break;
1294         }
1295         case AC_GETDEV_CHANGED:
1296         {
1297                 softc = (struct ada_softc *)periph->softc;
1298                 xpt_setup_ccb(&cgd.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
1299                 cgd.ccb_h.func_code = XPT_GDEV_TYPE;
1300                 xpt_action((union ccb *)&cgd);
1301
1302                 /*
1303                  * Set/clear support flags based on the new Identify data.
1304                  */
1305                 adasetflags(softc, &cgd);
1306
1307                 cam_periph_async(periph, code, path, arg);
1308                 break;
1309         }
1310         case AC_ADVINFO_CHANGED:
1311         {
1312                 uintptr_t buftype;
1313
1314                 buftype = (uintptr_t)arg;
1315                 if (buftype == CDAI_TYPE_PHYS_PATH) {
1316                         struct ada_softc *softc;
1317
1318                         softc = periph->softc;
1319                         disk_attr_changed(softc->disk, "GEOM::physpath",
1320                                           M_NOWAIT);
1321                 }
1322                 break;
1323         }
1324         case AC_SENT_BDR:
1325         case AC_BUS_RESET:
1326         {
1327                 softc = (struct ada_softc *)periph->softc;
1328                 cam_periph_async(periph, code, path, arg);
1329                 if (softc->state != ADA_STATE_NORMAL)
1330                         break;
1331                 xpt_setup_ccb(&cgd.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
1332                 cgd.ccb_h.func_code = XPT_GDEV_TYPE;
1333                 xpt_action((union ccb *)&cgd);
1334                 if (ADA_RA >= 0 && softc->flags & ADA_FLAG_CAN_RAHEAD)
1335                         softc->state = ADA_STATE_RAHEAD;
1336                 else if (ADA_WC >= 0 && softc->flags & ADA_FLAG_CAN_WCACHE)
1337                         softc->state = ADA_STATE_WCACHE;
1338                 else if ((softc->flags & ADA_FLAG_CAN_LOG)
1339                       && (softc->zone_mode != ADA_ZONE_NONE))
1340                         softc->state = ADA_STATE_LOGDIR;
1341                 else
1342                     break;
1343                 if (cam_periph_acquire(periph) != 0)
1344                         softc->state = ADA_STATE_NORMAL;
1345                 else
1346                         xpt_schedule(periph, CAM_PRIORITY_DEV);
1347         }
1348         default:
1349                 cam_periph_async(periph, code, path, arg);
1350                 break;
1351         }
1352 }
1353
1354 static int
1355 adazonemodesysctl(SYSCTL_HANDLER_ARGS)
1356 {
1357         char tmpbuf[40];
1358         struct ada_softc *softc;
1359         int error;
1360
1361         softc = (struct ada_softc *)arg1;
1362
1363         switch (softc->zone_mode) {
1364         case ADA_ZONE_DRIVE_MANAGED:
1365                 snprintf(tmpbuf, sizeof(tmpbuf), "Drive Managed");
1366                 break;
1367         case ADA_ZONE_HOST_AWARE:
1368                 snprintf(tmpbuf, sizeof(tmpbuf), "Host Aware");
1369                 break;
1370         case ADA_ZONE_HOST_MANAGED:
1371                 snprintf(tmpbuf, sizeof(tmpbuf), "Host Managed");
1372                 break;
1373         case ADA_ZONE_NONE:
1374         default:
1375                 snprintf(tmpbuf, sizeof(tmpbuf), "Not Zoned");
1376                 break;
1377         }
1378
1379         error = sysctl_handle_string(oidp, tmpbuf, sizeof(tmpbuf), req);
1380
1381         return (error);
1382 }
1383
1384 static int
1385 adazonesupsysctl(SYSCTL_HANDLER_ARGS)
1386 {
1387         char tmpbuf[180];
1388         struct ada_softc *softc;
1389         struct sbuf sb;
1390         int error, first;
1391         unsigned int i;
1392
1393         softc = (struct ada_softc *)arg1;
1394
1395         error = 0;
1396         first = 1;
1397         sbuf_new(&sb, tmpbuf, sizeof(tmpbuf), 0);
1398
1399         for (i = 0; i < sizeof(ada_zone_desc_table) /
1400              sizeof(ada_zone_desc_table[0]); i++) {
1401                 if (softc->zone_flags & ada_zone_desc_table[i].value) {
1402                         if (first == 0)
1403                                 sbuf_printf(&sb, ", ");
1404                         else
1405                                 first = 0;
1406                         sbuf_cat(&sb, ada_zone_desc_table[i].desc);
1407                 }
1408         }
1409
1410         if (first == 1)
1411                 sbuf_printf(&sb, "None");
1412
1413         sbuf_finish(&sb);
1414
1415         error = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
1416
1417         return (error);
1418 }
1419
1420
1421 static void
1422 adasysctlinit(void *context, int pending)
1423 {
1424         struct cam_periph *periph;
1425         struct ada_softc *softc;
1426         char tmpstr[32], tmpstr2[16];
1427
1428         periph = (struct cam_periph *)context;
1429
1430         /* periph was held for us when this task was enqueued */
1431         if ((periph->flags & CAM_PERIPH_INVALID) != 0) {
1432                 cam_periph_release(periph);
1433                 return;
1434         }
1435
1436         softc = (struct ada_softc *)periph->softc;
1437         snprintf(tmpstr, sizeof(tmpstr), "CAM ADA unit %d",periph->unit_number);
1438         snprintf(tmpstr2, sizeof(tmpstr2), "%d", periph->unit_number);
1439
1440         sysctl_ctx_init(&softc->sysctl_ctx);
1441         softc->flags |= ADA_FLAG_SCTX_INIT;
1442         softc->sysctl_tree = SYSCTL_ADD_NODE_WITH_LABEL(&softc->sysctl_ctx,
1443                 SYSCTL_STATIC_CHILDREN(_kern_cam_ada), OID_AUTO, tmpstr2,
1444                 CTLFLAG_RD, 0, tmpstr, "device_index");
1445         if (softc->sysctl_tree == NULL) {
1446                 printf("adasysctlinit: unable to allocate sysctl tree\n");
1447                 cam_periph_release(periph);
1448                 return;
1449         }
1450
1451         SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1452                 OID_AUTO, "delete_method", CTLTYPE_STRING | CTLFLAG_RW,
1453                 softc, 0, adadeletemethodsysctl, "A",
1454                 "BIO_DELETE execution method");
1455         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
1456                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
1457                 "trim_count", CTLFLAG_RD, &softc->trim_count,
1458                 "Total number of dsm commands sent");
1459         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
1460                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
1461                 "trim_ranges", CTLFLAG_RD, &softc->trim_ranges,
1462                 "Total number of ranges in dsm commands");
1463         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
1464                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
1465                 "trim_lbas", CTLFLAG_RD, &softc->trim_lbas,
1466                 "Total lbas in the dsm commands sent");
1467         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1468                 OID_AUTO, "read_ahead", CTLFLAG_RW | CTLFLAG_MPSAFE,
1469                 &softc->read_ahead, 0, "Enable disk read ahead.");
1470         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1471                 OID_AUTO, "write_cache", CTLFLAG_RW | CTLFLAG_MPSAFE,
1472                 &softc->write_cache, 0, "Enable disk write cache.");
1473         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1474                 OID_AUTO, "unmapped_io", CTLFLAG_RD | CTLFLAG_MPSAFE,
1475                 &softc->unmappedio, 0, "Unmapped I/O leaf");
1476         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1477                 OID_AUTO, "rotating", CTLFLAG_RD | CTLFLAG_MPSAFE,
1478                 &softc->rotating, 0, "Rotating media");
1479         SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1480                 OID_AUTO, "zone_mode", CTLTYPE_STRING | CTLFLAG_RD,
1481                 softc, 0, adazonemodesysctl, "A",
1482                 "Zone Mode");
1483         SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1484                 OID_AUTO, "zone_support", CTLTYPE_STRING | CTLFLAG_RD,
1485                 softc, 0, adazonesupsysctl, "A",
1486                 "Zone Support");
1487         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
1488                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
1489                 "optimal_seq_zones", CTLFLAG_RD, &softc->optimal_seq_zones,
1490                 "Optimal Number of Open Sequential Write Preferred Zones");
1491         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
1492                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
1493                 "optimal_nonseq_zones", CTLFLAG_RD,
1494                 &softc->optimal_nonseq_zones,
1495                 "Optimal Number of Non-Sequentially Written Sequential Write "
1496                 "Preferred Zones");
1497         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
1498                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
1499                 "max_seq_zones", CTLFLAG_RD, &softc->max_seq_zones,
1500                 "Maximum Number of Open Sequential Write Required Zones");
1501
1502 #ifdef CAM_TEST_FAILURE
1503         /*
1504          * Add a 'door bell' sysctl which allows one to set it from userland
1505          * and cause something bad to happen.  For the moment, we only allow
1506          * whacking the next read or write.
1507          */
1508         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1509                 OID_AUTO, "force_read_error", CTLFLAG_RW | CTLFLAG_MPSAFE,
1510                 &softc->force_read_error, 0,
1511                 "Force a read error for the next N reads.");
1512         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1513                 OID_AUTO, "force_write_error", CTLFLAG_RW | CTLFLAG_MPSAFE,
1514                 &softc->force_write_error, 0,
1515                 "Force a write error for the next N writes.");
1516         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1517                 OID_AUTO, "periodic_read_error", CTLFLAG_RW | CTLFLAG_MPSAFE,
1518                 &softc->periodic_read_error, 0,
1519                 "Force a read error every N reads (don't set too low).");
1520         SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1521                 OID_AUTO, "invalidate", CTLTYPE_U64 | CTLFLAG_RW | CTLFLAG_MPSAFE,
1522                 periph, 0, cam_periph_invalidate_sysctl, "I",
1523                 "Write 1 to invalidate the drive immediately");
1524 #endif
1525
1526 #ifdef CAM_IO_STATS
1527         softc->sysctl_stats_tree = SYSCTL_ADD_NODE(&softc->sysctl_stats_ctx,
1528                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO, "stats",
1529                 CTLFLAG_RD, 0, "Statistics");
1530         SYSCTL_ADD_INT(&softc->sysctl_stats_ctx,
1531                 SYSCTL_CHILDREN(softc->sysctl_stats_tree),
1532                 OID_AUTO, "timeouts", CTLFLAG_RD | CTLFLAG_MPSAFE,
1533                 &softc->timeouts, 0,
1534                 "Device timeouts reported by the SIM");
1535         SYSCTL_ADD_INT(&softc->sysctl_stats_ctx,
1536                 SYSCTL_CHILDREN(softc->sysctl_stats_tree),
1537                 OID_AUTO, "errors", CTLFLAG_RD | CTLFLAG_MPSAFE,
1538                 &softc->errors, 0,
1539                 "Transport errors reported by the SIM.");
1540         SYSCTL_ADD_INT(&softc->sysctl_stats_ctx,
1541                 SYSCTL_CHILDREN(softc->sysctl_stats_tree),
1542                 OID_AUTO, "pack_invalidations", CTLFLAG_RD | CTLFLAG_MPSAFE,
1543                 &softc->invalidations, 0,
1544                 "Device pack invalidations.");
1545 #endif
1546
1547         cam_iosched_sysctl_init(softc->cam_iosched, &softc->sysctl_ctx,
1548             softc->sysctl_tree);
1549
1550         cam_periph_release(periph);
1551 }
1552
1553 static int
1554 adagetattr(struct bio *bp)
1555 {
1556         int ret;
1557         struct cam_periph *periph;
1558
1559         periph = (struct cam_periph *)bp->bio_disk->d_drv1;
1560         cam_periph_lock(periph);
1561         ret = xpt_getattr(bp->bio_data, bp->bio_length, bp->bio_attribute,
1562             periph->path);
1563         cam_periph_unlock(periph);
1564         if (ret == 0)
1565                 bp->bio_completed = bp->bio_length;
1566         return ret;
1567 }
1568
1569 static int
1570 adadeletemethodsysctl(SYSCTL_HANDLER_ARGS)
1571 {
1572         char buf[16];
1573         const char *p;
1574         struct ada_softc *softc;
1575         int i, error, value, methods;
1576
1577         softc = (struct ada_softc *)arg1;
1578
1579         value = softc->delete_method;
1580         if (value < 0 || value > ADA_DELETE_MAX)
1581                 p = "UNKNOWN";
1582         else
1583                 p = ada_delete_method_names[value];
1584         strncpy(buf, p, sizeof(buf));
1585         error = sysctl_handle_string(oidp, buf, sizeof(buf), req);
1586         if (error != 0 || req->newptr == NULL)
1587                 return (error);
1588         methods = 1 << ADA_DELETE_DISABLE;
1589         if ((softc->flags & ADA_FLAG_CAN_CFA) &&
1590             !(softc->flags & ADA_FLAG_CAN_48BIT))
1591                 methods |= 1 << ADA_DELETE_CFA_ERASE;
1592         if (softc->flags & ADA_FLAG_CAN_TRIM)
1593                 methods |= 1 << ADA_DELETE_DSM_TRIM;
1594         if (softc->flags & ADA_FLAG_CAN_NCQ_TRIM)
1595                 methods |= 1 << ADA_DELETE_NCQ_DSM_TRIM;
1596         for (i = 0; i <= ADA_DELETE_MAX; i++) {
1597                 if (!(methods & (1 << i)) ||
1598                     strcmp(buf, ada_delete_method_names[i]) != 0)
1599                         continue;
1600                 softc->delete_method = i;
1601                 return (0);
1602         }
1603         return (EINVAL);
1604 }
1605
1606 static void
1607 adasetflags(struct ada_softc *softc, struct ccb_getdev *cgd)
1608 {
1609         if ((cgd->ident_data.capabilities1 & ATA_SUPPORT_DMA) &&
1610             (cgd->inq_flags & SID_DMA))
1611                 softc->flags |= ADA_FLAG_CAN_DMA;
1612         else
1613                 softc->flags &= ~ADA_FLAG_CAN_DMA;
1614
1615         if (cgd->ident_data.support.command2 & ATA_SUPPORT_ADDRESS48) {
1616                 softc->flags |= ADA_FLAG_CAN_48BIT;
1617                 if (cgd->inq_flags & SID_DMA48)
1618                         softc->flags |= ADA_FLAG_CAN_DMA48;
1619                 else
1620                         softc->flags &= ~ADA_FLAG_CAN_DMA48;
1621         } else
1622                 softc->flags &= ~(ADA_FLAG_CAN_48BIT | ADA_FLAG_CAN_DMA48);
1623
1624         if (cgd->ident_data.support.command2 & ATA_SUPPORT_FLUSHCACHE)
1625                 softc->flags |= ADA_FLAG_CAN_FLUSHCACHE;
1626         else
1627                 softc->flags &= ~ADA_FLAG_CAN_FLUSHCACHE;
1628
1629         if (cgd->ident_data.support.command1 & ATA_SUPPORT_POWERMGT)
1630                 softc->flags |= ADA_FLAG_CAN_POWERMGT;
1631         else
1632                 softc->flags &= ~ADA_FLAG_CAN_POWERMGT;
1633
1634         if ((cgd->ident_data.satacapabilities & ATA_SUPPORT_NCQ) &&
1635             (cgd->inq_flags & SID_DMA) && (cgd->inq_flags & SID_CmdQue))
1636                 softc->flags |= ADA_FLAG_CAN_NCQ;
1637         else
1638                 softc->flags &= ~ADA_FLAG_CAN_NCQ;
1639
1640         if ((cgd->ident_data.support_dsm & ATA_SUPPORT_DSM_TRIM) &&
1641             (cgd->inq_flags & SID_DMA)) {
1642                 softc->flags |= ADA_FLAG_CAN_TRIM;
1643                 softc->trim_max_ranges = TRIM_MAX_RANGES;
1644                 if (cgd->ident_data.max_dsm_blocks != 0) {
1645                         softc->trim_max_ranges =
1646                             min(cgd->ident_data.max_dsm_blocks *
1647                                 ATA_DSM_BLK_RANGES, softc->trim_max_ranges);
1648                 }
1649                 /*
1650                  * If we can do RCVSND_FPDMA_QUEUED commands, we may be able
1651                  * to do NCQ trims, if we support trims at all. We also need
1652                  * support from the SIM to do things properly. Perhaps we
1653                  * should look at log 13 dword 0 bit 0 and dword 1 bit 0 are
1654                  * set too...
1655                  */
1656                 if ((softc->quirks & ADA_Q_NCQ_TRIM_BROKEN) == 0 &&
1657                     (softc->flags & ADA_FLAG_PIM_ATA_EXT) != 0 &&
1658                     (cgd->ident_data.satacapabilities2 &
1659                      ATA_SUPPORT_RCVSND_FPDMA_QUEUED) != 0 &&
1660                     (softc->flags & ADA_FLAG_CAN_TRIM) != 0)
1661                         softc->flags |= ADA_FLAG_CAN_NCQ_TRIM;
1662                 else
1663                         softc->flags &= ~ADA_FLAG_CAN_NCQ_TRIM;
1664         } else
1665                 softc->flags &= ~(ADA_FLAG_CAN_TRIM | ADA_FLAG_CAN_NCQ_TRIM);
1666
1667         if (cgd->ident_data.support.command2 & ATA_SUPPORT_CFA)
1668                 softc->flags |= ADA_FLAG_CAN_CFA;
1669         else
1670                 softc->flags &= ~ADA_FLAG_CAN_CFA;
1671
1672         /*
1673          * Now that we've set the appropriate flags, setup the delete
1674          * method.
1675          */
1676         adasetdeletemethod(softc);
1677
1678         if ((cgd->ident_data.support.extension & ATA_SUPPORT_GENLOG)
1679          && ((softc->quirks & ADA_Q_LOG_BROKEN) == 0))
1680                 softc->flags |= ADA_FLAG_CAN_LOG;
1681         else
1682                 softc->flags &= ~ADA_FLAG_CAN_LOG;
1683
1684         if ((cgd->ident_data.support3 & ATA_SUPPORT_ZONE_MASK) ==
1685              ATA_SUPPORT_ZONE_HOST_AWARE)
1686                 softc->zone_mode = ADA_ZONE_HOST_AWARE;
1687         else if (((cgd->ident_data.support3 & ATA_SUPPORT_ZONE_MASK) ==
1688                    ATA_SUPPORT_ZONE_DEV_MANAGED)
1689               || (softc->quirks & ADA_Q_SMR_DM))
1690                 softc->zone_mode = ADA_ZONE_DRIVE_MANAGED;
1691         else
1692                 softc->zone_mode = ADA_ZONE_NONE;
1693
1694         if (cgd->ident_data.support.command1 & ATA_SUPPORT_LOOKAHEAD)
1695                 softc->flags |= ADA_FLAG_CAN_RAHEAD;
1696         else
1697                 softc->flags &= ~ADA_FLAG_CAN_RAHEAD;
1698
1699         if (cgd->ident_data.support.command1 & ATA_SUPPORT_WRITECACHE)
1700                 softc->flags |= ADA_FLAG_CAN_WCACHE;
1701         else
1702                 softc->flags &= ~ADA_FLAG_CAN_WCACHE;
1703 }
1704
1705 static cam_status
1706 adaregister(struct cam_periph *periph, void *arg)
1707 {
1708         struct ada_softc *softc;
1709         struct ccb_pathinq cpi;
1710         struct ccb_getdev *cgd;
1711         struct disk_params *dp;
1712         struct sbuf sb;
1713         char   *announce_buf;
1714         caddr_t match;
1715         u_int maxio;
1716         int quirks;
1717
1718         cgd = (struct ccb_getdev *)arg;
1719         if (cgd == NULL) {
1720                 printf("adaregister: no getdev CCB, can't register device\n");
1721                 return(CAM_REQ_CMP_ERR);
1722         }
1723
1724         softc = (struct ada_softc *)malloc(sizeof(*softc), M_DEVBUF,
1725             M_NOWAIT|M_ZERO);
1726
1727         if (softc == NULL) {
1728                 printf("adaregister: Unable to probe new device. "
1729                     "Unable to allocate softc\n");
1730                 return(CAM_REQ_CMP_ERR);
1731         }
1732
1733         announce_buf = softc->announce_temp;
1734         bzero(announce_buf, ADA_ANNOUNCETMP_SZ);
1735
1736         if (cam_iosched_init(&softc->cam_iosched, periph) != 0) {
1737                 printf("adaregister: Unable to probe new device. "
1738                        "Unable to allocate iosched memory\n");
1739                 free(softc, M_DEVBUF);
1740                 return(CAM_REQ_CMP_ERR);
1741         }
1742
1743         periph->softc = softc;
1744
1745         /*
1746          * See if this device has any quirks.
1747          */
1748         match = cam_quirkmatch((caddr_t)&cgd->ident_data,
1749                                (caddr_t)ada_quirk_table,
1750                                nitems(ada_quirk_table),
1751                                sizeof(*ada_quirk_table), ata_identify_match);
1752         if (match != NULL)
1753                 softc->quirks = ((struct ada_quirk_entry *)match)->quirks;
1754         else
1755                 softc->quirks = ADA_Q_NONE;
1756
1757         xpt_path_inq(&cpi, periph->path);
1758
1759         TASK_INIT(&softc->sysctl_task, 0, adasysctlinit, periph);
1760
1761         /*
1762          * Register this media as a disk
1763          */
1764         (void)cam_periph_hold(periph, PRIBIO);
1765         cam_periph_unlock(periph);
1766         snprintf(announce_buf, ADA_ANNOUNCETMP_SZ,
1767             "kern.cam.ada.%d.quirks", periph->unit_number);
1768         quirks = softc->quirks;
1769         TUNABLE_INT_FETCH(announce_buf, &quirks);
1770         softc->quirks = quirks;
1771         softc->read_ahead = -1;
1772         snprintf(announce_buf, ADA_ANNOUNCETMP_SZ,
1773             "kern.cam.ada.%d.read_ahead", periph->unit_number);
1774         TUNABLE_INT_FETCH(announce_buf, &softc->read_ahead);
1775         softc->write_cache = -1;
1776         snprintf(announce_buf, ADA_ANNOUNCETMP_SZ,
1777             "kern.cam.ada.%d.write_cache", periph->unit_number);
1778         TUNABLE_INT_FETCH(announce_buf, &softc->write_cache);
1779
1780         /*
1781          * Set support flags based on the Identify data and quirks.
1782          */
1783         adasetflags(softc, cgd);
1784
1785         /* Disable queue sorting for non-rotational media by default. */
1786         if (cgd->ident_data.media_rotation_rate == ATA_RATE_NON_ROTATING) {
1787                 softc->rotating = 0;
1788         } else {
1789                 softc->rotating = 1;
1790         }
1791         cam_iosched_set_sort_queue(softc->cam_iosched,  softc->rotating ? -1 : 0);
1792         adagetparams(periph, cgd);
1793         softc->disk = disk_alloc();
1794         softc->disk->d_rotation_rate = cgd->ident_data.media_rotation_rate;
1795         softc->disk->d_devstat = devstat_new_entry(periph->periph_name,
1796                           periph->unit_number, softc->params.secsize,
1797                           DEVSTAT_ALL_SUPPORTED,
1798                           DEVSTAT_TYPE_DIRECT |
1799                           XPORT_DEVSTAT_TYPE(cpi.transport),
1800                           DEVSTAT_PRIORITY_DISK);
1801         softc->disk->d_open = adaopen;
1802         softc->disk->d_close = adaclose;
1803         softc->disk->d_strategy = adastrategy;
1804         softc->disk->d_getattr = adagetattr;
1805         softc->disk->d_dump = adadump;
1806         softc->disk->d_gone = adadiskgonecb;
1807         softc->disk->d_name = "ada";
1808         softc->disk->d_drv1 = periph;
1809         maxio = cpi.maxio;              /* Honor max I/O size of SIM */
1810         if (maxio == 0)
1811                 maxio = DFLTPHYS;       /* traditional default */
1812         else if (maxio > MAXPHYS)
1813                 maxio = MAXPHYS;        /* for safety */
1814         if (softc->flags & ADA_FLAG_CAN_48BIT)
1815                 maxio = min(maxio, 65536 * softc->params.secsize);
1816         else                                    /* 28bit ATA command limit */
1817                 maxio = min(maxio, 256 * softc->params.secsize);
1818         softc->disk->d_maxsize = maxio;
1819         softc->disk->d_unit = periph->unit_number;
1820         softc->disk->d_flags = DISKFLAG_DIRECT_COMPLETION | DISKFLAG_CANZONE;
1821         if (softc->flags & ADA_FLAG_CAN_FLUSHCACHE)
1822                 softc->disk->d_flags |= DISKFLAG_CANFLUSHCACHE;
1823         /* Device lies about TRIM capability. */
1824         if ((softc->quirks & ADA_Q_NO_TRIM) &&
1825             (softc->flags & ADA_FLAG_CAN_TRIM))
1826                 softc->flags &= ~ADA_FLAG_CAN_TRIM;
1827         if (softc->flags & ADA_FLAG_CAN_TRIM) {
1828                 softc->disk->d_flags |= DISKFLAG_CANDELETE;
1829                 softc->disk->d_delmaxsize = softc->params.secsize *
1830                                             ATA_DSM_RANGE_MAX *
1831                                             softc->trim_max_ranges;
1832         } else if ((softc->flags & ADA_FLAG_CAN_CFA) &&
1833             !(softc->flags & ADA_FLAG_CAN_48BIT)) {
1834                 softc->disk->d_flags |= DISKFLAG_CANDELETE;
1835                 softc->disk->d_delmaxsize = 256 * softc->params.secsize;
1836         } else
1837                 softc->disk->d_delmaxsize = maxio;
1838         if ((cpi.hba_misc & PIM_UNMAPPED) != 0) {
1839                 softc->disk->d_flags |= DISKFLAG_UNMAPPED_BIO;
1840                 softc->unmappedio = 1;
1841         }
1842         if (cpi.hba_misc & PIM_ATA_EXT)
1843                 softc->flags |= ADA_FLAG_PIM_ATA_EXT;
1844         strlcpy(softc->disk->d_descr, cgd->ident_data.model,
1845             MIN(sizeof(softc->disk->d_descr), sizeof(cgd->ident_data.model)));
1846         strlcpy(softc->disk->d_ident, cgd->ident_data.serial,
1847             MIN(sizeof(softc->disk->d_ident), sizeof(cgd->ident_data.serial)));
1848         softc->disk->d_hba_vendor = cpi.hba_vendor;
1849         softc->disk->d_hba_device = cpi.hba_device;
1850         softc->disk->d_hba_subvendor = cpi.hba_subvendor;
1851         softc->disk->d_hba_subdevice = cpi.hba_subdevice;
1852
1853         softc->disk->d_sectorsize = softc->params.secsize;
1854         softc->disk->d_mediasize = (off_t)softc->params.sectors *
1855             softc->params.secsize;
1856         if (ata_physical_sector_size(&cgd->ident_data) !=
1857             softc->params.secsize) {
1858                 softc->disk->d_stripesize =
1859                     ata_physical_sector_size(&cgd->ident_data);
1860                 softc->disk->d_stripeoffset = (softc->disk->d_stripesize -
1861                     ata_logical_sector_offset(&cgd->ident_data)) %
1862                     softc->disk->d_stripesize;
1863         } else if (softc->quirks & ADA_Q_4K) {
1864                 softc->disk->d_stripesize = 4096;
1865                 softc->disk->d_stripeoffset = 0;
1866         }
1867         softc->disk->d_fwsectors = softc->params.secs_per_track;
1868         softc->disk->d_fwheads = softc->params.heads;
1869         ata_disk_firmware_geom_adjust(softc->disk);
1870
1871         /*
1872          * Acquire a reference to the periph before we register with GEOM.
1873          * We'll release this reference once GEOM calls us back (via
1874          * adadiskgonecb()) telling us that our provider has been freed.
1875          */
1876         if (cam_periph_acquire(periph) != 0) {
1877                 xpt_print(periph->path, "%s: lost periph during "
1878                           "registration!\n", __func__);
1879                 cam_periph_lock(periph);
1880                 return (CAM_REQ_CMP_ERR);
1881         }
1882         disk_create(softc->disk, DISK_VERSION);
1883         cam_periph_lock(periph);
1884
1885         dp = &softc->params;
1886         snprintf(announce_buf, ADA_ANNOUNCETMP_SZ,
1887             "%juMB (%ju %u byte sectors)",
1888             ((uintmax_t)dp->secsize * dp->sectors) / (1024 * 1024),
1889             (uintmax_t)dp->sectors, dp->secsize);
1890
1891         sbuf_new(&sb, softc->announce_buffer, ADA_ANNOUNCE_SZ, SBUF_FIXEDLEN);
1892         xpt_announce_periph_sbuf(periph, &sb, announce_buf);
1893         xpt_announce_quirks_sbuf(periph, &sb, softc->quirks, ADA_Q_BIT_STRING);
1894         sbuf_finish(&sb);
1895         sbuf_putbuf(&sb);
1896
1897         /*
1898          * Create our sysctl variables, now that we know
1899          * we have successfully attached.
1900          */
1901         if (cam_periph_acquire(periph) == 0)
1902                 taskqueue_enqueue(taskqueue_thread, &softc->sysctl_task);
1903
1904         /*
1905          * Add async callbacks for bus reset and
1906          * bus device reset calls.  I don't bother
1907          * checking if this fails as, in most cases,
1908          * the system will function just fine without
1909          * them and the only alternative would be to
1910          * not attach the device on failure.
1911          */
1912         xpt_register_async(AC_SENT_BDR | AC_BUS_RESET | AC_LOST_DEVICE |
1913             AC_GETDEV_CHANGED | AC_ADVINFO_CHANGED,
1914             adaasync, periph, periph->path);
1915
1916         /*
1917          * Schedule a periodic event to occasionally send an
1918          * ordered tag to a device.
1919          */
1920         callout_init_mtx(&softc->sendordered_c, cam_periph_mtx(periph), 0);
1921         callout_reset(&softc->sendordered_c,
1922             (ada_default_timeout * hz) / ADA_ORDEREDTAG_INTERVAL,
1923             adasendorderedtag, softc);
1924
1925         if (ADA_RA >= 0 && softc->flags & ADA_FLAG_CAN_RAHEAD) {
1926                 softc->state = ADA_STATE_RAHEAD;
1927         } else if (ADA_WC >= 0 && softc->flags & ADA_FLAG_CAN_WCACHE) {
1928                 softc->state = ADA_STATE_WCACHE;
1929         } else if ((softc->flags & ADA_FLAG_CAN_LOG)
1930                 && (softc->zone_mode != ADA_ZONE_NONE)) {
1931                 softc->state = ADA_STATE_LOGDIR;
1932         } else {
1933                 /*
1934                  * Nothing to probe, so we can just transition to the
1935                  * normal state.
1936                  */
1937                 adaprobedone(periph, NULL);
1938                 return(CAM_REQ_CMP);
1939         }
1940
1941         xpt_schedule(periph, CAM_PRIORITY_DEV);
1942
1943         return(CAM_REQ_CMP);
1944 }
1945
1946 static int
1947 ada_dsmtrim_req_create(struct ada_softc *softc, struct bio *bp, struct trim_request *req)
1948 {
1949         uint64_t lastlba = (uint64_t)-1, lbas = 0;
1950         int c, lastcount = 0, off, ranges = 0;
1951
1952         bzero(req, sizeof(*req));
1953         TAILQ_INIT(&req->bps);
1954         do {
1955                 uint64_t lba = bp->bio_pblkno;
1956                 int count = bp->bio_bcount / softc->params.secsize;
1957
1958                 /* Try to extend the previous range. */
1959                 if (lba == lastlba) {
1960                         c = min(count, ATA_DSM_RANGE_MAX - lastcount);
1961                         lastcount += c;
1962                         off = (ranges - 1) * ATA_DSM_RANGE_SIZE;
1963                         req->data[off + 6] = lastcount & 0xff;
1964                         req->data[off + 7] =
1965                                 (lastcount >> 8) & 0xff;
1966                         count -= c;
1967                         lba += c;
1968                         lbas += c;
1969                 }
1970
1971                 while (count > 0) {
1972                         c = min(count, ATA_DSM_RANGE_MAX);
1973                         off = ranges * ATA_DSM_RANGE_SIZE;
1974                         req->data[off + 0] = lba & 0xff;
1975                         req->data[off + 1] = (lba >> 8) & 0xff;
1976                         req->data[off + 2] = (lba >> 16) & 0xff;
1977                         req->data[off + 3] = (lba >> 24) & 0xff;
1978                         req->data[off + 4] = (lba >> 32) & 0xff;
1979                         req->data[off + 5] = (lba >> 40) & 0xff;
1980                         req->data[off + 6] = c & 0xff;
1981                         req->data[off + 7] = (c >> 8) & 0xff;
1982                         lba += c;
1983                         lbas += c;
1984                         count -= c;
1985                         lastcount = c;
1986                         ranges++;
1987                         /*
1988                          * Its the caller's responsibility to ensure the
1989                          * request will fit so we don't need to check for
1990                          * overrun here
1991                          */
1992                 }
1993                 lastlba = lba;
1994                 TAILQ_INSERT_TAIL(&req->bps, bp, bio_queue);
1995
1996                 bp = cam_iosched_next_trim(softc->cam_iosched);
1997                 if (bp == NULL)
1998                         break;
1999                 if (bp->bio_bcount / softc->params.secsize >
2000                     (softc->trim_max_ranges - ranges) * ATA_DSM_RANGE_MAX) {
2001                         cam_iosched_put_back_trim(softc->cam_iosched, bp);
2002                         break;
2003                 }
2004         } while (1);
2005         softc->trim_count++;
2006         softc->trim_ranges += ranges;
2007         softc->trim_lbas += lbas;
2008
2009         return (ranges);
2010 }
2011
2012 static void
2013 ada_dsmtrim(struct ada_softc *softc, struct bio *bp, struct ccb_ataio *ataio)
2014 {
2015         struct trim_request *req = &softc->trim_req;
2016         int ranges;
2017
2018         ranges = ada_dsmtrim_req_create(softc, bp, req);
2019         cam_fill_ataio(ataio,
2020             ada_retry_count,
2021             adadone,
2022             CAM_DIR_OUT,
2023             0,
2024             req->data,
2025             howmany(ranges, ATA_DSM_BLK_RANGES) * ATA_DSM_BLK_SIZE,
2026             ada_default_timeout * 1000);
2027         ata_48bit_cmd(ataio, ATA_DATA_SET_MANAGEMENT,
2028             ATA_DSM_TRIM, 0, howmany(ranges, ATA_DSM_BLK_RANGES));
2029 }
2030
2031 static void
2032 ada_ncq_dsmtrim(struct ada_softc *softc, struct bio *bp, struct ccb_ataio *ataio)
2033 {
2034         struct trim_request *req = &softc->trim_req;
2035         int ranges;
2036
2037         ranges = ada_dsmtrim_req_create(softc, bp, req);
2038         cam_fill_ataio(ataio,
2039             ada_retry_count,
2040             adadone,
2041             CAM_DIR_OUT,
2042             0,
2043             req->data,
2044             howmany(ranges, ATA_DSM_BLK_RANGES) * ATA_DSM_BLK_SIZE,
2045             ada_default_timeout * 1000);
2046         ata_ncq_cmd(ataio,
2047             ATA_SEND_FPDMA_QUEUED,
2048             0,
2049             howmany(ranges, ATA_DSM_BLK_RANGES));
2050         ataio->cmd.sector_count_exp = ATA_SFPDMA_DSM;
2051         ataio->ata_flags |= ATA_FLAG_AUX;
2052         ataio->aux = 1;
2053 }
2054
2055 static void
2056 ada_cfaerase(struct ada_softc *softc, struct bio *bp, struct ccb_ataio *ataio)
2057 {
2058         struct trim_request *req = &softc->trim_req;
2059         uint64_t lba = bp->bio_pblkno;
2060         uint16_t count = bp->bio_bcount / softc->params.secsize;
2061
2062         bzero(req, sizeof(*req));
2063         TAILQ_INIT(&req->bps);
2064         TAILQ_INSERT_TAIL(&req->bps, bp, bio_queue);
2065
2066         cam_fill_ataio(ataio,
2067             ada_retry_count,
2068             adadone,
2069             CAM_DIR_NONE,
2070             0,
2071             NULL,
2072             0,
2073             ada_default_timeout*1000);
2074
2075         if (count >= 256)
2076                 count = 0;
2077         ata_28bit_cmd(ataio, ATA_CFA_ERASE, 0, lba, count);
2078 }
2079
2080 static int
2081 ada_zone_bio_to_ata(int disk_zone_cmd)
2082 {
2083         switch (disk_zone_cmd) {
2084         case DISK_ZONE_OPEN:
2085                 return ATA_ZM_OPEN_ZONE;
2086         case DISK_ZONE_CLOSE:
2087                 return ATA_ZM_CLOSE_ZONE;
2088         case DISK_ZONE_FINISH:
2089                 return ATA_ZM_FINISH_ZONE;
2090         case DISK_ZONE_RWP:
2091                 return ATA_ZM_RWP;
2092         }
2093
2094         return -1;
2095 }
2096
2097 static int
2098 ada_zone_cmd(struct cam_periph *periph, union ccb *ccb, struct bio *bp,
2099              int *queue_ccb)
2100 {
2101         struct ada_softc *softc;
2102         int error;
2103
2104         error = 0;
2105
2106         if (bp->bio_cmd != BIO_ZONE) {
2107                 error = EINVAL;
2108                 goto bailout;
2109         }
2110
2111         softc = periph->softc;
2112
2113         switch (bp->bio_zone.zone_cmd) {
2114         case DISK_ZONE_OPEN:
2115         case DISK_ZONE_CLOSE:
2116         case DISK_ZONE_FINISH:
2117         case DISK_ZONE_RWP: {
2118                 int zone_flags;
2119                 int zone_sa;
2120                 uint64_t lba;
2121
2122                 zone_sa = ada_zone_bio_to_ata(bp->bio_zone.zone_cmd);
2123                 if (zone_sa == -1) {
2124                         xpt_print(periph->path, "Cannot translate zone "
2125                             "cmd %#x to ATA\n", bp->bio_zone.zone_cmd);
2126                         error = EINVAL;
2127                         goto bailout;
2128                 }
2129
2130                 zone_flags = 0;
2131                 lba = bp->bio_zone.zone_params.rwp.id;
2132
2133                 if (bp->bio_zone.zone_params.rwp.flags &
2134                     DISK_ZONE_RWP_FLAG_ALL)
2135                         zone_flags |= ZBC_OUT_ALL;
2136
2137                 ata_zac_mgmt_out(&ccb->ataio,
2138                                  /*retries*/ ada_retry_count,
2139                                  /*cbfcnp*/ adadone,
2140                                  /*use_ncq*/ (softc->flags &
2141                                               ADA_FLAG_PIM_ATA_EXT) ? 1 : 0,
2142                                  /*zm_action*/ zone_sa,
2143                                  /*zone_id*/ lba,
2144                                  /*zone_flags*/ zone_flags,
2145                                  /*sector_count*/ 0,
2146                                  /*data_ptr*/ NULL,
2147                                  /*dxfer_len*/ 0,
2148                                  /*timeout*/ ada_default_timeout * 1000);
2149                 *queue_ccb = 1;
2150
2151                 break;
2152         }
2153         case DISK_ZONE_REPORT_ZONES: {
2154                 uint8_t *rz_ptr;
2155                 uint32_t num_entries, alloc_size;
2156                 struct disk_zone_report *rep;
2157
2158                 rep = &bp->bio_zone.zone_params.report;
2159
2160                 num_entries = rep->entries_allocated;
2161                 if (num_entries == 0) {
2162                         xpt_print(periph->path, "No entries allocated for "
2163                             "Report Zones request\n");
2164                         error = EINVAL;
2165                         goto bailout;
2166                 }
2167                 alloc_size = sizeof(struct scsi_report_zones_hdr) +
2168                     (sizeof(struct scsi_report_zones_desc) * num_entries);
2169                 alloc_size = min(alloc_size, softc->disk->d_maxsize);
2170                 rz_ptr = malloc(alloc_size, M_ATADA, M_NOWAIT | M_ZERO);
2171                 if (rz_ptr == NULL) {
2172                         xpt_print(periph->path, "Unable to allocate memory "
2173                            "for Report Zones request\n");
2174                         error = ENOMEM;
2175                         goto bailout;
2176                 }
2177
2178                 ata_zac_mgmt_in(&ccb->ataio,
2179                                 /*retries*/ ada_retry_count,
2180                                 /*cbcfnp*/ adadone,
2181                                 /*use_ncq*/ (softc->flags &
2182                                              ADA_FLAG_PIM_ATA_EXT) ? 1 : 0,
2183                                 /*zm_action*/ ATA_ZM_REPORT_ZONES,
2184                                 /*zone_id*/ rep->starting_id,
2185                                 /*zone_flags*/ rep->rep_options,
2186                                 /*data_ptr*/ rz_ptr,
2187                                 /*dxfer_len*/ alloc_size,
2188                                 /*timeout*/ ada_default_timeout * 1000);
2189
2190                 /*
2191                  * For BIO_ZONE, this isn't normally needed.  However, it
2192                  * is used by devstat_end_transaction_bio() to determine
2193                  * how much data was transferred.
2194                  */
2195                 /*
2196                  * XXX KDM we have a problem.  But I'm not sure how to fix
2197                  * it.  devstat uses bio_bcount - bio_resid to calculate
2198                  * the amount of data transferred.   The GEOM disk code
2199                  * uses bio_length - bio_resid to calculate the amount of
2200                  * data in bio_completed.  We have different structure
2201                  * sizes above and below the ada(4) driver.  So, if we
2202                  * use the sizes above, the amount transferred won't be
2203                  * quite accurate for devstat.  If we use different sizes
2204                  * for bio_bcount and bio_length (above and below
2205                  * respectively), then the residual needs to match one or
2206                  * the other.  Everything is calculated after the bio
2207                  * leaves the driver, so changing the values around isn't
2208                  * really an option.  For now, just set the count to the
2209                  * passed in length.  This means that the calculations
2210                  * above (e.g. bio_completed) will be correct, but the
2211                  * amount of data reported to devstat will be slightly
2212                  * under or overstated.
2213                  */
2214                 bp->bio_bcount = bp->bio_length;
2215
2216                 *queue_ccb = 1;
2217
2218                 break;
2219         }
2220         case DISK_ZONE_GET_PARAMS: {
2221                 struct disk_zone_disk_params *params;
2222
2223                 params = &bp->bio_zone.zone_params.disk_params;
2224                 bzero(params, sizeof(*params));
2225
2226                 switch (softc->zone_mode) {
2227                 case ADA_ZONE_DRIVE_MANAGED:
2228                         params->zone_mode = DISK_ZONE_MODE_DRIVE_MANAGED;
2229                         break;
2230                 case ADA_ZONE_HOST_AWARE:
2231                         params->zone_mode = DISK_ZONE_MODE_HOST_AWARE;
2232                         break;
2233                 case ADA_ZONE_HOST_MANAGED:
2234                         params->zone_mode = DISK_ZONE_MODE_HOST_MANAGED;
2235                         break;
2236                 default:
2237                 case ADA_ZONE_NONE:
2238                         params->zone_mode = DISK_ZONE_MODE_NONE;
2239                         break;
2240                 }
2241
2242                 if (softc->zone_flags & ADA_ZONE_FLAG_URSWRZ)
2243                         params->flags |= DISK_ZONE_DISK_URSWRZ;
2244
2245                 if (softc->zone_flags & ADA_ZONE_FLAG_OPT_SEQ_SET) {
2246                         params->optimal_seq_zones = softc->optimal_seq_zones;
2247                         params->flags |= DISK_ZONE_OPT_SEQ_SET;
2248                 }
2249
2250                 if (softc->zone_flags & ADA_ZONE_FLAG_OPT_NONSEQ_SET) {
2251                         params->optimal_nonseq_zones =
2252                             softc->optimal_nonseq_zones;
2253                         params->flags |= DISK_ZONE_OPT_NONSEQ_SET;
2254                 }
2255
2256                 if (softc->zone_flags & ADA_ZONE_FLAG_MAX_SEQ_SET) {
2257                         params->max_seq_zones = softc->max_seq_zones;
2258                         params->flags |= DISK_ZONE_MAX_SEQ_SET;
2259                 }
2260                 if (softc->zone_flags & ADA_ZONE_FLAG_RZ_SUP)
2261                         params->flags |= DISK_ZONE_RZ_SUP;
2262
2263                 if (softc->zone_flags & ADA_ZONE_FLAG_OPEN_SUP)
2264                         params->flags |= DISK_ZONE_OPEN_SUP;
2265
2266                 if (softc->zone_flags & ADA_ZONE_FLAG_CLOSE_SUP)
2267                         params->flags |= DISK_ZONE_CLOSE_SUP;
2268
2269                 if (softc->zone_flags & ADA_ZONE_FLAG_FINISH_SUP)
2270                         params->flags |= DISK_ZONE_FINISH_SUP;
2271
2272                 if (softc->zone_flags & ADA_ZONE_FLAG_RWP_SUP)
2273                         params->flags |= DISK_ZONE_RWP_SUP;
2274                 break;
2275         }
2276         default:
2277                 break;
2278         }
2279 bailout:
2280         return (error);
2281 }
2282
2283 static void
2284 adastart(struct cam_periph *periph, union ccb *start_ccb)
2285 {
2286         struct ada_softc *softc = (struct ada_softc *)periph->softc;
2287         struct ccb_ataio *ataio = &start_ccb->ataio;
2288
2289         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("adastart\n"));
2290
2291         switch (softc->state) {
2292         case ADA_STATE_NORMAL:
2293         {
2294                 struct bio *bp;
2295                 u_int8_t tag_code;
2296
2297                 bp = cam_iosched_next_bio(softc->cam_iosched);
2298                 if (bp == NULL) {
2299                         xpt_release_ccb(start_ccb);
2300                         break;
2301                 }
2302
2303                 if ((bp->bio_flags & BIO_ORDERED) != 0 ||
2304                     (bp->bio_cmd != BIO_DELETE && (softc->flags & ADA_FLAG_NEED_OTAG) != 0)) {
2305                         softc->flags &= ~ADA_FLAG_NEED_OTAG;
2306                         softc->flags |= ADA_FLAG_WAS_OTAG;
2307                         tag_code = 0;
2308                 } else {
2309                         tag_code = 1;
2310                 }
2311                 switch (bp->bio_cmd) {
2312                 case BIO_WRITE:
2313                 case BIO_READ:
2314                 {
2315                         uint64_t lba = bp->bio_pblkno;
2316                         uint16_t count = bp->bio_bcount / softc->params.secsize;
2317                         void *data_ptr;
2318                         int rw_op;
2319
2320                         if (bp->bio_cmd == BIO_WRITE) {
2321                                 softc->flags |= ADA_FLAG_DIRTY;
2322                                 rw_op = CAM_DIR_OUT;
2323                         } else {
2324                                 rw_op = CAM_DIR_IN;
2325                         }
2326
2327                         data_ptr = bp->bio_data;
2328                         if ((bp->bio_flags & (BIO_UNMAPPED|BIO_VLIST)) != 0) {
2329                                 rw_op |= CAM_DATA_BIO;
2330                                 data_ptr = bp;
2331                         }
2332
2333 #ifdef CAM_TEST_FAILURE
2334                         int fail = 0;
2335
2336                         /*
2337                          * Support the failure ioctls.  If the command is a
2338                          * read, and there are pending forced read errors, or
2339                          * if a write and pending write errors, then fail this
2340                          * operation with EIO.  This is useful for testing
2341                          * purposes.  Also, support having every Nth read fail.
2342                          *
2343                          * This is a rather blunt tool.
2344                          */
2345                         if (bp->bio_cmd == BIO_READ) {
2346                                 if (softc->force_read_error) {
2347                                         softc->force_read_error--;
2348                                         fail = 1;
2349                                 }
2350                                 if (softc->periodic_read_error > 0) {
2351                                         if (++softc->periodic_read_count >=
2352                                             softc->periodic_read_error) {
2353                                                 softc->periodic_read_count = 0;
2354                                                 fail = 1;
2355                                         }
2356                                 }
2357                         } else {
2358                                 if (softc->force_write_error) {
2359                                         softc->force_write_error--;
2360                                         fail = 1;
2361                                 }
2362                         }
2363                         if (fail) {
2364                                 biofinish(bp, NULL, EIO);
2365                                 xpt_release_ccb(start_ccb);
2366                                 adaschedule(periph);
2367                                 return;
2368                         }
2369 #endif
2370                         KASSERT((bp->bio_flags & BIO_UNMAPPED) == 0 ||
2371                             round_page(bp->bio_bcount + bp->bio_ma_offset) /
2372                             PAGE_SIZE == bp->bio_ma_n,
2373                             ("Short bio %p", bp));
2374                         cam_fill_ataio(ataio,
2375                             ada_retry_count,
2376                             adadone,
2377                             rw_op,
2378                             0,
2379                             data_ptr,
2380                             bp->bio_bcount,
2381                             ada_default_timeout*1000);
2382
2383                         if ((softc->flags & ADA_FLAG_CAN_NCQ) && tag_code) {
2384                                 if (bp->bio_cmd == BIO_READ) {
2385                                         ata_ncq_cmd(ataio, ATA_READ_FPDMA_QUEUED,
2386                                             lba, count);
2387                                 } else {
2388                                         ata_ncq_cmd(ataio, ATA_WRITE_FPDMA_QUEUED,
2389                                             lba, count);
2390                                 }
2391                         } else if ((softc->flags & ADA_FLAG_CAN_48BIT) &&
2392                             (lba + count >= ATA_MAX_28BIT_LBA ||
2393                             count > 256)) {
2394                                 if (softc->flags & ADA_FLAG_CAN_DMA48) {
2395                                         if (bp->bio_cmd == BIO_READ) {
2396                                                 ata_48bit_cmd(ataio, ATA_READ_DMA48,
2397                                                     0, lba, count);
2398                                         } else {
2399                                                 ata_48bit_cmd(ataio, ATA_WRITE_DMA48,
2400                                                     0, lba, count);
2401                                         }
2402                                 } else {
2403                                         if (bp->bio_cmd == BIO_READ) {
2404                                                 ata_48bit_cmd(ataio, ATA_READ_MUL48,
2405                                                     0, lba, count);
2406                                         } else {
2407                                                 ata_48bit_cmd(ataio, ATA_WRITE_MUL48,
2408                                                     0, lba, count);
2409                                         }
2410                                 }
2411                         } else {
2412                                 if (count == 256)
2413                                         count = 0;
2414                                 if (softc->flags & ADA_FLAG_CAN_DMA) {
2415                                         if (bp->bio_cmd == BIO_READ) {
2416                                                 ata_28bit_cmd(ataio, ATA_READ_DMA,
2417                                                     0, lba, count);
2418                                         } else {
2419                                                 ata_28bit_cmd(ataio, ATA_WRITE_DMA,
2420                                                     0, lba, count);
2421                                         }
2422                                 } else {
2423                                         if (bp->bio_cmd == BIO_READ) {
2424                                                 ata_28bit_cmd(ataio, ATA_READ_MUL,
2425                                                     0, lba, count);
2426                                         } else {
2427                                                 ata_28bit_cmd(ataio, ATA_WRITE_MUL,
2428                                                     0, lba, count);
2429                                         }
2430                                 }
2431                         }
2432                         break;
2433                 }
2434                 case BIO_DELETE:
2435                         switch (softc->delete_method) {
2436                         case ADA_DELETE_NCQ_DSM_TRIM:
2437                                 ada_ncq_dsmtrim(softc, bp, ataio);
2438                                 break;
2439                         case ADA_DELETE_DSM_TRIM:
2440                                 ada_dsmtrim(softc, bp, ataio);
2441                                 break;
2442                         case ADA_DELETE_CFA_ERASE:
2443                                 ada_cfaerase(softc, bp, ataio);
2444                                 break;
2445                         default:
2446                                 biofinish(bp, NULL, EOPNOTSUPP);
2447                                 xpt_release_ccb(start_ccb);
2448                                 adaschedule(periph);
2449                                 return;
2450                         }
2451                         start_ccb->ccb_h.ccb_state = ADA_CCB_TRIM;
2452                         start_ccb->ccb_h.flags |= CAM_UNLOCKED;
2453                         cam_iosched_submit_trim(softc->cam_iosched);
2454                         goto out;
2455                 case BIO_FLUSH:
2456                         cam_fill_ataio(ataio,
2457                             1,
2458                             adadone,
2459                             CAM_DIR_NONE,
2460                             0,
2461                             NULL,
2462                             0,
2463                             ada_default_timeout*1000);
2464
2465                         if (softc->flags & ADA_FLAG_CAN_48BIT)
2466                                 ata_48bit_cmd(ataio, ATA_FLUSHCACHE48, 0, 0, 0);
2467                         else
2468                                 ata_28bit_cmd(ataio, ATA_FLUSHCACHE, 0, 0, 0);
2469                         break;
2470                 case BIO_ZONE: {
2471                         int error, queue_ccb;
2472
2473                         queue_ccb = 0;
2474
2475                         error = ada_zone_cmd(periph, start_ccb, bp, &queue_ccb);
2476                         if ((error != 0)
2477                          || (queue_ccb == 0)) {
2478                                 biofinish(bp, NULL, error);
2479                                 xpt_release_ccb(start_ccb);
2480                                 return;
2481                         }
2482                         break;
2483                 }
2484                 }
2485                 start_ccb->ccb_h.ccb_state = ADA_CCB_BUFFER_IO;
2486                 start_ccb->ccb_h.flags |= CAM_UNLOCKED;
2487 out:
2488                 start_ccb->ccb_h.ccb_bp = bp;
2489                 softc->outstanding_cmds++;
2490                 softc->refcount++;
2491                 cam_periph_unlock(periph);
2492                 xpt_action(start_ccb);
2493                 cam_periph_lock(periph);
2494
2495                 /* May have more work to do, so ensure we stay scheduled */
2496                 adaschedule(periph);
2497                 break;
2498         }
2499         case ADA_STATE_RAHEAD:
2500         case ADA_STATE_WCACHE:
2501         {
2502                 cam_fill_ataio(ataio,
2503                     1,
2504                     adadone,
2505                     CAM_DIR_NONE,
2506                     0,
2507                     NULL,
2508                     0,
2509                     ada_default_timeout*1000);
2510
2511                 if (softc->state == ADA_STATE_RAHEAD) {
2512                         ata_28bit_cmd(ataio, ATA_SETFEATURES, ADA_RA ?
2513                             ATA_SF_ENAB_RCACHE : ATA_SF_DIS_RCACHE, 0, 0);
2514                         start_ccb->ccb_h.ccb_state = ADA_CCB_RAHEAD;
2515                 } else {
2516                         ata_28bit_cmd(ataio, ATA_SETFEATURES, ADA_WC ?
2517                             ATA_SF_ENAB_WCACHE : ATA_SF_DIS_WCACHE, 0, 0);
2518                         start_ccb->ccb_h.ccb_state = ADA_CCB_WCACHE;
2519                 }
2520                 start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
2521                 xpt_action(start_ccb);
2522                 break;
2523         }
2524         case ADA_STATE_LOGDIR:
2525         {
2526                 struct ata_gp_log_dir *log_dir;
2527
2528                 if ((softc->flags & ADA_FLAG_CAN_LOG) == 0) {
2529                         adaprobedone(periph, start_ccb);
2530                         break;
2531                 }
2532
2533                 log_dir = malloc(sizeof(*log_dir), M_ATADA, M_NOWAIT|M_ZERO);
2534                 if (log_dir == NULL) {
2535                         xpt_print(periph->path, "Couldn't malloc log_dir "
2536                             "data\n");
2537                         softc->state = ADA_STATE_NORMAL;
2538                         xpt_release_ccb(start_ccb);
2539                         break;
2540                 }
2541
2542
2543                 ata_read_log(ataio,
2544                     /*retries*/1,
2545                     /*cbfcnp*/adadone,
2546                     /*log_address*/ ATA_LOG_DIRECTORY,
2547                     /*page_number*/ 0,
2548                     /*block_count*/ 1,
2549                     /*protocol*/ softc->flags & ADA_FLAG_CAN_DMA ?
2550                                  CAM_ATAIO_DMA : 0,
2551                     /*data_ptr*/ (uint8_t *)log_dir,
2552                     /*dxfer_len*/sizeof(*log_dir),
2553                     /*timeout*/ada_default_timeout*1000);
2554
2555                 start_ccb->ccb_h.ccb_state = ADA_CCB_LOGDIR;
2556                 xpt_action(start_ccb);
2557                 break;
2558         }
2559         case ADA_STATE_IDDIR:
2560         {
2561                 struct ata_identify_log_pages *id_dir;
2562
2563                 id_dir = malloc(sizeof(*id_dir), M_ATADA, M_NOWAIT | M_ZERO);
2564                 if (id_dir == NULL) {
2565                         xpt_print(periph->path, "Couldn't malloc id_dir "
2566                             "data\n");
2567                         adaprobedone(periph, start_ccb);
2568                         break;
2569                 }
2570
2571                 ata_read_log(ataio,
2572                     /*retries*/1,
2573                     /*cbfcnp*/adadone,
2574                     /*log_address*/ ATA_IDENTIFY_DATA_LOG,
2575                     /*page_number*/ ATA_IDL_PAGE_LIST,
2576                     /*block_count*/ 1,
2577                     /*protocol*/ softc->flags & ADA_FLAG_CAN_DMA ?
2578                                  CAM_ATAIO_DMA : 0,
2579                     /*data_ptr*/ (uint8_t *)id_dir,
2580                     /*dxfer_len*/ sizeof(*id_dir),
2581                     /*timeout*/ada_default_timeout*1000);
2582
2583                 start_ccb->ccb_h.ccb_state = ADA_CCB_IDDIR;
2584                 xpt_action(start_ccb);
2585                 break;
2586         }
2587         case ADA_STATE_SUP_CAP:
2588         {
2589                 struct ata_identify_log_sup_cap *sup_cap;
2590
2591                 sup_cap = malloc(sizeof(*sup_cap), M_ATADA, M_NOWAIT|M_ZERO);
2592                 if (sup_cap == NULL) {
2593                         xpt_print(periph->path, "Couldn't malloc sup_cap "
2594                             "data\n");
2595                         adaprobedone(periph, start_ccb);
2596                         break;
2597                 }
2598
2599                 ata_read_log(ataio,
2600                     /*retries*/1,
2601                     /*cbfcnp*/adadone,
2602                     /*log_address*/ ATA_IDENTIFY_DATA_LOG,
2603                     /*page_number*/ ATA_IDL_SUP_CAP,
2604                     /*block_count*/ 1,
2605                     /*protocol*/ softc->flags & ADA_FLAG_CAN_DMA ?
2606                                  CAM_ATAIO_DMA : 0,
2607                     /*data_ptr*/ (uint8_t *)sup_cap,
2608                     /*dxfer_len*/ sizeof(*sup_cap),
2609                     /*timeout*/ada_default_timeout*1000);
2610
2611                 start_ccb->ccb_h.ccb_state = ADA_CCB_SUP_CAP;
2612                 xpt_action(start_ccb);
2613                 break;
2614         }
2615         case ADA_STATE_ZONE:
2616         {
2617                 struct ata_zoned_info_log *ata_zone;
2618
2619                 ata_zone = malloc(sizeof(*ata_zone), M_ATADA, M_NOWAIT|M_ZERO);
2620                 if (ata_zone == NULL) {
2621                         xpt_print(periph->path, "Couldn't malloc ata_zone "
2622                             "data\n");
2623                         adaprobedone(periph, start_ccb);
2624                         break;
2625                 }
2626
2627                 ata_read_log(ataio,
2628                     /*retries*/1,
2629                     /*cbfcnp*/adadone,
2630                     /*log_address*/ ATA_IDENTIFY_DATA_LOG,
2631                     /*page_number*/ ATA_IDL_ZDI,
2632                     /*block_count*/ 1,
2633                     /*protocol*/ softc->flags & ADA_FLAG_CAN_DMA ?
2634                                  CAM_ATAIO_DMA : 0,
2635                     /*data_ptr*/ (uint8_t *)ata_zone,
2636                     /*dxfer_len*/ sizeof(*ata_zone),
2637                     /*timeout*/ada_default_timeout*1000);
2638
2639                 start_ccb->ccb_h.ccb_state = ADA_CCB_ZONE;
2640                 xpt_action(start_ccb);
2641                 break;
2642         }
2643         }
2644 }
2645
2646 static void
2647 adaprobedone(struct cam_periph *periph, union ccb *ccb)
2648 {
2649         struct ada_softc *softc;
2650
2651         softc = (struct ada_softc *)periph->softc;
2652
2653         if (ccb != NULL)
2654                 xpt_release_ccb(ccb);
2655
2656         softc->state = ADA_STATE_NORMAL;
2657         softc->flags |= ADA_FLAG_PROBED;
2658         adaschedule(periph);
2659         if ((softc->flags & ADA_FLAG_ANNOUNCED) == 0) {
2660                 softc->flags |= ADA_FLAG_ANNOUNCED;
2661                 cam_periph_unhold(periph);
2662         } else {
2663                 cam_periph_release_locked(periph);
2664         }
2665 }
2666
2667 static void
2668 adazonedone(struct cam_periph *periph, union ccb *ccb)
2669 {
2670         struct bio *bp;
2671
2672         bp = (struct bio *)ccb->ccb_h.ccb_bp;
2673
2674         switch (bp->bio_zone.zone_cmd) {
2675         case DISK_ZONE_OPEN:
2676         case DISK_ZONE_CLOSE:
2677         case DISK_ZONE_FINISH:
2678         case DISK_ZONE_RWP:
2679                 break;
2680         case DISK_ZONE_REPORT_ZONES: {
2681                 uint32_t avail_len;
2682                 struct disk_zone_report *rep;
2683                 struct scsi_report_zones_hdr *hdr;
2684                 struct scsi_report_zones_desc *desc;
2685                 struct disk_zone_rep_entry *entry;
2686                 uint32_t hdr_len, num_avail;
2687                 uint32_t num_to_fill, i;
2688
2689                 rep = &bp->bio_zone.zone_params.report;
2690                 avail_len = ccb->ataio.dxfer_len - ccb->ataio.resid;
2691                 /*
2692                  * Note that bio_resid isn't normally used for zone
2693                  * commands, but it is used by devstat_end_transaction_bio()
2694                  * to determine how much data was transferred.  Because
2695                  * the size of the SCSI/ATA data structures is different
2696                  * than the size of the BIO interface structures, the
2697                  * amount of data actually transferred from the drive will
2698                  * be different than the amount of data transferred to
2699                  * the user.
2700                  */
2701                 hdr = (struct scsi_report_zones_hdr *)ccb->ataio.data_ptr;
2702                 if (avail_len < sizeof(*hdr)) {
2703                         /*
2704                          * Is there a better error than EIO here?  We asked
2705                          * for at least the header, and we got less than
2706                          * that.
2707                          */
2708                         bp->bio_error = EIO;
2709                         bp->bio_flags |= BIO_ERROR;
2710                         bp->bio_resid = bp->bio_bcount;
2711                         break;
2712                 }
2713
2714                 hdr_len = le32dec(hdr->length);
2715                 if (hdr_len > 0)
2716                         rep->entries_available = hdr_len / sizeof(*desc);
2717                 else
2718                         rep->entries_available = 0;
2719                 /*
2720                  * NOTE: using the same values for the BIO version of the
2721                  * same field as the SCSI/ATA values.  This means we could
2722                  * get some additional values that aren't defined in bio.h
2723                  * if more values of the same field are defined later.
2724                  */
2725                 rep->header.same = hdr->byte4 & SRZ_SAME_MASK;
2726                 rep->header.maximum_lba = le64dec(hdr->maximum_lba);
2727                 /*
2728                  * If the drive reports no entries that match the query,
2729                  * we're done.
2730                  */
2731                 if (hdr_len == 0) {
2732                         rep->entries_filled = 0;
2733                         bp->bio_resid = bp->bio_bcount;
2734                         break;
2735                 }
2736
2737                 num_avail = min((avail_len - sizeof(*hdr)) / sizeof(*desc),
2738                                 hdr_len / sizeof(*desc));
2739                 /*
2740                  * If the drive didn't return any data, then we're done.
2741                  */
2742                 if (num_avail == 0) {
2743                         rep->entries_filled = 0;
2744                         bp->bio_resid = bp->bio_bcount;
2745                         break;
2746                 }
2747
2748                 num_to_fill = min(num_avail, rep->entries_allocated);
2749                 /*
2750                  * If the user didn't allocate any entries for us to fill,
2751                  * we're done.
2752                  */
2753                 if (num_to_fill == 0) {
2754                         rep->entries_filled = 0;
2755                         bp->bio_resid = bp->bio_bcount;
2756                         break;
2757                 }
2758
2759                 for (i = 0, desc = &hdr->desc_list[0], entry=&rep->entries[0];
2760                      i < num_to_fill; i++, desc++, entry++) {
2761                         /*
2762                          * NOTE: we're mapping the values here directly
2763                          * from the SCSI/ATA bit definitions to the bio.h
2764                          * definitions.  There is also a warning in
2765                          * disk_zone.h, but the impact is that if
2766                          * additional values are added in the SCSI/ATA
2767                          * specs these will be visible to consumers of
2768                          * this interface.
2769                          */
2770                         entry->zone_type = desc->zone_type & SRZ_TYPE_MASK;
2771                         entry->zone_condition =
2772                             (desc->zone_flags & SRZ_ZONE_COND_MASK) >>
2773                             SRZ_ZONE_COND_SHIFT;
2774                         entry->zone_flags |= desc->zone_flags &
2775                             (SRZ_ZONE_NON_SEQ|SRZ_ZONE_RESET);
2776                         entry->zone_length = le64dec(desc->zone_length);
2777                         entry->zone_start_lba = le64dec(desc->zone_start_lba);
2778                         entry->write_pointer_lba =
2779                             le64dec(desc->write_pointer_lba);
2780                 }
2781                 rep->entries_filled = num_to_fill;
2782                 /*
2783                  * Note that this residual is accurate from the user's
2784                  * standpoint, but the amount transferred isn't accurate
2785                  * from the standpoint of what actually came back from the
2786                  * drive.
2787                  */
2788                 bp->bio_resid = bp->bio_bcount - (num_to_fill * sizeof(*entry));
2789                 break;
2790         }
2791         case DISK_ZONE_GET_PARAMS:
2792         default:
2793                 /*
2794                  * In theory we should not get a GET_PARAMS bio, since it
2795                  * should be handled without queueing the command to the
2796                  * drive.
2797                  */
2798                 panic("%s: Invalid zone command %d", __func__,
2799                     bp->bio_zone.zone_cmd);
2800                 break;
2801         }
2802
2803         if (bp->bio_zone.zone_cmd == DISK_ZONE_REPORT_ZONES)
2804                 free(ccb->ataio.data_ptr, M_ATADA);
2805 }
2806
2807
2808 static void
2809 adadone(struct cam_periph *periph, union ccb *done_ccb)
2810 {
2811         struct ada_softc *softc;
2812         struct ccb_ataio *ataio;
2813         struct cam_path *path;
2814         uint32_t priority;
2815         int state;
2816
2817         softc = (struct ada_softc *)periph->softc;
2818         ataio = &done_ccb->ataio;
2819         path = done_ccb->ccb_h.path;
2820         priority = done_ccb->ccb_h.pinfo.priority;
2821
2822         CAM_DEBUG(path, CAM_DEBUG_TRACE, ("adadone\n"));
2823
2824         state = ataio->ccb_h.ccb_state & ADA_CCB_TYPE_MASK;
2825         switch (state) {
2826         case ADA_CCB_BUFFER_IO:
2827         case ADA_CCB_TRIM:
2828         {
2829                 struct bio *bp;
2830                 int error;
2831
2832                 cam_periph_lock(periph);
2833                 bp = (struct bio *)done_ccb->ccb_h.ccb_bp;
2834                 if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
2835                         error = adaerror(done_ccb, 0, 0);
2836                         if (error == ERESTART) {
2837                                 /* A retry was scheduled, so just return. */
2838                                 cam_periph_unlock(periph);
2839                                 return;
2840                         }
2841                         if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0)
2842                                 cam_release_devq(path,
2843                                                  /*relsim_flags*/0,
2844                                                  /*reduction*/0,
2845                                                  /*timeout*/0,
2846                                                  /*getcount_only*/0);
2847                         /*
2848                          * If we get an error on an NCQ DSM TRIM, fall back
2849                          * to a non-NCQ DSM TRIM forever. Please note that if
2850                          * CAN_NCQ_TRIM is set, CAN_TRIM is necessarily set too.
2851                          * However, for this one trim, we treat it as advisory
2852                          * and return success up the stack.
2853                          */
2854                         if (state == ADA_CCB_TRIM &&
2855                             error != 0 &&
2856                             (softc->flags & ADA_FLAG_CAN_NCQ_TRIM) != 0) {
2857                                 softc->flags &= ~ADA_FLAG_CAN_NCQ_TRIM;
2858                                 error = 0;
2859                                 adasetdeletemethod(softc);
2860                         }
2861                 } else {
2862                         if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0)
2863                                 panic("REQ_CMP with QFRZN");
2864
2865                         error = 0;
2866                 }
2867                 bp->bio_error = error;
2868                 if (error != 0) {
2869                         bp->bio_resid = bp->bio_bcount;
2870                         bp->bio_flags |= BIO_ERROR;
2871                 } else {
2872                         if (bp->bio_cmd == BIO_ZONE)
2873                                 adazonedone(periph, done_ccb);
2874                         else if (state == ADA_CCB_TRIM)
2875                                 bp->bio_resid = 0;
2876                         else
2877                                 bp->bio_resid = ataio->resid;
2878
2879                         if ((bp->bio_resid > 0)
2880                          && (bp->bio_cmd != BIO_ZONE))
2881                                 bp->bio_flags |= BIO_ERROR;
2882                 }
2883                 softc->outstanding_cmds--;
2884                 if (softc->outstanding_cmds == 0)
2885                         softc->flags |= ADA_FLAG_WAS_OTAG;
2886
2887                 /*
2888                  * We need to call cam_iosched before we call biodone so that we
2889                  * don't measure any activity that happens in the completion
2890                  * routine, which in the case of sendfile can be quite
2891                  * extensive.  Release the periph refcount taken in adastart()
2892                  * for each CCB.
2893                  */
2894                 cam_iosched_bio_complete(softc->cam_iosched, bp, done_ccb);
2895                 xpt_release_ccb(done_ccb);
2896                 KASSERT(softc->refcount >= 1, ("adadone softc %p refcount %d", softc, softc->refcount));
2897                 softc->refcount--;
2898                 if (state == ADA_CCB_TRIM) {
2899                         TAILQ_HEAD(, bio) queue;
2900                         struct bio *bp1;
2901
2902                         TAILQ_INIT(&queue);
2903                         TAILQ_CONCAT(&queue, &softc->trim_req.bps, bio_queue);
2904                         /*
2905                          * Normally, the xpt_release_ccb() above would make sure
2906                          * that when we have more work to do, that work would
2907                          * get kicked off. However, we specifically keep
2908                          * trim_running set to 0 before the call above to allow
2909                          * other I/O to progress when many BIO_DELETE requests
2910                          * are pushed down. We set trim_running to 0 and call
2911                          * daschedule again so that we don't stall if there are
2912                          * no other I/Os pending apart from BIO_DELETEs.
2913                          */
2914                         cam_iosched_trim_done(softc->cam_iosched);
2915                         adaschedule(periph);
2916                         cam_periph_unlock(periph);
2917                         while ((bp1 = TAILQ_FIRST(&queue)) != NULL) {
2918                                 TAILQ_REMOVE(&queue, bp1, bio_queue);
2919                                 bp1->bio_error = error;
2920                                 if (error != 0) {
2921                                         bp1->bio_flags |= BIO_ERROR;
2922                                         bp1->bio_resid = bp1->bio_bcount;
2923                                 } else
2924                                         bp1->bio_resid = 0;
2925                                 biodone(bp1);
2926                         }
2927                 } else {
2928                         adaschedule(periph);
2929                         cam_periph_unlock(periph);
2930                         biodone(bp);
2931                 }
2932                 return;
2933         }
2934         case ADA_CCB_RAHEAD:
2935         {
2936                 if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
2937                         if (adaerror(done_ccb, 0, 0) == ERESTART) {
2938                                 /* Drop freeze taken due to CAM_DEV_QFREEZE */
2939                                 cam_release_devq(path, 0, 0, 0, FALSE);
2940                                 return;
2941                         } else if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
2942                                 cam_release_devq(path,
2943                                     /*relsim_flags*/0,
2944                                     /*reduction*/0,
2945                                     /*timeout*/0,
2946                                     /*getcount_only*/0);
2947                         }
2948                 }
2949
2950                 /*
2951                  * Since our peripheral may be invalidated by an error
2952                  * above or an external event, we must release our CCB
2953                  * before releasing the reference on the peripheral.
2954                  * The peripheral will only go away once the last reference
2955                  * is removed, and we need it around for the CCB release
2956                  * operation.
2957                  */
2958
2959                 xpt_release_ccb(done_ccb);
2960                 softc->state = ADA_STATE_WCACHE;
2961                 xpt_schedule(periph, priority);
2962                 /* Drop freeze taken due to CAM_DEV_QFREEZE */
2963                 cam_release_devq(path, 0, 0, 0, FALSE);
2964                 return;
2965         }
2966         case ADA_CCB_WCACHE:
2967         {
2968                 if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
2969                         if (adaerror(done_ccb, 0, 0) == ERESTART) {
2970                                 /* Drop freeze taken due to CAM_DEV_QFREEZE */
2971                                 cam_release_devq(path, 0, 0, 0, FALSE);
2972                                 return;
2973                         } else if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
2974                                 cam_release_devq(path,
2975                                     /*relsim_flags*/0,
2976                                     /*reduction*/0,
2977                                     /*timeout*/0,
2978                                     /*getcount_only*/0);
2979                         }
2980                 }
2981
2982                 /* Drop freeze taken due to CAM_DEV_QFREEZE */
2983                 cam_release_devq(path, 0, 0, 0, FALSE);
2984
2985                 if ((softc->flags & ADA_FLAG_CAN_LOG)
2986                  && (softc->zone_mode != ADA_ZONE_NONE)) {
2987                         xpt_release_ccb(done_ccb);
2988                         softc->state = ADA_STATE_LOGDIR;
2989                         xpt_schedule(periph, priority);
2990                 } else {
2991                         adaprobedone(periph, done_ccb);
2992                 }
2993                 return;
2994         }
2995         case ADA_CCB_LOGDIR:
2996         {
2997                 int error;
2998
2999                 if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
3000                         error = 0;
3001                         softc->valid_logdir_len = 0;
3002                         bzero(&softc->ata_logdir, sizeof(softc->ata_logdir));
3003                         softc->valid_logdir_len =
3004                                 ataio->dxfer_len - ataio->resid;
3005                         if (softc->valid_logdir_len > 0)
3006                                 bcopy(ataio->data_ptr, &softc->ata_logdir,
3007                                     min(softc->valid_logdir_len,
3008                                         sizeof(softc->ata_logdir)));
3009                         /*
3010                          * Figure out whether the Identify Device log is
3011                          * supported.  The General Purpose log directory
3012                          * has a header, and lists the number of pages
3013                          * available for each GP log identified by the
3014                          * offset into the list.
3015                          */
3016                         if ((softc->valid_logdir_len >=
3017                             ((ATA_IDENTIFY_DATA_LOG + 1) * sizeof(uint16_t)))
3018                          && (le16dec(softc->ata_logdir.header) ==
3019                              ATA_GP_LOG_DIR_VERSION)
3020                          && (le16dec(&softc->ata_logdir.num_pages[
3021                              (ATA_IDENTIFY_DATA_LOG *
3022                              sizeof(uint16_t)) - sizeof(uint16_t)]) > 0)){
3023                                 softc->flags |= ADA_FLAG_CAN_IDLOG;
3024                         } else {
3025                                 softc->flags &= ~ADA_FLAG_CAN_IDLOG;
3026                         }
3027                 } else {
3028                         error = adaerror(done_ccb, CAM_RETRY_SELTO,
3029                                          SF_RETRY_UA|SF_NO_PRINT);
3030                         if (error == ERESTART)
3031                                 return;
3032                         else if (error != 0) {
3033                                 /*
3034                                  * If we can't get the ATA log directory,
3035                                  * then ATA logs are effectively not
3036                                  * supported even if the bit is set in the
3037                                  * identify data.
3038                                  */
3039                                 softc->flags &= ~(ADA_FLAG_CAN_LOG |
3040                                                   ADA_FLAG_CAN_IDLOG);
3041                                 if ((done_ccb->ccb_h.status &
3042                                      CAM_DEV_QFRZN) != 0) {
3043                                         /* Don't wedge this device's queue */
3044                                         cam_release_devq(done_ccb->ccb_h.path,
3045                                                          /*relsim_flags*/0,
3046                                                          /*reduction*/0,
3047                                                          /*timeout*/0,
3048                                                          /*getcount_only*/0);
3049                                 }
3050                         }
3051
3052
3053                 }
3054
3055                 free(ataio->data_ptr, M_ATADA);
3056
3057                 if ((error == 0)
3058                  && (softc->flags & ADA_FLAG_CAN_IDLOG)) {
3059                         softc->state = ADA_STATE_IDDIR;
3060                         xpt_release_ccb(done_ccb);
3061                         xpt_schedule(periph, priority);
3062                 } else
3063                         adaprobedone(periph, done_ccb);
3064
3065                 return;
3066         }
3067         case ADA_CCB_IDDIR: {
3068                 int error;
3069
3070                 if ((ataio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
3071                         off_t entries_offset, max_entries;
3072                         error = 0;
3073
3074                         softc->valid_iddir_len = 0;
3075                         bzero(&softc->ata_iddir, sizeof(softc->ata_iddir));
3076                         softc->flags &= ~(ADA_FLAG_CAN_SUPCAP |
3077                                           ADA_FLAG_CAN_ZONE);
3078                         softc->valid_iddir_len =
3079                                 ataio->dxfer_len - ataio->resid;
3080                         if (softc->valid_iddir_len > 0)
3081                                 bcopy(ataio->data_ptr, &softc->ata_iddir,
3082                                     min(softc->valid_iddir_len,
3083                                         sizeof(softc->ata_iddir)));
3084
3085                         entries_offset =
3086                             __offsetof(struct ata_identify_log_pages,entries);
3087                         max_entries = softc->valid_iddir_len - entries_offset;
3088                         if ((softc->valid_iddir_len > (entries_offset + 1))
3089                          && (le64dec(softc->ata_iddir.header) ==
3090                              ATA_IDLOG_REVISION)
3091                          && (softc->ata_iddir.entry_count > 0)) {
3092                                 int num_entries, i;
3093
3094                                 num_entries = softc->ata_iddir.entry_count;
3095                                 num_entries = min(num_entries,
3096                                    softc->valid_iddir_len - entries_offset);
3097                                 for (i = 0; i < num_entries &&
3098                                      i < max_entries; i++) {
3099                                         if (softc->ata_iddir.entries[i] ==
3100                                             ATA_IDL_SUP_CAP)
3101                                                 softc->flags |=
3102                                                     ADA_FLAG_CAN_SUPCAP;
3103                                         else if (softc->ata_iddir.entries[i]==
3104                                                  ATA_IDL_ZDI)
3105                                                 softc->flags |=
3106                                                     ADA_FLAG_CAN_ZONE;
3107
3108                                         if ((softc->flags &
3109                                              ADA_FLAG_CAN_SUPCAP)
3110                                          && (softc->flags &
3111                                              ADA_FLAG_CAN_ZONE))
3112                                                 break;
3113                                 }
3114                         }
3115                 } else {
3116                         error = adaerror(done_ccb, CAM_RETRY_SELTO,
3117                                          SF_RETRY_UA|SF_NO_PRINT);
3118                         if (error == ERESTART)
3119                                 return;
3120                         else if (error != 0) {
3121                                 /*
3122                                  * If we can't get the ATA Identify Data log
3123                                  * directory, then it effectively isn't
3124                                  * supported even if the ATA Log directory
3125                                  * a non-zero number of pages present for
3126                                  * this log.
3127                                  */
3128                                 softc->flags &= ~ADA_FLAG_CAN_IDLOG;
3129                                 if ((done_ccb->ccb_h.status &
3130                                      CAM_DEV_QFRZN) != 0) {
3131                                         /* Don't wedge this device's queue */
3132                                         cam_release_devq(done_ccb->ccb_h.path,
3133                                                          /*relsim_flags*/0,
3134                                                          /*reduction*/0,
3135                                                          /*timeout*/0,
3136                                                          /*getcount_only*/0);
3137                                 }
3138                         }
3139                 }
3140
3141                 free(ataio->data_ptr, M_ATADA);
3142
3143                 if ((error == 0)
3144                  && (softc->flags & ADA_FLAG_CAN_SUPCAP)) {
3145                         softc->state = ADA_STATE_SUP_CAP;
3146                         xpt_release_ccb(done_ccb);
3147                         xpt_schedule(periph, priority);
3148                 } else
3149                         adaprobedone(periph, done_ccb);
3150                 return;
3151         }
3152         case ADA_CCB_SUP_CAP: {
3153                 int error;
3154
3155                 if ((ataio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
3156                         uint32_t valid_len;
3157                         size_t needed_size;
3158                         struct ata_identify_log_sup_cap *sup_cap;
3159                         error = 0;
3160
3161                         sup_cap = (struct ata_identify_log_sup_cap *)
3162                             ataio->data_ptr;
3163                         valid_len = ataio->dxfer_len - ataio->resid;
3164                         needed_size =
3165                             __offsetof(struct ata_identify_log_sup_cap,
3166                             sup_zac_cap) + 1 + sizeof(sup_cap->sup_zac_cap);
3167                         if (valid_len >= needed_size) {
3168                                 uint64_t zoned, zac_cap;
3169
3170                                 zoned = le64dec(sup_cap->zoned_cap);
3171                                 if (zoned & ATA_ZONED_VALID) {
3172                                         /*
3173                                          * This should have already been
3174                                          * set, because this is also in the
3175                                          * ATA identify data.
3176                                          */
3177                                         if ((zoned & ATA_ZONED_MASK) ==
3178                                             ATA_SUPPORT_ZONE_HOST_AWARE)
3179                                                 softc->zone_mode =
3180                                                     ADA_ZONE_HOST_AWARE;
3181                                         else if ((zoned & ATA_ZONED_MASK) ==
3182                                             ATA_SUPPORT_ZONE_DEV_MANAGED)
3183                                                 softc->zone_mode =
3184                                                     ADA_ZONE_DRIVE_MANAGED;
3185                                 }
3186
3187                                 zac_cap = le64dec(sup_cap->sup_zac_cap);
3188                                 if (zac_cap & ATA_SUP_ZAC_CAP_VALID) {
3189                                         if (zac_cap & ATA_REPORT_ZONES_SUP)
3190                                                 softc->zone_flags |=
3191                                                     ADA_ZONE_FLAG_RZ_SUP;
3192                                         if (zac_cap & ATA_ND_OPEN_ZONE_SUP)
3193                                                 softc->zone_flags |=
3194                                                     ADA_ZONE_FLAG_OPEN_SUP;
3195                                         if (zac_cap & ATA_ND_CLOSE_ZONE_SUP)
3196                                                 softc->zone_flags |=
3197                                                     ADA_ZONE_FLAG_CLOSE_SUP;
3198                                         if (zac_cap & ATA_ND_FINISH_ZONE_SUP)
3199                                                 softc->zone_flags |=
3200                                                     ADA_ZONE_FLAG_FINISH_SUP;
3201                                         if (zac_cap & ATA_ND_RWP_SUP)
3202                                                 softc->zone_flags |=
3203                                                     ADA_ZONE_FLAG_RWP_SUP;
3204                                 } else {
3205                                         /*
3206                                          * This field was introduced in
3207                                          * ACS-4, r08 on April 28th, 2015.
3208                                          * If the drive firmware was written
3209                                          * to an earlier spec, it won't have
3210                                          * the field.  So, assume all
3211                                          * commands are supported.
3212                                          */
3213                                         softc->zone_flags |=
3214                                             ADA_ZONE_FLAG_SUP_MASK;
3215                                 }
3216                         }
3217                 } else {
3218                         error = adaerror(done_ccb, CAM_RETRY_SELTO,
3219                                          SF_RETRY_UA|SF_NO_PRINT);
3220                         if (error == ERESTART)
3221                                 return;
3222                         else if (error != 0) {
3223                                 /*
3224                                  * If we can't get the ATA Identify Data
3225                                  * Supported Capabilities page, clear the
3226                                  * flag...
3227                                  */
3228                                 softc->flags &= ~ADA_FLAG_CAN_SUPCAP;
3229                                 /*
3230                                  * And clear zone capabilities.
3231                                  */
3232                                 softc->zone_flags &= ~ADA_ZONE_FLAG_SUP_MASK;
3233                                 if ((done_ccb->ccb_h.status &
3234                                      CAM_DEV_QFRZN) != 0) {
3235                                         /* Don't wedge this device's queue */
3236                                         cam_release_devq(done_ccb->ccb_h.path,
3237                                                          /*relsim_flags*/0,
3238                                                          /*reduction*/0,
3239                                                          /*timeout*/0,
3240                                                          /*getcount_only*/0);
3241                                 }
3242                         }
3243                 }
3244
3245                 free(ataio->data_ptr, M_ATADA);
3246
3247                 if ((error == 0)
3248                  && (softc->flags & ADA_FLAG_CAN_ZONE)) {
3249                         softc->state = ADA_STATE_ZONE;
3250                         xpt_release_ccb(done_ccb);
3251                         xpt_schedule(periph, priority);
3252                 } else
3253                         adaprobedone(periph, done_ccb);
3254                 return;
3255         }
3256         case ADA_CCB_ZONE: {
3257                 int error;
3258
3259                 if ((ataio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
3260                         struct ata_zoned_info_log *zi_log;
3261                         uint32_t valid_len;
3262                         size_t needed_size;
3263
3264                         zi_log = (struct ata_zoned_info_log *)ataio->data_ptr;
3265
3266                         valid_len = ataio->dxfer_len - ataio->resid;
3267                         needed_size = __offsetof(struct ata_zoned_info_log,
3268                             version_info) + 1 + sizeof(zi_log->version_info);
3269                         if (valid_len >= needed_size) {
3270                                 uint64_t tmpvar;
3271
3272                                 tmpvar = le64dec(zi_log->zoned_cap);
3273                                 if (tmpvar & ATA_ZDI_CAP_VALID) {
3274                                         if (tmpvar & ATA_ZDI_CAP_URSWRZ)
3275                                                 softc->zone_flags |=
3276                                                     ADA_ZONE_FLAG_URSWRZ;
3277                                         else
3278                                                 softc->zone_flags &=
3279                                                     ~ADA_ZONE_FLAG_URSWRZ;
3280                                 }
3281                                 tmpvar = le64dec(zi_log->optimal_seq_zones);
3282                                 if (tmpvar & ATA_ZDI_OPT_SEQ_VALID) {
3283                                         softc->zone_flags |=
3284                                             ADA_ZONE_FLAG_OPT_SEQ_SET;
3285                                         softc->optimal_seq_zones = (tmpvar &
3286                                             ATA_ZDI_OPT_SEQ_MASK);
3287                                 } else {
3288                                         softc->zone_flags &=
3289                                             ~ADA_ZONE_FLAG_OPT_SEQ_SET;
3290                                         softc->optimal_seq_zones = 0;
3291                                 }
3292
3293                                 tmpvar =le64dec(zi_log->optimal_nonseq_zones);
3294                                 if (tmpvar & ATA_ZDI_OPT_NS_VALID) {
3295                                         softc->zone_flags |=
3296                                             ADA_ZONE_FLAG_OPT_NONSEQ_SET;
3297                                         softc->optimal_nonseq_zones =
3298                                             (tmpvar & ATA_ZDI_OPT_NS_MASK);
3299                                 } else {
3300                                         softc->zone_flags &=
3301                                             ~ADA_ZONE_FLAG_OPT_NONSEQ_SET;
3302                                         softc->optimal_nonseq_zones = 0;
3303                                 }
3304
3305                                 tmpvar = le64dec(zi_log->max_seq_req_zones);
3306                                 if (tmpvar & ATA_ZDI_MAX_SEQ_VALID) {
3307                                         softc->zone_flags |=
3308                                             ADA_ZONE_FLAG_MAX_SEQ_SET;
3309                                         softc->max_seq_zones =
3310                                             (tmpvar & ATA_ZDI_MAX_SEQ_MASK);
3311                                 } else {
3312                                         softc->zone_flags &=
3313                                             ~ADA_ZONE_FLAG_MAX_SEQ_SET;
3314                                         softc->max_seq_zones = 0;
3315                                 }
3316                         }
3317                 } else {
3318                         error = adaerror(done_ccb, CAM_RETRY_SELTO,
3319                                          SF_RETRY_UA|SF_NO_PRINT);
3320                         if (error == ERESTART)
3321                                 return;
3322                         else if (error != 0) {
3323                                 softc->flags &= ~ADA_FLAG_CAN_ZONE;
3324                                 softc->flags &= ~ADA_ZONE_FLAG_SET_MASK;
3325
3326                                 if ((done_ccb->ccb_h.status &
3327                                      CAM_DEV_QFRZN) != 0) {
3328                                         /* Don't wedge this device's queue */
3329                                         cam_release_devq(done_ccb->ccb_h.path,
3330                                                          /*relsim_flags*/0,
3331                                                          /*reduction*/0,
3332                                                          /*timeout*/0,
3333                                                          /*getcount_only*/0);
3334                                 }
3335                         }
3336                 }
3337                 free(ataio->data_ptr, M_ATADA);
3338
3339                 adaprobedone(periph, done_ccb);
3340                 return;
3341         }
3342         case ADA_CCB_DUMP:
3343                 /* No-op.  We're polling */
3344                 return;
3345         default:
3346                 break;
3347         }
3348         xpt_release_ccb(done_ccb);
3349 }
3350
3351 static int
3352 adaerror(union ccb *ccb, u_int32_t cam_flags, u_int32_t sense_flags)
3353 {
3354 #ifdef CAM_IO_STATS
3355         struct ada_softc *softc;
3356         struct cam_periph *periph;
3357
3358         periph = xpt_path_periph(ccb->ccb_h.path);
3359         softc = (struct ada_softc *)periph->softc;
3360
3361         switch (ccb->ccb_h.status & CAM_STATUS_MASK) {
3362         case CAM_CMD_TIMEOUT:
3363                 softc->timeouts++;
3364                 break;
3365         case CAM_REQ_ABORTED:
3366         case CAM_REQ_CMP_ERR:
3367         case CAM_REQ_TERMIO:
3368         case CAM_UNREC_HBA_ERROR:
3369         case CAM_DATA_RUN_ERR:
3370         case CAM_ATA_STATUS_ERROR:
3371                 softc->errors++;
3372                 break;
3373         default:
3374                 break;
3375         }
3376 #endif
3377
3378         return(cam_periph_error(ccb, cam_flags, sense_flags));
3379 }
3380
3381 static void
3382 adagetparams(struct cam_periph *periph, struct ccb_getdev *cgd)
3383 {
3384         struct ada_softc *softc = (struct ada_softc *)periph->softc;
3385         struct disk_params *dp = &softc->params;
3386         u_int64_t lbasize48;
3387         u_int32_t lbasize;
3388
3389         dp->secsize = ata_logical_sector_size(&cgd->ident_data);
3390         if ((cgd->ident_data.atavalid & ATA_FLAG_54_58) &&
3391                 cgd->ident_data.current_heads && cgd->ident_data.current_sectors) {
3392                 dp->heads = cgd->ident_data.current_heads;
3393                 dp->secs_per_track = cgd->ident_data.current_sectors;
3394                 dp->cylinders = cgd->ident_data.cylinders;
3395                 dp->sectors = (u_int32_t)cgd->ident_data.current_size_1 |
3396                           ((u_int32_t)cgd->ident_data.current_size_2 << 16);
3397         } else {
3398                 dp->heads = cgd->ident_data.heads;
3399                 dp->secs_per_track = cgd->ident_data.sectors;
3400                 dp->cylinders = cgd->ident_data.cylinders;
3401                 dp->sectors = cgd->ident_data.cylinders *
3402                               (u_int32_t)(dp->heads * dp->secs_per_track);
3403         }
3404         lbasize = (u_int32_t)cgd->ident_data.lba_size_1 |
3405                   ((u_int32_t)cgd->ident_data.lba_size_2 << 16);
3406
3407         /* use the 28bit LBA size if valid or bigger than the CHS mapping */
3408         if (cgd->ident_data.cylinders == 16383 || dp->sectors < lbasize)
3409                 dp->sectors = lbasize;
3410
3411         /* use the 48bit LBA size if valid */
3412         lbasize48 = ((u_int64_t)cgd->ident_data.lba_size48_1) |
3413                     ((u_int64_t)cgd->ident_data.lba_size48_2 << 16) |
3414                     ((u_int64_t)cgd->ident_data.lba_size48_3 << 32) |
3415                     ((u_int64_t)cgd->ident_data.lba_size48_4 << 48);
3416         if ((cgd->ident_data.support.command2 & ATA_SUPPORT_ADDRESS48) &&
3417             lbasize48 > ATA_MAX_28BIT_LBA)
3418                 dp->sectors = lbasize48;
3419 }
3420
3421 static void
3422 adasendorderedtag(void *arg)
3423 {
3424         struct ada_softc *softc = arg;
3425
3426         if (ada_send_ordered) {
3427                 if (softc->outstanding_cmds > 0) {
3428                         if ((softc->flags & ADA_FLAG_WAS_OTAG) == 0)
3429                                 softc->flags |= ADA_FLAG_NEED_OTAG;
3430                         softc->flags &= ~ADA_FLAG_WAS_OTAG;
3431                 }
3432         }
3433         /* Queue us up again */
3434         callout_reset(&softc->sendordered_c,
3435             (ada_default_timeout * hz) / ADA_ORDEREDTAG_INTERVAL,
3436             adasendorderedtag, softc);
3437 }
3438
3439 /*
3440  * Step through all ADA peripheral drivers, and if the device is still open,
3441  * sync the disk cache to physical media.
3442  */
3443 static void
3444 adaflush(void)
3445 {
3446         struct cam_periph *periph;
3447         struct ada_softc *softc;
3448         union ccb *ccb;
3449         int error;
3450
3451         CAM_PERIPH_FOREACH(periph, &adadriver) {
3452                 softc = (struct ada_softc *)periph->softc;
3453                 if (SCHEDULER_STOPPED()) {
3454                         /* If we paniced with the lock held, do not recurse. */
3455                         if (!cam_periph_owned(periph) &&
3456                             (softc->flags & ADA_FLAG_OPEN)) {
3457                                 adadump(softc->disk, NULL, 0, 0, 0);
3458                         }
3459                         continue;
3460                 }
3461                 cam_periph_lock(periph);
3462                 /*
3463                  * We only sync the cache if the drive is still open, and
3464                  * if the drive is capable of it..
3465                  */
3466                 if (((softc->flags & ADA_FLAG_OPEN) == 0) ||
3467                     (softc->flags & ADA_FLAG_CAN_FLUSHCACHE) == 0) {
3468                         cam_periph_unlock(periph);
3469                         continue;
3470                 }
3471
3472                 ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL);
3473                 cam_fill_ataio(&ccb->ataio,
3474                                     0,
3475                                     NULL,
3476                                     CAM_DIR_NONE,
3477                                     0,
3478                                     NULL,
3479                                     0,
3480                                     ada_default_timeout*1000);
3481                 if (softc->flags & ADA_FLAG_CAN_48BIT)
3482                         ata_48bit_cmd(&ccb->ataio, ATA_FLUSHCACHE48, 0, 0, 0);
3483                 else
3484                         ata_28bit_cmd(&ccb->ataio, ATA_FLUSHCACHE, 0, 0, 0);
3485
3486                 error = cam_periph_runccb(ccb, adaerror, /*cam_flags*/0,
3487                     /*sense_flags*/ SF_NO_RECOVERY | SF_NO_RETRY,
3488                     softc->disk->d_devstat);
3489                 if (error != 0)
3490                         xpt_print(periph->path, "Synchronize cache failed\n");
3491                 xpt_release_ccb(ccb);
3492                 cam_periph_unlock(periph);
3493         }
3494 }
3495
3496 static void
3497 adaspindown(uint8_t cmd, int flags)
3498 {
3499         struct cam_periph *periph;
3500         struct ada_softc *softc;
3501         struct ccb_ataio local_ccb;
3502         int error;
3503
3504         CAM_PERIPH_FOREACH(periph, &adadriver) {
3505                 /* If we paniced with lock held - not recurse here. */
3506                 if (cam_periph_owned(periph))
3507                         continue;
3508                 cam_periph_lock(periph);
3509                 softc = (struct ada_softc *)periph->softc;
3510                 /*
3511                  * We only spin-down the drive if it is capable of it..
3512                  */
3513                 if ((softc->flags & ADA_FLAG_CAN_POWERMGT) == 0) {
3514                         cam_periph_unlock(periph);
3515                         continue;
3516                 }
3517
3518                 if (bootverbose)
3519                         xpt_print(periph->path, "spin-down\n");
3520
3521                 memset(&local_ccb, 0, sizeof(local_ccb));
3522                 xpt_setup_ccb(&local_ccb.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
3523                 local_ccb.ccb_h.ccb_state = ADA_CCB_DUMP;
3524
3525                 cam_fill_ataio(&local_ccb,
3526                                     0,
3527                                     NULL,
3528                                     CAM_DIR_NONE | flags,
3529                                     0,
3530                                     NULL,
3531                                     0,
3532                                     ada_default_timeout*1000);
3533                 ata_28bit_cmd(&local_ccb, cmd, 0, 0, 0);
3534                 error = cam_periph_runccb((union ccb *)&local_ccb, adaerror,
3535                     /*cam_flags*/0, /*sense_flags*/ SF_NO_RECOVERY | SF_NO_RETRY,
3536                     softc->disk->d_devstat);
3537                 if (error != 0)
3538                         xpt_print(periph->path, "Spin-down disk failed\n");
3539                 cam_periph_unlock(periph);
3540         }
3541 }
3542
3543 static void
3544 adashutdown(void *arg, int howto)
3545 {
3546         int how;
3547
3548         adaflush();
3549
3550         /*
3551          * STANDBY IMMEDIATE saves any volatile data to the drive. It also spins
3552          * down hard drives. IDLE IMMEDIATE also saves the volatile data without
3553          * a spindown. We send the former when we expect to lose power soon. For
3554          * a warm boot, we send the latter to avoid a thundering herd of spinups
3555          * just after the kernel loads while probing. We have to do something to
3556          * flush the data because the BIOS in many systems resets the HBA
3557          * causing a COMINIT/COMRESET negotiation, which some drives interpret
3558          * as license to toss the volatile data, and others count as unclean
3559          * shutdown when in the Active PM state in SMART attributes.
3560          *
3561          * adaspindown will ensure that we don't send this to a drive that
3562          * doesn't support it.
3563          */
3564         if (ada_spindown_shutdown != 0) {
3565                 how = (howto & (RB_HALT | RB_POWEROFF | RB_POWERCYCLE)) ?
3566                     ATA_STANDBY_IMMEDIATE : ATA_IDLE_IMMEDIATE;
3567                 adaspindown(how, 0);
3568         }
3569 }
3570
3571 static void
3572 adasuspend(void *arg)
3573 {
3574
3575         adaflush();
3576         /*
3577          * SLEEP also fushes any volatile data, like STANDBY IMEDIATE,
3578          * so we don't need to send it as well.
3579          */
3580         if (ada_spindown_suspend != 0)
3581                 adaspindown(ATA_SLEEP, CAM_DEV_QFREEZE);
3582 }
3583
3584 static void
3585 adaresume(void *arg)
3586 {
3587         struct cam_periph *periph;
3588         struct ada_softc *softc;
3589
3590         if (ada_spindown_suspend == 0)
3591                 return;
3592
3593         CAM_PERIPH_FOREACH(periph, &adadriver) {
3594                 cam_periph_lock(periph);
3595                 softc = (struct ada_softc *)periph->softc;
3596                 /*
3597                  * We only spin-down the drive if it is capable of it..
3598                  */
3599                 if ((softc->flags & ADA_FLAG_CAN_POWERMGT) == 0) {
3600                         cam_periph_unlock(periph);
3601                         continue;
3602                 }
3603
3604                 if (bootverbose)
3605                         xpt_print(periph->path, "resume\n");
3606
3607                 /*
3608                  * Drop freeze taken due to CAM_DEV_QFREEZE flag set on
3609                  * sleep request.
3610                  */
3611                 cam_release_devq(periph->path,
3612                          /*relsim_flags*/0,
3613                          /*openings*/0,
3614                          /*timeout*/0,
3615                          /*getcount_only*/0);
3616
3617                 cam_periph_unlock(periph);
3618         }
3619 }
3620
3621 #endif /* _KERNEL */