sys/cam/ata/ata_da.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause
   3  *
   4  * Copyright (c) 2009 Alexander Motin <mav@FreeBSD.org>
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer,
  12  *    without modification, immediately at the beginning of the file.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  18  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  19  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  20  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  21  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  22  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  23  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  24  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  25  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  26  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27  */
  28
  29 #include "opt_ada.h"
  30
  31 #include <sys/param.h>
  32
  33 #ifdef _KERNEL
  34 #include <sys/systm.h>
  35 #include <sys/kernel.h>
  36 #include <sys/bio.h>
  37 #include <sys/sysctl.h>
  38 #include <sys/taskqueue.h>
  39 #include <sys/lock.h>
  40 #include <sys/mutex.h>
  41 #include <sys/conf.h>
  42 #include <sys/devicestat.h>
  43 #include <sys/eventhandler.h>
  44 #include <sys/malloc.h>
  45 #include <sys/endian.h>
  46 #include <sys/cons.h>
  47 #include <sys/proc.h>
  48 #include <sys/reboot.h>
  49 #include <sys/sbuf.h>
  50 #include <geom/geom.h>
  51 #include <geom/geom_disk.h>
  52 #endif /* _KERNEL */
  53
  54 #ifndef _KERNEL
  55 #include <stdio.h>
  56 #include <string.h>
  57 #endif /* _KERNEL */
  58
  59 #include <cam/cam.h>
  60 #include <cam/cam_ccb.h>
  61 #include <cam/cam_periph.h>
  62 #include <cam/cam_xpt_periph.h>
  63 #include <cam/scsi/scsi_all.h>
  64 #include <cam/scsi/scsi_da.h>
  65 #include <cam/cam_sim.h>
  66 #include <cam/cam_iosched.h>
  67
  68 #include <cam/ata/ata_all.h>
  69
  70 #ifdef _KERNEL
  71
  72 #define ATA_MAX_28BIT_LBA               268435455UL
  73
  74 extern int iosched_debug;
  75
  76 typedef enum {
  77         ADA_STATE_RAHEAD,
  78         ADA_STATE_WCACHE,
  79         ADA_STATE_LOGDIR,
  80         ADA_STATE_IDDIR,
  81         ADA_STATE_SUP_CAP,
  82         ADA_STATE_ZONE,
  83         ADA_STATE_NORMAL
  84 } ada_state;
  85
  86 typedef enum {
  87         ADA_FLAG_CAN_48BIT      = 0x00000002,
  88         ADA_FLAG_CAN_FLUSHCACHE = 0x00000004,
  89         ADA_FLAG_CAN_NCQ        = 0x00000008,
  90         ADA_FLAG_CAN_DMA        = 0x00000010,
  91         ADA_FLAG_NEED_OTAG      = 0x00000020,
  92         ADA_FLAG_WAS_OTAG       = 0x00000040,
  93         ADA_FLAG_CAN_TRIM       = 0x00000080,
  94         ADA_FLAG_OPEN           = 0x00000100,
  95         ADA_FLAG_SCTX_INIT      = 0x00000200,
  96         ADA_FLAG_CAN_CFA        = 0x00000400,
  97         ADA_FLAG_CAN_POWERMGT   = 0x00000800,
  98         ADA_FLAG_CAN_DMA48      = 0x00001000,
  99         ADA_FLAG_CAN_LOG        = 0x00002000,
 100         ADA_FLAG_CAN_IDLOG      = 0x00004000,
 101         ADA_FLAG_CAN_SUPCAP     = 0x00008000,
 102         ADA_FLAG_CAN_ZONE       = 0x00010000,
 103         ADA_FLAG_CAN_WCACHE     = 0x00020000,
 104         ADA_FLAG_CAN_RAHEAD     = 0x00040000,
 105         ADA_FLAG_PROBED         = 0x00080000,
 106         ADA_FLAG_ANNOUNCED      = 0x00100000,
 107         ADA_FLAG_DIRTY          = 0x00200000,
 108         ADA_FLAG_CAN_NCQ_TRIM   = 0x00400000,   /* CAN_TRIM also set */
 109         ADA_FLAG_PIM_ATA_EXT    = 0x00800000,
 110         ADA_FLAG_UNMAPPEDIO     = 0x01000000,
 111         ADA_FLAG_ROTATING       = 0x02000000
 112 } ada_flags;
 113 #define ADA_FLAG_STRING         \
 114         "\020"                  \
 115         "\002CAN_48BIT"         \
 116         "\003CAN_FLUSHCACHE"    \
 117         "\004CAN_NCQ"           \
 118         "\005CAN_DMA"           \
 119         "\006NEED_OTAG"         \
 120         "\007WAS_OTAG"          \
 121         "\010CAN_TRIM"          \
 122         "\011OPEN"              \
 123         "\012SCTX_INIT"         \
 124         "\013CAN_CFA"           \
 125         "\014CAN_POWERMGT"      \
 126         "\015CAN_DMA48"         \
 127         "\016CAN_LOG"           \
 128         "\017CAN_IDLOG"         \
 129         "\020CAN_SUPCAP"        \
 130         "\021CAN_ZONE"          \
 131         "\022CAN_WCACHE"        \
 132         "\023CAN_RAHEAD"        \
 133         "\024PROBED"            \
 134         "\025ANNOUNCED"         \
 135         "\026DIRTY"             \
 136         "\027CAN_NCQ_TRIM"      \
 137         "\030PIM_ATA_EXT"       \
 138         "\031UNMAPPEDIO"        \
 139         "\032ROTATING"
 140
 141 typedef enum {
 142         ADA_Q_NONE              = 0x00,
 143         ADA_Q_4K                = 0x01,
 144         ADA_Q_NCQ_TRIM_BROKEN   = 0x02,
 145         ADA_Q_LOG_BROKEN        = 0x04,
 146         ADA_Q_SMR_DM            = 0x08,
 147         ADA_Q_NO_TRIM           = 0x10,
 148         ADA_Q_128KB             = 0x20
 149 } ada_quirks;
 150
 151 #define ADA_Q_BIT_STRING        \
 152         "\020"                  \
 153         "\0014K"                \
 154         "\002NCQ_TRIM_BROKEN"   \
 155         "\003LOG_BROKEN"        \
 156         "\004SMR_DM"            \
 157         "\005NO_TRIM"           \
 158         "\006128KB"
 159
 160 typedef enum {
 161         ADA_CCB_RAHEAD          = 0x01,
 162         ADA_CCB_WCACHE          = 0x02,
 163         ADA_CCB_BUFFER_IO       = 0x03,
 164         ADA_CCB_DUMP            = 0x05,
 165         ADA_CCB_TRIM            = 0x06,
 166         ADA_CCB_LOGDIR          = 0x07,
 167         ADA_CCB_IDDIR           = 0x08,
 168         ADA_CCB_SUP_CAP         = 0x09,
 169         ADA_CCB_ZONE            = 0x0a,
 170         ADA_CCB_TYPE_MASK       = 0x0F,
 171 } ada_ccb_state;
 172
 173 typedef enum {
 174         ADA_ZONE_NONE           = 0x00,
 175         ADA_ZONE_DRIVE_MANAGED  = 0x01,
 176         ADA_ZONE_HOST_AWARE     = 0x02,
 177         ADA_ZONE_HOST_MANAGED   = 0x03
 178 } ada_zone_mode;
 179
 180 typedef enum {
 181         ADA_ZONE_FLAG_RZ_SUP            = 0x0001,
 182         ADA_ZONE_FLAG_OPEN_SUP          = 0x0002,
 183         ADA_ZONE_FLAG_CLOSE_SUP         = 0x0004,
 184         ADA_ZONE_FLAG_FINISH_SUP        = 0x0008,
 185         ADA_ZONE_FLAG_RWP_SUP           = 0x0010,
 186         ADA_ZONE_FLAG_SUP_MASK          = (ADA_ZONE_FLAG_RZ_SUP |
 187                                            ADA_ZONE_FLAG_OPEN_SUP |
 188                                            ADA_ZONE_FLAG_CLOSE_SUP |
 189                                            ADA_ZONE_FLAG_FINISH_SUP |
 190                                            ADA_ZONE_FLAG_RWP_SUP),
 191         ADA_ZONE_FLAG_URSWRZ            = 0x0020,
 192         ADA_ZONE_FLAG_OPT_SEQ_SET       = 0x0040,
 193         ADA_ZONE_FLAG_OPT_NONSEQ_SET    = 0x0080,
 194         ADA_ZONE_FLAG_MAX_SEQ_SET       = 0x0100,
 195         ADA_ZONE_FLAG_SET_MASK          = (ADA_ZONE_FLAG_OPT_SEQ_SET |
 196                                            ADA_ZONE_FLAG_OPT_NONSEQ_SET |
 197                                            ADA_ZONE_FLAG_MAX_SEQ_SET)
 198 } ada_zone_flags;
 199
 200 static struct ada_zone_desc {
 201         ada_zone_flags value;
 202         const char *desc;
 203 } ada_zone_desc_table[] = {
 204         {ADA_ZONE_FLAG_RZ_SUP, "Report Zones" },
 205         {ADA_ZONE_FLAG_OPEN_SUP, "Open" },
 206         {ADA_ZONE_FLAG_CLOSE_SUP, "Close" },
 207         {ADA_ZONE_FLAG_FINISH_SUP, "Finish" },
 208         {ADA_ZONE_FLAG_RWP_SUP, "Reset Write Pointer" },
 209 };
 210
 211 /* Offsets into our private area for storing information */
 212 #define ccb_state       ppriv_field0
 213 #define ccb_bp          ppriv_ptr1
 214
 215 typedef enum {
 216         ADA_DELETE_NONE,
 217         ADA_DELETE_DISABLE,
 218         ADA_DELETE_CFA_ERASE,
 219         ADA_DELETE_DSM_TRIM,
 220         ADA_DELETE_NCQ_DSM_TRIM,
 221         ADA_DELETE_MIN = ADA_DELETE_CFA_ERASE,
 222         ADA_DELETE_MAX = ADA_DELETE_NCQ_DSM_TRIM,
 223 } ada_delete_methods;
 224
 225 static const char *ada_delete_method_names[] =
 226     { "NONE", "DISABLE", "CFA_ERASE", "DSM_TRIM", "NCQ_DSM_TRIM" };
 227 #if 0
 228 static const char *ada_delete_method_desc[] =
 229     { "NONE", "DISABLED", "CFA Erase", "DSM Trim", "DSM Trim via NCQ" };
 230 #endif
 231
 232 struct disk_params {
 233         uint8_t  heads;
 234         uint8_t  secs_per_track;
 235         uint32_t cylinders;
 236         uint32_t secsize;       /* Number of bytes/logical sector */
 237         uint64_t sectors;       /* Total number sectors */
 238 };
 239
 240 #define TRIM_MAX_BLOCKS 8
 241 #define TRIM_MAX_RANGES (TRIM_MAX_BLOCKS * ATA_DSM_BLK_RANGES)
 242 struct trim_request {
 243         uint8_t         data[TRIM_MAX_RANGES * ATA_DSM_RANGE_SIZE];
 244         TAILQ_HEAD(, bio) bps;
 245 };
 246
 247 struct ada_softc {
 248         struct   cam_iosched_softc *cam_iosched;
 249         int      outstanding_cmds;      /* Number of active commands */
 250         int      refcount;              /* Active xpt_action() calls */
 251         ada_state state;
 252         ada_flags flags;
 253         ada_zone_mode zone_mode;
 254         ada_zone_flags zone_flags;
 255         struct ata_gp_log_dir ata_logdir;
 256         int valid_logdir_len;
 257         struct ata_identify_log_pages ata_iddir;
 258         int valid_iddir_len;
 259         uint64_t optimal_seq_zones;
 260         uint64_t optimal_nonseq_zones;
 261         uint64_t max_seq_zones;
 262         ada_quirks quirks;
 263         ada_delete_methods delete_method;
 264         int      trim_max_ranges;
 265         int      read_ahead;
 266         int      write_cache;
 267 #ifdef CAM_TEST_FAILURE
 268         int      force_read_error;
 269         int      force_write_error;
 270         int      periodic_read_error;
 271         int      periodic_read_count;
 272 #endif
 273         struct ccb_pathinq      cpi;
 274         struct disk_params      params;
 275         struct disk             *disk;
 276         struct task             sysctl_task;
 277         struct sysctl_ctx_list  sysctl_ctx;
 278         struct sysctl_oid       *sysctl_tree;
 279         struct callout          sendordered_c;
 280         struct trim_request     trim_req;
 281         uint64_t                trim_count;
 282         uint64_t                trim_ranges;
 283         uint64_t                trim_lbas;
 284 #ifdef CAM_IO_STATS
 285         struct sysctl_ctx_list  sysctl_stats_ctx;
 286         struct sysctl_oid       *sysctl_stats_tree;
 287         u_int   timeouts;
 288         u_int   errors;
 289         u_int   invalidations;
 290 #endif
 291 #define ADA_ANNOUNCETMP_SZ 80
 292         char    announce_temp[ADA_ANNOUNCETMP_SZ];
 293 #define ADA_ANNOUNCE_SZ 400
 294         char    announce_buffer[ADA_ANNOUNCE_SZ];
 295 };
 296
 297 static uma_zone_t ada_ccb_zone;
 298
 299 struct ada_quirk_entry {
 300         struct scsi_inquiry_pattern inq_pat;
 301         ada_quirks quirks;
 302 };
 303
 304 static struct ada_quirk_entry ada_quirk_table[] =
 305 {
 306         {
 307                 /* Sandisk X400 */
 308                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SanDisk?SD8SB8U1T00*", "X4162000*" },
 309                 /*quirks*/ADA_Q_128KB
 310         },
 311         {
 312                 /* Hitachi Advanced Format (4k) drives */
 313                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Hitachi H??????????E3*", "*" },
 314                 /*quirks*/ADA_Q_4K
 315         },
 316         {
 317                 /* Samsung Advanced Format (4k) drives */
 318                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG HD155UI*", "*" },
 319                 /*quirks*/ADA_Q_4K
 320         },
 321         {
 322                 /* Samsung Advanced Format (4k) drives */
 323                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG HD204UI*", "*" },
 324                 /*quirks*/ADA_Q_4K
 325         },
 326         {
 327                 /* Seagate Barracuda Green Advanced Format (4k) drives */
 328                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST????DL*", "*" },
 329                 /*quirks*/ADA_Q_4K
 330         },
 331         {
 332                 /* Seagate Barracuda Advanced Format (4k) drives */
 333                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST???DM*", "*" },
 334                 /*quirks*/ADA_Q_4K
 335         },
 336         {
 337                 /* Seagate Barracuda Advanced Format (4k) drives */
 338                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST????DM*", "*" },
 339                 /*quirks*/ADA_Q_4K
 340         },
 341         {
 342                 /* Seagate Momentus Advanced Format (4k) drives */
 343                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9500423AS*", "*" },
 344                 /*quirks*/ADA_Q_4K
 345         },
 346         {
 347                 /* Seagate Momentus Advanced Format (4k) drives */
 348                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9500424AS*", "*" },
 349                 /*quirks*/ADA_Q_4K
 350         },
 351         {
 352                 /* Seagate Momentus Advanced Format (4k) drives */
 353                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9640423AS*", "*" },
 354                 /*quirks*/ADA_Q_4K
 355         },
 356         {
 357                 /* Seagate Momentus Advanced Format (4k) drives */
 358                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9640424AS*", "*" },
 359                 /*quirks*/ADA_Q_4K
 360         },
 361         {
 362                 /* Seagate Momentus Advanced Format (4k) drives */
 363                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9750420AS*", "*" },
 364                 /*quirks*/ADA_Q_4K
 365         },
 366         {
 367                 /* Seagate Momentus Advanced Format (4k) drives */
 368                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9750422AS*", "*" },
 369                 /*quirks*/ADA_Q_4K
 370         },
 371         {
 372                 /* Seagate Momentus Advanced Format (4k) drives */
 373                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9750423AS*", "*" },
 374                 /*quirks*/ADA_Q_4K
 375         },
 376         {
 377                 /* Seagate Momentus Thin Advanced Format (4k) drives */
 378                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST???LT*", "*" },
 379                 /*quirks*/ADA_Q_4K
 380         },
 381         {
 382                 /* WDC Caviar Red Advanced Format (4k) drives */
 383                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD????CX*", "*" },
 384                 /*quirks*/ADA_Q_4K
 385         },
 386         {
 387                 /* WDC Caviar Green Advanced Format (4k) drives */
 388                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD????RS*", "*" },
 389                 /*quirks*/ADA_Q_4K
 390         },
 391         {
 392                 /* WDC Caviar Green/Red Advanced Format (4k) drives */
 393                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD????RX*", "*" },
 394                 /*quirks*/ADA_Q_4K
 395         },
 396         {
 397                 /* WDC Caviar Red Advanced Format (4k) drives */
 398                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD??????CX*", "*" },
 399                 /*quirks*/ADA_Q_4K
 400         },
 401         {
 402                 /* WDC Caviar Black Advanced Format (4k) drives */
 403                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD????AZEX*", "*" },
 404                 /*quirks*/ADA_Q_4K
 405         },
 406         {
 407                 /* WDC Caviar Black Advanced Format (4k) drives */
 408                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD????FZEX*", "*" },
 409                 /*quirks*/ADA_Q_4K
 410         },
 411         {
 412                 /* WDC Caviar Green Advanced Format (4k) drives */
 413                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD??????RS*", "*" },
 414                 /*quirks*/ADA_Q_4K
 415         },
 416         {
 417                 /* WDC Caviar Green Advanced Format (4k) drives */
 418                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD??????RX*", "*" },
 419                 /*quirks*/ADA_Q_4K
 420         },
 421         {
 422                 /* WDC Scorpio Black Advanced Format (4k) drives */
 423                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD???PKT*", "*" },
 424                 /*quirks*/ADA_Q_4K
 425         },
 426         {
 427                 /* WDC Scorpio Black Advanced Format (4k) drives */
 428                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD?????PKT*", "*" },
 429                 /*quirks*/ADA_Q_4K
 430         },
 431         {
 432                 /* WDC Scorpio Blue Advanced Format (4k) drives */
 433                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD???PVT*", "*" },
 434                 /*quirks*/ADA_Q_4K
 435         },
 436         {
 437                 /* WDC Scorpio Blue Advanced Format (4k) drives */
 438                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD?????PVT*", "*" },
 439                 /*quirks*/ADA_Q_4K
 440         },
 441         /* SSDs */
 442         {
 443                 /*
 444                  * Corsair Force 2 SSDs
 445                  * 4k optimised & trim only works in 4k requests + 4k aligned
 446                  */
 447                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair CSSD-F*", "*" },
 448                 /*quirks*/ADA_Q_4K
 449         },
 450         {
 451                 /*
 452                  * Corsair Force 3 SSDs
 453                  * 4k optimised & trim only works in 4k requests + 4k aligned
 454                  */
 455                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair Force 3*", "*" },
 456                 /*quirks*/ADA_Q_4K
 457         },
 458         {
 459                 /*
 460                  * Corsair Neutron GTX SSDs
 461                  * 4k optimised & trim only works in 4k requests + 4k aligned
 462                  */
 463                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair Neutron GTX*", "*" },
 464                 /*quirks*/ADA_Q_4K
 465         },
 466         {
 467                 /*
 468                  * Corsair Force GT & GS SSDs
 469                  * 4k optimised & trim only works in 4k requests + 4k aligned
 470                  */
 471                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair Force G*", "*" },
 472                 /*quirks*/ADA_Q_4K
 473         },
 474         {
 475                 /*
 476                  * Crucial M4 SSDs
 477                  * 4k optimised & trim only works in 4k requests + 4k aligned
 478                  */
 479                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "M4-CT???M4SSD2*", "*" },
 480                 /*quirks*/ADA_Q_4K
 481         },
 482         {
 483                 /*
 484                  * Crucial M500 SSDs MU07 firmware
 485                  * NCQ Trim works
 486                  */
 487                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Crucial CT*M500*", "MU07" },
 488                 /*quirks*/0
 489         },
 490         {
 491                 /*
 492                  * Crucial M500 SSDs all other firmware
 493                  * NCQ Trim doesn't work
 494                  */
 495                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Crucial CT*M500*", "*" },
 496                 /*quirks*/ADA_Q_NCQ_TRIM_BROKEN
 497         },
 498         {
 499                 /*
 500                  * Crucial M550 SSDs
 501                  * NCQ Trim doesn't work, but only on MU01 firmware
 502                  */
 503                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Crucial CT*M550*", "MU01" },
 504                 /*quirks*/ADA_Q_NCQ_TRIM_BROKEN
 505         },
 506         {
 507                 /*
 508                  * Crucial MX100 SSDs
 509                  * NCQ Trim doesn't work, but only on MU01 firmware
 510                  */
 511                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Crucial CT*MX100*", "MU01" },
 512                 /*quirks*/ADA_Q_NCQ_TRIM_BROKEN
 513         },
 514         {
 515                 /*
 516                  * Crucial RealSSD C300 SSDs
 517                  * 4k optimised
 518                  */
 519                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "C300-CTFDDAC???MAG*",
 520                 "*" }, /*quirks*/ADA_Q_4K
 521         },
 522         {
 523                 /*
 524                  * FCCT M500 SSDs
 525                  * NCQ Trim doesn't work
 526                  */
 527                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "FCCT*M500*", "*" },
 528                 /*quirks*/ADA_Q_NCQ_TRIM_BROKEN
 529         },
 530         {
 531                 /*
 532                  * Intel 320 Series SSDs
 533                  * 4k optimised & trim only works in 4k requests + 4k aligned
 534                  */
 535                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSA2CW*", "*" },
 536                 /*quirks*/ADA_Q_4K
 537         },
 538         {
 539                 /*
 540                  * Intel 330 Series SSDs
 541                  * 4k optimised & trim only works in 4k requests + 4k aligned
 542                  */
 543                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSC2CT*", "*" },
 544                 /*quirks*/ADA_Q_4K
 545         },
 546         {
 547                 /*
 548                  * Intel 510 Series SSDs
 549                  * 4k optimised & trim only works in 4k requests + 4k aligned
 550                  */
 551                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSC2MH*", "*" },
 552                 /*quirks*/ADA_Q_4K
 553         },
 554         {
 555                 /*
 556                  * Intel 520 Series SSDs
 557                  * 4k optimised & trim only works in 4k requests + 4k aligned
 558                  */
 559                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSC2BW*", "*" },
 560                 /*quirks*/ADA_Q_4K
 561         },
 562         {
 563                 /*
 564                  * Intel S3610 Series SSDs
 565                  * 4k optimised & trim only works in 4k requests + 4k aligned
 566                  */
 567                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSC2BX*", "*" },
 568                 /*quirks*/ADA_Q_4K
 569         },
 570         {
 571                 /*
 572                  * Intel X25-M Series SSDs
 573                  * 4k optimised & trim only works in 4k requests + 4k aligned
 574                  */
 575                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSA2M*", "*" },
 576                 /*quirks*/ADA_Q_4K
 577         },
 578         {
 579                 /*
 580                  * KingDian S200 60GB P0921B
 581                  * Trimming crash the SSD
 582                  */
 583                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "KingDian S200 *", "*" },
 584                 /*quirks*/ADA_Q_NO_TRIM
 585         },
 586         {
 587                 /*
 588                  * Kingston E100 Series SSDs
 589                  * 4k optimised & trim only works in 4k requests + 4k aligned
 590                  */
 591                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "KINGSTON SE100S3*", "*" },
 592                 /*quirks*/ADA_Q_4K
 593         },
 594         {
 595                 /*
 596                  * Kingston HyperX 3k SSDs
 597                  * 4k optimised & trim only works in 4k requests + 4k aligned
 598                  */
 599                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "KINGSTON SH103S3*", "*" },
 600                 /*quirks*/ADA_Q_4K
 601         },
 602         {
 603                 /*
 604                  * Marvell SSDs (entry taken from OpenSolaris)
 605                  * 4k optimised & trim only works in 4k requests + 4k aligned
 606                  */
 607                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "MARVELL SD88SA02*", "*" },
 608                 /*quirks*/ADA_Q_4K
 609         },
 610         {
 611                 /*
 612                  * Micron M500 SSDs firmware MU07
 613                  * NCQ Trim works?
 614                  */
 615                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Micron M500*", "MU07" },
 616                 /*quirks*/0
 617         },
 618         {
 619                 /*
 620                  * Micron M500 SSDs all other firmware
 621                  * NCQ Trim doesn't work
 622                  */
 623                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Micron M500*", "*" },
 624                 /*quirks*/ADA_Q_NCQ_TRIM_BROKEN
 625         },
 626         {
 627                 /*
 628                  * Micron M5[15]0 SSDs
 629                  * NCQ Trim doesn't work, but only MU01 firmware
 630                  */
 631                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Micron M5[15]0*", "MU01" },
 632                 /*quirks*/ADA_Q_NCQ_TRIM_BROKEN
 633         },
 634         {
 635                 /*
 636                  * Micron 5100 SSDs
 637                  * 4k optimised & trim only works in 4k requests + 4k aligned
 638                  */
 639                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Micron 5100 MTFDDAK*", "*" },
 640                 /*quirks*/ADA_Q_4K
 641         },
 642         {
 643                 /*
 644                  * OCZ Agility 2 SSDs
 645                  * 4k optimised & trim only works in 4k requests + 4k aligned
 646                  */
 647                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-AGILITY2*", "*" },
 648                 /*quirks*/ADA_Q_4K
 649         },
 650         {
 651                 /*
 652                  * OCZ Agility 3 SSDs
 653                  * 4k optimised & trim only works in 4k requests + 4k aligned
 654                  */
 655                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-AGILITY3*", "*" },
 656                 /*quirks*/ADA_Q_4K
 657         },
 658         {
 659                 /*
 660                  * OCZ Deneva R Series SSDs
 661                  * 4k optimised & trim only works in 4k requests + 4k aligned
 662                  */
 663                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "DENRSTE251M45*", "*" },
 664                 /*quirks*/ADA_Q_4K
 665         },
 666         {
 667                 /*
 668                  * OCZ Vertex 2 SSDs (inc pro series)
 669                  * 4k optimised & trim only works in 4k requests + 4k aligned
 670                  */
 671                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ?VERTEX2*", "*" },
 672                 /*quirks*/ADA_Q_4K
 673         },
 674         {
 675                 /*
 676                  * OCZ Vertex 3 SSDs
 677                  * 4k optimised & trim only works in 4k requests + 4k aligned
 678                  */
 679                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-VERTEX3*", "*" },
 680                 /*quirks*/ADA_Q_4K
 681         },
 682         {
 683                 /*
 684                  * OCZ Vertex 4 SSDs
 685                  * 4k optimised & trim only works in 4k requests + 4k aligned
 686                  */
 687                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-VERTEX4*", "*" },
 688                 /*quirks*/ADA_Q_4K
 689         },
 690         {
 691                 /*
 692                  * Samsung 750 SSDs
 693                  * 4k optimised, NCQ TRIM seems to work
 694                  */
 695                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Samsung SSD 750*", "*" },
 696                 /*quirks*/ADA_Q_4K
 697         },
 698         {
 699                 /*
 700                  * Samsung 830 Series SSDs
 701                  * 4k optimised, NCQ TRIM Broken (normal TRIM is fine)
 702                  */
 703                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG SSD 830 Series*", "*" },
 704                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 705         },
 706         {
 707                 /*
 708                  * Samsung 840 SSDs
 709                  * 4k optimised, NCQ TRIM Broken (normal TRIM is fine)
 710                  */
 711                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Samsung SSD 840*", "*" },
 712                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 713         },
 714         {
 715                 /*
 716                  * Samsung 845 SSDs
 717                  * 4k optimised, NCQ TRIM Broken (normal TRIM is fine)
 718                  */
 719                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Samsung SSD 845*", "*" },
 720                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 721         },
 722         {
 723                 /*
 724                  * Samsung 850 SSDs
 725                  * 4k optimised, NCQ TRIM broken (normal TRIM fine)
 726                  */
 727                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Samsung SSD 850*", "*" },
 728                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 729         },
 730         {
 731                 /*
 732                  * Samsung SM863 Series SSDs (MZ7KM*)
 733                  * 4k optimised, NCQ believed to be working
 734                  */
 735                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG MZ7KM*", "*" },
 736                 /*quirks*/ADA_Q_4K
 737         },
 738         {
 739                 /*
 740                  * Samsung 843T Series SSDs (MZ7WD*)
 741                  * Samsung PM851 Series SSDs (MZ7TE*)
 742                  * Samsung PM853T Series SSDs (MZ7GE*)
 743                  * 4k optimised, NCQ believed to be broken since these are
 744                  * appear to be built with the same controllers as the 840/850.
 745                  */
 746                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG MZ7*", "*" },
 747                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 748         },
 749         {
 750                 /*
 751                  * Same as for SAMSUNG MZ7* but enable the quirks for SSD
 752                  * starting with MZ7* too
 753                  */
 754                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "MZ7*", "*" },
 755                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 756         },
 757         {
 758                 /*
 759                  * Samsung PM851 Series SSDs Dell OEM
 760                  * device model          "SAMSUNG SSD PM851 mSATA 256GB"
 761                  * 4k optimised, NCQ broken
 762                  */
 763                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG SSD PM851*", "*" },
 764                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 765         },
 766         {
 767                 /*
 768                  * SuperTalent TeraDrive CT SSDs
 769                  * 4k optimised & trim only works in 4k requests + 4k aligned
 770                  */
 771                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "FTM??CT25H*", "*" },
 772                 /*quirks*/ADA_Q_4K
 773         },
 774         {
 775                 /*
 776                  * XceedIOPS SATA SSDs
 777                  * 4k optimised
 778                  */
 779                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SG9XCS2D*", "*" },
 780                 /*quirks*/ADA_Q_4K
 781         },
 782         {
 783                 /*
 784                  * Samsung drive that doesn't support READ LOG EXT or
 785                  * READ LOG DMA EXT, despite reporting that it does in
 786                  * ATA identify data:
 787                  * SAMSUNG HD200HJ KF100-06
 788                  */
 789                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG HD200*", "*" },
 790                 /*quirks*/ADA_Q_LOG_BROKEN
 791         },
 792         {
 793                 /*
 794                  * Samsung drive that doesn't support READ LOG EXT or
 795                  * READ LOG DMA EXT, despite reporting that it does in
 796                  * ATA identify data:
 797                  * SAMSUNG HD501LJ CR100-10
 798                  */
 799                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG HD501*", "*" },
 800                 /*quirks*/ADA_Q_LOG_BROKEN
 801         },
 802         {
 803                 /*
 804                  * Seagate Lamarr 8TB Shingled Magnetic Recording (SMR)
 805                  * Drive Managed SATA hard drive.  This drive doesn't report
 806                  * in firmware that it is a drive managed SMR drive.
 807                  */
 808                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST8000AS000[23]*", "*" },
 809                 /*quirks*/ADA_Q_SMR_DM
 810         },
 811         {
 812                 /* WD Green SSD */
 813                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WDS?????G0*", "*" },
 814                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 815         },
 816         {
 817                 /* Default */
 818                 {
 819                   T_ANY, SIP_MEDIA_REMOVABLE|SIP_MEDIA_FIXED,
 820                   /*vendor*/"*", /*product*/"*", /*revision*/"*"
 821                 },
 822                 /*quirks*/0
 823         },
 824 };
 825
 826 static  disk_strategy_t adastrategy;
 827 static  dumper_t        adadump;
 828 static  periph_init_t   adainit;
 829 static  void            adadiskgonecb(struct disk *dp);
 830 static  periph_oninv_t  adaoninvalidate;
 831 static  periph_dtor_t   adacleanup;
 832 static  void            adaasync(void *callback_arg, uint32_t code,
 833                                 struct cam_path *path, void *arg);
 834 static  int             adabitsysctl(SYSCTL_HANDLER_ARGS);
 835 static  int             adaflagssysctl(SYSCTL_HANDLER_ARGS);
 836 static  int             adazonesupsysctl(SYSCTL_HANDLER_ARGS);
 837 static  void            adasysctlinit(void *context, int pending);
 838 static  int             adagetattr(struct bio *bp);
 839 static  void            adasetflags(struct ada_softc *softc,
 840                                     struct ccb_getdev *cgd);
 841 static void             adasetgeom(struct ada_softc *softc,
 842                                    struct ccb_getdev *cgd);
 843 static  periph_ctor_t   adaregister;
 844 static  void            ada_dsmtrim(struct ada_softc *softc, struct bio *bp,
 845                                     struct ccb_ataio *ataio);
 846 static  void            ada_cfaerase(struct ada_softc *softc, struct bio *bp,
 847                                      struct ccb_ataio *ataio);
 848 static  int             ada_zone_bio_to_ata(int disk_zone_cmd);
 849 static  int             ada_zone_cmd(struct cam_periph *periph, union ccb *ccb,
 850                                      struct bio *bp, int *queue_ccb);
 851 static  periph_start_t  adastart;
 852 static  void            adaprobedone(struct cam_periph *periph, union ccb *ccb);
 853 static  void            adazonedone(struct cam_periph *periph, union ccb *ccb);
 854 static  void            adadone(struct cam_periph *periph,
 855                                union ccb *done_ccb);
 856 static  int             adaerror(union ccb *ccb, uint32_t cam_flags,
 857                                 uint32_t sense_flags);
 858 static callout_func_t   adasendorderedtag;
 859 static void             adashutdown(void *arg, int howto);
 860 static void             adasuspend(void *arg);
 861 static void             adaresume(void *arg);
 862
 863 #ifndef ADA_DEFAULT_TIMEOUT
 864 #define ADA_DEFAULT_TIMEOUT 30  /* Timeout in seconds */
 865 #endif
 866
 867 #ifndef ADA_DEFAULT_RETRY
 868 #define ADA_DEFAULT_RETRY       4
 869 #endif
 870
 871 #ifndef ADA_DEFAULT_SEND_ORDERED
 872 #define ADA_DEFAULT_SEND_ORDERED        1
 873 #endif
 874
 875 #ifndef ADA_DEFAULT_SPINDOWN_SHUTDOWN
 876 #define ADA_DEFAULT_SPINDOWN_SHUTDOWN   1
 877 #endif
 878
 879 #ifndef ADA_DEFAULT_SPINDOWN_SUSPEND
 880 #define ADA_DEFAULT_SPINDOWN_SUSPEND    1
 881 #endif
 882
 883 #ifndef ADA_DEFAULT_READ_AHEAD
 884 #define ADA_DEFAULT_READ_AHEAD  1
 885 #endif
 886
 887 #ifndef ADA_DEFAULT_WRITE_CACHE
 888 #define ADA_DEFAULT_WRITE_CACHE 1
 889 #endif
 890
 891 #define ADA_RA  (softc->read_ahead >= 0 ? \
 892                  softc->read_ahead : ada_read_ahead)
 893 #define ADA_WC  (softc->write_cache >= 0 ? \
 894                  softc->write_cache : ada_write_cache)
 895
 896 static int ada_retry_count = ADA_DEFAULT_RETRY;
 897 static int ada_default_timeout = ADA_DEFAULT_TIMEOUT;
 898 static int ada_send_ordered = ADA_DEFAULT_SEND_ORDERED;
 899 static int ada_spindown_shutdown = ADA_DEFAULT_SPINDOWN_SHUTDOWN;
 900 static int ada_spindown_suspend = ADA_DEFAULT_SPINDOWN_SUSPEND;
 901 static int ada_read_ahead = ADA_DEFAULT_READ_AHEAD;
 902 static int ada_write_cache = ADA_DEFAULT_WRITE_CACHE;
 903 static int ada_enable_biospeedup = 1;
 904 static int ada_enable_uma_ccbs = 1;
 905
 906 static SYSCTL_NODE(_kern_cam, OID_AUTO, ada, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
 907     "CAM Direct Access Disk driver");
 908 SYSCTL_INT(_kern_cam_ada, OID_AUTO, retry_count, CTLFLAG_RWTUN,
 909            &ada_retry_count, 0, "Normal I/O retry count");
 910 SYSCTL_INT(_kern_cam_ada, OID_AUTO, default_timeout, CTLFLAG_RWTUN,
 911            &ada_default_timeout, 0, "Normal I/O timeout (in seconds)");
 912 SYSCTL_INT(_kern_cam_ada, OID_AUTO, send_ordered, CTLFLAG_RWTUN,
 913            &ada_send_ordered, 0, "Send Ordered Tags");
 914 SYSCTL_INT(_kern_cam_ada, OID_AUTO, spindown_shutdown, CTLFLAG_RWTUN,
 915            &ada_spindown_shutdown, 0, "Spin down upon shutdown");
 916 SYSCTL_INT(_kern_cam_ada, OID_AUTO, spindown_suspend, CTLFLAG_RWTUN,
 917            &ada_spindown_suspend, 0, "Spin down upon suspend");
 918 SYSCTL_INT(_kern_cam_ada, OID_AUTO, read_ahead, CTLFLAG_RWTUN,
 919            &ada_read_ahead, 0, "Enable disk read-ahead");
 920 SYSCTL_INT(_kern_cam_ada, OID_AUTO, write_cache, CTLFLAG_RWTUN,
 921            &ada_write_cache, 0, "Enable disk write cache");
 922 SYSCTL_INT(_kern_cam_ada, OID_AUTO, enable_biospeedup, CTLFLAG_RDTUN,
 923            &ada_enable_biospeedup, 0, "Enable BIO_SPEEDUP processing");
 924 SYSCTL_INT(_kern_cam_ada, OID_AUTO, enable_uma_ccbs, CTLFLAG_RWTUN,
 925             &ada_enable_uma_ccbs, 0, "Use UMA for CCBs");
 926
 927 /*
 928  * ADA_ORDEREDTAG_INTERVAL determines how often, relative
 929  * to the default timeout, we check to see whether an ordered
 930  * tagged transaction is appropriate to prevent simple tag
 931  * starvation.  Since we'd like to ensure that there is at least
 932  * 1/2 of the timeout length left for a starved transaction to
 933  * complete after we've sent an ordered tag, we must poll at least
 934  * four times in every timeout period.  This takes care of the worst
 935  * case where a starved transaction starts during an interval that
 936  * meets the requirement "don't send an ordered tag" test so it takes
 937  * us two intervals to determine that a tag must be sent.
 938  */
 939 #ifndef ADA_ORDEREDTAG_INTERVAL
 940 #define ADA_ORDEREDTAG_INTERVAL 4
 941 #endif
 942
 943 static struct periph_driver adadriver =
 944 {
 945         adainit, "ada",
 946         TAILQ_HEAD_INITIALIZER(adadriver.units), /* generation */ 0
 947 };
 948
 949 static int adadeletemethodsysctl(SYSCTL_HANDLER_ARGS);
 950
 951 PERIPHDRIVER_DECLARE(ada, adadriver);
 952
 953 static MALLOC_DEFINE(M_ATADA, "ata_da", "ata_da buffers");
 954
 955 static int
 956 adaopen(struct disk *dp)
 957 {
 958         struct cam_periph *periph;
 959         struct ada_softc *softc;
 960         int error;
 961
 962         periph = (struct cam_periph *)dp->d_drv1;
 963         if (cam_periph_acquire(periph) != 0) {
 964                 return(ENXIO);
 965         }
 966
 967         cam_periph_lock(periph);
 968         if ((error = cam_periph_hold(periph, PRIBIO|PCATCH)) != 0) {
 969                 cam_periph_unlock(periph);
 970                 cam_periph_release(periph);
 971                 return (error);
 972         }
 973
 974         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE | CAM_DEBUG_PERIPH,
 975             ("adaopen\n"));
 976
 977         softc = (struct ada_softc *)periph->softc;
 978         softc->flags |= ADA_FLAG_OPEN;
 979
 980         cam_periph_unhold(periph);
 981         cam_periph_unlock(periph);
 982         return (0);
 983 }
 984
 985 static int
 986 adaclose(struct disk *dp)
 987 {
 988         struct  cam_periph *periph;
 989         struct  ada_softc *softc;
 990         union ccb *ccb;
 991         int error;
 992
 993         periph = (struct cam_periph *)dp->d_drv1;
 994         softc = (struct ada_softc *)periph->softc;
 995         cam_periph_lock(periph);
 996
 997         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE | CAM_DEBUG_PERIPH,
 998             ("adaclose\n"));
 999
1000         /* We only sync the cache if the drive is capable of it. */
1001         if ((softc->flags & ADA_FLAG_DIRTY) != 0 &&
1002             (softc->flags & ADA_FLAG_CAN_FLUSHCACHE) != 0 &&
1003             (periph->flags & CAM_PERIPH_INVALID) == 0 &&
1004             cam_periph_hold(periph, PRIBIO) == 0) {
1005                 ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL);
1006                 cam_fill_ataio(&ccb->ataio,
1007                                     1,
1008                                     NULL,
1009                                     CAM_DIR_NONE,
1010                                     0,
1011                                     NULL,
1012                                     0,
1013                                     ada_default_timeout*1000);
1014
1015                 if (softc->flags & ADA_FLAG_CAN_48BIT)
1016                         ata_48bit_cmd(&ccb->ataio, ATA_FLUSHCACHE48, 0, 0, 0);
1017                 else
1018                         ata_28bit_cmd(&ccb->ataio, ATA_FLUSHCACHE, 0, 0, 0);
1019                 error = cam_periph_runccb(ccb, adaerror, /*cam_flags*/0,
1020                     /*sense_flags*/0, softc->disk->d_devstat);
1021
1022                 if (error != 0)
1023                         xpt_print(periph->path, "Synchronize cache failed\n");
1024                 softc->flags &= ~ADA_FLAG_DIRTY;
1025                 xpt_release_ccb(ccb);
1026                 cam_periph_unhold(periph);
1027         }
1028
1029         softc->flags &= ~ADA_FLAG_OPEN;
1030
1031         while (softc->refcount != 0)
1032                 cam_periph_sleep(periph, &softc->refcount, PRIBIO, "adaclose", 1);
1033         cam_periph_unlock(periph);
1034         cam_periph_release(periph);
1035         return (0);
1036 }
1037
1038 static void
1039 adaschedule(struct cam_periph *periph)
1040 {
1041         struct ada_softc *softc = (struct ada_softc *)periph->softc;
1042
1043         if (softc->state != ADA_STATE_NORMAL)
1044                 return;
1045
1046         cam_iosched_schedule(softc->cam_iosched, periph);
1047 }
1048
1049 /*
1050  * Actually translate the requested transfer into one the physical driver
1051  * can understand.  The transfer is described by a buf and will include
1052  * only one physical transfer.
1053  */
1054 static void
1055 adastrategy(struct bio *bp)
1056 {
1057         struct cam_periph *periph;
1058         struct ada_softc *softc;
1059
1060         periph = (struct cam_periph *)bp->bio_disk->d_drv1;
1061         softc = (struct ada_softc *)periph->softc;
1062
1063         cam_periph_lock(periph);
1064
1065         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("adastrategy(%p)\n", bp));
1066
1067         /*
1068          * If the device has been made invalid, error out
1069          */
1070         if ((periph->flags & CAM_PERIPH_INVALID) != 0) {
1071                 cam_periph_unlock(periph);
1072                 biofinish(bp, NULL, ENXIO);
1073                 return;
1074         }
1075
1076         /*
1077          * Zone commands must be ordered, because they can depend on the
1078          * effects of previously issued commands, and they may affect
1079          * commands after them.
1080          */
1081         if (bp->bio_cmd == BIO_ZONE)
1082                 bp->bio_flags |= BIO_ORDERED;
1083
1084         /*
1085          * Place it in the queue of disk activities for this disk
1086          */
1087         cam_iosched_queue_work(softc->cam_iosched, bp);
1088
1089         /*
1090          * Schedule ourselves for performing the work.
1091          */
1092         adaschedule(periph);
1093         cam_periph_unlock(periph);
1094
1095         return;
1096 }
1097
1098 static int
1099 adadump(void *arg, void *virtual, off_t offset, size_t length)
1100 {
1101         struct      cam_periph *periph;
1102         struct      ada_softc *softc;
1103         u_int       secsize;
1104         struct      ccb_ataio ataio;
1105         struct      disk *dp;
1106         uint64_t    lba;
1107         uint16_t    count;
1108         int         error = 0;
1109
1110         dp = arg;
1111         periph = dp->d_drv1;
1112         softc = (struct ada_softc *)periph->softc;
1113         secsize = softc->params.secsize;
1114         lba = offset / secsize;
1115         count = length / secsize;
1116         if ((periph->flags & CAM_PERIPH_INVALID) != 0)
1117                 return (ENXIO);
1118
1119         memset(&ataio, 0, sizeof(ataio));
1120         if (length > 0) {
1121                 xpt_setup_ccb(&ataio.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
1122                 ataio.ccb_h.ccb_state = ADA_CCB_DUMP;
1123                 cam_fill_ataio(&ataio,
1124                     0,
1125                     NULL,
1126                     CAM_DIR_OUT,
1127                     0,
1128                     (uint8_t *) virtual,
1129                     length,
1130                     ada_default_timeout*1000);
1131                 if ((softc->flags & ADA_FLAG_CAN_48BIT) &&
1132                     (lba + count >= ATA_MAX_28BIT_LBA ||
1133                     count >= 256)) {
1134                         ata_48bit_cmd(&ataio, ATA_WRITE_DMA48,
1135                             0, lba, count);
1136                 } else {
1137                         ata_28bit_cmd(&ataio, ATA_WRITE_DMA,
1138                             0, lba, count);
1139                 }
1140                 error = cam_periph_runccb((union ccb *)&ataio, adaerror,
1141                     0, SF_NO_RECOVERY | SF_NO_RETRY, NULL);
1142                 if (error != 0)
1143                         printf("Aborting dump due to I/O error.\n");
1144
1145                 return (error);
1146         }
1147
1148         if (softc->flags & ADA_FLAG_CAN_FLUSHCACHE) {
1149                 xpt_setup_ccb(&ataio.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
1150
1151                 /*
1152                  * Tell the drive to flush its internal cache. if we
1153                  * can't flush in 5s we have big problems. No need to
1154                  * wait the default 60s to detect problems.
1155                  */
1156                 ataio.ccb_h.ccb_state = ADA_CCB_DUMP;
1157                 cam_fill_ataio(&ataio,
1158                                     0,
1159                                     NULL,
1160                                     CAM_DIR_NONE,
1161                                     0,
1162                                     NULL,
1163                                     0,
1164                                     5*1000);
1165
1166                 if (softc->flags & ADA_FLAG_CAN_48BIT)
1167                         ata_48bit_cmd(&ataio, ATA_FLUSHCACHE48, 0, 0, 0);
1168                 else
1169                         ata_28bit_cmd(&ataio, ATA_FLUSHCACHE, 0, 0, 0);
1170                 error = cam_periph_runccb((union ccb *)&ataio, adaerror,
1171                     0, SF_NO_RECOVERY | SF_NO_RETRY, NULL);
1172                 if (error != 0)
1173                         xpt_print(periph->path, "Synchronize cache failed\n");
1174         }
1175         return (error);
1176 }
1177
1178 static void
1179 adainit(void)
1180 {
1181         cam_status status;
1182
1183         ada_ccb_zone = uma_zcreate("ada_ccb",
1184             sizeof(struct ccb_ataio), NULL, NULL, NULL, NULL,
1185             UMA_ALIGN_PTR, 0);
1186
1187         /*
1188          * Install a global async callback.  This callback will
1189          * receive async callbacks like "new device found".
1190          */
1191         status = xpt_register_async(AC_FOUND_DEVICE, adaasync, NULL, NULL);
1192
1193         if (status != CAM_REQ_CMP) {
1194                 printf("ada: Failed to attach master async callback "
1195                        "due to status 0x%x!\n", status);
1196         } else if (ada_send_ordered) {
1197                 /* Register our event handlers */
1198                 if ((EVENTHANDLER_REGISTER(power_suspend, adasuspend,
1199                                            NULL, EVENTHANDLER_PRI_LAST)) == NULL)
1200                     printf("adainit: power event registration failed!\n");
1201                 if ((EVENTHANDLER_REGISTER(power_resume, adaresume,
1202                                            NULL, EVENTHANDLER_PRI_LAST)) == NULL)
1203                     printf("adainit: power event registration failed!\n");
1204                 if ((EVENTHANDLER_REGISTER(shutdown_post_sync, adashutdown,
1205                                            NULL, SHUTDOWN_PRI_DEFAULT)) == NULL)
1206                     printf("adainit: shutdown event registration failed!\n");
1207         }
1208 }
1209
1210 /*
1211  * Callback from GEOM, called when it has finished cleaning up its
1212  * resources.
1213  */
1214 static void
1215 adadiskgonecb(struct disk *dp)
1216 {
1217         struct cam_periph *periph;
1218
1219         periph = (struct cam_periph *)dp->d_drv1;
1220
1221         cam_periph_release(periph);
1222 }
1223
1224 static void
1225 adaoninvalidate(struct cam_periph *periph)
1226 {
1227         struct ada_softc *softc;
1228
1229         softc = (struct ada_softc *)periph->softc;
1230
1231         /*
1232          * De-register any async callbacks.
1233          */
1234         xpt_register_async(0, adaasync, periph, periph->path);
1235 #ifdef CAM_IO_STATS
1236         softc->invalidations++;
1237 #endif
1238
1239         /*
1240          * Return all queued I/O with ENXIO. Transactions may be queued up here
1241          * for retry (since we are called while there's other transactions
1242          * pending). Any requests in the hardware will drain before ndacleanup
1243          * is called.
1244          */
1245         cam_iosched_flush(softc->cam_iosched, NULL, ENXIO);
1246
1247         /*
1248          * Tell GEOM that we've gone away, we'll get a callback when it is
1249          * done cleaning up its resources.
1250          */
1251         disk_gone(softc->disk);
1252 }
1253
1254 static void
1255 adacleanup(struct cam_periph *periph)
1256 {
1257         struct ada_softc *softc;
1258
1259         softc = (struct ada_softc *)periph->softc;
1260
1261         cam_periph_unlock(periph);
1262
1263         cam_iosched_fini(softc->cam_iosched);
1264
1265         /*
1266          * If we can't free the sysctl tree, oh well...
1267          */
1268         if ((softc->flags & ADA_FLAG_SCTX_INIT) != 0) {
1269 #ifdef CAM_IO_STATS
1270                 if (sysctl_ctx_free(&softc->sysctl_stats_ctx) != 0)
1271                         xpt_print(periph->path,
1272                             "can't remove sysctl stats context\n");
1273 #endif
1274                 if (sysctl_ctx_free(&softc->sysctl_ctx) != 0)
1275                         xpt_print(periph->path,
1276                             "can't remove sysctl context\n");
1277         }
1278
1279         disk_destroy(softc->disk);
1280         callout_drain(&softc->sendordered_c);
1281         free(softc, M_DEVBUF);
1282         cam_periph_lock(periph);
1283 }
1284
1285 static void
1286 adasetdeletemethod(struct ada_softc *softc)
1287 {
1288
1289         if (softc->flags & ADA_FLAG_CAN_NCQ_TRIM)
1290                 softc->delete_method = ADA_DELETE_NCQ_DSM_TRIM;
1291         else if (softc->flags & ADA_FLAG_CAN_TRIM)
1292                 softc->delete_method = ADA_DELETE_DSM_TRIM;
1293         else if ((softc->flags & ADA_FLAG_CAN_CFA) && !(softc->flags & ADA_FLAG_CAN_48BIT))
1294                 softc->delete_method = ADA_DELETE_CFA_ERASE;
1295         else
1296                 softc->delete_method = ADA_DELETE_NONE;
1297 }
1298
1299 static void
1300 adaasync(void *callback_arg, uint32_t code,
1301         struct cam_path *path, void *arg)
1302 {
1303         struct ccb_getdev cgd;
1304         struct cam_periph *periph;
1305         struct ada_softc *softc;
1306
1307         periph = (struct cam_periph *)callback_arg;
1308         switch (code) {
1309         case AC_FOUND_DEVICE:
1310         {
1311                 struct ccb_getdev *cgd;
1312                 cam_status status;
1313
1314                 cgd = (struct ccb_getdev *)arg;
1315                 if (cgd == NULL)
1316                         break;
1317
1318                 if (cgd->protocol != PROTO_ATA)
1319                         break;
1320
1321                 /*
1322                  * Allocate a peripheral instance for
1323                  * this device and start the probe
1324                  * process.
1325                  */
1326                 status = cam_periph_alloc(adaregister, adaoninvalidate,
1327                                           adacleanup, adastart,
1328                                           "ada", CAM_PERIPH_BIO,
1329                                           path, adaasync,
1330                                           AC_FOUND_DEVICE, cgd);
1331
1332                 if (status != CAM_REQ_CMP
1333                  && status != CAM_REQ_INPROG)
1334                         printf("adaasync: Unable to attach to new device "
1335                                 "due to status 0x%x\n", status);
1336                 break;
1337         }
1338         case AC_GETDEV_CHANGED:
1339         {
1340                 softc = (struct ada_softc *)periph->softc;
1341                 memset(&cgd, 0, sizeof(cgd));
1342                 xpt_setup_ccb(&cgd.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
1343                 cgd.ccb_h.func_code = XPT_GDEV_TYPE;
1344                 xpt_action((union ccb *)&cgd);
1345
1346                 /*
1347                  * Update our information based on the new Identify data.
1348                  */
1349                 adasetflags(softc, &cgd);
1350                 adasetgeom(softc, &cgd);
1351                 disk_resize(softc->disk, M_NOWAIT);
1352                 break;
1353         }
1354         case AC_ADVINFO_CHANGED:
1355         {
1356                 uintptr_t buftype;
1357
1358                 buftype = (uintptr_t)arg;
1359                 if (buftype == CDAI_TYPE_PHYS_PATH) {
1360                         struct ada_softc *softc;
1361
1362                         softc = periph->softc;
1363                         disk_attr_changed(softc->disk, "GEOM::physpath",
1364                                           M_NOWAIT);
1365                 }
1366                 break;
1367         }
1368         case AC_SENT_BDR:
1369         case AC_BUS_RESET:
1370         {
1371                 softc = (struct ada_softc *)periph->softc;
1372                 if (softc->state != ADA_STATE_NORMAL)
1373                         break;
1374                 if (ADA_RA >= 0 && softc->flags & ADA_FLAG_CAN_RAHEAD)
1375                         softc->state = ADA_STATE_RAHEAD;
1376                 else if (ADA_WC >= 0 && softc->flags & ADA_FLAG_CAN_WCACHE)
1377                         softc->state = ADA_STATE_WCACHE;
1378                 else if ((softc->flags & ADA_FLAG_CAN_LOG)
1379                       && (softc->zone_mode != ADA_ZONE_NONE))
1380                         softc->state = ADA_STATE_LOGDIR;
1381                 else
1382                         break;
1383                 if (cam_periph_acquire(periph) != 0)
1384                         softc->state = ADA_STATE_NORMAL;
1385                 else
1386                         xpt_schedule(periph, CAM_PRIORITY_DEV);
1387                 break;
1388         }
1389         default:
1390                 break;
1391         }
1392         cam_periph_async(periph, code, path, arg);
1393 }
1394
1395 static int
1396 adazonemodesysctl(SYSCTL_HANDLER_ARGS)
1397 {
1398         char tmpbuf[40];
1399         struct ada_softc *softc;
1400         int error;
1401
1402         softc = (struct ada_softc *)arg1;
1403
1404         switch (softc->zone_mode) {
1405         case ADA_ZONE_DRIVE_MANAGED:
1406                 snprintf(tmpbuf, sizeof(tmpbuf), "Drive Managed");
1407                 break;
1408         case ADA_ZONE_HOST_AWARE:
1409                 snprintf(tmpbuf, sizeof(tmpbuf), "Host Aware");
1410                 break;
1411         case ADA_ZONE_HOST_MANAGED:
1412                 snprintf(tmpbuf, sizeof(tmpbuf), "Host Managed");
1413                 break;
1414         case ADA_ZONE_NONE:
1415         default:
1416                 snprintf(tmpbuf, sizeof(tmpbuf), "Not Zoned");
1417                 break;
1418         }
1419
1420         error = sysctl_handle_string(oidp, tmpbuf, sizeof(tmpbuf), req);
1421
1422         return (error);
1423 }
1424
1425 static int
1426 adazonesupsysctl(SYSCTL_HANDLER_ARGS)
1427 {
1428         struct ada_softc *softc;
1429         struct sbuf sb;
1430         int error, first;
1431         unsigned int i;
1432
1433         softc = (struct ada_softc *)arg1;
1434
1435         first = 1;
1436         sbuf_new_for_sysctl(&sb, NULL, 0, req);
1437
1438         for (i = 0; i < sizeof(ada_zone_desc_table) /
1439              sizeof(ada_zone_desc_table[0]); i++) {
1440                 if (softc->zone_flags & ada_zone_desc_table[i].value) {
1441                         if (first == 0)
1442                                 sbuf_cat(&sb, ", ");
1443                         else
1444                                 first = 0;
1445                         sbuf_cat(&sb, ada_zone_desc_table[i].desc);
1446                 }
1447         }
1448
1449         if (first == 1)
1450                 sbuf_cat(&sb, "None");
1451
1452         error = sbuf_finish(&sb);
1453         sbuf_delete(&sb);
1454         return (error);
1455 }
1456
1457 static void
1458 adasysctlinit(void *context, int pending)
1459 {
1460         struct cam_periph *periph;
1461         struct ada_softc *softc;
1462         char tmpstr[32], tmpstr2[16];
1463
1464         periph = (struct cam_periph *)context;
1465
1466         /* periph was held for us when this task was enqueued */
1467         if ((periph->flags & CAM_PERIPH_INVALID) != 0) {
1468                 cam_periph_release(periph);
1469                 return;
1470         }
1471
1472         softc = (struct ada_softc *)periph->softc;
1473         snprintf(tmpstr, sizeof(tmpstr), "CAM ADA unit %d",periph->unit_number);
1474         snprintf(tmpstr2, sizeof(tmpstr2), "%d", periph->unit_number);
1475
1476         sysctl_ctx_init(&softc->sysctl_ctx);
1477         softc->flags |= ADA_FLAG_SCTX_INIT;
1478         softc->sysctl_tree = SYSCTL_ADD_NODE_WITH_LABEL(&softc->sysctl_ctx,
1479                 SYSCTL_STATIC_CHILDREN(_kern_cam_ada), OID_AUTO, tmpstr2,
1480                 CTLFLAG_RD | CTLFLAG_MPSAFE, 0, tmpstr, "device_index");
1481         if (softc->sysctl_tree == NULL) {
1482                 printf("adasysctlinit: unable to allocate sysctl tree\n");
1483                 cam_periph_release(periph);
1484                 return;
1485         }
1486
1487         SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1488                 OID_AUTO, "delete_method",
1489                 CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE,
1490                 softc, 0, adadeletemethodsysctl, "A",
1491                 "BIO_DELETE execution method");
1492         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
1493                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
1494                 "trim_count", CTLFLAG_RD, &softc->trim_count,
1495                 "Total number of dsm commands sent");
1496         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
1497                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
1498                 "trim_ranges", CTLFLAG_RD, &softc->trim_ranges,
1499                 "Total number of ranges in dsm commands");
1500         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
1501                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
1502                 "trim_lbas", CTLFLAG_RD, &softc->trim_lbas,
1503                 "Total lbas in the dsm commands sent");
1504         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1505                 OID_AUTO, "read_ahead", CTLFLAG_RW | CTLFLAG_MPSAFE,
1506                 &softc->read_ahead, 0, "Enable disk read ahead.");
1507         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1508                 OID_AUTO, "write_cache", CTLFLAG_RW | CTLFLAG_MPSAFE,
1509                 &softc->write_cache, 0, "Enable disk write cache.");
1510         SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1511                 OID_AUTO, "zone_mode",
1512                 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE,
1513                 softc, 0, adazonemodesysctl, "A",
1514                 "Zone Mode");
1515         SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1516                 OID_AUTO, "zone_support",
1517                 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE,
1518                 softc, 0, adazonesupsysctl, "A",
1519                 "Zone Support");
1520         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
1521                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
1522                 "optimal_seq_zones", CTLFLAG_RD, &softc->optimal_seq_zones,
1523                 "Optimal Number of Open Sequential Write Preferred Zones");
1524         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
1525                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
1526                 "optimal_nonseq_zones", CTLFLAG_RD,
1527                 &softc->optimal_nonseq_zones,
1528                 "Optimal Number of Non-Sequentially Written Sequential Write "
1529                 "Preferred Zones");
1530         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
1531                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
1532                 "max_seq_zones", CTLFLAG_RD, &softc->max_seq_zones,
1533                 "Maximum Number of Open Sequential Write Required Zones");
1534         SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1535             OID_AUTO, "flags", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE,
1536             softc, 0, adaflagssysctl, "A",
1537             "Flags for drive");
1538         SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1539             OID_AUTO, "unmapped_io", CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MPSAFE,
1540             &softc->flags, (u_int)ADA_FLAG_UNMAPPEDIO, adabitsysctl, "I",
1541             "Use unmapped I/O. This sysctl is *DEPRECATED*, gone in FreeBSD 15");
1542         SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1543             OID_AUTO, "rotating", CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MPSAFE,
1544             &softc->flags, (u_int)ADA_FLAG_ROTATING, adabitsysctl, "I",
1545             "Rotating media. This sysctl is *DEPRECATED*, gone in FreeBSD 15");
1546
1547 #ifdef CAM_TEST_FAILURE
1548         /*
1549          * Add a 'door bell' sysctl which allows one to set it from userland
1550          * and cause something bad to happen.  For the moment, we only allow
1551          * whacking the next read or write.
1552          */
1553         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1554                 OID_AUTO, "force_read_error", CTLFLAG_RW | CTLFLAG_MPSAFE,
1555                 &softc->force_read_error, 0,
1556                 "Force a read error for the next N reads.");
1557         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1558                 OID_AUTO, "force_write_error", CTLFLAG_RW | CTLFLAG_MPSAFE,
1559                 &softc->force_write_error, 0,
1560                 "Force a write error for the next N writes.");
1561         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1562                 OID_AUTO, "periodic_read_error", CTLFLAG_RW | CTLFLAG_MPSAFE,
1563                 &softc->periodic_read_error, 0,
1564                 "Force a read error every N reads (don't set too low).");
1565         SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1566                 OID_AUTO, "invalidate", CTLTYPE_U64 | CTLFLAG_RW | CTLFLAG_MPSAFE,
1567                 periph, 0, cam_periph_invalidate_sysctl, "I",
1568                 "Write 1 to invalidate the drive immediately");
1569 #endif
1570
1571 #ifdef CAM_IO_STATS
1572         softc->sysctl_stats_tree = SYSCTL_ADD_NODE(&softc->sysctl_stats_ctx,
1573                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO, "stats",
1574                 CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "Statistics");
1575         SYSCTL_ADD_INT(&softc->sysctl_stats_ctx,
1576                 SYSCTL_CHILDREN(softc->sysctl_stats_tree),
1577                 OID_AUTO, "timeouts", CTLFLAG_RD | CTLFLAG_MPSAFE,
1578                 &softc->timeouts, 0,
1579                 "Device timeouts reported by the SIM");
1580         SYSCTL_ADD_INT(&softc->sysctl_stats_ctx,
1581                 SYSCTL_CHILDREN(softc->sysctl_stats_tree),
1582                 OID_AUTO, "errors", CTLFLAG_RD | CTLFLAG_MPSAFE,
1583                 &softc->errors, 0,
1584                 "Transport errors reported by the SIM.");
1585         SYSCTL_ADD_INT(&softc->sysctl_stats_ctx,
1586                 SYSCTL_CHILDREN(softc->sysctl_stats_tree),
1587                 OID_AUTO, "pack_invalidations", CTLFLAG_RD | CTLFLAG_MPSAFE,
1588                 &softc->invalidations, 0,
1589                 "Device pack invalidations.");
1590 #endif
1591
1592         cam_iosched_sysctl_init(softc->cam_iosched, &softc->sysctl_ctx,
1593             softc->sysctl_tree);
1594
1595         cam_periph_release(periph);
1596 }
1597
1598 static int
1599 adagetattr(struct bio *bp)
1600 {
1601         int ret;
1602         struct cam_periph *periph;
1603
1604         if (g_handleattr_int(bp, "GEOM::canspeedup", ada_enable_biospeedup))
1605                 return (EJUSTRETURN);
1606
1607         periph = (struct cam_periph *)bp->bio_disk->d_drv1;
1608         cam_periph_lock(periph);
1609         ret = xpt_getattr(bp->bio_data, bp->bio_length, bp->bio_attribute,
1610             periph->path);
1611         cam_periph_unlock(periph);
1612         if (ret == 0)
1613                 bp->bio_completed = bp->bio_length;
1614         return ret;
1615 }
1616
1617 static int
1618 adadeletemethodsysctl(SYSCTL_HANDLER_ARGS)
1619 {
1620         char buf[16];
1621         const char *p;
1622         struct ada_softc *softc;
1623         int i, error, value, methods;
1624
1625         softc = (struct ada_softc *)arg1;
1626
1627         value = softc->delete_method;
1628         if (value < 0 || value > ADA_DELETE_MAX)
1629                 p = "UNKNOWN";
1630         else
1631                 p = ada_delete_method_names[value];
1632         strncpy(buf, p, sizeof(buf));
1633         error = sysctl_handle_string(oidp, buf, sizeof(buf), req);
1634         if (error != 0 || req->newptr == NULL)
1635                 return (error);
1636         methods = 1 << ADA_DELETE_DISABLE;
1637         if ((softc->flags & ADA_FLAG_CAN_CFA) &&
1638             !(softc->flags & ADA_FLAG_CAN_48BIT))
1639                 methods |= 1 << ADA_DELETE_CFA_ERASE;
1640         if (softc->flags & ADA_FLAG_CAN_TRIM)
1641                 methods |= 1 << ADA_DELETE_DSM_TRIM;
1642         if (softc->flags & ADA_FLAG_CAN_NCQ_TRIM)
1643                 methods |= 1 << ADA_DELETE_NCQ_DSM_TRIM;
1644         for (i = 0; i <= ADA_DELETE_MAX; i++) {
1645                 if (!(methods & (1 << i)) ||
1646                     strcmp(buf, ada_delete_method_names[i]) != 0)
1647                         continue;
1648                 softc->delete_method = i;
1649                 return (0);
1650         }
1651         return (EINVAL);
1652 }
1653
1654 static int
1655 adabitsysctl(SYSCTL_HANDLER_ARGS)
1656 {
1657         u_int *flags = arg1;
1658         u_int test = arg2;
1659         int tmpout, error;
1660
1661         tmpout = !!(*flags & test);
1662         error = SYSCTL_OUT(req, &tmpout, sizeof(tmpout));
1663         if (error || !req->newptr)
1664                 return (error);
1665
1666         return (EPERM);
1667 }
1668
1669 static int
1670 adaflagssysctl(SYSCTL_HANDLER_ARGS)
1671 {
1672         struct sbuf sbuf;
1673         struct ada_softc *softc = arg1;
1674         int error;
1675
1676         sbuf_new_for_sysctl(&sbuf, NULL, 0, req);
1677         if (softc->flags != 0)
1678                 sbuf_printf(&sbuf, "0x%b", (unsigned)softc->flags, ADA_FLAG_STRING);
1679         else
1680                 sbuf_putc(&sbuf, '0');
1681         error = sbuf_finish(&sbuf);
1682         sbuf_delete(&sbuf);
1683
1684         return (error);
1685 }
1686
1687 static void
1688 adasetflags(struct ada_softc *softc, struct ccb_getdev *cgd)
1689 {
1690         if ((cgd->ident_data.capabilities1 & ATA_SUPPORT_DMA) &&
1691             (cgd->inq_flags & SID_DMA))
1692                 softc->flags |= ADA_FLAG_CAN_DMA;
1693         else
1694                 softc->flags &= ~ADA_FLAG_CAN_DMA;
1695
1696         if (cgd->ident_data.support.command2 & ATA_SUPPORT_ADDRESS48) {
1697                 softc->flags |= ADA_FLAG_CAN_48BIT;
1698                 if (cgd->inq_flags & SID_DMA48)
1699                         softc->flags |= ADA_FLAG_CAN_DMA48;
1700                 else
1701                         softc->flags &= ~ADA_FLAG_CAN_DMA48;
1702         } else
1703                 softc->flags &= ~(ADA_FLAG_CAN_48BIT | ADA_FLAG_CAN_DMA48);
1704
1705         if (cgd->ident_data.support.command2 & ATA_SUPPORT_FLUSHCACHE)
1706                 softc->flags |= ADA_FLAG_CAN_FLUSHCACHE;
1707         else
1708                 softc->flags &= ~ADA_FLAG_CAN_FLUSHCACHE;
1709
1710         if (cgd->ident_data.support.command1 & ATA_SUPPORT_POWERMGT)
1711                 softc->flags |= ADA_FLAG_CAN_POWERMGT;
1712         else
1713                 softc->flags &= ~ADA_FLAG_CAN_POWERMGT;
1714
1715         if ((cgd->ident_data.satacapabilities & ATA_SUPPORT_NCQ) &&
1716             (cgd->inq_flags & SID_DMA) && (cgd->inq_flags & SID_CmdQue))
1717                 softc->flags |= ADA_FLAG_CAN_NCQ;
1718         else
1719                 softc->flags &= ~ADA_FLAG_CAN_NCQ;
1720
1721         if ((cgd->ident_data.support_dsm & ATA_SUPPORT_DSM_TRIM) &&
1722             (cgd->inq_flags & SID_DMA) &&
1723             (softc->quirks & ADA_Q_NO_TRIM) == 0) {
1724                 softc->flags |= ADA_FLAG_CAN_TRIM;
1725                 softc->trim_max_ranges = TRIM_MAX_RANGES;
1726                 if (cgd->ident_data.max_dsm_blocks != 0) {
1727                         softc->trim_max_ranges =
1728                             min(cgd->ident_data.max_dsm_blocks *
1729                                 ATA_DSM_BLK_RANGES, softc->trim_max_ranges);
1730                 }
1731                 /*
1732                  * If we can do RCVSND_FPDMA_QUEUED commands, we may be able
1733                  * to do NCQ trims, if we support trims at all. We also need
1734                  * support from the SIM to do things properly. Perhaps we
1735                  * should look at log 13 dword 0 bit 0 and dword 1 bit 0 are
1736                  * set too...
1737                  */
1738                 if ((softc->quirks & ADA_Q_NCQ_TRIM_BROKEN) == 0 &&
1739                     (softc->flags & ADA_FLAG_PIM_ATA_EXT) != 0 &&
1740                     (cgd->ident_data.satacapabilities2 &
1741                      ATA_SUPPORT_RCVSND_FPDMA_QUEUED) != 0 &&
1742                     (softc->flags & ADA_FLAG_CAN_TRIM) != 0)
1743                         softc->flags |= ADA_FLAG_CAN_NCQ_TRIM;
1744                 else
1745                         softc->flags &= ~ADA_FLAG_CAN_NCQ_TRIM;
1746         } else
1747                 softc->flags &= ~(ADA_FLAG_CAN_TRIM | ADA_FLAG_CAN_NCQ_TRIM);
1748
1749         if (cgd->ident_data.support.command2 & ATA_SUPPORT_CFA)
1750                 softc->flags |= ADA_FLAG_CAN_CFA;
1751         else
1752                 softc->flags &= ~ADA_FLAG_CAN_CFA;
1753
1754         /*
1755          * Now that we've set the appropriate flags, setup the delete
1756          * method.
1757          */
1758         adasetdeletemethod(softc);
1759
1760         if ((cgd->ident_data.support.extension & ATA_SUPPORT_GENLOG)
1761          && ((softc->quirks & ADA_Q_LOG_BROKEN) == 0))
1762                 softc->flags |= ADA_FLAG_CAN_LOG;
1763         else
1764                 softc->flags &= ~ADA_FLAG_CAN_LOG;
1765
1766         if ((cgd->ident_data.support3 & ATA_SUPPORT_ZONE_MASK) ==
1767              ATA_SUPPORT_ZONE_HOST_AWARE)
1768                 softc->zone_mode = ADA_ZONE_HOST_AWARE;
1769         else if (((cgd->ident_data.support3 & ATA_SUPPORT_ZONE_MASK) ==
1770                    ATA_SUPPORT_ZONE_DEV_MANAGED)
1771               || (softc->quirks & ADA_Q_SMR_DM))
1772                 softc->zone_mode = ADA_ZONE_DRIVE_MANAGED;
1773         else
1774                 softc->zone_mode = ADA_ZONE_NONE;
1775
1776         if (cgd->ident_data.support.command1 & ATA_SUPPORT_LOOKAHEAD)
1777                 softc->flags |= ADA_FLAG_CAN_RAHEAD;
1778         else
1779                 softc->flags &= ~ADA_FLAG_CAN_RAHEAD;
1780
1781         if (cgd->ident_data.support.command1 & ATA_SUPPORT_WRITECACHE)
1782                 softc->flags |= ADA_FLAG_CAN_WCACHE;
1783         else
1784                 softc->flags &= ~ADA_FLAG_CAN_WCACHE;
1785 }
1786
1787 static cam_status
1788 adaregister(struct cam_periph *periph, void *arg)
1789 {
1790         struct ada_softc *softc;
1791         struct ccb_getdev *cgd;
1792         struct disk_params *dp;
1793         struct sbuf sb;
1794         char   *announce_buf;
1795         caddr_t match;
1796         int quirks;
1797
1798         cgd = (struct ccb_getdev *)arg;
1799         if (cgd == NULL) {
1800                 printf("adaregister: no getdev CCB, can't register device\n");
1801                 return(CAM_REQ_CMP_ERR);
1802         }
1803
1804         softc = (struct ada_softc *)malloc(sizeof(*softc), M_DEVBUF,
1805             M_NOWAIT|M_ZERO);
1806
1807         if (softc == NULL) {
1808                 printf("adaregister: Unable to probe new device. "
1809                     "Unable to allocate softc\n");
1810                 return(CAM_REQ_CMP_ERR);
1811         }
1812
1813         announce_buf = softc->announce_temp;
1814         bzero(announce_buf, ADA_ANNOUNCETMP_SZ);
1815
1816         if (cam_iosched_init(&softc->cam_iosched, periph) != 0) {
1817                 printf("adaregister: Unable to probe new device. "
1818                        "Unable to allocate iosched memory\n");
1819                 free(softc, M_DEVBUF);
1820                 return(CAM_REQ_CMP_ERR);
1821         }
1822
1823         periph->softc = softc;
1824         xpt_path_inq(&softc->cpi, periph->path);
1825
1826         /*
1827          * See if this device has any quirks.
1828          */
1829         match = cam_quirkmatch((caddr_t)&cgd->ident_data,
1830                                (caddr_t)ada_quirk_table,
1831                                nitems(ada_quirk_table),
1832                                sizeof(*ada_quirk_table), ata_identify_match);
1833         if (match != NULL)
1834                 softc->quirks = ((struct ada_quirk_entry *)match)->quirks;
1835         else
1836                 softc->quirks = ADA_Q_NONE;
1837
1838         TASK_INIT(&softc->sysctl_task, 0, adasysctlinit, periph);
1839
1840         /*
1841          * Take a reference on the periph while adastart is called to finish
1842          * the probe.  The reference will be dropped in adaprobedone at the
1843          * end of probe.
1844          */
1845         (void)cam_periph_acquire(periph);
1846         cam_periph_unlock(periph);
1847         snprintf(announce_buf, ADA_ANNOUNCETMP_SZ,
1848             "kern.cam.ada.%d.quirks", periph->unit_number);
1849         quirks = softc->quirks;
1850         TUNABLE_INT_FETCH(announce_buf, &quirks);
1851         softc->quirks = quirks;
1852         softc->read_ahead = -1;
1853         snprintf(announce_buf, ADA_ANNOUNCETMP_SZ,
1854             "kern.cam.ada.%d.read_ahead", periph->unit_number);
1855         TUNABLE_INT_FETCH(announce_buf, &softc->read_ahead);
1856         softc->write_cache = -1;
1857         snprintf(announce_buf, ADA_ANNOUNCETMP_SZ,
1858             "kern.cam.ada.%d.write_cache", periph->unit_number);
1859         TUNABLE_INT_FETCH(announce_buf, &softc->write_cache);
1860
1861         /*
1862          * Let XPT know we can use UMA-allocated CCBs.
1863          */
1864         if (ada_enable_uma_ccbs) {
1865                 KASSERT(ada_ccb_zone != NULL,
1866                     ("%s: NULL ada_ccb_zone", __func__));
1867                 periph->ccb_zone = ada_ccb_zone;
1868         }
1869
1870         /*
1871          * Set support flags based on the Identify data and quirks.
1872          */
1873         adasetflags(softc, cgd);
1874         if (softc->cpi.hba_misc & PIM_ATA_EXT)
1875                 softc->flags |= ADA_FLAG_PIM_ATA_EXT;
1876
1877         /* Disable queue sorting for non-rotational media by default. */
1878         if (cgd->ident_data.media_rotation_rate == ATA_RATE_NON_ROTATING) {
1879                 softc->flags &= ~ADA_FLAG_ROTATING;
1880         } else {
1881                 softc->flags |= ADA_FLAG_ROTATING;
1882         }
1883         cam_iosched_set_sort_queue(softc->cam_iosched,
1884             (softc->flags & ADA_FLAG_ROTATING) ? -1 : 0);
1885         softc->disk = disk_alloc();
1886         adasetgeom(softc, cgd);
1887         softc->disk->d_devstat = devstat_new_entry(periph->periph_name,
1888                           periph->unit_number, softc->params.secsize,
1889                           DEVSTAT_ALL_SUPPORTED,
1890                           DEVSTAT_TYPE_DIRECT |
1891                           XPORT_DEVSTAT_TYPE(softc->cpi.transport),
1892                           DEVSTAT_PRIORITY_DISK);
1893         softc->disk->d_open = adaopen;
1894         softc->disk->d_close = adaclose;
1895         softc->disk->d_strategy = adastrategy;
1896         softc->disk->d_getattr = adagetattr;
1897         if (cam_sim_pollable(periph->sim))
1898                 softc->disk->d_dump = adadump;
1899         softc->disk->d_gone = adadiskgonecb;
1900         softc->disk->d_name = "ada";
1901         softc->disk->d_drv1 = periph;
1902         softc->disk->d_unit = periph->unit_number;
1903         cam_periph_lock(periph);
1904
1905         dp = &softc->params;
1906         snprintf(announce_buf, ADA_ANNOUNCETMP_SZ,
1907             "%juMB (%ju %u byte sectors)",
1908             ((uintmax_t)dp->secsize * dp->sectors) / (1024 * 1024),
1909             (uintmax_t)dp->sectors, dp->secsize);
1910
1911         sbuf_new(&sb, softc->announce_buffer, ADA_ANNOUNCE_SZ, SBUF_FIXEDLEN);
1912         xpt_announce_periph_sbuf(periph, &sb, announce_buf);
1913         xpt_announce_quirks_sbuf(periph, &sb, softc->quirks, ADA_Q_BIT_STRING);
1914         sbuf_finish(&sb);
1915         sbuf_putbuf(&sb);
1916
1917         /*
1918          * Create our sysctl variables, now that we know
1919          * we have successfully attached.
1920          */
1921         if (cam_periph_acquire(periph) == 0)
1922                 taskqueue_enqueue(taskqueue_thread, &softc->sysctl_task);
1923
1924         /*
1925          * Add async callbacks for bus reset and
1926          * bus device reset calls.  I don't bother
1927          * checking if this fails as, in most cases,
1928          * the system will function just fine without
1929          * them and the only alternative would be to
1930          * not attach the device on failure.
1931          */
1932         xpt_register_async(AC_SENT_BDR | AC_BUS_RESET | AC_LOST_DEVICE |
1933             AC_GETDEV_CHANGED | AC_ADVINFO_CHANGED,
1934             adaasync, periph, periph->path);
1935
1936         /*
1937          * Schedule a periodic event to occasionally send an
1938          * ordered tag to a device.
1939          */
1940         callout_init_mtx(&softc->sendordered_c, cam_periph_mtx(periph), 0);
1941         callout_reset_sbt(&softc->sendordered_c,
1942             SBT_1S / ADA_ORDEREDTAG_INTERVAL * ada_default_timeout, 0,
1943             adasendorderedtag, softc, C_PREL(1));
1944
1945         /* Released after probe when disk_create() call pass it to GEOM. */
1946         cam_periph_hold_boot(periph);
1947
1948         if (ADA_RA >= 0 && softc->flags & ADA_FLAG_CAN_RAHEAD) {
1949                 softc->state = ADA_STATE_RAHEAD;
1950         } else if (ADA_WC >= 0 && softc->flags & ADA_FLAG_CAN_WCACHE) {
1951                 softc->state = ADA_STATE_WCACHE;
1952         } else if ((softc->flags & ADA_FLAG_CAN_LOG)
1953                 && (softc->zone_mode != ADA_ZONE_NONE)) {
1954                 softc->state = ADA_STATE_LOGDIR;
1955         } else {
1956                 /*
1957                  * Nothing to probe, so we can just transition to the
1958                  * normal state.
1959                  */
1960                 adaprobedone(periph, NULL);
1961                 return(CAM_REQ_CMP);
1962         }
1963
1964         xpt_schedule(periph, CAM_PRIORITY_DEV);
1965         return(CAM_REQ_CMP);
1966 }
1967
1968 static int
1969 ada_dsmtrim_req_create(struct ada_softc *softc, struct bio *bp, struct trim_request *req)
1970 {
1971         uint64_t lastlba = (uint64_t)-1, lbas = 0;
1972         int c, lastcount = 0, off, ranges = 0;
1973
1974         bzero(req, sizeof(*req));
1975         TAILQ_INIT(&req->bps);
1976         do {
1977                 uint64_t lba = bp->bio_pblkno;
1978                 int count = bp->bio_bcount / softc->params.secsize;
1979
1980                 /* Try to extend the previous range. */
1981                 if (lba == lastlba) {
1982                         c = min(count, ATA_DSM_RANGE_MAX - lastcount);
1983                         lastcount += c;
1984                         off = (ranges - 1) * ATA_DSM_RANGE_SIZE;
1985                         req->data[off + 6] = lastcount & 0xff;
1986                         req->data[off + 7] =
1987                                 (lastcount >> 8) & 0xff;
1988                         count -= c;
1989                         lba += c;
1990                         lbas += c;
1991                 }
1992
1993                 while (count > 0) {
1994                         c = min(count, ATA_DSM_RANGE_MAX);
1995                         off = ranges * ATA_DSM_RANGE_SIZE;
1996                         req->data[off + 0] = lba & 0xff;
1997                         req->data[off + 1] = (lba >> 8) & 0xff;
1998                         req->data[off + 2] = (lba >> 16) & 0xff;
1999                         req->data[off + 3] = (lba >> 24) & 0xff;
2000                         req->data[off + 4] = (lba >> 32) & 0xff;
2001                         req->data[off + 5] = (lba >> 40) & 0xff;
2002                         req->data[off + 6] = c & 0xff;
2003                         req->data[off + 7] = (c >> 8) & 0xff;
2004                         lba += c;
2005                         lbas += c;
2006                         count -= c;
2007                         lastcount = c;
2008                         ranges++;
2009                         /*
2010                          * Its the caller's responsibility to ensure the
2011                          * request will fit so we don't need to check for
2012                          * overrun here
2013                          */
2014                 }
2015                 lastlba = lba;
2016                 TAILQ_INSERT_TAIL(&req->bps, bp, bio_queue);
2017
2018                 bp = cam_iosched_next_trim(softc->cam_iosched);
2019                 if (bp == NULL)
2020                         break;
2021                 if (bp->bio_bcount / softc->params.secsize >
2022                     (softc->trim_max_ranges - ranges) * ATA_DSM_RANGE_MAX) {
2023                         cam_iosched_put_back_trim(softc->cam_iosched, bp);
2024                         break;
2025                 }
2026         } while (1);
2027         softc->trim_count++;
2028         softc->trim_ranges += ranges;
2029         softc->trim_lbas += lbas;
2030
2031         return (ranges);
2032 }
2033
2034 static void
2035 ada_dsmtrim(struct ada_softc *softc, struct bio *bp, struct ccb_ataio *ataio)
2036 {
2037         struct trim_request *req = &softc->trim_req;
2038         int ranges;
2039
2040         ranges = ada_dsmtrim_req_create(softc, bp, req);
2041         cam_fill_ataio(ataio,
2042             ada_retry_count,
2043             adadone,
2044             CAM_DIR_OUT,
2045             0,
2046             req->data,
2047             howmany(ranges, ATA_DSM_BLK_RANGES) * ATA_DSM_BLK_SIZE,
2048             ada_default_timeout * 1000);
2049         ata_48bit_cmd(ataio, ATA_DATA_SET_MANAGEMENT,
2050             ATA_DSM_TRIM, 0, howmany(ranges, ATA_DSM_BLK_RANGES));
2051 }
2052
2053 static void
2054 ada_ncq_dsmtrim(struct ada_softc *softc, struct bio *bp, struct ccb_ataio *ataio)
2055 {
2056         struct trim_request *req = &softc->trim_req;
2057         int ranges;
2058
2059         ranges = ada_dsmtrim_req_create(softc, bp, req);
2060         cam_fill_ataio(ataio,
2061             ada_retry_count,
2062             adadone,
2063             CAM_DIR_OUT,
2064             0,
2065             req->data,
2066             howmany(ranges, ATA_DSM_BLK_RANGES) * ATA_DSM_BLK_SIZE,
2067             ada_default_timeout * 1000);
2068         ata_ncq_cmd(ataio,
2069             ATA_SEND_FPDMA_QUEUED,
2070             0,
2071             howmany(ranges, ATA_DSM_BLK_RANGES));
2072         ataio->cmd.sector_count_exp = ATA_SFPDMA_DSM;
2073         ataio->ata_flags |= ATA_FLAG_AUX;
2074         ataio->aux = 1;
2075 }
2076
2077 static void
2078 ada_cfaerase(struct ada_softc *softc, struct bio *bp, struct ccb_ataio *ataio)
2079 {
2080         struct trim_request *req = &softc->trim_req;
2081         uint64_t lba = bp->bio_pblkno;
2082         uint16_t count = bp->bio_bcount / softc->params.secsize;
2083
2084         bzero(req, sizeof(*req));
2085         TAILQ_INIT(&req->bps);
2086         TAILQ_INSERT_TAIL(&req->bps, bp, bio_queue);
2087
2088         cam_fill_ataio(ataio,
2089             ada_retry_count,
2090             adadone,
2091             CAM_DIR_NONE,
2092             0,
2093             NULL,
2094             0,
2095             ada_default_timeout*1000);
2096
2097         if (count >= 256)
2098                 count = 0;
2099         ata_28bit_cmd(ataio, ATA_CFA_ERASE, 0, lba, count);
2100 }
2101
2102 static int
2103 ada_zone_bio_to_ata(int disk_zone_cmd)
2104 {
2105         switch (disk_zone_cmd) {
2106         case DISK_ZONE_OPEN:
2107                 return ATA_ZM_OPEN_ZONE;
2108         case DISK_ZONE_CLOSE:
2109                 return ATA_ZM_CLOSE_ZONE;
2110         case DISK_ZONE_FINISH:
2111                 return ATA_ZM_FINISH_ZONE;
2112         case DISK_ZONE_RWP:
2113                 return ATA_ZM_RWP;
2114         }
2115
2116         return -1;
2117 }
2118
2119 static int
2120 ada_zone_cmd(struct cam_periph *periph, union ccb *ccb, struct bio *bp,
2121              int *queue_ccb)
2122 {
2123         struct ada_softc *softc;
2124         int error;
2125
2126         error = 0;
2127
2128         if (bp->bio_cmd != BIO_ZONE) {
2129                 error = EINVAL;
2130                 goto bailout;
2131         }
2132
2133         softc = periph->softc;
2134
2135         switch (bp->bio_zone.zone_cmd) {
2136         case DISK_ZONE_OPEN:
2137         case DISK_ZONE_CLOSE:
2138         case DISK_ZONE_FINISH:
2139         case DISK_ZONE_RWP: {
2140                 int zone_flags;
2141                 int zone_sa;
2142                 uint64_t lba;
2143
2144                 zone_sa = ada_zone_bio_to_ata(bp->bio_zone.zone_cmd);
2145                 if (zone_sa == -1) {
2146                         xpt_print(periph->path, "Cannot translate zone "
2147                             "cmd %#x to ATA\n", bp->bio_zone.zone_cmd);
2148                         error = EINVAL;
2149                         goto bailout;
2150                 }
2151
2152                 zone_flags = 0;
2153                 lba = bp->bio_zone.zone_params.rwp.id;
2154
2155                 if (bp->bio_zone.zone_params.rwp.flags &
2156                     DISK_ZONE_RWP_FLAG_ALL)
2157                         zone_flags |= ZBC_OUT_ALL;
2158
2159                 ata_zac_mgmt_out(&ccb->ataio,
2160                                  /*retries*/ ada_retry_count,
2161                                  /*cbfcnp*/ adadone,
2162                                  /*use_ncq*/ (softc->flags &
2163                                               ADA_FLAG_PIM_ATA_EXT) ? 1 : 0,
2164                                  /*zm_action*/ zone_sa,
2165                                  /*zone_id*/ lba,
2166                                  /*zone_flags*/ zone_flags,
2167                                  /*sector_count*/ 0,
2168                                  /*data_ptr*/ NULL,
2169                                  /*dxfer_len*/ 0,
2170                                  /*timeout*/ ada_default_timeout * 1000);
2171                 *queue_ccb = 1;
2172
2173                 break;
2174         }
2175         case DISK_ZONE_REPORT_ZONES: {
2176                 uint8_t *rz_ptr;
2177                 uint32_t num_entries, alloc_size;
2178                 struct disk_zone_report *rep;
2179
2180                 rep = &bp->bio_zone.zone_params.report;
2181
2182                 num_entries = rep->entries_allocated;
2183                 if (num_entries == 0) {
2184                         xpt_print(periph->path, "No entries allocated for "
2185                             "Report Zones request\n");
2186                         error = EINVAL;
2187                         goto bailout;
2188                 }
2189                 alloc_size = sizeof(struct scsi_report_zones_hdr) +
2190                     (sizeof(struct scsi_report_zones_desc) * num_entries);
2191                 alloc_size = min(alloc_size, softc->disk->d_maxsize);
2192                 rz_ptr = malloc(alloc_size, M_ATADA, M_NOWAIT | M_ZERO);
2193                 if (rz_ptr == NULL) {
2194                         xpt_print(periph->path, "Unable to allocate memory "
2195                            "for Report Zones request\n");
2196                         error = ENOMEM;
2197                         goto bailout;
2198                 }
2199
2200                 ata_zac_mgmt_in(&ccb->ataio,
2201                                 /*retries*/ ada_retry_count,
2202                                 /*cbcfnp*/ adadone,
2203                                 /*use_ncq*/ (softc->flags &
2204                                              ADA_FLAG_PIM_ATA_EXT) ? 1 : 0,
2205                                 /*zm_action*/ ATA_ZM_REPORT_ZONES,
2206                                 /*zone_id*/ rep->starting_id,
2207                                 /*zone_flags*/ rep->rep_options,
2208                                 /*data_ptr*/ rz_ptr,
2209                                 /*dxfer_len*/ alloc_size,
2210                                 /*timeout*/ ada_default_timeout * 1000);
2211
2212                 /*
2213                  * For BIO_ZONE, this isn't normally needed.  However, it
2214                  * is used by devstat_end_transaction_bio() to determine
2215                  * how much data was transferred.
2216                  */
2217                 /*
2218                  * XXX KDM we have a problem.  But I'm not sure how to fix
2219                  * it.  devstat uses bio_bcount - bio_resid to calculate
2220                  * the amount of data transferred.   The GEOM disk code
2221                  * uses bio_length - bio_resid to calculate the amount of
2222                  * data in bio_completed.  We have different structure
2223                  * sizes above and below the ada(4) driver.  So, if we
2224                  * use the sizes above, the amount transferred won't be
2225                  * quite accurate for devstat.  If we use different sizes
2226                  * for bio_bcount and bio_length (above and below
2227                  * respectively), then the residual needs to match one or
2228                  * the other.  Everything is calculated after the bio
2229                  * leaves the driver, so changing the values around isn't
2230                  * really an option.  For now, just set the count to the
2231                  * passed in length.  This means that the calculations
2232                  * above (e.g. bio_completed) will be correct, but the
2233                  * amount of data reported to devstat will be slightly
2234                  * under or overstated.
2235                  */
2236                 bp->bio_bcount = bp->bio_length;
2237
2238                 *queue_ccb = 1;
2239
2240                 break;
2241         }
2242         case DISK_ZONE_GET_PARAMS: {
2243                 struct disk_zone_disk_params *params;
2244
2245                 params = &bp->bio_zone.zone_params.disk_params;
2246                 bzero(params, sizeof(*params));
2247
2248                 switch (softc->zone_mode) {
2249                 case ADA_ZONE_DRIVE_MANAGED:
2250                         params->zone_mode = DISK_ZONE_MODE_DRIVE_MANAGED;
2251                         break;
2252                 case ADA_ZONE_HOST_AWARE:
2253                         params->zone_mode = DISK_ZONE_MODE_HOST_AWARE;
2254                         break;
2255                 case ADA_ZONE_HOST_MANAGED:
2256                         params->zone_mode = DISK_ZONE_MODE_HOST_MANAGED;
2257                         break;
2258                 default:
2259                 case ADA_ZONE_NONE:
2260                         params->zone_mode = DISK_ZONE_MODE_NONE;
2261                         break;
2262                 }
2263
2264                 if (softc->zone_flags & ADA_ZONE_FLAG_URSWRZ)
2265                         params->flags |= DISK_ZONE_DISK_URSWRZ;
2266
2267                 if (softc->zone_flags & ADA_ZONE_FLAG_OPT_SEQ_SET) {
2268                         params->optimal_seq_zones = softc->optimal_seq_zones;
2269                         params->flags |= DISK_ZONE_OPT_SEQ_SET;
2270                 }
2271
2272                 if (softc->zone_flags & ADA_ZONE_FLAG_OPT_NONSEQ_SET) {
2273                         params->optimal_nonseq_zones =
2274                             softc->optimal_nonseq_zones;
2275                         params->flags |= DISK_ZONE_OPT_NONSEQ_SET;
2276                 }
2277
2278                 if (softc->zone_flags & ADA_ZONE_FLAG_MAX_SEQ_SET) {
2279                         params->max_seq_zones = softc->max_seq_zones;
2280                         params->flags |= DISK_ZONE_MAX_SEQ_SET;
2281                 }
2282                 if (softc->zone_flags & ADA_ZONE_FLAG_RZ_SUP)
2283                         params->flags |= DISK_ZONE_RZ_SUP;
2284
2285                 if (softc->zone_flags & ADA_ZONE_FLAG_OPEN_SUP)
2286                         params->flags |= DISK_ZONE_OPEN_SUP;
2287
2288                 if (softc->zone_flags & ADA_ZONE_FLAG_CLOSE_SUP)
2289                         params->flags |= DISK_ZONE_CLOSE_SUP;
2290
2291                 if (softc->zone_flags & ADA_ZONE_FLAG_FINISH_SUP)
2292                         params->flags |= DISK_ZONE_FINISH_SUP;
2293
2294                 if (softc->zone_flags & ADA_ZONE_FLAG_RWP_SUP)
2295                         params->flags |= DISK_ZONE_RWP_SUP;
2296                 break;
2297         }
2298         default:
2299                 break;
2300         }
2301 bailout:
2302         return (error);
2303 }
2304
2305 static void
2306 adastart(struct cam_periph *periph, union ccb *start_ccb)
2307 {
2308         struct ada_softc *softc = (struct ada_softc *)periph->softc;
2309         struct ccb_ataio *ataio = &start_ccb->ataio;
2310
2311         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("adastart\n"));
2312
2313         switch (softc->state) {
2314         case ADA_STATE_NORMAL:
2315         {
2316                 struct bio *bp;
2317                 uint8_t tag_code;
2318
2319                 bp = cam_iosched_next_bio(softc->cam_iosched);
2320                 if (bp == NULL) {
2321                         xpt_release_ccb(start_ccb);
2322                         break;
2323                 }
2324
2325                 if ((bp->bio_flags & BIO_ORDERED) != 0 ||
2326                     (bp->bio_cmd != BIO_DELETE && (softc->flags & ADA_FLAG_NEED_OTAG) != 0)) {
2327                         softc->flags &= ~ADA_FLAG_NEED_OTAG;
2328                         softc->flags |= ADA_FLAG_WAS_OTAG;
2329                         tag_code = 0;
2330                 } else {
2331                         tag_code = 1;
2332                 }
2333                 switch (bp->bio_cmd) {
2334                 case BIO_WRITE:
2335                 case BIO_READ:
2336                 {
2337                         uint64_t lba = bp->bio_pblkno;
2338                         uint16_t count = bp->bio_bcount / softc->params.secsize;
2339                         void *data_ptr;
2340                         int rw_op;
2341
2342                         if (bp->bio_cmd == BIO_WRITE) {
2343                                 softc->flags |= ADA_FLAG_DIRTY;
2344                                 rw_op = CAM_DIR_OUT;
2345                         } else {
2346                                 rw_op = CAM_DIR_IN;
2347                         }
2348
2349                         data_ptr = bp->bio_data;
2350                         if ((bp->bio_flags & (BIO_UNMAPPED|BIO_VLIST)) != 0) {
2351                                 rw_op |= CAM_DATA_BIO;
2352                                 data_ptr = bp;
2353                         }
2354
2355 #ifdef CAM_TEST_FAILURE
2356                         int fail = 0;
2357
2358                         /*
2359                          * Support the failure ioctls.  If the command is a
2360                          * read, and there are pending forced read errors, or
2361                          * if a write and pending write errors, then fail this
2362                          * operation with EIO.  This is useful for testing
2363                          * purposes.  Also, support having every Nth read fail.
2364                          *
2365                          * This is a rather blunt tool.
2366                          */
2367                         if (bp->bio_cmd == BIO_READ) {
2368                                 if (softc->force_read_error) {
2369                                         softc->force_read_error--;
2370                                         fail = 1;
2371                                 }
2372                                 if (softc->periodic_read_error > 0) {
2373                                         if (++softc->periodic_read_count >=
2374                                             softc->periodic_read_error) {
2375                                                 softc->periodic_read_count = 0;
2376                                                 fail = 1;
2377                                         }
2378                                 }
2379                         } else {
2380                                 if (softc->force_write_error) {
2381                                         softc->force_write_error--;
2382                                         fail = 1;
2383                                 }
2384                         }
2385                         if (fail) {
2386                                 biofinish(bp, NULL, EIO);
2387                                 xpt_release_ccb(start_ccb);
2388                                 adaschedule(periph);
2389                                 return;
2390                         }
2391 #endif
2392                         KASSERT((bp->bio_flags & BIO_UNMAPPED) == 0 ||
2393                             round_page(bp->bio_bcount + bp->bio_ma_offset) /
2394                             PAGE_SIZE == bp->bio_ma_n,
2395                             ("Short bio %p", bp));
2396                         cam_fill_ataio(ataio,
2397                             ada_retry_count,
2398                             adadone,
2399                             rw_op,
2400                             0,
2401                             data_ptr,
2402                             bp->bio_bcount,
2403                             ada_default_timeout*1000);
2404
2405                         if ((softc->flags & ADA_FLAG_CAN_NCQ) && tag_code) {
2406                                 if (bp->bio_cmd == BIO_READ) {
2407                                         ata_ncq_cmd(ataio, ATA_READ_FPDMA_QUEUED,
2408                                             lba, count);
2409                                 } else {
2410                                         ata_ncq_cmd(ataio, ATA_WRITE_FPDMA_QUEUED,
2411                                             lba, count);
2412                                 }
2413                         } else if ((softc->flags & ADA_FLAG_CAN_48BIT) &&
2414                             (lba + count >= ATA_MAX_28BIT_LBA ||
2415                             count > 256)) {
2416                                 if (softc->flags & ADA_FLAG_CAN_DMA48) {
2417                                         if (bp->bio_cmd == BIO_READ) {
2418                                                 ata_48bit_cmd(ataio, ATA_READ_DMA48,
2419                                                     0, lba, count);
2420                                         } else {
2421                                                 ata_48bit_cmd(ataio, ATA_WRITE_DMA48,
2422                                                     0, lba, count);
2423                                         }
2424                                 } else {
2425                                         if (bp->bio_cmd == BIO_READ) {
2426                                                 ata_48bit_cmd(ataio, ATA_READ_MUL48,
2427                                                     0, lba, count);
2428                                         } else {
2429                                                 ata_48bit_cmd(ataio, ATA_WRITE_MUL48,
2430                                                     0, lba, count);
2431                                         }
2432                                 }
2433                         } else {
2434                                 if (count == 256)
2435                                         count = 0;
2436                                 if (softc->flags & ADA_FLAG_CAN_DMA) {
2437                                         if (bp->bio_cmd == BIO_READ) {
2438                                                 ata_28bit_cmd(ataio, ATA_READ_DMA,
2439                                                     0, lba, count);
2440                                         } else {
2441                                                 ata_28bit_cmd(ataio, ATA_WRITE_DMA,
2442                                                     0, lba, count);
2443                                         }
2444                                 } else {
2445                                         if (bp->bio_cmd == BIO_READ) {
2446                                                 ata_28bit_cmd(ataio, ATA_READ_MUL,
2447                                                     0, lba, count);
2448                                         } else {
2449                                                 ata_28bit_cmd(ataio, ATA_WRITE_MUL,
2450                                                     0, lba, count);
2451                                         }
2452                                 }
2453                         }
2454                         break;
2455                 }
2456                 case BIO_DELETE:
2457                         switch (softc->delete_method) {
2458                         case ADA_DELETE_NCQ_DSM_TRIM:
2459                                 ada_ncq_dsmtrim(softc, bp, ataio);
2460                                 break;
2461                         case ADA_DELETE_DSM_TRIM:
2462                                 ada_dsmtrim(softc, bp, ataio);
2463                                 break;
2464                         case ADA_DELETE_CFA_ERASE:
2465                                 ada_cfaerase(softc, bp, ataio);
2466                                 break;
2467                         default:
2468                                 biofinish(bp, NULL, EOPNOTSUPP);
2469                                 xpt_release_ccb(start_ccb);
2470                                 adaschedule(periph);
2471                                 return;
2472                         }
2473                         start_ccb->ccb_h.ccb_state = ADA_CCB_TRIM;
2474                         start_ccb->ccb_h.flags |= CAM_UNLOCKED;
2475                         cam_iosched_submit_trim(softc->cam_iosched);
2476                         goto out;
2477                 case BIO_FLUSH:
2478                         cam_fill_ataio(ataio,
2479                             1,
2480                             adadone,
2481                             CAM_DIR_NONE,
2482                             0,
2483                             NULL,
2484                             0,
2485                             ada_default_timeout*1000);
2486
2487                         if (softc->flags & ADA_FLAG_CAN_48BIT)
2488                                 ata_48bit_cmd(ataio, ATA_FLUSHCACHE48, 0, 0, 0);
2489                         else
2490                                 ata_28bit_cmd(ataio, ATA_FLUSHCACHE, 0, 0, 0);
2491                         break;
2492                 case BIO_ZONE: {
2493                         int error, queue_ccb;
2494
2495                         queue_ccb = 0;
2496
2497                         error = ada_zone_cmd(periph, start_ccb, bp, &queue_ccb);
2498                         if ((error != 0)
2499                          || (queue_ccb == 0)) {
2500                                 biofinish(bp, NULL, error);
2501                                 xpt_release_ccb(start_ccb);
2502                                 return;
2503                         }
2504                         break;
2505                 }
2506                 default:
2507                         biofinish(bp, NULL, EOPNOTSUPP);
2508                         xpt_release_ccb(start_ccb);
2509                         return;
2510                 }
2511                 start_ccb->ccb_h.ccb_state = ADA_CCB_BUFFER_IO;
2512                 start_ccb->ccb_h.flags |= CAM_UNLOCKED;
2513 out:
2514                 start_ccb->ccb_h.ccb_bp = bp;
2515                 softc->outstanding_cmds++;
2516                 softc->refcount++;
2517                 cam_periph_unlock(periph);
2518                 xpt_action(start_ccb);
2519                 cam_periph_lock(periph);
2520
2521                 /* May have more work to do, so ensure we stay scheduled */
2522                 adaschedule(periph);
2523                 break;
2524         }
2525         case ADA_STATE_RAHEAD:
2526         case ADA_STATE_WCACHE:
2527         {
2528                 cam_fill_ataio(ataio,
2529                     1,
2530                     adadone,
2531                     CAM_DIR_NONE,
2532                     0,
2533                     NULL,
2534                     0,
2535                     ada_default_timeout*1000);
2536
2537                 if (softc->state == ADA_STATE_RAHEAD) {
2538                         ata_28bit_cmd(ataio, ATA_SETFEATURES, ADA_RA ?
2539                             ATA_SF_ENAB_RCACHE : ATA_SF_DIS_RCACHE, 0, 0);
2540                         start_ccb->ccb_h.ccb_state = ADA_CCB_RAHEAD;
2541                 } else {
2542                         ata_28bit_cmd(ataio, ATA_SETFEATURES, ADA_WC ?
2543                             ATA_SF_ENAB_WCACHE : ATA_SF_DIS_WCACHE, 0, 0);
2544                         start_ccb->ccb_h.ccb_state = ADA_CCB_WCACHE;
2545                 }
2546                 start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
2547                 xpt_action(start_ccb);
2548                 break;
2549         }
2550         case ADA_STATE_LOGDIR:
2551         {
2552                 struct ata_gp_log_dir *log_dir;
2553
2554                 if ((softc->flags & ADA_FLAG_CAN_LOG) == 0) {
2555                         adaprobedone(periph, start_ccb);
2556                         break;
2557                 }
2558
2559                 log_dir = malloc(sizeof(*log_dir), M_ATADA, M_NOWAIT|M_ZERO);
2560                 if (log_dir == NULL) {
2561                         xpt_print(periph->path, "Couldn't malloc log_dir "
2562                             "data\n");
2563                         softc->state = ADA_STATE_NORMAL;
2564                         xpt_release_ccb(start_ccb);
2565                         break;
2566                 }
2567
2568                 ata_read_log(ataio,
2569                     /*retries*/1,
2570                     /*cbfcnp*/adadone,
2571                     /*log_address*/ ATA_LOG_DIRECTORY,
2572                     /*page_number*/ 0,
2573                     /*block_count*/ 1,
2574                     /*protocol*/ softc->flags & ADA_FLAG_CAN_DMA ?
2575                                  CAM_ATAIO_DMA : 0,
2576                     /*data_ptr*/ (uint8_t *)log_dir,
2577                     /*dxfer_len*/sizeof(*log_dir),
2578                     /*timeout*/ada_default_timeout*1000);
2579
2580                 start_ccb->ccb_h.ccb_state = ADA_CCB_LOGDIR;
2581                 xpt_action(start_ccb);
2582                 break;
2583         }
2584         case ADA_STATE_IDDIR:
2585         {
2586                 struct ata_identify_log_pages *id_dir;
2587
2588                 id_dir = malloc(sizeof(*id_dir), M_ATADA, M_NOWAIT | M_ZERO);
2589                 if (id_dir == NULL) {
2590                         xpt_print(periph->path, "Couldn't malloc id_dir "
2591                             "data\n");
2592                         adaprobedone(periph, start_ccb);
2593                         break;
2594                 }
2595
2596                 ata_read_log(ataio,
2597                     /*retries*/1,
2598                     /*cbfcnp*/adadone,
2599                     /*log_address*/ ATA_IDENTIFY_DATA_LOG,
2600                     /*page_number*/ ATA_IDL_PAGE_LIST,
2601                     /*block_count*/ 1,
2602                     /*protocol*/ softc->flags & ADA_FLAG_CAN_DMA ?
2603                                  CAM_ATAIO_DMA : 0,
2604                     /*data_ptr*/ (uint8_t *)id_dir,
2605                     /*dxfer_len*/ sizeof(*id_dir),
2606                     /*timeout*/ada_default_timeout*1000);
2607
2608                 start_ccb->ccb_h.ccb_state = ADA_CCB_IDDIR;
2609                 xpt_action(start_ccb);
2610                 break;
2611         }
2612         case ADA_STATE_SUP_CAP:
2613         {
2614                 struct ata_identify_log_sup_cap *sup_cap;
2615
2616                 sup_cap = malloc(sizeof(*sup_cap), M_ATADA, M_NOWAIT|M_ZERO);
2617                 if (sup_cap == NULL) {
2618                         xpt_print(periph->path, "Couldn't malloc sup_cap "
2619                             "data\n");
2620                         adaprobedone(periph, start_ccb);
2621                         break;
2622                 }
2623
2624                 ata_read_log(ataio,
2625                     /*retries*/1,
2626                     /*cbfcnp*/adadone,
2627                     /*log_address*/ ATA_IDENTIFY_DATA_LOG,
2628                     /*page_number*/ ATA_IDL_SUP_CAP,
2629                     /*block_count*/ 1,
2630                     /*protocol*/ softc->flags & ADA_FLAG_CAN_DMA ?
2631                                  CAM_ATAIO_DMA : 0,
2632                     /*data_ptr*/ (uint8_t *)sup_cap,
2633                     /*dxfer_len*/ sizeof(*sup_cap),
2634                     /*timeout*/ada_default_timeout*1000);
2635
2636                 start_ccb->ccb_h.ccb_state = ADA_CCB_SUP_CAP;
2637                 xpt_action(start_ccb);
2638                 break;
2639         }
2640         case ADA_STATE_ZONE:
2641         {
2642                 struct ata_zoned_info_log *ata_zone;
2643
2644                 ata_zone = malloc(sizeof(*ata_zone), M_ATADA, M_NOWAIT|M_ZERO);
2645                 if (ata_zone == NULL) {
2646                         xpt_print(periph->path, "Couldn't malloc ata_zone "
2647                             "data\n");
2648                         adaprobedone(periph, start_ccb);
2649                         break;
2650                 }
2651
2652                 ata_read_log(ataio,
2653                     /*retries*/1,
2654                     /*cbfcnp*/adadone,
2655                     /*log_address*/ ATA_IDENTIFY_DATA_LOG,
2656                     /*page_number*/ ATA_IDL_ZDI,
2657                     /*block_count*/ 1,
2658                     /*protocol*/ softc->flags & ADA_FLAG_CAN_DMA ?
2659                                  CAM_ATAIO_DMA : 0,
2660                     /*data_ptr*/ (uint8_t *)ata_zone,
2661                     /*dxfer_len*/ sizeof(*ata_zone),
2662                     /*timeout*/ada_default_timeout*1000);
2663
2664                 start_ccb->ccb_h.ccb_state = ADA_CCB_ZONE;
2665                 xpt_action(start_ccb);
2666                 break;
2667         }
2668         }
2669 }
2670
2671 static void
2672 adaprobedone(struct cam_periph *periph, union ccb *ccb)
2673 {
2674         struct ada_softc *softc;
2675
2676         softc = (struct ada_softc *)periph->softc;
2677
2678         /*
2679          * Since our peripheral may be invalidated by an error we must release
2680          * our CCB before releasing the reference on the peripheral.  The
2681          * peripheral will only go away once the last reference is removed, and
2682          * we need it around for the CCB release operation.
2683          */
2684         if (ccb != NULL)
2685                 xpt_release_ccb(ccb);
2686
2687         softc->state = ADA_STATE_NORMAL;
2688         softc->flags |= ADA_FLAG_PROBED;
2689         adaschedule(periph);
2690         if ((softc->flags & ADA_FLAG_ANNOUNCED) == 0) {
2691                 softc->flags |= ADA_FLAG_ANNOUNCED;
2692
2693                 /*
2694                  * We'll release this reference once GEOM calls us back via
2695                  * adadiskgonecb(), telling us that our provider has been freed.
2696                  */
2697                 if (cam_periph_acquire(periph) == 0)
2698                         disk_create(softc->disk, DISK_VERSION);
2699
2700                 cam_periph_release_boot(periph);
2701         }
2702         cam_periph_release_locked(periph);
2703 }
2704
2705 static void
2706 adazonedone(struct cam_periph *periph, union ccb *ccb)
2707 {
2708         struct bio *bp;
2709
2710         bp = (struct bio *)ccb->ccb_h.ccb_bp;
2711
2712         switch (bp->bio_zone.zone_cmd) {
2713         case DISK_ZONE_OPEN:
2714         case DISK_ZONE_CLOSE:
2715         case DISK_ZONE_FINISH:
2716         case DISK_ZONE_RWP:
2717                 break;
2718         case DISK_ZONE_REPORT_ZONES: {
2719                 uint32_t avail_len;
2720                 struct disk_zone_report *rep;
2721                 struct scsi_report_zones_hdr *hdr;
2722                 struct scsi_report_zones_desc *desc;
2723                 struct disk_zone_rep_entry *entry;
2724                 uint32_t hdr_len, num_avail;
2725                 uint32_t num_to_fill, i;
2726
2727                 rep = &bp->bio_zone.zone_params.report;
2728                 avail_len = ccb->ataio.dxfer_len - ccb->ataio.resid;
2729                 /*
2730                  * Note that bio_resid isn't normally used for zone
2731                  * commands, but it is used by devstat_end_transaction_bio()
2732                  * to determine how much data was transferred.  Because
2733                  * the size of the SCSI/ATA data structures is different
2734                  * than the size of the BIO interface structures, the
2735                  * amount of data actually transferred from the drive will
2736                  * be different than the amount of data transferred to
2737                  * the user.
2738                  */
2739                 hdr = (struct scsi_report_zones_hdr *)ccb->ataio.data_ptr;
2740                 if (avail_len < sizeof(*hdr)) {
2741                         /*
2742                          * Is there a better error than EIO here?  We asked
2743                          * for at least the header, and we got less than
2744                          * that.
2745                          */
2746                         bp->bio_error = EIO;
2747                         bp->bio_flags |= BIO_ERROR;
2748                         bp->bio_resid = bp->bio_bcount;
2749                         break;
2750                 }
2751
2752                 hdr_len = le32dec(hdr->length);
2753                 if (hdr_len > 0)
2754                         rep->entries_available = hdr_len / sizeof(*desc);
2755                 else
2756                         rep->entries_available = 0;
2757                 /*
2758                  * NOTE: using the same values for the BIO version of the
2759                  * same field as the SCSI/ATA values.  This means we could
2760                  * get some additional values that aren't defined in bio.h
2761                  * if more values of the same field are defined later.
2762                  */
2763                 rep->header.same = hdr->byte4 & SRZ_SAME_MASK;
2764                 rep->header.maximum_lba = le64dec(hdr->maximum_lba);
2765                 /*
2766                  * If the drive reports no entries that match the query,
2767                  * we're done.
2768                  */
2769                 if (hdr_len == 0) {
2770                         rep->entries_filled = 0;
2771                         bp->bio_resid = bp->bio_bcount;
2772                         break;
2773                 }
2774
2775                 num_avail = min((avail_len - sizeof(*hdr)) / sizeof(*desc),
2776                                 hdr_len / sizeof(*desc));
2777                 /*
2778                  * If the drive didn't return any data, then we're done.
2779                  */
2780                 if (num_avail == 0) {
2781                         rep->entries_filled = 0;
2782                         bp->bio_resid = bp->bio_bcount;
2783                         break;
2784                 }
2785
2786                 num_to_fill = min(num_avail, rep->entries_allocated);
2787                 /*
2788                  * If the user didn't allocate any entries for us to fill,
2789                  * we're done.
2790                  */
2791                 if (num_to_fill == 0) {
2792                         rep->entries_filled = 0;
2793                         bp->bio_resid = bp->bio_bcount;
2794                         break;
2795                 }
2796
2797                 for (i = 0, desc = &hdr->desc_list[0], entry=&rep->entries[0];
2798                      i < num_to_fill; i++, desc++, entry++) {
2799                         /*
2800                          * NOTE: we're mapping the values here directly
2801                          * from the SCSI/ATA bit definitions to the bio.h
2802                          * definitions.  There is also a warning in
2803                          * disk_zone.h, but the impact is that if
2804                          * additional values are added in the SCSI/ATA
2805                          * specs these will be visible to consumers of
2806                          * this interface.
2807                          */
2808                         entry->zone_type = desc->zone_type & SRZ_TYPE_MASK;
2809                         entry->zone_condition =
2810                             (desc->zone_flags & SRZ_ZONE_COND_MASK) >>
2811                             SRZ_ZONE_COND_SHIFT;
2812                         entry->zone_flags |= desc->zone_flags &
2813                             (SRZ_ZONE_NON_SEQ|SRZ_ZONE_RESET);
2814                         entry->zone_length = le64dec(desc->zone_length);
2815                         entry->zone_start_lba = le64dec(desc->zone_start_lba);
2816                         entry->write_pointer_lba =
2817                             le64dec(desc->write_pointer_lba);
2818                 }
2819                 rep->entries_filled = num_to_fill;
2820                 /*
2821                  * Note that this residual is accurate from the user's
2822                  * standpoint, but the amount transferred isn't accurate
2823                  * from the standpoint of what actually came back from the
2824                  * drive.
2825                  */
2826                 bp->bio_resid = bp->bio_bcount - (num_to_fill * sizeof(*entry));
2827                 break;
2828         }
2829         case DISK_ZONE_GET_PARAMS:
2830         default:
2831                 /*
2832                  * In theory we should not get a GET_PARAMS bio, since it
2833                  * should be handled without queueing the command to the
2834                  * drive.
2835                  */
2836                 panic("%s: Invalid zone command %d", __func__,
2837                     bp->bio_zone.zone_cmd);
2838                 break;
2839         }
2840
2841         if (bp->bio_zone.zone_cmd == DISK_ZONE_REPORT_ZONES)
2842                 free(ccb->ataio.data_ptr, M_ATADA);
2843 }
2844
2845 static void
2846 adadone(struct cam_periph *periph, union ccb *done_ccb)
2847 {
2848         struct ada_softc *softc;
2849         struct ccb_ataio *ataio;
2850         struct cam_path *path;
2851         uint32_t priority;
2852         int state;
2853
2854         softc = (struct ada_softc *)periph->softc;
2855         ataio = &done_ccb->ataio;
2856         path = done_ccb->ccb_h.path;
2857         priority = done_ccb->ccb_h.pinfo.priority;
2858
2859         CAM_DEBUG(path, CAM_DEBUG_TRACE, ("adadone\n"));
2860
2861         state = ataio->ccb_h.ccb_state & ADA_CCB_TYPE_MASK;
2862         switch (state) {
2863         case ADA_CCB_BUFFER_IO:
2864         case ADA_CCB_TRIM:
2865         {
2866                 struct bio *bp;
2867                 int error;
2868
2869                 cam_periph_lock(periph);
2870                 bp = (struct bio *)done_ccb->ccb_h.ccb_bp;
2871                 if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
2872                         error = adaerror(done_ccb, CAM_RETRY_SELTO, 0);
2873                         if (error == ERESTART) {
2874                                 /* A retry was scheduled, so just return. */
2875                                 cam_periph_unlock(periph);
2876                                 return;
2877                         }
2878                         if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0)
2879                                 cam_release_devq(path,
2880                                                  /*relsim_flags*/0,
2881                                                  /*reduction*/0,
2882                                                  /*timeout*/0,
2883                                                  /*getcount_only*/0);
2884                         /*
2885                          * If we get an error on an NCQ DSM TRIM, fall back
2886                          * to a non-NCQ DSM TRIM forever. Please note that if
2887                          * CAN_NCQ_TRIM is set, CAN_TRIM is necessarily set too.
2888                          * However, for this one trim, we treat it as advisory
2889                          * and return success up the stack.
2890                          */
2891                         if (state == ADA_CCB_TRIM &&
2892                             error != 0 &&
2893                             (softc->flags & ADA_FLAG_CAN_NCQ_TRIM) != 0) {
2894                                 softc->flags &= ~ADA_FLAG_CAN_NCQ_TRIM;
2895                                 error = 0;
2896                                 adasetdeletemethod(softc);
2897                         }
2898                 } else {
2899                         if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0)
2900                                 panic("REQ_CMP with QFRZN");
2901
2902                         error = 0;
2903                 }
2904                 bp->bio_error = error;
2905                 if (error != 0) {
2906                         bp->bio_resid = bp->bio_bcount;
2907                         bp->bio_flags |= BIO_ERROR;
2908                 } else {
2909                         if (bp->bio_cmd == BIO_ZONE)
2910                                 adazonedone(periph, done_ccb);
2911                         else if (state == ADA_CCB_TRIM)
2912                                 bp->bio_resid = 0;
2913                         else
2914                                 bp->bio_resid = ataio->resid;
2915
2916                         if ((bp->bio_resid > 0)
2917                          && (bp->bio_cmd != BIO_ZONE))
2918                                 bp->bio_flags |= BIO_ERROR;
2919                 }
2920                 softc->outstanding_cmds--;
2921                 if (softc->outstanding_cmds == 0)
2922                         softc->flags |= ADA_FLAG_WAS_OTAG;
2923
2924                 /*
2925                  * We need to call cam_iosched before we call biodone so that we
2926                  * don't measure any activity that happens in the completion
2927                  * routine, which in the case of sendfile can be quite
2928                  * extensive.  Release the periph refcount taken in adastart()
2929                  * for each CCB.
2930                  */
2931                 cam_iosched_bio_complete(softc->cam_iosched, bp, done_ccb);
2932                 xpt_release_ccb(done_ccb);
2933                 KASSERT(softc->refcount >= 1, ("adadone softc %p refcount %d", softc, softc->refcount));
2934                 softc->refcount--;
2935                 if (state == ADA_CCB_TRIM) {
2936                         TAILQ_HEAD(, bio) queue;
2937                         struct bio *bp1;
2938
2939                         TAILQ_INIT(&queue);
2940                         TAILQ_CONCAT(&queue, &softc->trim_req.bps, bio_queue);
2941                         /*
2942                          * Normally, the xpt_release_ccb() above would make sure
2943                          * that when we have more work to do, that work would
2944                          * get kicked off. However, we specifically keep
2945                          * trim_running set to 0 before the call above to allow
2946                          * other I/O to progress when many BIO_DELETE requests
2947                          * are pushed down. We set trim_running to 0 and call
2948                          * daschedule again so that we don't stall if there are
2949                          * no other I/Os pending apart from BIO_DELETEs.
2950                          */
2951                         cam_iosched_trim_done(softc->cam_iosched);
2952                         adaschedule(periph);
2953                         cam_periph_unlock(periph);
2954                         while ((bp1 = TAILQ_FIRST(&queue)) != NULL) {
2955                                 TAILQ_REMOVE(&queue, bp1, bio_queue);
2956                                 bp1->bio_error = error;
2957                                 if (error != 0) {
2958                                         bp1->bio_flags |= BIO_ERROR;
2959                                         bp1->bio_resid = bp1->bio_bcount;
2960                                 } else
2961                                         bp1->bio_resid = 0;
2962                                 biodone(bp1);
2963                         }
2964                 } else {
2965                         adaschedule(periph);
2966                         cam_periph_unlock(periph);
2967                         biodone(bp);
2968                 }
2969                 return;
2970         }
2971         case ADA_CCB_RAHEAD:
2972         {
2973                 if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
2974                         if (adaerror(done_ccb, 0, 0) == ERESTART) {
2975                                 /* Drop freeze taken due to CAM_DEV_QFREEZE */
2976                                 cam_release_devq(path, 0, 0, 0, FALSE);
2977                                 return;
2978                         } else if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
2979                                 cam_release_devq(path,
2980                                     /*relsim_flags*/0,
2981                                     /*reduction*/0,
2982                                     /*timeout*/0,
2983                                     /*getcount_only*/0);
2984                         }
2985                 }
2986
2987                 xpt_release_ccb(done_ccb);
2988                 softc->state = ADA_STATE_WCACHE;
2989                 xpt_schedule(periph, priority);
2990                 /* Drop freeze taken due to CAM_DEV_QFREEZE */
2991                 cam_release_devq(path, 0, 0, 0, FALSE);
2992                 return;
2993         }
2994         case ADA_CCB_WCACHE:
2995         {
2996                 if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
2997                         if (adaerror(done_ccb, 0, 0) == ERESTART) {
2998                                 /* Drop freeze taken due to CAM_DEV_QFREEZE */
2999                                 cam_release_devq(path, 0, 0, 0, FALSE);
3000                                 return;
3001                         } else if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
3002                                 cam_release_devq(path,
3003                                     /*relsim_flags*/0,
3004                                     /*reduction*/0,
3005                                     /*timeout*/0,
3006                                     /*getcount_only*/0);
3007                         }
3008                 }
3009
3010                 /* Drop freeze taken due to CAM_DEV_QFREEZE */
3011                 cam_release_devq(path, 0, 0, 0, FALSE);
3012
3013                 if ((softc->flags & ADA_FLAG_CAN_LOG)
3014                  && (softc->zone_mode != ADA_ZONE_NONE)) {
3015                         xpt_release_ccb(done_ccb);
3016                         softc->state = ADA_STATE_LOGDIR;
3017                         xpt_schedule(periph, priority);
3018                 } else {
3019                         adaprobedone(periph, done_ccb);
3020                 }
3021                 return;
3022         }
3023         case ADA_CCB_LOGDIR:
3024         {
3025                 int error;
3026
3027                 if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
3028                         error = 0;
3029                         softc->valid_logdir_len = 0;
3030                         bzero(&softc->ata_logdir, sizeof(softc->ata_logdir));
3031                         softc->valid_logdir_len =
3032                                 ataio->dxfer_len - ataio->resid;
3033                         if (softc->valid_logdir_len > 0)
3034                                 bcopy(ataio->data_ptr, &softc->ata_logdir,
3035                                     min(softc->valid_logdir_len,
3036                                         sizeof(softc->ata_logdir)));
3037                         /*
3038                          * Figure out whether the Identify Device log is
3039                          * supported.  The General Purpose log directory
3040                          * has a header, and lists the number of pages
3041                          * available for each GP log identified by the
3042                          * offset into the list.
3043                          */
3044                         if ((softc->valid_logdir_len >=
3045                             ((ATA_IDENTIFY_DATA_LOG + 1) * sizeof(uint16_t)))
3046                          && (le16dec(softc->ata_logdir.header) ==
3047                              ATA_GP_LOG_DIR_VERSION)
3048                          && (le16dec(&softc->ata_logdir.num_pages[
3049                              (ATA_IDENTIFY_DATA_LOG *
3050                              sizeof(uint16_t)) - sizeof(uint16_t)]) > 0)){
3051                                 softc->flags |= ADA_FLAG_CAN_IDLOG;
3052                         } else {
3053                                 softc->flags &= ~ADA_FLAG_CAN_IDLOG;
3054                         }
3055                 } else {
3056                         error = adaerror(done_ccb, CAM_RETRY_SELTO,
3057                                          SF_RETRY_UA|SF_NO_PRINT);
3058                         if (error == ERESTART)
3059                                 return;
3060                         else if (error != 0) {
3061                                 /*
3062                                  * If we can't get the ATA log directory,
3063                                  * then ATA logs are effectively not
3064                                  * supported even if the bit is set in the
3065                                  * identify data.
3066                                  */
3067                                 softc->flags &= ~(ADA_FLAG_CAN_LOG |
3068                                                   ADA_FLAG_CAN_IDLOG);
3069                                 if ((done_ccb->ccb_h.status &
3070                                      CAM_DEV_QFRZN) != 0) {
3071                                         /* Don't wedge this device's queue */
3072                                         cam_release_devq(done_ccb->ccb_h.path,
3073                                                          /*relsim_flags*/0,
3074                                                          /*reduction*/0,
3075                                                          /*timeout*/0,
3076                                                          /*getcount_only*/0);
3077                                 }
3078                         }
3079                 }
3080
3081                 free(ataio->data_ptr, M_ATADA);
3082
3083                 if ((error == 0)
3084                  && (softc->flags & ADA_FLAG_CAN_IDLOG)) {
3085                         softc->state = ADA_STATE_IDDIR;
3086                         xpt_release_ccb(done_ccb);
3087                         xpt_schedule(periph, priority);
3088                 } else
3089                         adaprobedone(periph, done_ccb);
3090
3091                 return;
3092         }
3093         case ADA_CCB_IDDIR: {
3094                 int error;
3095
3096                 if ((ataio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
3097                         off_t entries_offset, max_entries;
3098                         error = 0;
3099
3100                         softc->valid_iddir_len = 0;
3101                         bzero(&softc->ata_iddir, sizeof(softc->ata_iddir));
3102                         softc->flags &= ~(ADA_FLAG_CAN_SUPCAP |
3103                                           ADA_FLAG_CAN_ZONE);
3104                         softc->valid_iddir_len =
3105                                 ataio->dxfer_len - ataio->resid;
3106                         if (softc->valid_iddir_len > 0)
3107                                 bcopy(ataio->data_ptr, &softc->ata_iddir,
3108                                     min(softc->valid_iddir_len,
3109                                         sizeof(softc->ata_iddir)));
3110
3111                         entries_offset =
3112                             __offsetof(struct ata_identify_log_pages,entries);
3113                         max_entries = softc->valid_iddir_len - entries_offset;
3114                         if ((softc->valid_iddir_len > (entries_offset + 1))
3115                          && (le64dec(softc->ata_iddir.header) ==
3116                              ATA_IDLOG_REVISION)
3117                          && (softc->ata_iddir.entry_count > 0)) {
3118                                 int num_entries, i;
3119
3120                                 num_entries = softc->ata_iddir.entry_count;
3121                                 num_entries = min(num_entries,
3122                                    softc->valid_iddir_len - entries_offset);
3123                                 for (i = 0; i < num_entries &&
3124                                      i < max_entries; i++) {
3125                                         if (softc->ata_iddir.entries[i] ==
3126                                             ATA_IDL_SUP_CAP)
3127                                                 softc->flags |=
3128                                                     ADA_FLAG_CAN_SUPCAP;
3129                                         else if (softc->ata_iddir.entries[i]==
3130                                                  ATA_IDL_ZDI)
3131                                                 softc->flags |=
3132                                                     ADA_FLAG_CAN_ZONE;
3133
3134                                         if ((softc->flags &
3135                                              ADA_FLAG_CAN_SUPCAP)
3136                                          && (softc->flags &
3137                                              ADA_FLAG_CAN_ZONE))
3138                                                 break;
3139                                 }
3140                         }
3141                 } else {
3142                         error = adaerror(done_ccb, CAM_RETRY_SELTO,
3143                                          SF_RETRY_UA|SF_NO_PRINT);
3144                         if (error == ERESTART)
3145                                 return;
3146                         else if (error != 0) {
3147                                 /*
3148                                  * If we can't get the ATA Identify Data log
3149                                  * directory, then it effectively isn't
3150                                  * supported even if the ATA Log directory
3151                                  * a non-zero number of pages present for
3152                                  * this log.
3153                                  */
3154                                 softc->flags &= ~ADA_FLAG_CAN_IDLOG;
3155                                 if ((done_ccb->ccb_h.status &
3156                                      CAM_DEV_QFRZN) != 0) {
3157                                         /* Don't wedge this device's queue */
3158                                         cam_release_devq(done_ccb->ccb_h.path,
3159                                                          /*relsim_flags*/0,
3160                                                          /*reduction*/0,
3161                                                          /*timeout*/0,
3162                                                          /*getcount_only*/0);
3163                                 }
3164                         }
3165                 }
3166
3167                 free(ataio->data_ptr, M_ATADA);
3168
3169                 if ((error == 0)
3170                  && (softc->flags & ADA_FLAG_CAN_SUPCAP)) {
3171                         softc->state = ADA_STATE_SUP_CAP;
3172                         xpt_release_ccb(done_ccb);
3173                         xpt_schedule(periph, priority);
3174                 } else
3175                         adaprobedone(periph, done_ccb);
3176                 return;
3177         }
3178         case ADA_CCB_SUP_CAP: {
3179                 int error;
3180
3181                 if ((ataio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
3182                         uint32_t valid_len;
3183                         size_t needed_size;
3184                         struct ata_identify_log_sup_cap *sup_cap;
3185                         error = 0;
3186
3187                         sup_cap = (struct ata_identify_log_sup_cap *)
3188                             ataio->data_ptr;
3189                         valid_len = ataio->dxfer_len - ataio->resid;
3190                         needed_size =
3191                             __offsetof(struct ata_identify_log_sup_cap,
3192                             sup_zac_cap) + 1 + sizeof(sup_cap->sup_zac_cap);
3193                         if (valid_len >= needed_size) {
3194                                 uint64_t zoned, zac_cap;
3195
3196                                 zoned = le64dec(sup_cap->zoned_cap);
3197                                 if (zoned & ATA_ZONED_VALID) {
3198                                         /*
3199                                          * This should have already been
3200                                          * set, because this is also in the
3201                                          * ATA identify data.
3202                                          */
3203                                         if ((zoned & ATA_ZONED_MASK) ==
3204                                             ATA_SUPPORT_ZONE_HOST_AWARE)
3205                                                 softc->zone_mode =
3206                                                     ADA_ZONE_HOST_AWARE;
3207                                         else if ((zoned & ATA_ZONED_MASK) ==
3208                                             ATA_SUPPORT_ZONE_DEV_MANAGED)
3209                                                 softc->zone_mode =
3210                                                     ADA_ZONE_DRIVE_MANAGED;
3211                                 }
3212
3213                                 zac_cap = le64dec(sup_cap->sup_zac_cap);
3214                                 if (zac_cap & ATA_SUP_ZAC_CAP_VALID) {
3215                                         if (zac_cap & ATA_REPORT_ZONES_SUP)
3216                                                 softc->zone_flags |=
3217                                                     ADA_ZONE_FLAG_RZ_SUP;
3218                                         if (zac_cap & ATA_ND_OPEN_ZONE_SUP)
3219                                                 softc->zone_flags |=
3220                                                     ADA_ZONE_FLAG_OPEN_SUP;
3221                                         if (zac_cap & ATA_ND_CLOSE_ZONE_SUP)
3222                                                 softc->zone_flags |=
3223                                                     ADA_ZONE_FLAG_CLOSE_SUP;
3224                                         if (zac_cap & ATA_ND_FINISH_ZONE_SUP)
3225                                                 softc->zone_flags |=
3226                                                     ADA_ZONE_FLAG_FINISH_SUP;
3227                                         if (zac_cap & ATA_ND_RWP_SUP)
3228                                                 softc->zone_flags |=
3229                                                     ADA_ZONE_FLAG_RWP_SUP;
3230                                 } else {
3231                                         /*
3232                                          * This field was introduced in
3233                                          * ACS-4, r08 on April 28th, 2015.
3234                                          * If the drive firmware was written
3235                                          * to an earlier spec, it won't have
3236                                          * the field.  So, assume all
3237                                          * commands are supported.
3238                                          */
3239                                         softc->zone_flags |=
3240                                             ADA_ZONE_FLAG_SUP_MASK;
3241                                 }
3242                         }
3243                 } else {
3244                         error = adaerror(done_ccb, CAM_RETRY_SELTO,
3245                                          SF_RETRY_UA|SF_NO_PRINT);
3246                         if (error == ERESTART)
3247                                 return;
3248                         else if (error != 0) {
3249                                 /*
3250                                  * If we can't get the ATA Identify Data
3251                                  * Supported Capabilities page, clear the
3252                                  * flag...
3253                                  */
3254                                 softc->flags &= ~ADA_FLAG_CAN_SUPCAP;
3255                                 /*
3256                                  * And clear zone capabilities.
3257                                  */
3258                                 softc->zone_flags &= ~ADA_ZONE_FLAG_SUP_MASK;
3259                                 if ((done_ccb->ccb_h.status &
3260                                      CAM_DEV_QFRZN) != 0) {
3261                                         /* Don't wedge this device's queue */
3262                                         cam_release_devq(done_ccb->ccb_h.path,
3263                                                          /*relsim_flags*/0,
3264                                                          /*reduction*/0,
3265                                                          /*timeout*/0,
3266                                                          /*getcount_only*/0);
3267                                 }
3268                         }
3269                 }
3270
3271                 free(ataio->data_ptr, M_ATADA);
3272
3273                 if ((error == 0)
3274                  && (softc->flags & ADA_FLAG_CAN_ZONE)) {
3275                         softc->state = ADA_STATE_ZONE;
3276                         xpt_release_ccb(done_ccb);
3277                         xpt_schedule(periph, priority);
3278                 } else
3279                         adaprobedone(periph, done_ccb);
3280                 return;
3281         }
3282         case ADA_CCB_ZONE: {
3283                 int error;
3284
3285                 if ((ataio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
3286                         struct ata_zoned_info_log *zi_log;
3287                         uint32_t valid_len;
3288                         size_t needed_size;
3289
3290                         zi_log = (struct ata_zoned_info_log *)ataio->data_ptr;
3291
3292                         valid_len = ataio->dxfer_len - ataio->resid;
3293                         needed_size = __offsetof(struct ata_zoned_info_log,
3294                             version_info) + 1 + sizeof(zi_log->version_info);
3295                         if (valid_len >= needed_size) {
3296                                 uint64_t tmpvar;
3297
3298                                 tmpvar = le64dec(zi_log->zoned_cap);
3299                                 if (tmpvar & ATA_ZDI_CAP_VALID) {
3300                                         if (tmpvar & ATA_ZDI_CAP_URSWRZ)
3301                                                 softc->zone_flags |=
3302                                                     ADA_ZONE_FLAG_URSWRZ;
3303                                         else
3304                                                 softc->zone_flags &=
3305                                                     ~ADA_ZONE_FLAG_URSWRZ;
3306                                 }
3307                                 tmpvar = le64dec(zi_log->optimal_seq_zones);
3308                                 if (tmpvar & ATA_ZDI_OPT_SEQ_VALID) {
3309                                         softc->zone_flags |=
3310                                             ADA_ZONE_FLAG_OPT_SEQ_SET;
3311                                         softc->optimal_seq_zones = (tmpvar &
3312                                             ATA_ZDI_OPT_SEQ_MASK);
3313                                 } else {
3314                                         softc->zone_flags &=
3315                                             ~ADA_ZONE_FLAG_OPT_SEQ_SET;
3316                                         softc->optimal_seq_zones = 0;
3317                                 }
3318
3319                                 tmpvar =le64dec(zi_log->optimal_nonseq_zones);
3320                                 if (tmpvar & ATA_ZDI_OPT_NS_VALID) {
3321                                         softc->zone_flags |=
3322                                             ADA_ZONE_FLAG_OPT_NONSEQ_SET;
3323                                         softc->optimal_nonseq_zones =
3324                                             (tmpvar & ATA_ZDI_OPT_NS_MASK);
3325                                 } else {
3326                                         softc->zone_flags &=
3327                                             ~ADA_ZONE_FLAG_OPT_NONSEQ_SET;
3328                                         softc->optimal_nonseq_zones = 0;
3329                                 }
3330
3331                                 tmpvar = le64dec(zi_log->max_seq_req_zones);
3332                                 if (tmpvar & ATA_ZDI_MAX_SEQ_VALID) {
3333                                         softc->zone_flags |=
3334                                             ADA_ZONE_FLAG_MAX_SEQ_SET;
3335                                         softc->max_seq_zones =
3336                                             (tmpvar & ATA_ZDI_MAX_SEQ_MASK);
3337                                 } else {
3338                                         softc->zone_flags &=
3339                                             ~ADA_ZONE_FLAG_MAX_SEQ_SET;
3340                                         softc->max_seq_zones = 0;
3341                                 }
3342                         }
3343                 } else {
3344                         error = adaerror(done_ccb, CAM_RETRY_SELTO,
3345                                          SF_RETRY_UA|SF_NO_PRINT);
3346                         if (error == ERESTART)
3347                                 return;
3348                         else if (error != 0) {
3349                                 softc->flags &= ~ADA_FLAG_CAN_ZONE;
3350                                 softc->flags &= ~ADA_ZONE_FLAG_SET_MASK;
3351
3352                                 if ((done_ccb->ccb_h.status &
3353                                      CAM_DEV_QFRZN) != 0) {
3354                                         /* Don't wedge this device's queue */
3355                                         cam_release_devq(done_ccb->ccb_h.path,
3356                                                          /*relsim_flags*/0,
3357                                                          /*reduction*/0,
3358                                                          /*timeout*/0,
3359                                                          /*getcount_only*/0);
3360                                 }
3361                         }
3362                 }
3363                 free(ataio->data_ptr, M_ATADA);
3364
3365                 adaprobedone(periph, done_ccb);
3366                 return;
3367         }
3368         case ADA_CCB_DUMP:
3369                 /* No-op.  We're polling */
3370                 return;
3371         default:
3372                 break;
3373         }
3374         xpt_release_ccb(done_ccb);
3375 }
3376
3377 static int
3378 adaerror(union ccb *ccb, uint32_t cam_flags, uint32_t sense_flags)
3379 {
3380 #ifdef CAM_IO_STATS
3381         struct ada_softc *softc;
3382         struct cam_periph *periph;
3383
3384         periph = xpt_path_periph(ccb->ccb_h.path);
3385         softc = (struct ada_softc *)periph->softc;
3386
3387         switch (ccb->ccb_h.status & CAM_STATUS_MASK) {
3388         case CAM_CMD_TIMEOUT:
3389                 softc->timeouts++;
3390                 break;
3391         case CAM_REQ_ABORTED:
3392         case CAM_REQ_CMP_ERR:
3393         case CAM_REQ_TERMIO:
3394         case CAM_UNREC_HBA_ERROR:
3395         case CAM_DATA_RUN_ERR:
3396         case CAM_ATA_STATUS_ERROR:
3397                 softc->errors++;
3398                 break;
3399         default:
3400                 break;
3401         }
3402 #endif
3403
3404         return(cam_periph_error(ccb, cam_flags, sense_flags));
3405 }
3406
3407 static void
3408 adasetgeom(struct ada_softc *softc, struct ccb_getdev *cgd)
3409 {
3410         struct disk_params *dp = &softc->params;
3411         uint64_t lbasize48;
3412         uint32_t lbasize;
3413         u_int maxio, d_flags;
3414         size_t tmpsize;
3415
3416         dp->secsize = ata_logical_sector_size(&cgd->ident_data);
3417         if ((cgd->ident_data.atavalid & ATA_FLAG_54_58) &&
3418             cgd->ident_data.current_heads != 0 &&
3419             cgd->ident_data.current_sectors != 0) {
3420                 dp->heads = cgd->ident_data.current_heads;
3421                 dp->secs_per_track = cgd->ident_data.current_sectors;
3422                 dp->cylinders = cgd->ident_data.cylinders;
3423                 dp->sectors = (uint32_t)cgd->ident_data.current_size_1 |
3424                           ((uint32_t)cgd->ident_data.current_size_2 << 16);
3425         } else {
3426                 dp->heads = cgd->ident_data.heads;
3427                 dp->secs_per_track = cgd->ident_data.sectors;
3428                 dp->cylinders = cgd->ident_data.cylinders;
3429                 dp->sectors = cgd->ident_data.cylinders *
3430                               (uint32_t)(dp->heads * dp->secs_per_track);
3431         }
3432         lbasize = (uint32_t)cgd->ident_data.lba_size_1 |
3433                   ((uint32_t)cgd->ident_data.lba_size_2 << 16);
3434
3435         /* use the 28bit LBA size if valid or bigger than the CHS mapping */
3436         if (cgd->ident_data.cylinders == 16383 || dp->sectors < lbasize)
3437                 dp->sectors = lbasize;
3438
3439         /* use the 48bit LBA size if valid */
3440         lbasize48 = ((uint64_t)cgd->ident_data.lba_size48_1) |
3441                     ((uint64_t)cgd->ident_data.lba_size48_2 << 16) |
3442                     ((uint64_t)cgd->ident_data.lba_size48_3 << 32) |
3443                     ((uint64_t)cgd->ident_data.lba_size48_4 << 48);
3444         if ((cgd->ident_data.support.command2 & ATA_SUPPORT_ADDRESS48) &&
3445             lbasize48 > ATA_MAX_28BIT_LBA)
3446                 dp->sectors = lbasize48;
3447
3448         maxio = softc->cpi.maxio;               /* Honor max I/O size of SIM */
3449         if (maxio == 0)
3450                 maxio = DFLTPHYS;       /* traditional default */
3451         else if (maxio > maxphys)
3452                 maxio = maxphys;        /* for safety */
3453         if (softc->flags & ADA_FLAG_CAN_48BIT)
3454                 maxio = min(maxio, 65536 * softc->params.secsize);
3455         else                                    /* 28bit ATA command limit */
3456                 maxio = min(maxio, 256 * softc->params.secsize);
3457         if (softc->quirks & ADA_Q_128KB)
3458                 maxio = min(maxio, 128 * 1024);
3459         softc->disk->d_maxsize = maxio;
3460         d_flags = DISKFLAG_DIRECT_COMPLETION | DISKFLAG_CANZONE;
3461         if (softc->flags & ADA_FLAG_CAN_FLUSHCACHE)
3462                 d_flags |= DISKFLAG_CANFLUSHCACHE;
3463         if (softc->flags & ADA_FLAG_CAN_TRIM) {
3464                 d_flags |= DISKFLAG_CANDELETE;
3465                 softc->disk->d_delmaxsize = softc->params.secsize *
3466                     ATA_DSM_RANGE_MAX * softc->trim_max_ranges;
3467         } else if ((softc->flags & ADA_FLAG_CAN_CFA) &&
3468             !(softc->flags & ADA_FLAG_CAN_48BIT)) {
3469                 d_flags |= DISKFLAG_CANDELETE;
3470                 softc->disk->d_delmaxsize = 256 * softc->params.secsize;
3471         } else
3472                 softc->disk->d_delmaxsize = maxio;
3473         if ((softc->cpi.hba_misc & PIM_UNMAPPED) != 0) {
3474                 d_flags |= DISKFLAG_UNMAPPED_BIO;
3475                 softc->flags |= ADA_FLAG_UNMAPPEDIO;
3476         }
3477         softc->disk->d_flags = d_flags;
3478
3479         /*
3480          * ata_param_fixup will strip trailing padding spaces and add a NUL,
3481          * but if the field has no padding (as is common for serial numbers)
3482          * there will still be no NUL terminator. We cannot use strlcpy, since
3483          * it keeps reading src until it finds a NUL in order to compute the
3484          * return value (and will truncate the final character due to having a
3485          * single dsize rather than separate ssize and dsize), and strncpy does
3486          * not add a NUL to the destination if it reaches the character limit.
3487          */
3488         tmpsize = MIN(sizeof(softc->disk->d_descr) - 1,
3489             sizeof(cgd->ident_data.model));
3490         memcpy(softc->disk->d_descr, cgd->ident_data.model, tmpsize);
3491         softc->disk->d_descr[tmpsize] = '\0';
3492
3493         tmpsize = MIN(sizeof(softc->disk->d_ident) - 1,
3494             sizeof(cgd->ident_data.serial));
3495         memcpy(softc->disk->d_ident, cgd->ident_data.serial, tmpsize);
3496         softc->disk->d_ident[tmpsize] = '\0';
3497
3498         softc->disk->d_sectorsize = softc->params.secsize;
3499         softc->disk->d_mediasize = (off_t)softc->params.sectors *
3500             softc->params.secsize;
3501         if (ata_physical_sector_size(&cgd->ident_data) !=
3502             softc->params.secsize) {
3503                 softc->disk->d_stripesize =
3504                     ata_physical_sector_size(&cgd->ident_data);
3505                 softc->disk->d_stripeoffset = (softc->disk->d_stripesize -
3506                     ata_logical_sector_offset(&cgd->ident_data)) %
3507                     softc->disk->d_stripesize;
3508         } else if (softc->quirks & ADA_Q_4K) {
3509                 softc->disk->d_stripesize = 4096;
3510                 softc->disk->d_stripeoffset = 0;
3511         }
3512         softc->disk->d_fwsectors = softc->params.secs_per_track;
3513         softc->disk->d_fwheads = softc->params.heads;
3514         softc->disk->d_rotation_rate = cgd->ident_data.media_rotation_rate;
3515         snprintf(softc->disk->d_attachment, sizeof(softc->disk->d_attachment),
3516             "%s%d", softc->cpi.dev_name, softc->cpi.unit_number);
3517 }
3518
3519 static void
3520 adasendorderedtag(void *arg)
3521 {
3522         struct ada_softc *softc = arg;
3523
3524         if (ada_send_ordered) {
3525                 if (softc->outstanding_cmds > 0) {
3526                         if ((softc->flags & ADA_FLAG_WAS_OTAG) == 0)
3527                                 softc->flags |= ADA_FLAG_NEED_OTAG;
3528                         softc->flags &= ~ADA_FLAG_WAS_OTAG;
3529                 }
3530         }
3531
3532         /* Queue us up again */
3533         callout_schedule_sbt(&softc->sendordered_c,
3534             SBT_1S / ADA_ORDEREDTAG_INTERVAL * ada_default_timeout, 0,
3535             C_PREL(1));
3536 }
3537
3538 /*
3539  * Step through all ADA peripheral drivers, and if the device is still open,
3540  * sync the disk cache to physical media.
3541  */
3542 static void
3543 adaflush(void)
3544 {
3545         struct cam_periph *periph;
3546         struct ada_softc *softc;
3547         union ccb *ccb;
3548         int error;
3549
3550         CAM_PERIPH_FOREACH(periph, &adadriver) {
3551                 softc = (struct ada_softc *)periph->softc;
3552                 if (SCHEDULER_STOPPED()) {
3553                         /* If we panicked with the lock held, do not recurse. */
3554                         if (!cam_periph_owned(periph) &&
3555                             (softc->flags & ADA_FLAG_OPEN)) {
3556                                 adadump(softc->disk, NULL, 0, 0);
3557                         }
3558                         continue;
3559                 }
3560                 cam_periph_lock(periph);
3561                 /*
3562                  * We only sync the cache if the drive is still open, and
3563                  * if the drive is capable of it..
3564                  */
3565                 if (((softc->flags & ADA_FLAG_OPEN) == 0) ||
3566                     (softc->flags & ADA_FLAG_CAN_FLUSHCACHE) == 0) {
3567                         cam_periph_unlock(periph);
3568                         continue;
3569                 }
3570
3571                 ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL);
3572                 cam_fill_ataio(&ccb->ataio,
3573                                     0,
3574                                     NULL,
3575                                     CAM_DIR_NONE,
3576                                     0,
3577                                     NULL,
3578                                     0,
3579                                     ada_default_timeout*1000);
3580                 if (softc->flags & ADA_FLAG_CAN_48BIT)
3581                         ata_48bit_cmd(&ccb->ataio, ATA_FLUSHCACHE48, 0, 0, 0);
3582                 else
3583                         ata_28bit_cmd(&ccb->ataio, ATA_FLUSHCACHE, 0, 0, 0);
3584
3585                 error = cam_periph_runccb(ccb, adaerror, /*cam_flags*/0,
3586                     /*sense_flags*/ SF_NO_RECOVERY | SF_NO_RETRY,
3587                     softc->disk->d_devstat);
3588                 if (error != 0)
3589                         xpt_print(periph->path, "Synchronize cache failed\n");
3590                 xpt_release_ccb(ccb);
3591                 cam_periph_unlock(periph);
3592         }
3593 }
3594
3595 static void
3596 adaspindown(uint8_t cmd, int flags)
3597 {
3598         struct cam_periph *periph;
3599         struct ada_softc *softc;
3600         struct ccb_ataio local_ccb;
3601         int error;
3602         int mode;
3603
3604         CAM_PERIPH_FOREACH(periph, &adadriver) {
3605                 /* If we panicked with lock held - not recurse here. */
3606                 if (cam_periph_owned(periph))
3607                         continue;
3608                 cam_periph_lock(periph);
3609                 softc = (struct ada_softc *)periph->softc;
3610                 /*
3611                  * We only spin-down the drive if it is capable of it..
3612                  */
3613                 if ((softc->flags & ADA_FLAG_CAN_POWERMGT) == 0) {
3614                         cam_periph_unlock(periph);
3615                         continue;
3616                 }
3617
3618                 /*
3619                  * Additionally check if we would spin up the drive instead of
3620                  * spinning it down.
3621                  */
3622                 if (cmd == ATA_IDLE_IMMEDIATE) {
3623                         memset(&local_ccb, 0, sizeof(local_ccb));
3624                         xpt_setup_ccb(&local_ccb.ccb_h, periph->path,
3625                             CAM_PRIORITY_NORMAL);
3626                         local_ccb.ccb_h.ccb_state = ADA_CCB_DUMP;
3627
3628                         cam_fill_ataio(&local_ccb, 0, NULL, CAM_DIR_NONE,
3629                             0, NULL, 0, ada_default_timeout * 1000);
3630                         ata_28bit_cmd(&local_ccb, ATA_CHECK_POWER_MODE,
3631                             0, 0, 0);
3632                         local_ccb.cmd.flags |= CAM_ATAIO_NEEDRESULT;
3633
3634                         error = cam_periph_runccb((union ccb *)&local_ccb,
3635                             adaerror, /*cam_flags*/0,
3636                             /*sense_flags*/ SF_NO_RECOVERY | SF_NO_RETRY,
3637                             softc->disk->d_devstat);
3638                         if (error != 0) {
3639                                 xpt_print(periph->path,
3640                                     "Failed to read current power mode\n");
3641                         } else {
3642                                 mode = local_ccb.res.sector_count;
3643 #ifdef DIAGNOSTIC
3644                                 if (bootverbose) {
3645                                         xpt_print(periph->path,
3646                                             "disk power mode 0x%02x\n", mode);
3647                                 }
3648 #endif
3649                                 switch (mode) {
3650                                 case ATA_PM_STANDBY:
3651                                 case ATA_PM_STANDBY_Y:
3652                                         if (bootverbose) {
3653                                                 xpt_print(periph->path,
3654                                                     "already spun down\n");
3655                                         }
3656                                         cam_periph_unlock(periph);
3657                                         continue;
3658                                 default:
3659                                         break;
3660                                 }
3661                         }
3662                 }
3663
3664                 if (bootverbose)
3665                         xpt_print(periph->path, "spin-down\n");
3666
3667                 memset(&local_ccb, 0, sizeof(local_ccb));
3668                 xpt_setup_ccb(&local_ccb.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
3669                 local_ccb.ccb_h.ccb_state = ADA_CCB_DUMP;
3670
3671                 cam_fill_ataio(&local_ccb,
3672                                     0,
3673                                     NULL,
3674                                     CAM_DIR_NONE | flags,
3675                                     0,
3676                                     NULL,
3677                                     0,
3678                                     ada_default_timeout*1000);
3679                 ata_28bit_cmd(&local_ccb, cmd, 0, 0, 0);
3680                 error = cam_periph_runccb((union ccb *)&local_ccb, adaerror,
3681                     /*cam_flags*/0, /*sense_flags*/ SF_NO_RECOVERY | SF_NO_RETRY,
3682                     softc->disk->d_devstat);
3683                 if (error != 0)
3684                         xpt_print(periph->path, "Spin-down disk failed\n");
3685                 cam_periph_unlock(periph);
3686         }
3687 }
3688
3689 static void
3690 adashutdown(void *arg, int howto)
3691 {
3692         int how;
3693
3694         if ((howto & RB_NOSYNC) != 0)
3695                 return;
3696
3697         adaflush();
3698
3699         /*
3700          * STANDBY IMMEDIATE saves any volatile data to the drive. It also spins
3701          * down hard drives. IDLE IMMEDIATE also saves the volatile data without
3702          * a spindown. We send the former when we expect to lose power soon. For
3703          * a warm boot, we send the latter to avoid a thundering herd of spinups
3704          * just after the kernel loads while probing. We have to do something to
3705          * flush the data because the BIOS in many systems resets the HBA
3706          * causing a COMINIT/COMRESET negotiation, which some drives interpret
3707          * as license to toss the volatile data, and others count as unclean
3708          * shutdown when in the Active PM state in SMART attributes.
3709          *
3710          * adaspindown will ensure that we don't send this to a drive that
3711          * doesn't support it.
3712          */
3713         if (ada_spindown_shutdown != 0) {
3714                 how = (howto & (RB_HALT | RB_POWEROFF | RB_POWERCYCLE)) ?
3715                     ATA_STANDBY_IMMEDIATE : ATA_IDLE_IMMEDIATE;
3716                 adaspindown(how, 0);
3717         }
3718 }
3719
3720 static void
3721 adasuspend(void *arg)
3722 {
3723
3724         adaflush();
3725         /*
3726          * SLEEP also fushes any volatile data, like STANDBY IMEDIATE,
3727          * so we don't need to send it as well.
3728          */
3729         if (ada_spindown_suspend != 0)
3730                 adaspindown(ATA_SLEEP, CAM_DEV_QFREEZE);
3731 }
3732
3733 static void
3734 adaresume(void *arg)
3735 {
3736         struct cam_periph *periph;
3737         struct ada_softc *softc;
3738
3739         if (ada_spindown_suspend == 0)
3740                 return;
3741
3742         CAM_PERIPH_FOREACH(periph, &adadriver) {
3743                 cam_periph_lock(periph);
3744                 softc = (struct ada_softc *)periph->softc;
3745                 /*
3746                  * We only spin-down the drive if it is capable of it..
3747                  */
3748                 if ((softc->flags & ADA_FLAG_CAN_POWERMGT) == 0) {
3749                         cam_periph_unlock(periph);
3750                         continue;
3751                 }
3752
3753                 if (bootverbose)
3754                         xpt_print(periph->path, "resume\n");
3755
3756                 /*
3757                  * Drop freeze taken due to CAM_DEV_QFREEZE flag set on
3758                  * sleep request.
3759                  */
3760                 cam_release_devq(periph->path,
3761                          /*relsim_flags*/0,
3762                          /*openings*/0,
3763                          /*timeout*/0,
3764                          /*getcount_only*/0);
3765
3766                 cam_periph_unlock(periph);
3767         }
3768 }
3769
3770 #endif /* _KERNEL */