sys/cam/ata/ata_da.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2009 Alexander Motin <mav@FreeBSD.org>
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer,
  12  *    without modification, immediately at the beginning of the file.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  18  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  19  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  20  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  21  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  22  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  23  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  24  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  25  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  26  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27  */
  28
  29 #include <sys/cdefs.h>
  30 __FBSDID("$FreeBSD$");
  31
  32 #include "opt_ada.h"
  33
  34 #include <sys/param.h>
  35
  36 #ifdef _KERNEL
  37 #include <sys/systm.h>
  38 #include <sys/kernel.h>
  39 #include <sys/bio.h>
  40 #include <sys/sysctl.h>
  41 #include <sys/taskqueue.h>
  42 #include <sys/lock.h>
  43 #include <sys/mutex.h>
  44 #include <sys/conf.h>
  45 #include <sys/devicestat.h>
  46 #include <sys/eventhandler.h>
  47 #include <sys/malloc.h>
  48 #include <sys/endian.h>
  49 #include <sys/cons.h>
  50 #include <sys/proc.h>
  51 #include <sys/reboot.h>
  52 #include <sys/sbuf.h>
  53 #include <geom/geom.h>
  54 #include <geom/geom_disk.h>
  55 #endif /* _KERNEL */
  56
  57 #ifndef _KERNEL
  58 #include <stdio.h>
  59 #include <string.h>
  60 #endif /* _KERNEL */
  61
  62 #include <cam/cam.h>
  63 #include <cam/cam_ccb.h>
  64 #include <cam/cam_periph.h>
  65 #include <cam/cam_xpt_periph.h>
  66 #include <cam/scsi/scsi_all.h>
  67 #include <cam/scsi/scsi_da.h>
  68 #include <cam/cam_sim.h>
  69 #include <cam/cam_iosched.h>
  70
  71 #include <cam/ata/ata_all.h>
  72
  73 #ifdef _KERNEL
  74
  75 #define ATA_MAX_28BIT_LBA               268435455UL
  76
  77 extern int iosched_debug;
  78
  79 typedef enum {
  80         ADA_STATE_RAHEAD,
  81         ADA_STATE_WCACHE,
  82         ADA_STATE_LOGDIR,
  83         ADA_STATE_IDDIR,
  84         ADA_STATE_SUP_CAP,
  85         ADA_STATE_ZONE,
  86         ADA_STATE_NORMAL
  87 } ada_state;
  88
  89 typedef enum {
  90         ADA_FLAG_CAN_48BIT      = 0x00000002,
  91         ADA_FLAG_CAN_FLUSHCACHE = 0x00000004,
  92         ADA_FLAG_CAN_NCQ        = 0x00000008,
  93         ADA_FLAG_CAN_DMA        = 0x00000010,
  94         ADA_FLAG_NEED_OTAG      = 0x00000020,
  95         ADA_FLAG_WAS_OTAG       = 0x00000040,
  96         ADA_FLAG_CAN_TRIM       = 0x00000080,
  97         ADA_FLAG_OPEN           = 0x00000100,
  98         ADA_FLAG_SCTX_INIT      = 0x00000200,
  99         ADA_FLAG_CAN_CFA        = 0x00000400,
 100         ADA_FLAG_CAN_POWERMGT   = 0x00000800,
 101         ADA_FLAG_CAN_DMA48      = 0x00001000,
 102         ADA_FLAG_CAN_LOG        = 0x00002000,
 103         ADA_FLAG_CAN_IDLOG      = 0x00004000,
 104         ADA_FLAG_CAN_SUPCAP     = 0x00008000,
 105         ADA_FLAG_CAN_ZONE       = 0x00010000,
 106         ADA_FLAG_CAN_WCACHE     = 0x00020000,
 107         ADA_FLAG_CAN_RAHEAD     = 0x00040000,
 108         ADA_FLAG_PROBED         = 0x00080000,
 109         ADA_FLAG_ANNOUNCED      = 0x00100000,
 110         ADA_FLAG_DIRTY          = 0x00200000,
 111         ADA_FLAG_CAN_NCQ_TRIM   = 0x00400000,   /* CAN_TRIM also set */
 112         ADA_FLAG_PIM_ATA_EXT    = 0x00800000,
 113         ADA_FLAG_UNMAPPEDIO     = 0x01000000,
 114         ADA_FLAG_ROTATING       = 0x02000000
 115 } ada_flags;
 116 #define ADA_FLAG_STRING         \
 117         "\020"                  \
 118         "\002CAN_48BIT"         \
 119         "\003CAN_FLUSHCACHE"    \
 120         "\004CAN_NCQ"           \
 121         "\005CAN_DMA"           \
 122         "\006NEED_OTAG"         \
 123         "\007WAS_OTAG"          \
 124         "\010CAN_TRIM"          \
 125         "\011OPEN"              \
 126         "\012SCTX_INIT"         \
 127         "\013CAN_CFA"           \
 128         "\014CAN_POWERMGT"      \
 129         "\015CAN_DMA48"         \
 130         "\016CAN_LOG"           \
 131         "\017CAN_IDLOG"         \
 132         "\020CAN_SUPCAP"        \
 133         "\021CAN_ZONE"          \
 134         "\022CAN_WCACHE"        \
 135         "\023CAN_RAHEAD"        \
 136         "\024PROBED"            \
 137         "\025ANNOUNCED"         \
 138         "\026DIRTY"             \
 139         "\027CAN_NCQ_TRIM"      \
 140         "\030PIM_ATA_EXT"       \
 141         "\031UNMAPPEDIO"        \
 142         "\032ROTATING"
 143
 144 typedef enum {
 145         ADA_Q_NONE              = 0x00,
 146         ADA_Q_4K                = 0x01,
 147         ADA_Q_NCQ_TRIM_BROKEN   = 0x02,
 148         ADA_Q_LOG_BROKEN        = 0x04,
 149         ADA_Q_SMR_DM            = 0x08,
 150         ADA_Q_NO_TRIM           = 0x10,
 151         ADA_Q_128KB             = 0x20
 152 } ada_quirks;
 153
 154 #define ADA_Q_BIT_STRING        \
 155         "\020"                  \
 156         "\0014K"                \
 157         "\002NCQ_TRIM_BROKEN"   \
 158         "\003LOG_BROKEN"        \
 159         "\004SMR_DM"            \
 160         "\005NO_TRIM"           \
 161         "\006128KB"
 162
 163 typedef enum {
 164         ADA_CCB_RAHEAD          = 0x01,
 165         ADA_CCB_WCACHE          = 0x02,
 166         ADA_CCB_BUFFER_IO       = 0x03,
 167         ADA_CCB_DUMP            = 0x05,
 168         ADA_CCB_TRIM            = 0x06,
 169         ADA_CCB_LOGDIR          = 0x07,
 170         ADA_CCB_IDDIR           = 0x08,
 171         ADA_CCB_SUP_CAP         = 0x09,
 172         ADA_CCB_ZONE            = 0x0a,
 173         ADA_CCB_TYPE_MASK       = 0x0F,
 174 } ada_ccb_state;
 175
 176 typedef enum {
 177         ADA_ZONE_NONE           = 0x00,
 178         ADA_ZONE_DRIVE_MANAGED  = 0x01,
 179         ADA_ZONE_HOST_AWARE     = 0x02,
 180         ADA_ZONE_HOST_MANAGED   = 0x03
 181 } ada_zone_mode;
 182
 183 typedef enum {
 184         ADA_ZONE_FLAG_RZ_SUP            = 0x0001,
 185         ADA_ZONE_FLAG_OPEN_SUP          = 0x0002,
 186         ADA_ZONE_FLAG_CLOSE_SUP         = 0x0004,
 187         ADA_ZONE_FLAG_FINISH_SUP        = 0x0008,
 188         ADA_ZONE_FLAG_RWP_SUP           = 0x0010,
 189         ADA_ZONE_FLAG_SUP_MASK          = (ADA_ZONE_FLAG_RZ_SUP |
 190                                            ADA_ZONE_FLAG_OPEN_SUP |
 191                                            ADA_ZONE_FLAG_CLOSE_SUP |
 192                                            ADA_ZONE_FLAG_FINISH_SUP |
 193                                            ADA_ZONE_FLAG_RWP_SUP),
 194         ADA_ZONE_FLAG_URSWRZ            = 0x0020,
 195         ADA_ZONE_FLAG_OPT_SEQ_SET       = 0x0040,
 196         ADA_ZONE_FLAG_OPT_NONSEQ_SET    = 0x0080,
 197         ADA_ZONE_FLAG_MAX_SEQ_SET       = 0x0100,
 198         ADA_ZONE_FLAG_SET_MASK          = (ADA_ZONE_FLAG_OPT_SEQ_SET |
 199                                            ADA_ZONE_FLAG_OPT_NONSEQ_SET |
 200                                            ADA_ZONE_FLAG_MAX_SEQ_SET)
 201 } ada_zone_flags;
 202
 203 static struct ada_zone_desc {
 204         ada_zone_flags value;
 205         const char *desc;
 206 } ada_zone_desc_table[] = {
 207         {ADA_ZONE_FLAG_RZ_SUP, "Report Zones" },
 208         {ADA_ZONE_FLAG_OPEN_SUP, "Open" },
 209         {ADA_ZONE_FLAG_CLOSE_SUP, "Close" },
 210         {ADA_ZONE_FLAG_FINISH_SUP, "Finish" },
 211         {ADA_ZONE_FLAG_RWP_SUP, "Reset Write Pointer" },
 212 };
 213
 214 /* Offsets into our private area for storing information */
 215 #define ccb_state       ppriv_field0
 216 #define ccb_bp          ppriv_ptr1
 217
 218 typedef enum {
 219         ADA_DELETE_NONE,
 220         ADA_DELETE_DISABLE,
 221         ADA_DELETE_CFA_ERASE,
 222         ADA_DELETE_DSM_TRIM,
 223         ADA_DELETE_NCQ_DSM_TRIM,
 224         ADA_DELETE_MIN = ADA_DELETE_CFA_ERASE,
 225         ADA_DELETE_MAX = ADA_DELETE_NCQ_DSM_TRIM,
 226 } ada_delete_methods;
 227
 228 static const char *ada_delete_method_names[] =
 229     { "NONE", "DISABLE", "CFA_ERASE", "DSM_TRIM", "NCQ_DSM_TRIM" };
 230 #if 0
 231 static const char *ada_delete_method_desc[] =
 232     { "NONE", "DISABLED", "CFA Erase", "DSM Trim", "DSM Trim via NCQ" };
 233 #endif
 234
 235 struct disk_params {
 236         u_int8_t  heads;
 237         u_int8_t  secs_per_track;
 238         u_int32_t cylinders;
 239         u_int32_t secsize;      /* Number of bytes/logical sector */
 240         u_int64_t sectors;      /* Total number sectors */
 241 };
 242
 243 #define TRIM_MAX_BLOCKS 8
 244 #define TRIM_MAX_RANGES (TRIM_MAX_BLOCKS * ATA_DSM_BLK_RANGES)
 245 struct trim_request {
 246         uint8_t         data[TRIM_MAX_RANGES * ATA_DSM_RANGE_SIZE];
 247         TAILQ_HEAD(, bio) bps;
 248 };
 249
 250 struct ada_softc {
 251         struct   cam_iosched_softc *cam_iosched;
 252         int      outstanding_cmds;      /* Number of active commands */
 253         int      refcount;              /* Active xpt_action() calls */
 254         ada_state state;
 255         ada_flags flags;
 256         ada_zone_mode zone_mode;
 257         ada_zone_flags zone_flags;
 258         struct ata_gp_log_dir ata_logdir;
 259         int valid_logdir_len;
 260         struct ata_identify_log_pages ata_iddir;
 261         int valid_iddir_len;
 262         uint64_t optimal_seq_zones;
 263         uint64_t optimal_nonseq_zones;
 264         uint64_t max_seq_zones;
 265         ada_quirks quirks;
 266         ada_delete_methods delete_method;
 267         int      trim_max_ranges;
 268         int      read_ahead;
 269         int      write_cache;
 270 #ifdef CAM_TEST_FAILURE
 271         int      force_read_error;
 272         int      force_write_error;
 273         int      periodic_read_error;
 274         int      periodic_read_count;
 275 #endif
 276         struct ccb_pathinq      cpi;
 277         struct disk_params      params;
 278         struct disk             *disk;
 279         struct task             sysctl_task;
 280         struct sysctl_ctx_list  sysctl_ctx;
 281         struct sysctl_oid       *sysctl_tree;
 282         struct callout          sendordered_c;
 283         struct trim_request     trim_req;
 284         uint64_t                trim_count;
 285         uint64_t                trim_ranges;
 286         uint64_t                trim_lbas;
 287 #ifdef CAM_IO_STATS
 288         struct sysctl_ctx_list  sysctl_stats_ctx;
 289         struct sysctl_oid       *sysctl_stats_tree;
 290         u_int   timeouts;
 291         u_int   errors;
 292         u_int   invalidations;
 293 #endif
 294 #define ADA_ANNOUNCETMP_SZ 80
 295         char    announce_temp[ADA_ANNOUNCETMP_SZ];
 296 #define ADA_ANNOUNCE_SZ 400
 297         char    announce_buffer[ADA_ANNOUNCE_SZ];
 298 };
 299
 300 static uma_zone_t ada_ccb_zone;
 301
 302 struct ada_quirk_entry {
 303         struct scsi_inquiry_pattern inq_pat;
 304         ada_quirks quirks;
 305 };
 306
 307 static struct ada_quirk_entry ada_quirk_table[] =
 308 {
 309         {
 310                 /* Sandisk X400 */
 311                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SanDisk?SD8SB8U1T00*", "X4162000*" },
 312                 /*quirks*/ADA_Q_128KB
 313         },
 314         {
 315                 /* Hitachi Advanced Format (4k) drives */
 316                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Hitachi H??????????E3*", "*" },
 317                 /*quirks*/ADA_Q_4K
 318         },
 319         {
 320                 /* Samsung Advanced Format (4k) drives */
 321                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG HD155UI*", "*" },
 322                 /*quirks*/ADA_Q_4K
 323         },
 324         {
 325                 /* Samsung Advanced Format (4k) drives */
 326                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG HD204UI*", "*" },
 327                 /*quirks*/ADA_Q_4K
 328         },
 329         {
 330                 /* Seagate Barracuda Green Advanced Format (4k) drives */
 331                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST????DL*", "*" },
 332                 /*quirks*/ADA_Q_4K
 333         },
 334         {
 335                 /* Seagate Barracuda Advanced Format (4k) drives */
 336                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST???DM*", "*" },
 337                 /*quirks*/ADA_Q_4K
 338         },
 339         {
 340                 /* Seagate Barracuda Advanced Format (4k) drives */
 341                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST????DM*", "*" },
 342                 /*quirks*/ADA_Q_4K
 343         },
 344         {
 345                 /* Seagate Momentus Advanced Format (4k) drives */
 346                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9500423AS*", "*" },
 347                 /*quirks*/ADA_Q_4K
 348         },
 349         {
 350                 /* Seagate Momentus Advanced Format (4k) drives */
 351                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9500424AS*", "*" },
 352                 /*quirks*/ADA_Q_4K
 353         },
 354         {
 355                 /* Seagate Momentus Advanced Format (4k) drives */
 356                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9640423AS*", "*" },
 357                 /*quirks*/ADA_Q_4K
 358         },
 359         {
 360                 /* Seagate Momentus Advanced Format (4k) drives */
 361                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9640424AS*", "*" },
 362                 /*quirks*/ADA_Q_4K
 363         },
 364         {
 365                 /* Seagate Momentus Advanced Format (4k) drives */
 366                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9750420AS*", "*" },
 367                 /*quirks*/ADA_Q_4K
 368         },
 369         {
 370                 /* Seagate Momentus Advanced Format (4k) drives */
 371                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9750422AS*", "*" },
 372                 /*quirks*/ADA_Q_4K
 373         },
 374         {
 375                 /* Seagate Momentus Advanced Format (4k) drives */
 376                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9750423AS*", "*" },
 377                 /*quirks*/ADA_Q_4K
 378         },
 379         {
 380                 /* Seagate Momentus Thin Advanced Format (4k) drives */
 381                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST???LT*", "*" },
 382                 /*quirks*/ADA_Q_4K
 383         },
 384         {
 385                 /* WDC Caviar Red Advanced Format (4k) drives */
 386                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD????CX*", "*" },
 387                 /*quirks*/ADA_Q_4K
 388         },
 389         {
 390                 /* WDC Caviar Green Advanced Format (4k) drives */
 391                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD????RS*", "*" },
 392                 /*quirks*/ADA_Q_4K
 393         },
 394         {
 395                 /* WDC Caviar Green/Red Advanced Format (4k) drives */
 396                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD????RX*", "*" },
 397                 /*quirks*/ADA_Q_4K
 398         },
 399         {
 400                 /* WDC Caviar Red Advanced Format (4k) drives */
 401                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD??????CX*", "*" },
 402                 /*quirks*/ADA_Q_4K
 403         },
 404         {
 405                 /* WDC Caviar Black Advanced Format (4k) drives */
 406                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD????AZEX*", "*" },
 407                 /*quirks*/ADA_Q_4K
 408         },
 409         {
 410                 /* WDC Caviar Black Advanced Format (4k) drives */
 411                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD????FZEX*", "*" },
 412                 /*quirks*/ADA_Q_4K
 413         },
 414         {
 415                 /* WDC Caviar Green Advanced Format (4k) drives */
 416                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD??????RS*", "*" },
 417                 /*quirks*/ADA_Q_4K
 418         },
 419         {
 420                 /* WDC Caviar Green Advanced Format (4k) drives */
 421                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD??????RX*", "*" },
 422                 /*quirks*/ADA_Q_4K
 423         },
 424         {
 425                 /* WDC Scorpio Black Advanced Format (4k) drives */
 426                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD???PKT*", "*" },
 427                 /*quirks*/ADA_Q_4K
 428         },
 429         {
 430                 /* WDC Scorpio Black Advanced Format (4k) drives */
 431                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD?????PKT*", "*" },
 432                 /*quirks*/ADA_Q_4K
 433         },
 434         {
 435                 /* WDC Scorpio Blue Advanced Format (4k) drives */
 436                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD???PVT*", "*" },
 437                 /*quirks*/ADA_Q_4K
 438         },
 439         {
 440                 /* WDC Scorpio Blue Advanced Format (4k) drives */
 441                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD?????PVT*", "*" },
 442                 /*quirks*/ADA_Q_4K
 443         },
 444         /* SSDs */
 445         {
 446                 /*
 447                  * Corsair Force 2 SSDs
 448                  * 4k optimised & trim only works in 4k requests + 4k aligned
 449                  */
 450                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair CSSD-F*", "*" },
 451                 /*quirks*/ADA_Q_4K
 452         },
 453         {
 454                 /*
 455                  * Corsair Force 3 SSDs
 456                  * 4k optimised & trim only works in 4k requests + 4k aligned
 457                  */
 458                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair Force 3*", "*" },
 459                 /*quirks*/ADA_Q_4K
 460         },
 461         {
 462                 /*
 463                  * Corsair Neutron GTX SSDs
 464                  * 4k optimised & trim only works in 4k requests + 4k aligned
 465                  */
 466                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair Neutron GTX*", "*" },
 467                 /*quirks*/ADA_Q_4K
 468         },
 469         {
 470                 /*
 471                  * Corsair Force GT & GS SSDs
 472                  * 4k optimised & trim only works in 4k requests + 4k aligned
 473                  */
 474                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair Force G*", "*" },
 475                 /*quirks*/ADA_Q_4K
 476         },
 477         {
 478                 /*
 479                  * Crucial M4 SSDs
 480                  * 4k optimised & trim only works in 4k requests + 4k aligned
 481                  */
 482                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "M4-CT???M4SSD2*", "*" },
 483                 /*quirks*/ADA_Q_4K
 484         },
 485         {
 486                 /*
 487                  * Crucial M500 SSDs MU07 firmware
 488                  * NCQ Trim works
 489                  */
 490                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Crucial CT*M500*", "MU07" },
 491                 /*quirks*/0
 492         },
 493         {
 494                 /*
 495                  * Crucial M500 SSDs all other firmware
 496                  * NCQ Trim doesn't work
 497                  */
 498                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Crucial CT*M500*", "*" },
 499                 /*quirks*/ADA_Q_NCQ_TRIM_BROKEN
 500         },
 501         {
 502                 /*
 503                  * Crucial M550 SSDs
 504                  * NCQ Trim doesn't work, but only on MU01 firmware
 505                  */
 506                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Crucial CT*M550*", "MU01" },
 507                 /*quirks*/ADA_Q_NCQ_TRIM_BROKEN
 508         },
 509         {
 510                 /*
 511                  * Crucial MX100 SSDs
 512                  * NCQ Trim doesn't work, but only on MU01 firmware
 513                  */
 514                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Crucial CT*MX100*", "MU01" },
 515                 /*quirks*/ADA_Q_NCQ_TRIM_BROKEN
 516         },
 517         {
 518                 /*
 519                  * Crucial RealSSD C300 SSDs
 520                  * 4k optimised
 521                  */
 522                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "C300-CTFDDAC???MAG*",
 523                 "*" }, /*quirks*/ADA_Q_4K
 524         },
 525         {
 526                 /*
 527                  * FCCT M500 SSDs
 528                  * NCQ Trim doesn't work
 529                  */
 530                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "FCCT*M500*", "*" },
 531                 /*quirks*/ADA_Q_NCQ_TRIM_BROKEN
 532         },
 533         {
 534                 /*
 535                  * Intel 320 Series SSDs
 536                  * 4k optimised & trim only works in 4k requests + 4k aligned
 537                  */
 538                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSA2CW*", "*" },
 539                 /*quirks*/ADA_Q_4K
 540         },
 541         {
 542                 /*
 543                  * Intel 330 Series SSDs
 544                  * 4k optimised & trim only works in 4k requests + 4k aligned
 545                  */
 546                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSC2CT*", "*" },
 547                 /*quirks*/ADA_Q_4K
 548         },
 549         {
 550                 /*
 551                  * Intel 510 Series SSDs
 552                  * 4k optimised & trim only works in 4k requests + 4k aligned
 553                  */
 554                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSC2MH*", "*" },
 555                 /*quirks*/ADA_Q_4K
 556         },
 557         {
 558                 /*
 559                  * Intel 520 Series SSDs
 560                  * 4k optimised & trim only works in 4k requests + 4k aligned
 561                  */
 562                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSC2BW*", "*" },
 563                 /*quirks*/ADA_Q_4K
 564         },
 565         {
 566                 /*
 567                  * Intel S3610 Series SSDs
 568                  * 4k optimised & trim only works in 4k requests + 4k aligned
 569                  */
 570                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSC2BX*", "*" },
 571                 /*quirks*/ADA_Q_4K
 572         },
 573         {
 574                 /*
 575                  * Intel X25-M Series SSDs
 576                  * 4k optimised & trim only works in 4k requests + 4k aligned
 577                  */
 578                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSA2M*", "*" },
 579                 /*quirks*/ADA_Q_4K
 580         },
 581         {
 582                 /*
 583                  * KingDian S200 60GB P0921B
 584                  * Trimming crash the SSD
 585                  */
 586                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "KingDian S200 *", "*" },
 587                 /*quirks*/ADA_Q_NO_TRIM
 588         },
 589         {
 590                 /*
 591                  * Kingston E100 Series SSDs
 592                  * 4k optimised & trim only works in 4k requests + 4k aligned
 593                  */
 594                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "KINGSTON SE100S3*", "*" },
 595                 /*quirks*/ADA_Q_4K
 596         },
 597         {
 598                 /*
 599                  * Kingston HyperX 3k SSDs
 600                  * 4k optimised & trim only works in 4k requests + 4k aligned
 601                  */
 602                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "KINGSTON SH103S3*", "*" },
 603                 /*quirks*/ADA_Q_4K
 604         },
 605         {
 606                 /*
 607                  * Marvell SSDs (entry taken from OpenSolaris)
 608                  * 4k optimised & trim only works in 4k requests + 4k aligned
 609                  */
 610                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "MARVELL SD88SA02*", "*" },
 611                 /*quirks*/ADA_Q_4K
 612         },
 613         {
 614                 /*
 615                  * Micron M500 SSDs firmware MU07
 616                  * NCQ Trim works?
 617                  */
 618                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Micron M500*", "MU07" },
 619                 /*quirks*/0
 620         },
 621         {
 622                 /*
 623                  * Micron M500 SSDs all other firmware
 624                  * NCQ Trim doesn't work
 625                  */
 626                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Micron M500*", "*" },
 627                 /*quirks*/ADA_Q_NCQ_TRIM_BROKEN
 628         },
 629         {
 630                 /*
 631                  * Micron M5[15]0 SSDs
 632                  * NCQ Trim doesn't work, but only MU01 firmware
 633                  */
 634                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Micron M5[15]0*", "MU01" },
 635                 /*quirks*/ADA_Q_NCQ_TRIM_BROKEN
 636         },
 637         {
 638                 /*
 639                  * Micron 5100 SSDs
 640                  * 4k optimised & trim only works in 4k requests + 4k aligned
 641                  */
 642                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Micron 5100 MTFDDAK*", "*" },
 643                 /*quirks*/ADA_Q_4K
 644         },
 645         {
 646                 /*
 647                  * OCZ Agility 2 SSDs
 648                  * 4k optimised & trim only works in 4k requests + 4k aligned
 649                  */
 650                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-AGILITY2*", "*" },
 651                 /*quirks*/ADA_Q_4K
 652         },
 653         {
 654                 /*
 655                  * OCZ Agility 3 SSDs
 656                  * 4k optimised & trim only works in 4k requests + 4k aligned
 657                  */
 658                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-AGILITY3*", "*" },
 659                 /*quirks*/ADA_Q_4K
 660         },
 661         {
 662                 /*
 663                  * OCZ Deneva R Series SSDs
 664                  * 4k optimised & trim only works in 4k requests + 4k aligned
 665                  */
 666                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "DENRSTE251M45*", "*" },
 667                 /*quirks*/ADA_Q_4K
 668         },
 669         {
 670                 /*
 671                  * OCZ Vertex 2 SSDs (inc pro series)
 672                  * 4k optimised & trim only works in 4k requests + 4k aligned
 673                  */
 674                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ?VERTEX2*", "*" },
 675                 /*quirks*/ADA_Q_4K
 676         },
 677         {
 678                 /*
 679                  * OCZ Vertex 3 SSDs
 680                  * 4k optimised & trim only works in 4k requests + 4k aligned
 681                  */
 682                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-VERTEX3*", "*" },
 683                 /*quirks*/ADA_Q_4K
 684         },
 685         {
 686                 /*
 687                  * OCZ Vertex 4 SSDs
 688                  * 4k optimised & trim only works in 4k requests + 4k aligned
 689                  */
 690                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-VERTEX4*", "*" },
 691                 /*quirks*/ADA_Q_4K
 692         },
 693         {
 694                 /*
 695                  * Samsung 750 SSDs
 696                  * 4k optimised, NCQ TRIM seems to work
 697                  */
 698                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Samsung SSD 750*", "*" },
 699                 /*quirks*/ADA_Q_4K
 700         },
 701         {
 702                 /*
 703                  * Samsung 830 Series SSDs
 704                  * 4k optimised, NCQ TRIM Broken (normal TRIM is fine)
 705                  */
 706                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG SSD 830 Series*", "*" },
 707                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 708         },
 709         {
 710                 /*
 711                  * Samsung 840 SSDs
 712                  * 4k optimised, NCQ TRIM Broken (normal TRIM is fine)
 713                  */
 714                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Samsung SSD 840*", "*" },
 715                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 716         },
 717         {
 718                 /*
 719                  * Samsung 845 SSDs
 720                  * 4k optimised, NCQ TRIM Broken (normal TRIM is fine)
 721                  */
 722                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Samsung SSD 845*", "*" },
 723                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 724         },
 725         {
 726                 /*
 727                  * Samsung 850 SSDs
 728                  * 4k optimised, NCQ TRIM broken (normal TRIM fine)
 729                  */
 730                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Samsung SSD 850*", "*" },
 731                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 732         },
 733         {
 734                 /*
 735                  * Samsung SM863 Series SSDs (MZ7KM*)
 736                  * 4k optimised, NCQ believed to be working
 737                  */
 738                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG MZ7KM*", "*" },
 739                 /*quirks*/ADA_Q_4K
 740         },
 741         {
 742                 /*
 743                  * Samsung 843T Series SSDs (MZ7WD*)
 744                  * Samsung PM851 Series SSDs (MZ7TE*)
 745                  * Samsung PM853T Series SSDs (MZ7GE*)
 746                  * 4k optimised, NCQ believed to be broken since these are
 747                  * appear to be built with the same controllers as the 840/850.
 748                  */
 749                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG MZ7*", "*" },
 750                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 751         },
 752         {
 753                 /*
 754                  * Same as for SAMSUNG MZ7* but enable the quirks for SSD
 755                  * starting with MZ7* too
 756                  */
 757                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "MZ7*", "*" },
 758                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 759         },
 760         {
 761                 /*
 762                  * Samsung PM851 Series SSDs Dell OEM
 763                  * device model          "SAMSUNG SSD PM851 mSATA 256GB"
 764                  * 4k optimised, NCQ broken
 765                  */
 766                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG SSD PM851*", "*" },
 767                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 768         },
 769         {
 770                 /*
 771                  * SuperTalent TeraDrive CT SSDs
 772                  * 4k optimised & trim only works in 4k requests + 4k aligned
 773                  */
 774                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "FTM??CT25H*", "*" },
 775                 /*quirks*/ADA_Q_4K
 776         },
 777         {
 778                 /*
 779                  * XceedIOPS SATA SSDs
 780                  * 4k optimised
 781                  */
 782                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SG9XCS2D*", "*" },
 783                 /*quirks*/ADA_Q_4K
 784         },
 785         {
 786                 /*
 787                  * Samsung drive that doesn't support READ LOG EXT or
 788                  * READ LOG DMA EXT, despite reporting that it does in
 789                  * ATA identify data:
 790                  * SAMSUNG HD200HJ KF100-06
 791                  */
 792                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG HD200*", "*" },
 793                 /*quirks*/ADA_Q_LOG_BROKEN
 794         },
 795         {
 796                 /*
 797                  * Samsung drive that doesn't support READ LOG EXT or
 798                  * READ LOG DMA EXT, despite reporting that it does in
 799                  * ATA identify data:
 800                  * SAMSUNG HD501LJ CR100-10
 801                  */
 802                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG HD501*", "*" },
 803                 /*quirks*/ADA_Q_LOG_BROKEN
 804         },
 805         {
 806                 /*
 807                  * Seagate Lamarr 8TB Shingled Magnetic Recording (SMR)
 808                  * Drive Managed SATA hard drive.  This drive doesn't report
 809                  * in firmware that it is a drive managed SMR drive.
 810                  */
 811                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST8000AS000[23]*", "*" },
 812                 /*quirks*/ADA_Q_SMR_DM
 813         },
 814         {
 815                 /* WD Green SSD */
 816                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WDS?????G0*", "*" },
 817                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 818         },
 819         {
 820                 /* Default */
 821                 {
 822                   T_ANY, SIP_MEDIA_REMOVABLE|SIP_MEDIA_FIXED,
 823                   /*vendor*/"*", /*product*/"*", /*revision*/"*"
 824                 },
 825                 /*quirks*/0
 826         },
 827 };
 828
 829 static  disk_strategy_t adastrategy;
 830 static  dumper_t        adadump;
 831 static  periph_init_t   adainit;
 832 static  void            adadiskgonecb(struct disk *dp);
 833 static  periph_oninv_t  adaoninvalidate;
 834 static  periph_dtor_t   adacleanup;
 835 static  void            adaasync(void *callback_arg, u_int32_t code,
 836                                 struct cam_path *path, void *arg);
 837 static  int             adabitsysctl(SYSCTL_HANDLER_ARGS);
 838 static  int             adaflagssysctl(SYSCTL_HANDLER_ARGS);
 839 static  int             adazonesupsysctl(SYSCTL_HANDLER_ARGS);
 840 static  void            adasysctlinit(void *context, int pending);
 841 static  int             adagetattr(struct bio *bp);
 842 static  void            adasetflags(struct ada_softc *softc,
 843                                     struct ccb_getdev *cgd);
 844 static void             adasetgeom(struct ada_softc *softc,
 845                                    struct ccb_getdev *cgd);
 846 static  periph_ctor_t   adaregister;
 847 static  void            ada_dsmtrim(struct ada_softc *softc, struct bio *bp,
 848                                     struct ccb_ataio *ataio);
 849 static  void            ada_cfaerase(struct ada_softc *softc, struct bio *bp,
 850                                      struct ccb_ataio *ataio);
 851 static  int             ada_zone_bio_to_ata(int disk_zone_cmd);
 852 static  int             ada_zone_cmd(struct cam_periph *periph, union ccb *ccb,
 853                                      struct bio *bp, int *queue_ccb);
 854 static  periph_start_t  adastart;
 855 static  void            adaprobedone(struct cam_periph *periph, union ccb *ccb);
 856 static  void            adazonedone(struct cam_periph *periph, union ccb *ccb);
 857 static  void            adadone(struct cam_periph *periph,
 858                                union ccb *done_ccb);
 859 static  int             adaerror(union ccb *ccb, u_int32_t cam_flags,
 860                                 u_int32_t sense_flags);
 861 static callout_func_t   adasendorderedtag;
 862 static void             adashutdown(void *arg, int howto);
 863 static void             adasuspend(void *arg);
 864 static void             adaresume(void *arg);
 865
 866 #ifndef ADA_DEFAULT_TIMEOUT
 867 #define ADA_DEFAULT_TIMEOUT 30  /* Timeout in seconds */
 868 #endif
 869
 870 #ifndef ADA_DEFAULT_RETRY
 871 #define ADA_DEFAULT_RETRY       4
 872 #endif
 873
 874 #ifndef ADA_DEFAULT_SEND_ORDERED
 875 #define ADA_DEFAULT_SEND_ORDERED        1
 876 #endif
 877
 878 #ifndef ADA_DEFAULT_SPINDOWN_SHUTDOWN
 879 #define ADA_DEFAULT_SPINDOWN_SHUTDOWN   1
 880 #endif
 881
 882 #ifndef ADA_DEFAULT_SPINDOWN_SUSPEND
 883 #define ADA_DEFAULT_SPINDOWN_SUSPEND    1
 884 #endif
 885
 886 #ifndef ADA_DEFAULT_READ_AHEAD
 887 #define ADA_DEFAULT_READ_AHEAD  1
 888 #endif
 889
 890 #ifndef ADA_DEFAULT_WRITE_CACHE
 891 #define ADA_DEFAULT_WRITE_CACHE 1
 892 #endif
 893
 894 #define ADA_RA  (softc->read_ahead >= 0 ? \
 895                  softc->read_ahead : ada_read_ahead)
 896 #define ADA_WC  (softc->write_cache >= 0 ? \
 897                  softc->write_cache : ada_write_cache)
 898
 899 static int ada_retry_count = ADA_DEFAULT_RETRY;
 900 static int ada_default_timeout = ADA_DEFAULT_TIMEOUT;
 901 static int ada_send_ordered = ADA_DEFAULT_SEND_ORDERED;
 902 static int ada_spindown_shutdown = ADA_DEFAULT_SPINDOWN_SHUTDOWN;
 903 static int ada_spindown_suspend = ADA_DEFAULT_SPINDOWN_SUSPEND;
 904 static int ada_read_ahead = ADA_DEFAULT_READ_AHEAD;
 905 static int ada_write_cache = ADA_DEFAULT_WRITE_CACHE;
 906 static int ada_enable_biospeedup = 1;
 907 static int ada_enable_uma_ccbs = 1;
 908
 909 static SYSCTL_NODE(_kern_cam, OID_AUTO, ada, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
 910     "CAM Direct Access Disk driver");
 911 SYSCTL_INT(_kern_cam_ada, OID_AUTO, retry_count, CTLFLAG_RWTUN,
 912            &ada_retry_count, 0, "Normal I/O retry count");
 913 SYSCTL_INT(_kern_cam_ada, OID_AUTO, default_timeout, CTLFLAG_RWTUN,
 914            &ada_default_timeout, 0, "Normal I/O timeout (in seconds)");
 915 SYSCTL_INT(_kern_cam_ada, OID_AUTO, send_ordered, CTLFLAG_RWTUN,
 916            &ada_send_ordered, 0, "Send Ordered Tags");
 917 SYSCTL_INT(_kern_cam_ada, OID_AUTO, spindown_shutdown, CTLFLAG_RWTUN,
 918            &ada_spindown_shutdown, 0, "Spin down upon shutdown");
 919 SYSCTL_INT(_kern_cam_ada, OID_AUTO, spindown_suspend, CTLFLAG_RWTUN,
 920            &ada_spindown_suspend, 0, "Spin down upon suspend");
 921 SYSCTL_INT(_kern_cam_ada, OID_AUTO, read_ahead, CTLFLAG_RWTUN,
 922            &ada_read_ahead, 0, "Enable disk read-ahead");
 923 SYSCTL_INT(_kern_cam_ada, OID_AUTO, write_cache, CTLFLAG_RWTUN,
 924            &ada_write_cache, 0, "Enable disk write cache");
 925 SYSCTL_INT(_kern_cam_ada, OID_AUTO, enable_biospeedup, CTLFLAG_RDTUN,
 926            &ada_enable_biospeedup, 0, "Enable BIO_SPEEDUP processing");
 927 SYSCTL_INT(_kern_cam_ada, OID_AUTO, enable_uma_ccbs, CTLFLAG_RWTUN,
 928             &ada_enable_uma_ccbs, 0, "Use UMA for CCBs");
 929
 930 /*
 931  * ADA_ORDEREDTAG_INTERVAL determines how often, relative
 932  * to the default timeout, we check to see whether an ordered
 933  * tagged transaction is appropriate to prevent simple tag
 934  * starvation.  Since we'd like to ensure that there is at least
 935  * 1/2 of the timeout length left for a starved transaction to
 936  * complete after we've sent an ordered tag, we must poll at least
 937  * four times in every timeout period.  This takes care of the worst
 938  * case where a starved transaction starts during an interval that
 939  * meets the requirement "don't send an ordered tag" test so it takes
 940  * us two intervals to determine that a tag must be sent.
 941  */
 942 #ifndef ADA_ORDEREDTAG_INTERVAL
 943 #define ADA_ORDEREDTAG_INTERVAL 4
 944 #endif
 945
 946 static struct periph_driver adadriver =
 947 {
 948         adainit, "ada",
 949         TAILQ_HEAD_INITIALIZER(adadriver.units), /* generation */ 0
 950 };
 951
 952 static int adadeletemethodsysctl(SYSCTL_HANDLER_ARGS);
 953
 954 PERIPHDRIVER_DECLARE(ada, adadriver);
 955
 956 static MALLOC_DEFINE(M_ATADA, "ata_da", "ata_da buffers");
 957
 958 static int
 959 adaopen(struct disk *dp)
 960 {
 961         struct cam_periph *periph;
 962         struct ada_softc *softc;
 963         int error;
 964
 965         periph = (struct cam_periph *)dp->d_drv1;
 966         if (cam_periph_acquire(periph) != 0) {
 967                 return(ENXIO);
 968         }
 969
 970         cam_periph_lock(periph);
 971         if ((error = cam_periph_hold(periph, PRIBIO|PCATCH)) != 0) {
 972                 cam_periph_unlock(periph);
 973                 cam_periph_release(periph);
 974                 return (error);
 975         }
 976
 977         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE | CAM_DEBUG_PERIPH,
 978             ("adaopen\n"));
 979
 980         softc = (struct ada_softc *)periph->softc;
 981         softc->flags |= ADA_FLAG_OPEN;
 982
 983         cam_periph_unhold(periph);
 984         cam_periph_unlock(periph);
 985         return (0);
 986 }
 987
 988 static int
 989 adaclose(struct disk *dp)
 990 {
 991         struct  cam_periph *periph;
 992         struct  ada_softc *softc;
 993         union ccb *ccb;
 994         int error;
 995
 996         periph = (struct cam_periph *)dp->d_drv1;
 997         softc = (struct ada_softc *)periph->softc;
 998         cam_periph_lock(periph);
 999
1000         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE | CAM_DEBUG_PERIPH,
1001             ("adaclose\n"));
1002
1003         /* We only sync the cache if the drive is capable of it. */
1004         if ((softc->flags & ADA_FLAG_DIRTY) != 0 &&
1005             (softc->flags & ADA_FLAG_CAN_FLUSHCACHE) != 0 &&
1006             (periph->flags & CAM_PERIPH_INVALID) == 0 &&
1007             cam_periph_hold(periph, PRIBIO) == 0) {
1008                 ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL);
1009                 cam_fill_ataio(&ccb->ataio,
1010                                     1,
1011                                     NULL,
1012                                     CAM_DIR_NONE,
1013                                     0,
1014                                     NULL,
1015                                     0,
1016                                     ada_default_timeout*1000);
1017
1018                 if (softc->flags & ADA_FLAG_CAN_48BIT)
1019                         ata_48bit_cmd(&ccb->ataio, ATA_FLUSHCACHE48, 0, 0, 0);
1020                 else
1021                         ata_28bit_cmd(&ccb->ataio, ATA_FLUSHCACHE, 0, 0, 0);
1022                 error = cam_periph_runccb(ccb, adaerror, /*cam_flags*/0,
1023                     /*sense_flags*/0, softc->disk->d_devstat);
1024
1025                 if (error != 0)
1026                         xpt_print(periph->path, "Synchronize cache failed\n");
1027                 softc->flags &= ~ADA_FLAG_DIRTY;
1028                 xpt_release_ccb(ccb);
1029                 cam_periph_unhold(periph);
1030         }
1031
1032         softc->flags &= ~ADA_FLAG_OPEN;
1033
1034         while (softc->refcount != 0)
1035                 cam_periph_sleep(periph, &softc->refcount, PRIBIO, "adaclose", 1);
1036         cam_periph_unlock(periph);
1037         cam_periph_release(periph);
1038         return (0);
1039 }
1040
1041 static void
1042 adaschedule(struct cam_periph *periph)
1043 {
1044         struct ada_softc *softc = (struct ada_softc *)periph->softc;
1045
1046         if (softc->state != ADA_STATE_NORMAL)
1047                 return;
1048
1049         cam_iosched_schedule(softc->cam_iosched, periph);
1050 }
1051
1052 /*
1053  * Actually translate the requested transfer into one the physical driver
1054  * can understand.  The transfer is described by a buf and will include
1055  * only one physical transfer.
1056  */
1057 static void
1058 adastrategy(struct bio *bp)
1059 {
1060         struct cam_periph *periph;
1061         struct ada_softc *softc;
1062
1063         periph = (struct cam_periph *)bp->bio_disk->d_drv1;
1064         softc = (struct ada_softc *)periph->softc;
1065
1066         cam_periph_lock(periph);
1067
1068         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("adastrategy(%p)\n", bp));
1069
1070         /*
1071          * If the device has been made invalid, error out
1072          */
1073         if ((periph->flags & CAM_PERIPH_INVALID) != 0) {
1074                 cam_periph_unlock(periph);
1075                 biofinish(bp, NULL, ENXIO);
1076                 return;
1077         }
1078
1079         /*
1080          * Zone commands must be ordered, because they can depend on the
1081          * effects of previously issued commands, and they may affect
1082          * commands after them.
1083          */
1084         if (bp->bio_cmd == BIO_ZONE)
1085                 bp->bio_flags |= BIO_ORDERED;
1086
1087         /*
1088          * Place it in the queue of disk activities for this disk
1089          */
1090         cam_iosched_queue_work(softc->cam_iosched, bp);
1091
1092         /*
1093          * Schedule ourselves for performing the work.
1094          */
1095         adaschedule(periph);
1096         cam_periph_unlock(periph);
1097
1098         return;
1099 }
1100
1101 static int
1102 adadump(void *arg, void *virtual, vm_offset_t physical, off_t offset, size_t length)
1103 {
1104         struct      cam_periph *periph;
1105         struct      ada_softc *softc;
1106         u_int       secsize;
1107         struct      ccb_ataio ataio;
1108         struct      disk *dp;
1109         uint64_t    lba;
1110         uint16_t    count;
1111         int         error = 0;
1112
1113         dp = arg;
1114         periph = dp->d_drv1;
1115         softc = (struct ada_softc *)periph->softc;
1116         secsize = softc->params.secsize;
1117         lba = offset / secsize;
1118         count = length / secsize;
1119         if ((periph->flags & CAM_PERIPH_INVALID) != 0)
1120                 return (ENXIO);
1121
1122         memset(&ataio, 0, sizeof(ataio));
1123         if (length > 0) {
1124                 xpt_setup_ccb(&ataio.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
1125                 ataio.ccb_h.ccb_state = ADA_CCB_DUMP;
1126                 cam_fill_ataio(&ataio,
1127                     0,
1128                     NULL,
1129                     CAM_DIR_OUT,
1130                     0,
1131                     (u_int8_t *) virtual,
1132                     length,
1133                     ada_default_timeout*1000);
1134                 if ((softc->flags & ADA_FLAG_CAN_48BIT) &&
1135                     (lba + count >= ATA_MAX_28BIT_LBA ||
1136                     count >= 256)) {
1137                         ata_48bit_cmd(&ataio, ATA_WRITE_DMA48,
1138                             0, lba, count);
1139                 } else {
1140                         ata_28bit_cmd(&ataio, ATA_WRITE_DMA,
1141                             0, lba, count);
1142                 }
1143                 error = cam_periph_runccb((union ccb *)&ataio, adaerror,
1144                     0, SF_NO_RECOVERY | SF_NO_RETRY, NULL);
1145                 if (error != 0)
1146                         printf("Aborting dump due to I/O error.\n");
1147
1148                 return (error);
1149         }
1150
1151         if (softc->flags & ADA_FLAG_CAN_FLUSHCACHE) {
1152                 xpt_setup_ccb(&ataio.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
1153
1154                 /*
1155                  * Tell the drive to flush its internal cache. if we
1156                  * can't flush in 5s we have big problems. No need to
1157                  * wait the default 60s to detect problems.
1158                  */
1159                 ataio.ccb_h.ccb_state = ADA_CCB_DUMP;
1160                 cam_fill_ataio(&ataio,
1161                                     0,
1162                                     NULL,
1163                                     CAM_DIR_NONE,
1164                                     0,
1165                                     NULL,
1166                                     0,
1167                                     5*1000);
1168
1169                 if (softc->flags & ADA_FLAG_CAN_48BIT)
1170                         ata_48bit_cmd(&ataio, ATA_FLUSHCACHE48, 0, 0, 0);
1171                 else
1172                         ata_28bit_cmd(&ataio, ATA_FLUSHCACHE, 0, 0, 0);
1173                 error = cam_periph_runccb((union ccb *)&ataio, adaerror,
1174                     0, SF_NO_RECOVERY | SF_NO_RETRY, NULL);
1175                 if (error != 0)
1176                         xpt_print(periph->path, "Synchronize cache failed\n");
1177         }
1178         return (error);
1179 }
1180
1181 static void
1182 adainit(void)
1183 {
1184         cam_status status;
1185
1186         ada_ccb_zone = uma_zcreate("ada_ccb",
1187             sizeof(struct ccb_ataio), NULL, NULL, NULL, NULL,
1188             UMA_ALIGN_PTR, 0);
1189
1190         /*
1191          * Install a global async callback.  This callback will
1192          * receive async callbacks like "new device found".
1193          */
1194         status = xpt_register_async(AC_FOUND_DEVICE, adaasync, NULL, NULL);
1195
1196         if (status != CAM_REQ_CMP) {
1197                 printf("ada: Failed to attach master async callback "
1198                        "due to status 0x%x!\n", status);
1199         } else if (ada_send_ordered) {
1200                 /* Register our event handlers */
1201                 if ((EVENTHANDLER_REGISTER(power_suspend, adasuspend,
1202                                            NULL, EVENTHANDLER_PRI_LAST)) == NULL)
1203                     printf("adainit: power event registration failed!\n");
1204                 if ((EVENTHANDLER_REGISTER(power_resume, adaresume,
1205                                            NULL, EVENTHANDLER_PRI_LAST)) == NULL)
1206                     printf("adainit: power event registration failed!\n");
1207                 if ((EVENTHANDLER_REGISTER(shutdown_post_sync, adashutdown,
1208                                            NULL, SHUTDOWN_PRI_DEFAULT)) == NULL)
1209                     printf("adainit: shutdown event registration failed!\n");
1210         }
1211 }
1212
1213 /*
1214  * Callback from GEOM, called when it has finished cleaning up its
1215  * resources.
1216  */
1217 static void
1218 adadiskgonecb(struct disk *dp)
1219 {
1220         struct cam_periph *periph;
1221
1222         periph = (struct cam_periph *)dp->d_drv1;
1223
1224         cam_periph_release(periph);
1225 }
1226
1227 static void
1228 adaoninvalidate(struct cam_periph *periph)
1229 {
1230         struct ada_softc *softc;
1231
1232         softc = (struct ada_softc *)periph->softc;
1233
1234         /*
1235          * De-register any async callbacks.
1236          */
1237         xpt_register_async(0, adaasync, periph, periph->path);
1238 #ifdef CAM_IO_STATS
1239         softc->invalidations++;
1240 #endif
1241
1242         /*
1243          * Return all queued I/O with ENXIO.
1244          * XXX Handle any transactions queued to the card
1245          *     with XPT_ABORT_CCB.
1246          */
1247         cam_iosched_flush(softc->cam_iosched, NULL, ENXIO);
1248
1249         disk_gone(softc->disk);
1250 }
1251
1252 static void
1253 adacleanup(struct cam_periph *periph)
1254 {
1255         struct ada_softc *softc;
1256
1257         softc = (struct ada_softc *)periph->softc;
1258
1259         cam_periph_unlock(periph);
1260
1261         cam_iosched_fini(softc->cam_iosched);
1262
1263         /*
1264          * If we can't free the sysctl tree, oh well...
1265          */
1266         if ((softc->flags & ADA_FLAG_SCTX_INIT) != 0) {
1267 #ifdef CAM_IO_STATS
1268                 if (sysctl_ctx_free(&softc->sysctl_stats_ctx) != 0)
1269                         xpt_print(periph->path,
1270                             "can't remove sysctl stats context\n");
1271 #endif
1272                 if (sysctl_ctx_free(&softc->sysctl_ctx) != 0)
1273                         xpt_print(periph->path,
1274                             "can't remove sysctl context\n");
1275         }
1276
1277         disk_destroy(softc->disk);
1278         callout_drain(&softc->sendordered_c);
1279         free(softc, M_DEVBUF);
1280         cam_periph_lock(periph);
1281 }
1282
1283 static void
1284 adasetdeletemethod(struct ada_softc *softc)
1285 {
1286
1287         if (softc->flags & ADA_FLAG_CAN_NCQ_TRIM)
1288                 softc->delete_method = ADA_DELETE_NCQ_DSM_TRIM;
1289         else if (softc->flags & ADA_FLAG_CAN_TRIM)
1290                 softc->delete_method = ADA_DELETE_DSM_TRIM;
1291         else if ((softc->flags & ADA_FLAG_CAN_CFA) && !(softc->flags & ADA_FLAG_CAN_48BIT))
1292                 softc->delete_method = ADA_DELETE_CFA_ERASE;
1293         else
1294                 softc->delete_method = ADA_DELETE_NONE;
1295 }
1296
1297 static void
1298 adaasync(void *callback_arg, u_int32_t code,
1299         struct cam_path *path, void *arg)
1300 {
1301         struct ccb_getdev cgd;
1302         struct cam_periph *periph;
1303         struct ada_softc *softc;
1304
1305         periph = (struct cam_periph *)callback_arg;
1306         switch (code) {
1307         case AC_FOUND_DEVICE:
1308         {
1309                 struct ccb_getdev *cgd;
1310                 cam_status status;
1311
1312                 cgd = (struct ccb_getdev *)arg;
1313                 if (cgd == NULL)
1314                         break;
1315
1316                 if (cgd->protocol != PROTO_ATA)
1317                         break;
1318
1319                 /*
1320                  * Allocate a peripheral instance for
1321                  * this device and start the probe
1322                  * process.
1323                  */
1324                 status = cam_periph_alloc(adaregister, adaoninvalidate,
1325                                           adacleanup, adastart,
1326                                           "ada", CAM_PERIPH_BIO,
1327                                           path, adaasync,
1328                                           AC_FOUND_DEVICE, cgd);
1329
1330                 if (status != CAM_REQ_CMP
1331                  && status != CAM_REQ_INPROG)
1332                         printf("adaasync: Unable to attach to new device "
1333                                 "due to status 0x%x\n", status);
1334                 break;
1335         }
1336         case AC_GETDEV_CHANGED:
1337         {
1338                 softc = (struct ada_softc *)periph->softc;
1339                 memset(&cgd, 0, sizeof(cgd));
1340                 xpt_setup_ccb(&cgd.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
1341                 cgd.ccb_h.func_code = XPT_GDEV_TYPE;
1342                 xpt_action((union ccb *)&cgd);
1343
1344                 /*
1345                  * Update our information based on the new Identify data.
1346                  */
1347                 adasetflags(softc, &cgd);
1348                 adasetgeom(softc, &cgd);
1349                 disk_resize(softc->disk, M_NOWAIT);
1350
1351                 cam_periph_async(periph, code, path, arg);
1352                 break;
1353         }
1354         case AC_ADVINFO_CHANGED:
1355         {
1356                 uintptr_t buftype;
1357
1358                 buftype = (uintptr_t)arg;
1359                 if (buftype == CDAI_TYPE_PHYS_PATH) {
1360                         struct ada_softc *softc;
1361
1362                         softc = periph->softc;
1363                         disk_attr_changed(softc->disk, "GEOM::physpath",
1364                                           M_NOWAIT);
1365                 }
1366                 break;
1367         }
1368         case AC_SENT_BDR:
1369         case AC_BUS_RESET:
1370         {
1371                 softc = (struct ada_softc *)periph->softc;
1372                 cam_periph_async(periph, code, path, arg);
1373                 if (softc->state != ADA_STATE_NORMAL)
1374                         break;
1375                 memset(&cgd, 0, sizeof(cgd));
1376                 xpt_setup_ccb(&cgd.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
1377                 cgd.ccb_h.func_code = XPT_GDEV_TYPE;
1378                 xpt_action((union ccb *)&cgd);
1379                 if (ADA_RA >= 0 && softc->flags & ADA_FLAG_CAN_RAHEAD)
1380                         softc->state = ADA_STATE_RAHEAD;
1381                 else if (ADA_WC >= 0 && softc->flags & ADA_FLAG_CAN_WCACHE)
1382                         softc->state = ADA_STATE_WCACHE;
1383                 else if ((softc->flags & ADA_FLAG_CAN_LOG)
1384                       && (softc->zone_mode != ADA_ZONE_NONE))
1385                         softc->state = ADA_STATE_LOGDIR;
1386                 else
1387                     break;
1388                 if (cam_periph_acquire(periph) != 0)
1389                         softc->state = ADA_STATE_NORMAL;
1390                 else
1391                         xpt_schedule(periph, CAM_PRIORITY_DEV);
1392         }
1393         default:
1394                 cam_periph_async(periph, code, path, arg);
1395                 break;
1396         }
1397 }
1398
1399 static int
1400 adazonemodesysctl(SYSCTL_HANDLER_ARGS)
1401 {
1402         char tmpbuf[40];
1403         struct ada_softc *softc;
1404         int error;
1405
1406         softc = (struct ada_softc *)arg1;
1407
1408         switch (softc->zone_mode) {
1409         case ADA_ZONE_DRIVE_MANAGED:
1410                 snprintf(tmpbuf, sizeof(tmpbuf), "Drive Managed");
1411                 break;
1412         case ADA_ZONE_HOST_AWARE:
1413                 snprintf(tmpbuf, sizeof(tmpbuf), "Host Aware");
1414                 break;
1415         case ADA_ZONE_HOST_MANAGED:
1416                 snprintf(tmpbuf, sizeof(tmpbuf), "Host Managed");
1417                 break;
1418         case ADA_ZONE_NONE:
1419         default:
1420                 snprintf(tmpbuf, sizeof(tmpbuf), "Not Zoned");
1421                 break;
1422         }
1423
1424         error = sysctl_handle_string(oidp, tmpbuf, sizeof(tmpbuf), req);
1425
1426         return (error);
1427 }
1428
1429 static int
1430 adazonesupsysctl(SYSCTL_HANDLER_ARGS)
1431 {
1432         char tmpbuf[180];
1433         struct ada_softc *softc;
1434         struct sbuf sb;
1435         int error, first;
1436         unsigned int i;
1437
1438         softc = (struct ada_softc *)arg1;
1439
1440         error = 0;
1441         first = 1;
1442         sbuf_new(&sb, tmpbuf, sizeof(tmpbuf), 0);
1443
1444         for (i = 0; i < sizeof(ada_zone_desc_table) /
1445              sizeof(ada_zone_desc_table[0]); i++) {
1446                 if (softc->zone_flags & ada_zone_desc_table[i].value) {
1447                         if (first == 0)
1448                                 sbuf_printf(&sb, ", ");
1449                         else
1450                                 first = 0;
1451                         sbuf_cat(&sb, ada_zone_desc_table[i].desc);
1452                 }
1453         }
1454
1455         if (first == 1)
1456                 sbuf_printf(&sb, "None");
1457
1458         sbuf_finish(&sb);
1459
1460         error = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
1461
1462         return (error);
1463 }
1464
1465 static void
1466 adasysctlinit(void *context, int pending)
1467 {
1468         struct cam_periph *periph;
1469         struct ada_softc *softc;
1470         char tmpstr[32], tmpstr2[16];
1471
1472         periph = (struct cam_periph *)context;
1473
1474         /* periph was held for us when this task was enqueued */
1475         if ((periph->flags & CAM_PERIPH_INVALID) != 0) {
1476                 cam_periph_release(periph);
1477                 return;
1478         }
1479
1480         softc = (struct ada_softc *)periph->softc;
1481         snprintf(tmpstr, sizeof(tmpstr), "CAM ADA unit %d",periph->unit_number);
1482         snprintf(tmpstr2, sizeof(tmpstr2), "%d", periph->unit_number);
1483
1484         sysctl_ctx_init(&softc->sysctl_ctx);
1485         softc->flags |= ADA_FLAG_SCTX_INIT;
1486         softc->sysctl_tree = SYSCTL_ADD_NODE_WITH_LABEL(&softc->sysctl_ctx,
1487                 SYSCTL_STATIC_CHILDREN(_kern_cam_ada), OID_AUTO, tmpstr2,
1488                 CTLFLAG_RD | CTLFLAG_MPSAFE, 0, tmpstr, "device_index");
1489         if (softc->sysctl_tree == NULL) {
1490                 printf("adasysctlinit: unable to allocate sysctl tree\n");
1491                 cam_periph_release(periph);
1492                 return;
1493         }
1494
1495         SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1496                 OID_AUTO, "delete_method",
1497                 CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
1498                 softc, 0, adadeletemethodsysctl, "A",
1499                 "BIO_DELETE execution method");
1500         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
1501                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
1502                 "trim_count", CTLFLAG_RD, &softc->trim_count,
1503                 "Total number of dsm commands sent");
1504         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
1505                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
1506                 "trim_ranges", CTLFLAG_RD, &softc->trim_ranges,
1507                 "Total number of ranges in dsm commands");
1508         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
1509                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
1510                 "trim_lbas", CTLFLAG_RD, &softc->trim_lbas,
1511                 "Total lbas in the dsm commands sent");
1512         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1513                 OID_AUTO, "read_ahead", CTLFLAG_RW | CTLFLAG_MPSAFE,
1514                 &softc->read_ahead, 0, "Enable disk read ahead.");
1515         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1516                 OID_AUTO, "write_cache", CTLFLAG_RW | CTLFLAG_MPSAFE,
1517                 &softc->write_cache, 0, "Enable disk write cache.");
1518         SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1519                 OID_AUTO, "zone_mode",
1520                 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_NEEDGIANT,
1521                 softc, 0, adazonemodesysctl, "A",
1522                 "Zone Mode");
1523         SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1524                 OID_AUTO, "zone_support",
1525                 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_NEEDGIANT,
1526                 softc, 0, adazonesupsysctl, "A",
1527                 "Zone Support");
1528         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
1529                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
1530                 "optimal_seq_zones", CTLFLAG_RD, &softc->optimal_seq_zones,
1531                 "Optimal Number of Open Sequential Write Preferred Zones");
1532         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
1533                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
1534                 "optimal_nonseq_zones", CTLFLAG_RD,
1535                 &softc->optimal_nonseq_zones,
1536                 "Optimal Number of Non-Sequentially Written Sequential Write "
1537                 "Preferred Zones");
1538         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
1539                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
1540                 "max_seq_zones", CTLFLAG_RD, &softc->max_seq_zones,
1541                 "Maximum Number of Open Sequential Write Required Zones");
1542         SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1543             OID_AUTO, "flags", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE,
1544             softc, 0, adaflagssysctl, "A",
1545             "Flags for drive");
1546         SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1547             OID_AUTO, "unmapped_io", CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MPSAFE,
1548             &softc->flags, (u_int)ADA_FLAG_UNMAPPEDIO, adabitsysctl, "I",
1549             "Unmapped I/O support *DEPRECATED* gone in FreeBSD 14");
1550         SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1551             OID_AUTO, "rotating", CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MPSAFE,
1552             &softc->flags, (u_int)ADA_FLAG_ROTATING, adabitsysctl, "I",
1553             "Rotating media *DEPRECATED* gone in FreeBSD 14");
1554
1555 #ifdef CAM_TEST_FAILURE
1556         /*
1557          * Add a 'door bell' sysctl which allows one to set it from userland
1558          * and cause something bad to happen.  For the moment, we only allow
1559          * whacking the next read or write.
1560          */
1561         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1562                 OID_AUTO, "force_read_error", CTLFLAG_RW | CTLFLAG_MPSAFE,
1563                 &softc->force_read_error, 0,
1564                 "Force a read error for the next N reads.");
1565         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1566                 OID_AUTO, "force_write_error", CTLFLAG_RW | CTLFLAG_MPSAFE,
1567                 &softc->force_write_error, 0,
1568                 "Force a write error for the next N writes.");
1569         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1570                 OID_AUTO, "periodic_read_error", CTLFLAG_RW | CTLFLAG_MPSAFE,
1571                 &softc->periodic_read_error, 0,
1572                 "Force a read error every N reads (don't set too low).");
1573         SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1574                 OID_AUTO, "invalidate", CTLTYPE_U64 | CTLFLAG_RW | CTLFLAG_MPSAFE,
1575                 periph, 0, cam_periph_invalidate_sysctl, "I",
1576                 "Write 1 to invalidate the drive immediately");
1577 #endif
1578
1579 #ifdef CAM_IO_STATS
1580         softc->sysctl_stats_tree = SYSCTL_ADD_NODE(&softc->sysctl_stats_ctx,
1581                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO, "stats",
1582                 CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "Statistics");
1583         SYSCTL_ADD_INT(&softc->sysctl_stats_ctx,
1584                 SYSCTL_CHILDREN(softc->sysctl_stats_tree),
1585                 OID_AUTO, "timeouts", CTLFLAG_RD | CTLFLAG_MPSAFE,
1586                 &softc->timeouts, 0,
1587                 "Device timeouts reported by the SIM");
1588         SYSCTL_ADD_INT(&softc->sysctl_stats_ctx,
1589                 SYSCTL_CHILDREN(softc->sysctl_stats_tree),
1590                 OID_AUTO, "errors", CTLFLAG_RD | CTLFLAG_MPSAFE,
1591                 &softc->errors, 0,
1592                 "Transport errors reported by the SIM.");
1593         SYSCTL_ADD_INT(&softc->sysctl_stats_ctx,
1594                 SYSCTL_CHILDREN(softc->sysctl_stats_tree),
1595                 OID_AUTO, "pack_invalidations", CTLFLAG_RD | CTLFLAG_MPSAFE,
1596                 &softc->invalidations, 0,
1597                 "Device pack invalidations.");
1598 #endif
1599
1600         cam_iosched_sysctl_init(softc->cam_iosched, &softc->sysctl_ctx,
1601             softc->sysctl_tree);
1602
1603         cam_periph_release(periph);
1604 }
1605
1606 static int
1607 adagetattr(struct bio *bp)
1608 {
1609         int ret;
1610         struct cam_periph *periph;
1611
1612         if (g_handleattr_int(bp, "GEOM::canspeedup", ada_enable_biospeedup))
1613                 return (EJUSTRETURN);
1614
1615         periph = (struct cam_periph *)bp->bio_disk->d_drv1;
1616         cam_periph_lock(periph);
1617         ret = xpt_getattr(bp->bio_data, bp->bio_length, bp->bio_attribute,
1618             periph->path);
1619         cam_periph_unlock(periph);
1620         if (ret == 0)
1621                 bp->bio_completed = bp->bio_length;
1622         return ret;
1623 }
1624
1625 static int
1626 adadeletemethodsysctl(SYSCTL_HANDLER_ARGS)
1627 {
1628         char buf[16];
1629         const char *p;
1630         struct ada_softc *softc;
1631         int i, error, value, methods;
1632
1633         softc = (struct ada_softc *)arg1;
1634
1635         value = softc->delete_method;
1636         if (value < 0 || value > ADA_DELETE_MAX)
1637                 p = "UNKNOWN";
1638         else
1639                 p = ada_delete_method_names[value];
1640         strncpy(buf, p, sizeof(buf));
1641         error = sysctl_handle_string(oidp, buf, sizeof(buf), req);
1642         if (error != 0 || req->newptr == NULL)
1643                 return (error);
1644         methods = 1 << ADA_DELETE_DISABLE;
1645         if ((softc->flags & ADA_FLAG_CAN_CFA) &&
1646             !(softc->flags & ADA_FLAG_CAN_48BIT))
1647                 methods |= 1 << ADA_DELETE_CFA_ERASE;
1648         if (softc->flags & ADA_FLAG_CAN_TRIM)
1649                 methods |= 1 << ADA_DELETE_DSM_TRIM;
1650         if (softc->flags & ADA_FLAG_CAN_NCQ_TRIM)
1651                 methods |= 1 << ADA_DELETE_NCQ_DSM_TRIM;
1652         for (i = 0; i <= ADA_DELETE_MAX; i++) {
1653                 if (!(methods & (1 << i)) ||
1654                     strcmp(buf, ada_delete_method_names[i]) != 0)
1655                         continue;
1656                 softc->delete_method = i;
1657                 return (0);
1658         }
1659         return (EINVAL);
1660 }
1661
1662 static int
1663 adabitsysctl(SYSCTL_HANDLER_ARGS)
1664 {
1665         u_int *flags = arg1;
1666         u_int test = arg2;
1667         int tmpout, error;
1668
1669         tmpout = !!(*flags & test);
1670         error = SYSCTL_OUT(req, &tmpout, sizeof(tmpout));
1671         if (error || !req->newptr)
1672                 return (error);
1673
1674         return (EPERM);
1675 }
1676
1677 static int
1678 adaflagssysctl(SYSCTL_HANDLER_ARGS)
1679 {
1680         struct sbuf sbuf;
1681         struct ada_softc *softc = arg1;
1682         int error;
1683
1684         sbuf_new_for_sysctl(&sbuf, NULL, 0, req);
1685         if (softc->flags != 0)
1686                 sbuf_printf(&sbuf, "0x%b", (unsigned)softc->flags, ADA_FLAG_STRING);
1687         else
1688                 sbuf_printf(&sbuf, "0");
1689         error = sbuf_finish(&sbuf);
1690         sbuf_delete(&sbuf);
1691
1692         return (error);
1693 }
1694
1695 static void
1696 adasetflags(struct ada_softc *softc, struct ccb_getdev *cgd)
1697 {
1698         if ((cgd->ident_data.capabilities1 & ATA_SUPPORT_DMA) &&
1699             (cgd->inq_flags & SID_DMA))
1700                 softc->flags |= ADA_FLAG_CAN_DMA;
1701         else
1702                 softc->flags &= ~ADA_FLAG_CAN_DMA;
1703
1704         if (cgd->ident_data.support.command2 & ATA_SUPPORT_ADDRESS48) {
1705                 softc->flags |= ADA_FLAG_CAN_48BIT;
1706                 if (cgd->inq_flags & SID_DMA48)
1707                         softc->flags |= ADA_FLAG_CAN_DMA48;
1708                 else
1709                         softc->flags &= ~ADA_FLAG_CAN_DMA48;
1710         } else
1711                 softc->flags &= ~(ADA_FLAG_CAN_48BIT | ADA_FLAG_CAN_DMA48);
1712
1713         if (cgd->ident_data.support.command2 & ATA_SUPPORT_FLUSHCACHE)
1714                 softc->flags |= ADA_FLAG_CAN_FLUSHCACHE;
1715         else
1716                 softc->flags &= ~ADA_FLAG_CAN_FLUSHCACHE;
1717
1718         if (cgd->ident_data.support.command1 & ATA_SUPPORT_POWERMGT)
1719                 softc->flags |= ADA_FLAG_CAN_POWERMGT;
1720         else
1721                 softc->flags &= ~ADA_FLAG_CAN_POWERMGT;
1722
1723         if ((cgd->ident_data.satacapabilities & ATA_SUPPORT_NCQ) &&
1724             (cgd->inq_flags & SID_DMA) && (cgd->inq_flags & SID_CmdQue))
1725                 softc->flags |= ADA_FLAG_CAN_NCQ;
1726         else
1727                 softc->flags &= ~ADA_FLAG_CAN_NCQ;
1728
1729         if ((cgd->ident_data.support_dsm & ATA_SUPPORT_DSM_TRIM) &&
1730             (cgd->inq_flags & SID_DMA) &&
1731             (softc->quirks & ADA_Q_NO_TRIM) == 0) {
1732                 softc->flags |= ADA_FLAG_CAN_TRIM;
1733                 softc->trim_max_ranges = TRIM_MAX_RANGES;
1734                 if (cgd->ident_data.max_dsm_blocks != 0) {
1735                         softc->trim_max_ranges =
1736                             min(cgd->ident_data.max_dsm_blocks *
1737                                 ATA_DSM_BLK_RANGES, softc->trim_max_ranges);
1738                 }
1739                 /*
1740                  * If we can do RCVSND_FPDMA_QUEUED commands, we may be able
1741                  * to do NCQ trims, if we support trims at all. We also need
1742                  * support from the SIM to do things properly. Perhaps we
1743                  * should look at log 13 dword 0 bit 0 and dword 1 bit 0 are
1744                  * set too...
1745                  */
1746                 if ((softc->quirks & ADA_Q_NCQ_TRIM_BROKEN) == 0 &&
1747                     (softc->flags & ADA_FLAG_PIM_ATA_EXT) != 0 &&
1748                     (cgd->ident_data.satacapabilities2 &
1749                      ATA_SUPPORT_RCVSND_FPDMA_QUEUED) != 0 &&
1750                     (softc->flags & ADA_FLAG_CAN_TRIM) != 0)
1751                         softc->flags |= ADA_FLAG_CAN_NCQ_TRIM;
1752                 else
1753                         softc->flags &= ~ADA_FLAG_CAN_NCQ_TRIM;
1754         } else
1755                 softc->flags &= ~(ADA_FLAG_CAN_TRIM | ADA_FLAG_CAN_NCQ_TRIM);
1756
1757         if (cgd->ident_data.support.command2 & ATA_SUPPORT_CFA)
1758                 softc->flags |= ADA_FLAG_CAN_CFA;
1759         else
1760                 softc->flags &= ~ADA_FLAG_CAN_CFA;
1761
1762         /*
1763          * Now that we've set the appropriate flags, setup the delete
1764          * method.
1765          */
1766         adasetdeletemethod(softc);
1767
1768         if ((cgd->ident_data.support.extension & ATA_SUPPORT_GENLOG)
1769          && ((softc->quirks & ADA_Q_LOG_BROKEN) == 0))
1770                 softc->flags |= ADA_FLAG_CAN_LOG;
1771         else
1772                 softc->flags &= ~ADA_FLAG_CAN_LOG;
1773
1774         if ((cgd->ident_data.support3 & ATA_SUPPORT_ZONE_MASK) ==
1775              ATA_SUPPORT_ZONE_HOST_AWARE)
1776                 softc->zone_mode = ADA_ZONE_HOST_AWARE;
1777         else if (((cgd->ident_data.support3 & ATA_SUPPORT_ZONE_MASK) ==
1778                    ATA_SUPPORT_ZONE_DEV_MANAGED)
1779               || (softc->quirks & ADA_Q_SMR_DM))
1780                 softc->zone_mode = ADA_ZONE_DRIVE_MANAGED;
1781         else
1782                 softc->zone_mode = ADA_ZONE_NONE;
1783
1784         if (cgd->ident_data.support.command1 & ATA_SUPPORT_LOOKAHEAD)
1785                 softc->flags |= ADA_FLAG_CAN_RAHEAD;
1786         else
1787                 softc->flags &= ~ADA_FLAG_CAN_RAHEAD;
1788
1789         if (cgd->ident_data.support.command1 & ATA_SUPPORT_WRITECACHE)
1790                 softc->flags |= ADA_FLAG_CAN_WCACHE;
1791         else
1792                 softc->flags &= ~ADA_FLAG_CAN_WCACHE;
1793 }
1794
1795 static cam_status
1796 adaregister(struct cam_periph *periph, void *arg)
1797 {
1798         struct ada_softc *softc;
1799         struct ccb_getdev *cgd;
1800         struct disk_params *dp;
1801         struct sbuf sb;
1802         char   *announce_buf;
1803         caddr_t match;
1804         int quirks;
1805
1806         cgd = (struct ccb_getdev *)arg;
1807         if (cgd == NULL) {
1808                 printf("adaregister: no getdev CCB, can't register device\n");
1809                 return(CAM_REQ_CMP_ERR);
1810         }
1811
1812         softc = (struct ada_softc *)malloc(sizeof(*softc), M_DEVBUF,
1813             M_NOWAIT|M_ZERO);
1814
1815         if (softc == NULL) {
1816                 printf("adaregister: Unable to probe new device. "
1817                     "Unable to allocate softc\n");
1818                 return(CAM_REQ_CMP_ERR);
1819         }
1820
1821         announce_buf = softc->announce_temp;
1822         bzero(announce_buf, ADA_ANNOUNCETMP_SZ);
1823
1824         if (cam_iosched_init(&softc->cam_iosched, periph) != 0) {
1825                 printf("adaregister: Unable to probe new device. "
1826                        "Unable to allocate iosched memory\n");
1827                 free(softc, M_DEVBUF);
1828                 return(CAM_REQ_CMP_ERR);
1829         }
1830
1831         periph->softc = softc;
1832         xpt_path_inq(&softc->cpi, periph->path);
1833
1834         /*
1835          * See if this device has any quirks.
1836          */
1837         match = cam_quirkmatch((caddr_t)&cgd->ident_data,
1838                                (caddr_t)ada_quirk_table,
1839                                nitems(ada_quirk_table),
1840                                sizeof(*ada_quirk_table), ata_identify_match);
1841         if (match != NULL)
1842                 softc->quirks = ((struct ada_quirk_entry *)match)->quirks;
1843         else
1844                 softc->quirks = ADA_Q_NONE;
1845
1846         TASK_INIT(&softc->sysctl_task, 0, adasysctlinit, periph);
1847
1848         /*
1849          * Register this media as a disk
1850          */
1851         (void)cam_periph_hold(periph, PRIBIO);
1852         cam_periph_unlock(periph);
1853         snprintf(announce_buf, ADA_ANNOUNCETMP_SZ,
1854             "kern.cam.ada.%d.quirks", periph->unit_number);
1855         quirks = softc->quirks;
1856         TUNABLE_INT_FETCH(announce_buf, &quirks);
1857         softc->quirks = quirks;
1858         softc->read_ahead = -1;
1859         snprintf(announce_buf, ADA_ANNOUNCETMP_SZ,
1860             "kern.cam.ada.%d.read_ahead", periph->unit_number);
1861         TUNABLE_INT_FETCH(announce_buf, &softc->read_ahead);
1862         softc->write_cache = -1;
1863         snprintf(announce_buf, ADA_ANNOUNCETMP_SZ,
1864             "kern.cam.ada.%d.write_cache", periph->unit_number);
1865         TUNABLE_INT_FETCH(announce_buf, &softc->write_cache);
1866
1867         /*
1868          * Let XPT know we can use UMA-allocated CCBs.
1869          */
1870         if (ada_enable_uma_ccbs) {
1871                 KASSERT(ada_ccb_zone != NULL,
1872                     ("%s: NULL ada_ccb_zone", __func__));
1873                 periph->ccb_zone = ada_ccb_zone;
1874         }
1875
1876         /*
1877          * Set support flags based on the Identify data and quirks.
1878          */
1879         adasetflags(softc, cgd);
1880         if (softc->cpi.hba_misc & PIM_ATA_EXT)
1881                 softc->flags |= ADA_FLAG_PIM_ATA_EXT;
1882
1883         /* Disable queue sorting for non-rotational media by default. */
1884         if (cgd->ident_data.media_rotation_rate == ATA_RATE_NON_ROTATING) {
1885                 softc->flags &= ~ADA_FLAG_ROTATING;
1886         } else {
1887                 softc->flags |= ADA_FLAG_ROTATING;
1888         }
1889         cam_iosched_set_sort_queue(softc->cam_iosched,
1890             (softc->flags & ADA_FLAG_ROTATING) ? -1 : 0);
1891         softc->disk = disk_alloc();
1892         adasetgeom(softc, cgd);
1893         softc->disk->d_devstat = devstat_new_entry(periph->periph_name,
1894                           periph->unit_number, softc->params.secsize,
1895                           DEVSTAT_ALL_SUPPORTED,
1896                           DEVSTAT_TYPE_DIRECT |
1897                           XPORT_DEVSTAT_TYPE(softc->cpi.transport),
1898                           DEVSTAT_PRIORITY_DISK);
1899         softc->disk->d_open = adaopen;
1900         softc->disk->d_close = adaclose;
1901         softc->disk->d_strategy = adastrategy;
1902         softc->disk->d_getattr = adagetattr;
1903         if (cam_sim_pollable(periph->sim))
1904                 softc->disk->d_dump = adadump;
1905         softc->disk->d_gone = adadiskgonecb;
1906         softc->disk->d_name = "ada";
1907         softc->disk->d_drv1 = periph;
1908         softc->disk->d_unit = periph->unit_number;
1909
1910         /*
1911          * Acquire a reference to the periph before we register with GEOM.
1912          * We'll release this reference once GEOM calls us back (via
1913          * adadiskgonecb()) telling us that our provider has been freed.
1914          */
1915         if (cam_periph_acquire(periph) != 0) {
1916                 xpt_print(periph->path, "%s: lost periph during "
1917                           "registration!\n", __func__);
1918                 cam_periph_lock(periph);
1919                 return (CAM_REQ_CMP_ERR);
1920         }
1921         disk_create(softc->disk, DISK_VERSION);
1922         cam_periph_lock(periph);
1923
1924         dp = &softc->params;
1925         snprintf(announce_buf, ADA_ANNOUNCETMP_SZ,
1926             "%juMB (%ju %u byte sectors)",
1927             ((uintmax_t)dp->secsize * dp->sectors) / (1024 * 1024),
1928             (uintmax_t)dp->sectors, dp->secsize);
1929
1930         sbuf_new(&sb, softc->announce_buffer, ADA_ANNOUNCE_SZ, SBUF_FIXEDLEN);
1931         xpt_announce_periph_sbuf(periph, &sb, announce_buf);
1932         xpt_announce_quirks_sbuf(periph, &sb, softc->quirks, ADA_Q_BIT_STRING);
1933         sbuf_finish(&sb);
1934         sbuf_putbuf(&sb);
1935
1936         /*
1937          * Create our sysctl variables, now that we know
1938          * we have successfully attached.
1939          */
1940         if (cam_periph_acquire(periph) == 0)
1941                 taskqueue_enqueue(taskqueue_thread, &softc->sysctl_task);
1942
1943         /*
1944          * Add async callbacks for bus reset and
1945          * bus device reset calls.  I don't bother
1946          * checking if this fails as, in most cases,
1947          * the system will function just fine without
1948          * them and the only alternative would be to
1949          * not attach the device on failure.
1950          */
1951         xpt_register_async(AC_SENT_BDR | AC_BUS_RESET | AC_LOST_DEVICE |
1952             AC_GETDEV_CHANGED | AC_ADVINFO_CHANGED,
1953             adaasync, periph, periph->path);
1954
1955         /*
1956          * Schedule a periodic event to occasionally send an
1957          * ordered tag to a device.
1958          */
1959         callout_init_mtx(&softc->sendordered_c, cam_periph_mtx(periph), 0);
1960         callout_reset(&softc->sendordered_c,
1961             (ada_default_timeout * hz) / ADA_ORDEREDTAG_INTERVAL,
1962             adasendorderedtag, softc);
1963
1964         if (ADA_RA >= 0 && softc->flags & ADA_FLAG_CAN_RAHEAD) {
1965                 softc->state = ADA_STATE_RAHEAD;
1966         } else if (ADA_WC >= 0 && softc->flags & ADA_FLAG_CAN_WCACHE) {
1967                 softc->state = ADA_STATE_WCACHE;
1968         } else if ((softc->flags & ADA_FLAG_CAN_LOG)
1969                 && (softc->zone_mode != ADA_ZONE_NONE)) {
1970                 softc->state = ADA_STATE_LOGDIR;
1971         } else {
1972                 /*
1973                  * Nothing to probe, so we can just transition to the
1974                  * normal state.
1975                  */
1976                 adaprobedone(periph, NULL);
1977                 return(CAM_REQ_CMP);
1978         }
1979
1980         xpt_schedule(periph, CAM_PRIORITY_DEV);
1981
1982         return(CAM_REQ_CMP);
1983 }
1984
1985 static int
1986 ada_dsmtrim_req_create(struct ada_softc *softc, struct bio *bp, struct trim_request *req)
1987 {
1988         uint64_t lastlba = (uint64_t)-1, lbas = 0;
1989         int c, lastcount = 0, off, ranges = 0;
1990
1991         bzero(req, sizeof(*req));
1992         TAILQ_INIT(&req->bps);
1993         do {
1994                 uint64_t lba = bp->bio_pblkno;
1995                 int count = bp->bio_bcount / softc->params.secsize;
1996
1997                 /* Try to extend the previous range. */
1998                 if (lba == lastlba) {
1999                         c = min(count, ATA_DSM_RANGE_MAX - lastcount);
2000                         lastcount += c;
2001                         off = (ranges - 1) * ATA_DSM_RANGE_SIZE;
2002                         req->data[off + 6] = lastcount & 0xff;
2003                         req->data[off + 7] =
2004                                 (lastcount >> 8) & 0xff;
2005                         count -= c;
2006                         lba += c;
2007                         lbas += c;
2008                 }
2009
2010                 while (count > 0) {
2011                         c = min(count, ATA_DSM_RANGE_MAX);
2012                         off = ranges * ATA_DSM_RANGE_SIZE;
2013                         req->data[off + 0] = lba & 0xff;
2014                         req->data[off + 1] = (lba >> 8) & 0xff;
2015                         req->data[off + 2] = (lba >> 16) & 0xff;
2016                         req->data[off + 3] = (lba >> 24) & 0xff;
2017                         req->data[off + 4] = (lba >> 32) & 0xff;
2018                         req->data[off + 5] = (lba >> 40) & 0xff;
2019                         req->data[off + 6] = c & 0xff;
2020                         req->data[off + 7] = (c >> 8) & 0xff;
2021                         lba += c;
2022                         lbas += c;
2023                         count -= c;
2024                         lastcount = c;
2025                         ranges++;
2026                         /*
2027                          * Its the caller's responsibility to ensure the
2028                          * request will fit so we don't need to check for
2029                          * overrun here
2030                          */
2031                 }
2032                 lastlba = lba;
2033                 TAILQ_INSERT_TAIL(&req->bps, bp, bio_queue);
2034
2035                 bp = cam_iosched_next_trim(softc->cam_iosched);
2036                 if (bp == NULL)
2037                         break;
2038                 if (bp->bio_bcount / softc->params.secsize >
2039                     (softc->trim_max_ranges - ranges) * ATA_DSM_RANGE_MAX) {
2040                         cam_iosched_put_back_trim(softc->cam_iosched, bp);
2041                         break;
2042                 }
2043         } while (1);
2044         softc->trim_count++;
2045         softc->trim_ranges += ranges;
2046         softc->trim_lbas += lbas;
2047
2048         return (ranges);
2049 }
2050
2051 static void
2052 ada_dsmtrim(struct ada_softc *softc, struct bio *bp, struct ccb_ataio *ataio)
2053 {
2054         struct trim_request *req = &softc->trim_req;
2055         int ranges;
2056
2057         ranges = ada_dsmtrim_req_create(softc, bp, req);
2058         cam_fill_ataio(ataio,
2059             ada_retry_count,
2060             adadone,
2061             CAM_DIR_OUT,
2062             0,
2063             req->data,
2064             howmany(ranges, ATA_DSM_BLK_RANGES) * ATA_DSM_BLK_SIZE,
2065             ada_default_timeout * 1000);
2066         ata_48bit_cmd(ataio, ATA_DATA_SET_MANAGEMENT,
2067             ATA_DSM_TRIM, 0, howmany(ranges, ATA_DSM_BLK_RANGES));
2068 }
2069
2070 static void
2071 ada_ncq_dsmtrim(struct ada_softc *softc, struct bio *bp, struct ccb_ataio *ataio)
2072 {
2073         struct trim_request *req = &softc->trim_req;
2074         int ranges;
2075
2076         ranges = ada_dsmtrim_req_create(softc, bp, req);
2077         cam_fill_ataio(ataio,
2078             ada_retry_count,
2079             adadone,
2080             CAM_DIR_OUT,
2081             0,
2082             req->data,
2083             howmany(ranges, ATA_DSM_BLK_RANGES) * ATA_DSM_BLK_SIZE,
2084             ada_default_timeout * 1000);
2085         ata_ncq_cmd(ataio,
2086             ATA_SEND_FPDMA_QUEUED,
2087             0,
2088             howmany(ranges, ATA_DSM_BLK_RANGES));
2089         ataio->cmd.sector_count_exp = ATA_SFPDMA_DSM;
2090         ataio->ata_flags |= ATA_FLAG_AUX;
2091         ataio->aux = 1;
2092 }
2093
2094 static void
2095 ada_cfaerase(struct ada_softc *softc, struct bio *bp, struct ccb_ataio *ataio)
2096 {
2097         struct trim_request *req = &softc->trim_req;
2098         uint64_t lba = bp->bio_pblkno;
2099         uint16_t count = bp->bio_bcount / softc->params.secsize;
2100
2101         bzero(req, sizeof(*req));
2102         TAILQ_INIT(&req->bps);
2103         TAILQ_INSERT_TAIL(&req->bps, bp, bio_queue);
2104
2105         cam_fill_ataio(ataio,
2106             ada_retry_count,
2107             adadone,
2108             CAM_DIR_NONE,
2109             0,
2110             NULL,
2111             0,
2112             ada_default_timeout*1000);
2113
2114         if (count >= 256)
2115                 count = 0;
2116         ata_28bit_cmd(ataio, ATA_CFA_ERASE, 0, lba, count);
2117 }
2118
2119 static int
2120 ada_zone_bio_to_ata(int disk_zone_cmd)
2121 {
2122         switch (disk_zone_cmd) {
2123         case DISK_ZONE_OPEN:
2124                 return ATA_ZM_OPEN_ZONE;
2125         case DISK_ZONE_CLOSE:
2126                 return ATA_ZM_CLOSE_ZONE;
2127         case DISK_ZONE_FINISH:
2128                 return ATA_ZM_FINISH_ZONE;
2129         case DISK_ZONE_RWP:
2130                 return ATA_ZM_RWP;
2131         }
2132
2133         return -1;
2134 }
2135
2136 static int
2137 ada_zone_cmd(struct cam_periph *periph, union ccb *ccb, struct bio *bp,
2138              int *queue_ccb)
2139 {
2140         struct ada_softc *softc;
2141         int error;
2142
2143         error = 0;
2144
2145         if (bp->bio_cmd != BIO_ZONE) {
2146                 error = EINVAL;
2147                 goto bailout;
2148         }
2149
2150         softc = periph->softc;
2151
2152         switch (bp->bio_zone.zone_cmd) {
2153         case DISK_ZONE_OPEN:
2154         case DISK_ZONE_CLOSE:
2155         case DISK_ZONE_FINISH:
2156         case DISK_ZONE_RWP: {
2157                 int zone_flags;
2158                 int zone_sa;
2159                 uint64_t lba;
2160
2161                 zone_sa = ada_zone_bio_to_ata(bp->bio_zone.zone_cmd);
2162                 if (zone_sa == -1) {
2163                         xpt_print(periph->path, "Cannot translate zone "
2164                             "cmd %#x to ATA\n", bp->bio_zone.zone_cmd);
2165                         error = EINVAL;
2166                         goto bailout;
2167                 }
2168
2169                 zone_flags = 0;
2170                 lba = bp->bio_zone.zone_params.rwp.id;
2171
2172                 if (bp->bio_zone.zone_params.rwp.flags &
2173                     DISK_ZONE_RWP_FLAG_ALL)
2174                         zone_flags |= ZBC_OUT_ALL;
2175
2176                 ata_zac_mgmt_out(&ccb->ataio,
2177                                  /*retries*/ ada_retry_count,
2178                                  /*cbfcnp*/ adadone,
2179                                  /*use_ncq*/ (softc->flags &
2180                                               ADA_FLAG_PIM_ATA_EXT) ? 1 : 0,
2181                                  /*zm_action*/ zone_sa,
2182                                  /*zone_id*/ lba,
2183                                  /*zone_flags*/ zone_flags,
2184                                  /*sector_count*/ 0,
2185                                  /*data_ptr*/ NULL,
2186                                  /*dxfer_len*/ 0,
2187                                  /*timeout*/ ada_default_timeout * 1000);
2188                 *queue_ccb = 1;
2189
2190                 break;
2191         }
2192         case DISK_ZONE_REPORT_ZONES: {
2193                 uint8_t *rz_ptr;
2194                 uint32_t num_entries, alloc_size;
2195                 struct disk_zone_report *rep;
2196
2197                 rep = &bp->bio_zone.zone_params.report;
2198
2199                 num_entries = rep->entries_allocated;
2200                 if (num_entries == 0) {
2201                         xpt_print(periph->path, "No entries allocated for "
2202                             "Report Zones request\n");
2203                         error = EINVAL;
2204                         goto bailout;
2205                 }
2206                 alloc_size = sizeof(struct scsi_report_zones_hdr) +
2207                     (sizeof(struct scsi_report_zones_desc) * num_entries);
2208                 alloc_size = min(alloc_size, softc->disk->d_maxsize);
2209                 rz_ptr = malloc(alloc_size, M_ATADA, M_NOWAIT | M_ZERO);
2210                 if (rz_ptr == NULL) {
2211                         xpt_print(periph->path, "Unable to allocate memory "
2212                            "for Report Zones request\n");
2213                         error = ENOMEM;
2214                         goto bailout;
2215                 }
2216
2217                 ata_zac_mgmt_in(&ccb->ataio,
2218                                 /*retries*/ ada_retry_count,
2219                                 /*cbcfnp*/ adadone,
2220                                 /*use_ncq*/ (softc->flags &
2221                                              ADA_FLAG_PIM_ATA_EXT) ? 1 : 0,
2222                                 /*zm_action*/ ATA_ZM_REPORT_ZONES,
2223                                 /*zone_id*/ rep->starting_id,
2224                                 /*zone_flags*/ rep->rep_options,
2225                                 /*data_ptr*/ rz_ptr,
2226                                 /*dxfer_len*/ alloc_size,
2227                                 /*timeout*/ ada_default_timeout * 1000);
2228
2229                 /*
2230                  * For BIO_ZONE, this isn't normally needed.  However, it
2231                  * is used by devstat_end_transaction_bio() to determine
2232                  * how much data was transferred.
2233                  */
2234                 /*
2235                  * XXX KDM we have a problem.  But I'm not sure how to fix
2236                  * it.  devstat uses bio_bcount - bio_resid to calculate
2237                  * the amount of data transferred.   The GEOM disk code
2238                  * uses bio_length - bio_resid to calculate the amount of
2239                  * data in bio_completed.  We have different structure
2240                  * sizes above and below the ada(4) driver.  So, if we
2241                  * use the sizes above, the amount transferred won't be
2242                  * quite accurate for devstat.  If we use different sizes
2243                  * for bio_bcount and bio_length (above and below
2244                  * respectively), then the residual needs to match one or
2245                  * the other.  Everything is calculated after the bio
2246                  * leaves the driver, so changing the values around isn't
2247                  * really an option.  For now, just set the count to the
2248                  * passed in length.  This means that the calculations
2249                  * above (e.g. bio_completed) will be correct, but the
2250                  * amount of data reported to devstat will be slightly
2251                  * under or overstated.
2252                  */
2253                 bp->bio_bcount = bp->bio_length;
2254
2255                 *queue_ccb = 1;
2256
2257                 break;
2258         }
2259         case DISK_ZONE_GET_PARAMS: {
2260                 struct disk_zone_disk_params *params;
2261
2262                 params = &bp->bio_zone.zone_params.disk_params;
2263                 bzero(params, sizeof(*params));
2264
2265                 switch (softc->zone_mode) {
2266                 case ADA_ZONE_DRIVE_MANAGED:
2267                         params->zone_mode = DISK_ZONE_MODE_DRIVE_MANAGED;
2268                         break;
2269                 case ADA_ZONE_HOST_AWARE:
2270                         params->zone_mode = DISK_ZONE_MODE_HOST_AWARE;
2271                         break;
2272                 case ADA_ZONE_HOST_MANAGED:
2273                         params->zone_mode = DISK_ZONE_MODE_HOST_MANAGED;
2274                         break;
2275                 default:
2276                 case ADA_ZONE_NONE:
2277                         params->zone_mode = DISK_ZONE_MODE_NONE;
2278                         break;
2279                 }
2280
2281                 if (softc->zone_flags & ADA_ZONE_FLAG_URSWRZ)
2282                         params->flags |= DISK_ZONE_DISK_URSWRZ;
2283
2284                 if (softc->zone_flags & ADA_ZONE_FLAG_OPT_SEQ_SET) {
2285                         params->optimal_seq_zones = softc->optimal_seq_zones;
2286                         params->flags |= DISK_ZONE_OPT_SEQ_SET;
2287                 }
2288
2289                 if (softc->zone_flags & ADA_ZONE_FLAG_OPT_NONSEQ_SET) {
2290                         params->optimal_nonseq_zones =
2291                             softc->optimal_nonseq_zones;
2292                         params->flags |= DISK_ZONE_OPT_NONSEQ_SET;
2293                 }
2294
2295                 if (softc->zone_flags & ADA_ZONE_FLAG_MAX_SEQ_SET) {
2296                         params->max_seq_zones = softc->max_seq_zones;
2297                         params->flags |= DISK_ZONE_MAX_SEQ_SET;
2298                 }
2299                 if (softc->zone_flags & ADA_ZONE_FLAG_RZ_SUP)
2300                         params->flags |= DISK_ZONE_RZ_SUP;
2301
2302                 if (softc->zone_flags & ADA_ZONE_FLAG_OPEN_SUP)
2303                         params->flags |= DISK_ZONE_OPEN_SUP;
2304
2305                 if (softc->zone_flags & ADA_ZONE_FLAG_CLOSE_SUP)
2306                         params->flags |= DISK_ZONE_CLOSE_SUP;
2307
2308                 if (softc->zone_flags & ADA_ZONE_FLAG_FINISH_SUP)
2309                         params->flags |= DISK_ZONE_FINISH_SUP;
2310
2311                 if (softc->zone_flags & ADA_ZONE_FLAG_RWP_SUP)
2312                         params->flags |= DISK_ZONE_RWP_SUP;
2313                 break;
2314         }
2315         default:
2316                 break;
2317         }
2318 bailout:
2319         return (error);
2320 }
2321
2322 static void
2323 adastart(struct cam_periph *periph, union ccb *start_ccb)
2324 {
2325         struct ada_softc *softc = (struct ada_softc *)periph->softc;
2326         struct ccb_ataio *ataio = &start_ccb->ataio;
2327
2328         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("adastart\n"));
2329
2330         switch (softc->state) {
2331         case ADA_STATE_NORMAL:
2332         {
2333                 struct bio *bp;
2334                 u_int8_t tag_code;
2335
2336                 bp = cam_iosched_next_bio(softc->cam_iosched);
2337                 if (bp == NULL) {
2338                         xpt_release_ccb(start_ccb);
2339                         break;
2340                 }
2341
2342                 if ((bp->bio_flags & BIO_ORDERED) != 0 ||
2343                     (bp->bio_cmd != BIO_DELETE && (softc->flags & ADA_FLAG_NEED_OTAG) != 0)) {
2344                         softc->flags &= ~ADA_FLAG_NEED_OTAG;
2345                         softc->flags |= ADA_FLAG_WAS_OTAG;
2346                         tag_code = 0;
2347                 } else {
2348                         tag_code = 1;
2349                 }
2350                 switch (bp->bio_cmd) {
2351                 case BIO_WRITE:
2352                 case BIO_READ:
2353                 {
2354                         uint64_t lba = bp->bio_pblkno;
2355                         uint16_t count = bp->bio_bcount / softc->params.secsize;
2356                         void *data_ptr;
2357                         int rw_op;
2358
2359                         if (bp->bio_cmd == BIO_WRITE) {
2360                                 softc->flags |= ADA_FLAG_DIRTY;
2361                                 rw_op = CAM_DIR_OUT;
2362                         } else {
2363                                 rw_op = CAM_DIR_IN;
2364                         }
2365
2366                         data_ptr = bp->bio_data;
2367                         if ((bp->bio_flags & (BIO_UNMAPPED|BIO_VLIST)) != 0) {
2368                                 rw_op |= CAM_DATA_BIO;
2369                                 data_ptr = bp;
2370                         }
2371
2372 #ifdef CAM_TEST_FAILURE
2373                         int fail = 0;
2374
2375                         /*
2376                          * Support the failure ioctls.  If the command is a
2377                          * read, and there are pending forced read errors, or
2378                          * if a write and pending write errors, then fail this
2379                          * operation with EIO.  This is useful for testing
2380                          * purposes.  Also, support having every Nth read fail.
2381                          *
2382                          * This is a rather blunt tool.
2383                          */
2384                         if (bp->bio_cmd == BIO_READ) {
2385                                 if (softc->force_read_error) {
2386                                         softc->force_read_error--;
2387                                         fail = 1;
2388                                 }
2389                                 if (softc->periodic_read_error > 0) {
2390                                         if (++softc->periodic_read_count >=
2391                                             softc->periodic_read_error) {
2392                                                 softc->periodic_read_count = 0;
2393                                                 fail = 1;
2394                                         }
2395                                 }
2396                         } else {
2397                                 if (softc->force_write_error) {
2398                                         softc->force_write_error--;
2399                                         fail = 1;
2400                                 }
2401                         }
2402                         if (fail) {
2403                                 biofinish(bp, NULL, EIO);
2404                                 xpt_release_ccb(start_ccb);
2405                                 adaschedule(periph);
2406                                 return;
2407                         }
2408 #endif
2409                         KASSERT((bp->bio_flags & BIO_UNMAPPED) == 0 ||
2410                             round_page(bp->bio_bcount + bp->bio_ma_offset) /
2411                             PAGE_SIZE == bp->bio_ma_n,
2412                             ("Short bio %p", bp));
2413                         cam_fill_ataio(ataio,
2414                             ada_retry_count,
2415                             adadone,
2416                             rw_op,
2417                             0,
2418                             data_ptr,
2419                             bp->bio_bcount,
2420                             ada_default_timeout*1000);
2421
2422                         if ((softc->flags & ADA_FLAG_CAN_NCQ) && tag_code) {
2423                                 if (bp->bio_cmd == BIO_READ) {
2424                                         ata_ncq_cmd(ataio, ATA_READ_FPDMA_QUEUED,
2425                                             lba, count);
2426                                 } else {
2427                                         ata_ncq_cmd(ataio, ATA_WRITE_FPDMA_QUEUED,
2428                                             lba, count);
2429                                 }
2430                         } else if ((softc->flags & ADA_FLAG_CAN_48BIT) &&
2431                             (lba + count >= ATA_MAX_28BIT_LBA ||
2432                             count > 256)) {
2433                                 if (softc->flags & ADA_FLAG_CAN_DMA48) {
2434                                         if (bp->bio_cmd == BIO_READ) {
2435                                                 ata_48bit_cmd(ataio, ATA_READ_DMA48,
2436                                                     0, lba, count);
2437                                         } else {
2438                                                 ata_48bit_cmd(ataio, ATA_WRITE_DMA48,
2439                                                     0, lba, count);
2440                                         }
2441                                 } else {
2442                                         if (bp->bio_cmd == BIO_READ) {
2443                                                 ata_48bit_cmd(ataio, ATA_READ_MUL48,
2444                                                     0, lba, count);
2445                                         } else {
2446                                                 ata_48bit_cmd(ataio, ATA_WRITE_MUL48,
2447                                                     0, lba, count);
2448                                         }
2449                                 }
2450                         } else {
2451                                 if (count == 256)
2452                                         count = 0;
2453                                 if (softc->flags & ADA_FLAG_CAN_DMA) {
2454                                         if (bp->bio_cmd == BIO_READ) {
2455                                                 ata_28bit_cmd(ataio, ATA_READ_DMA,
2456                                                     0, lba, count);
2457                                         } else {
2458                                                 ata_28bit_cmd(ataio, ATA_WRITE_DMA,
2459                                                     0, lba, count);
2460                                         }
2461                                 } else {
2462                                         if (bp->bio_cmd == BIO_READ) {
2463                                                 ata_28bit_cmd(ataio, ATA_READ_MUL,
2464                                                     0, lba, count);
2465                                         } else {
2466                                                 ata_28bit_cmd(ataio, ATA_WRITE_MUL,
2467                                                     0, lba, count);
2468                                         }
2469                                 }
2470                         }
2471                         break;
2472                 }
2473                 case BIO_DELETE:
2474                         switch (softc->delete_method) {
2475                         case ADA_DELETE_NCQ_DSM_TRIM:
2476                                 ada_ncq_dsmtrim(softc, bp, ataio);
2477                                 break;
2478                         case ADA_DELETE_DSM_TRIM:
2479                                 ada_dsmtrim(softc, bp, ataio);
2480                                 break;
2481                         case ADA_DELETE_CFA_ERASE:
2482                                 ada_cfaerase(softc, bp, ataio);
2483                                 break;
2484                         default:
2485                                 biofinish(bp, NULL, EOPNOTSUPP);
2486                                 xpt_release_ccb(start_ccb);
2487                                 adaschedule(periph);
2488                                 return;
2489                         }
2490                         start_ccb->ccb_h.ccb_state = ADA_CCB_TRIM;
2491                         start_ccb->ccb_h.flags |= CAM_UNLOCKED;
2492                         cam_iosched_submit_trim(softc->cam_iosched);
2493                         goto out;
2494                 case BIO_FLUSH:
2495                         cam_fill_ataio(ataio,
2496                             1,
2497                             adadone,
2498                             CAM_DIR_NONE,
2499                             0,
2500                             NULL,
2501                             0,
2502                             ada_default_timeout*1000);
2503
2504                         if (softc->flags & ADA_FLAG_CAN_48BIT)
2505                                 ata_48bit_cmd(ataio, ATA_FLUSHCACHE48, 0, 0, 0);
2506                         else
2507                                 ata_28bit_cmd(ataio, ATA_FLUSHCACHE, 0, 0, 0);
2508                         break;
2509                 case BIO_ZONE: {
2510                         int error, queue_ccb;
2511
2512                         queue_ccb = 0;
2513
2514                         error = ada_zone_cmd(periph, start_ccb, bp, &queue_ccb);
2515                         if ((error != 0)
2516                          || (queue_ccb == 0)) {
2517                                 biofinish(bp, NULL, error);
2518                                 xpt_release_ccb(start_ccb);
2519                                 return;
2520                         }
2521                         break;
2522                 }
2523                 default:
2524                         biofinish(bp, NULL, EOPNOTSUPP);
2525                         xpt_release_ccb(start_ccb);
2526                         return;
2527                 }
2528                 start_ccb->ccb_h.ccb_state = ADA_CCB_BUFFER_IO;
2529                 start_ccb->ccb_h.flags |= CAM_UNLOCKED;
2530 out:
2531                 start_ccb->ccb_h.ccb_bp = bp;
2532                 softc->outstanding_cmds++;
2533                 softc->refcount++;
2534                 cam_periph_unlock(periph);
2535                 xpt_action(start_ccb);
2536                 cam_periph_lock(periph);
2537
2538                 /* May have more work to do, so ensure we stay scheduled */
2539                 adaschedule(periph);
2540                 break;
2541         }
2542         case ADA_STATE_RAHEAD:
2543         case ADA_STATE_WCACHE:
2544         {
2545                 cam_fill_ataio(ataio,
2546                     1,
2547                     adadone,
2548                     CAM_DIR_NONE,
2549                     0,
2550                     NULL,
2551                     0,
2552                     ada_default_timeout*1000);
2553
2554                 if (softc->state == ADA_STATE_RAHEAD) {
2555                         ata_28bit_cmd(ataio, ATA_SETFEATURES, ADA_RA ?
2556                             ATA_SF_ENAB_RCACHE : ATA_SF_DIS_RCACHE, 0, 0);
2557                         start_ccb->ccb_h.ccb_state = ADA_CCB_RAHEAD;
2558                 } else {
2559                         ata_28bit_cmd(ataio, ATA_SETFEATURES, ADA_WC ?
2560                             ATA_SF_ENAB_WCACHE : ATA_SF_DIS_WCACHE, 0, 0);
2561                         start_ccb->ccb_h.ccb_state = ADA_CCB_WCACHE;
2562                 }
2563                 start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
2564                 xpt_action(start_ccb);
2565                 break;
2566         }
2567         case ADA_STATE_LOGDIR:
2568         {
2569                 struct ata_gp_log_dir *log_dir;
2570
2571                 if ((softc->flags & ADA_FLAG_CAN_LOG) == 0) {
2572                         adaprobedone(periph, start_ccb);
2573                         break;
2574                 }
2575
2576                 log_dir = malloc(sizeof(*log_dir), M_ATADA, M_NOWAIT|M_ZERO);
2577                 if (log_dir == NULL) {
2578                         xpt_print(periph->path, "Couldn't malloc log_dir "
2579                             "data\n");
2580                         softc->state = ADA_STATE_NORMAL;
2581                         xpt_release_ccb(start_ccb);
2582                         break;
2583                 }
2584
2585                 ata_read_log(ataio,
2586                     /*retries*/1,
2587                     /*cbfcnp*/adadone,
2588                     /*log_address*/ ATA_LOG_DIRECTORY,
2589                     /*page_number*/ 0,
2590                     /*block_count*/ 1,
2591                     /*protocol*/ softc->flags & ADA_FLAG_CAN_DMA ?
2592                                  CAM_ATAIO_DMA : 0,
2593                     /*data_ptr*/ (uint8_t *)log_dir,
2594                     /*dxfer_len*/sizeof(*log_dir),
2595                     /*timeout*/ada_default_timeout*1000);
2596
2597                 start_ccb->ccb_h.ccb_state = ADA_CCB_LOGDIR;
2598                 xpt_action(start_ccb);
2599                 break;
2600         }
2601         case ADA_STATE_IDDIR:
2602         {
2603                 struct ata_identify_log_pages *id_dir;
2604
2605                 id_dir = malloc(sizeof(*id_dir), M_ATADA, M_NOWAIT | M_ZERO);
2606                 if (id_dir == NULL) {
2607                         xpt_print(periph->path, "Couldn't malloc id_dir "
2608                             "data\n");
2609                         adaprobedone(periph, start_ccb);
2610                         break;
2611                 }
2612
2613                 ata_read_log(ataio,
2614                     /*retries*/1,
2615                     /*cbfcnp*/adadone,
2616                     /*log_address*/ ATA_IDENTIFY_DATA_LOG,
2617                     /*page_number*/ ATA_IDL_PAGE_LIST,
2618                     /*block_count*/ 1,
2619                     /*protocol*/ softc->flags & ADA_FLAG_CAN_DMA ?
2620                                  CAM_ATAIO_DMA : 0,
2621                     /*data_ptr*/ (uint8_t *)id_dir,
2622                     /*dxfer_len*/ sizeof(*id_dir),
2623                     /*timeout*/ada_default_timeout*1000);
2624
2625                 start_ccb->ccb_h.ccb_state = ADA_CCB_IDDIR;
2626                 xpt_action(start_ccb);
2627                 break;
2628         }
2629         case ADA_STATE_SUP_CAP:
2630         {
2631                 struct ata_identify_log_sup_cap *sup_cap;
2632
2633                 sup_cap = malloc(sizeof(*sup_cap), M_ATADA, M_NOWAIT|M_ZERO);
2634                 if (sup_cap == NULL) {
2635                         xpt_print(periph->path, "Couldn't malloc sup_cap "
2636                             "data\n");
2637                         adaprobedone(periph, start_ccb);
2638                         break;
2639                 }
2640
2641                 ata_read_log(ataio,
2642                     /*retries*/1,
2643                     /*cbfcnp*/adadone,
2644                     /*log_address*/ ATA_IDENTIFY_DATA_LOG,
2645                     /*page_number*/ ATA_IDL_SUP_CAP,
2646                     /*block_count*/ 1,
2647                     /*protocol*/ softc->flags & ADA_FLAG_CAN_DMA ?
2648                                  CAM_ATAIO_DMA : 0,
2649                     /*data_ptr*/ (uint8_t *)sup_cap,
2650                     /*dxfer_len*/ sizeof(*sup_cap),
2651                     /*timeout*/ada_default_timeout*1000);
2652
2653                 start_ccb->ccb_h.ccb_state = ADA_CCB_SUP_CAP;
2654                 xpt_action(start_ccb);
2655                 break;
2656         }
2657         case ADA_STATE_ZONE:
2658         {
2659                 struct ata_zoned_info_log *ata_zone;
2660
2661                 ata_zone = malloc(sizeof(*ata_zone), M_ATADA, M_NOWAIT|M_ZERO);
2662                 if (ata_zone == NULL) {
2663                         xpt_print(periph->path, "Couldn't malloc ata_zone "
2664                             "data\n");
2665                         adaprobedone(periph, start_ccb);
2666                         break;
2667                 }
2668
2669                 ata_read_log(ataio,
2670                     /*retries*/1,
2671                     /*cbfcnp*/adadone,
2672                     /*log_address*/ ATA_IDENTIFY_DATA_LOG,
2673                     /*page_number*/ ATA_IDL_ZDI,
2674                     /*block_count*/ 1,
2675                     /*protocol*/ softc->flags & ADA_FLAG_CAN_DMA ?
2676                                  CAM_ATAIO_DMA : 0,
2677                     /*data_ptr*/ (uint8_t *)ata_zone,
2678                     /*dxfer_len*/ sizeof(*ata_zone),
2679                     /*timeout*/ada_default_timeout*1000);
2680
2681                 start_ccb->ccb_h.ccb_state = ADA_CCB_ZONE;
2682                 xpt_action(start_ccb);
2683                 break;
2684         }
2685         }
2686 }
2687
2688 static void
2689 adaprobedone(struct cam_periph *periph, union ccb *ccb)
2690 {
2691         struct ada_softc *softc;
2692
2693         softc = (struct ada_softc *)periph->softc;
2694
2695         if (ccb != NULL)
2696                 xpt_release_ccb(ccb);
2697
2698         softc->state = ADA_STATE_NORMAL;
2699         softc->flags |= ADA_FLAG_PROBED;
2700         adaschedule(periph);
2701         if ((softc->flags & ADA_FLAG_ANNOUNCED) == 0) {
2702                 softc->flags |= ADA_FLAG_ANNOUNCED;
2703                 cam_periph_unhold(periph);
2704         } else {
2705                 cam_periph_release_locked(periph);
2706         }
2707 }
2708
2709 static void
2710 adazonedone(struct cam_periph *periph, union ccb *ccb)
2711 {
2712         struct bio *bp;
2713
2714         bp = (struct bio *)ccb->ccb_h.ccb_bp;
2715
2716         switch (bp->bio_zone.zone_cmd) {
2717         case DISK_ZONE_OPEN:
2718         case DISK_ZONE_CLOSE:
2719         case DISK_ZONE_FINISH:
2720         case DISK_ZONE_RWP:
2721                 break;
2722         case DISK_ZONE_REPORT_ZONES: {
2723                 uint32_t avail_len;
2724                 struct disk_zone_report *rep;
2725                 struct scsi_report_zones_hdr *hdr;
2726                 struct scsi_report_zones_desc *desc;
2727                 struct disk_zone_rep_entry *entry;
2728                 uint32_t hdr_len, num_avail;
2729                 uint32_t num_to_fill, i;
2730
2731                 rep = &bp->bio_zone.zone_params.report;
2732                 avail_len = ccb->ataio.dxfer_len - ccb->ataio.resid;
2733                 /*
2734                  * Note that bio_resid isn't normally used for zone
2735                  * commands, but it is used by devstat_end_transaction_bio()
2736                  * to determine how much data was transferred.  Because
2737                  * the size of the SCSI/ATA data structures is different
2738                  * than the size of the BIO interface structures, the
2739                  * amount of data actually transferred from the drive will
2740                  * be different than the amount of data transferred to
2741                  * the user.
2742                  */
2743                 hdr = (struct scsi_report_zones_hdr *)ccb->ataio.data_ptr;
2744                 if (avail_len < sizeof(*hdr)) {
2745                         /*
2746                          * Is there a better error than EIO here?  We asked
2747                          * for at least the header, and we got less than
2748                          * that.
2749                          */
2750                         bp->bio_error = EIO;
2751                         bp->bio_flags |= BIO_ERROR;
2752                         bp->bio_resid = bp->bio_bcount;
2753                         break;
2754                 }
2755
2756                 hdr_len = le32dec(hdr->length);
2757                 if (hdr_len > 0)
2758                         rep->entries_available = hdr_len / sizeof(*desc);
2759                 else
2760                         rep->entries_available = 0;
2761                 /*
2762                  * NOTE: using the same values for the BIO version of the
2763                  * same field as the SCSI/ATA values.  This means we could
2764                  * get some additional values that aren't defined in bio.h
2765                  * if more values of the same field are defined later.
2766                  */
2767                 rep->header.same = hdr->byte4 & SRZ_SAME_MASK;
2768                 rep->header.maximum_lba = le64dec(hdr->maximum_lba);
2769                 /*
2770                  * If the drive reports no entries that match the query,
2771                  * we're done.
2772                  */
2773                 if (hdr_len == 0) {
2774                         rep->entries_filled = 0;
2775                         bp->bio_resid = bp->bio_bcount;
2776                         break;
2777                 }
2778
2779                 num_avail = min((avail_len - sizeof(*hdr)) / sizeof(*desc),
2780                                 hdr_len / sizeof(*desc));
2781                 /*
2782                  * If the drive didn't return any data, then we're done.
2783                  */
2784                 if (num_avail == 0) {
2785                         rep->entries_filled = 0;
2786                         bp->bio_resid = bp->bio_bcount;
2787                         break;
2788                 }
2789
2790                 num_to_fill = min(num_avail, rep->entries_allocated);
2791                 /*
2792                  * If the user didn't allocate any entries for us to fill,
2793                  * we're done.
2794                  */
2795                 if (num_to_fill == 0) {
2796                         rep->entries_filled = 0;
2797                         bp->bio_resid = bp->bio_bcount;
2798                         break;
2799                 }
2800
2801                 for (i = 0, desc = &hdr->desc_list[0], entry=&rep->entries[0];
2802                      i < num_to_fill; i++, desc++, entry++) {
2803                         /*
2804                          * NOTE: we're mapping the values here directly
2805                          * from the SCSI/ATA bit definitions to the bio.h
2806                          * definitions.  There is also a warning in
2807                          * disk_zone.h, but the impact is that if
2808                          * additional values are added in the SCSI/ATA
2809                          * specs these will be visible to consumers of
2810                          * this interface.
2811                          */
2812                         entry->zone_type = desc->zone_type & SRZ_TYPE_MASK;
2813                         entry->zone_condition =
2814                             (desc->zone_flags & SRZ_ZONE_COND_MASK) >>
2815                             SRZ_ZONE_COND_SHIFT;
2816                         entry->zone_flags |= desc->zone_flags &
2817                             (SRZ_ZONE_NON_SEQ|SRZ_ZONE_RESET);
2818                         entry->zone_length = le64dec(desc->zone_length);
2819                         entry->zone_start_lba = le64dec(desc->zone_start_lba);
2820                         entry->write_pointer_lba =
2821                             le64dec(desc->write_pointer_lba);
2822                 }
2823                 rep->entries_filled = num_to_fill;
2824                 /*
2825                  * Note that this residual is accurate from the user's
2826                  * standpoint, but the amount transferred isn't accurate
2827                  * from the standpoint of what actually came back from the
2828                  * drive.
2829                  */
2830                 bp->bio_resid = bp->bio_bcount - (num_to_fill * sizeof(*entry));
2831                 break;
2832         }
2833         case DISK_ZONE_GET_PARAMS:
2834         default:
2835                 /*
2836                  * In theory we should not get a GET_PARAMS bio, since it
2837                  * should be handled without queueing the command to the
2838                  * drive.
2839                  */
2840                 panic("%s: Invalid zone command %d", __func__,
2841                     bp->bio_zone.zone_cmd);
2842                 break;
2843         }
2844
2845         if (bp->bio_zone.zone_cmd == DISK_ZONE_REPORT_ZONES)
2846                 free(ccb->ataio.data_ptr, M_ATADA);
2847 }
2848
2849 static void
2850 adadone(struct cam_periph *periph, union ccb *done_ccb)
2851 {
2852         struct ada_softc *softc;
2853         struct ccb_ataio *ataio;
2854         struct cam_path *path;
2855         uint32_t priority;
2856         int state;
2857
2858         softc = (struct ada_softc *)periph->softc;
2859         ataio = &done_ccb->ataio;
2860         path = done_ccb->ccb_h.path;
2861         priority = done_ccb->ccb_h.pinfo.priority;
2862
2863         CAM_DEBUG(path, CAM_DEBUG_TRACE, ("adadone\n"));
2864
2865         state = ataio->ccb_h.ccb_state & ADA_CCB_TYPE_MASK;
2866         switch (state) {
2867         case ADA_CCB_BUFFER_IO:
2868         case ADA_CCB_TRIM:
2869         {
2870                 struct bio *bp;
2871                 int error;
2872
2873                 cam_periph_lock(periph);
2874                 bp = (struct bio *)done_ccb->ccb_h.ccb_bp;
2875                 if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
2876                         error = adaerror(done_ccb, 0, 0);
2877                         if (error == ERESTART) {
2878                                 /* A retry was scheduled, so just return. */
2879                                 cam_periph_unlock(periph);
2880                                 return;
2881                         }
2882                         if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0)
2883                                 cam_release_devq(path,
2884                                                  /*relsim_flags*/0,
2885                                                  /*reduction*/0,
2886                                                  /*timeout*/0,
2887                                                  /*getcount_only*/0);
2888                         /*
2889                          * If we get an error on an NCQ DSM TRIM, fall back
2890                          * to a non-NCQ DSM TRIM forever. Please note that if
2891                          * CAN_NCQ_TRIM is set, CAN_TRIM is necessarily set too.
2892                          * However, for this one trim, we treat it as advisory
2893                          * and return success up the stack.
2894                          */
2895                         if (state == ADA_CCB_TRIM &&
2896                             error != 0 &&
2897                             (softc->flags & ADA_FLAG_CAN_NCQ_TRIM) != 0) {
2898                                 softc->flags &= ~ADA_FLAG_CAN_NCQ_TRIM;
2899                                 error = 0;
2900                                 adasetdeletemethod(softc);
2901                         }
2902                 } else {
2903                         if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0)
2904                                 panic("REQ_CMP with QFRZN");
2905
2906                         error = 0;
2907                 }
2908                 bp->bio_error = error;
2909                 if (error != 0) {
2910                         bp->bio_resid = bp->bio_bcount;
2911                         bp->bio_flags |= BIO_ERROR;
2912                 } else {
2913                         if (bp->bio_cmd == BIO_ZONE)
2914                                 adazonedone(periph, done_ccb);
2915                         else if (state == ADA_CCB_TRIM)
2916                                 bp->bio_resid = 0;
2917                         else
2918                                 bp->bio_resid = ataio->resid;
2919
2920                         if ((bp->bio_resid > 0)
2921                          && (bp->bio_cmd != BIO_ZONE))
2922                                 bp->bio_flags |= BIO_ERROR;
2923                 }
2924                 softc->outstanding_cmds--;
2925                 if (softc->outstanding_cmds == 0)
2926                         softc->flags |= ADA_FLAG_WAS_OTAG;
2927
2928                 /*
2929                  * We need to call cam_iosched before we call biodone so that we
2930                  * don't measure any activity that happens in the completion
2931                  * routine, which in the case of sendfile can be quite
2932                  * extensive.  Release the periph refcount taken in adastart()
2933                  * for each CCB.
2934                  */
2935                 cam_iosched_bio_complete(softc->cam_iosched, bp, done_ccb);
2936                 xpt_release_ccb(done_ccb);
2937                 KASSERT(softc->refcount >= 1, ("adadone softc %p refcount %d", softc, softc->refcount));
2938                 softc->refcount--;
2939                 if (state == ADA_CCB_TRIM) {
2940                         TAILQ_HEAD(, bio) queue;
2941                         struct bio *bp1;
2942
2943                         TAILQ_INIT(&queue);
2944                         TAILQ_CONCAT(&queue, &softc->trim_req.bps, bio_queue);
2945                         /*
2946                          * Normally, the xpt_release_ccb() above would make sure
2947                          * that when we have more work to do, that work would
2948                          * get kicked off. However, we specifically keep
2949                          * trim_running set to 0 before the call above to allow
2950                          * other I/O to progress when many BIO_DELETE requests
2951                          * are pushed down. We set trim_running to 0 and call
2952                          * daschedule again so that we don't stall if there are
2953                          * no other I/Os pending apart from BIO_DELETEs.
2954                          */
2955                         cam_iosched_trim_done(softc->cam_iosched);
2956                         adaschedule(periph);
2957                         cam_periph_unlock(periph);
2958                         while ((bp1 = TAILQ_FIRST(&queue)) != NULL) {
2959                                 TAILQ_REMOVE(&queue, bp1, bio_queue);
2960                                 bp1->bio_error = error;
2961                                 if (error != 0) {
2962                                         bp1->bio_flags |= BIO_ERROR;
2963                                         bp1->bio_resid = bp1->bio_bcount;
2964                                 } else
2965                                         bp1->bio_resid = 0;
2966                                 biodone(bp1);
2967                         }
2968                 } else {
2969                         adaschedule(periph);
2970                         cam_periph_unlock(periph);
2971                         biodone(bp);
2972                 }
2973                 return;
2974         }
2975         case ADA_CCB_RAHEAD:
2976         {
2977                 if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
2978                         if (adaerror(done_ccb, 0, 0) == ERESTART) {
2979                                 /* Drop freeze taken due to CAM_DEV_QFREEZE */
2980                                 cam_release_devq(path, 0, 0, 0, FALSE);
2981                                 return;
2982                         } else if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
2983                                 cam_release_devq(path,
2984                                     /*relsim_flags*/0,
2985                                     /*reduction*/0,
2986                                     /*timeout*/0,
2987                                     /*getcount_only*/0);
2988                         }
2989                 }
2990
2991                 /*
2992                  * Since our peripheral may be invalidated by an error
2993                  * above or an external event, we must release our CCB
2994                  * before releasing the reference on the peripheral.
2995                  * The peripheral will only go away once the last reference
2996                  * is removed, and we need it around for the CCB release
2997                  * operation.
2998                  */
2999
3000                 xpt_release_ccb(done_ccb);
3001                 softc->state = ADA_STATE_WCACHE;
3002                 xpt_schedule(periph, priority);
3003                 /* Drop freeze taken due to CAM_DEV_QFREEZE */
3004                 cam_release_devq(path, 0, 0, 0, FALSE);
3005                 return;
3006         }
3007         case ADA_CCB_WCACHE:
3008         {
3009                 if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
3010                         if (adaerror(done_ccb, 0, 0) == ERESTART) {
3011                                 /* Drop freeze taken due to CAM_DEV_QFREEZE */
3012                                 cam_release_devq(path, 0, 0, 0, FALSE);
3013                                 return;
3014                         } else if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
3015                                 cam_release_devq(path,
3016                                     /*relsim_flags*/0,
3017                                     /*reduction*/0,
3018                                     /*timeout*/0,
3019                                     /*getcount_only*/0);
3020                         }
3021                 }
3022
3023                 /* Drop freeze taken due to CAM_DEV_QFREEZE */
3024                 cam_release_devq(path, 0, 0, 0, FALSE);
3025
3026                 if ((softc->flags & ADA_FLAG_CAN_LOG)
3027                  && (softc->zone_mode != ADA_ZONE_NONE)) {
3028                         xpt_release_ccb(done_ccb);
3029                         softc->state = ADA_STATE_LOGDIR;
3030                         xpt_schedule(periph, priority);
3031                 } else {
3032                         adaprobedone(periph, done_ccb);
3033                 }
3034                 return;
3035         }
3036         case ADA_CCB_LOGDIR:
3037         {
3038                 int error;
3039
3040                 if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
3041                         error = 0;
3042                         softc->valid_logdir_len = 0;
3043                         bzero(&softc->ata_logdir, sizeof(softc->ata_logdir));
3044                         softc->valid_logdir_len =
3045                                 ataio->dxfer_len - ataio->resid;
3046                         if (softc->valid_logdir_len > 0)
3047                                 bcopy(ataio->data_ptr, &softc->ata_logdir,
3048                                     min(softc->valid_logdir_len,
3049                                         sizeof(softc->ata_logdir)));
3050                         /*
3051                          * Figure out whether the Identify Device log is
3052                          * supported.  The General Purpose log directory
3053                          * has a header, and lists the number of pages
3054                          * available for each GP log identified by the
3055                          * offset into the list.
3056                          */
3057                         if ((softc->valid_logdir_len >=
3058                             ((ATA_IDENTIFY_DATA_LOG + 1) * sizeof(uint16_t)))
3059                          && (le16dec(softc->ata_logdir.header) ==
3060                              ATA_GP_LOG_DIR_VERSION)
3061                          && (le16dec(&softc->ata_logdir.num_pages[
3062                              (ATA_IDENTIFY_DATA_LOG *
3063                              sizeof(uint16_t)) - sizeof(uint16_t)]) > 0)){
3064                                 softc->flags |= ADA_FLAG_CAN_IDLOG;
3065                         } else {
3066                                 softc->flags &= ~ADA_FLAG_CAN_IDLOG;
3067                         }
3068                 } else {
3069                         error = adaerror(done_ccb, CAM_RETRY_SELTO,
3070                                          SF_RETRY_UA|SF_NO_PRINT);
3071                         if (error == ERESTART)
3072                                 return;
3073                         else if (error != 0) {
3074                                 /*
3075                                  * If we can't get the ATA log directory,
3076                                  * then ATA logs are effectively not
3077                                  * supported even if the bit is set in the
3078                                  * identify data.
3079                                  */
3080                                 softc->flags &= ~(ADA_FLAG_CAN_LOG |
3081                                                   ADA_FLAG_CAN_IDLOG);
3082                                 if ((done_ccb->ccb_h.status &
3083                                      CAM_DEV_QFRZN) != 0) {
3084                                         /* Don't wedge this device's queue */
3085                                         cam_release_devq(done_ccb->ccb_h.path,
3086                                                          /*relsim_flags*/0,
3087                                                          /*reduction*/0,
3088                                                          /*timeout*/0,
3089                                                          /*getcount_only*/0);
3090                                 }
3091                         }
3092                 }
3093
3094                 free(ataio->data_ptr, M_ATADA);
3095
3096                 if ((error == 0)
3097                  && (softc->flags & ADA_FLAG_CAN_IDLOG)) {
3098                         softc->state = ADA_STATE_IDDIR;
3099                         xpt_release_ccb(done_ccb);
3100                         xpt_schedule(periph, priority);
3101                 } else
3102                         adaprobedone(periph, done_ccb);
3103
3104                 return;
3105         }
3106         case ADA_CCB_IDDIR: {
3107                 int error;
3108
3109                 if ((ataio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
3110                         off_t entries_offset, max_entries;
3111                         error = 0;
3112
3113                         softc->valid_iddir_len = 0;
3114                         bzero(&softc->ata_iddir, sizeof(softc->ata_iddir));
3115                         softc->flags &= ~(ADA_FLAG_CAN_SUPCAP |
3116                                           ADA_FLAG_CAN_ZONE);
3117                         softc->valid_iddir_len =
3118                                 ataio->dxfer_len - ataio->resid;
3119                         if (softc->valid_iddir_len > 0)
3120                                 bcopy(ataio->data_ptr, &softc->ata_iddir,
3121                                     min(softc->valid_iddir_len,
3122                                         sizeof(softc->ata_iddir)));
3123
3124                         entries_offset =
3125                             __offsetof(struct ata_identify_log_pages,entries);
3126                         max_entries = softc->valid_iddir_len - entries_offset;
3127                         if ((softc->valid_iddir_len > (entries_offset + 1))
3128                          && (le64dec(softc->ata_iddir.header) ==
3129                              ATA_IDLOG_REVISION)
3130                          && (softc->ata_iddir.entry_count > 0)) {
3131                                 int num_entries, i;
3132
3133                                 num_entries = softc->ata_iddir.entry_count;
3134                                 num_entries = min(num_entries,
3135                                    softc->valid_iddir_len - entries_offset);
3136                                 for (i = 0; i < num_entries &&
3137                                      i < max_entries; i++) {
3138                                         if (softc->ata_iddir.entries[i] ==
3139                                             ATA_IDL_SUP_CAP)
3140                                                 softc->flags |=
3141                                                     ADA_FLAG_CAN_SUPCAP;
3142                                         else if (softc->ata_iddir.entries[i]==
3143                                                  ATA_IDL_ZDI)
3144                                                 softc->flags |=
3145                                                     ADA_FLAG_CAN_ZONE;
3146
3147                                         if ((softc->flags &
3148                                              ADA_FLAG_CAN_SUPCAP)
3149                                          && (softc->flags &
3150                                              ADA_FLAG_CAN_ZONE))
3151                                                 break;
3152                                 }
3153                         }
3154                 } else {
3155                         error = adaerror(done_ccb, CAM_RETRY_SELTO,
3156                                          SF_RETRY_UA|SF_NO_PRINT);
3157                         if (error == ERESTART)
3158                                 return;
3159                         else if (error != 0) {
3160                                 /*
3161                                  * If we can't get the ATA Identify Data log
3162                                  * directory, then it effectively isn't
3163                                  * supported even if the ATA Log directory
3164                                  * a non-zero number of pages present for
3165                                  * this log.
3166                                  */
3167                                 softc->flags &= ~ADA_FLAG_CAN_IDLOG;
3168                                 if ((done_ccb->ccb_h.status &
3169                                      CAM_DEV_QFRZN) != 0) {
3170                                         /* Don't wedge this device's queue */
3171                                         cam_release_devq(done_ccb->ccb_h.path,
3172                                                          /*relsim_flags*/0,
3173                                                          /*reduction*/0,
3174                                                          /*timeout*/0,
3175                                                          /*getcount_only*/0);
3176                                 }
3177                         }
3178                 }
3179
3180                 free(ataio->data_ptr, M_ATADA);
3181
3182                 if ((error == 0)
3183                  && (softc->flags & ADA_FLAG_CAN_SUPCAP)) {
3184                         softc->state = ADA_STATE_SUP_CAP;
3185                         xpt_release_ccb(done_ccb);
3186                         xpt_schedule(periph, priority);
3187                 } else
3188                         adaprobedone(periph, done_ccb);
3189                 return;
3190         }
3191         case ADA_CCB_SUP_CAP: {
3192                 int error;
3193
3194                 if ((ataio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
3195                         uint32_t valid_len;
3196                         size_t needed_size;
3197                         struct ata_identify_log_sup_cap *sup_cap;
3198                         error = 0;
3199
3200                         sup_cap = (struct ata_identify_log_sup_cap *)
3201                             ataio->data_ptr;
3202                         valid_len = ataio->dxfer_len - ataio->resid;
3203                         needed_size =
3204                             __offsetof(struct ata_identify_log_sup_cap,
3205                             sup_zac_cap) + 1 + sizeof(sup_cap->sup_zac_cap);
3206                         if (valid_len >= needed_size) {
3207                                 uint64_t zoned, zac_cap;
3208
3209                                 zoned = le64dec(sup_cap->zoned_cap);
3210                                 if (zoned & ATA_ZONED_VALID) {
3211                                         /*
3212                                          * This should have already been
3213                                          * set, because this is also in the
3214                                          * ATA identify data.
3215                                          */
3216                                         if ((zoned & ATA_ZONED_MASK) ==
3217                                             ATA_SUPPORT_ZONE_HOST_AWARE)
3218                                                 softc->zone_mode =
3219                                                     ADA_ZONE_HOST_AWARE;
3220                                         else if ((zoned & ATA_ZONED_MASK) ==
3221                                             ATA_SUPPORT_ZONE_DEV_MANAGED)
3222                                                 softc->zone_mode =
3223                                                     ADA_ZONE_DRIVE_MANAGED;
3224                                 }
3225
3226                                 zac_cap = le64dec(sup_cap->sup_zac_cap);
3227                                 if (zac_cap & ATA_SUP_ZAC_CAP_VALID) {
3228                                         if (zac_cap & ATA_REPORT_ZONES_SUP)
3229                                                 softc->zone_flags |=
3230                                                     ADA_ZONE_FLAG_RZ_SUP;
3231                                         if (zac_cap & ATA_ND_OPEN_ZONE_SUP)
3232                                                 softc->zone_flags |=
3233                                                     ADA_ZONE_FLAG_OPEN_SUP;
3234                                         if (zac_cap & ATA_ND_CLOSE_ZONE_SUP)
3235                                                 softc->zone_flags |=
3236                                                     ADA_ZONE_FLAG_CLOSE_SUP;
3237                                         if (zac_cap & ATA_ND_FINISH_ZONE_SUP)
3238                                                 softc->zone_flags |=
3239                                                     ADA_ZONE_FLAG_FINISH_SUP;
3240                                         if (zac_cap & ATA_ND_RWP_SUP)
3241                                                 softc->zone_flags |=
3242                                                     ADA_ZONE_FLAG_RWP_SUP;
3243                                 } else {
3244                                         /*
3245                                          * This field was introduced in
3246                                          * ACS-4, r08 on April 28th, 2015.
3247                                          * If the drive firmware was written
3248                                          * to an earlier spec, it won't have
3249                                          * the field.  So, assume all
3250                                          * commands are supported.
3251                                          */
3252                                         softc->zone_flags |=
3253                                             ADA_ZONE_FLAG_SUP_MASK;
3254                                 }
3255                         }
3256                 } else {
3257                         error = adaerror(done_ccb, CAM_RETRY_SELTO,
3258                                          SF_RETRY_UA|SF_NO_PRINT);
3259                         if (error == ERESTART)
3260                                 return;
3261                         else if (error != 0) {
3262                                 /*
3263                                  * If we can't get the ATA Identify Data
3264                                  * Supported Capabilities page, clear the
3265                                  * flag...
3266                                  */
3267                                 softc->flags &= ~ADA_FLAG_CAN_SUPCAP;
3268                                 /*
3269                                  * And clear zone capabilities.
3270                                  */
3271                                 softc->zone_flags &= ~ADA_ZONE_FLAG_SUP_MASK;
3272                                 if ((done_ccb->ccb_h.status &
3273                                      CAM_DEV_QFRZN) != 0) {
3274                                         /* Don't wedge this device's queue */
3275                                         cam_release_devq(done_ccb->ccb_h.path,
3276                                                          /*relsim_flags*/0,
3277                                                          /*reduction*/0,
3278                                                          /*timeout*/0,
3279                                                          /*getcount_only*/0);
3280                                 }
3281                         }
3282                 }
3283
3284                 free(ataio->data_ptr, M_ATADA);
3285
3286                 if ((error == 0)
3287                  && (softc->flags & ADA_FLAG_CAN_ZONE)) {
3288                         softc->state = ADA_STATE_ZONE;
3289                         xpt_release_ccb(done_ccb);
3290                         xpt_schedule(periph, priority);
3291                 } else
3292                         adaprobedone(periph, done_ccb);
3293                 return;
3294         }
3295         case ADA_CCB_ZONE: {
3296                 int error;
3297
3298                 if ((ataio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
3299                         struct ata_zoned_info_log *zi_log;
3300                         uint32_t valid_len;
3301                         size_t needed_size;
3302
3303                         zi_log = (struct ata_zoned_info_log *)ataio->data_ptr;
3304
3305                         valid_len = ataio->dxfer_len - ataio->resid;
3306                         needed_size = __offsetof(struct ata_zoned_info_log,
3307                             version_info) + 1 + sizeof(zi_log->version_info);
3308                         if (valid_len >= needed_size) {
3309                                 uint64_t tmpvar;
3310
3311                                 tmpvar = le64dec(zi_log->zoned_cap);
3312                                 if (tmpvar & ATA_ZDI_CAP_VALID) {
3313                                         if (tmpvar & ATA_ZDI_CAP_URSWRZ)
3314                                                 softc->zone_flags |=
3315                                                     ADA_ZONE_FLAG_URSWRZ;
3316                                         else
3317                                                 softc->zone_flags &=
3318                                                     ~ADA_ZONE_FLAG_URSWRZ;
3319                                 }
3320                                 tmpvar = le64dec(zi_log->optimal_seq_zones);
3321                                 if (tmpvar & ATA_ZDI_OPT_SEQ_VALID) {
3322                                         softc->zone_flags |=
3323                                             ADA_ZONE_FLAG_OPT_SEQ_SET;
3324                                         softc->optimal_seq_zones = (tmpvar &
3325                                             ATA_ZDI_OPT_SEQ_MASK);
3326                                 } else {
3327                                         softc->zone_flags &=
3328                                             ~ADA_ZONE_FLAG_OPT_SEQ_SET;
3329                                         softc->optimal_seq_zones = 0;
3330                                 }
3331
3332                                 tmpvar =le64dec(zi_log->optimal_nonseq_zones);
3333                                 if (tmpvar & ATA_ZDI_OPT_NS_VALID) {
3334                                         softc->zone_flags |=
3335                                             ADA_ZONE_FLAG_OPT_NONSEQ_SET;
3336                                         softc->optimal_nonseq_zones =
3337                                             (tmpvar & ATA_ZDI_OPT_NS_MASK);
3338                                 } else {
3339                                         softc->zone_flags &=
3340                                             ~ADA_ZONE_FLAG_OPT_NONSEQ_SET;
3341                                         softc->optimal_nonseq_zones = 0;
3342                                 }
3343
3344                                 tmpvar = le64dec(zi_log->max_seq_req_zones);
3345                                 if (tmpvar & ATA_ZDI_MAX_SEQ_VALID) {
3346                                         softc->zone_flags |=
3347                                             ADA_ZONE_FLAG_MAX_SEQ_SET;
3348                                         softc->max_seq_zones =
3349                                             (tmpvar & ATA_ZDI_MAX_SEQ_MASK);
3350                                 } else {
3351                                         softc->zone_flags &=
3352                                             ~ADA_ZONE_FLAG_MAX_SEQ_SET;
3353                                         softc->max_seq_zones = 0;
3354                                 }
3355                         }
3356                 } else {
3357                         error = adaerror(done_ccb, CAM_RETRY_SELTO,
3358                                          SF_RETRY_UA|SF_NO_PRINT);
3359                         if (error == ERESTART)
3360                                 return;
3361                         else if (error != 0) {
3362                                 softc->flags &= ~ADA_FLAG_CAN_ZONE;
3363                                 softc->flags &= ~ADA_ZONE_FLAG_SET_MASK;
3364
3365                                 if ((done_ccb->ccb_h.status &
3366                                      CAM_DEV_QFRZN) != 0) {
3367                                         /* Don't wedge this device's queue */
3368                                         cam_release_devq(done_ccb->ccb_h.path,
3369                                                          /*relsim_flags*/0,
3370                                                          /*reduction*/0,
3371                                                          /*timeout*/0,
3372                                                          /*getcount_only*/0);
3373                                 }
3374                         }
3375                 }
3376                 free(ataio->data_ptr, M_ATADA);
3377
3378                 adaprobedone(periph, done_ccb);
3379                 return;
3380         }
3381         case ADA_CCB_DUMP:
3382                 /* No-op.  We're polling */
3383                 return;
3384         default:
3385                 break;
3386         }
3387         xpt_release_ccb(done_ccb);
3388 }
3389
3390 static int
3391 adaerror(union ccb *ccb, u_int32_t cam_flags, u_int32_t sense_flags)
3392 {
3393 #ifdef CAM_IO_STATS
3394         struct ada_softc *softc;
3395         struct cam_periph *periph;
3396
3397         periph = xpt_path_periph(ccb->ccb_h.path);
3398         softc = (struct ada_softc *)periph->softc;
3399
3400         switch (ccb->ccb_h.status & CAM_STATUS_MASK) {
3401         case CAM_CMD_TIMEOUT:
3402                 softc->timeouts++;
3403                 break;
3404         case CAM_REQ_ABORTED:
3405         case CAM_REQ_CMP_ERR:
3406         case CAM_REQ_TERMIO:
3407         case CAM_UNREC_HBA_ERROR:
3408         case CAM_DATA_RUN_ERR:
3409         case CAM_ATA_STATUS_ERROR:
3410                 softc->errors++;
3411                 break;
3412         default:
3413                 break;
3414         }
3415 #endif
3416
3417         return(cam_periph_error(ccb, cam_flags, sense_flags));
3418 }
3419
3420 static void
3421 adasetgeom(struct ada_softc *softc, struct ccb_getdev *cgd)
3422 {
3423         struct disk_params *dp = &softc->params;
3424         u_int64_t lbasize48;
3425         u_int32_t lbasize;
3426         u_int maxio, d_flags;
3427
3428         dp->secsize = ata_logical_sector_size(&cgd->ident_data);
3429         if ((cgd->ident_data.atavalid & ATA_FLAG_54_58) &&
3430             cgd->ident_data.current_heads != 0 &&
3431             cgd->ident_data.current_sectors != 0) {
3432                 dp->heads = cgd->ident_data.current_heads;
3433                 dp->secs_per_track = cgd->ident_data.current_sectors;
3434                 dp->cylinders = cgd->ident_data.cylinders;
3435                 dp->sectors = (u_int32_t)cgd->ident_data.current_size_1 |
3436                           ((u_int32_t)cgd->ident_data.current_size_2 << 16);
3437         } else {
3438                 dp->heads = cgd->ident_data.heads;
3439                 dp->secs_per_track = cgd->ident_data.sectors;
3440                 dp->cylinders = cgd->ident_data.cylinders;
3441                 dp->sectors = cgd->ident_data.cylinders *
3442                               (u_int32_t)(dp->heads * dp->secs_per_track);
3443         }
3444         lbasize = (u_int32_t)cgd->ident_data.lba_size_1 |
3445                   ((u_int32_t)cgd->ident_data.lba_size_2 << 16);
3446
3447         /* use the 28bit LBA size if valid or bigger than the CHS mapping */
3448         if (cgd->ident_data.cylinders == 16383 || dp->sectors < lbasize)
3449                 dp->sectors = lbasize;
3450
3451         /* use the 48bit LBA size if valid */
3452         lbasize48 = ((u_int64_t)cgd->ident_data.lba_size48_1) |
3453                     ((u_int64_t)cgd->ident_data.lba_size48_2 << 16) |
3454                     ((u_int64_t)cgd->ident_data.lba_size48_3 << 32) |
3455                     ((u_int64_t)cgd->ident_data.lba_size48_4 << 48);
3456         if ((cgd->ident_data.support.command2 & ATA_SUPPORT_ADDRESS48) &&
3457             lbasize48 > ATA_MAX_28BIT_LBA)
3458                 dp->sectors = lbasize48;
3459
3460         maxio = softc->cpi.maxio;               /* Honor max I/O size of SIM */
3461         if (maxio == 0)
3462                 maxio = DFLTPHYS;       /* traditional default */
3463         else if (maxio > maxphys)
3464                 maxio = maxphys;        /* for safety */
3465         if (softc->flags & ADA_FLAG_CAN_48BIT)
3466                 maxio = min(maxio, 65536 * softc->params.secsize);
3467         else                                    /* 28bit ATA command limit */
3468                 maxio = min(maxio, 256 * softc->params.secsize);
3469         if (softc->quirks & ADA_Q_128KB)
3470                 maxio = min(maxio, 128 * 1024);
3471         softc->disk->d_maxsize = maxio;
3472         d_flags = DISKFLAG_DIRECT_COMPLETION | DISKFLAG_CANZONE;
3473         if (softc->flags & ADA_FLAG_CAN_FLUSHCACHE)
3474                 d_flags |= DISKFLAG_CANFLUSHCACHE;
3475         if (softc->flags & ADA_FLAG_CAN_TRIM) {
3476                 d_flags |= DISKFLAG_CANDELETE;
3477                 softc->disk->d_delmaxsize = softc->params.secsize *
3478                     ATA_DSM_RANGE_MAX * softc->trim_max_ranges;
3479         } else if ((softc->flags & ADA_FLAG_CAN_CFA) &&
3480             !(softc->flags & ADA_FLAG_CAN_48BIT)) {
3481                 d_flags |= DISKFLAG_CANDELETE;
3482                 softc->disk->d_delmaxsize = 256 * softc->params.secsize;
3483         } else
3484                 softc->disk->d_delmaxsize = maxio;
3485         if ((softc->cpi.hba_misc & PIM_UNMAPPED) != 0) {
3486                 d_flags |= DISKFLAG_UNMAPPED_BIO;
3487                 softc->flags |= ADA_FLAG_UNMAPPEDIO;
3488         }
3489         softc->disk->d_flags = d_flags;
3490         strlcpy(softc->disk->d_descr, cgd->ident_data.model,
3491             MIN(sizeof(softc->disk->d_descr), sizeof(cgd->ident_data.model)));
3492         strlcpy(softc->disk->d_ident, cgd->ident_data.serial,
3493             MIN(sizeof(softc->disk->d_ident), sizeof(cgd->ident_data.serial)));
3494
3495         softc->disk->d_sectorsize = softc->params.secsize;
3496         softc->disk->d_mediasize = (off_t)softc->params.sectors *
3497             softc->params.secsize;
3498         if (ata_physical_sector_size(&cgd->ident_data) !=
3499             softc->params.secsize) {
3500                 softc->disk->d_stripesize =
3501                     ata_physical_sector_size(&cgd->ident_data);
3502                 softc->disk->d_stripeoffset = (softc->disk->d_stripesize -
3503                     ata_logical_sector_offset(&cgd->ident_data)) %
3504                     softc->disk->d_stripesize;
3505         } else if (softc->quirks & ADA_Q_4K) {
3506                 softc->disk->d_stripesize = 4096;
3507                 softc->disk->d_stripeoffset = 0;
3508         }
3509         softc->disk->d_fwsectors = softc->params.secs_per_track;
3510         softc->disk->d_fwheads = softc->params.heads;
3511         softc->disk->d_rotation_rate = cgd->ident_data.media_rotation_rate;
3512         snprintf(softc->disk->d_attachment, sizeof(softc->disk->d_attachment),
3513             "%s%d", softc->cpi.dev_name, softc->cpi.unit_number);
3514 }
3515
3516 static void
3517 adasendorderedtag(void *arg)
3518 {
3519         struct ada_softc *softc = arg;
3520
3521         if (ada_send_ordered) {
3522                 if (softc->outstanding_cmds > 0) {
3523                         if ((softc->flags & ADA_FLAG_WAS_OTAG) == 0)
3524                                 softc->flags |= ADA_FLAG_NEED_OTAG;
3525                         softc->flags &= ~ADA_FLAG_WAS_OTAG;
3526                 }
3527         }
3528         /* Queue us up again */
3529         callout_reset(&softc->sendordered_c,
3530             (ada_default_timeout * hz) / ADA_ORDEREDTAG_INTERVAL,
3531             adasendorderedtag, softc);
3532 }
3533
3534 /*
3535  * Step through all ADA peripheral drivers, and if the device is still open,
3536  * sync the disk cache to physical media.
3537  */
3538 static void
3539 adaflush(void)
3540 {
3541         struct cam_periph *periph;
3542         struct ada_softc *softc;
3543         union ccb *ccb;
3544         int error;
3545
3546         CAM_PERIPH_FOREACH(periph, &adadriver) {
3547                 softc = (struct ada_softc *)periph->softc;
3548                 if (SCHEDULER_STOPPED()) {
3549                         /* If we paniced with the lock held, do not recurse. */
3550                         if (!cam_periph_owned(periph) &&
3551                             (softc->flags & ADA_FLAG_OPEN)) {
3552                                 adadump(softc->disk, NULL, 0, 0, 0);
3553                         }
3554                         continue;
3555                 }
3556                 cam_periph_lock(periph);
3557                 /*
3558                  * We only sync the cache if the drive is still open, and
3559                  * if the drive is capable of it..
3560                  */
3561                 if (((softc->flags & ADA_FLAG_OPEN) == 0) ||
3562                     (softc->flags & ADA_FLAG_CAN_FLUSHCACHE) == 0) {
3563                         cam_periph_unlock(periph);
3564                         continue;
3565                 }
3566
3567                 ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL);
3568                 cam_fill_ataio(&ccb->ataio,
3569                                     0,
3570                                     NULL,
3571                                     CAM_DIR_NONE,
3572                                     0,
3573                                     NULL,
3574                                     0,
3575                                     ada_default_timeout*1000);
3576                 if (softc->flags & ADA_FLAG_CAN_48BIT)
3577                         ata_48bit_cmd(&ccb->ataio, ATA_FLUSHCACHE48, 0, 0, 0);
3578                 else
3579                         ata_28bit_cmd(&ccb->ataio, ATA_FLUSHCACHE, 0, 0, 0);
3580
3581                 error = cam_periph_runccb(ccb, adaerror, /*cam_flags*/0,
3582                     /*sense_flags*/ SF_NO_RECOVERY | SF_NO_RETRY,
3583                     softc->disk->d_devstat);
3584                 if (error != 0)
3585                         xpt_print(periph->path, "Synchronize cache failed\n");
3586                 xpt_release_ccb(ccb);
3587                 cam_periph_unlock(periph);
3588         }
3589 }
3590
3591 static void
3592 adaspindown(uint8_t cmd, int flags)
3593 {
3594         struct cam_periph *periph;
3595         struct ada_softc *softc;
3596         struct ccb_ataio local_ccb;
3597         int error;
3598
3599         CAM_PERIPH_FOREACH(periph, &adadriver) {
3600                 /* If we paniced with lock held - not recurse here. */
3601                 if (cam_periph_owned(periph))
3602                         continue;
3603                 cam_periph_lock(periph);
3604                 softc = (struct ada_softc *)periph->softc;
3605                 /*
3606                  * We only spin-down the drive if it is capable of it..
3607                  */
3608                 if ((softc->flags & ADA_FLAG_CAN_POWERMGT) == 0) {
3609                         cam_periph_unlock(periph);
3610                         continue;
3611                 }
3612
3613                 if (bootverbose)
3614                         xpt_print(periph->path, "spin-down\n");
3615
3616                 memset(&local_ccb, 0, sizeof(local_ccb));
3617                 xpt_setup_ccb(&local_ccb.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
3618                 local_ccb.ccb_h.ccb_state = ADA_CCB_DUMP;
3619
3620                 cam_fill_ataio(&local_ccb,
3621                                     0,
3622                                     NULL,
3623                                     CAM_DIR_NONE | flags,
3624                                     0,
3625                                     NULL,
3626                                     0,
3627                                     ada_default_timeout*1000);
3628                 ata_28bit_cmd(&local_ccb, cmd, 0, 0, 0);
3629                 error = cam_periph_runccb((union ccb *)&local_ccb, adaerror,
3630                     /*cam_flags*/0, /*sense_flags*/ SF_NO_RECOVERY | SF_NO_RETRY,
3631                     softc->disk->d_devstat);
3632                 if (error != 0)
3633                         xpt_print(periph->path, "Spin-down disk failed\n");
3634                 cam_periph_unlock(periph);
3635         }
3636 }
3637
3638 static void
3639 adashutdown(void *arg, int howto)
3640 {
3641         int how;
3642
3643         adaflush();
3644
3645         /*
3646          * STANDBY IMMEDIATE saves any volatile data to the drive. It also spins
3647          * down hard drives. IDLE IMMEDIATE also saves the volatile data without
3648          * a spindown. We send the former when we expect to lose power soon. For
3649          * a warm boot, we send the latter to avoid a thundering herd of spinups
3650          * just after the kernel loads while probing. We have to do something to
3651          * flush the data because the BIOS in many systems resets the HBA
3652          * causing a COMINIT/COMRESET negotiation, which some drives interpret
3653          * as license to toss the volatile data, and others count as unclean
3654          * shutdown when in the Active PM state in SMART attributes.
3655          *
3656          * adaspindown will ensure that we don't send this to a drive that
3657          * doesn't support it.
3658          */
3659         if (ada_spindown_shutdown != 0) {
3660                 how = (howto & (RB_HALT | RB_POWEROFF | RB_POWERCYCLE)) ?
3661                     ATA_STANDBY_IMMEDIATE : ATA_IDLE_IMMEDIATE;
3662                 adaspindown(how, 0);
3663         }
3664 }
3665
3666 static void
3667 adasuspend(void *arg)
3668 {
3669
3670         adaflush();
3671         /*
3672          * SLEEP also fushes any volatile data, like STANDBY IMEDIATE,
3673          * so we don't need to send it as well.
3674          */
3675         if (ada_spindown_suspend != 0)
3676                 adaspindown(ATA_SLEEP, CAM_DEV_QFREEZE);
3677 }
3678
3679 static void
3680 adaresume(void *arg)
3681 {
3682         struct cam_periph *periph;
3683         struct ada_softc *softc;
3684
3685         if (ada_spindown_suspend == 0)
3686                 return;
3687
3688         CAM_PERIPH_FOREACH(periph, &adadriver) {
3689                 cam_periph_lock(periph);
3690                 softc = (struct ada_softc *)periph->softc;
3691                 /*
3692                  * We only spin-down the drive if it is capable of it..
3693                  */
3694                 if ((softc->flags & ADA_FLAG_CAN_POWERMGT) == 0) {
3695                         cam_periph_unlock(periph);
3696                         continue;
3697                 }
3698
3699                 if (bootverbose)
3700                         xpt_print(periph->path, "resume\n");
3701
3702                 /*
3703                  * Drop freeze taken due to CAM_DEV_QFREEZE flag set on
3704                  * sleep request.
3705                  */
3706                 cam_release_devq(periph->path,
3707                          /*relsim_flags*/0,
3708                          /*openings*/0,
3709                          /*timeout*/0,
3710                          /*getcount_only*/0);
3711
3712                 cam_periph_unlock(periph);
3713         }
3714 }
3715
3716 #endif /* _KERNEL */