sys/cam/ata/ata_da.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2009 Alexander Motin <mav@FreeBSD.org>
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer,
  12  *    without modification, immediately at the beginning of the file.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  18  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  19  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  20  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  21  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  22  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  23  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  24  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  25  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  26  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27  */
  28
  29 #include <sys/cdefs.h>
  30 __FBSDID("$FreeBSD$");
  31
  32 #include "opt_ada.h"
  33
  34 #include <sys/param.h>
  35
  36 #ifdef _KERNEL
  37 #include <sys/systm.h>
  38 #include <sys/kernel.h>
  39 #include <sys/bio.h>
  40 #include <sys/sysctl.h>
  41 #include <sys/taskqueue.h>
  42 #include <sys/lock.h>
  43 #include <sys/mutex.h>
  44 #include <sys/conf.h>
  45 #include <sys/devicestat.h>
  46 #include <sys/eventhandler.h>
  47 #include <sys/malloc.h>
  48 #include <sys/endian.h>
  49 #include <sys/cons.h>
  50 #include <sys/proc.h>
  51 #include <sys/reboot.h>
  52 #include <sys/sbuf.h>
  53 #include <geom/geom_disk.h>
  54 #endif /* _KERNEL */
  55
  56 #ifndef _KERNEL
  57 #include <stdio.h>
  58 #include <string.h>
  59 #endif /* _KERNEL */
  60
  61 #include <cam/cam.h>
  62 #include <cam/cam_ccb.h>
  63 #include <cam/cam_periph.h>
  64 #include <cam/cam_xpt_periph.h>
  65 #include <cam/scsi/scsi_all.h>
  66 #include <cam/scsi/scsi_da.h>
  67 #include <cam/cam_sim.h>
  68 #include <cam/cam_iosched.h>
  69
  70 #include <cam/ata/ata_all.h>
  71
  72 #include <machine/md_var.h>     /* geometry translation */
  73
  74 #ifdef _KERNEL
  75
  76 #define ATA_MAX_28BIT_LBA               268435455UL
  77
  78 extern int iosched_debug;
  79
  80 typedef enum {
  81         ADA_STATE_RAHEAD,
  82         ADA_STATE_WCACHE,
  83         ADA_STATE_LOGDIR,
  84         ADA_STATE_IDDIR,
  85         ADA_STATE_SUP_CAP,
  86         ADA_STATE_ZONE,
  87         ADA_STATE_NORMAL
  88 } ada_state;
  89
  90 typedef enum {
  91         ADA_FLAG_CAN_48BIT      = 0x00000002,
  92         ADA_FLAG_CAN_FLUSHCACHE = 0x00000004,
  93         ADA_FLAG_CAN_NCQ        = 0x00000008,
  94         ADA_FLAG_CAN_DMA        = 0x00000010,
  95         ADA_FLAG_NEED_OTAG      = 0x00000020,
  96         ADA_FLAG_WAS_OTAG       = 0x00000040,
  97         ADA_FLAG_CAN_TRIM       = 0x00000080,
  98         ADA_FLAG_OPEN           = 0x00000100,
  99         ADA_FLAG_SCTX_INIT      = 0x00000200,
 100         ADA_FLAG_CAN_CFA        = 0x00000400,
 101         ADA_FLAG_CAN_POWERMGT   = 0x00000800,
 102         ADA_FLAG_CAN_DMA48      = 0x00001000,
 103         ADA_FLAG_CAN_LOG        = 0x00002000,
 104         ADA_FLAG_CAN_IDLOG      = 0x00004000,
 105         ADA_FLAG_CAN_SUPCAP     = 0x00008000,
 106         ADA_FLAG_CAN_ZONE       = 0x00010000,
 107         ADA_FLAG_CAN_WCACHE     = 0x00020000,
 108         ADA_FLAG_CAN_RAHEAD     = 0x00040000,
 109         ADA_FLAG_PROBED         = 0x00080000,
 110         ADA_FLAG_ANNOUNCED      = 0x00100000,
 111         ADA_FLAG_DIRTY          = 0x00200000,
 112         ADA_FLAG_CAN_NCQ_TRIM   = 0x00400000,   /* CAN_TRIM also set */
 113         ADA_FLAG_PIM_ATA_EXT    = 0x00800000
 114 } ada_flags;
 115
 116 typedef enum {
 117         ADA_Q_NONE              = 0x00,
 118         ADA_Q_4K                = 0x01,
 119         ADA_Q_NCQ_TRIM_BROKEN   = 0x02,
 120         ADA_Q_LOG_BROKEN        = 0x04,
 121         ADA_Q_SMR_DM            = 0x08,
 122         ADA_Q_NO_TRIM           = 0x10,
 123         ADA_Q_128KB             = 0x20
 124 } ada_quirks;
 125
 126 #define ADA_Q_BIT_STRING        \
 127         "\020"                  \
 128         "\0014K"                \
 129         "\002NCQ_TRIM_BROKEN"   \
 130         "\003LOG_BROKEN"        \
 131         "\004SMR_DM"            \
 132         "\005NO_TRIM"           \
 133         "\006128KB"
 134
 135 typedef enum {
 136         ADA_CCB_RAHEAD          = 0x01,
 137         ADA_CCB_WCACHE          = 0x02,
 138         ADA_CCB_BUFFER_IO       = 0x03,
 139         ADA_CCB_DUMP            = 0x05,
 140         ADA_CCB_TRIM            = 0x06,
 141         ADA_CCB_LOGDIR          = 0x07,
 142         ADA_CCB_IDDIR           = 0x08,
 143         ADA_CCB_SUP_CAP         = 0x09,
 144         ADA_CCB_ZONE            = 0x0a,
 145         ADA_CCB_TYPE_MASK       = 0x0F,
 146 } ada_ccb_state;
 147
 148 typedef enum {
 149         ADA_ZONE_NONE           = 0x00,
 150         ADA_ZONE_DRIVE_MANAGED  = 0x01,
 151         ADA_ZONE_HOST_AWARE     = 0x02,
 152         ADA_ZONE_HOST_MANAGED   = 0x03
 153 } ada_zone_mode;
 154
 155 typedef enum {
 156         ADA_ZONE_FLAG_RZ_SUP            = 0x0001,
 157         ADA_ZONE_FLAG_OPEN_SUP          = 0x0002,
 158         ADA_ZONE_FLAG_CLOSE_SUP         = 0x0004,
 159         ADA_ZONE_FLAG_FINISH_SUP        = 0x0008,
 160         ADA_ZONE_FLAG_RWP_SUP           = 0x0010,
 161         ADA_ZONE_FLAG_SUP_MASK          = (ADA_ZONE_FLAG_RZ_SUP |
 162                                            ADA_ZONE_FLAG_OPEN_SUP |
 163                                            ADA_ZONE_FLAG_CLOSE_SUP |
 164                                            ADA_ZONE_FLAG_FINISH_SUP |
 165                                            ADA_ZONE_FLAG_RWP_SUP),
 166         ADA_ZONE_FLAG_URSWRZ            = 0x0020,
 167         ADA_ZONE_FLAG_OPT_SEQ_SET       = 0x0040,
 168         ADA_ZONE_FLAG_OPT_NONSEQ_SET    = 0x0080,
 169         ADA_ZONE_FLAG_MAX_SEQ_SET       = 0x0100,
 170         ADA_ZONE_FLAG_SET_MASK          = (ADA_ZONE_FLAG_OPT_SEQ_SET |
 171                                            ADA_ZONE_FLAG_OPT_NONSEQ_SET |
 172                                            ADA_ZONE_FLAG_MAX_SEQ_SET)
 173 } ada_zone_flags;
 174
 175 static struct ada_zone_desc {
 176         ada_zone_flags value;
 177         const char *desc;
 178 } ada_zone_desc_table[] = {
 179         {ADA_ZONE_FLAG_RZ_SUP, "Report Zones" },
 180         {ADA_ZONE_FLAG_OPEN_SUP, "Open" },
 181         {ADA_ZONE_FLAG_CLOSE_SUP, "Close" },
 182         {ADA_ZONE_FLAG_FINISH_SUP, "Finish" },
 183         {ADA_ZONE_FLAG_RWP_SUP, "Reset Write Pointer" },
 184 };
 185
 186
 187 /* Offsets into our private area for storing information */
 188 #define ccb_state       ppriv_field0
 189 #define ccb_bp          ppriv_ptr1
 190
 191 typedef enum {
 192         ADA_DELETE_NONE,
 193         ADA_DELETE_DISABLE,
 194         ADA_DELETE_CFA_ERASE,
 195         ADA_DELETE_DSM_TRIM,
 196         ADA_DELETE_NCQ_DSM_TRIM,
 197         ADA_DELETE_MIN = ADA_DELETE_CFA_ERASE,
 198         ADA_DELETE_MAX = ADA_DELETE_NCQ_DSM_TRIM,
 199 } ada_delete_methods;
 200
 201 static const char *ada_delete_method_names[] =
 202     { "NONE", "DISABLE", "CFA_ERASE", "DSM_TRIM", "NCQ_DSM_TRIM" };
 203 #if 0
 204 static const char *ada_delete_method_desc[] =
 205     { "NONE", "DISABLED", "CFA Erase", "DSM Trim", "DSM Trim via NCQ" };
 206 #endif
 207
 208 struct disk_params {
 209         u_int8_t  heads;
 210         u_int8_t  secs_per_track;
 211         u_int32_t cylinders;
 212         u_int32_t secsize;      /* Number of bytes/logical sector */
 213         u_int64_t sectors;      /* Total number sectors */
 214 };
 215
 216 #define TRIM_MAX_BLOCKS 8
 217 #define TRIM_MAX_RANGES (TRIM_MAX_BLOCKS * ATA_DSM_BLK_RANGES)
 218 struct trim_request {
 219         uint8_t         data[TRIM_MAX_RANGES * ATA_DSM_RANGE_SIZE];
 220         TAILQ_HEAD(, bio) bps;
 221 };
 222
 223 struct ada_softc {
 224         struct   cam_iosched_softc *cam_iosched;
 225         int      outstanding_cmds;      /* Number of active commands */
 226         int      refcount;              /* Active xpt_action() calls */
 227         ada_state state;
 228         ada_flags flags;
 229         ada_zone_mode zone_mode;
 230         ada_zone_flags zone_flags;
 231         struct ata_gp_log_dir ata_logdir;
 232         int valid_logdir_len;
 233         struct ata_identify_log_pages ata_iddir;
 234         int valid_iddir_len;
 235         uint64_t optimal_seq_zones;
 236         uint64_t optimal_nonseq_zones;
 237         uint64_t max_seq_zones;
 238         ada_quirks quirks;
 239         ada_delete_methods delete_method;
 240         int      trim_max_ranges;
 241         int      read_ahead;
 242         int      write_cache;
 243         int      unmappedio;
 244         int      rotating;
 245 #ifdef CAM_TEST_FAILURE
 246         int      force_read_error;
 247         int      force_write_error;
 248         int      periodic_read_error;
 249         int      periodic_read_count;
 250 #endif
 251         struct   disk_params params;
 252         struct   disk *disk;
 253         struct task             sysctl_task;
 254         struct sysctl_ctx_list  sysctl_ctx;
 255         struct sysctl_oid       *sysctl_tree;
 256         struct callout          sendordered_c;
 257         struct trim_request     trim_req;
 258         uint64_t                trim_count;
 259         uint64_t                trim_ranges;
 260         uint64_t                trim_lbas;
 261 #ifdef CAM_IO_STATS
 262         struct sysctl_ctx_list  sysctl_stats_ctx;
 263         struct sysctl_oid       *sysctl_stats_tree;
 264         u_int   timeouts;
 265         u_int   errors;
 266         u_int   invalidations;
 267 #endif
 268 #define ADA_ANNOUNCETMP_SZ 80
 269         char    announce_temp[ADA_ANNOUNCETMP_SZ];
 270 #define ADA_ANNOUNCE_SZ 400
 271         char    announce_buffer[ADA_ANNOUNCE_SZ];
 272 };
 273
 274 struct ada_quirk_entry {
 275         struct scsi_inquiry_pattern inq_pat;
 276         ada_quirks quirks;
 277 };
 278
 279 static struct ada_quirk_entry ada_quirk_table[] =
 280 {
 281         {
 282                 /* Sandisk X400 */
 283                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SanDisk?SD8SB8U1T00*", "X4162000*" },
 284                 /*quirks*/ADA_Q_128KB
 285         },
 286         {
 287                 /* Hitachi Advanced Format (4k) drives */
 288                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Hitachi H??????????E3*", "*" },
 289                 /*quirks*/ADA_Q_4K
 290         },
 291         {
 292                 /* Samsung Advanced Format (4k) drives */
 293                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG HD155UI*", "*" },
 294                 /*quirks*/ADA_Q_4K
 295         },
 296         {
 297                 /* Samsung Advanced Format (4k) drives */
 298                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG HD204UI*", "*" },
 299                 /*quirks*/ADA_Q_4K
 300         },
 301         {
 302                 /* Seagate Barracuda Green Advanced Format (4k) drives */
 303                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST????DL*", "*" },
 304                 /*quirks*/ADA_Q_4K
 305         },
 306         {
 307                 /* Seagate Barracuda Advanced Format (4k) drives */
 308                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST???DM*", "*" },
 309                 /*quirks*/ADA_Q_4K
 310         },
 311         {
 312                 /* Seagate Barracuda Advanced Format (4k) drives */
 313                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST????DM*", "*" },
 314                 /*quirks*/ADA_Q_4K
 315         },
 316         {
 317                 /* Seagate Momentus Advanced Format (4k) drives */
 318                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9500423AS*", "*" },
 319                 /*quirks*/ADA_Q_4K
 320         },
 321         {
 322                 /* Seagate Momentus Advanced Format (4k) drives */
 323                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9500424AS*", "*" },
 324                 /*quirks*/ADA_Q_4K
 325         },
 326         {
 327                 /* Seagate Momentus Advanced Format (4k) drives */
 328                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9640423AS*", "*" },
 329                 /*quirks*/ADA_Q_4K
 330         },
 331         {
 332                 /* Seagate Momentus Advanced Format (4k) drives */
 333                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9640424AS*", "*" },
 334                 /*quirks*/ADA_Q_4K
 335         },
 336         {
 337                 /* Seagate Momentus Advanced Format (4k) drives */
 338                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9750420AS*", "*" },
 339                 /*quirks*/ADA_Q_4K
 340         },
 341         {
 342                 /* Seagate Momentus Advanced Format (4k) drives */
 343                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9750422AS*", "*" },
 344                 /*quirks*/ADA_Q_4K
 345         },
 346         {
 347                 /* Seagate Momentus Advanced Format (4k) drives */
 348                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9750423AS*", "*" },
 349                 /*quirks*/ADA_Q_4K
 350         },
 351         {
 352                 /* Seagate Momentus Thin Advanced Format (4k) drives */
 353                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST???LT*", "*" },
 354                 /*quirks*/ADA_Q_4K
 355         },
 356         {
 357                 /* WDC Caviar Red Advanced Format (4k) drives */
 358                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD????CX*", "*" },
 359                 /*quirks*/ADA_Q_4K
 360         },
 361         {
 362                 /* WDC Caviar Green Advanced Format (4k) drives */
 363                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD????RS*", "*" },
 364                 /*quirks*/ADA_Q_4K
 365         },
 366         {
 367                 /* WDC Caviar Green/Red Advanced Format (4k) drives */
 368                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD????RX*", "*" },
 369                 /*quirks*/ADA_Q_4K
 370         },
 371         {
 372                 /* WDC Caviar Red Advanced Format (4k) drives */
 373                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD??????CX*", "*" },
 374                 /*quirks*/ADA_Q_4K
 375         },
 376         {
 377                 /* WDC Caviar Black Advanced Format (4k) drives */
 378                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD????AZEX*", "*" },
 379                 /*quirks*/ADA_Q_4K
 380         },
 381         {
 382                 /* WDC Caviar Black Advanced Format (4k) drives */
 383                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD????FZEX*", "*" },
 384                 /*quirks*/ADA_Q_4K
 385         },
 386         {
 387                 /* WDC Caviar Green Advanced Format (4k) drives */
 388                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD??????RS*", "*" },
 389                 /*quirks*/ADA_Q_4K
 390         },
 391         {
 392                 /* WDC Caviar Green Advanced Format (4k) drives */
 393                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD??????RX*", "*" },
 394                 /*quirks*/ADA_Q_4K
 395         },
 396         {
 397                 /* WDC Scorpio Black Advanced Format (4k) drives */
 398                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD???PKT*", "*" },
 399                 /*quirks*/ADA_Q_4K
 400         },
 401         {
 402                 /* WDC Scorpio Black Advanced Format (4k) drives */
 403                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD?????PKT*", "*" },
 404                 /*quirks*/ADA_Q_4K
 405         },
 406         {
 407                 /* WDC Scorpio Blue Advanced Format (4k) drives */
 408                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD???PVT*", "*" },
 409                 /*quirks*/ADA_Q_4K
 410         },
 411         {
 412                 /* WDC Scorpio Blue Advanced Format (4k) drives */
 413                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD?????PVT*", "*" },
 414                 /*quirks*/ADA_Q_4K
 415         },
 416         /* SSDs */
 417         {
 418                 /*
 419                  * Corsair Force 2 SSDs
 420                  * 4k optimised & trim only works in 4k requests + 4k aligned
 421                  */
 422                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair CSSD-F*", "*" },
 423                 /*quirks*/ADA_Q_4K
 424         },
 425         {
 426                 /*
 427                  * Corsair Force 3 SSDs
 428                  * 4k optimised & trim only works in 4k requests + 4k aligned
 429                  */
 430                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair Force 3*", "*" },
 431                 /*quirks*/ADA_Q_4K
 432         },
 433         {
 434                 /*
 435                  * Corsair Neutron GTX SSDs
 436                  * 4k optimised & trim only works in 4k requests + 4k aligned
 437                  */
 438                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair Neutron GTX*", "*" },
 439                 /*quirks*/ADA_Q_4K
 440         },
 441         {
 442                 /*
 443                  * Corsair Force GT & GS SSDs
 444                  * 4k optimised & trim only works in 4k requests + 4k aligned
 445                  */
 446                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair Force G*", "*" },
 447                 /*quirks*/ADA_Q_4K
 448         },
 449         {
 450                 /*
 451                  * Crucial M4 SSDs
 452                  * 4k optimised & trim only works in 4k requests + 4k aligned
 453                  */
 454                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "M4-CT???M4SSD2*", "*" },
 455                 /*quirks*/ADA_Q_4K
 456         },
 457         {
 458                 /*
 459                  * Crucial M500 SSDs MU07 firmware
 460                  * NCQ Trim works
 461                  */
 462                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Crucial CT*M500*", "MU07" },
 463                 /*quirks*/0
 464         },
 465         {
 466                 /*
 467                  * Crucial M500 SSDs all other firmware
 468                  * NCQ Trim doesn't work
 469                  */
 470                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Crucial CT*M500*", "*" },
 471                 /*quirks*/ADA_Q_NCQ_TRIM_BROKEN
 472         },
 473         {
 474                 /*
 475                  * Crucial M550 SSDs
 476                  * NCQ Trim doesn't work, but only on MU01 firmware
 477                  */
 478                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Crucial CT*M550*", "MU01" },
 479                 /*quirks*/ADA_Q_NCQ_TRIM_BROKEN
 480         },
 481         {
 482                 /*
 483                  * Crucial MX100 SSDs
 484                  * NCQ Trim doesn't work, but only on MU01 firmware
 485                  */
 486                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Crucial CT*MX100*", "MU01" },
 487                 /*quirks*/ADA_Q_NCQ_TRIM_BROKEN
 488         },
 489         {
 490                 /*
 491                  * Crucial RealSSD C300 SSDs
 492                  * 4k optimised
 493                  */
 494                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "C300-CTFDDAC???MAG*",
 495                 "*" }, /*quirks*/ADA_Q_4K
 496         },
 497         {
 498                 /*
 499                  * FCCT M500 SSDs
 500                  * NCQ Trim doesn't work
 501                  */
 502                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "FCCT*M500*", "*" },
 503                 /*quirks*/ADA_Q_NCQ_TRIM_BROKEN
 504         },
 505         {
 506                 /*
 507                  * Intel 320 Series SSDs
 508                  * 4k optimised & trim only works in 4k requests + 4k aligned
 509                  */
 510                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSA2CW*", "*" },
 511                 /*quirks*/ADA_Q_4K
 512         },
 513         {
 514                 /*
 515                  * Intel 330 Series SSDs
 516                  * 4k optimised & trim only works in 4k requests + 4k aligned
 517                  */
 518                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSC2CT*", "*" },
 519                 /*quirks*/ADA_Q_4K
 520         },
 521         {
 522                 /*
 523                  * Intel 510 Series SSDs
 524                  * 4k optimised & trim only works in 4k requests + 4k aligned
 525                  */
 526                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSC2MH*", "*" },
 527                 /*quirks*/ADA_Q_4K
 528         },
 529         {
 530                 /*
 531                  * Intel 520 Series SSDs
 532                  * 4k optimised & trim only works in 4k requests + 4k aligned
 533                  */
 534                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSC2BW*", "*" },
 535                 /*quirks*/ADA_Q_4K
 536         },
 537         {
 538                 /*
 539                  * Intel S3610 Series SSDs
 540                  * 4k optimised & trim only works in 4k requests + 4k aligned
 541                  */
 542                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSC2BX*", "*" },
 543                 /*quirks*/ADA_Q_4K
 544         },
 545         {
 546                 /*
 547                  * Intel X25-M Series SSDs
 548                  * 4k optimised & trim only works in 4k requests + 4k aligned
 549                  */
 550                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSA2M*", "*" },
 551                 /*quirks*/ADA_Q_4K
 552         },
 553         {
 554                 /*
 555                  * KingDian S200 60GB P0921B
 556                  * Trimming crash the SSD
 557                  */
 558                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "KingDian S200 *", "*" },
 559                 /*quirks*/ADA_Q_NO_TRIM
 560         },
 561         {
 562                 /*
 563                  * Kingston E100 Series SSDs
 564                  * 4k optimised & trim only works in 4k requests + 4k aligned
 565                  */
 566                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "KINGSTON SE100S3*", "*" },
 567                 /*quirks*/ADA_Q_4K
 568         },
 569         {
 570                 /*
 571                  * Kingston HyperX 3k SSDs
 572                  * 4k optimised & trim only works in 4k requests + 4k aligned
 573                  */
 574                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "KINGSTON SH103S3*", "*" },
 575                 /*quirks*/ADA_Q_4K
 576         },
 577         {
 578                 /*
 579                  * Marvell SSDs (entry taken from OpenSolaris)
 580                  * 4k optimised & trim only works in 4k requests + 4k aligned
 581                  */
 582                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "MARVELL SD88SA02*", "*" },
 583                 /*quirks*/ADA_Q_4K
 584         },
 585         {
 586                 /*
 587                  * Micron M500 SSDs firmware MU07
 588                  * NCQ Trim works?
 589                  */
 590                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Micron M500*", "MU07" },
 591                 /*quirks*/0
 592         },
 593         {
 594                 /*
 595                  * Micron M500 SSDs all other firmware
 596                  * NCQ Trim doesn't work
 597                  */
 598                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Micron M500*", "*" },
 599                 /*quirks*/ADA_Q_NCQ_TRIM_BROKEN
 600         },
 601         {
 602                 /*
 603                  * Micron M5[15]0 SSDs
 604                  * NCQ Trim doesn't work, but only MU01 firmware
 605                  */
 606                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Micron M5[15]0*", "MU01" },
 607                 /*quirks*/ADA_Q_NCQ_TRIM_BROKEN
 608         },
 609         {
 610                 /*
 611                  * Micron 5100 SSDs
 612                  * 4k optimised & trim only works in 4k requests + 4k aligned
 613                  */
 614                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Micron 5100 MTFDDAK*", "*" },
 615                 /*quirks*/ADA_Q_4K
 616         },
 617         {
 618                 /*
 619                  * OCZ Agility 2 SSDs
 620                  * 4k optimised & trim only works in 4k requests + 4k aligned
 621                  */
 622                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-AGILITY2*", "*" },
 623                 /*quirks*/ADA_Q_4K
 624         },
 625         {
 626                 /*
 627                  * OCZ Agility 3 SSDs
 628                  * 4k optimised & trim only works in 4k requests + 4k aligned
 629                  */
 630                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-AGILITY3*", "*" },
 631                 /*quirks*/ADA_Q_4K
 632         },
 633         {
 634                 /*
 635                  * OCZ Deneva R Series SSDs
 636                  * 4k optimised & trim only works in 4k requests + 4k aligned
 637                  */
 638                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "DENRSTE251M45*", "*" },
 639                 /*quirks*/ADA_Q_4K
 640         },
 641         {
 642                 /*
 643                  * OCZ Vertex 2 SSDs (inc pro series)
 644                  * 4k optimised & trim only works in 4k requests + 4k aligned
 645                  */
 646                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ?VERTEX2*", "*" },
 647                 /*quirks*/ADA_Q_4K
 648         },
 649         {
 650                 /*
 651                  * OCZ Vertex 3 SSDs
 652                  * 4k optimised & trim only works in 4k requests + 4k aligned
 653                  */
 654                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-VERTEX3*", "*" },
 655                 /*quirks*/ADA_Q_4K
 656         },
 657         {
 658                 /*
 659                  * OCZ Vertex 4 SSDs
 660                  * 4k optimised & trim only works in 4k requests + 4k aligned
 661                  */
 662                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-VERTEX4*", "*" },
 663                 /*quirks*/ADA_Q_4K
 664         },
 665         {
 666                 /*
 667                  * Samsung 750 SSDs
 668                  * 4k optimised, NCQ TRIM seems to work
 669                  */
 670                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Samsung SSD 750*", "*" },
 671                 /*quirks*/ADA_Q_4K
 672         },
 673         {
 674                 /*
 675                  * Samsung 830 Series SSDs
 676                  * 4k optimised, NCQ TRIM Broken (normal TRIM is fine)
 677                  */
 678                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG SSD 830 Series*", "*" },
 679                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 680         },
 681         {
 682                 /*
 683                  * Samsung 840 SSDs
 684                  * 4k optimised, NCQ TRIM Broken (normal TRIM is fine)
 685                  */
 686                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Samsung SSD 840*", "*" },
 687                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 688         },
 689         {
 690                 /*
 691                  * Samsung 845 SSDs
 692                  * 4k optimised, NCQ TRIM Broken (normal TRIM is fine)
 693                  */
 694                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Samsung SSD 845*", "*" },
 695                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 696         },
 697         {
 698                 /*
 699                  * Samsung 850 SSDs
 700                  * 4k optimised, NCQ TRIM broken (normal TRIM fine)
 701                  */
 702                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Samsung SSD 850*", "*" },
 703                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 704         },
 705         {
 706                 /*
 707                  * Samsung SM863 Series SSDs (MZ7KM*)
 708                  * 4k optimised, NCQ believed to be working
 709                  */
 710                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG MZ7KM*", "*" },
 711                 /*quirks*/ADA_Q_4K
 712         },
 713         {
 714                 /*
 715                  * Samsung 843T Series SSDs (MZ7WD*)
 716                  * Samsung PM851 Series SSDs (MZ7TE*)
 717                  * Samsung PM853T Series SSDs (MZ7GE*)
 718                  * 4k optimised, NCQ believed to be broken since these are
 719                  * appear to be built with the same controllers as the 840/850.
 720                  */
 721                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG MZ7*", "*" },
 722                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 723         },
 724         {
 725                 /*
 726                  * Same as for SAMSUNG MZ7* but enable the quirks for SSD
 727                  * starting with MZ7* too
 728                  */
 729                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "MZ7*", "*" },
 730                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 731         },
 732         {
 733                 /*
 734                  * Samsung PM851 Series SSDs Dell OEM
 735                  * device model          "SAMSUNG SSD PM851 mSATA 256GB"
 736                  * 4k optimised, NCQ broken
 737                  */
 738                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG SSD PM851*", "*" },
 739                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 740         },
 741         {
 742                 /*
 743                  * SuperTalent TeraDrive CT SSDs
 744                  * 4k optimised & trim only works in 4k requests + 4k aligned
 745                  */
 746                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "FTM??CT25H*", "*" },
 747                 /*quirks*/ADA_Q_4K
 748         },
 749         {
 750                 /*
 751                  * XceedIOPS SATA SSDs
 752                  * 4k optimised
 753                  */
 754                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SG9XCS2D*", "*" },
 755                 /*quirks*/ADA_Q_4K
 756         },
 757         {
 758                 /*
 759                  * Samsung drive that doesn't support READ LOG EXT or
 760                  * READ LOG DMA EXT, despite reporting that it does in
 761                  * ATA identify data:
 762                  * SAMSUNG HD200HJ KF100-06
 763                  */
 764                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG HD200*", "*" },
 765                 /*quirks*/ADA_Q_LOG_BROKEN
 766         },
 767         {
 768                 /*
 769                  * Samsung drive that doesn't support READ LOG EXT or
 770                  * READ LOG DMA EXT, despite reporting that it does in
 771                  * ATA identify data:
 772                  * SAMSUNG HD501LJ CR100-10
 773                  */
 774                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG HD501*", "*" },
 775                 /*quirks*/ADA_Q_LOG_BROKEN
 776         },
 777         {
 778                 /*
 779                  * Seagate Lamarr 8TB Shingled Magnetic Recording (SMR)
 780                  * Drive Managed SATA hard drive.  This drive doesn't report
 781                  * in firmware that it is a drive managed SMR drive.
 782                  */
 783                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST8000AS000[23]*", "*" },
 784                 /*quirks*/ADA_Q_SMR_DM
 785         },
 786         {
 787                 /* Default */
 788                 {
 789                   T_ANY, SIP_MEDIA_REMOVABLE|SIP_MEDIA_FIXED,
 790                   /*vendor*/"*", /*product*/"*", /*revision*/"*"
 791                 },
 792                 /*quirks*/0
 793         },
 794 };
 795
 796 static  disk_strategy_t adastrategy;
 797 static  dumper_t        adadump;
 798 static  periph_init_t   adainit;
 799 static  void            adadiskgonecb(struct disk *dp);
 800 static  periph_oninv_t  adaoninvalidate;
 801 static  periph_dtor_t   adacleanup;
 802 static  void            adaasync(void *callback_arg, u_int32_t code,
 803                                 struct cam_path *path, void *arg);
 804 static  int             adazonemodesysctl(SYSCTL_HANDLER_ARGS);
 805 static  int             adazonesupsysctl(SYSCTL_HANDLER_ARGS);
 806 static  void            adasysctlinit(void *context, int pending);
 807 static  int             adagetattr(struct bio *bp);
 808 static  void            adasetflags(struct ada_softc *softc,
 809                                     struct ccb_getdev *cgd);
 810 static  periph_ctor_t   adaregister;
 811 static  void            ada_dsmtrim(struct ada_softc *softc, struct bio *bp,
 812                                     struct ccb_ataio *ataio);
 813 static  void            ada_cfaerase(struct ada_softc *softc, struct bio *bp,
 814                                      struct ccb_ataio *ataio);
 815 static  int             ada_zone_bio_to_ata(int disk_zone_cmd);
 816 static  int             ada_zone_cmd(struct cam_periph *periph, union ccb *ccb,
 817                                      struct bio *bp, int *queue_ccb);
 818 static  periph_start_t  adastart;
 819 static  void            adaprobedone(struct cam_periph *periph, union ccb *ccb);
 820 static  void            adazonedone(struct cam_periph *periph, union ccb *ccb);
 821 static  void            adadone(struct cam_periph *periph,
 822                                union ccb *done_ccb);
 823 static  int             adaerror(union ccb *ccb, u_int32_t cam_flags,
 824                                 u_int32_t sense_flags);
 825 static void             adagetparams(struct cam_periph *periph,
 826                                 struct ccb_getdev *cgd);
 827 static timeout_t        adasendorderedtag;
 828 static void             adashutdown(void *arg, int howto);
 829 static void             adasuspend(void *arg);
 830 static void             adaresume(void *arg);
 831
 832 #ifndef ADA_DEFAULT_TIMEOUT
 833 #define ADA_DEFAULT_TIMEOUT 30  /* Timeout in seconds */
 834 #endif
 835
 836 #ifndef ADA_DEFAULT_RETRY
 837 #define ADA_DEFAULT_RETRY       4
 838 #endif
 839
 840 #ifndef ADA_DEFAULT_SEND_ORDERED
 841 #define ADA_DEFAULT_SEND_ORDERED        1
 842 #endif
 843
 844 #ifndef ADA_DEFAULT_SPINDOWN_SHUTDOWN
 845 #define ADA_DEFAULT_SPINDOWN_SHUTDOWN   1
 846 #endif
 847
 848 #ifndef ADA_DEFAULT_SPINDOWN_SUSPEND
 849 #define ADA_DEFAULT_SPINDOWN_SUSPEND    1
 850 #endif
 851
 852 #ifndef ADA_DEFAULT_READ_AHEAD
 853 #define ADA_DEFAULT_READ_AHEAD  1
 854 #endif
 855
 856 #ifndef ADA_DEFAULT_WRITE_CACHE
 857 #define ADA_DEFAULT_WRITE_CACHE 1
 858 #endif
 859
 860 #define ADA_RA  (softc->read_ahead >= 0 ? \
 861                  softc->read_ahead : ada_read_ahead)
 862 #define ADA_WC  (softc->write_cache >= 0 ? \
 863                  softc->write_cache : ada_write_cache)
 864
 865 /*
 866  * Most platforms map firmware geometry to actual, but some don't.  If
 867  * not overridden, default to nothing.
 868  */
 869 #ifndef ata_disk_firmware_geom_adjust
 870 #define ata_disk_firmware_geom_adjust(disk)
 871 #endif
 872
 873 static int ada_retry_count = ADA_DEFAULT_RETRY;
 874 static int ada_default_timeout = ADA_DEFAULT_TIMEOUT;
 875 static int ada_send_ordered = ADA_DEFAULT_SEND_ORDERED;
 876 static int ada_spindown_shutdown = ADA_DEFAULT_SPINDOWN_SHUTDOWN;
 877 static int ada_spindown_suspend = ADA_DEFAULT_SPINDOWN_SUSPEND;
 878 static int ada_read_ahead = ADA_DEFAULT_READ_AHEAD;
 879 static int ada_write_cache = ADA_DEFAULT_WRITE_CACHE;
 880
 881 static SYSCTL_NODE(_kern_cam, OID_AUTO, ada, CTLFLAG_RD, 0,
 882             "CAM Direct Access Disk driver");
 883 SYSCTL_INT(_kern_cam_ada, OID_AUTO, retry_count, CTLFLAG_RWTUN,
 884            &ada_retry_count, 0, "Normal I/O retry count");
 885 SYSCTL_INT(_kern_cam_ada, OID_AUTO, default_timeout, CTLFLAG_RWTUN,
 886            &ada_default_timeout, 0, "Normal I/O timeout (in seconds)");
 887 SYSCTL_INT(_kern_cam_ada, OID_AUTO, send_ordered, CTLFLAG_RWTUN,
 888            &ada_send_ordered, 0, "Send Ordered Tags");
 889 SYSCTL_INT(_kern_cam_ada, OID_AUTO, spindown_shutdown, CTLFLAG_RWTUN,
 890            &ada_spindown_shutdown, 0, "Spin down upon shutdown");
 891 SYSCTL_INT(_kern_cam_ada, OID_AUTO, spindown_suspend, CTLFLAG_RWTUN,
 892            &ada_spindown_suspend, 0, "Spin down upon suspend");
 893 SYSCTL_INT(_kern_cam_ada, OID_AUTO, read_ahead, CTLFLAG_RWTUN,
 894            &ada_read_ahead, 0, "Enable disk read-ahead");
 895 SYSCTL_INT(_kern_cam_ada, OID_AUTO, write_cache, CTLFLAG_RWTUN,
 896            &ada_write_cache, 0, "Enable disk write cache");
 897
 898 /*
 899  * ADA_ORDEREDTAG_INTERVAL determines how often, relative
 900  * to the default timeout, we check to see whether an ordered
 901  * tagged transaction is appropriate to prevent simple tag
 902  * starvation.  Since we'd like to ensure that there is at least
 903  * 1/2 of the timeout length left for a starved transaction to
 904  * complete after we've sent an ordered tag, we must poll at least
 905  * four times in every timeout period.  This takes care of the worst
 906  * case where a starved transaction starts during an interval that
 907  * meets the requirement "don't send an ordered tag" test so it takes
 908  * us two intervals to determine that a tag must be sent.
 909  */
 910 #ifndef ADA_ORDEREDTAG_INTERVAL
 911 #define ADA_ORDEREDTAG_INTERVAL 4
 912 #endif
 913
 914 static struct periph_driver adadriver =
 915 {
 916         adainit, "ada",
 917         TAILQ_HEAD_INITIALIZER(adadriver.units), /* generation */ 0
 918 };
 919
 920 static int adadeletemethodsysctl(SYSCTL_HANDLER_ARGS);
 921
 922 PERIPHDRIVER_DECLARE(ada, adadriver);
 923
 924 static MALLOC_DEFINE(M_ATADA, "ata_da", "ata_da buffers");
 925
 926 static int
 927 adaopen(struct disk *dp)
 928 {
 929         struct cam_periph *periph;
 930         struct ada_softc *softc;
 931         int error;
 932
 933         periph = (struct cam_periph *)dp->d_drv1;
 934         if (cam_periph_acquire(periph) != 0) {
 935                 return(ENXIO);
 936         }
 937
 938         cam_periph_lock(periph);
 939         if ((error = cam_periph_hold(periph, PRIBIO|PCATCH)) != 0) {
 940                 cam_periph_unlock(periph);
 941                 cam_periph_release(periph);
 942                 return (error);
 943         }
 944
 945         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE | CAM_DEBUG_PERIPH,
 946             ("adaopen\n"));
 947
 948         softc = (struct ada_softc *)periph->softc;
 949         softc->flags |= ADA_FLAG_OPEN;
 950
 951         cam_periph_unhold(periph);
 952         cam_periph_unlock(periph);
 953         return (0);
 954 }
 955
 956 static int
 957 adaclose(struct disk *dp)
 958 {
 959         struct  cam_periph *periph;
 960         struct  ada_softc *softc;
 961         union ccb *ccb;
 962         int error;
 963
 964         periph = (struct cam_periph *)dp->d_drv1;
 965         softc = (struct ada_softc *)periph->softc;
 966         cam_periph_lock(periph);
 967
 968         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE | CAM_DEBUG_PERIPH,
 969             ("adaclose\n"));
 970
 971         /* We only sync the cache if the drive is capable of it. */
 972         if ((softc->flags & ADA_FLAG_DIRTY) != 0 &&
 973             (softc->flags & ADA_FLAG_CAN_FLUSHCACHE) != 0 &&
 974             (periph->flags & CAM_PERIPH_INVALID) == 0 &&
 975             cam_periph_hold(periph, PRIBIO) == 0) {
 976
 977                 ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL);
 978                 cam_fill_ataio(&ccb->ataio,
 979                                     1,
 980                                     NULL,
 981                                     CAM_DIR_NONE,
 982                                     0,
 983                                     NULL,
 984                                     0,
 985                                     ada_default_timeout*1000);
 986
 987                 if (softc->flags & ADA_FLAG_CAN_48BIT)
 988                         ata_48bit_cmd(&ccb->ataio, ATA_FLUSHCACHE48, 0, 0, 0);
 989                 else
 990                         ata_28bit_cmd(&ccb->ataio, ATA_FLUSHCACHE, 0, 0, 0);
 991                 error = cam_periph_runccb(ccb, adaerror, /*cam_flags*/0,
 992                     /*sense_flags*/0, softc->disk->d_devstat);
 993
 994                 if (error != 0)
 995                         xpt_print(periph->path, "Synchronize cache failed\n");
 996                 softc->flags &= ~ADA_FLAG_DIRTY;
 997                 xpt_release_ccb(ccb);
 998                 cam_periph_unhold(periph);
 999         }
1000
1001         softc->flags &= ~ADA_FLAG_OPEN;
1002
1003         while (softc->refcount != 0)
1004                 cam_periph_sleep(periph, &softc->refcount, PRIBIO, "adaclose", 1);
1005         cam_periph_unlock(periph);
1006         cam_periph_release(periph);
1007         return (0);
1008 }
1009
1010 static void
1011 adaschedule(struct cam_periph *periph)
1012 {
1013         struct ada_softc *softc = (struct ada_softc *)periph->softc;
1014
1015         if (softc->state != ADA_STATE_NORMAL)
1016                 return;
1017
1018         cam_iosched_schedule(softc->cam_iosched, periph);
1019 }
1020
1021 /*
1022  * Actually translate the requested transfer into one the physical driver
1023  * can understand.  The transfer is described by a buf and will include
1024  * only one physical transfer.
1025  */
1026 static void
1027 adastrategy(struct bio *bp)
1028 {
1029         struct cam_periph *periph;
1030         struct ada_softc *softc;
1031
1032         periph = (struct cam_periph *)bp->bio_disk->d_drv1;
1033         softc = (struct ada_softc *)periph->softc;
1034
1035         cam_periph_lock(periph);
1036
1037         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("adastrategy(%p)\n", bp));
1038
1039         /*
1040          * If the device has been made invalid, error out
1041          */
1042         if ((periph->flags & CAM_PERIPH_INVALID) != 0) {
1043                 cam_periph_unlock(periph);
1044                 biofinish(bp, NULL, ENXIO);
1045                 return;
1046         }
1047
1048         /*
1049          * Zone commands must be ordered, because they can depend on the
1050          * effects of previously issued commands, and they may affect
1051          * commands after them.
1052          */
1053         if (bp->bio_cmd == BIO_ZONE)
1054                 bp->bio_flags |= BIO_ORDERED;
1055
1056         /*
1057          * Place it in the queue of disk activities for this disk
1058          */
1059         cam_iosched_queue_work(softc->cam_iosched, bp);
1060
1061         /*
1062          * Schedule ourselves for performing the work.
1063          */
1064         adaschedule(periph);
1065         cam_periph_unlock(periph);
1066
1067         return;
1068 }
1069
1070 static int
1071 adadump(void *arg, void *virtual, vm_offset_t physical, off_t offset, size_t length)
1072 {
1073         struct      cam_periph *periph;
1074         struct      ada_softc *softc;
1075         u_int       secsize;
1076         struct      ccb_ataio ataio;
1077         struct      disk *dp;
1078         uint64_t    lba;
1079         uint16_t    count;
1080         int         error = 0;
1081
1082         dp = arg;
1083         periph = dp->d_drv1;
1084         softc = (struct ada_softc *)periph->softc;
1085         secsize = softc->params.secsize;
1086         lba = offset / secsize;
1087         count = length / secsize;
1088         if ((periph->flags & CAM_PERIPH_INVALID) != 0)
1089                 return (ENXIO);
1090
1091         memset(&ataio, 0, sizeof(ataio));
1092         if (length > 0) {
1093                 xpt_setup_ccb(&ataio.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
1094                 ataio.ccb_h.ccb_state = ADA_CCB_DUMP;
1095                 cam_fill_ataio(&ataio,
1096                     0,
1097                     NULL,
1098                     CAM_DIR_OUT,
1099                     0,
1100                     (u_int8_t *) virtual,
1101                     length,
1102                     ada_default_timeout*1000);
1103                 if ((softc->flags & ADA_FLAG_CAN_48BIT) &&
1104                     (lba + count >= ATA_MAX_28BIT_LBA ||
1105                     count >= 256)) {
1106                         ata_48bit_cmd(&ataio, ATA_WRITE_DMA48,
1107                             0, lba, count);
1108                 } else {
1109                         ata_28bit_cmd(&ataio, ATA_WRITE_DMA,
1110                             0, lba, count);
1111                 }
1112                 error = cam_periph_runccb((union ccb *)&ataio, adaerror,
1113                     0, SF_NO_RECOVERY | SF_NO_RETRY, NULL);
1114                 if (error != 0)
1115                         printf("Aborting dump due to I/O error.\n");
1116
1117                 return (error);
1118         }
1119
1120         if (softc->flags & ADA_FLAG_CAN_FLUSHCACHE) {
1121                 xpt_setup_ccb(&ataio.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
1122
1123                 /*
1124                  * Tell the drive to flush its internal cache. if we
1125                  * can't flush in 5s we have big problems. No need to
1126                  * wait the default 60s to detect problems.
1127                  */
1128                 ataio.ccb_h.ccb_state = ADA_CCB_DUMP;
1129                 cam_fill_ataio(&ataio,
1130                                     0,
1131                                     NULL,
1132                                     CAM_DIR_NONE,
1133                                     0,
1134                                     NULL,
1135                                     0,
1136                                     5*1000);
1137
1138                 if (softc->flags & ADA_FLAG_CAN_48BIT)
1139                         ata_48bit_cmd(&ataio, ATA_FLUSHCACHE48, 0, 0, 0);
1140                 else
1141                         ata_28bit_cmd(&ataio, ATA_FLUSHCACHE, 0, 0, 0);
1142                 error = cam_periph_runccb((union ccb *)&ataio, adaerror,
1143                     0, SF_NO_RECOVERY | SF_NO_RETRY, NULL);
1144                 if (error != 0)
1145                         xpt_print(periph->path, "Synchronize cache failed\n");
1146         }
1147         return (error);
1148 }
1149
1150 static void
1151 adainit(void)
1152 {
1153         cam_status status;
1154
1155         /*
1156          * Install a global async callback.  This callback will
1157          * receive async callbacks like "new device found".
1158          */
1159         status = xpt_register_async(AC_FOUND_DEVICE, adaasync, NULL, NULL);
1160
1161         if (status != CAM_REQ_CMP) {
1162                 printf("ada: Failed to attach master async callback "
1163                        "due to status 0x%x!\n", status);
1164         } else if (ada_send_ordered) {
1165
1166                 /* Register our event handlers */
1167                 if ((EVENTHANDLER_REGISTER(power_suspend, adasuspend,
1168                                            NULL, EVENTHANDLER_PRI_LAST)) == NULL)
1169                     printf("adainit: power event registration failed!\n");
1170                 if ((EVENTHANDLER_REGISTER(power_resume, adaresume,
1171                                            NULL, EVENTHANDLER_PRI_LAST)) == NULL)
1172                     printf("adainit: power event registration failed!\n");
1173                 if ((EVENTHANDLER_REGISTER(shutdown_post_sync, adashutdown,
1174                                            NULL, SHUTDOWN_PRI_DEFAULT)) == NULL)
1175                     printf("adainit: shutdown event registration failed!\n");
1176         }
1177 }
1178
1179 /*
1180  * Callback from GEOM, called when it has finished cleaning up its
1181  * resources.
1182  */
1183 static void
1184 adadiskgonecb(struct disk *dp)
1185 {
1186         struct cam_periph *periph;
1187
1188         periph = (struct cam_periph *)dp->d_drv1;
1189
1190         cam_periph_release(periph);
1191 }
1192
1193 static void
1194 adaoninvalidate(struct cam_periph *periph)
1195 {
1196         struct ada_softc *softc;
1197
1198         softc = (struct ada_softc *)periph->softc;
1199
1200         /*
1201          * De-register any async callbacks.
1202          */
1203         xpt_register_async(0, adaasync, periph, periph->path);
1204 #ifdef CAM_IO_STATS
1205         softc->invalidations++;
1206 #endif
1207
1208         /*
1209          * Return all queued I/O with ENXIO.
1210          * XXX Handle any transactions queued to the card
1211          *     with XPT_ABORT_CCB.
1212          */
1213         cam_iosched_flush(softc->cam_iosched, NULL, ENXIO);
1214
1215         disk_gone(softc->disk);
1216 }
1217
1218 static void
1219 adacleanup(struct cam_periph *periph)
1220 {
1221         struct ada_softc *softc;
1222
1223         softc = (struct ada_softc *)periph->softc;
1224
1225         cam_periph_unlock(periph);
1226
1227         cam_iosched_fini(softc->cam_iosched);
1228
1229         /*
1230          * If we can't free the sysctl tree, oh well...
1231          */
1232         if ((softc->flags & ADA_FLAG_SCTX_INIT) != 0) {
1233 #ifdef CAM_IO_STATS
1234                 if (sysctl_ctx_free(&softc->sysctl_stats_ctx) != 0)
1235                         xpt_print(periph->path,
1236                             "can't remove sysctl stats context\n");
1237 #endif
1238                 if (sysctl_ctx_free(&softc->sysctl_ctx) != 0)
1239                         xpt_print(periph->path,
1240                             "can't remove sysctl context\n");
1241         }
1242
1243         disk_destroy(softc->disk);
1244         callout_drain(&softc->sendordered_c);
1245         free(softc, M_DEVBUF);
1246         cam_periph_lock(periph);
1247 }
1248
1249 static void
1250 adasetdeletemethod(struct ada_softc *softc)
1251 {
1252
1253         if (softc->flags & ADA_FLAG_CAN_NCQ_TRIM)
1254                 softc->delete_method = ADA_DELETE_NCQ_DSM_TRIM;
1255         else if (softc->flags & ADA_FLAG_CAN_TRIM)
1256                 softc->delete_method = ADA_DELETE_DSM_TRIM;
1257         else if ((softc->flags & ADA_FLAG_CAN_CFA) && !(softc->flags & ADA_FLAG_CAN_48BIT))
1258                 softc->delete_method = ADA_DELETE_CFA_ERASE;
1259         else
1260                 softc->delete_method = ADA_DELETE_NONE;
1261 }
1262
1263 static void
1264 adaasync(void *callback_arg, u_int32_t code,
1265         struct cam_path *path, void *arg)
1266 {
1267         struct ccb_getdev cgd;
1268         struct cam_periph *periph;
1269         struct ada_softc *softc;
1270
1271         periph = (struct cam_periph *)callback_arg;
1272         switch (code) {
1273         case AC_FOUND_DEVICE:
1274         {
1275                 struct ccb_getdev *cgd;
1276                 cam_status status;
1277
1278                 cgd = (struct ccb_getdev *)arg;
1279                 if (cgd == NULL)
1280                         break;
1281
1282                 if (cgd->protocol != PROTO_ATA)
1283                         break;
1284
1285                 /*
1286                  * Allocate a peripheral instance for
1287                  * this device and start the probe
1288                  * process.
1289                  */
1290                 status = cam_periph_alloc(adaregister, adaoninvalidate,
1291                                           adacleanup, adastart,
1292                                           "ada", CAM_PERIPH_BIO,
1293                                           path, adaasync,
1294                                           AC_FOUND_DEVICE, cgd);
1295
1296                 if (status != CAM_REQ_CMP
1297                  && status != CAM_REQ_INPROG)
1298                         printf("adaasync: Unable to attach to new device "
1299                                 "due to status 0x%x\n", status);
1300                 break;
1301         }
1302         case AC_GETDEV_CHANGED:
1303         {
1304                 softc = (struct ada_softc *)periph->softc;
1305                 xpt_setup_ccb(&cgd.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
1306                 cgd.ccb_h.func_code = XPT_GDEV_TYPE;
1307                 xpt_action((union ccb *)&cgd);
1308
1309                 /*
1310                  * Set/clear support flags based on the new Identify data.
1311                  */
1312                 adasetflags(softc, &cgd);
1313
1314                 cam_periph_async(periph, code, path, arg);
1315                 break;
1316         }
1317         case AC_ADVINFO_CHANGED:
1318         {
1319                 uintptr_t buftype;
1320
1321                 buftype = (uintptr_t)arg;
1322                 if (buftype == CDAI_TYPE_PHYS_PATH) {
1323                         struct ada_softc *softc;
1324
1325                         softc = periph->softc;
1326                         disk_attr_changed(softc->disk, "GEOM::physpath",
1327                                           M_NOWAIT);
1328                 }
1329                 break;
1330         }
1331         case AC_SENT_BDR:
1332         case AC_BUS_RESET:
1333         {
1334                 softc = (struct ada_softc *)periph->softc;
1335                 cam_periph_async(periph, code, path, arg);
1336                 if (softc->state != ADA_STATE_NORMAL)
1337                         break;
1338                 xpt_setup_ccb(&cgd.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
1339                 cgd.ccb_h.func_code = XPT_GDEV_TYPE;
1340                 xpt_action((union ccb *)&cgd);
1341                 if (ADA_RA >= 0 && softc->flags & ADA_FLAG_CAN_RAHEAD)
1342                         softc->state = ADA_STATE_RAHEAD;
1343                 else if (ADA_WC >= 0 && softc->flags & ADA_FLAG_CAN_WCACHE)
1344                         softc->state = ADA_STATE_WCACHE;
1345                 else if ((softc->flags & ADA_FLAG_CAN_LOG)
1346                       && (softc->zone_mode != ADA_ZONE_NONE))
1347                         softc->state = ADA_STATE_LOGDIR;
1348                 else
1349                     break;
1350                 if (cam_periph_acquire(periph) != 0)
1351                         softc->state = ADA_STATE_NORMAL;
1352                 else
1353                         xpt_schedule(periph, CAM_PRIORITY_DEV);
1354         }
1355         default:
1356                 cam_periph_async(periph, code, path, arg);
1357                 break;
1358         }
1359 }
1360
1361 static int
1362 adazonemodesysctl(SYSCTL_HANDLER_ARGS)
1363 {
1364         char tmpbuf[40];
1365         struct ada_softc *softc;
1366         int error;
1367
1368         softc = (struct ada_softc *)arg1;
1369
1370         switch (softc->zone_mode) {
1371         case ADA_ZONE_DRIVE_MANAGED:
1372                 snprintf(tmpbuf, sizeof(tmpbuf), "Drive Managed");
1373                 break;
1374         case ADA_ZONE_HOST_AWARE:
1375                 snprintf(tmpbuf, sizeof(tmpbuf), "Host Aware");
1376                 break;
1377         case ADA_ZONE_HOST_MANAGED:
1378                 snprintf(tmpbuf, sizeof(tmpbuf), "Host Managed");
1379                 break;
1380         case ADA_ZONE_NONE:
1381         default:
1382                 snprintf(tmpbuf, sizeof(tmpbuf), "Not Zoned");
1383                 break;
1384         }
1385
1386         error = sysctl_handle_string(oidp, tmpbuf, sizeof(tmpbuf), req);
1387
1388         return (error);
1389 }
1390
1391 static int
1392 adazonesupsysctl(SYSCTL_HANDLER_ARGS)
1393 {
1394         char tmpbuf[180];
1395         struct ada_softc *softc;
1396         struct sbuf sb;
1397         int error, first;
1398         unsigned int i;
1399
1400         softc = (struct ada_softc *)arg1;
1401
1402         error = 0;
1403         first = 1;
1404         sbuf_new(&sb, tmpbuf, sizeof(tmpbuf), 0);
1405
1406         for (i = 0; i < sizeof(ada_zone_desc_table) /
1407              sizeof(ada_zone_desc_table[0]); i++) {
1408                 if (softc->zone_flags & ada_zone_desc_table[i].value) {
1409                         if (first == 0)
1410                                 sbuf_printf(&sb, ", ");
1411                         else
1412                                 first = 0;
1413                         sbuf_cat(&sb, ada_zone_desc_table[i].desc);
1414                 }
1415         }
1416
1417         if (first == 1)
1418                 sbuf_printf(&sb, "None");
1419
1420         sbuf_finish(&sb);
1421
1422         error = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
1423
1424         return (error);
1425 }
1426
1427
1428 static void
1429 adasysctlinit(void *context, int pending)
1430 {
1431         struct cam_periph *periph;
1432         struct ada_softc *softc;
1433         char tmpstr[32], tmpstr2[16];
1434
1435         periph = (struct cam_periph *)context;
1436
1437         /* periph was held for us when this task was enqueued */
1438         if ((periph->flags & CAM_PERIPH_INVALID) != 0) {
1439                 cam_periph_release(periph);
1440                 return;
1441         }
1442
1443         softc = (struct ada_softc *)periph->softc;
1444         snprintf(tmpstr, sizeof(tmpstr), "CAM ADA unit %d",periph->unit_number);
1445         snprintf(tmpstr2, sizeof(tmpstr2), "%d", periph->unit_number);
1446
1447         sysctl_ctx_init(&softc->sysctl_ctx);
1448         softc->flags |= ADA_FLAG_SCTX_INIT;
1449         softc->sysctl_tree = SYSCTL_ADD_NODE_WITH_LABEL(&softc->sysctl_ctx,
1450                 SYSCTL_STATIC_CHILDREN(_kern_cam_ada), OID_AUTO, tmpstr2,
1451                 CTLFLAG_RD, 0, tmpstr, "device_index");
1452         if (softc->sysctl_tree == NULL) {
1453                 printf("adasysctlinit: unable to allocate sysctl tree\n");
1454                 cam_periph_release(periph);
1455                 return;
1456         }
1457
1458         SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1459                 OID_AUTO, "delete_method", CTLTYPE_STRING | CTLFLAG_RW,
1460                 softc, 0, adadeletemethodsysctl, "A",
1461                 "BIO_DELETE execution method");
1462         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
1463                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
1464                 "trim_count", CTLFLAG_RD, &softc->trim_count,
1465                 "Total number of dsm commands sent");
1466         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
1467                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
1468                 "trim_ranges", CTLFLAG_RD, &softc->trim_ranges,
1469                 "Total number of ranges in dsm commands");
1470         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
1471                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
1472                 "trim_lbas", CTLFLAG_RD, &softc->trim_lbas,
1473                 "Total lbas in the dsm commands sent");
1474         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1475                 OID_AUTO, "read_ahead", CTLFLAG_RW | CTLFLAG_MPSAFE,
1476                 &softc->read_ahead, 0, "Enable disk read ahead.");
1477         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1478                 OID_AUTO, "write_cache", CTLFLAG_RW | CTLFLAG_MPSAFE,
1479                 &softc->write_cache, 0, "Enable disk write cache.");
1480         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1481                 OID_AUTO, "unmapped_io", CTLFLAG_RD | CTLFLAG_MPSAFE,
1482                 &softc->unmappedio, 0, "Unmapped I/O leaf");
1483         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1484                 OID_AUTO, "rotating", CTLFLAG_RD | CTLFLAG_MPSAFE,
1485                 &softc->rotating, 0, "Rotating media");
1486         SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1487                 OID_AUTO, "zone_mode", CTLTYPE_STRING | CTLFLAG_RD,
1488                 softc, 0, adazonemodesysctl, "A",
1489                 "Zone Mode");
1490         SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1491                 OID_AUTO, "zone_support", CTLTYPE_STRING | CTLFLAG_RD,
1492                 softc, 0, adazonesupsysctl, "A",
1493                 "Zone Support");
1494         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
1495                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
1496                 "optimal_seq_zones", CTLFLAG_RD, &softc->optimal_seq_zones,
1497                 "Optimal Number of Open Sequential Write Preferred Zones");
1498         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
1499                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
1500                 "optimal_nonseq_zones", CTLFLAG_RD,
1501                 &softc->optimal_nonseq_zones,
1502                 "Optimal Number of Non-Sequentially Written Sequential Write "
1503                 "Preferred Zones");
1504         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
1505                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
1506                 "max_seq_zones", CTLFLAG_RD, &softc->max_seq_zones,
1507                 "Maximum Number of Open Sequential Write Required Zones");
1508
1509 #ifdef CAM_TEST_FAILURE
1510         /*
1511          * Add a 'door bell' sysctl which allows one to set it from userland
1512          * and cause something bad to happen.  For the moment, we only allow
1513          * whacking the next read or write.
1514          */
1515         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1516                 OID_AUTO, "force_read_error", CTLFLAG_RW | CTLFLAG_MPSAFE,
1517                 &softc->force_read_error, 0,
1518                 "Force a read error for the next N reads.");
1519         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1520                 OID_AUTO, "force_write_error", CTLFLAG_RW | CTLFLAG_MPSAFE,
1521                 &softc->force_write_error, 0,
1522                 "Force a write error for the next N writes.");
1523         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1524                 OID_AUTO, "periodic_read_error", CTLFLAG_RW | CTLFLAG_MPSAFE,
1525                 &softc->periodic_read_error, 0,
1526                 "Force a read error every N reads (don't set too low).");
1527         SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1528                 OID_AUTO, "invalidate", CTLTYPE_U64 | CTLFLAG_RW | CTLFLAG_MPSAFE,
1529                 periph, 0, cam_periph_invalidate_sysctl, "I",
1530                 "Write 1 to invalidate the drive immediately");
1531 #endif
1532
1533 #ifdef CAM_IO_STATS
1534         softc->sysctl_stats_tree = SYSCTL_ADD_NODE(&softc->sysctl_stats_ctx,
1535                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO, "stats",
1536                 CTLFLAG_RD, 0, "Statistics");
1537         SYSCTL_ADD_INT(&softc->sysctl_stats_ctx,
1538                 SYSCTL_CHILDREN(softc->sysctl_stats_tree),
1539                 OID_AUTO, "timeouts", CTLFLAG_RD | CTLFLAG_MPSAFE,
1540                 &softc->timeouts, 0,
1541                 "Device timeouts reported by the SIM");
1542         SYSCTL_ADD_INT(&softc->sysctl_stats_ctx,
1543                 SYSCTL_CHILDREN(softc->sysctl_stats_tree),
1544                 OID_AUTO, "errors", CTLFLAG_RD | CTLFLAG_MPSAFE,
1545                 &softc->errors, 0,
1546                 "Transport errors reported by the SIM.");
1547         SYSCTL_ADD_INT(&softc->sysctl_stats_ctx,
1548                 SYSCTL_CHILDREN(softc->sysctl_stats_tree),
1549                 OID_AUTO, "pack_invalidations", CTLFLAG_RD | CTLFLAG_MPSAFE,
1550                 &softc->invalidations, 0,
1551                 "Device pack invalidations.");
1552 #endif
1553
1554         cam_iosched_sysctl_init(softc->cam_iosched, &softc->sysctl_ctx,
1555             softc->sysctl_tree);
1556
1557         cam_periph_release(periph);
1558 }
1559
1560 static int
1561 adagetattr(struct bio *bp)
1562 {
1563         int ret;
1564         struct cam_periph *periph;
1565
1566         periph = (struct cam_periph *)bp->bio_disk->d_drv1;
1567         cam_periph_lock(periph);
1568         ret = xpt_getattr(bp->bio_data, bp->bio_length, bp->bio_attribute,
1569             periph->path);
1570         cam_periph_unlock(periph);
1571         if (ret == 0)
1572                 bp->bio_completed = bp->bio_length;
1573         return ret;
1574 }
1575
1576 static int
1577 adadeletemethodsysctl(SYSCTL_HANDLER_ARGS)
1578 {
1579         char buf[16];
1580         const char *p;
1581         struct ada_softc *softc;
1582         int i, error, value, methods;
1583
1584         softc = (struct ada_softc *)arg1;
1585
1586         value = softc->delete_method;
1587         if (value < 0 || value > ADA_DELETE_MAX)
1588                 p = "UNKNOWN";
1589         else
1590                 p = ada_delete_method_names[value];
1591         strncpy(buf, p, sizeof(buf));
1592         error = sysctl_handle_string(oidp, buf, sizeof(buf), req);
1593         if (error != 0 || req->newptr == NULL)
1594                 return (error);
1595         methods = 1 << ADA_DELETE_DISABLE;
1596         if ((softc->flags & ADA_FLAG_CAN_CFA) &&
1597             !(softc->flags & ADA_FLAG_CAN_48BIT))
1598                 methods |= 1 << ADA_DELETE_CFA_ERASE;
1599         if (softc->flags & ADA_FLAG_CAN_TRIM)
1600                 methods |= 1 << ADA_DELETE_DSM_TRIM;
1601         if (softc->flags & ADA_FLAG_CAN_NCQ_TRIM)
1602                 methods |= 1 << ADA_DELETE_NCQ_DSM_TRIM;
1603         for (i = 0; i <= ADA_DELETE_MAX; i++) {
1604                 if (!(methods & (1 << i)) ||
1605                     strcmp(buf, ada_delete_method_names[i]) != 0)
1606                         continue;
1607                 softc->delete_method = i;
1608                 return (0);
1609         }
1610         return (EINVAL);
1611 }
1612
1613 static void
1614 adasetflags(struct ada_softc *softc, struct ccb_getdev *cgd)
1615 {
1616         if ((cgd->ident_data.capabilities1 & ATA_SUPPORT_DMA) &&
1617             (cgd->inq_flags & SID_DMA))
1618                 softc->flags |= ADA_FLAG_CAN_DMA;
1619         else
1620                 softc->flags &= ~ADA_FLAG_CAN_DMA;
1621
1622         if (cgd->ident_data.support.command2 & ATA_SUPPORT_ADDRESS48) {
1623                 softc->flags |= ADA_FLAG_CAN_48BIT;
1624                 if (cgd->inq_flags & SID_DMA48)
1625                         softc->flags |= ADA_FLAG_CAN_DMA48;
1626                 else
1627                         softc->flags &= ~ADA_FLAG_CAN_DMA48;
1628         } else
1629                 softc->flags &= ~(ADA_FLAG_CAN_48BIT | ADA_FLAG_CAN_DMA48);
1630
1631         if (cgd->ident_data.support.command2 & ATA_SUPPORT_FLUSHCACHE)
1632                 softc->flags |= ADA_FLAG_CAN_FLUSHCACHE;
1633         else
1634                 softc->flags &= ~ADA_FLAG_CAN_FLUSHCACHE;
1635
1636         if (cgd->ident_data.support.command1 & ATA_SUPPORT_POWERMGT)
1637                 softc->flags |= ADA_FLAG_CAN_POWERMGT;
1638         else
1639                 softc->flags &= ~ADA_FLAG_CAN_POWERMGT;
1640
1641         if ((cgd->ident_data.satacapabilities & ATA_SUPPORT_NCQ) &&
1642             (cgd->inq_flags & SID_DMA) && (cgd->inq_flags & SID_CmdQue))
1643                 softc->flags |= ADA_FLAG_CAN_NCQ;
1644         else
1645                 softc->flags &= ~ADA_FLAG_CAN_NCQ;
1646
1647         if ((cgd->ident_data.support_dsm & ATA_SUPPORT_DSM_TRIM) &&
1648             (cgd->inq_flags & SID_DMA)) {
1649                 softc->flags |= ADA_FLAG_CAN_TRIM;
1650                 softc->trim_max_ranges = TRIM_MAX_RANGES;
1651                 if (cgd->ident_data.max_dsm_blocks != 0) {
1652                         softc->trim_max_ranges =
1653                             min(cgd->ident_data.max_dsm_blocks *
1654                                 ATA_DSM_BLK_RANGES, softc->trim_max_ranges);
1655                 }
1656                 /*
1657                  * If we can do RCVSND_FPDMA_QUEUED commands, we may be able
1658                  * to do NCQ trims, if we support trims at all. We also need
1659                  * support from the SIM to do things properly. Perhaps we
1660                  * should look at log 13 dword 0 bit 0 and dword 1 bit 0 are
1661                  * set too...
1662                  */
1663                 if ((softc->quirks & ADA_Q_NCQ_TRIM_BROKEN) == 0 &&
1664                     (softc->flags & ADA_FLAG_PIM_ATA_EXT) != 0 &&
1665                     (cgd->ident_data.satacapabilities2 &
1666                      ATA_SUPPORT_RCVSND_FPDMA_QUEUED) != 0 &&
1667                     (softc->flags & ADA_FLAG_CAN_TRIM) != 0)
1668                         softc->flags |= ADA_FLAG_CAN_NCQ_TRIM;
1669                 else
1670                         softc->flags &= ~ADA_FLAG_CAN_NCQ_TRIM;
1671         } else
1672                 softc->flags &= ~(ADA_FLAG_CAN_TRIM | ADA_FLAG_CAN_NCQ_TRIM);
1673
1674         if (cgd->ident_data.support.command2 & ATA_SUPPORT_CFA)
1675                 softc->flags |= ADA_FLAG_CAN_CFA;
1676         else
1677                 softc->flags &= ~ADA_FLAG_CAN_CFA;
1678
1679         /*
1680          * Now that we've set the appropriate flags, setup the delete
1681          * method.
1682          */
1683         adasetdeletemethod(softc);
1684
1685         if ((cgd->ident_data.support.extension & ATA_SUPPORT_GENLOG)
1686          && ((softc->quirks & ADA_Q_LOG_BROKEN) == 0))
1687                 softc->flags |= ADA_FLAG_CAN_LOG;
1688         else
1689                 softc->flags &= ~ADA_FLAG_CAN_LOG;
1690
1691         if ((cgd->ident_data.support3 & ATA_SUPPORT_ZONE_MASK) ==
1692              ATA_SUPPORT_ZONE_HOST_AWARE)
1693                 softc->zone_mode = ADA_ZONE_HOST_AWARE;
1694         else if (((cgd->ident_data.support3 & ATA_SUPPORT_ZONE_MASK) ==
1695                    ATA_SUPPORT_ZONE_DEV_MANAGED)
1696               || (softc->quirks & ADA_Q_SMR_DM))
1697                 softc->zone_mode = ADA_ZONE_DRIVE_MANAGED;
1698         else
1699                 softc->zone_mode = ADA_ZONE_NONE;
1700
1701         if (cgd->ident_data.support.command1 & ATA_SUPPORT_LOOKAHEAD)
1702                 softc->flags |= ADA_FLAG_CAN_RAHEAD;
1703         else
1704                 softc->flags &= ~ADA_FLAG_CAN_RAHEAD;
1705
1706         if (cgd->ident_data.support.command1 & ATA_SUPPORT_WRITECACHE)
1707                 softc->flags |= ADA_FLAG_CAN_WCACHE;
1708         else
1709                 softc->flags &= ~ADA_FLAG_CAN_WCACHE;
1710 }
1711
1712 static cam_status
1713 adaregister(struct cam_periph *periph, void *arg)
1714 {
1715         struct ada_softc *softc;
1716         struct ccb_pathinq cpi;
1717         struct ccb_getdev *cgd;
1718         struct disk_params *dp;
1719         struct sbuf sb;
1720         char   *announce_buf;
1721         caddr_t match;
1722         u_int maxio;
1723         int quirks;
1724
1725         cgd = (struct ccb_getdev *)arg;
1726         if (cgd == NULL) {
1727                 printf("adaregister: no getdev CCB, can't register device\n");
1728                 return(CAM_REQ_CMP_ERR);
1729         }
1730
1731         softc = (struct ada_softc *)malloc(sizeof(*softc), M_DEVBUF,
1732             M_NOWAIT|M_ZERO);
1733
1734         if (softc == NULL) {
1735                 printf("adaregister: Unable to probe new device. "
1736                     "Unable to allocate softc\n");
1737                 return(CAM_REQ_CMP_ERR);
1738         }
1739
1740         announce_buf = softc->announce_temp;
1741         bzero(announce_buf, ADA_ANNOUNCETMP_SZ);
1742
1743         if (cam_iosched_init(&softc->cam_iosched, periph) != 0) {
1744                 printf("adaregister: Unable to probe new device. "
1745                        "Unable to allocate iosched memory\n");
1746                 free(softc, M_DEVBUF);
1747                 return(CAM_REQ_CMP_ERR);
1748         }
1749
1750         periph->softc = softc;
1751
1752         /*
1753          * See if this device has any quirks.
1754          */
1755         match = cam_quirkmatch((caddr_t)&cgd->ident_data,
1756                                (caddr_t)ada_quirk_table,
1757                                nitems(ada_quirk_table),
1758                                sizeof(*ada_quirk_table), ata_identify_match);
1759         if (match != NULL)
1760                 softc->quirks = ((struct ada_quirk_entry *)match)->quirks;
1761         else
1762                 softc->quirks = ADA_Q_NONE;
1763
1764         xpt_path_inq(&cpi, periph->path);
1765
1766         TASK_INIT(&softc->sysctl_task, 0, adasysctlinit, periph);
1767
1768         /*
1769          * Register this media as a disk
1770          */
1771         (void)cam_periph_hold(periph, PRIBIO);
1772         cam_periph_unlock(periph);
1773         snprintf(announce_buf, ADA_ANNOUNCETMP_SZ,
1774             "kern.cam.ada.%d.quirks", periph->unit_number);
1775         quirks = softc->quirks;
1776         TUNABLE_INT_FETCH(announce_buf, &quirks);
1777         softc->quirks = quirks;
1778         softc->read_ahead = -1;
1779         snprintf(announce_buf, ADA_ANNOUNCETMP_SZ,
1780             "kern.cam.ada.%d.read_ahead", periph->unit_number);
1781         TUNABLE_INT_FETCH(announce_buf, &softc->read_ahead);
1782         softc->write_cache = -1;
1783         snprintf(announce_buf, ADA_ANNOUNCETMP_SZ,
1784             "kern.cam.ada.%d.write_cache", periph->unit_number);
1785         TUNABLE_INT_FETCH(announce_buf, &softc->write_cache);
1786
1787         /*
1788          * Set support flags based on the Identify data and quirks.
1789          */
1790         adasetflags(softc, cgd);
1791
1792         /* Disable queue sorting for non-rotational media by default. */
1793         if (cgd->ident_data.media_rotation_rate == ATA_RATE_NON_ROTATING) {
1794                 softc->rotating = 0;
1795         } else {
1796                 softc->rotating = 1;
1797         }
1798         cam_iosched_set_sort_queue(softc->cam_iosched,  softc->rotating ? -1 : 0);
1799         adagetparams(periph, cgd);
1800         softc->disk = disk_alloc();
1801         softc->disk->d_rotation_rate = cgd->ident_data.media_rotation_rate;
1802         softc->disk->d_devstat = devstat_new_entry(periph->periph_name,
1803                           periph->unit_number, softc->params.secsize,
1804                           DEVSTAT_ALL_SUPPORTED,
1805                           DEVSTAT_TYPE_DIRECT |
1806                           XPORT_DEVSTAT_TYPE(cpi.transport),
1807                           DEVSTAT_PRIORITY_DISK);
1808         softc->disk->d_open = adaopen;
1809         softc->disk->d_close = adaclose;
1810         softc->disk->d_strategy = adastrategy;
1811         softc->disk->d_getattr = adagetattr;
1812         softc->disk->d_dump = adadump;
1813         softc->disk->d_gone = adadiskgonecb;
1814         softc->disk->d_name = "ada";
1815         softc->disk->d_drv1 = periph;
1816         maxio = cpi.maxio;              /* Honor max I/O size of SIM */
1817         if (maxio == 0)
1818                 maxio = DFLTPHYS;       /* traditional default */
1819         else if (maxio > MAXPHYS)
1820                 maxio = MAXPHYS;        /* for safety */
1821         if (softc->flags & ADA_FLAG_CAN_48BIT)
1822                 maxio = min(maxio, 65536 * softc->params.secsize);
1823         else                                    /* 28bit ATA command limit */
1824                 maxio = min(maxio, 256 * softc->params.secsize);
1825         if (softc->quirks & ADA_Q_128KB)
1826                 maxio = min(maxio, 128 * 1024);
1827         softc->disk->d_maxsize = maxio;
1828         softc->disk->d_unit = periph->unit_number;
1829         softc->disk->d_flags = DISKFLAG_DIRECT_COMPLETION | DISKFLAG_CANZONE;
1830         if (softc->flags & ADA_FLAG_CAN_FLUSHCACHE)
1831                 softc->disk->d_flags |= DISKFLAG_CANFLUSHCACHE;
1832         /* Device lies about TRIM capability. */
1833         if ((softc->quirks & ADA_Q_NO_TRIM) &&
1834             (softc->flags & ADA_FLAG_CAN_TRIM))
1835                 softc->flags &= ~ADA_FLAG_CAN_TRIM;
1836         if (softc->flags & ADA_FLAG_CAN_TRIM) {
1837                 softc->disk->d_flags |= DISKFLAG_CANDELETE;
1838                 softc->disk->d_delmaxsize = softc->params.secsize *
1839                                             ATA_DSM_RANGE_MAX *
1840                                             softc->trim_max_ranges;
1841         } else if ((softc->flags & ADA_FLAG_CAN_CFA) &&
1842             !(softc->flags & ADA_FLAG_CAN_48BIT)) {
1843                 softc->disk->d_flags |= DISKFLAG_CANDELETE;
1844                 softc->disk->d_delmaxsize = 256 * softc->params.secsize;
1845         } else
1846                 softc->disk->d_delmaxsize = maxio;
1847         if ((cpi.hba_misc & PIM_UNMAPPED) != 0) {
1848                 softc->disk->d_flags |= DISKFLAG_UNMAPPED_BIO;
1849                 softc->unmappedio = 1;
1850         }
1851         if (cpi.hba_misc & PIM_ATA_EXT)
1852                 softc->flags |= ADA_FLAG_PIM_ATA_EXT;
1853         strlcpy(softc->disk->d_descr, cgd->ident_data.model,
1854             MIN(sizeof(softc->disk->d_descr), sizeof(cgd->ident_data.model)));
1855         strlcpy(softc->disk->d_ident, cgd->ident_data.serial,
1856             MIN(sizeof(softc->disk->d_ident), sizeof(cgd->ident_data.serial)));
1857         softc->disk->d_hba_vendor = cpi.hba_vendor;
1858         softc->disk->d_hba_device = cpi.hba_device;
1859         softc->disk->d_hba_subvendor = cpi.hba_subvendor;
1860         softc->disk->d_hba_subdevice = cpi.hba_subdevice;
1861
1862         softc->disk->d_sectorsize = softc->params.secsize;
1863         softc->disk->d_mediasize = (off_t)softc->params.sectors *
1864             softc->params.secsize;
1865         if (ata_physical_sector_size(&cgd->ident_data) !=
1866             softc->params.secsize) {
1867                 softc->disk->d_stripesize =
1868                     ata_physical_sector_size(&cgd->ident_data);
1869                 softc->disk->d_stripeoffset = (softc->disk->d_stripesize -
1870                     ata_logical_sector_offset(&cgd->ident_data)) %
1871                     softc->disk->d_stripesize;
1872         } else if (softc->quirks & ADA_Q_4K) {
1873                 softc->disk->d_stripesize = 4096;
1874                 softc->disk->d_stripeoffset = 0;
1875         }
1876         softc->disk->d_fwsectors = softc->params.secs_per_track;
1877         softc->disk->d_fwheads = softc->params.heads;
1878         ata_disk_firmware_geom_adjust(softc->disk);
1879
1880         /*
1881          * Acquire a reference to the periph before we register with GEOM.
1882          * We'll release this reference once GEOM calls us back (via
1883          * adadiskgonecb()) telling us that our provider has been freed.
1884          */
1885         if (cam_periph_acquire(periph) != 0) {
1886                 xpt_print(periph->path, "%s: lost periph during "
1887                           "registration!\n", __func__);
1888                 cam_periph_lock(periph);
1889                 return (CAM_REQ_CMP_ERR);
1890         }
1891         disk_create(softc->disk, DISK_VERSION);
1892         cam_periph_lock(periph);
1893
1894         dp = &softc->params;
1895         snprintf(announce_buf, ADA_ANNOUNCETMP_SZ,
1896             "%juMB (%ju %u byte sectors)",
1897             ((uintmax_t)dp->secsize * dp->sectors) / (1024 * 1024),
1898             (uintmax_t)dp->sectors, dp->secsize);
1899
1900         sbuf_new(&sb, softc->announce_buffer, ADA_ANNOUNCE_SZ, SBUF_FIXEDLEN);
1901         xpt_announce_periph_sbuf(periph, &sb, announce_buf);
1902         xpt_announce_quirks_sbuf(periph, &sb, softc->quirks, ADA_Q_BIT_STRING);
1903         sbuf_finish(&sb);
1904         sbuf_putbuf(&sb);
1905
1906         /*
1907          * Create our sysctl variables, now that we know
1908          * we have successfully attached.
1909          */
1910         if (cam_periph_acquire(periph) == 0)
1911                 taskqueue_enqueue(taskqueue_thread, &softc->sysctl_task);
1912
1913         /*
1914          * Add async callbacks for bus reset and
1915          * bus device reset calls.  I don't bother
1916          * checking if this fails as, in most cases,
1917          * the system will function just fine without
1918          * them and the only alternative would be to
1919          * not attach the device on failure.
1920          */
1921         xpt_register_async(AC_SENT_BDR | AC_BUS_RESET | AC_LOST_DEVICE |
1922             AC_GETDEV_CHANGED | AC_ADVINFO_CHANGED,
1923             adaasync, periph, periph->path);
1924
1925         /*
1926          * Schedule a periodic event to occasionally send an
1927          * ordered tag to a device.
1928          */
1929         callout_init_mtx(&softc->sendordered_c, cam_periph_mtx(periph), 0);
1930         callout_reset(&softc->sendordered_c,
1931             (ada_default_timeout * hz) / ADA_ORDEREDTAG_INTERVAL,
1932             adasendorderedtag, softc);
1933
1934         if (ADA_RA >= 0 && softc->flags & ADA_FLAG_CAN_RAHEAD) {
1935                 softc->state = ADA_STATE_RAHEAD;
1936         } else if (ADA_WC >= 0 && softc->flags & ADA_FLAG_CAN_WCACHE) {
1937                 softc->state = ADA_STATE_WCACHE;
1938         } else if ((softc->flags & ADA_FLAG_CAN_LOG)
1939                 && (softc->zone_mode != ADA_ZONE_NONE)) {
1940                 softc->state = ADA_STATE_LOGDIR;
1941         } else {
1942                 /*
1943                  * Nothing to probe, so we can just transition to the
1944                  * normal state.
1945                  */
1946                 adaprobedone(periph, NULL);
1947                 return(CAM_REQ_CMP);
1948         }
1949
1950         xpt_schedule(periph, CAM_PRIORITY_DEV);
1951
1952         return(CAM_REQ_CMP);
1953 }
1954
1955 static int
1956 ada_dsmtrim_req_create(struct ada_softc *softc, struct bio *bp, struct trim_request *req)
1957 {
1958         uint64_t lastlba = (uint64_t)-1, lbas = 0;
1959         int c, lastcount = 0, off, ranges = 0;
1960
1961         bzero(req, sizeof(*req));
1962         TAILQ_INIT(&req->bps);
1963         do {
1964                 uint64_t lba = bp->bio_pblkno;
1965                 int count = bp->bio_bcount / softc->params.secsize;
1966
1967                 /* Try to extend the previous range. */
1968                 if (lba == lastlba) {
1969                         c = min(count, ATA_DSM_RANGE_MAX - lastcount);
1970                         lastcount += c;
1971                         off = (ranges - 1) * ATA_DSM_RANGE_SIZE;
1972                         req->data[off + 6] = lastcount & 0xff;
1973                         req->data[off + 7] =
1974                                 (lastcount >> 8) & 0xff;
1975                         count -= c;
1976                         lba += c;
1977                         lbas += c;
1978                 }
1979
1980                 while (count > 0) {
1981                         c = min(count, ATA_DSM_RANGE_MAX);
1982                         off = ranges * ATA_DSM_RANGE_SIZE;
1983                         req->data[off + 0] = lba & 0xff;
1984                         req->data[off + 1] = (lba >> 8) & 0xff;
1985                         req->data[off + 2] = (lba >> 16) & 0xff;
1986                         req->data[off + 3] = (lba >> 24) & 0xff;
1987                         req->data[off + 4] = (lba >> 32) & 0xff;
1988                         req->data[off + 5] = (lba >> 40) & 0xff;
1989                         req->data[off + 6] = c & 0xff;
1990                         req->data[off + 7] = (c >> 8) & 0xff;
1991                         lba += c;
1992                         lbas += c;
1993                         count -= c;
1994                         lastcount = c;
1995                         ranges++;
1996                         /*
1997                          * Its the caller's responsibility to ensure the
1998                          * request will fit so we don't need to check for
1999                          * overrun here
2000                          */
2001                 }
2002                 lastlba = lba;
2003                 TAILQ_INSERT_TAIL(&req->bps, bp, bio_queue);
2004
2005                 bp = cam_iosched_next_trim(softc->cam_iosched);
2006                 if (bp == NULL)
2007                         break;
2008                 if (bp->bio_bcount / softc->params.secsize >
2009                     (softc->trim_max_ranges - ranges) * ATA_DSM_RANGE_MAX) {
2010                         cam_iosched_put_back_trim(softc->cam_iosched, bp);
2011                         break;
2012                 }
2013         } while (1);
2014         softc->trim_count++;
2015         softc->trim_ranges += ranges;
2016         softc->trim_lbas += lbas;
2017
2018         return (ranges);
2019 }
2020
2021 static void
2022 ada_dsmtrim(struct ada_softc *softc, struct bio *bp, struct ccb_ataio *ataio)
2023 {
2024         struct trim_request *req = &softc->trim_req;
2025         int ranges;
2026
2027         ranges = ada_dsmtrim_req_create(softc, bp, req);
2028         cam_fill_ataio(ataio,
2029             ada_retry_count,
2030             adadone,
2031             CAM_DIR_OUT,
2032             0,
2033             req->data,
2034             howmany(ranges, ATA_DSM_BLK_RANGES) * ATA_DSM_BLK_SIZE,
2035             ada_default_timeout * 1000);
2036         ata_48bit_cmd(ataio, ATA_DATA_SET_MANAGEMENT,
2037             ATA_DSM_TRIM, 0, howmany(ranges, ATA_DSM_BLK_RANGES));
2038 }
2039
2040 static void
2041 ada_ncq_dsmtrim(struct ada_softc *softc, struct bio *bp, struct ccb_ataio *ataio)
2042 {
2043         struct trim_request *req = &softc->trim_req;
2044         int ranges;
2045
2046         ranges = ada_dsmtrim_req_create(softc, bp, req);
2047         cam_fill_ataio(ataio,
2048             ada_retry_count,
2049             adadone,
2050             CAM_DIR_OUT,
2051             0,
2052             req->data,
2053             howmany(ranges, ATA_DSM_BLK_RANGES) * ATA_DSM_BLK_SIZE,
2054             ada_default_timeout * 1000);
2055         ata_ncq_cmd(ataio,
2056             ATA_SEND_FPDMA_QUEUED,
2057             0,
2058             howmany(ranges, ATA_DSM_BLK_RANGES));
2059         ataio->cmd.sector_count_exp = ATA_SFPDMA_DSM;
2060         ataio->ata_flags |= ATA_FLAG_AUX;
2061         ataio->aux = 1;
2062 }
2063
2064 static void
2065 ada_cfaerase(struct ada_softc *softc, struct bio *bp, struct ccb_ataio *ataio)
2066 {
2067         struct trim_request *req = &softc->trim_req;
2068         uint64_t lba = bp->bio_pblkno;
2069         uint16_t count = bp->bio_bcount / softc->params.secsize;
2070
2071         bzero(req, sizeof(*req));
2072         TAILQ_INIT(&req->bps);
2073         TAILQ_INSERT_TAIL(&req->bps, bp, bio_queue);
2074
2075         cam_fill_ataio(ataio,
2076             ada_retry_count,
2077             adadone,
2078             CAM_DIR_NONE,
2079             0,
2080             NULL,
2081             0,
2082             ada_default_timeout*1000);
2083
2084         if (count >= 256)
2085                 count = 0;
2086         ata_28bit_cmd(ataio, ATA_CFA_ERASE, 0, lba, count);
2087 }
2088
2089 static int
2090 ada_zone_bio_to_ata(int disk_zone_cmd)
2091 {
2092         switch (disk_zone_cmd) {
2093         case DISK_ZONE_OPEN:
2094                 return ATA_ZM_OPEN_ZONE;
2095         case DISK_ZONE_CLOSE:
2096                 return ATA_ZM_CLOSE_ZONE;
2097         case DISK_ZONE_FINISH:
2098                 return ATA_ZM_FINISH_ZONE;
2099         case DISK_ZONE_RWP:
2100                 return ATA_ZM_RWP;
2101         }
2102
2103         return -1;
2104 }
2105
2106 static int
2107 ada_zone_cmd(struct cam_periph *periph, union ccb *ccb, struct bio *bp,
2108              int *queue_ccb)
2109 {
2110         struct ada_softc *softc;
2111         int error;
2112
2113         error = 0;
2114
2115         if (bp->bio_cmd != BIO_ZONE) {
2116                 error = EINVAL;
2117                 goto bailout;
2118         }
2119
2120         softc = periph->softc;
2121
2122         switch (bp->bio_zone.zone_cmd) {
2123         case DISK_ZONE_OPEN:
2124         case DISK_ZONE_CLOSE:
2125         case DISK_ZONE_FINISH:
2126         case DISK_ZONE_RWP: {
2127                 int zone_flags;
2128                 int zone_sa;
2129                 uint64_t lba;
2130
2131                 zone_sa = ada_zone_bio_to_ata(bp->bio_zone.zone_cmd);
2132                 if (zone_sa == -1) {
2133                         xpt_print(periph->path, "Cannot translate zone "
2134                             "cmd %#x to ATA\n", bp->bio_zone.zone_cmd);
2135                         error = EINVAL;
2136                         goto bailout;
2137                 }
2138
2139                 zone_flags = 0;
2140                 lba = bp->bio_zone.zone_params.rwp.id;
2141
2142                 if (bp->bio_zone.zone_params.rwp.flags &
2143                     DISK_ZONE_RWP_FLAG_ALL)
2144                         zone_flags |= ZBC_OUT_ALL;
2145
2146                 ata_zac_mgmt_out(&ccb->ataio,
2147                                  /*retries*/ ada_retry_count,
2148                                  /*cbfcnp*/ adadone,
2149                                  /*use_ncq*/ (softc->flags &
2150                                               ADA_FLAG_PIM_ATA_EXT) ? 1 : 0,
2151                                  /*zm_action*/ zone_sa,
2152                                  /*zone_id*/ lba,
2153                                  /*zone_flags*/ zone_flags,
2154                                  /*sector_count*/ 0,
2155                                  /*data_ptr*/ NULL,
2156                                  /*dxfer_len*/ 0,
2157                                  /*timeout*/ ada_default_timeout * 1000);
2158                 *queue_ccb = 1;
2159
2160                 break;
2161         }
2162         case DISK_ZONE_REPORT_ZONES: {
2163                 uint8_t *rz_ptr;
2164                 uint32_t num_entries, alloc_size;
2165                 struct disk_zone_report *rep;
2166
2167                 rep = &bp->bio_zone.zone_params.report;
2168
2169                 num_entries = rep->entries_allocated;
2170                 if (num_entries == 0) {
2171                         xpt_print(periph->path, "No entries allocated for "
2172                             "Report Zones request\n");
2173                         error = EINVAL;
2174                         goto bailout;
2175                 }
2176                 alloc_size = sizeof(struct scsi_report_zones_hdr) +
2177                     (sizeof(struct scsi_report_zones_desc) * num_entries);
2178                 alloc_size = min(alloc_size, softc->disk->d_maxsize);
2179                 rz_ptr = malloc(alloc_size, M_ATADA, M_NOWAIT | M_ZERO);
2180                 if (rz_ptr == NULL) {
2181                         xpt_print(periph->path, "Unable to allocate memory "
2182                            "for Report Zones request\n");
2183                         error = ENOMEM;
2184                         goto bailout;
2185                 }
2186
2187                 ata_zac_mgmt_in(&ccb->ataio,
2188                                 /*retries*/ ada_retry_count,
2189                                 /*cbcfnp*/ adadone,
2190                                 /*use_ncq*/ (softc->flags &
2191                                              ADA_FLAG_PIM_ATA_EXT) ? 1 : 0,
2192                                 /*zm_action*/ ATA_ZM_REPORT_ZONES,
2193                                 /*zone_id*/ rep->starting_id,
2194                                 /*zone_flags*/ rep->rep_options,
2195                                 /*data_ptr*/ rz_ptr,
2196                                 /*dxfer_len*/ alloc_size,
2197                                 /*timeout*/ ada_default_timeout * 1000);
2198
2199                 /*
2200                  * For BIO_ZONE, this isn't normally needed.  However, it
2201                  * is used by devstat_end_transaction_bio() to determine
2202                  * how much data was transferred.
2203                  */
2204                 /*
2205                  * XXX KDM we have a problem.  But I'm not sure how to fix
2206                  * it.  devstat uses bio_bcount - bio_resid to calculate
2207                  * the amount of data transferred.   The GEOM disk code
2208                  * uses bio_length - bio_resid to calculate the amount of
2209                  * data in bio_completed.  We have different structure
2210                  * sizes above and below the ada(4) driver.  So, if we
2211                  * use the sizes above, the amount transferred won't be
2212                  * quite accurate for devstat.  If we use different sizes
2213                  * for bio_bcount and bio_length (above and below
2214                  * respectively), then the residual needs to match one or
2215                  * the other.  Everything is calculated after the bio
2216                  * leaves the driver, so changing the values around isn't
2217                  * really an option.  For now, just set the count to the
2218                  * passed in length.  This means that the calculations
2219                  * above (e.g. bio_completed) will be correct, but the
2220                  * amount of data reported to devstat will be slightly
2221                  * under or overstated.
2222                  */
2223                 bp->bio_bcount = bp->bio_length;
2224
2225                 *queue_ccb = 1;
2226
2227                 break;
2228         }
2229         case DISK_ZONE_GET_PARAMS: {
2230                 struct disk_zone_disk_params *params;
2231
2232                 params = &bp->bio_zone.zone_params.disk_params;
2233                 bzero(params, sizeof(*params));
2234
2235                 switch (softc->zone_mode) {
2236                 case ADA_ZONE_DRIVE_MANAGED:
2237                         params->zone_mode = DISK_ZONE_MODE_DRIVE_MANAGED;
2238                         break;
2239                 case ADA_ZONE_HOST_AWARE:
2240                         params->zone_mode = DISK_ZONE_MODE_HOST_AWARE;
2241                         break;
2242                 case ADA_ZONE_HOST_MANAGED:
2243                         params->zone_mode = DISK_ZONE_MODE_HOST_MANAGED;
2244                         break;
2245                 default:
2246                 case ADA_ZONE_NONE:
2247                         params->zone_mode = DISK_ZONE_MODE_NONE;
2248                         break;
2249                 }
2250
2251                 if (softc->zone_flags & ADA_ZONE_FLAG_URSWRZ)
2252                         params->flags |= DISK_ZONE_DISK_URSWRZ;
2253
2254                 if (softc->zone_flags & ADA_ZONE_FLAG_OPT_SEQ_SET) {
2255                         params->optimal_seq_zones = softc->optimal_seq_zones;
2256                         params->flags |= DISK_ZONE_OPT_SEQ_SET;
2257                 }
2258
2259                 if (softc->zone_flags & ADA_ZONE_FLAG_OPT_NONSEQ_SET) {
2260                         params->optimal_nonseq_zones =
2261                             softc->optimal_nonseq_zones;
2262                         params->flags |= DISK_ZONE_OPT_NONSEQ_SET;
2263                 }
2264
2265                 if (softc->zone_flags & ADA_ZONE_FLAG_MAX_SEQ_SET) {
2266                         params->max_seq_zones = softc->max_seq_zones;
2267                         params->flags |= DISK_ZONE_MAX_SEQ_SET;
2268                 }
2269                 if (softc->zone_flags & ADA_ZONE_FLAG_RZ_SUP)
2270                         params->flags |= DISK_ZONE_RZ_SUP;
2271
2272                 if (softc->zone_flags & ADA_ZONE_FLAG_OPEN_SUP)
2273                         params->flags |= DISK_ZONE_OPEN_SUP;
2274
2275                 if (softc->zone_flags & ADA_ZONE_FLAG_CLOSE_SUP)
2276                         params->flags |= DISK_ZONE_CLOSE_SUP;
2277
2278                 if (softc->zone_flags & ADA_ZONE_FLAG_FINISH_SUP)
2279                         params->flags |= DISK_ZONE_FINISH_SUP;
2280
2281                 if (softc->zone_flags & ADA_ZONE_FLAG_RWP_SUP)
2282                         params->flags |= DISK_ZONE_RWP_SUP;
2283                 break;
2284         }
2285         default:
2286                 break;
2287         }
2288 bailout:
2289         return (error);
2290 }
2291
2292 static void
2293 adastart(struct cam_periph *periph, union ccb *start_ccb)
2294 {
2295         struct ada_softc *softc = (struct ada_softc *)periph->softc;
2296         struct ccb_ataio *ataio = &start_ccb->ataio;
2297
2298         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("adastart\n"));
2299
2300         switch (softc->state) {
2301         case ADA_STATE_NORMAL:
2302         {
2303                 struct bio *bp;
2304                 u_int8_t tag_code;
2305
2306                 bp = cam_iosched_next_bio(softc->cam_iosched);
2307                 if (bp == NULL) {
2308                         xpt_release_ccb(start_ccb);
2309                         break;
2310                 }
2311
2312                 if ((bp->bio_flags & BIO_ORDERED) != 0 ||
2313                     (bp->bio_cmd != BIO_DELETE && (softc->flags & ADA_FLAG_NEED_OTAG) != 0)) {
2314                         softc->flags &= ~ADA_FLAG_NEED_OTAG;
2315                         softc->flags |= ADA_FLAG_WAS_OTAG;
2316                         tag_code = 0;
2317                 } else {
2318                         tag_code = 1;
2319                 }
2320                 switch (bp->bio_cmd) {
2321                 case BIO_WRITE:
2322                 case BIO_READ:
2323                 {
2324                         uint64_t lba = bp->bio_pblkno;
2325                         uint16_t count = bp->bio_bcount / softc->params.secsize;
2326                         void *data_ptr;
2327                         int rw_op;
2328
2329                         if (bp->bio_cmd == BIO_WRITE) {
2330                                 softc->flags |= ADA_FLAG_DIRTY;
2331                                 rw_op = CAM_DIR_OUT;
2332                         } else {
2333                                 rw_op = CAM_DIR_IN;
2334                         }
2335
2336                         data_ptr = bp->bio_data;
2337                         if ((bp->bio_flags & (BIO_UNMAPPED|BIO_VLIST)) != 0) {
2338                                 rw_op |= CAM_DATA_BIO;
2339                                 data_ptr = bp;
2340                         }
2341
2342 #ifdef CAM_TEST_FAILURE
2343                         int fail = 0;
2344
2345                         /*
2346                          * Support the failure ioctls.  If the command is a
2347                          * read, and there are pending forced read errors, or
2348                          * if a write and pending write errors, then fail this
2349                          * operation with EIO.  This is useful for testing
2350                          * purposes.  Also, support having every Nth read fail.
2351                          *
2352                          * This is a rather blunt tool.
2353                          */
2354                         if (bp->bio_cmd == BIO_READ) {
2355                                 if (softc->force_read_error) {
2356                                         softc->force_read_error--;
2357                                         fail = 1;
2358                                 }
2359                                 if (softc->periodic_read_error > 0) {
2360                                         if (++softc->periodic_read_count >=
2361                                             softc->periodic_read_error) {
2362                                                 softc->periodic_read_count = 0;
2363                                                 fail = 1;
2364                                         }
2365                                 }
2366                         } else {
2367                                 if (softc->force_write_error) {
2368                                         softc->force_write_error--;
2369                                         fail = 1;
2370                                 }
2371                         }
2372                         if (fail) {
2373                                 biofinish(bp, NULL, EIO);
2374                                 xpt_release_ccb(start_ccb);
2375                                 adaschedule(periph);
2376                                 return;
2377                         }
2378 #endif
2379                         KASSERT((bp->bio_flags & BIO_UNMAPPED) == 0 ||
2380                             round_page(bp->bio_bcount + bp->bio_ma_offset) /
2381                             PAGE_SIZE == bp->bio_ma_n,
2382                             ("Short bio %p", bp));
2383                         cam_fill_ataio(ataio,
2384                             ada_retry_count,
2385                             adadone,
2386                             rw_op,
2387                             0,
2388                             data_ptr,
2389                             bp->bio_bcount,
2390                             ada_default_timeout*1000);
2391
2392                         if ((softc->flags & ADA_FLAG_CAN_NCQ) && tag_code) {
2393                                 if (bp->bio_cmd == BIO_READ) {
2394                                         ata_ncq_cmd(ataio, ATA_READ_FPDMA_QUEUED,
2395                                             lba, count);
2396                                 } else {
2397                                         ata_ncq_cmd(ataio, ATA_WRITE_FPDMA_QUEUED,
2398                                             lba, count);
2399                                 }
2400                         } else if ((softc->flags & ADA_FLAG_CAN_48BIT) &&
2401                             (lba + count >= ATA_MAX_28BIT_LBA ||
2402                             count > 256)) {
2403                                 if (softc->flags & ADA_FLAG_CAN_DMA48) {
2404                                         if (bp->bio_cmd == BIO_READ) {
2405                                                 ata_48bit_cmd(ataio, ATA_READ_DMA48,
2406                                                     0, lba, count);
2407                                         } else {
2408                                                 ata_48bit_cmd(ataio, ATA_WRITE_DMA48,
2409                                                     0, lba, count);
2410                                         }
2411                                 } else {
2412                                         if (bp->bio_cmd == BIO_READ) {
2413                                                 ata_48bit_cmd(ataio, ATA_READ_MUL48,
2414                                                     0, lba, count);
2415                                         } else {
2416                                                 ata_48bit_cmd(ataio, ATA_WRITE_MUL48,
2417                                                     0, lba, count);
2418                                         }
2419                                 }
2420                         } else {
2421                                 if (count == 256)
2422                                         count = 0;
2423                                 if (softc->flags & ADA_FLAG_CAN_DMA) {
2424                                         if (bp->bio_cmd == BIO_READ) {
2425                                                 ata_28bit_cmd(ataio, ATA_READ_DMA,
2426                                                     0, lba, count);
2427                                         } else {
2428                                                 ata_28bit_cmd(ataio, ATA_WRITE_DMA,
2429                                                     0, lba, count);
2430                                         }
2431                                 } else {
2432                                         if (bp->bio_cmd == BIO_READ) {
2433                                                 ata_28bit_cmd(ataio, ATA_READ_MUL,
2434                                                     0, lba, count);
2435                                         } else {
2436                                                 ata_28bit_cmd(ataio, ATA_WRITE_MUL,
2437                                                     0, lba, count);
2438                                         }
2439                                 }
2440                         }
2441                         break;
2442                 }
2443                 case BIO_DELETE:
2444                         switch (softc->delete_method) {
2445                         case ADA_DELETE_NCQ_DSM_TRIM:
2446                                 ada_ncq_dsmtrim(softc, bp, ataio);
2447                                 break;
2448                         case ADA_DELETE_DSM_TRIM:
2449                                 ada_dsmtrim(softc, bp, ataio);
2450                                 break;
2451                         case ADA_DELETE_CFA_ERASE:
2452                                 ada_cfaerase(softc, bp, ataio);
2453                                 break;
2454                         default:
2455                                 biofinish(bp, NULL, EOPNOTSUPP);
2456                                 xpt_release_ccb(start_ccb);
2457                                 adaschedule(periph);
2458                                 return;
2459                         }
2460                         start_ccb->ccb_h.ccb_state = ADA_CCB_TRIM;
2461                         start_ccb->ccb_h.flags |= CAM_UNLOCKED;
2462                         cam_iosched_submit_trim(softc->cam_iosched);
2463                         goto out;
2464                 case BIO_FLUSH:
2465                         cam_fill_ataio(ataio,
2466                             1,
2467                             adadone,
2468                             CAM_DIR_NONE,
2469                             0,
2470                             NULL,
2471                             0,
2472                             ada_default_timeout*1000);
2473
2474                         if (softc->flags & ADA_FLAG_CAN_48BIT)
2475                                 ata_48bit_cmd(ataio, ATA_FLUSHCACHE48, 0, 0, 0);
2476                         else
2477                                 ata_28bit_cmd(ataio, ATA_FLUSHCACHE, 0, 0, 0);
2478                         break;
2479                 case BIO_ZONE: {
2480                         int error, queue_ccb;
2481
2482                         queue_ccb = 0;
2483
2484                         error = ada_zone_cmd(periph, start_ccb, bp, &queue_ccb);
2485                         if ((error != 0)
2486                          || (queue_ccb == 0)) {
2487                                 biofinish(bp, NULL, error);
2488                                 xpt_release_ccb(start_ccb);
2489                                 return;
2490                         }
2491                         break;
2492                 }
2493                 }
2494                 start_ccb->ccb_h.ccb_state = ADA_CCB_BUFFER_IO;
2495                 start_ccb->ccb_h.flags |= CAM_UNLOCKED;
2496 out:
2497                 start_ccb->ccb_h.ccb_bp = bp;
2498                 softc->outstanding_cmds++;
2499                 softc->refcount++;
2500                 cam_periph_unlock(periph);
2501                 xpt_action(start_ccb);
2502                 cam_periph_lock(periph);
2503
2504                 /* May have more work to do, so ensure we stay scheduled */
2505                 adaschedule(periph);
2506                 break;
2507         }
2508         case ADA_STATE_RAHEAD:
2509         case ADA_STATE_WCACHE:
2510         {
2511                 cam_fill_ataio(ataio,
2512                     1,
2513                     adadone,
2514                     CAM_DIR_NONE,
2515                     0,
2516                     NULL,
2517                     0,
2518                     ada_default_timeout*1000);
2519
2520                 if (softc->state == ADA_STATE_RAHEAD) {
2521                         ata_28bit_cmd(ataio, ATA_SETFEATURES, ADA_RA ?
2522                             ATA_SF_ENAB_RCACHE : ATA_SF_DIS_RCACHE, 0, 0);
2523                         start_ccb->ccb_h.ccb_state = ADA_CCB_RAHEAD;
2524                 } else {
2525                         ata_28bit_cmd(ataio, ATA_SETFEATURES, ADA_WC ?
2526                             ATA_SF_ENAB_WCACHE : ATA_SF_DIS_WCACHE, 0, 0);
2527                         start_ccb->ccb_h.ccb_state = ADA_CCB_WCACHE;
2528                 }
2529                 start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
2530                 xpt_action(start_ccb);
2531                 break;
2532         }
2533         case ADA_STATE_LOGDIR:
2534         {
2535                 struct ata_gp_log_dir *log_dir;
2536
2537                 if ((softc->flags & ADA_FLAG_CAN_LOG) == 0) {
2538                         adaprobedone(periph, start_ccb);
2539                         break;
2540                 }
2541
2542                 log_dir = malloc(sizeof(*log_dir), M_ATADA, M_NOWAIT|M_ZERO);
2543                 if (log_dir == NULL) {
2544                         xpt_print(periph->path, "Couldn't malloc log_dir "
2545                             "data\n");
2546                         softc->state = ADA_STATE_NORMAL;
2547                         xpt_release_ccb(start_ccb);
2548                         break;
2549                 }
2550
2551
2552                 ata_read_log(ataio,
2553                     /*retries*/1,
2554                     /*cbfcnp*/adadone,
2555                     /*log_address*/ ATA_LOG_DIRECTORY,
2556                     /*page_number*/ 0,
2557                     /*block_count*/ 1,
2558                     /*protocol*/ softc->flags & ADA_FLAG_CAN_DMA ?
2559                                  CAM_ATAIO_DMA : 0,
2560                     /*data_ptr*/ (uint8_t *)log_dir,
2561                     /*dxfer_len*/sizeof(*log_dir),
2562                     /*timeout*/ada_default_timeout*1000);
2563
2564                 start_ccb->ccb_h.ccb_state = ADA_CCB_LOGDIR;
2565                 xpt_action(start_ccb);
2566                 break;
2567         }
2568         case ADA_STATE_IDDIR:
2569         {
2570                 struct ata_identify_log_pages *id_dir;
2571
2572                 id_dir = malloc(sizeof(*id_dir), M_ATADA, M_NOWAIT | M_ZERO);
2573                 if (id_dir == NULL) {
2574                         xpt_print(periph->path, "Couldn't malloc id_dir "
2575                             "data\n");
2576                         adaprobedone(periph, start_ccb);
2577                         break;
2578                 }
2579
2580                 ata_read_log(ataio,
2581                     /*retries*/1,
2582                     /*cbfcnp*/adadone,
2583                     /*log_address*/ ATA_IDENTIFY_DATA_LOG,
2584                     /*page_number*/ ATA_IDL_PAGE_LIST,
2585                     /*block_count*/ 1,
2586                     /*protocol*/ softc->flags & ADA_FLAG_CAN_DMA ?
2587                                  CAM_ATAIO_DMA : 0,
2588                     /*data_ptr*/ (uint8_t *)id_dir,
2589                     /*dxfer_len*/ sizeof(*id_dir),
2590                     /*timeout*/ada_default_timeout*1000);
2591
2592                 start_ccb->ccb_h.ccb_state = ADA_CCB_IDDIR;
2593                 xpt_action(start_ccb);
2594                 break;
2595         }
2596         case ADA_STATE_SUP_CAP:
2597         {
2598                 struct ata_identify_log_sup_cap *sup_cap;
2599
2600                 sup_cap = malloc(sizeof(*sup_cap), M_ATADA, M_NOWAIT|M_ZERO);
2601                 if (sup_cap == NULL) {
2602                         xpt_print(periph->path, "Couldn't malloc sup_cap "
2603                             "data\n");
2604                         adaprobedone(periph, start_ccb);
2605                         break;
2606                 }
2607
2608                 ata_read_log(ataio,
2609                     /*retries*/1,
2610                     /*cbfcnp*/adadone,
2611                     /*log_address*/ ATA_IDENTIFY_DATA_LOG,
2612                     /*page_number*/ ATA_IDL_SUP_CAP,
2613                     /*block_count*/ 1,
2614                     /*protocol*/ softc->flags & ADA_FLAG_CAN_DMA ?
2615                                  CAM_ATAIO_DMA : 0,
2616                     /*data_ptr*/ (uint8_t *)sup_cap,
2617                     /*dxfer_len*/ sizeof(*sup_cap),
2618                     /*timeout*/ada_default_timeout*1000);
2619
2620                 start_ccb->ccb_h.ccb_state = ADA_CCB_SUP_CAP;
2621                 xpt_action(start_ccb);
2622                 break;
2623         }
2624         case ADA_STATE_ZONE:
2625         {
2626                 struct ata_zoned_info_log *ata_zone;
2627
2628                 ata_zone = malloc(sizeof(*ata_zone), M_ATADA, M_NOWAIT|M_ZERO);
2629                 if (ata_zone == NULL) {
2630                         xpt_print(periph->path, "Couldn't malloc ata_zone "
2631                             "data\n");
2632                         adaprobedone(periph, start_ccb);
2633                         break;
2634                 }
2635
2636                 ata_read_log(ataio,
2637                     /*retries*/1,
2638                     /*cbfcnp*/adadone,
2639                     /*log_address*/ ATA_IDENTIFY_DATA_LOG,
2640                     /*page_number*/ ATA_IDL_ZDI,
2641                     /*block_count*/ 1,
2642                     /*protocol*/ softc->flags & ADA_FLAG_CAN_DMA ?
2643                                  CAM_ATAIO_DMA : 0,
2644                     /*data_ptr*/ (uint8_t *)ata_zone,
2645                     /*dxfer_len*/ sizeof(*ata_zone),
2646                     /*timeout*/ada_default_timeout*1000);
2647
2648                 start_ccb->ccb_h.ccb_state = ADA_CCB_ZONE;
2649                 xpt_action(start_ccb);
2650                 break;
2651         }
2652         }
2653 }
2654
2655 static void
2656 adaprobedone(struct cam_periph *periph, union ccb *ccb)
2657 {
2658         struct ada_softc *softc;
2659
2660         softc = (struct ada_softc *)periph->softc;
2661
2662         if (ccb != NULL)
2663                 xpt_release_ccb(ccb);
2664
2665         softc->state = ADA_STATE_NORMAL;
2666         softc->flags |= ADA_FLAG_PROBED;
2667         adaschedule(periph);
2668         if ((softc->flags & ADA_FLAG_ANNOUNCED) == 0) {
2669                 softc->flags |= ADA_FLAG_ANNOUNCED;
2670                 cam_periph_unhold(periph);
2671         } else {
2672                 cam_periph_release_locked(periph);
2673         }
2674 }
2675
2676 static void
2677 adazonedone(struct cam_periph *periph, union ccb *ccb)
2678 {
2679         struct bio *bp;
2680
2681         bp = (struct bio *)ccb->ccb_h.ccb_bp;
2682
2683         switch (bp->bio_zone.zone_cmd) {
2684         case DISK_ZONE_OPEN:
2685         case DISK_ZONE_CLOSE:
2686         case DISK_ZONE_FINISH:
2687         case DISK_ZONE_RWP:
2688                 break;
2689         case DISK_ZONE_REPORT_ZONES: {
2690                 uint32_t avail_len;
2691                 struct disk_zone_report *rep;
2692                 struct scsi_report_zones_hdr *hdr;
2693                 struct scsi_report_zones_desc *desc;
2694                 struct disk_zone_rep_entry *entry;
2695                 uint32_t hdr_len, num_avail;
2696                 uint32_t num_to_fill, i;
2697
2698                 rep = &bp->bio_zone.zone_params.report;
2699                 avail_len = ccb->ataio.dxfer_len - ccb->ataio.resid;
2700                 /*
2701                  * Note that bio_resid isn't normally used for zone
2702                  * commands, but it is used by devstat_end_transaction_bio()
2703                  * to determine how much data was transferred.  Because
2704                  * the size of the SCSI/ATA data structures is different
2705                  * than the size of the BIO interface structures, the
2706                  * amount of data actually transferred from the drive will
2707                  * be different than the amount of data transferred to
2708                  * the user.
2709                  */
2710                 hdr = (struct scsi_report_zones_hdr *)ccb->ataio.data_ptr;
2711                 if (avail_len < sizeof(*hdr)) {
2712                         /*
2713                          * Is there a better error than EIO here?  We asked
2714                          * for at least the header, and we got less than
2715                          * that.
2716                          */
2717                         bp->bio_error = EIO;
2718                         bp->bio_flags |= BIO_ERROR;
2719                         bp->bio_resid = bp->bio_bcount;
2720                         break;
2721                 }
2722
2723                 hdr_len = le32dec(hdr->length);
2724                 if (hdr_len > 0)
2725                         rep->entries_available = hdr_len / sizeof(*desc);
2726                 else
2727                         rep->entries_available = 0;
2728                 /*
2729                  * NOTE: using the same values for the BIO version of the
2730                  * same field as the SCSI/ATA values.  This means we could
2731                  * get some additional values that aren't defined in bio.h
2732                  * if more values of the same field are defined later.
2733                  */
2734                 rep->header.same = hdr->byte4 & SRZ_SAME_MASK;
2735                 rep->header.maximum_lba = le64dec(hdr->maximum_lba);
2736                 /*
2737                  * If the drive reports no entries that match the query,
2738                  * we're done.
2739                  */
2740                 if (hdr_len == 0) {
2741                         rep->entries_filled = 0;
2742                         bp->bio_resid = bp->bio_bcount;
2743                         break;
2744                 }
2745
2746                 num_avail = min((avail_len - sizeof(*hdr)) / sizeof(*desc),
2747                                 hdr_len / sizeof(*desc));
2748                 /*
2749                  * If the drive didn't return any data, then we're done.
2750                  */
2751                 if (num_avail == 0) {
2752                         rep->entries_filled = 0;
2753                         bp->bio_resid = bp->bio_bcount;
2754                         break;
2755                 }
2756
2757                 num_to_fill = min(num_avail, rep->entries_allocated);
2758                 /*
2759                  * If the user didn't allocate any entries for us to fill,
2760                  * we're done.
2761                  */
2762                 if (num_to_fill == 0) {
2763                         rep->entries_filled = 0;
2764                         bp->bio_resid = bp->bio_bcount;
2765                         break;
2766                 }
2767
2768                 for (i = 0, desc = &hdr->desc_list[0], entry=&rep->entries[0];
2769                      i < num_to_fill; i++, desc++, entry++) {
2770                         /*
2771                          * NOTE: we're mapping the values here directly
2772                          * from the SCSI/ATA bit definitions to the bio.h
2773                          * definitions.  There is also a warning in
2774                          * disk_zone.h, but the impact is that if
2775                          * additional values are added in the SCSI/ATA
2776                          * specs these will be visible to consumers of
2777                          * this interface.
2778                          */
2779                         entry->zone_type = desc->zone_type & SRZ_TYPE_MASK;
2780                         entry->zone_condition =
2781                             (desc->zone_flags & SRZ_ZONE_COND_MASK) >>
2782                             SRZ_ZONE_COND_SHIFT;
2783                         entry->zone_flags |= desc->zone_flags &
2784                             (SRZ_ZONE_NON_SEQ|SRZ_ZONE_RESET);
2785                         entry->zone_length = le64dec(desc->zone_length);
2786                         entry->zone_start_lba = le64dec(desc->zone_start_lba);
2787                         entry->write_pointer_lba =
2788                             le64dec(desc->write_pointer_lba);
2789                 }
2790                 rep->entries_filled = num_to_fill;
2791                 /*
2792                  * Note that this residual is accurate from the user's
2793                  * standpoint, but the amount transferred isn't accurate
2794                  * from the standpoint of what actually came back from the
2795                  * drive.
2796                  */
2797                 bp->bio_resid = bp->bio_bcount - (num_to_fill * sizeof(*entry));
2798                 break;
2799         }
2800         case DISK_ZONE_GET_PARAMS:
2801         default:
2802                 /*
2803                  * In theory we should not get a GET_PARAMS bio, since it
2804                  * should be handled without queueing the command to the
2805                  * drive.
2806                  */
2807                 panic("%s: Invalid zone command %d", __func__,
2808                     bp->bio_zone.zone_cmd);
2809                 break;
2810         }
2811
2812         if (bp->bio_zone.zone_cmd == DISK_ZONE_REPORT_ZONES)
2813                 free(ccb->ataio.data_ptr, M_ATADA);
2814 }
2815
2816
2817 static void
2818 adadone(struct cam_periph *periph, union ccb *done_ccb)
2819 {
2820         struct ada_softc *softc;
2821         struct ccb_ataio *ataio;
2822         struct cam_path *path;
2823         uint32_t priority;
2824         int state;
2825
2826         softc = (struct ada_softc *)periph->softc;
2827         ataio = &done_ccb->ataio;
2828         path = done_ccb->ccb_h.path;
2829         priority = done_ccb->ccb_h.pinfo.priority;
2830
2831         CAM_DEBUG(path, CAM_DEBUG_TRACE, ("adadone\n"));
2832
2833         state = ataio->ccb_h.ccb_state & ADA_CCB_TYPE_MASK;
2834         switch (state) {
2835         case ADA_CCB_BUFFER_IO:
2836         case ADA_CCB_TRIM:
2837         {
2838                 struct bio *bp;
2839                 int error;
2840
2841                 cam_periph_lock(periph);
2842                 bp = (struct bio *)done_ccb->ccb_h.ccb_bp;
2843                 if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
2844                         error = adaerror(done_ccb, 0, 0);
2845                         if (error == ERESTART) {
2846                                 /* A retry was scheduled, so just return. */
2847                                 cam_periph_unlock(periph);
2848                                 return;
2849                         }
2850                         if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0)
2851                                 cam_release_devq(path,
2852                                                  /*relsim_flags*/0,
2853                                                  /*reduction*/0,
2854                                                  /*timeout*/0,
2855                                                  /*getcount_only*/0);
2856                         /*
2857                          * If we get an error on an NCQ DSM TRIM, fall back
2858                          * to a non-NCQ DSM TRIM forever. Please note that if
2859                          * CAN_NCQ_TRIM is set, CAN_TRIM is necessarily set too.
2860                          * However, for this one trim, we treat it as advisory
2861                          * and return success up the stack.
2862                          */
2863                         if (state == ADA_CCB_TRIM &&
2864                             error != 0 &&
2865                             (softc->flags & ADA_FLAG_CAN_NCQ_TRIM) != 0) {
2866                                 softc->flags &= ~ADA_FLAG_CAN_NCQ_TRIM;
2867                                 error = 0;
2868                                 adasetdeletemethod(softc);
2869                         }
2870                 } else {
2871                         if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0)
2872                                 panic("REQ_CMP with QFRZN");
2873
2874                         error = 0;
2875                 }
2876                 bp->bio_error = error;
2877                 if (error != 0) {
2878                         bp->bio_resid = bp->bio_bcount;
2879                         bp->bio_flags |= BIO_ERROR;
2880                 } else {
2881                         if (bp->bio_cmd == BIO_ZONE)
2882                                 adazonedone(periph, done_ccb);
2883                         else if (state == ADA_CCB_TRIM)
2884                                 bp->bio_resid = 0;
2885                         else
2886                                 bp->bio_resid = ataio->resid;
2887
2888                         if ((bp->bio_resid > 0)
2889                          && (bp->bio_cmd != BIO_ZONE))
2890                                 bp->bio_flags |= BIO_ERROR;
2891                 }
2892                 softc->outstanding_cmds--;
2893                 if (softc->outstanding_cmds == 0)
2894                         softc->flags |= ADA_FLAG_WAS_OTAG;
2895
2896                 /*
2897                  * We need to call cam_iosched before we call biodone so that we
2898                  * don't measure any activity that happens in the completion
2899                  * routine, which in the case of sendfile can be quite
2900                  * extensive.  Release the periph refcount taken in adastart()
2901                  * for each CCB.
2902                  */
2903                 cam_iosched_bio_complete(softc->cam_iosched, bp, done_ccb);
2904                 xpt_release_ccb(done_ccb);
2905                 KASSERT(softc->refcount >= 1, ("adadone softc %p refcount %d", softc, softc->refcount));
2906                 softc->refcount--;
2907                 if (state == ADA_CCB_TRIM) {
2908                         TAILQ_HEAD(, bio) queue;
2909                         struct bio *bp1;
2910
2911                         TAILQ_INIT(&queue);
2912                         TAILQ_CONCAT(&queue, &softc->trim_req.bps, bio_queue);
2913                         /*
2914                          * Normally, the xpt_release_ccb() above would make sure
2915                          * that when we have more work to do, that work would
2916                          * get kicked off. However, we specifically keep
2917                          * trim_running set to 0 before the call above to allow
2918                          * other I/O to progress when many BIO_DELETE requests
2919                          * are pushed down. We set trim_running to 0 and call
2920                          * daschedule again so that we don't stall if there are
2921                          * no other I/Os pending apart from BIO_DELETEs.
2922                          */
2923                         cam_iosched_trim_done(softc->cam_iosched);
2924                         adaschedule(periph);
2925                         cam_periph_unlock(periph);
2926                         while ((bp1 = TAILQ_FIRST(&queue)) != NULL) {
2927                                 TAILQ_REMOVE(&queue, bp1, bio_queue);
2928                                 bp1->bio_error = error;
2929                                 if (error != 0) {
2930                                         bp1->bio_flags |= BIO_ERROR;
2931                                         bp1->bio_resid = bp1->bio_bcount;
2932                                 } else
2933                                         bp1->bio_resid = 0;
2934                                 biodone(bp1);
2935                         }
2936                 } else {
2937                         adaschedule(periph);
2938                         cam_periph_unlock(periph);
2939                         biodone(bp);
2940                 }
2941                 return;
2942         }
2943         case ADA_CCB_RAHEAD:
2944         {
2945                 if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
2946                         if (adaerror(done_ccb, 0, 0) == ERESTART) {
2947                                 /* Drop freeze taken due to CAM_DEV_QFREEZE */
2948                                 cam_release_devq(path, 0, 0, 0, FALSE);
2949                                 return;
2950                         } else if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
2951                                 cam_release_devq(path,
2952                                     /*relsim_flags*/0,
2953                                     /*reduction*/0,
2954                                     /*timeout*/0,
2955                                     /*getcount_only*/0);
2956                         }
2957                 }
2958
2959                 /*
2960                  * Since our peripheral may be invalidated by an error
2961                  * above or an external event, we must release our CCB
2962                  * before releasing the reference on the peripheral.
2963                  * The peripheral will only go away once the last reference
2964                  * is removed, and we need it around for the CCB release
2965                  * operation.
2966                  */
2967
2968                 xpt_release_ccb(done_ccb);
2969                 softc->state = ADA_STATE_WCACHE;
2970                 xpt_schedule(periph, priority);
2971                 /* Drop freeze taken due to CAM_DEV_QFREEZE */
2972                 cam_release_devq(path, 0, 0, 0, FALSE);
2973                 return;
2974         }
2975         case ADA_CCB_WCACHE:
2976         {
2977                 if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
2978                         if (adaerror(done_ccb, 0, 0) == ERESTART) {
2979                                 /* Drop freeze taken due to CAM_DEV_QFREEZE */
2980                                 cam_release_devq(path, 0, 0, 0, FALSE);
2981                                 return;
2982                         } else if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
2983                                 cam_release_devq(path,
2984                                     /*relsim_flags*/0,
2985                                     /*reduction*/0,
2986                                     /*timeout*/0,
2987                                     /*getcount_only*/0);
2988                         }
2989                 }
2990
2991                 /* Drop freeze taken due to CAM_DEV_QFREEZE */
2992                 cam_release_devq(path, 0, 0, 0, FALSE);
2993
2994                 if ((softc->flags & ADA_FLAG_CAN_LOG)
2995                  && (softc->zone_mode != ADA_ZONE_NONE)) {
2996                         xpt_release_ccb(done_ccb);
2997                         softc->state = ADA_STATE_LOGDIR;
2998                         xpt_schedule(periph, priority);
2999                 } else {
3000                         adaprobedone(periph, done_ccb);
3001                 }
3002                 return;
3003         }
3004         case ADA_CCB_LOGDIR:
3005         {
3006                 int error;
3007
3008                 if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
3009                         error = 0;
3010                         softc->valid_logdir_len = 0;
3011                         bzero(&softc->ata_logdir, sizeof(softc->ata_logdir));
3012                         softc->valid_logdir_len =
3013                                 ataio->dxfer_len - ataio->resid;
3014                         if (softc->valid_logdir_len > 0)
3015                                 bcopy(ataio->data_ptr, &softc->ata_logdir,
3016                                     min(softc->valid_logdir_len,
3017                                         sizeof(softc->ata_logdir)));
3018                         /*
3019                          * Figure out whether the Identify Device log is
3020                          * supported.  The General Purpose log directory
3021                          * has a header, and lists the number of pages
3022                          * available for each GP log identified by the
3023                          * offset into the list.
3024                          */
3025                         if ((softc->valid_logdir_len >=
3026                             ((ATA_IDENTIFY_DATA_LOG + 1) * sizeof(uint16_t)))
3027                          && (le16dec(softc->ata_logdir.header) ==
3028                              ATA_GP_LOG_DIR_VERSION)
3029                          && (le16dec(&softc->ata_logdir.num_pages[
3030                              (ATA_IDENTIFY_DATA_LOG *
3031                              sizeof(uint16_t)) - sizeof(uint16_t)]) > 0)){
3032                                 softc->flags |= ADA_FLAG_CAN_IDLOG;
3033                         } else {
3034                                 softc->flags &= ~ADA_FLAG_CAN_IDLOG;
3035                         }
3036                 } else {
3037                         error = adaerror(done_ccb, CAM_RETRY_SELTO,
3038                                          SF_RETRY_UA|SF_NO_PRINT);
3039                         if (error == ERESTART)
3040                                 return;
3041                         else if (error != 0) {
3042                                 /*
3043                                  * If we can't get the ATA log directory,
3044                                  * then ATA logs are effectively not
3045                                  * supported even if the bit is set in the
3046                                  * identify data.
3047                                  */
3048                                 softc->flags &= ~(ADA_FLAG_CAN_LOG |
3049                                                   ADA_FLAG_CAN_IDLOG);
3050                                 if ((done_ccb->ccb_h.status &
3051                                      CAM_DEV_QFRZN) != 0) {
3052                                         /* Don't wedge this device's queue */
3053                                         cam_release_devq(done_ccb->ccb_h.path,
3054                                                          /*relsim_flags*/0,
3055                                                          /*reduction*/0,
3056                                                          /*timeout*/0,
3057                                                          /*getcount_only*/0);
3058                                 }
3059                         }
3060
3061
3062                 }
3063
3064                 free(ataio->data_ptr, M_ATADA);
3065
3066                 if ((error == 0)
3067                  && (softc->flags & ADA_FLAG_CAN_IDLOG)) {
3068                         softc->state = ADA_STATE_IDDIR;
3069                         xpt_release_ccb(done_ccb);
3070                         xpt_schedule(periph, priority);
3071                 } else
3072                         adaprobedone(periph, done_ccb);
3073
3074                 return;
3075         }
3076         case ADA_CCB_IDDIR: {
3077                 int error;
3078
3079                 if ((ataio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
3080                         off_t entries_offset, max_entries;
3081                         error = 0;
3082
3083                         softc->valid_iddir_len = 0;
3084                         bzero(&softc->ata_iddir, sizeof(softc->ata_iddir));
3085                         softc->flags &= ~(ADA_FLAG_CAN_SUPCAP |
3086                                           ADA_FLAG_CAN_ZONE);
3087                         softc->valid_iddir_len =
3088                                 ataio->dxfer_len - ataio->resid;
3089                         if (softc->valid_iddir_len > 0)
3090                                 bcopy(ataio->data_ptr, &softc->ata_iddir,
3091                                     min(softc->valid_iddir_len,
3092                                         sizeof(softc->ata_iddir)));
3093
3094                         entries_offset =
3095                             __offsetof(struct ata_identify_log_pages,entries);
3096                         max_entries = softc->valid_iddir_len - entries_offset;
3097                         if ((softc->valid_iddir_len > (entries_offset + 1))
3098                          && (le64dec(softc->ata_iddir.header) ==
3099                              ATA_IDLOG_REVISION)
3100                          && (softc->ata_iddir.entry_count > 0)) {
3101                                 int num_entries, i;
3102
3103                                 num_entries = softc->ata_iddir.entry_count;
3104                                 num_entries = min(num_entries,
3105                                    softc->valid_iddir_len - entries_offset);
3106                                 for (i = 0; i < num_entries &&
3107                                      i < max_entries; i++) {
3108                                         if (softc->ata_iddir.entries[i] ==
3109                                             ATA_IDL_SUP_CAP)
3110                                                 softc->flags |=
3111                                                     ADA_FLAG_CAN_SUPCAP;
3112                                         else if (softc->ata_iddir.entries[i]==
3113                                                  ATA_IDL_ZDI)
3114                                                 softc->flags |=
3115                                                     ADA_FLAG_CAN_ZONE;
3116
3117                                         if ((softc->flags &
3118                                              ADA_FLAG_CAN_SUPCAP)
3119                                          && (softc->flags &
3120                                              ADA_FLAG_CAN_ZONE))
3121                                                 break;
3122                                 }
3123                         }
3124                 } else {
3125                         error = adaerror(done_ccb, CAM_RETRY_SELTO,
3126                                          SF_RETRY_UA|SF_NO_PRINT);
3127                         if (error == ERESTART)
3128                                 return;
3129                         else if (error != 0) {
3130                                 /*
3131                                  * If we can't get the ATA Identify Data log
3132                                  * directory, then it effectively isn't
3133                                  * supported even if the ATA Log directory
3134                                  * a non-zero number of pages present for
3135                                  * this log.
3136                                  */
3137                                 softc->flags &= ~ADA_FLAG_CAN_IDLOG;
3138                                 if ((done_ccb->ccb_h.status &
3139                                      CAM_DEV_QFRZN) != 0) {
3140                                         /* Don't wedge this device's queue */
3141                                         cam_release_devq(done_ccb->ccb_h.path,
3142                                                          /*relsim_flags*/0,
3143                                                          /*reduction*/0,
3144                                                          /*timeout*/0,
3145                                                          /*getcount_only*/0);
3146                                 }
3147                         }
3148                 }
3149
3150                 free(ataio->data_ptr, M_ATADA);
3151
3152                 if ((error == 0)
3153                  && (softc->flags & ADA_FLAG_CAN_SUPCAP)) {
3154                         softc->state = ADA_STATE_SUP_CAP;
3155                         xpt_release_ccb(done_ccb);
3156                         xpt_schedule(periph, priority);
3157                 } else
3158                         adaprobedone(periph, done_ccb);
3159                 return;
3160         }
3161         case ADA_CCB_SUP_CAP: {
3162                 int error;
3163
3164                 if ((ataio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
3165                         uint32_t valid_len;
3166                         size_t needed_size;
3167                         struct ata_identify_log_sup_cap *sup_cap;
3168                         error = 0;
3169
3170                         sup_cap = (struct ata_identify_log_sup_cap *)
3171                             ataio->data_ptr;
3172                         valid_len = ataio->dxfer_len - ataio->resid;
3173                         needed_size =
3174                             __offsetof(struct ata_identify_log_sup_cap,
3175                             sup_zac_cap) + 1 + sizeof(sup_cap->sup_zac_cap);
3176                         if (valid_len >= needed_size) {
3177                                 uint64_t zoned, zac_cap;
3178
3179                                 zoned = le64dec(sup_cap->zoned_cap);
3180                                 if (zoned & ATA_ZONED_VALID) {
3181                                         /*
3182                                          * This should have already been
3183                                          * set, because this is also in the
3184                                          * ATA identify data.
3185                                          */
3186                                         if ((zoned & ATA_ZONED_MASK) ==
3187                                             ATA_SUPPORT_ZONE_HOST_AWARE)
3188                                                 softc->zone_mode =
3189                                                     ADA_ZONE_HOST_AWARE;
3190                                         else if ((zoned & ATA_ZONED_MASK) ==
3191                                             ATA_SUPPORT_ZONE_DEV_MANAGED)
3192                                                 softc->zone_mode =
3193                                                     ADA_ZONE_DRIVE_MANAGED;
3194                                 }
3195
3196                                 zac_cap = le64dec(sup_cap->sup_zac_cap);
3197                                 if (zac_cap & ATA_SUP_ZAC_CAP_VALID) {
3198                                         if (zac_cap & ATA_REPORT_ZONES_SUP)
3199                                                 softc->zone_flags |=
3200                                                     ADA_ZONE_FLAG_RZ_SUP;
3201                                         if (zac_cap & ATA_ND_OPEN_ZONE_SUP)
3202                                                 softc->zone_flags |=
3203                                                     ADA_ZONE_FLAG_OPEN_SUP;
3204                                         if (zac_cap & ATA_ND_CLOSE_ZONE_SUP)
3205                                                 softc->zone_flags |=
3206                                                     ADA_ZONE_FLAG_CLOSE_SUP;
3207                                         if (zac_cap & ATA_ND_FINISH_ZONE_SUP)
3208                                                 softc->zone_flags |=
3209                                                     ADA_ZONE_FLAG_FINISH_SUP;
3210                                         if (zac_cap & ATA_ND_RWP_SUP)
3211                                                 softc->zone_flags |=
3212                                                     ADA_ZONE_FLAG_RWP_SUP;
3213                                 } else {
3214                                         /*
3215                                          * This field was introduced in
3216                                          * ACS-4, r08 on April 28th, 2015.
3217                                          * If the drive firmware was written
3218                                          * to an earlier spec, it won't have
3219                                          * the field.  So, assume all
3220                                          * commands are supported.
3221                                          */
3222                                         softc->zone_flags |=
3223                                             ADA_ZONE_FLAG_SUP_MASK;
3224                                 }
3225                         }
3226                 } else {
3227                         error = adaerror(done_ccb, CAM_RETRY_SELTO,
3228                                          SF_RETRY_UA|SF_NO_PRINT);
3229                         if (error == ERESTART)
3230                                 return;
3231                         else if (error != 0) {
3232                                 /*
3233                                  * If we can't get the ATA Identify Data
3234                                  * Supported Capabilities page, clear the
3235                                  * flag...
3236                                  */
3237                                 softc->flags &= ~ADA_FLAG_CAN_SUPCAP;
3238                                 /*
3239                                  * And clear zone capabilities.
3240                                  */
3241                                 softc->zone_flags &= ~ADA_ZONE_FLAG_SUP_MASK;
3242                                 if ((done_ccb->ccb_h.status &
3243                                      CAM_DEV_QFRZN) != 0) {
3244                                         /* Don't wedge this device's queue */
3245                                         cam_release_devq(done_ccb->ccb_h.path,
3246                                                          /*relsim_flags*/0,
3247                                                          /*reduction*/0,
3248                                                          /*timeout*/0,
3249                                                          /*getcount_only*/0);
3250                                 }
3251                         }
3252                 }
3253
3254                 free(ataio->data_ptr, M_ATADA);
3255
3256                 if ((error == 0)
3257                  && (softc->flags & ADA_FLAG_CAN_ZONE)) {
3258                         softc->state = ADA_STATE_ZONE;
3259                         xpt_release_ccb(done_ccb);
3260                         xpt_schedule(periph, priority);
3261                 } else
3262                         adaprobedone(periph, done_ccb);
3263                 return;
3264         }
3265         case ADA_CCB_ZONE: {
3266                 int error;
3267
3268                 if ((ataio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
3269                         struct ata_zoned_info_log *zi_log;
3270                         uint32_t valid_len;
3271                         size_t needed_size;
3272
3273                         zi_log = (struct ata_zoned_info_log *)ataio->data_ptr;
3274
3275                         valid_len = ataio->dxfer_len - ataio->resid;
3276                         needed_size = __offsetof(struct ata_zoned_info_log,
3277                             version_info) + 1 + sizeof(zi_log->version_info);
3278                         if (valid_len >= needed_size) {
3279                                 uint64_t tmpvar;
3280
3281                                 tmpvar = le64dec(zi_log->zoned_cap);
3282                                 if (tmpvar & ATA_ZDI_CAP_VALID) {
3283                                         if (tmpvar & ATA_ZDI_CAP_URSWRZ)
3284                                                 softc->zone_flags |=
3285                                                     ADA_ZONE_FLAG_URSWRZ;
3286                                         else
3287                                                 softc->zone_flags &=
3288                                                     ~ADA_ZONE_FLAG_URSWRZ;
3289                                 }
3290                                 tmpvar = le64dec(zi_log->optimal_seq_zones);
3291                                 if (tmpvar & ATA_ZDI_OPT_SEQ_VALID) {
3292                                         softc->zone_flags |=
3293                                             ADA_ZONE_FLAG_OPT_SEQ_SET;
3294                                         softc->optimal_seq_zones = (tmpvar &
3295                                             ATA_ZDI_OPT_SEQ_MASK);
3296                                 } else {
3297                                         softc->zone_flags &=
3298                                             ~ADA_ZONE_FLAG_OPT_SEQ_SET;
3299                                         softc->optimal_seq_zones = 0;
3300                                 }
3301
3302                                 tmpvar =le64dec(zi_log->optimal_nonseq_zones);
3303                                 if (tmpvar & ATA_ZDI_OPT_NS_VALID) {
3304                                         softc->zone_flags |=
3305                                             ADA_ZONE_FLAG_OPT_NONSEQ_SET;
3306                                         softc->optimal_nonseq_zones =
3307                                             (tmpvar & ATA_ZDI_OPT_NS_MASK);
3308                                 } else {
3309                                         softc->zone_flags &=
3310                                             ~ADA_ZONE_FLAG_OPT_NONSEQ_SET;
3311                                         softc->optimal_nonseq_zones = 0;
3312                                 }
3313
3314                                 tmpvar = le64dec(zi_log->max_seq_req_zones);
3315                                 if (tmpvar & ATA_ZDI_MAX_SEQ_VALID) {
3316                                         softc->zone_flags |=
3317                                             ADA_ZONE_FLAG_MAX_SEQ_SET;
3318                                         softc->max_seq_zones =
3319                                             (tmpvar & ATA_ZDI_MAX_SEQ_MASK);
3320                                 } else {
3321                                         softc->zone_flags &=
3322                                             ~ADA_ZONE_FLAG_MAX_SEQ_SET;
3323                                         softc->max_seq_zones = 0;
3324                                 }
3325                         }
3326                 } else {
3327                         error = adaerror(done_ccb, CAM_RETRY_SELTO,
3328                                          SF_RETRY_UA|SF_NO_PRINT);
3329                         if (error == ERESTART)
3330                                 return;
3331                         else if (error != 0) {
3332                                 softc->flags &= ~ADA_FLAG_CAN_ZONE;
3333                                 softc->flags &= ~ADA_ZONE_FLAG_SET_MASK;
3334
3335                                 if ((done_ccb->ccb_h.status &
3336                                      CAM_DEV_QFRZN) != 0) {
3337                                         /* Don't wedge this device's queue */
3338                                         cam_release_devq(done_ccb->ccb_h.path,
3339                                                          /*relsim_flags*/0,
3340                                                          /*reduction*/0,
3341                                                          /*timeout*/0,
3342                                                          /*getcount_only*/0);
3343                                 }
3344                         }
3345                 }
3346                 free(ataio->data_ptr, M_ATADA);
3347
3348                 adaprobedone(periph, done_ccb);
3349                 return;
3350         }
3351         case ADA_CCB_DUMP:
3352                 /* No-op.  We're polling */
3353                 return;
3354         default:
3355                 break;
3356         }
3357         xpt_release_ccb(done_ccb);
3358 }
3359
3360 static int
3361 adaerror(union ccb *ccb, u_int32_t cam_flags, u_int32_t sense_flags)
3362 {
3363 #ifdef CAM_IO_STATS
3364         struct ada_softc *softc;
3365         struct cam_periph *periph;
3366
3367         periph = xpt_path_periph(ccb->ccb_h.path);
3368         softc = (struct ada_softc *)periph->softc;
3369
3370         switch (ccb->ccb_h.status & CAM_STATUS_MASK) {
3371         case CAM_CMD_TIMEOUT:
3372                 softc->timeouts++;
3373                 break;
3374         case CAM_REQ_ABORTED:
3375         case CAM_REQ_CMP_ERR:
3376         case CAM_REQ_TERMIO:
3377         case CAM_UNREC_HBA_ERROR:
3378         case CAM_DATA_RUN_ERR:
3379         case CAM_ATA_STATUS_ERROR:
3380                 softc->errors++;
3381                 break;
3382         default:
3383                 break;
3384         }
3385 #endif
3386
3387         return(cam_periph_error(ccb, cam_flags, sense_flags));
3388 }
3389
3390 static void
3391 adagetparams(struct cam_periph *periph, struct ccb_getdev *cgd)
3392 {
3393         struct ada_softc *softc = (struct ada_softc *)periph->softc;
3394         struct disk_params *dp = &softc->params;
3395         u_int64_t lbasize48;
3396         u_int32_t lbasize;
3397
3398         dp->secsize = ata_logical_sector_size(&cgd->ident_data);
3399         if ((cgd->ident_data.atavalid & ATA_FLAG_54_58) &&
3400                 cgd->ident_data.current_heads && cgd->ident_data.current_sectors) {
3401                 dp->heads = cgd->ident_data.current_heads;
3402                 dp->secs_per_track = cgd->ident_data.current_sectors;
3403                 dp->cylinders = cgd->ident_data.cylinders;
3404                 dp->sectors = (u_int32_t)cgd->ident_data.current_size_1 |
3405                           ((u_int32_t)cgd->ident_data.current_size_2 << 16);
3406         } else {
3407                 dp->heads = cgd->ident_data.heads;
3408                 dp->secs_per_track = cgd->ident_data.sectors;
3409                 dp->cylinders = cgd->ident_data.cylinders;
3410                 dp->sectors = cgd->ident_data.cylinders *
3411                               (u_int32_t)(dp->heads * dp->secs_per_track);
3412         }
3413         lbasize = (u_int32_t)cgd->ident_data.lba_size_1 |
3414                   ((u_int32_t)cgd->ident_data.lba_size_2 << 16);
3415
3416         /* use the 28bit LBA size if valid or bigger than the CHS mapping */
3417         if (cgd->ident_data.cylinders == 16383 || dp->sectors < lbasize)
3418                 dp->sectors = lbasize;
3419
3420         /* use the 48bit LBA size if valid */
3421         lbasize48 = ((u_int64_t)cgd->ident_data.lba_size48_1) |
3422                     ((u_int64_t)cgd->ident_data.lba_size48_2 << 16) |
3423                     ((u_int64_t)cgd->ident_data.lba_size48_3 << 32) |
3424                     ((u_int64_t)cgd->ident_data.lba_size48_4 << 48);
3425         if ((cgd->ident_data.support.command2 & ATA_SUPPORT_ADDRESS48) &&
3426             lbasize48 > ATA_MAX_28BIT_LBA)
3427                 dp->sectors = lbasize48;
3428 }
3429
3430 static void
3431 adasendorderedtag(void *arg)
3432 {
3433         struct ada_softc *softc = arg;
3434
3435         if (ada_send_ordered) {
3436                 if (softc->outstanding_cmds > 0) {
3437                         if ((softc->flags & ADA_FLAG_WAS_OTAG) == 0)
3438                                 softc->flags |= ADA_FLAG_NEED_OTAG;
3439                         softc->flags &= ~ADA_FLAG_WAS_OTAG;
3440                 }
3441         }
3442         /* Queue us up again */
3443         callout_reset(&softc->sendordered_c,
3444             (ada_default_timeout * hz) / ADA_ORDEREDTAG_INTERVAL,
3445             adasendorderedtag, softc);
3446 }
3447
3448 /*
3449  * Step through all ADA peripheral drivers, and if the device is still open,
3450  * sync the disk cache to physical media.
3451  */
3452 static void
3453 adaflush(void)
3454 {
3455         struct cam_periph *periph;
3456         struct ada_softc *softc;
3457         union ccb *ccb;
3458         int error;
3459
3460         CAM_PERIPH_FOREACH(periph, &adadriver) {
3461                 softc = (struct ada_softc *)periph->softc;
3462                 if (SCHEDULER_STOPPED()) {
3463                         /* If we paniced with the lock held, do not recurse. */
3464                         if (!cam_periph_owned(periph) &&
3465                             (softc->flags & ADA_FLAG_OPEN)) {
3466                                 adadump(softc->disk, NULL, 0, 0, 0);
3467                         }
3468                         continue;
3469                 }
3470                 cam_periph_lock(periph);
3471                 /*
3472                  * We only sync the cache if the drive is still open, and
3473                  * if the drive is capable of it..
3474                  */
3475                 if (((softc->flags & ADA_FLAG_OPEN) == 0) ||
3476                     (softc->flags & ADA_FLAG_CAN_FLUSHCACHE) == 0) {
3477                         cam_periph_unlock(periph);
3478                         continue;
3479                 }
3480
3481                 ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL);
3482                 cam_fill_ataio(&ccb->ataio,
3483                                     0,
3484                                     NULL,
3485                                     CAM_DIR_NONE,
3486                                     0,
3487                                     NULL,
3488                                     0,
3489                                     ada_default_timeout*1000);
3490                 if (softc->flags & ADA_FLAG_CAN_48BIT)
3491                         ata_48bit_cmd(&ccb->ataio, ATA_FLUSHCACHE48, 0, 0, 0);
3492                 else
3493                         ata_28bit_cmd(&ccb->ataio, ATA_FLUSHCACHE, 0, 0, 0);
3494
3495                 error = cam_periph_runccb(ccb, adaerror, /*cam_flags*/0,
3496                     /*sense_flags*/ SF_NO_RECOVERY | SF_NO_RETRY,
3497                     softc->disk->d_devstat);
3498                 if (error != 0)
3499                         xpt_print(periph->path, "Synchronize cache failed\n");
3500                 xpt_release_ccb(ccb);
3501                 cam_periph_unlock(periph);
3502         }
3503 }
3504
3505 static void
3506 adaspindown(uint8_t cmd, int flags)
3507 {
3508         struct cam_periph *periph;
3509         struct ada_softc *softc;
3510         struct ccb_ataio local_ccb;
3511         int error;
3512
3513         CAM_PERIPH_FOREACH(periph, &adadriver) {
3514                 /* If we paniced with lock held - not recurse here. */
3515                 if (cam_periph_owned(periph))
3516                         continue;
3517                 cam_periph_lock(periph);
3518                 softc = (struct ada_softc *)periph->softc;
3519                 /*
3520                  * We only spin-down the drive if it is capable of it..
3521                  */
3522                 if ((softc->flags & ADA_FLAG_CAN_POWERMGT) == 0) {
3523                         cam_periph_unlock(periph);
3524                         continue;
3525                 }
3526
3527                 if (bootverbose)
3528                         xpt_print(periph->path, "spin-down\n");
3529
3530                 memset(&local_ccb, 0, sizeof(local_ccb));
3531                 xpt_setup_ccb(&local_ccb.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
3532                 local_ccb.ccb_h.ccb_state = ADA_CCB_DUMP;
3533
3534                 cam_fill_ataio(&local_ccb,
3535                                     0,
3536                                     NULL,
3537                                     CAM_DIR_NONE | flags,
3538                                     0,
3539                                     NULL,
3540                                     0,
3541                                     ada_default_timeout*1000);
3542                 ata_28bit_cmd(&local_ccb, cmd, 0, 0, 0);
3543                 error = cam_periph_runccb((union ccb *)&local_ccb, adaerror,
3544                     /*cam_flags*/0, /*sense_flags*/ SF_NO_RECOVERY | SF_NO_RETRY,
3545                     softc->disk->d_devstat);
3546                 if (error != 0)
3547                         xpt_print(periph->path, "Spin-down disk failed\n");
3548                 cam_periph_unlock(periph);
3549         }
3550 }
3551
3552 static void
3553 adashutdown(void *arg, int howto)
3554 {
3555         int how;
3556
3557         adaflush();
3558
3559         /*
3560          * STANDBY IMMEDIATE saves any volatile data to the drive. It also spins
3561          * down hard drives. IDLE IMMEDIATE also saves the volatile data without
3562          * a spindown. We send the former when we expect to lose power soon. For
3563          * a warm boot, we send the latter to avoid a thundering herd of spinups
3564          * just after the kernel loads while probing. We have to do something to
3565          * flush the data because the BIOS in many systems resets the HBA
3566          * causing a COMINIT/COMRESET negotiation, which some drives interpret
3567          * as license to toss the volatile data, and others count as unclean
3568          * shutdown when in the Active PM state in SMART attributes.
3569          *
3570          * adaspindown will ensure that we don't send this to a drive that
3571          * doesn't support it.
3572          */
3573         if (ada_spindown_shutdown != 0) {
3574                 how = (howto & (RB_HALT | RB_POWEROFF | RB_POWERCYCLE)) ?
3575                     ATA_STANDBY_IMMEDIATE : ATA_IDLE_IMMEDIATE;
3576                 adaspindown(how, 0);
3577         }
3578 }
3579
3580 static void
3581 adasuspend(void *arg)
3582 {
3583
3584         adaflush();
3585         /*
3586          * SLEEP also fushes any volatile data, like STANDBY IMEDIATE,
3587          * so we don't need to send it as well.
3588          */
3589         if (ada_spindown_suspend != 0)
3590                 adaspindown(ATA_SLEEP, CAM_DEV_QFREEZE);
3591 }
3592
3593 static void
3594 adaresume(void *arg)
3595 {
3596         struct cam_periph *periph;
3597         struct ada_softc *softc;
3598
3599         if (ada_spindown_suspend == 0)
3600                 return;
3601
3602         CAM_PERIPH_FOREACH(periph, &adadriver) {
3603                 cam_periph_lock(periph);
3604                 softc = (struct ada_softc *)periph->softc;
3605                 /*
3606                  * We only spin-down the drive if it is capable of it..
3607                  */
3608                 if ((softc->flags & ADA_FLAG_CAN_POWERMGT) == 0) {
3609                         cam_periph_unlock(periph);
3610                         continue;
3611                 }
3612
3613                 if (bootverbose)
3614                         xpt_print(periph->path, "resume\n");
3615
3616                 /*
3617                  * Drop freeze taken due to CAM_DEV_QFREEZE flag set on
3618                  * sleep request.
3619                  */
3620                 cam_release_devq(periph->path,
3621                          /*relsim_flags*/0,
3622                          /*openings*/0,
3623                          /*timeout*/0,
3624                          /*getcount_only*/0);
3625
3626                 cam_periph_unlock(periph);
3627         }
3628 }
3629
3630 #endif /* _KERNEL */