sys/cam/ata/ata_da.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2009 Alexander Motin <mav@FreeBSD.org>
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer,
  12  *    without modification, immediately at the beginning of the file.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  18  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  19  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  20  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  21  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  22  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  23  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  24  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  25  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  26  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27  */
  28
  29 #include <sys/cdefs.h>
  30 __FBSDID("$FreeBSD$");
  31
  32 #include "opt_ada.h"
  33
  34 #include <sys/param.h>
  35
  36 #ifdef _KERNEL
  37 #include <sys/systm.h>
  38 #include <sys/kernel.h>
  39 #include <sys/bio.h>
  40 #include <sys/sysctl.h>
  41 #include <sys/taskqueue.h>
  42 #include <sys/lock.h>
  43 #include <sys/mutex.h>
  44 #include <sys/conf.h>
  45 #include <sys/devicestat.h>
  46 #include <sys/eventhandler.h>
  47 #include <sys/malloc.h>
  48 #include <sys/endian.h>
  49 #include <sys/cons.h>
  50 #include <sys/proc.h>
  51 #include <sys/reboot.h>
  52 #include <sys/sbuf.h>
  53 #include <geom/geom_disk.h>
  54 #endif /* _KERNEL */
  55
  56 #ifndef _KERNEL
  57 #include <stdio.h>
  58 #include <string.h>
  59 #endif /* _KERNEL */
  60
  61 #include <cam/cam.h>
  62 #include <cam/cam_ccb.h>
  63 #include <cam/cam_periph.h>
  64 #include <cam/cam_xpt_periph.h>
  65 #include <cam/scsi/scsi_all.h>
  66 #include <cam/scsi/scsi_da.h>
  67 #include <cam/cam_sim.h>
  68 #include <cam/cam_iosched.h>
  69
  70 #include <cam/ata/ata_all.h>
  71
  72 #include <machine/md_var.h>     /* geometry translation */
  73
  74 #ifdef _KERNEL
  75
  76 #define ATA_MAX_28BIT_LBA               268435455UL
  77
  78 extern int iosched_debug;
  79
  80 typedef enum {
  81         ADA_STATE_RAHEAD,
  82         ADA_STATE_WCACHE,
  83         ADA_STATE_LOGDIR,
  84         ADA_STATE_IDDIR,
  85         ADA_STATE_SUP_CAP,
  86         ADA_STATE_ZONE,
  87         ADA_STATE_NORMAL
  88 } ada_state;
  89
  90 typedef enum {
  91         ADA_FLAG_CAN_48BIT      = 0x00000002,
  92         ADA_FLAG_CAN_FLUSHCACHE = 0x00000004,
  93         ADA_FLAG_CAN_NCQ        = 0x00000008,
  94         ADA_FLAG_CAN_DMA        = 0x00000010,
  95         ADA_FLAG_NEED_OTAG      = 0x00000020,
  96         ADA_FLAG_WAS_OTAG       = 0x00000040,
  97         ADA_FLAG_CAN_TRIM       = 0x00000080,
  98         ADA_FLAG_OPEN           = 0x00000100,
  99         ADA_FLAG_SCTX_INIT      = 0x00000200,
 100         ADA_FLAG_CAN_CFA        = 0x00000400,
 101         ADA_FLAG_CAN_POWERMGT   = 0x00000800,
 102         ADA_FLAG_CAN_DMA48      = 0x00001000,
 103         ADA_FLAG_CAN_LOG        = 0x00002000,
 104         ADA_FLAG_CAN_IDLOG      = 0x00004000,
 105         ADA_FLAG_CAN_SUPCAP     = 0x00008000,
 106         ADA_FLAG_CAN_ZONE       = 0x00010000,
 107         ADA_FLAG_CAN_WCACHE     = 0x00020000,
 108         ADA_FLAG_CAN_RAHEAD     = 0x00040000,
 109         ADA_FLAG_PROBED         = 0x00080000,
 110         ADA_FLAG_ANNOUNCED      = 0x00100000,
 111         ADA_FLAG_DIRTY          = 0x00200000,
 112         ADA_FLAG_CAN_NCQ_TRIM   = 0x00400000,   /* CAN_TRIM also set */
 113         ADA_FLAG_PIM_ATA_EXT    = 0x00800000
 114 } ada_flags;
 115
 116 typedef enum {
 117         ADA_Q_NONE              = 0x00,
 118         ADA_Q_4K                = 0x01,
 119         ADA_Q_NCQ_TRIM_BROKEN   = 0x02,
 120         ADA_Q_LOG_BROKEN        = 0x04,
 121         ADA_Q_SMR_DM            = 0x08,
 122         ADA_Q_NO_TRIM           = 0x10,
 123         ADA_Q_128KB             = 0x20
 124 } ada_quirks;
 125
 126 #define ADA_Q_BIT_STRING        \
 127         "\020"                  \
 128         "\0014K"                \
 129         "\002NCQ_TRIM_BROKEN"   \
 130         "\003LOG_BROKEN"        \
 131         "\004SMR_DM"            \
 132         "\005NO_TRIM"           \
 133         "\006128KB"
 134
 135 typedef enum {
 136         ADA_CCB_RAHEAD          = 0x01,
 137         ADA_CCB_WCACHE          = 0x02,
 138         ADA_CCB_BUFFER_IO       = 0x03,
 139         ADA_CCB_DUMP            = 0x05,
 140         ADA_CCB_TRIM            = 0x06,
 141         ADA_CCB_LOGDIR          = 0x07,
 142         ADA_CCB_IDDIR           = 0x08,
 143         ADA_CCB_SUP_CAP         = 0x09,
 144         ADA_CCB_ZONE            = 0x0a,
 145         ADA_CCB_TYPE_MASK       = 0x0F,
 146 } ada_ccb_state;
 147
 148 typedef enum {
 149         ADA_ZONE_NONE           = 0x00,
 150         ADA_ZONE_DRIVE_MANAGED  = 0x01,
 151         ADA_ZONE_HOST_AWARE     = 0x02,
 152         ADA_ZONE_HOST_MANAGED   = 0x03
 153 } ada_zone_mode;
 154
 155 typedef enum {
 156         ADA_ZONE_FLAG_RZ_SUP            = 0x0001,
 157         ADA_ZONE_FLAG_OPEN_SUP          = 0x0002,
 158         ADA_ZONE_FLAG_CLOSE_SUP         = 0x0004,
 159         ADA_ZONE_FLAG_FINISH_SUP        = 0x0008,
 160         ADA_ZONE_FLAG_RWP_SUP           = 0x0010,
 161         ADA_ZONE_FLAG_SUP_MASK          = (ADA_ZONE_FLAG_RZ_SUP |
 162                                            ADA_ZONE_FLAG_OPEN_SUP |
 163                                            ADA_ZONE_FLAG_CLOSE_SUP |
 164                                            ADA_ZONE_FLAG_FINISH_SUP |
 165                                            ADA_ZONE_FLAG_RWP_SUP),
 166         ADA_ZONE_FLAG_URSWRZ            = 0x0020,
 167         ADA_ZONE_FLAG_OPT_SEQ_SET       = 0x0040,
 168         ADA_ZONE_FLAG_OPT_NONSEQ_SET    = 0x0080,
 169         ADA_ZONE_FLAG_MAX_SEQ_SET       = 0x0100,
 170         ADA_ZONE_FLAG_SET_MASK          = (ADA_ZONE_FLAG_OPT_SEQ_SET |
 171                                            ADA_ZONE_FLAG_OPT_NONSEQ_SET |
 172                                            ADA_ZONE_FLAG_MAX_SEQ_SET)
 173 } ada_zone_flags;
 174
 175 static struct ada_zone_desc {
 176         ada_zone_flags value;
 177         const char *desc;
 178 } ada_zone_desc_table[] = {
 179         {ADA_ZONE_FLAG_RZ_SUP, "Report Zones" },
 180         {ADA_ZONE_FLAG_OPEN_SUP, "Open" },
 181         {ADA_ZONE_FLAG_CLOSE_SUP, "Close" },
 182         {ADA_ZONE_FLAG_FINISH_SUP, "Finish" },
 183         {ADA_ZONE_FLAG_RWP_SUP, "Reset Write Pointer" },
 184 };
 185
 186
 187 /* Offsets into our private area for storing information */
 188 #define ccb_state       ppriv_field0
 189 #define ccb_bp          ppriv_ptr1
 190
 191 typedef enum {
 192         ADA_DELETE_NONE,
 193         ADA_DELETE_DISABLE,
 194         ADA_DELETE_CFA_ERASE,
 195         ADA_DELETE_DSM_TRIM,
 196         ADA_DELETE_NCQ_DSM_TRIM,
 197         ADA_DELETE_MIN = ADA_DELETE_CFA_ERASE,
 198         ADA_DELETE_MAX = ADA_DELETE_NCQ_DSM_TRIM,
 199 } ada_delete_methods;
 200
 201 static const char *ada_delete_method_names[] =
 202     { "NONE", "DISABLE", "CFA_ERASE", "DSM_TRIM", "NCQ_DSM_TRIM" };
 203 #if 0
 204 static const char *ada_delete_method_desc[] =
 205     { "NONE", "DISABLED", "CFA Erase", "DSM Trim", "DSM Trim via NCQ" };
 206 #endif
 207
 208 struct disk_params {
 209         u_int8_t  heads;
 210         u_int8_t  secs_per_track;
 211         u_int32_t cylinders;
 212         u_int32_t secsize;      /* Number of bytes/logical sector */
 213         u_int64_t sectors;      /* Total number sectors */
 214 };
 215
 216 #define TRIM_MAX_BLOCKS 8
 217 #define TRIM_MAX_RANGES (TRIM_MAX_BLOCKS * ATA_DSM_BLK_RANGES)
 218 struct trim_request {
 219         uint8_t         data[TRIM_MAX_RANGES * ATA_DSM_RANGE_SIZE];
 220         TAILQ_HEAD(, bio) bps;
 221 };
 222
 223 struct ada_softc {
 224         struct   cam_iosched_softc *cam_iosched;
 225         int      outstanding_cmds;      /* Number of active commands */
 226         int      refcount;              /* Active xpt_action() calls */
 227         ada_state state;
 228         ada_flags flags;
 229         ada_zone_mode zone_mode;
 230         ada_zone_flags zone_flags;
 231         struct ata_gp_log_dir ata_logdir;
 232         int valid_logdir_len;
 233         struct ata_identify_log_pages ata_iddir;
 234         int valid_iddir_len;
 235         uint64_t optimal_seq_zones;
 236         uint64_t optimal_nonseq_zones;
 237         uint64_t max_seq_zones;
 238         ada_quirks quirks;
 239         ada_delete_methods delete_method;
 240         int      trim_max_ranges;
 241         int      read_ahead;
 242         int      write_cache;
 243         int      unmappedio;
 244         int      rotating;
 245 #ifdef CAM_TEST_FAILURE
 246         int      force_read_error;
 247         int      force_write_error;
 248         int      periodic_read_error;
 249         int      periodic_read_count;
 250 #endif
 251         struct ccb_pathinq      cpi;
 252         struct disk_params      params;
 253         struct disk             *disk;
 254         struct task             sysctl_task;
 255         struct sysctl_ctx_list  sysctl_ctx;
 256         struct sysctl_oid       *sysctl_tree;
 257         struct callout          sendordered_c;
 258         struct trim_request     trim_req;
 259         uint64_t                trim_count;
 260         uint64_t                trim_ranges;
 261         uint64_t                trim_lbas;
 262 #ifdef CAM_IO_STATS
 263         struct sysctl_ctx_list  sysctl_stats_ctx;
 264         struct sysctl_oid       *sysctl_stats_tree;
 265         u_int   timeouts;
 266         u_int   errors;
 267         u_int   invalidations;
 268 #endif
 269 #define ADA_ANNOUNCETMP_SZ 80
 270         char    announce_temp[ADA_ANNOUNCETMP_SZ];
 271 #define ADA_ANNOUNCE_SZ 400
 272         char    announce_buffer[ADA_ANNOUNCE_SZ];
 273 };
 274
 275 struct ada_quirk_entry {
 276         struct scsi_inquiry_pattern inq_pat;
 277         ada_quirks quirks;
 278 };
 279
 280 static struct ada_quirk_entry ada_quirk_table[] =
 281 {
 282         {
 283                 /* Sandisk X400 */
 284                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SanDisk?SD8SB8U1T00*", "X4162000*" },
 285                 /*quirks*/ADA_Q_128KB
 286         },
 287         {
 288                 /* Hitachi Advanced Format (4k) drives */
 289                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Hitachi H??????????E3*", "*" },
 290                 /*quirks*/ADA_Q_4K
 291         },
 292         {
 293                 /* Samsung Advanced Format (4k) drives */
 294                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG HD155UI*", "*" },
 295                 /*quirks*/ADA_Q_4K
 296         },
 297         {
 298                 /* Samsung Advanced Format (4k) drives */
 299                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG HD204UI*", "*" },
 300                 /*quirks*/ADA_Q_4K
 301         },
 302         {
 303                 /* Seagate Barracuda Green Advanced Format (4k) drives */
 304                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST????DL*", "*" },
 305                 /*quirks*/ADA_Q_4K
 306         },
 307         {
 308                 /* Seagate Barracuda Advanced Format (4k) drives */
 309                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST???DM*", "*" },
 310                 /*quirks*/ADA_Q_4K
 311         },
 312         {
 313                 /* Seagate Barracuda Advanced Format (4k) drives */
 314                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST????DM*", "*" },
 315                 /*quirks*/ADA_Q_4K
 316         },
 317         {
 318                 /* Seagate Momentus Advanced Format (4k) drives */
 319                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9500423AS*", "*" },
 320                 /*quirks*/ADA_Q_4K
 321         },
 322         {
 323                 /* Seagate Momentus Advanced Format (4k) drives */
 324                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9500424AS*", "*" },
 325                 /*quirks*/ADA_Q_4K
 326         },
 327         {
 328                 /* Seagate Momentus Advanced Format (4k) drives */
 329                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9640423AS*", "*" },
 330                 /*quirks*/ADA_Q_4K
 331         },
 332         {
 333                 /* Seagate Momentus Advanced Format (4k) drives */
 334                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9640424AS*", "*" },
 335                 /*quirks*/ADA_Q_4K
 336         },
 337         {
 338                 /* Seagate Momentus Advanced Format (4k) drives */
 339                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9750420AS*", "*" },
 340                 /*quirks*/ADA_Q_4K
 341         },
 342         {
 343                 /* Seagate Momentus Advanced Format (4k) drives */
 344                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9750422AS*", "*" },
 345                 /*quirks*/ADA_Q_4K
 346         },
 347         {
 348                 /* Seagate Momentus Advanced Format (4k) drives */
 349                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9750423AS*", "*" },
 350                 /*quirks*/ADA_Q_4K
 351         },
 352         {
 353                 /* Seagate Momentus Thin Advanced Format (4k) drives */
 354                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST???LT*", "*" },
 355                 /*quirks*/ADA_Q_4K
 356         },
 357         {
 358                 /* WDC Caviar Red Advanced Format (4k) drives */
 359                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD????CX*", "*" },
 360                 /*quirks*/ADA_Q_4K
 361         },
 362         {
 363                 /* WDC Caviar Green Advanced Format (4k) drives */
 364                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD????RS*", "*" },
 365                 /*quirks*/ADA_Q_4K
 366         },
 367         {
 368                 /* WDC Caviar Green/Red Advanced Format (4k) drives */
 369                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD????RX*", "*" },
 370                 /*quirks*/ADA_Q_4K
 371         },
 372         {
 373                 /* WDC Caviar Red Advanced Format (4k) drives */
 374                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD??????CX*", "*" },
 375                 /*quirks*/ADA_Q_4K
 376         },
 377         {
 378                 /* WDC Caviar Black Advanced Format (4k) drives */
 379                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD????AZEX*", "*" },
 380                 /*quirks*/ADA_Q_4K
 381         },
 382         {
 383                 /* WDC Caviar Black Advanced Format (4k) drives */
 384                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD????FZEX*", "*" },
 385                 /*quirks*/ADA_Q_4K
 386         },
 387         {
 388                 /* WDC Caviar Green Advanced Format (4k) drives */
 389                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD??????RS*", "*" },
 390                 /*quirks*/ADA_Q_4K
 391         },
 392         {
 393                 /* WDC Caviar Green Advanced Format (4k) drives */
 394                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD??????RX*", "*" },
 395                 /*quirks*/ADA_Q_4K
 396         },
 397         {
 398                 /* WDC Scorpio Black Advanced Format (4k) drives */
 399                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD???PKT*", "*" },
 400                 /*quirks*/ADA_Q_4K
 401         },
 402         {
 403                 /* WDC Scorpio Black Advanced Format (4k) drives */
 404                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD?????PKT*", "*" },
 405                 /*quirks*/ADA_Q_4K
 406         },
 407         {
 408                 /* WDC Scorpio Blue Advanced Format (4k) drives */
 409                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD???PVT*", "*" },
 410                 /*quirks*/ADA_Q_4K
 411         },
 412         {
 413                 /* WDC Scorpio Blue Advanced Format (4k) drives */
 414                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD?????PVT*", "*" },
 415                 /*quirks*/ADA_Q_4K
 416         },
 417         /* SSDs */
 418         {
 419                 /*
 420                  * Corsair Force 2 SSDs
 421                  * 4k optimised & trim only works in 4k requests + 4k aligned
 422                  */
 423                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair CSSD-F*", "*" },
 424                 /*quirks*/ADA_Q_4K
 425         },
 426         {
 427                 /*
 428                  * Corsair Force 3 SSDs
 429                  * 4k optimised & trim only works in 4k requests + 4k aligned
 430                  */
 431                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair Force 3*", "*" },
 432                 /*quirks*/ADA_Q_4K
 433         },
 434         {
 435                 /*
 436                  * Corsair Neutron GTX SSDs
 437                  * 4k optimised & trim only works in 4k requests + 4k aligned
 438                  */
 439                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair Neutron GTX*", "*" },
 440                 /*quirks*/ADA_Q_4K
 441         },
 442         {
 443                 /*
 444                  * Corsair Force GT & GS SSDs
 445                  * 4k optimised & trim only works in 4k requests + 4k aligned
 446                  */
 447                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair Force G*", "*" },
 448                 /*quirks*/ADA_Q_4K
 449         },
 450         {
 451                 /*
 452                  * Crucial M4 SSDs
 453                  * 4k optimised & trim only works in 4k requests + 4k aligned
 454                  */
 455                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "M4-CT???M4SSD2*", "*" },
 456                 /*quirks*/ADA_Q_4K
 457         },
 458         {
 459                 /*
 460                  * Crucial M500 SSDs MU07 firmware
 461                  * NCQ Trim works
 462                  */
 463                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Crucial CT*M500*", "MU07" },
 464                 /*quirks*/0
 465         },
 466         {
 467                 /*
 468                  * Crucial M500 SSDs all other firmware
 469                  * NCQ Trim doesn't work
 470                  */
 471                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Crucial CT*M500*", "*" },
 472                 /*quirks*/ADA_Q_NCQ_TRIM_BROKEN
 473         },
 474         {
 475                 /*
 476                  * Crucial M550 SSDs
 477                  * NCQ Trim doesn't work, but only on MU01 firmware
 478                  */
 479                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Crucial CT*M550*", "MU01" },
 480                 /*quirks*/ADA_Q_NCQ_TRIM_BROKEN
 481         },
 482         {
 483                 /*
 484                  * Crucial MX100 SSDs
 485                  * NCQ Trim doesn't work, but only on MU01 firmware
 486                  */
 487                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Crucial CT*MX100*", "MU01" },
 488                 /*quirks*/ADA_Q_NCQ_TRIM_BROKEN
 489         },
 490         {
 491                 /*
 492                  * Crucial RealSSD C300 SSDs
 493                  * 4k optimised
 494                  */
 495                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "C300-CTFDDAC???MAG*",
 496                 "*" }, /*quirks*/ADA_Q_4K
 497         },
 498         {
 499                 /*
 500                  * FCCT M500 SSDs
 501                  * NCQ Trim doesn't work
 502                  */
 503                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "FCCT*M500*", "*" },
 504                 /*quirks*/ADA_Q_NCQ_TRIM_BROKEN
 505         },
 506         {
 507                 /*
 508                  * Intel 320 Series SSDs
 509                  * 4k optimised & trim only works in 4k requests + 4k aligned
 510                  */
 511                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSA2CW*", "*" },
 512                 /*quirks*/ADA_Q_4K
 513         },
 514         {
 515                 /*
 516                  * Intel 330 Series SSDs
 517                  * 4k optimised & trim only works in 4k requests + 4k aligned
 518                  */
 519                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSC2CT*", "*" },
 520                 /*quirks*/ADA_Q_4K
 521         },
 522         {
 523                 /*
 524                  * Intel 510 Series SSDs
 525                  * 4k optimised & trim only works in 4k requests + 4k aligned
 526                  */
 527                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSC2MH*", "*" },
 528                 /*quirks*/ADA_Q_4K
 529         },
 530         {
 531                 /*
 532                  * Intel 520 Series SSDs
 533                  * 4k optimised & trim only works in 4k requests + 4k aligned
 534                  */
 535                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSC2BW*", "*" },
 536                 /*quirks*/ADA_Q_4K
 537         },
 538         {
 539                 /*
 540                  * Intel S3610 Series SSDs
 541                  * 4k optimised & trim only works in 4k requests + 4k aligned
 542                  */
 543                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSC2BX*", "*" },
 544                 /*quirks*/ADA_Q_4K
 545         },
 546         {
 547                 /*
 548                  * Intel X25-M Series SSDs
 549                  * 4k optimised & trim only works in 4k requests + 4k aligned
 550                  */
 551                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSA2M*", "*" },
 552                 /*quirks*/ADA_Q_4K
 553         },
 554         {
 555                 /*
 556                  * KingDian S200 60GB P0921B
 557                  * Trimming crash the SSD
 558                  */
 559                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "KingDian S200 *", "*" },
 560                 /*quirks*/ADA_Q_NO_TRIM
 561         },
 562         {
 563                 /*
 564                  * Kingston E100 Series SSDs
 565                  * 4k optimised & trim only works in 4k requests + 4k aligned
 566                  */
 567                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "KINGSTON SE100S3*", "*" },
 568                 /*quirks*/ADA_Q_4K
 569         },
 570         {
 571                 /*
 572                  * Kingston HyperX 3k SSDs
 573                  * 4k optimised & trim only works in 4k requests + 4k aligned
 574                  */
 575                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "KINGSTON SH103S3*", "*" },
 576                 /*quirks*/ADA_Q_4K
 577         },
 578         {
 579                 /*
 580                  * Marvell SSDs (entry taken from OpenSolaris)
 581                  * 4k optimised & trim only works in 4k requests + 4k aligned
 582                  */
 583                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "MARVELL SD88SA02*", "*" },
 584                 /*quirks*/ADA_Q_4K
 585         },
 586         {
 587                 /*
 588                  * Micron M500 SSDs firmware MU07
 589                  * NCQ Trim works?
 590                  */
 591                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Micron M500*", "MU07" },
 592                 /*quirks*/0
 593         },
 594         {
 595                 /*
 596                  * Micron M500 SSDs all other firmware
 597                  * NCQ Trim doesn't work
 598                  */
 599                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Micron M500*", "*" },
 600                 /*quirks*/ADA_Q_NCQ_TRIM_BROKEN
 601         },
 602         {
 603                 /*
 604                  * Micron M5[15]0 SSDs
 605                  * NCQ Trim doesn't work, but only MU01 firmware
 606                  */
 607                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Micron M5[15]0*", "MU01" },
 608                 /*quirks*/ADA_Q_NCQ_TRIM_BROKEN
 609         },
 610         {
 611                 /*
 612                  * Micron 5100 SSDs
 613                  * 4k optimised & trim only works in 4k requests + 4k aligned
 614                  */
 615                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Micron 5100 MTFDDAK*", "*" },
 616                 /*quirks*/ADA_Q_4K
 617         },
 618         {
 619                 /*
 620                  * OCZ Agility 2 SSDs
 621                  * 4k optimised & trim only works in 4k requests + 4k aligned
 622                  */
 623                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-AGILITY2*", "*" },
 624                 /*quirks*/ADA_Q_4K
 625         },
 626         {
 627                 /*
 628                  * OCZ Agility 3 SSDs
 629                  * 4k optimised & trim only works in 4k requests + 4k aligned
 630                  */
 631                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-AGILITY3*", "*" },
 632                 /*quirks*/ADA_Q_4K
 633         },
 634         {
 635                 /*
 636                  * OCZ Deneva R Series SSDs
 637                  * 4k optimised & trim only works in 4k requests + 4k aligned
 638                  */
 639                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "DENRSTE251M45*", "*" },
 640                 /*quirks*/ADA_Q_4K
 641         },
 642         {
 643                 /*
 644                  * OCZ Vertex 2 SSDs (inc pro series)
 645                  * 4k optimised & trim only works in 4k requests + 4k aligned
 646                  */
 647                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ?VERTEX2*", "*" },
 648                 /*quirks*/ADA_Q_4K
 649         },
 650         {
 651                 /*
 652                  * OCZ Vertex 3 SSDs
 653                  * 4k optimised & trim only works in 4k requests + 4k aligned
 654                  */
 655                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-VERTEX3*", "*" },
 656                 /*quirks*/ADA_Q_4K
 657         },
 658         {
 659                 /*
 660                  * OCZ Vertex 4 SSDs
 661                  * 4k optimised & trim only works in 4k requests + 4k aligned
 662                  */
 663                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-VERTEX4*", "*" },
 664                 /*quirks*/ADA_Q_4K
 665         },
 666         {
 667                 /*
 668                  * Samsung 750 SSDs
 669                  * 4k optimised, NCQ TRIM seems to work
 670                  */
 671                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Samsung SSD 750*", "*" },
 672                 /*quirks*/ADA_Q_4K
 673         },
 674         {
 675                 /*
 676                  * Samsung 830 Series SSDs
 677                  * 4k optimised, NCQ TRIM Broken (normal TRIM is fine)
 678                  */
 679                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG SSD 830 Series*", "*" },
 680                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 681         },
 682         {
 683                 /*
 684                  * Samsung 840 SSDs
 685                  * 4k optimised, NCQ TRIM Broken (normal TRIM is fine)
 686                  */
 687                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Samsung SSD 840*", "*" },
 688                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 689         },
 690         {
 691                 /*
 692                  * Samsung 845 SSDs
 693                  * 4k optimised, NCQ TRIM Broken (normal TRIM is fine)
 694                  */
 695                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Samsung SSD 845*", "*" },
 696                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 697         },
 698         {
 699                 /*
 700                  * Samsung 850 SSDs
 701                  * 4k optimised, NCQ TRIM broken (normal TRIM fine)
 702                  */
 703                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Samsung SSD 850*", "*" },
 704                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 705         },
 706         {
 707                 /*
 708                  * Samsung SM863 Series SSDs (MZ7KM*)
 709                  * 4k optimised, NCQ believed to be working
 710                  */
 711                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG MZ7KM*", "*" },
 712                 /*quirks*/ADA_Q_4K
 713         },
 714         {
 715                 /*
 716                  * Samsung 843T Series SSDs (MZ7WD*)
 717                  * Samsung PM851 Series SSDs (MZ7TE*)
 718                  * Samsung PM853T Series SSDs (MZ7GE*)
 719                  * 4k optimised, NCQ believed to be broken since these are
 720                  * appear to be built with the same controllers as the 840/850.
 721                  */
 722                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG MZ7*", "*" },
 723                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 724         },
 725         {
 726                 /*
 727                  * Same as for SAMSUNG MZ7* but enable the quirks for SSD
 728                  * starting with MZ7* too
 729                  */
 730                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "MZ7*", "*" },
 731                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 732         },
 733         {
 734                 /*
 735                  * Samsung PM851 Series SSDs Dell OEM
 736                  * device model          "SAMSUNG SSD PM851 mSATA 256GB"
 737                  * 4k optimised, NCQ broken
 738                  */
 739                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG SSD PM851*", "*" },
 740                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 741         },
 742         {
 743                 /*
 744                  * SuperTalent TeraDrive CT SSDs
 745                  * 4k optimised & trim only works in 4k requests + 4k aligned
 746                  */
 747                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "FTM??CT25H*", "*" },
 748                 /*quirks*/ADA_Q_4K
 749         },
 750         {
 751                 /*
 752                  * XceedIOPS SATA SSDs
 753                  * 4k optimised
 754                  */
 755                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SG9XCS2D*", "*" },
 756                 /*quirks*/ADA_Q_4K
 757         },
 758         {
 759                 /*
 760                  * Samsung drive that doesn't support READ LOG EXT or
 761                  * READ LOG DMA EXT, despite reporting that it does in
 762                  * ATA identify data:
 763                  * SAMSUNG HD200HJ KF100-06
 764                  */
 765                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG HD200*", "*" },
 766                 /*quirks*/ADA_Q_LOG_BROKEN
 767         },
 768         {
 769                 /*
 770                  * Samsung drive that doesn't support READ LOG EXT or
 771                  * READ LOG DMA EXT, despite reporting that it does in
 772                  * ATA identify data:
 773                  * SAMSUNG HD501LJ CR100-10
 774                  */
 775                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG HD501*", "*" },
 776                 /*quirks*/ADA_Q_LOG_BROKEN
 777         },
 778         {
 779                 /*
 780                  * Seagate Lamarr 8TB Shingled Magnetic Recording (SMR)
 781                  * Drive Managed SATA hard drive.  This drive doesn't report
 782                  * in firmware that it is a drive managed SMR drive.
 783                  */
 784                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST8000AS000[23]*", "*" },
 785                 /*quirks*/ADA_Q_SMR_DM
 786         },
 787         {
 788                 /* Default */
 789                 {
 790                   T_ANY, SIP_MEDIA_REMOVABLE|SIP_MEDIA_FIXED,
 791                   /*vendor*/"*", /*product*/"*", /*revision*/"*"
 792                 },
 793                 /*quirks*/0
 794         },
 795 };
 796
 797 static  disk_strategy_t adastrategy;
 798 static  dumper_t        adadump;
 799 static  periph_init_t   adainit;
 800 static  void            adadiskgonecb(struct disk *dp);
 801 static  periph_oninv_t  adaoninvalidate;
 802 static  periph_dtor_t   adacleanup;
 803 static  void            adaasync(void *callback_arg, u_int32_t code,
 804                                 struct cam_path *path, void *arg);
 805 static  int             adazonemodesysctl(SYSCTL_HANDLER_ARGS);
 806 static  int             adazonesupsysctl(SYSCTL_HANDLER_ARGS);
 807 static  void            adasysctlinit(void *context, int pending);
 808 static  int             adagetattr(struct bio *bp);
 809 static  void            adasetflags(struct ada_softc *softc,
 810                                     struct ccb_getdev *cgd);
 811 static void             adasetgeom(struct ada_softc *softc,
 812                                    struct ccb_getdev *cgd);
 813 static  periph_ctor_t   adaregister;
 814 static  void            ada_dsmtrim(struct ada_softc *softc, struct bio *bp,
 815                                     struct ccb_ataio *ataio);
 816 static  void            ada_cfaerase(struct ada_softc *softc, struct bio *bp,
 817                                      struct ccb_ataio *ataio);
 818 static  int             ada_zone_bio_to_ata(int disk_zone_cmd);
 819 static  int             ada_zone_cmd(struct cam_periph *periph, union ccb *ccb,
 820                                      struct bio *bp, int *queue_ccb);
 821 static  periph_start_t  adastart;
 822 static  void            adaprobedone(struct cam_periph *periph, union ccb *ccb);
 823 static  void            adazonedone(struct cam_periph *periph, union ccb *ccb);
 824 static  void            adadone(struct cam_periph *periph,
 825                                union ccb *done_ccb);
 826 static  int             adaerror(union ccb *ccb, u_int32_t cam_flags,
 827                                 u_int32_t sense_flags);
 828 static timeout_t        adasendorderedtag;
 829 static void             adashutdown(void *arg, int howto);
 830 static void             adasuspend(void *arg);
 831 static void             adaresume(void *arg);
 832
 833 #ifndef ADA_DEFAULT_TIMEOUT
 834 #define ADA_DEFAULT_TIMEOUT 30  /* Timeout in seconds */
 835 #endif
 836
 837 #ifndef ADA_DEFAULT_RETRY
 838 #define ADA_DEFAULT_RETRY       4
 839 #endif
 840
 841 #ifndef ADA_DEFAULT_SEND_ORDERED
 842 #define ADA_DEFAULT_SEND_ORDERED        1
 843 #endif
 844
 845 #ifndef ADA_DEFAULT_SPINDOWN_SHUTDOWN
 846 #define ADA_DEFAULT_SPINDOWN_SHUTDOWN   1
 847 #endif
 848
 849 #ifndef ADA_DEFAULT_SPINDOWN_SUSPEND
 850 #define ADA_DEFAULT_SPINDOWN_SUSPEND    1
 851 #endif
 852
 853 #ifndef ADA_DEFAULT_READ_AHEAD
 854 #define ADA_DEFAULT_READ_AHEAD  1
 855 #endif
 856
 857 #ifndef ADA_DEFAULT_WRITE_CACHE
 858 #define ADA_DEFAULT_WRITE_CACHE 1
 859 #endif
 860
 861 #define ADA_RA  (softc->read_ahead >= 0 ? \
 862                  softc->read_ahead : ada_read_ahead)
 863 #define ADA_WC  (softc->write_cache >= 0 ? \
 864                  softc->write_cache : ada_write_cache)
 865
 866 /*
 867  * Most platforms map firmware geometry to actual, but some don't.  If
 868  * not overridden, default to nothing.
 869  */
 870 #ifndef ata_disk_firmware_geom_adjust
 871 #define ata_disk_firmware_geom_adjust(disk)
 872 #endif
 873
 874 static int ada_retry_count = ADA_DEFAULT_RETRY;
 875 static int ada_default_timeout = ADA_DEFAULT_TIMEOUT;
 876 static int ada_send_ordered = ADA_DEFAULT_SEND_ORDERED;
 877 static int ada_spindown_shutdown = ADA_DEFAULT_SPINDOWN_SHUTDOWN;
 878 static int ada_spindown_suspend = ADA_DEFAULT_SPINDOWN_SUSPEND;
 879 static int ada_read_ahead = ADA_DEFAULT_READ_AHEAD;
 880 static int ada_write_cache = ADA_DEFAULT_WRITE_CACHE;
 881
 882 static SYSCTL_NODE(_kern_cam, OID_AUTO, ada, CTLFLAG_RD, 0,
 883             "CAM Direct Access Disk driver");
 884 SYSCTL_INT(_kern_cam_ada, OID_AUTO, retry_count, CTLFLAG_RWTUN,
 885            &ada_retry_count, 0, "Normal I/O retry count");
 886 SYSCTL_INT(_kern_cam_ada, OID_AUTO, default_timeout, CTLFLAG_RWTUN,
 887            &ada_default_timeout, 0, "Normal I/O timeout (in seconds)");
 888 SYSCTL_INT(_kern_cam_ada, OID_AUTO, send_ordered, CTLFLAG_RWTUN,
 889            &ada_send_ordered, 0, "Send Ordered Tags");
 890 SYSCTL_INT(_kern_cam_ada, OID_AUTO, spindown_shutdown, CTLFLAG_RWTUN,
 891            &ada_spindown_shutdown, 0, "Spin down upon shutdown");
 892 SYSCTL_INT(_kern_cam_ada, OID_AUTO, spindown_suspend, CTLFLAG_RWTUN,
 893            &ada_spindown_suspend, 0, "Spin down upon suspend");
 894 SYSCTL_INT(_kern_cam_ada, OID_AUTO, read_ahead, CTLFLAG_RWTUN,
 895            &ada_read_ahead, 0, "Enable disk read-ahead");
 896 SYSCTL_INT(_kern_cam_ada, OID_AUTO, write_cache, CTLFLAG_RWTUN,
 897            &ada_write_cache, 0, "Enable disk write cache");
 898
 899 /*
 900  * ADA_ORDEREDTAG_INTERVAL determines how often, relative
 901  * to the default timeout, we check to see whether an ordered
 902  * tagged transaction is appropriate to prevent simple tag
 903  * starvation.  Since we'd like to ensure that there is at least
 904  * 1/2 of the timeout length left for a starved transaction to
 905  * complete after we've sent an ordered tag, we must poll at least
 906  * four times in every timeout period.  This takes care of the worst
 907  * case where a starved transaction starts during an interval that
 908  * meets the requirement "don't send an ordered tag" test so it takes
 909  * us two intervals to determine that a tag must be sent.
 910  */
 911 #ifndef ADA_ORDEREDTAG_INTERVAL
 912 #define ADA_ORDEREDTAG_INTERVAL 4
 913 #endif
 914
 915 static struct periph_driver adadriver =
 916 {
 917         adainit, "ada",
 918         TAILQ_HEAD_INITIALIZER(adadriver.units), /* generation */ 0
 919 };
 920
 921 static int adadeletemethodsysctl(SYSCTL_HANDLER_ARGS);
 922
 923 PERIPHDRIVER_DECLARE(ada, adadriver);
 924
 925 static MALLOC_DEFINE(M_ATADA, "ata_da", "ata_da buffers");
 926
 927 static int
 928 adaopen(struct disk *dp)
 929 {
 930         struct cam_periph *periph;
 931         struct ada_softc *softc;
 932         int error;
 933
 934         periph = (struct cam_periph *)dp->d_drv1;
 935         if (cam_periph_acquire(periph) != 0) {
 936                 return(ENXIO);
 937         }
 938
 939         cam_periph_lock(periph);
 940         if ((error = cam_periph_hold(periph, PRIBIO|PCATCH)) != 0) {
 941                 cam_periph_unlock(periph);
 942                 cam_periph_release(periph);
 943                 return (error);
 944         }
 945
 946         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE | CAM_DEBUG_PERIPH,
 947             ("adaopen\n"));
 948
 949         softc = (struct ada_softc *)periph->softc;
 950         softc->flags |= ADA_FLAG_OPEN;
 951
 952         cam_periph_unhold(periph);
 953         cam_periph_unlock(periph);
 954         return (0);
 955 }
 956
 957 static int
 958 adaclose(struct disk *dp)
 959 {
 960         struct  cam_periph *periph;
 961         struct  ada_softc *softc;
 962         union ccb *ccb;
 963         int error;
 964
 965         periph = (struct cam_periph *)dp->d_drv1;
 966         softc = (struct ada_softc *)periph->softc;
 967         cam_periph_lock(periph);
 968
 969         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE | CAM_DEBUG_PERIPH,
 970             ("adaclose\n"));
 971
 972         /* We only sync the cache if the drive is capable of it. */
 973         if ((softc->flags & ADA_FLAG_DIRTY) != 0 &&
 974             (softc->flags & ADA_FLAG_CAN_FLUSHCACHE) != 0 &&
 975             (periph->flags & CAM_PERIPH_INVALID) == 0 &&
 976             cam_periph_hold(periph, PRIBIO) == 0) {
 977
 978                 ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL);
 979                 cam_fill_ataio(&ccb->ataio,
 980                                     1,
 981                                     NULL,
 982                                     CAM_DIR_NONE,
 983                                     0,
 984                                     NULL,
 985                                     0,
 986                                     ada_default_timeout*1000);
 987
 988                 if (softc->flags & ADA_FLAG_CAN_48BIT)
 989                         ata_48bit_cmd(&ccb->ataio, ATA_FLUSHCACHE48, 0, 0, 0);
 990                 else
 991                         ata_28bit_cmd(&ccb->ataio, ATA_FLUSHCACHE, 0, 0, 0);
 992                 error = cam_periph_runccb(ccb, adaerror, /*cam_flags*/0,
 993                     /*sense_flags*/0, softc->disk->d_devstat);
 994
 995                 if (error != 0)
 996                         xpt_print(periph->path, "Synchronize cache failed\n");
 997                 softc->flags &= ~ADA_FLAG_DIRTY;
 998                 xpt_release_ccb(ccb);
 999                 cam_periph_unhold(periph);
1000         }
1001
1002         softc->flags &= ~ADA_FLAG_OPEN;
1003
1004         while (softc->refcount != 0)
1005                 cam_periph_sleep(periph, &softc->refcount, PRIBIO, "adaclose", 1);
1006         cam_periph_unlock(periph);
1007         cam_periph_release(periph);
1008         return (0);
1009 }
1010
1011 static void
1012 adaschedule(struct cam_periph *periph)
1013 {
1014         struct ada_softc *softc = (struct ada_softc *)periph->softc;
1015
1016         if (softc->state != ADA_STATE_NORMAL)
1017                 return;
1018
1019         cam_iosched_schedule(softc->cam_iosched, periph);
1020 }
1021
1022 /*
1023  * Actually translate the requested transfer into one the physical driver
1024  * can understand.  The transfer is described by a buf and will include
1025  * only one physical transfer.
1026  */
1027 static void
1028 adastrategy(struct bio *bp)
1029 {
1030         struct cam_periph *periph;
1031         struct ada_softc *softc;
1032
1033         periph = (struct cam_periph *)bp->bio_disk->d_drv1;
1034         softc = (struct ada_softc *)periph->softc;
1035
1036         cam_periph_lock(periph);
1037
1038         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("adastrategy(%p)\n", bp));
1039
1040         /*
1041          * If the device has been made invalid, error out
1042          */
1043         if ((periph->flags & CAM_PERIPH_INVALID) != 0) {
1044                 cam_periph_unlock(periph);
1045                 biofinish(bp, NULL, ENXIO);
1046                 return;
1047         }
1048
1049         /*
1050          * Zone commands must be ordered, because they can depend on the
1051          * effects of previously issued commands, and they may affect
1052          * commands after them.
1053          */
1054         if (bp->bio_cmd == BIO_ZONE)
1055                 bp->bio_flags |= BIO_ORDERED;
1056
1057         /*
1058          * Place it in the queue of disk activities for this disk
1059          */
1060         cam_iosched_queue_work(softc->cam_iosched, bp);
1061
1062         /*
1063          * Schedule ourselves for performing the work.
1064          */
1065         adaschedule(periph);
1066         cam_periph_unlock(periph);
1067
1068         return;
1069 }
1070
1071 static int
1072 adadump(void *arg, void *virtual, vm_offset_t physical, off_t offset, size_t length)
1073 {
1074         struct      cam_periph *periph;
1075         struct      ada_softc *softc;
1076         u_int       secsize;
1077         struct      ccb_ataio ataio;
1078         struct      disk *dp;
1079         uint64_t    lba;
1080         uint16_t    count;
1081         int         error = 0;
1082
1083         dp = arg;
1084         periph = dp->d_drv1;
1085         softc = (struct ada_softc *)periph->softc;
1086         secsize = softc->params.secsize;
1087         lba = offset / secsize;
1088         count = length / secsize;
1089         if ((periph->flags & CAM_PERIPH_INVALID) != 0)
1090                 return (ENXIO);
1091
1092         memset(&ataio, 0, sizeof(ataio));
1093         if (length > 0) {
1094                 xpt_setup_ccb(&ataio.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
1095                 ataio.ccb_h.ccb_state = ADA_CCB_DUMP;
1096                 cam_fill_ataio(&ataio,
1097                     0,
1098                     NULL,
1099                     CAM_DIR_OUT,
1100                     0,
1101                     (u_int8_t *) virtual,
1102                     length,
1103                     ada_default_timeout*1000);
1104                 if ((softc->flags & ADA_FLAG_CAN_48BIT) &&
1105                     (lba + count >= ATA_MAX_28BIT_LBA ||
1106                     count >= 256)) {
1107                         ata_48bit_cmd(&ataio, ATA_WRITE_DMA48,
1108                             0, lba, count);
1109                 } else {
1110                         ata_28bit_cmd(&ataio, ATA_WRITE_DMA,
1111                             0, lba, count);
1112                 }
1113                 error = cam_periph_runccb((union ccb *)&ataio, adaerror,
1114                     0, SF_NO_RECOVERY | SF_NO_RETRY, NULL);
1115                 if (error != 0)
1116                         printf("Aborting dump due to I/O error.\n");
1117
1118                 return (error);
1119         }
1120
1121         if (softc->flags & ADA_FLAG_CAN_FLUSHCACHE) {
1122                 xpt_setup_ccb(&ataio.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
1123
1124                 /*
1125                  * Tell the drive to flush its internal cache. if we
1126                  * can't flush in 5s we have big problems. No need to
1127                  * wait the default 60s to detect problems.
1128                  */
1129                 ataio.ccb_h.ccb_state = ADA_CCB_DUMP;
1130                 cam_fill_ataio(&ataio,
1131                                     0,
1132                                     NULL,
1133                                     CAM_DIR_NONE,
1134                                     0,
1135                                     NULL,
1136                                     0,
1137                                     5*1000);
1138
1139                 if (softc->flags & ADA_FLAG_CAN_48BIT)
1140                         ata_48bit_cmd(&ataio, ATA_FLUSHCACHE48, 0, 0, 0);
1141                 else
1142                         ata_28bit_cmd(&ataio, ATA_FLUSHCACHE, 0, 0, 0);
1143                 error = cam_periph_runccb((union ccb *)&ataio, adaerror,
1144                     0, SF_NO_RECOVERY | SF_NO_RETRY, NULL);
1145                 if (error != 0)
1146                         xpt_print(periph->path, "Synchronize cache failed\n");
1147         }
1148         return (error);
1149 }
1150
1151 static void
1152 adainit(void)
1153 {
1154         cam_status status;
1155
1156         /*
1157          * Install a global async callback.  This callback will
1158          * receive async callbacks like "new device found".
1159          */
1160         status = xpt_register_async(AC_FOUND_DEVICE, adaasync, NULL, NULL);
1161
1162         if (status != CAM_REQ_CMP) {
1163                 printf("ada: Failed to attach master async callback "
1164                        "due to status 0x%x!\n", status);
1165         } else if (ada_send_ordered) {
1166
1167                 /* Register our event handlers */
1168                 if ((EVENTHANDLER_REGISTER(power_suspend, adasuspend,
1169                                            NULL, EVENTHANDLER_PRI_LAST)) == NULL)
1170                     printf("adainit: power event registration failed!\n");
1171                 if ((EVENTHANDLER_REGISTER(power_resume, adaresume,
1172                                            NULL, EVENTHANDLER_PRI_LAST)) == NULL)
1173                     printf("adainit: power event registration failed!\n");
1174                 if ((EVENTHANDLER_REGISTER(shutdown_post_sync, adashutdown,
1175                                            NULL, SHUTDOWN_PRI_DEFAULT)) == NULL)
1176                     printf("adainit: shutdown event registration failed!\n");
1177         }
1178 }
1179
1180 /*
1181  * Callback from GEOM, called when it has finished cleaning up its
1182  * resources.
1183  */
1184 static void
1185 adadiskgonecb(struct disk *dp)
1186 {
1187         struct cam_periph *periph;
1188
1189         periph = (struct cam_periph *)dp->d_drv1;
1190
1191         cam_periph_release(periph);
1192 }
1193
1194 static void
1195 adaoninvalidate(struct cam_periph *periph)
1196 {
1197         struct ada_softc *softc;
1198
1199         softc = (struct ada_softc *)periph->softc;
1200
1201         /*
1202          * De-register any async callbacks.
1203          */
1204         xpt_register_async(0, adaasync, periph, periph->path);
1205 #ifdef CAM_IO_STATS
1206         softc->invalidations++;
1207 #endif
1208
1209         /*
1210          * Return all queued I/O with ENXIO.
1211          * XXX Handle any transactions queued to the card
1212          *     with XPT_ABORT_CCB.
1213          */
1214         cam_iosched_flush(softc->cam_iosched, NULL, ENXIO);
1215
1216         disk_gone(softc->disk);
1217 }
1218
1219 static void
1220 adacleanup(struct cam_periph *periph)
1221 {
1222         struct ada_softc *softc;
1223
1224         softc = (struct ada_softc *)periph->softc;
1225
1226         cam_periph_unlock(periph);
1227
1228         cam_iosched_fini(softc->cam_iosched);
1229
1230         /*
1231          * If we can't free the sysctl tree, oh well...
1232          */
1233         if ((softc->flags & ADA_FLAG_SCTX_INIT) != 0) {
1234 #ifdef CAM_IO_STATS
1235                 if (sysctl_ctx_free(&softc->sysctl_stats_ctx) != 0)
1236                         xpt_print(periph->path,
1237                             "can't remove sysctl stats context\n");
1238 #endif
1239                 if (sysctl_ctx_free(&softc->sysctl_ctx) != 0)
1240                         xpt_print(periph->path,
1241                             "can't remove sysctl context\n");
1242         }
1243
1244         disk_destroy(softc->disk);
1245         callout_drain(&softc->sendordered_c);
1246         free(softc, M_DEVBUF);
1247         cam_periph_lock(periph);
1248 }
1249
1250 static void
1251 adasetdeletemethod(struct ada_softc *softc)
1252 {
1253
1254         if (softc->flags & ADA_FLAG_CAN_NCQ_TRIM)
1255                 softc->delete_method = ADA_DELETE_NCQ_DSM_TRIM;
1256         else if (softc->flags & ADA_FLAG_CAN_TRIM)
1257                 softc->delete_method = ADA_DELETE_DSM_TRIM;
1258         else if ((softc->flags & ADA_FLAG_CAN_CFA) && !(softc->flags & ADA_FLAG_CAN_48BIT))
1259                 softc->delete_method = ADA_DELETE_CFA_ERASE;
1260         else
1261                 softc->delete_method = ADA_DELETE_NONE;
1262 }
1263
1264 static void
1265 adaasync(void *callback_arg, u_int32_t code,
1266         struct cam_path *path, void *arg)
1267 {
1268         struct ccb_getdev cgd;
1269         struct cam_periph *periph;
1270         struct ada_softc *softc;
1271
1272         periph = (struct cam_periph *)callback_arg;
1273         switch (code) {
1274         case AC_FOUND_DEVICE:
1275         {
1276                 struct ccb_getdev *cgd;
1277                 cam_status status;
1278
1279                 cgd = (struct ccb_getdev *)arg;
1280                 if (cgd == NULL)
1281                         break;
1282
1283                 if (cgd->protocol != PROTO_ATA)
1284                         break;
1285
1286                 /*
1287                  * Allocate a peripheral instance for
1288                  * this device and start the probe
1289                  * process.
1290                  */
1291                 status = cam_periph_alloc(adaregister, adaoninvalidate,
1292                                           adacleanup, adastart,
1293                                           "ada", CAM_PERIPH_BIO,
1294                                           path, adaasync,
1295                                           AC_FOUND_DEVICE, cgd);
1296
1297                 if (status != CAM_REQ_CMP
1298                  && status != CAM_REQ_INPROG)
1299                         printf("adaasync: Unable to attach to new device "
1300                                 "due to status 0x%x\n", status);
1301                 break;
1302         }
1303         case AC_GETDEV_CHANGED:
1304         {
1305                 softc = (struct ada_softc *)periph->softc;
1306                 xpt_setup_ccb(&cgd.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
1307                 cgd.ccb_h.func_code = XPT_GDEV_TYPE;
1308                 xpt_action((union ccb *)&cgd);
1309
1310                 /*
1311                  * Update our information based on the new Identify data.
1312                  */
1313                 adasetflags(softc, &cgd);
1314                 adasetgeom(softc, &cgd);
1315                 disk_resize(softc->disk, M_NOWAIT);
1316
1317                 cam_periph_async(periph, code, path, arg);
1318                 break;
1319         }
1320         case AC_ADVINFO_CHANGED:
1321         {
1322                 uintptr_t buftype;
1323
1324                 buftype = (uintptr_t)arg;
1325                 if (buftype == CDAI_TYPE_PHYS_PATH) {
1326                         struct ada_softc *softc;
1327
1328                         softc = periph->softc;
1329                         disk_attr_changed(softc->disk, "GEOM::physpath",
1330                                           M_NOWAIT);
1331                 }
1332                 break;
1333         }
1334         case AC_SENT_BDR:
1335         case AC_BUS_RESET:
1336         {
1337                 softc = (struct ada_softc *)periph->softc;
1338                 cam_periph_async(periph, code, path, arg);
1339                 if (softc->state != ADA_STATE_NORMAL)
1340                         break;
1341                 xpt_setup_ccb(&cgd.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
1342                 cgd.ccb_h.func_code = XPT_GDEV_TYPE;
1343                 xpt_action((union ccb *)&cgd);
1344                 if (ADA_RA >= 0 && softc->flags & ADA_FLAG_CAN_RAHEAD)
1345                         softc->state = ADA_STATE_RAHEAD;
1346                 else if (ADA_WC >= 0 && softc->flags & ADA_FLAG_CAN_WCACHE)
1347                         softc->state = ADA_STATE_WCACHE;
1348                 else if ((softc->flags & ADA_FLAG_CAN_LOG)
1349                       && (softc->zone_mode != ADA_ZONE_NONE))
1350                         softc->state = ADA_STATE_LOGDIR;
1351                 else
1352                     break;
1353                 if (cam_periph_acquire(periph) != 0)
1354                         softc->state = ADA_STATE_NORMAL;
1355                 else
1356                         xpt_schedule(periph, CAM_PRIORITY_DEV);
1357         }
1358         default:
1359                 cam_periph_async(periph, code, path, arg);
1360                 break;
1361         }
1362 }
1363
1364 static int
1365 adazonemodesysctl(SYSCTL_HANDLER_ARGS)
1366 {
1367         char tmpbuf[40];
1368         struct ada_softc *softc;
1369         int error;
1370
1371         softc = (struct ada_softc *)arg1;
1372
1373         switch (softc->zone_mode) {
1374         case ADA_ZONE_DRIVE_MANAGED:
1375                 snprintf(tmpbuf, sizeof(tmpbuf), "Drive Managed");
1376                 break;
1377         case ADA_ZONE_HOST_AWARE:
1378                 snprintf(tmpbuf, sizeof(tmpbuf), "Host Aware");
1379                 break;
1380         case ADA_ZONE_HOST_MANAGED:
1381                 snprintf(tmpbuf, sizeof(tmpbuf), "Host Managed");
1382                 break;
1383         case ADA_ZONE_NONE:
1384         default:
1385                 snprintf(tmpbuf, sizeof(tmpbuf), "Not Zoned");
1386                 break;
1387         }
1388
1389         error = sysctl_handle_string(oidp, tmpbuf, sizeof(tmpbuf), req);
1390
1391         return (error);
1392 }
1393
1394 static int
1395 adazonesupsysctl(SYSCTL_HANDLER_ARGS)
1396 {
1397         char tmpbuf[180];
1398         struct ada_softc *softc;
1399         struct sbuf sb;
1400         int error, first;
1401         unsigned int i;
1402
1403         softc = (struct ada_softc *)arg1;
1404
1405         error = 0;
1406         first = 1;
1407         sbuf_new(&sb, tmpbuf, sizeof(tmpbuf), 0);
1408
1409         for (i = 0; i < sizeof(ada_zone_desc_table) /
1410              sizeof(ada_zone_desc_table[0]); i++) {
1411                 if (softc->zone_flags & ada_zone_desc_table[i].value) {
1412                         if (first == 0)
1413                                 sbuf_printf(&sb, ", ");
1414                         else
1415                                 first = 0;
1416                         sbuf_cat(&sb, ada_zone_desc_table[i].desc);
1417                 }
1418         }
1419
1420         if (first == 1)
1421                 sbuf_printf(&sb, "None");
1422
1423         sbuf_finish(&sb);
1424
1425         error = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
1426
1427         return (error);
1428 }
1429
1430
1431 static void
1432 adasysctlinit(void *context, int pending)
1433 {
1434         struct cam_periph *periph;
1435         struct ada_softc *softc;
1436         char tmpstr[32], tmpstr2[16];
1437
1438         periph = (struct cam_periph *)context;
1439
1440         /* periph was held for us when this task was enqueued */
1441         if ((periph->flags & CAM_PERIPH_INVALID) != 0) {
1442                 cam_periph_release(periph);
1443                 return;
1444         }
1445
1446         softc = (struct ada_softc *)periph->softc;
1447         snprintf(tmpstr, sizeof(tmpstr), "CAM ADA unit %d",periph->unit_number);
1448         snprintf(tmpstr2, sizeof(tmpstr2), "%d", periph->unit_number);
1449
1450         sysctl_ctx_init(&softc->sysctl_ctx);
1451         softc->flags |= ADA_FLAG_SCTX_INIT;
1452         softc->sysctl_tree = SYSCTL_ADD_NODE_WITH_LABEL(&softc->sysctl_ctx,
1453                 SYSCTL_STATIC_CHILDREN(_kern_cam_ada), OID_AUTO, tmpstr2,
1454                 CTLFLAG_RD, 0, tmpstr, "device_index");
1455         if (softc->sysctl_tree == NULL) {
1456                 printf("adasysctlinit: unable to allocate sysctl tree\n");
1457                 cam_periph_release(periph);
1458                 return;
1459         }
1460
1461         SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1462                 OID_AUTO, "delete_method", CTLTYPE_STRING | CTLFLAG_RW,
1463                 softc, 0, adadeletemethodsysctl, "A",
1464                 "BIO_DELETE execution method");
1465         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
1466                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
1467                 "trim_count", CTLFLAG_RD, &softc->trim_count,
1468                 "Total number of dsm commands sent");
1469         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
1470                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
1471                 "trim_ranges", CTLFLAG_RD, &softc->trim_ranges,
1472                 "Total number of ranges in dsm commands");
1473         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
1474                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
1475                 "trim_lbas", CTLFLAG_RD, &softc->trim_lbas,
1476                 "Total lbas in the dsm commands sent");
1477         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1478                 OID_AUTO, "read_ahead", CTLFLAG_RW | CTLFLAG_MPSAFE,
1479                 &softc->read_ahead, 0, "Enable disk read ahead.");
1480         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1481                 OID_AUTO, "write_cache", CTLFLAG_RW | CTLFLAG_MPSAFE,
1482                 &softc->write_cache, 0, "Enable disk write cache.");
1483         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1484                 OID_AUTO, "unmapped_io", CTLFLAG_RD | CTLFLAG_MPSAFE,
1485                 &softc->unmappedio, 0, "Unmapped I/O leaf");
1486         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1487                 OID_AUTO, "rotating", CTLFLAG_RD | CTLFLAG_MPSAFE,
1488                 &softc->rotating, 0, "Rotating media");
1489         SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1490                 OID_AUTO, "zone_mode", CTLTYPE_STRING | CTLFLAG_RD,
1491                 softc, 0, adazonemodesysctl, "A",
1492                 "Zone Mode");
1493         SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1494                 OID_AUTO, "zone_support", CTLTYPE_STRING | CTLFLAG_RD,
1495                 softc, 0, adazonesupsysctl, "A",
1496                 "Zone Support");
1497         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
1498                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
1499                 "optimal_seq_zones", CTLFLAG_RD, &softc->optimal_seq_zones,
1500                 "Optimal Number of Open Sequential Write Preferred Zones");
1501         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
1502                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
1503                 "optimal_nonseq_zones", CTLFLAG_RD,
1504                 &softc->optimal_nonseq_zones,
1505                 "Optimal Number of Non-Sequentially Written Sequential Write "
1506                 "Preferred Zones");
1507         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
1508                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
1509                 "max_seq_zones", CTLFLAG_RD, &softc->max_seq_zones,
1510                 "Maximum Number of Open Sequential Write Required Zones");
1511
1512 #ifdef CAM_TEST_FAILURE
1513         /*
1514          * Add a 'door bell' sysctl which allows one to set it from userland
1515          * and cause something bad to happen.  For the moment, we only allow
1516          * whacking the next read or write.
1517          */
1518         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1519                 OID_AUTO, "force_read_error", CTLFLAG_RW | CTLFLAG_MPSAFE,
1520                 &softc->force_read_error, 0,
1521                 "Force a read error for the next N reads.");
1522         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1523                 OID_AUTO, "force_write_error", CTLFLAG_RW | CTLFLAG_MPSAFE,
1524                 &softc->force_write_error, 0,
1525                 "Force a write error for the next N writes.");
1526         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1527                 OID_AUTO, "periodic_read_error", CTLFLAG_RW | CTLFLAG_MPSAFE,
1528                 &softc->periodic_read_error, 0,
1529                 "Force a read error every N reads (don't set too low).");
1530         SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1531                 OID_AUTO, "invalidate", CTLTYPE_U64 | CTLFLAG_RW | CTLFLAG_MPSAFE,
1532                 periph, 0, cam_periph_invalidate_sysctl, "I",
1533                 "Write 1 to invalidate the drive immediately");
1534 #endif
1535
1536 #ifdef CAM_IO_STATS
1537         softc->sysctl_stats_tree = SYSCTL_ADD_NODE(&softc->sysctl_stats_ctx,
1538                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO, "stats",
1539                 CTLFLAG_RD, 0, "Statistics");
1540         SYSCTL_ADD_INT(&softc->sysctl_stats_ctx,
1541                 SYSCTL_CHILDREN(softc->sysctl_stats_tree),
1542                 OID_AUTO, "timeouts", CTLFLAG_RD | CTLFLAG_MPSAFE,
1543                 &softc->timeouts, 0,
1544                 "Device timeouts reported by the SIM");
1545         SYSCTL_ADD_INT(&softc->sysctl_stats_ctx,
1546                 SYSCTL_CHILDREN(softc->sysctl_stats_tree),
1547                 OID_AUTO, "errors", CTLFLAG_RD | CTLFLAG_MPSAFE,
1548                 &softc->errors, 0,
1549                 "Transport errors reported by the SIM.");
1550         SYSCTL_ADD_INT(&softc->sysctl_stats_ctx,
1551                 SYSCTL_CHILDREN(softc->sysctl_stats_tree),
1552                 OID_AUTO, "pack_invalidations", CTLFLAG_RD | CTLFLAG_MPSAFE,
1553                 &softc->invalidations, 0,
1554                 "Device pack invalidations.");
1555 #endif
1556
1557         cam_iosched_sysctl_init(softc->cam_iosched, &softc->sysctl_ctx,
1558             softc->sysctl_tree);
1559
1560         cam_periph_release(periph);
1561 }
1562
1563 static int
1564 adagetattr(struct bio *bp)
1565 {
1566         int ret;
1567         struct cam_periph *periph;
1568
1569         periph = (struct cam_periph *)bp->bio_disk->d_drv1;
1570         cam_periph_lock(periph);
1571         ret = xpt_getattr(bp->bio_data, bp->bio_length, bp->bio_attribute,
1572             periph->path);
1573         cam_periph_unlock(periph);
1574         if (ret == 0)
1575                 bp->bio_completed = bp->bio_length;
1576         return ret;
1577 }
1578
1579 static int
1580 adadeletemethodsysctl(SYSCTL_HANDLER_ARGS)
1581 {
1582         char buf[16];
1583         const char *p;
1584         struct ada_softc *softc;
1585         int i, error, value, methods;
1586
1587         softc = (struct ada_softc *)arg1;
1588
1589         value = softc->delete_method;
1590         if (value < 0 || value > ADA_DELETE_MAX)
1591                 p = "UNKNOWN";
1592         else
1593                 p = ada_delete_method_names[value];
1594         strncpy(buf, p, sizeof(buf));
1595         error = sysctl_handle_string(oidp, buf, sizeof(buf), req);
1596         if (error != 0 || req->newptr == NULL)
1597                 return (error);
1598         methods = 1 << ADA_DELETE_DISABLE;
1599         if ((softc->flags & ADA_FLAG_CAN_CFA) &&
1600             !(softc->flags & ADA_FLAG_CAN_48BIT))
1601                 methods |= 1 << ADA_DELETE_CFA_ERASE;
1602         if (softc->flags & ADA_FLAG_CAN_TRIM)
1603                 methods |= 1 << ADA_DELETE_DSM_TRIM;
1604         if (softc->flags & ADA_FLAG_CAN_NCQ_TRIM)
1605                 methods |= 1 << ADA_DELETE_NCQ_DSM_TRIM;
1606         for (i = 0; i <= ADA_DELETE_MAX; i++) {
1607                 if (!(methods & (1 << i)) ||
1608                     strcmp(buf, ada_delete_method_names[i]) != 0)
1609                         continue;
1610                 softc->delete_method = i;
1611                 return (0);
1612         }
1613         return (EINVAL);
1614 }
1615
1616 static void
1617 adasetflags(struct ada_softc *softc, struct ccb_getdev *cgd)
1618 {
1619         if ((cgd->ident_data.capabilities1 & ATA_SUPPORT_DMA) &&
1620             (cgd->inq_flags & SID_DMA))
1621                 softc->flags |= ADA_FLAG_CAN_DMA;
1622         else
1623                 softc->flags &= ~ADA_FLAG_CAN_DMA;
1624
1625         if (cgd->ident_data.support.command2 & ATA_SUPPORT_ADDRESS48) {
1626                 softc->flags |= ADA_FLAG_CAN_48BIT;
1627                 if (cgd->inq_flags & SID_DMA48)
1628                         softc->flags |= ADA_FLAG_CAN_DMA48;
1629                 else
1630                         softc->flags &= ~ADA_FLAG_CAN_DMA48;
1631         } else
1632                 softc->flags &= ~(ADA_FLAG_CAN_48BIT | ADA_FLAG_CAN_DMA48);
1633
1634         if (cgd->ident_data.support.command2 & ATA_SUPPORT_FLUSHCACHE)
1635                 softc->flags |= ADA_FLAG_CAN_FLUSHCACHE;
1636         else
1637                 softc->flags &= ~ADA_FLAG_CAN_FLUSHCACHE;
1638
1639         if (cgd->ident_data.support.command1 & ATA_SUPPORT_POWERMGT)
1640                 softc->flags |= ADA_FLAG_CAN_POWERMGT;
1641         else
1642                 softc->flags &= ~ADA_FLAG_CAN_POWERMGT;
1643
1644         if ((cgd->ident_data.satacapabilities & ATA_SUPPORT_NCQ) &&
1645             (cgd->inq_flags & SID_DMA) && (cgd->inq_flags & SID_CmdQue))
1646                 softc->flags |= ADA_FLAG_CAN_NCQ;
1647         else
1648                 softc->flags &= ~ADA_FLAG_CAN_NCQ;
1649
1650         if ((cgd->ident_data.support_dsm & ATA_SUPPORT_DSM_TRIM) &&
1651             (cgd->inq_flags & SID_DMA) &&
1652             (softc->quirks & ADA_Q_NO_TRIM) == 0) {
1653                 softc->flags |= ADA_FLAG_CAN_TRIM;
1654                 softc->trim_max_ranges = TRIM_MAX_RANGES;
1655                 if (cgd->ident_data.max_dsm_blocks != 0) {
1656                         softc->trim_max_ranges =
1657                             min(cgd->ident_data.max_dsm_blocks *
1658                                 ATA_DSM_BLK_RANGES, softc->trim_max_ranges);
1659                 }
1660                 /*
1661                  * If we can do RCVSND_FPDMA_QUEUED commands, we may be able
1662                  * to do NCQ trims, if we support trims at all. We also need
1663                  * support from the SIM to do things properly. Perhaps we
1664                  * should look at log 13 dword 0 bit 0 and dword 1 bit 0 are
1665                  * set too...
1666                  */
1667                 if ((softc->quirks & ADA_Q_NCQ_TRIM_BROKEN) == 0 &&
1668                     (softc->flags & ADA_FLAG_PIM_ATA_EXT) != 0 &&
1669                     (cgd->ident_data.satacapabilities2 &
1670                      ATA_SUPPORT_RCVSND_FPDMA_QUEUED) != 0 &&
1671                     (softc->flags & ADA_FLAG_CAN_TRIM) != 0)
1672                         softc->flags |= ADA_FLAG_CAN_NCQ_TRIM;
1673                 else
1674                         softc->flags &= ~ADA_FLAG_CAN_NCQ_TRIM;
1675         } else
1676                 softc->flags &= ~(ADA_FLAG_CAN_TRIM | ADA_FLAG_CAN_NCQ_TRIM);
1677
1678         if (cgd->ident_data.support.command2 & ATA_SUPPORT_CFA)
1679                 softc->flags |= ADA_FLAG_CAN_CFA;
1680         else
1681                 softc->flags &= ~ADA_FLAG_CAN_CFA;
1682
1683         /*
1684          * Now that we've set the appropriate flags, setup the delete
1685          * method.
1686          */
1687         adasetdeletemethod(softc);
1688
1689         if ((cgd->ident_data.support.extension & ATA_SUPPORT_GENLOG)
1690          && ((softc->quirks & ADA_Q_LOG_BROKEN) == 0))
1691                 softc->flags |= ADA_FLAG_CAN_LOG;
1692         else
1693                 softc->flags &= ~ADA_FLAG_CAN_LOG;
1694
1695         if ((cgd->ident_data.support3 & ATA_SUPPORT_ZONE_MASK) ==
1696              ATA_SUPPORT_ZONE_HOST_AWARE)
1697                 softc->zone_mode = ADA_ZONE_HOST_AWARE;
1698         else if (((cgd->ident_data.support3 & ATA_SUPPORT_ZONE_MASK) ==
1699                    ATA_SUPPORT_ZONE_DEV_MANAGED)
1700               || (softc->quirks & ADA_Q_SMR_DM))
1701                 softc->zone_mode = ADA_ZONE_DRIVE_MANAGED;
1702         else
1703                 softc->zone_mode = ADA_ZONE_NONE;
1704
1705         if (cgd->ident_data.support.command1 & ATA_SUPPORT_LOOKAHEAD)
1706                 softc->flags |= ADA_FLAG_CAN_RAHEAD;
1707         else
1708                 softc->flags &= ~ADA_FLAG_CAN_RAHEAD;
1709
1710         if (cgd->ident_data.support.command1 & ATA_SUPPORT_WRITECACHE)
1711                 softc->flags |= ADA_FLAG_CAN_WCACHE;
1712         else
1713                 softc->flags &= ~ADA_FLAG_CAN_WCACHE;
1714 }
1715
1716 static cam_status
1717 adaregister(struct cam_periph *periph, void *arg)
1718 {
1719         struct ada_softc *softc;
1720         struct ccb_getdev *cgd;
1721         struct disk_params *dp;
1722         struct sbuf sb;
1723         char   *announce_buf;
1724         caddr_t match;
1725         int quirks;
1726
1727         cgd = (struct ccb_getdev *)arg;
1728         if (cgd == NULL) {
1729                 printf("adaregister: no getdev CCB, can't register device\n");
1730                 return(CAM_REQ_CMP_ERR);
1731         }
1732
1733         softc = (struct ada_softc *)malloc(sizeof(*softc), M_DEVBUF,
1734             M_NOWAIT|M_ZERO);
1735
1736         if (softc == NULL) {
1737                 printf("adaregister: Unable to probe new device. "
1738                     "Unable to allocate softc\n");
1739                 return(CAM_REQ_CMP_ERR);
1740         }
1741
1742         announce_buf = softc->announce_temp;
1743         bzero(announce_buf, ADA_ANNOUNCETMP_SZ);
1744
1745         if (cam_iosched_init(&softc->cam_iosched, periph) != 0) {
1746                 printf("adaregister: Unable to probe new device. "
1747                        "Unable to allocate iosched memory\n");
1748                 free(softc, M_DEVBUF);
1749                 return(CAM_REQ_CMP_ERR);
1750         }
1751
1752         periph->softc = softc;
1753         xpt_path_inq(&softc->cpi, periph->path);
1754
1755         /*
1756          * See if this device has any quirks.
1757          */
1758         match = cam_quirkmatch((caddr_t)&cgd->ident_data,
1759                                (caddr_t)ada_quirk_table,
1760                                nitems(ada_quirk_table),
1761                                sizeof(*ada_quirk_table), ata_identify_match);
1762         if (match != NULL)
1763                 softc->quirks = ((struct ada_quirk_entry *)match)->quirks;
1764         else
1765                 softc->quirks = ADA_Q_NONE;
1766
1767         TASK_INIT(&softc->sysctl_task, 0, adasysctlinit, periph);
1768
1769         /*
1770          * Register this media as a disk
1771          */
1772         (void)cam_periph_hold(periph, PRIBIO);
1773         cam_periph_unlock(periph);
1774         snprintf(announce_buf, ADA_ANNOUNCETMP_SZ,
1775             "kern.cam.ada.%d.quirks", periph->unit_number);
1776         quirks = softc->quirks;
1777         TUNABLE_INT_FETCH(announce_buf, &quirks);
1778         softc->quirks = quirks;
1779         softc->read_ahead = -1;
1780         snprintf(announce_buf, ADA_ANNOUNCETMP_SZ,
1781             "kern.cam.ada.%d.read_ahead", periph->unit_number);
1782         TUNABLE_INT_FETCH(announce_buf, &softc->read_ahead);
1783         softc->write_cache = -1;
1784         snprintf(announce_buf, ADA_ANNOUNCETMP_SZ,
1785             "kern.cam.ada.%d.write_cache", periph->unit_number);
1786         TUNABLE_INT_FETCH(announce_buf, &softc->write_cache);
1787
1788         /*
1789          * Set support flags based on the Identify data and quirks.
1790          */
1791         adasetflags(softc, cgd);
1792         if (softc->cpi.hba_misc & PIM_ATA_EXT)
1793                 softc->flags |= ADA_FLAG_PIM_ATA_EXT;
1794
1795         /* Disable queue sorting for non-rotational media by default. */
1796         if (cgd->ident_data.media_rotation_rate == ATA_RATE_NON_ROTATING) {
1797                 softc->rotating = 0;
1798         } else {
1799                 softc->rotating = 1;
1800         }
1801         cam_iosched_set_sort_queue(softc->cam_iosched,  softc->rotating ? -1 : 0);
1802         softc->disk = disk_alloc();
1803         adasetgeom(softc, cgd);
1804         softc->disk->d_devstat = devstat_new_entry(periph->periph_name,
1805                           periph->unit_number, softc->params.secsize,
1806                           DEVSTAT_ALL_SUPPORTED,
1807                           DEVSTAT_TYPE_DIRECT |
1808                           XPORT_DEVSTAT_TYPE(softc->cpi.transport),
1809                           DEVSTAT_PRIORITY_DISK);
1810         softc->disk->d_open = adaopen;
1811         softc->disk->d_close = adaclose;
1812         softc->disk->d_strategy = adastrategy;
1813         softc->disk->d_getattr = adagetattr;
1814         softc->disk->d_dump = adadump;
1815         softc->disk->d_gone = adadiskgonecb;
1816         softc->disk->d_name = "ada";
1817         softc->disk->d_drv1 = periph;
1818         softc->disk->d_unit = periph->unit_number;
1819
1820         /*
1821          * Acquire a reference to the periph before we register with GEOM.
1822          * We'll release this reference once GEOM calls us back (via
1823          * adadiskgonecb()) telling us that our provider has been freed.
1824          */
1825         if (cam_periph_acquire(periph) != 0) {
1826                 xpt_print(periph->path, "%s: lost periph during "
1827                           "registration!\n", __func__);
1828                 cam_periph_lock(periph);
1829                 return (CAM_REQ_CMP_ERR);
1830         }
1831         disk_create(softc->disk, DISK_VERSION);
1832         cam_periph_lock(periph);
1833
1834         dp = &softc->params;
1835         snprintf(announce_buf, ADA_ANNOUNCETMP_SZ,
1836             "%juMB (%ju %u byte sectors)",
1837             ((uintmax_t)dp->secsize * dp->sectors) / (1024 * 1024),
1838             (uintmax_t)dp->sectors, dp->secsize);
1839
1840         sbuf_new(&sb, softc->announce_buffer, ADA_ANNOUNCE_SZ, SBUF_FIXEDLEN);
1841         xpt_announce_periph_sbuf(periph, &sb, announce_buf);
1842         xpt_announce_quirks_sbuf(periph, &sb, softc->quirks, ADA_Q_BIT_STRING);
1843         sbuf_finish(&sb);
1844         sbuf_putbuf(&sb);
1845
1846         /*
1847          * Create our sysctl variables, now that we know
1848          * we have successfully attached.
1849          */
1850         if (cam_periph_acquire(periph) == 0)
1851                 taskqueue_enqueue(taskqueue_thread, &softc->sysctl_task);
1852
1853         /*
1854          * Add async callbacks for bus reset and
1855          * bus device reset calls.  I don't bother
1856          * checking if this fails as, in most cases,
1857          * the system will function just fine without
1858          * them and the only alternative would be to
1859          * not attach the device on failure.
1860          */
1861         xpt_register_async(AC_SENT_BDR | AC_BUS_RESET | AC_LOST_DEVICE |
1862             AC_GETDEV_CHANGED | AC_ADVINFO_CHANGED,
1863             adaasync, periph, periph->path);
1864
1865         /*
1866          * Schedule a periodic event to occasionally send an
1867          * ordered tag to a device.
1868          */
1869         callout_init_mtx(&softc->sendordered_c, cam_periph_mtx(periph), 0);
1870         callout_reset(&softc->sendordered_c,
1871             (ada_default_timeout * hz) / ADA_ORDEREDTAG_INTERVAL,
1872             adasendorderedtag, softc);
1873
1874         if (ADA_RA >= 0 && softc->flags & ADA_FLAG_CAN_RAHEAD) {
1875                 softc->state = ADA_STATE_RAHEAD;
1876         } else if (ADA_WC >= 0 && softc->flags & ADA_FLAG_CAN_WCACHE) {
1877                 softc->state = ADA_STATE_WCACHE;
1878         } else if ((softc->flags & ADA_FLAG_CAN_LOG)
1879                 && (softc->zone_mode != ADA_ZONE_NONE)) {
1880                 softc->state = ADA_STATE_LOGDIR;
1881         } else {
1882                 /*
1883                  * Nothing to probe, so we can just transition to the
1884                  * normal state.
1885                  */
1886                 adaprobedone(periph, NULL);
1887                 return(CAM_REQ_CMP);
1888         }
1889
1890         xpt_schedule(periph, CAM_PRIORITY_DEV);
1891
1892         return(CAM_REQ_CMP);
1893 }
1894
1895 static int
1896 ada_dsmtrim_req_create(struct ada_softc *softc, struct bio *bp, struct trim_request *req)
1897 {
1898         uint64_t lastlba = (uint64_t)-1, lbas = 0;
1899         int c, lastcount = 0, off, ranges = 0;
1900
1901         bzero(req, sizeof(*req));
1902         TAILQ_INIT(&req->bps);
1903         do {
1904                 uint64_t lba = bp->bio_pblkno;
1905                 int count = bp->bio_bcount / softc->params.secsize;
1906
1907                 /* Try to extend the previous range. */
1908                 if (lba == lastlba) {
1909                         c = min(count, ATA_DSM_RANGE_MAX - lastcount);
1910                         lastcount += c;
1911                         off = (ranges - 1) * ATA_DSM_RANGE_SIZE;
1912                         req->data[off + 6] = lastcount & 0xff;
1913                         req->data[off + 7] =
1914                                 (lastcount >> 8) & 0xff;
1915                         count -= c;
1916                         lba += c;
1917                         lbas += c;
1918                 }
1919
1920                 while (count > 0) {
1921                         c = min(count, ATA_DSM_RANGE_MAX);
1922                         off = ranges * ATA_DSM_RANGE_SIZE;
1923                         req->data[off + 0] = lba & 0xff;
1924                         req->data[off + 1] = (lba >> 8) & 0xff;
1925                         req->data[off + 2] = (lba >> 16) & 0xff;
1926                         req->data[off + 3] = (lba >> 24) & 0xff;
1927                         req->data[off + 4] = (lba >> 32) & 0xff;
1928                         req->data[off + 5] = (lba >> 40) & 0xff;
1929                         req->data[off + 6] = c & 0xff;
1930                         req->data[off + 7] = (c >> 8) & 0xff;
1931                         lba += c;
1932                         lbas += c;
1933                         count -= c;
1934                         lastcount = c;
1935                         ranges++;
1936                         /*
1937                          * Its the caller's responsibility to ensure the
1938                          * request will fit so we don't need to check for
1939                          * overrun here
1940                          */
1941                 }
1942                 lastlba = lba;
1943                 TAILQ_INSERT_TAIL(&req->bps, bp, bio_queue);
1944
1945                 bp = cam_iosched_next_trim(softc->cam_iosched);
1946                 if (bp == NULL)
1947                         break;
1948                 if (bp->bio_bcount / softc->params.secsize >
1949                     (softc->trim_max_ranges - ranges) * ATA_DSM_RANGE_MAX) {
1950                         cam_iosched_put_back_trim(softc->cam_iosched, bp);
1951                         break;
1952                 }
1953         } while (1);
1954         softc->trim_count++;
1955         softc->trim_ranges += ranges;
1956         softc->trim_lbas += lbas;
1957
1958         return (ranges);
1959 }
1960
1961 static void
1962 ada_dsmtrim(struct ada_softc *softc, struct bio *bp, struct ccb_ataio *ataio)
1963 {
1964         struct trim_request *req = &softc->trim_req;
1965         int ranges;
1966
1967         ranges = ada_dsmtrim_req_create(softc, bp, req);
1968         cam_fill_ataio(ataio,
1969             ada_retry_count,
1970             adadone,
1971             CAM_DIR_OUT,
1972             0,
1973             req->data,
1974             howmany(ranges, ATA_DSM_BLK_RANGES) * ATA_DSM_BLK_SIZE,
1975             ada_default_timeout * 1000);
1976         ata_48bit_cmd(ataio, ATA_DATA_SET_MANAGEMENT,
1977             ATA_DSM_TRIM, 0, howmany(ranges, ATA_DSM_BLK_RANGES));
1978 }
1979
1980 static void
1981 ada_ncq_dsmtrim(struct ada_softc *softc, struct bio *bp, struct ccb_ataio *ataio)
1982 {
1983         struct trim_request *req = &softc->trim_req;
1984         int ranges;
1985
1986         ranges = ada_dsmtrim_req_create(softc, bp, req);
1987         cam_fill_ataio(ataio,
1988             ada_retry_count,
1989             adadone,
1990             CAM_DIR_OUT,
1991             0,
1992             req->data,
1993             howmany(ranges, ATA_DSM_BLK_RANGES) * ATA_DSM_BLK_SIZE,
1994             ada_default_timeout * 1000);
1995         ata_ncq_cmd(ataio,
1996             ATA_SEND_FPDMA_QUEUED,
1997             0,
1998             howmany(ranges, ATA_DSM_BLK_RANGES));
1999         ataio->cmd.sector_count_exp = ATA_SFPDMA_DSM;
2000         ataio->ata_flags |= ATA_FLAG_AUX;
2001         ataio->aux = 1;
2002 }
2003
2004 static void
2005 ada_cfaerase(struct ada_softc *softc, struct bio *bp, struct ccb_ataio *ataio)
2006 {
2007         struct trim_request *req = &softc->trim_req;
2008         uint64_t lba = bp->bio_pblkno;
2009         uint16_t count = bp->bio_bcount / softc->params.secsize;
2010
2011         bzero(req, sizeof(*req));
2012         TAILQ_INIT(&req->bps);
2013         TAILQ_INSERT_TAIL(&req->bps, bp, bio_queue);
2014
2015         cam_fill_ataio(ataio,
2016             ada_retry_count,
2017             adadone,
2018             CAM_DIR_NONE,
2019             0,
2020             NULL,
2021             0,
2022             ada_default_timeout*1000);
2023
2024         if (count >= 256)
2025                 count = 0;
2026         ata_28bit_cmd(ataio, ATA_CFA_ERASE, 0, lba, count);
2027 }
2028
2029 static int
2030 ada_zone_bio_to_ata(int disk_zone_cmd)
2031 {
2032         switch (disk_zone_cmd) {
2033         case DISK_ZONE_OPEN:
2034                 return ATA_ZM_OPEN_ZONE;
2035         case DISK_ZONE_CLOSE:
2036                 return ATA_ZM_CLOSE_ZONE;
2037         case DISK_ZONE_FINISH:
2038                 return ATA_ZM_FINISH_ZONE;
2039         case DISK_ZONE_RWP:
2040                 return ATA_ZM_RWP;
2041         }
2042
2043         return -1;
2044 }
2045
2046 static int
2047 ada_zone_cmd(struct cam_periph *periph, union ccb *ccb, struct bio *bp,
2048              int *queue_ccb)
2049 {
2050         struct ada_softc *softc;
2051         int error;
2052
2053         error = 0;
2054
2055         if (bp->bio_cmd != BIO_ZONE) {
2056                 error = EINVAL;
2057                 goto bailout;
2058         }
2059
2060         softc = periph->softc;
2061
2062         switch (bp->bio_zone.zone_cmd) {
2063         case DISK_ZONE_OPEN:
2064         case DISK_ZONE_CLOSE:
2065         case DISK_ZONE_FINISH:
2066         case DISK_ZONE_RWP: {
2067                 int zone_flags;
2068                 int zone_sa;
2069                 uint64_t lba;
2070
2071                 zone_sa = ada_zone_bio_to_ata(bp->bio_zone.zone_cmd);
2072                 if (zone_sa == -1) {
2073                         xpt_print(periph->path, "Cannot translate zone "
2074                             "cmd %#x to ATA\n", bp->bio_zone.zone_cmd);
2075                         error = EINVAL;
2076                         goto bailout;
2077                 }
2078
2079                 zone_flags = 0;
2080                 lba = bp->bio_zone.zone_params.rwp.id;
2081
2082                 if (bp->bio_zone.zone_params.rwp.flags &
2083                     DISK_ZONE_RWP_FLAG_ALL)
2084                         zone_flags |= ZBC_OUT_ALL;
2085
2086                 ata_zac_mgmt_out(&ccb->ataio,
2087                                  /*retries*/ ada_retry_count,
2088                                  /*cbfcnp*/ adadone,
2089                                  /*use_ncq*/ (softc->flags &
2090                                               ADA_FLAG_PIM_ATA_EXT) ? 1 : 0,
2091                                  /*zm_action*/ zone_sa,
2092                                  /*zone_id*/ lba,
2093                                  /*zone_flags*/ zone_flags,
2094                                  /*sector_count*/ 0,
2095                                  /*data_ptr*/ NULL,
2096                                  /*dxfer_len*/ 0,
2097                                  /*timeout*/ ada_default_timeout * 1000);
2098                 *queue_ccb = 1;
2099
2100                 break;
2101         }
2102         case DISK_ZONE_REPORT_ZONES: {
2103                 uint8_t *rz_ptr;
2104                 uint32_t num_entries, alloc_size;
2105                 struct disk_zone_report *rep;
2106
2107                 rep = &bp->bio_zone.zone_params.report;
2108
2109                 num_entries = rep->entries_allocated;
2110                 if (num_entries == 0) {
2111                         xpt_print(periph->path, "No entries allocated for "
2112                             "Report Zones request\n");
2113                         error = EINVAL;
2114                         goto bailout;
2115                 }
2116                 alloc_size = sizeof(struct scsi_report_zones_hdr) +
2117                     (sizeof(struct scsi_report_zones_desc) * num_entries);
2118                 alloc_size = min(alloc_size, softc->disk->d_maxsize);
2119                 rz_ptr = malloc(alloc_size, M_ATADA, M_NOWAIT | M_ZERO);
2120                 if (rz_ptr == NULL) {
2121                         xpt_print(periph->path, "Unable to allocate memory "
2122                            "for Report Zones request\n");
2123                         error = ENOMEM;
2124                         goto bailout;
2125                 }
2126
2127                 ata_zac_mgmt_in(&ccb->ataio,
2128                                 /*retries*/ ada_retry_count,
2129                                 /*cbcfnp*/ adadone,
2130                                 /*use_ncq*/ (softc->flags &
2131                                              ADA_FLAG_PIM_ATA_EXT) ? 1 : 0,
2132                                 /*zm_action*/ ATA_ZM_REPORT_ZONES,
2133                                 /*zone_id*/ rep->starting_id,
2134                                 /*zone_flags*/ rep->rep_options,
2135                                 /*data_ptr*/ rz_ptr,
2136                                 /*dxfer_len*/ alloc_size,
2137                                 /*timeout*/ ada_default_timeout * 1000);
2138
2139                 /*
2140                  * For BIO_ZONE, this isn't normally needed.  However, it
2141                  * is used by devstat_end_transaction_bio() to determine
2142                  * how much data was transferred.
2143                  */
2144                 /*
2145                  * XXX KDM we have a problem.  But I'm not sure how to fix
2146                  * it.  devstat uses bio_bcount - bio_resid to calculate
2147                  * the amount of data transferred.   The GEOM disk code
2148                  * uses bio_length - bio_resid to calculate the amount of
2149                  * data in bio_completed.  We have different structure
2150                  * sizes above and below the ada(4) driver.  So, if we
2151                  * use the sizes above, the amount transferred won't be
2152                  * quite accurate for devstat.  If we use different sizes
2153                  * for bio_bcount and bio_length (above and below
2154                  * respectively), then the residual needs to match one or
2155                  * the other.  Everything is calculated after the bio
2156                  * leaves the driver, so changing the values around isn't
2157                  * really an option.  For now, just set the count to the
2158                  * passed in length.  This means that the calculations
2159                  * above (e.g. bio_completed) will be correct, but the
2160                  * amount of data reported to devstat will be slightly
2161                  * under or overstated.
2162                  */
2163                 bp->bio_bcount = bp->bio_length;
2164
2165                 *queue_ccb = 1;
2166
2167                 break;
2168         }
2169         case DISK_ZONE_GET_PARAMS: {
2170                 struct disk_zone_disk_params *params;
2171
2172                 params = &bp->bio_zone.zone_params.disk_params;
2173                 bzero(params, sizeof(*params));
2174
2175                 switch (softc->zone_mode) {
2176                 case ADA_ZONE_DRIVE_MANAGED:
2177                         params->zone_mode = DISK_ZONE_MODE_DRIVE_MANAGED;
2178                         break;
2179                 case ADA_ZONE_HOST_AWARE:
2180                         params->zone_mode = DISK_ZONE_MODE_HOST_AWARE;
2181                         break;
2182                 case ADA_ZONE_HOST_MANAGED:
2183                         params->zone_mode = DISK_ZONE_MODE_HOST_MANAGED;
2184                         break;
2185                 default:
2186                 case ADA_ZONE_NONE:
2187                         params->zone_mode = DISK_ZONE_MODE_NONE;
2188                         break;
2189                 }
2190
2191                 if (softc->zone_flags & ADA_ZONE_FLAG_URSWRZ)
2192                         params->flags |= DISK_ZONE_DISK_URSWRZ;
2193
2194                 if (softc->zone_flags & ADA_ZONE_FLAG_OPT_SEQ_SET) {
2195                         params->optimal_seq_zones = softc->optimal_seq_zones;
2196                         params->flags |= DISK_ZONE_OPT_SEQ_SET;
2197                 }
2198
2199                 if (softc->zone_flags & ADA_ZONE_FLAG_OPT_NONSEQ_SET) {
2200                         params->optimal_nonseq_zones =
2201                             softc->optimal_nonseq_zones;
2202                         params->flags |= DISK_ZONE_OPT_NONSEQ_SET;
2203                 }
2204
2205                 if (softc->zone_flags & ADA_ZONE_FLAG_MAX_SEQ_SET) {
2206                         params->max_seq_zones = softc->max_seq_zones;
2207                         params->flags |= DISK_ZONE_MAX_SEQ_SET;
2208                 }
2209                 if (softc->zone_flags & ADA_ZONE_FLAG_RZ_SUP)
2210                         params->flags |= DISK_ZONE_RZ_SUP;
2211
2212                 if (softc->zone_flags & ADA_ZONE_FLAG_OPEN_SUP)
2213                         params->flags |= DISK_ZONE_OPEN_SUP;
2214
2215                 if (softc->zone_flags & ADA_ZONE_FLAG_CLOSE_SUP)
2216                         params->flags |= DISK_ZONE_CLOSE_SUP;
2217
2218                 if (softc->zone_flags & ADA_ZONE_FLAG_FINISH_SUP)
2219                         params->flags |= DISK_ZONE_FINISH_SUP;
2220
2221                 if (softc->zone_flags & ADA_ZONE_FLAG_RWP_SUP)
2222                         params->flags |= DISK_ZONE_RWP_SUP;
2223                 break;
2224         }
2225         default:
2226                 break;
2227         }
2228 bailout:
2229         return (error);
2230 }
2231
2232 static void
2233 adastart(struct cam_periph *periph, union ccb *start_ccb)
2234 {
2235         struct ada_softc *softc = (struct ada_softc *)periph->softc;
2236         struct ccb_ataio *ataio = &start_ccb->ataio;
2237
2238         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("adastart\n"));
2239
2240         switch (softc->state) {
2241         case ADA_STATE_NORMAL:
2242         {
2243                 struct bio *bp;
2244                 u_int8_t tag_code;
2245
2246                 bp = cam_iosched_next_bio(softc->cam_iosched);
2247                 if (bp == NULL) {
2248                         xpt_release_ccb(start_ccb);
2249                         break;
2250                 }
2251
2252                 if ((bp->bio_flags & BIO_ORDERED) != 0 ||
2253                     (bp->bio_cmd != BIO_DELETE && (softc->flags & ADA_FLAG_NEED_OTAG) != 0)) {
2254                         softc->flags &= ~ADA_FLAG_NEED_OTAG;
2255                         softc->flags |= ADA_FLAG_WAS_OTAG;
2256                         tag_code = 0;
2257                 } else {
2258                         tag_code = 1;
2259                 }
2260                 switch (bp->bio_cmd) {
2261                 case BIO_WRITE:
2262                 case BIO_READ:
2263                 {
2264                         uint64_t lba = bp->bio_pblkno;
2265                         uint16_t count = bp->bio_bcount / softc->params.secsize;
2266                         void *data_ptr;
2267                         int rw_op;
2268
2269                         if (bp->bio_cmd == BIO_WRITE) {
2270                                 softc->flags |= ADA_FLAG_DIRTY;
2271                                 rw_op = CAM_DIR_OUT;
2272                         } else {
2273                                 rw_op = CAM_DIR_IN;
2274                         }
2275
2276                         data_ptr = bp->bio_data;
2277                         if ((bp->bio_flags & (BIO_UNMAPPED|BIO_VLIST)) != 0) {
2278                                 rw_op |= CAM_DATA_BIO;
2279                                 data_ptr = bp;
2280                         }
2281
2282 #ifdef CAM_TEST_FAILURE
2283                         int fail = 0;
2284
2285                         /*
2286                          * Support the failure ioctls.  If the command is a
2287                          * read, and there are pending forced read errors, or
2288                          * if a write and pending write errors, then fail this
2289                          * operation with EIO.  This is useful for testing
2290                          * purposes.  Also, support having every Nth read fail.
2291                          *
2292                          * This is a rather blunt tool.
2293                          */
2294                         if (bp->bio_cmd == BIO_READ) {
2295                                 if (softc->force_read_error) {
2296                                         softc->force_read_error--;
2297                                         fail = 1;
2298                                 }
2299                                 if (softc->periodic_read_error > 0) {
2300                                         if (++softc->periodic_read_count >=
2301                                             softc->periodic_read_error) {
2302                                                 softc->periodic_read_count = 0;
2303                                                 fail = 1;
2304                                         }
2305                                 }
2306                         } else {
2307                                 if (softc->force_write_error) {
2308                                         softc->force_write_error--;
2309                                         fail = 1;
2310                                 }
2311                         }
2312                         if (fail) {
2313                                 biofinish(bp, NULL, EIO);
2314                                 xpt_release_ccb(start_ccb);
2315                                 adaschedule(periph);
2316                                 return;
2317                         }
2318 #endif
2319                         KASSERT((bp->bio_flags & BIO_UNMAPPED) == 0 ||
2320                             round_page(bp->bio_bcount + bp->bio_ma_offset) /
2321                             PAGE_SIZE == bp->bio_ma_n,
2322                             ("Short bio %p", bp));
2323                         cam_fill_ataio(ataio,
2324                             ada_retry_count,
2325                             adadone,
2326                             rw_op,
2327                             0,
2328                             data_ptr,
2329                             bp->bio_bcount,
2330                             ada_default_timeout*1000);
2331
2332                         if ((softc->flags & ADA_FLAG_CAN_NCQ) && tag_code) {
2333                                 if (bp->bio_cmd == BIO_READ) {
2334                                         ata_ncq_cmd(ataio, ATA_READ_FPDMA_QUEUED,
2335                                             lba, count);
2336                                 } else {
2337                                         ata_ncq_cmd(ataio, ATA_WRITE_FPDMA_QUEUED,
2338                                             lba, count);
2339                                 }
2340                         } else if ((softc->flags & ADA_FLAG_CAN_48BIT) &&
2341                             (lba + count >= ATA_MAX_28BIT_LBA ||
2342                             count > 256)) {
2343                                 if (softc->flags & ADA_FLAG_CAN_DMA48) {
2344                                         if (bp->bio_cmd == BIO_READ) {
2345                                                 ata_48bit_cmd(ataio, ATA_READ_DMA48,
2346                                                     0, lba, count);
2347                                         } else {
2348                                                 ata_48bit_cmd(ataio, ATA_WRITE_DMA48,
2349                                                     0, lba, count);
2350                                         }
2351                                 } else {
2352                                         if (bp->bio_cmd == BIO_READ) {
2353                                                 ata_48bit_cmd(ataio, ATA_READ_MUL48,
2354                                                     0, lba, count);
2355                                         } else {
2356                                                 ata_48bit_cmd(ataio, ATA_WRITE_MUL48,
2357                                                     0, lba, count);
2358                                         }
2359                                 }
2360                         } else {
2361                                 if (count == 256)
2362                                         count = 0;
2363                                 if (softc->flags & ADA_FLAG_CAN_DMA) {
2364                                         if (bp->bio_cmd == BIO_READ) {
2365                                                 ata_28bit_cmd(ataio, ATA_READ_DMA,
2366                                                     0, lba, count);
2367                                         } else {
2368                                                 ata_28bit_cmd(ataio, ATA_WRITE_DMA,
2369                                                     0, lba, count);
2370                                         }
2371                                 } else {
2372                                         if (bp->bio_cmd == BIO_READ) {
2373                                                 ata_28bit_cmd(ataio, ATA_READ_MUL,
2374                                                     0, lba, count);
2375                                         } else {
2376                                                 ata_28bit_cmd(ataio, ATA_WRITE_MUL,
2377                                                     0, lba, count);
2378                                         }
2379                                 }
2380                         }
2381                         break;
2382                 }
2383                 case BIO_DELETE:
2384                         switch (softc->delete_method) {
2385                         case ADA_DELETE_NCQ_DSM_TRIM:
2386                                 ada_ncq_dsmtrim(softc, bp, ataio);
2387                                 break;
2388                         case ADA_DELETE_DSM_TRIM:
2389                                 ada_dsmtrim(softc, bp, ataio);
2390                                 break;
2391                         case ADA_DELETE_CFA_ERASE:
2392                                 ada_cfaerase(softc, bp, ataio);
2393                                 break;
2394                         default:
2395                                 biofinish(bp, NULL, EOPNOTSUPP);
2396                                 xpt_release_ccb(start_ccb);
2397                                 adaschedule(periph);
2398                                 return;
2399                         }
2400                         start_ccb->ccb_h.ccb_state = ADA_CCB_TRIM;
2401                         start_ccb->ccb_h.flags |= CAM_UNLOCKED;
2402                         cam_iosched_submit_trim(softc->cam_iosched);
2403                         goto out;
2404                 case BIO_FLUSH:
2405                         cam_fill_ataio(ataio,
2406                             1,
2407                             adadone,
2408                             CAM_DIR_NONE,
2409                             0,
2410                             NULL,
2411                             0,
2412                             ada_default_timeout*1000);
2413
2414                         if (softc->flags & ADA_FLAG_CAN_48BIT)
2415                                 ata_48bit_cmd(ataio, ATA_FLUSHCACHE48, 0, 0, 0);
2416                         else
2417                                 ata_28bit_cmd(ataio, ATA_FLUSHCACHE, 0, 0, 0);
2418                         break;
2419                 case BIO_ZONE: {
2420                         int error, queue_ccb;
2421
2422                         queue_ccb = 0;
2423
2424                         error = ada_zone_cmd(periph, start_ccb, bp, &queue_ccb);
2425                         if ((error != 0)
2426                          || (queue_ccb == 0)) {
2427                                 biofinish(bp, NULL, error);
2428                                 xpt_release_ccb(start_ccb);
2429                                 return;
2430                         }
2431                         break;
2432                 }
2433                 }
2434                 start_ccb->ccb_h.ccb_state = ADA_CCB_BUFFER_IO;
2435                 start_ccb->ccb_h.flags |= CAM_UNLOCKED;
2436 out:
2437                 start_ccb->ccb_h.ccb_bp = bp;
2438                 softc->outstanding_cmds++;
2439                 softc->refcount++;
2440                 cam_periph_unlock(periph);
2441                 xpt_action(start_ccb);
2442                 cam_periph_lock(periph);
2443
2444                 /* May have more work to do, so ensure we stay scheduled */
2445                 adaschedule(periph);
2446                 break;
2447         }
2448         case ADA_STATE_RAHEAD:
2449         case ADA_STATE_WCACHE:
2450         {
2451                 cam_fill_ataio(ataio,
2452                     1,
2453                     adadone,
2454                     CAM_DIR_NONE,
2455                     0,
2456                     NULL,
2457                     0,
2458                     ada_default_timeout*1000);
2459
2460                 if (softc->state == ADA_STATE_RAHEAD) {
2461                         ata_28bit_cmd(ataio, ATA_SETFEATURES, ADA_RA ?
2462                             ATA_SF_ENAB_RCACHE : ATA_SF_DIS_RCACHE, 0, 0);
2463                         start_ccb->ccb_h.ccb_state = ADA_CCB_RAHEAD;
2464                 } else {
2465                         ata_28bit_cmd(ataio, ATA_SETFEATURES, ADA_WC ?
2466                             ATA_SF_ENAB_WCACHE : ATA_SF_DIS_WCACHE, 0, 0);
2467                         start_ccb->ccb_h.ccb_state = ADA_CCB_WCACHE;
2468                 }
2469                 start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
2470                 xpt_action(start_ccb);
2471                 break;
2472         }
2473         case ADA_STATE_LOGDIR:
2474         {
2475                 struct ata_gp_log_dir *log_dir;
2476
2477                 if ((softc->flags & ADA_FLAG_CAN_LOG) == 0) {
2478                         adaprobedone(periph, start_ccb);
2479                         break;
2480                 }
2481
2482                 log_dir = malloc(sizeof(*log_dir), M_ATADA, M_NOWAIT|M_ZERO);
2483                 if (log_dir == NULL) {
2484                         xpt_print(periph->path, "Couldn't malloc log_dir "
2485                             "data\n");
2486                         softc->state = ADA_STATE_NORMAL;
2487                         xpt_release_ccb(start_ccb);
2488                         break;
2489                 }
2490
2491
2492                 ata_read_log(ataio,
2493                     /*retries*/1,
2494                     /*cbfcnp*/adadone,
2495                     /*log_address*/ ATA_LOG_DIRECTORY,
2496                     /*page_number*/ 0,
2497                     /*block_count*/ 1,
2498                     /*protocol*/ softc->flags & ADA_FLAG_CAN_DMA ?
2499                                  CAM_ATAIO_DMA : 0,
2500                     /*data_ptr*/ (uint8_t *)log_dir,
2501                     /*dxfer_len*/sizeof(*log_dir),
2502                     /*timeout*/ada_default_timeout*1000);
2503
2504                 start_ccb->ccb_h.ccb_state = ADA_CCB_LOGDIR;
2505                 xpt_action(start_ccb);
2506                 break;
2507         }
2508         case ADA_STATE_IDDIR:
2509         {
2510                 struct ata_identify_log_pages *id_dir;
2511
2512                 id_dir = malloc(sizeof(*id_dir), M_ATADA, M_NOWAIT | M_ZERO);
2513                 if (id_dir == NULL) {
2514                         xpt_print(periph->path, "Couldn't malloc id_dir "
2515                             "data\n");
2516                         adaprobedone(periph, start_ccb);
2517                         break;
2518                 }
2519
2520                 ata_read_log(ataio,
2521                     /*retries*/1,
2522                     /*cbfcnp*/adadone,
2523                     /*log_address*/ ATA_IDENTIFY_DATA_LOG,
2524                     /*page_number*/ ATA_IDL_PAGE_LIST,
2525                     /*block_count*/ 1,
2526                     /*protocol*/ softc->flags & ADA_FLAG_CAN_DMA ?
2527                                  CAM_ATAIO_DMA : 0,
2528                     /*data_ptr*/ (uint8_t *)id_dir,
2529                     /*dxfer_len*/ sizeof(*id_dir),
2530                     /*timeout*/ada_default_timeout*1000);
2531
2532                 start_ccb->ccb_h.ccb_state = ADA_CCB_IDDIR;
2533                 xpt_action(start_ccb);
2534                 break;
2535         }
2536         case ADA_STATE_SUP_CAP:
2537         {
2538                 struct ata_identify_log_sup_cap *sup_cap;
2539
2540                 sup_cap = malloc(sizeof(*sup_cap), M_ATADA, M_NOWAIT|M_ZERO);
2541                 if (sup_cap == NULL) {
2542                         xpt_print(periph->path, "Couldn't malloc sup_cap "
2543                             "data\n");
2544                         adaprobedone(periph, start_ccb);
2545                         break;
2546                 }
2547
2548                 ata_read_log(ataio,
2549                     /*retries*/1,
2550                     /*cbfcnp*/adadone,
2551                     /*log_address*/ ATA_IDENTIFY_DATA_LOG,
2552                     /*page_number*/ ATA_IDL_SUP_CAP,
2553                     /*block_count*/ 1,
2554                     /*protocol*/ softc->flags & ADA_FLAG_CAN_DMA ?
2555                                  CAM_ATAIO_DMA : 0,
2556                     /*data_ptr*/ (uint8_t *)sup_cap,
2557                     /*dxfer_len*/ sizeof(*sup_cap),
2558                     /*timeout*/ada_default_timeout*1000);
2559
2560                 start_ccb->ccb_h.ccb_state = ADA_CCB_SUP_CAP;
2561                 xpt_action(start_ccb);
2562                 break;
2563         }
2564         case ADA_STATE_ZONE:
2565         {
2566                 struct ata_zoned_info_log *ata_zone;
2567
2568                 ata_zone = malloc(sizeof(*ata_zone), M_ATADA, M_NOWAIT|M_ZERO);
2569                 if (ata_zone == NULL) {
2570                         xpt_print(periph->path, "Couldn't malloc ata_zone "
2571                             "data\n");
2572                         adaprobedone(periph, start_ccb);
2573                         break;
2574                 }
2575
2576                 ata_read_log(ataio,
2577                     /*retries*/1,
2578                     /*cbfcnp*/adadone,
2579                     /*log_address*/ ATA_IDENTIFY_DATA_LOG,
2580                     /*page_number*/ ATA_IDL_ZDI,
2581                     /*block_count*/ 1,
2582                     /*protocol*/ softc->flags & ADA_FLAG_CAN_DMA ?
2583                                  CAM_ATAIO_DMA : 0,
2584                     /*data_ptr*/ (uint8_t *)ata_zone,
2585                     /*dxfer_len*/ sizeof(*ata_zone),
2586                     /*timeout*/ada_default_timeout*1000);
2587
2588                 start_ccb->ccb_h.ccb_state = ADA_CCB_ZONE;
2589                 xpt_action(start_ccb);
2590                 break;
2591         }
2592         }
2593 }
2594
2595 static void
2596 adaprobedone(struct cam_periph *periph, union ccb *ccb)
2597 {
2598         struct ada_softc *softc;
2599
2600         softc = (struct ada_softc *)periph->softc;
2601
2602         if (ccb != NULL)
2603                 xpt_release_ccb(ccb);
2604
2605         softc->state = ADA_STATE_NORMAL;
2606         softc->flags |= ADA_FLAG_PROBED;
2607         adaschedule(periph);
2608         if ((softc->flags & ADA_FLAG_ANNOUNCED) == 0) {
2609                 softc->flags |= ADA_FLAG_ANNOUNCED;
2610                 cam_periph_unhold(periph);
2611         } else {
2612                 cam_periph_release_locked(periph);
2613         }
2614 }
2615
2616 static void
2617 adazonedone(struct cam_periph *periph, union ccb *ccb)
2618 {
2619         struct bio *bp;
2620
2621         bp = (struct bio *)ccb->ccb_h.ccb_bp;
2622
2623         switch (bp->bio_zone.zone_cmd) {
2624         case DISK_ZONE_OPEN:
2625         case DISK_ZONE_CLOSE:
2626         case DISK_ZONE_FINISH:
2627         case DISK_ZONE_RWP:
2628                 break;
2629         case DISK_ZONE_REPORT_ZONES: {
2630                 uint32_t avail_len;
2631                 struct disk_zone_report *rep;
2632                 struct scsi_report_zones_hdr *hdr;
2633                 struct scsi_report_zones_desc *desc;
2634                 struct disk_zone_rep_entry *entry;
2635                 uint32_t hdr_len, num_avail;
2636                 uint32_t num_to_fill, i;
2637
2638                 rep = &bp->bio_zone.zone_params.report;
2639                 avail_len = ccb->ataio.dxfer_len - ccb->ataio.resid;
2640                 /*
2641                  * Note that bio_resid isn't normally used for zone
2642                  * commands, but it is used by devstat_end_transaction_bio()
2643                  * to determine how much data was transferred.  Because
2644                  * the size of the SCSI/ATA data structures is different
2645                  * than the size of the BIO interface structures, the
2646                  * amount of data actually transferred from the drive will
2647                  * be different than the amount of data transferred to
2648                  * the user.
2649                  */
2650                 hdr = (struct scsi_report_zones_hdr *)ccb->ataio.data_ptr;
2651                 if (avail_len < sizeof(*hdr)) {
2652                         /*
2653                          * Is there a better error than EIO here?  We asked
2654                          * for at least the header, and we got less than
2655                          * that.
2656                          */
2657                         bp->bio_error = EIO;
2658                         bp->bio_flags |= BIO_ERROR;
2659                         bp->bio_resid = bp->bio_bcount;
2660                         break;
2661                 }
2662
2663                 hdr_len = le32dec(hdr->length);
2664                 if (hdr_len > 0)
2665                         rep->entries_available = hdr_len / sizeof(*desc);
2666                 else
2667                         rep->entries_available = 0;
2668                 /*
2669                  * NOTE: using the same values for the BIO version of the
2670                  * same field as the SCSI/ATA values.  This means we could
2671                  * get some additional values that aren't defined in bio.h
2672                  * if more values of the same field are defined later.
2673                  */
2674                 rep->header.same = hdr->byte4 & SRZ_SAME_MASK;
2675                 rep->header.maximum_lba = le64dec(hdr->maximum_lba);
2676                 /*
2677                  * If the drive reports no entries that match the query,
2678                  * we're done.
2679                  */
2680                 if (hdr_len == 0) {
2681                         rep->entries_filled = 0;
2682                         bp->bio_resid = bp->bio_bcount;
2683                         break;
2684                 }
2685
2686                 num_avail = min((avail_len - sizeof(*hdr)) / sizeof(*desc),
2687                                 hdr_len / sizeof(*desc));
2688                 /*
2689                  * If the drive didn't return any data, then we're done.
2690                  */
2691                 if (num_avail == 0) {
2692                         rep->entries_filled = 0;
2693                         bp->bio_resid = bp->bio_bcount;
2694                         break;
2695                 }
2696
2697                 num_to_fill = min(num_avail, rep->entries_allocated);
2698                 /*
2699                  * If the user didn't allocate any entries for us to fill,
2700                  * we're done.
2701                  */
2702                 if (num_to_fill == 0) {
2703                         rep->entries_filled = 0;
2704                         bp->bio_resid = bp->bio_bcount;
2705                         break;
2706                 }
2707
2708                 for (i = 0, desc = &hdr->desc_list[0], entry=&rep->entries[0];
2709                      i < num_to_fill; i++, desc++, entry++) {
2710                         /*
2711                          * NOTE: we're mapping the values here directly
2712                          * from the SCSI/ATA bit definitions to the bio.h
2713                          * definitions.  There is also a warning in
2714                          * disk_zone.h, but the impact is that if
2715                          * additional values are added in the SCSI/ATA
2716                          * specs these will be visible to consumers of
2717                          * this interface.
2718                          */
2719                         entry->zone_type = desc->zone_type & SRZ_TYPE_MASK;
2720                         entry->zone_condition =
2721                             (desc->zone_flags & SRZ_ZONE_COND_MASK) >>
2722                             SRZ_ZONE_COND_SHIFT;
2723                         entry->zone_flags |= desc->zone_flags &
2724                             (SRZ_ZONE_NON_SEQ|SRZ_ZONE_RESET);
2725                         entry->zone_length = le64dec(desc->zone_length);
2726                         entry->zone_start_lba = le64dec(desc->zone_start_lba);
2727                         entry->write_pointer_lba =
2728                             le64dec(desc->write_pointer_lba);
2729                 }
2730                 rep->entries_filled = num_to_fill;
2731                 /*
2732                  * Note that this residual is accurate from the user's
2733                  * standpoint, but the amount transferred isn't accurate
2734                  * from the standpoint of what actually came back from the
2735                  * drive.
2736                  */
2737                 bp->bio_resid = bp->bio_bcount - (num_to_fill * sizeof(*entry));
2738                 break;
2739         }
2740         case DISK_ZONE_GET_PARAMS:
2741         default:
2742                 /*
2743                  * In theory we should not get a GET_PARAMS bio, since it
2744                  * should be handled without queueing the command to the
2745                  * drive.
2746                  */
2747                 panic("%s: Invalid zone command %d", __func__,
2748                     bp->bio_zone.zone_cmd);
2749                 break;
2750         }
2751
2752         if (bp->bio_zone.zone_cmd == DISK_ZONE_REPORT_ZONES)
2753                 free(ccb->ataio.data_ptr, M_ATADA);
2754 }
2755
2756
2757 static void
2758 adadone(struct cam_periph *periph, union ccb *done_ccb)
2759 {
2760         struct ada_softc *softc;
2761         struct ccb_ataio *ataio;
2762         struct cam_path *path;
2763         uint32_t priority;
2764         int state;
2765
2766         softc = (struct ada_softc *)periph->softc;
2767         ataio = &done_ccb->ataio;
2768         path = done_ccb->ccb_h.path;
2769         priority = done_ccb->ccb_h.pinfo.priority;
2770
2771         CAM_DEBUG(path, CAM_DEBUG_TRACE, ("adadone\n"));
2772
2773         state = ataio->ccb_h.ccb_state & ADA_CCB_TYPE_MASK;
2774         switch (state) {
2775         case ADA_CCB_BUFFER_IO:
2776         case ADA_CCB_TRIM:
2777         {
2778                 struct bio *bp;
2779                 int error;
2780
2781                 cam_periph_lock(periph);
2782                 bp = (struct bio *)done_ccb->ccb_h.ccb_bp;
2783                 if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
2784                         error = adaerror(done_ccb, 0, 0);
2785                         if (error == ERESTART) {
2786                                 /* A retry was scheduled, so just return. */
2787                                 cam_periph_unlock(periph);
2788                                 return;
2789                         }
2790                         if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0)
2791                                 cam_release_devq(path,
2792                                                  /*relsim_flags*/0,
2793                                                  /*reduction*/0,
2794                                                  /*timeout*/0,
2795                                                  /*getcount_only*/0);
2796                         /*
2797                          * If we get an error on an NCQ DSM TRIM, fall back
2798                          * to a non-NCQ DSM TRIM forever. Please note that if
2799                          * CAN_NCQ_TRIM is set, CAN_TRIM is necessarily set too.
2800                          * However, for this one trim, we treat it as advisory
2801                          * and return success up the stack.
2802                          */
2803                         if (state == ADA_CCB_TRIM &&
2804                             error != 0 &&
2805                             (softc->flags & ADA_FLAG_CAN_NCQ_TRIM) != 0) {
2806                                 softc->flags &= ~ADA_FLAG_CAN_NCQ_TRIM;
2807                                 error = 0;
2808                                 adasetdeletemethod(softc);
2809                         }
2810                 } else {
2811                         if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0)
2812                                 panic("REQ_CMP with QFRZN");
2813
2814                         error = 0;
2815                 }
2816                 bp->bio_error = error;
2817                 if (error != 0) {
2818                         bp->bio_resid = bp->bio_bcount;
2819                         bp->bio_flags |= BIO_ERROR;
2820                 } else {
2821                         if (bp->bio_cmd == BIO_ZONE)
2822                                 adazonedone(periph, done_ccb);
2823                         else if (state == ADA_CCB_TRIM)
2824                                 bp->bio_resid = 0;
2825                         else
2826                                 bp->bio_resid = ataio->resid;
2827
2828                         if ((bp->bio_resid > 0)
2829                          && (bp->bio_cmd != BIO_ZONE))
2830                                 bp->bio_flags |= BIO_ERROR;
2831                 }
2832                 softc->outstanding_cmds--;
2833                 if (softc->outstanding_cmds == 0)
2834                         softc->flags |= ADA_FLAG_WAS_OTAG;
2835
2836                 /*
2837                  * We need to call cam_iosched before we call biodone so that we
2838                  * don't measure any activity that happens in the completion
2839                  * routine, which in the case of sendfile can be quite
2840                  * extensive.  Release the periph refcount taken in adastart()
2841                  * for each CCB.
2842                  */
2843                 cam_iosched_bio_complete(softc->cam_iosched, bp, done_ccb);
2844                 xpt_release_ccb(done_ccb);
2845                 KASSERT(softc->refcount >= 1, ("adadone softc %p refcount %d", softc, softc->refcount));
2846                 softc->refcount--;
2847                 if (state == ADA_CCB_TRIM) {
2848                         TAILQ_HEAD(, bio) queue;
2849                         struct bio *bp1;
2850
2851                         TAILQ_INIT(&queue);
2852                         TAILQ_CONCAT(&queue, &softc->trim_req.bps, bio_queue);
2853                         /*
2854                          * Normally, the xpt_release_ccb() above would make sure
2855                          * that when we have more work to do, that work would
2856                          * get kicked off. However, we specifically keep
2857                          * trim_running set to 0 before the call above to allow
2858                          * other I/O to progress when many BIO_DELETE requests
2859                          * are pushed down. We set trim_running to 0 and call
2860                          * daschedule again so that we don't stall if there are
2861                          * no other I/Os pending apart from BIO_DELETEs.
2862                          */
2863                         cam_iosched_trim_done(softc->cam_iosched);
2864                         adaschedule(periph);
2865                         cam_periph_unlock(periph);
2866                         while ((bp1 = TAILQ_FIRST(&queue)) != NULL) {
2867                                 TAILQ_REMOVE(&queue, bp1, bio_queue);
2868                                 bp1->bio_error = error;
2869                                 if (error != 0) {
2870                                         bp1->bio_flags |= BIO_ERROR;
2871                                         bp1->bio_resid = bp1->bio_bcount;
2872                                 } else
2873                                         bp1->bio_resid = 0;
2874                                 biodone(bp1);
2875                         }
2876                 } else {
2877                         adaschedule(periph);
2878                         cam_periph_unlock(periph);
2879                         biodone(bp);
2880                 }
2881                 return;
2882         }
2883         case ADA_CCB_RAHEAD:
2884         {
2885                 if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
2886                         if (adaerror(done_ccb, 0, 0) == ERESTART) {
2887                                 /* Drop freeze taken due to CAM_DEV_QFREEZE */
2888                                 cam_release_devq(path, 0, 0, 0, FALSE);
2889                                 return;
2890                         } else if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
2891                                 cam_release_devq(path,
2892                                     /*relsim_flags*/0,
2893                                     /*reduction*/0,
2894                                     /*timeout*/0,
2895                                     /*getcount_only*/0);
2896                         }
2897                 }
2898
2899                 /*
2900                  * Since our peripheral may be invalidated by an error
2901                  * above or an external event, we must release our CCB
2902                  * before releasing the reference on the peripheral.
2903                  * The peripheral will only go away once the last reference
2904                  * is removed, and we need it around for the CCB release
2905                  * operation.
2906                  */
2907
2908                 xpt_release_ccb(done_ccb);
2909                 softc->state = ADA_STATE_WCACHE;
2910                 xpt_schedule(periph, priority);
2911                 /* Drop freeze taken due to CAM_DEV_QFREEZE */
2912                 cam_release_devq(path, 0, 0, 0, FALSE);
2913                 return;
2914         }
2915         case ADA_CCB_WCACHE:
2916         {
2917                 if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
2918                         if (adaerror(done_ccb, 0, 0) == ERESTART) {
2919                                 /* Drop freeze taken due to CAM_DEV_QFREEZE */
2920                                 cam_release_devq(path, 0, 0, 0, FALSE);
2921                                 return;
2922                         } else if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
2923                                 cam_release_devq(path,
2924                                     /*relsim_flags*/0,
2925                                     /*reduction*/0,
2926                                     /*timeout*/0,
2927                                     /*getcount_only*/0);
2928                         }
2929                 }
2930
2931                 /* Drop freeze taken due to CAM_DEV_QFREEZE */
2932                 cam_release_devq(path, 0, 0, 0, FALSE);
2933
2934                 if ((softc->flags & ADA_FLAG_CAN_LOG)
2935                  && (softc->zone_mode != ADA_ZONE_NONE)) {
2936                         xpt_release_ccb(done_ccb);
2937                         softc->state = ADA_STATE_LOGDIR;
2938                         xpt_schedule(periph, priority);
2939                 } else {
2940                         adaprobedone(periph, done_ccb);
2941                 }
2942                 return;
2943         }
2944         case ADA_CCB_LOGDIR:
2945         {
2946                 int error;
2947
2948                 if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
2949                         error = 0;
2950                         softc->valid_logdir_len = 0;
2951                         bzero(&softc->ata_logdir, sizeof(softc->ata_logdir));
2952                         softc->valid_logdir_len =
2953                                 ataio->dxfer_len - ataio->resid;
2954                         if (softc->valid_logdir_len > 0)
2955                                 bcopy(ataio->data_ptr, &softc->ata_logdir,
2956                                     min(softc->valid_logdir_len,
2957                                         sizeof(softc->ata_logdir)));
2958                         /*
2959                          * Figure out whether the Identify Device log is
2960                          * supported.  The General Purpose log directory
2961                          * has a header, and lists the number of pages
2962                          * available for each GP log identified by the
2963                          * offset into the list.
2964                          */
2965                         if ((softc->valid_logdir_len >=
2966                             ((ATA_IDENTIFY_DATA_LOG + 1) * sizeof(uint16_t)))
2967                          && (le16dec(softc->ata_logdir.header) ==
2968                              ATA_GP_LOG_DIR_VERSION)
2969                          && (le16dec(&softc->ata_logdir.num_pages[
2970                              (ATA_IDENTIFY_DATA_LOG *
2971                              sizeof(uint16_t)) - sizeof(uint16_t)]) > 0)){
2972                                 softc->flags |= ADA_FLAG_CAN_IDLOG;
2973                         } else {
2974                                 softc->flags &= ~ADA_FLAG_CAN_IDLOG;
2975                         }
2976                 } else {
2977                         error = adaerror(done_ccb, CAM_RETRY_SELTO,
2978                                          SF_RETRY_UA|SF_NO_PRINT);
2979                         if (error == ERESTART)
2980                                 return;
2981                         else if (error != 0) {
2982                                 /*
2983                                  * If we can't get the ATA log directory,
2984                                  * then ATA logs are effectively not
2985                                  * supported even if the bit is set in the
2986                                  * identify data.
2987                                  */
2988                                 softc->flags &= ~(ADA_FLAG_CAN_LOG |
2989                                                   ADA_FLAG_CAN_IDLOG);
2990                                 if ((done_ccb->ccb_h.status &
2991                                      CAM_DEV_QFRZN) != 0) {
2992                                         /* Don't wedge this device's queue */
2993                                         cam_release_devq(done_ccb->ccb_h.path,
2994                                                          /*relsim_flags*/0,
2995                                                          /*reduction*/0,
2996                                                          /*timeout*/0,
2997                                                          /*getcount_only*/0);
2998                                 }
2999                         }
3000
3001
3002                 }
3003
3004                 free(ataio->data_ptr, M_ATADA);
3005
3006                 if ((error == 0)
3007                  && (softc->flags & ADA_FLAG_CAN_IDLOG)) {
3008                         softc->state = ADA_STATE_IDDIR;
3009                         xpt_release_ccb(done_ccb);
3010                         xpt_schedule(periph, priority);
3011                 } else
3012                         adaprobedone(periph, done_ccb);
3013
3014                 return;
3015         }
3016         case ADA_CCB_IDDIR: {
3017                 int error;
3018
3019                 if ((ataio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
3020                         off_t entries_offset, max_entries;
3021                         error = 0;
3022
3023                         softc->valid_iddir_len = 0;
3024                         bzero(&softc->ata_iddir, sizeof(softc->ata_iddir));
3025                         softc->flags &= ~(ADA_FLAG_CAN_SUPCAP |
3026                                           ADA_FLAG_CAN_ZONE);
3027                         softc->valid_iddir_len =
3028                                 ataio->dxfer_len - ataio->resid;
3029                         if (softc->valid_iddir_len > 0)
3030                                 bcopy(ataio->data_ptr, &softc->ata_iddir,
3031                                     min(softc->valid_iddir_len,
3032                                         sizeof(softc->ata_iddir)));
3033
3034                         entries_offset =
3035                             __offsetof(struct ata_identify_log_pages,entries);
3036                         max_entries = softc->valid_iddir_len - entries_offset;
3037                         if ((softc->valid_iddir_len > (entries_offset + 1))
3038                          && (le64dec(softc->ata_iddir.header) ==
3039                              ATA_IDLOG_REVISION)
3040                          && (softc->ata_iddir.entry_count > 0)) {
3041                                 int num_entries, i;
3042
3043                                 num_entries = softc->ata_iddir.entry_count;
3044                                 num_entries = min(num_entries,
3045                                    softc->valid_iddir_len - entries_offset);
3046                                 for (i = 0; i < num_entries &&
3047                                      i < max_entries; i++) {
3048                                         if (softc->ata_iddir.entries[i] ==
3049                                             ATA_IDL_SUP_CAP)
3050                                                 softc->flags |=
3051                                                     ADA_FLAG_CAN_SUPCAP;
3052                                         else if (softc->ata_iddir.entries[i]==
3053                                                  ATA_IDL_ZDI)
3054                                                 softc->flags |=
3055                                                     ADA_FLAG_CAN_ZONE;
3056
3057                                         if ((softc->flags &
3058                                              ADA_FLAG_CAN_SUPCAP)
3059                                          && (softc->flags &
3060                                              ADA_FLAG_CAN_ZONE))
3061                                                 break;
3062                                 }
3063                         }
3064                 } else {
3065                         error = adaerror(done_ccb, CAM_RETRY_SELTO,
3066                                          SF_RETRY_UA|SF_NO_PRINT);
3067                         if (error == ERESTART)
3068                                 return;
3069                         else if (error != 0) {
3070                                 /*
3071                                  * If we can't get the ATA Identify Data log
3072                                  * directory, then it effectively isn't
3073                                  * supported even if the ATA Log directory
3074                                  * a non-zero number of pages present for
3075                                  * this log.
3076                                  */
3077                                 softc->flags &= ~ADA_FLAG_CAN_IDLOG;
3078                                 if ((done_ccb->ccb_h.status &
3079                                      CAM_DEV_QFRZN) != 0) {
3080                                         /* Don't wedge this device's queue */
3081                                         cam_release_devq(done_ccb->ccb_h.path,
3082                                                          /*relsim_flags*/0,
3083                                                          /*reduction*/0,
3084                                                          /*timeout*/0,
3085                                                          /*getcount_only*/0);
3086                                 }
3087                         }
3088                 }
3089
3090                 free(ataio->data_ptr, M_ATADA);
3091
3092                 if ((error == 0)
3093                  && (softc->flags & ADA_FLAG_CAN_SUPCAP)) {
3094                         softc->state = ADA_STATE_SUP_CAP;
3095                         xpt_release_ccb(done_ccb);
3096                         xpt_schedule(periph, priority);
3097                 } else
3098                         adaprobedone(periph, done_ccb);
3099                 return;
3100         }
3101         case ADA_CCB_SUP_CAP: {
3102                 int error;
3103
3104                 if ((ataio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
3105                         uint32_t valid_len;
3106                         size_t needed_size;
3107                         struct ata_identify_log_sup_cap *sup_cap;
3108                         error = 0;
3109
3110                         sup_cap = (struct ata_identify_log_sup_cap *)
3111                             ataio->data_ptr;
3112                         valid_len = ataio->dxfer_len - ataio->resid;
3113                         needed_size =
3114                             __offsetof(struct ata_identify_log_sup_cap,
3115                             sup_zac_cap) + 1 + sizeof(sup_cap->sup_zac_cap);
3116                         if (valid_len >= needed_size) {
3117                                 uint64_t zoned, zac_cap;
3118
3119                                 zoned = le64dec(sup_cap->zoned_cap);
3120                                 if (zoned & ATA_ZONED_VALID) {
3121                                         /*
3122                                          * This should have already been
3123                                          * set, because this is also in the
3124                                          * ATA identify data.
3125                                          */
3126                                         if ((zoned & ATA_ZONED_MASK) ==
3127                                             ATA_SUPPORT_ZONE_HOST_AWARE)
3128                                                 softc->zone_mode =
3129                                                     ADA_ZONE_HOST_AWARE;
3130                                         else if ((zoned & ATA_ZONED_MASK) ==
3131                                             ATA_SUPPORT_ZONE_DEV_MANAGED)
3132                                                 softc->zone_mode =
3133                                                     ADA_ZONE_DRIVE_MANAGED;
3134                                 }
3135
3136                                 zac_cap = le64dec(sup_cap->sup_zac_cap);
3137                                 if (zac_cap & ATA_SUP_ZAC_CAP_VALID) {
3138                                         if (zac_cap & ATA_REPORT_ZONES_SUP)
3139                                                 softc->zone_flags |=
3140                                                     ADA_ZONE_FLAG_RZ_SUP;
3141                                         if (zac_cap & ATA_ND_OPEN_ZONE_SUP)
3142                                                 softc->zone_flags |=
3143                                                     ADA_ZONE_FLAG_OPEN_SUP;
3144                                         if (zac_cap & ATA_ND_CLOSE_ZONE_SUP)
3145                                                 softc->zone_flags |=
3146                                                     ADA_ZONE_FLAG_CLOSE_SUP;
3147                                         if (zac_cap & ATA_ND_FINISH_ZONE_SUP)
3148                                                 softc->zone_flags |=
3149                                                     ADA_ZONE_FLAG_FINISH_SUP;
3150                                         if (zac_cap & ATA_ND_RWP_SUP)
3151                                                 softc->zone_flags |=
3152                                                     ADA_ZONE_FLAG_RWP_SUP;
3153                                 } else {
3154                                         /*
3155                                          * This field was introduced in
3156                                          * ACS-4, r08 on April 28th, 2015.
3157                                          * If the drive firmware was written
3158                                          * to an earlier spec, it won't have
3159                                          * the field.  So, assume all
3160                                          * commands are supported.
3161                                          */
3162                                         softc->zone_flags |=
3163                                             ADA_ZONE_FLAG_SUP_MASK;
3164                                 }
3165                         }
3166                 } else {
3167                         error = adaerror(done_ccb, CAM_RETRY_SELTO,
3168                                          SF_RETRY_UA|SF_NO_PRINT);
3169                         if (error == ERESTART)
3170                                 return;
3171                         else if (error != 0) {
3172                                 /*
3173                                  * If we can't get the ATA Identify Data
3174                                  * Supported Capabilities page, clear the
3175                                  * flag...
3176                                  */
3177                                 softc->flags &= ~ADA_FLAG_CAN_SUPCAP;
3178                                 /*
3179                                  * And clear zone capabilities.
3180                                  */
3181                                 softc->zone_flags &= ~ADA_ZONE_FLAG_SUP_MASK;
3182                                 if ((done_ccb->ccb_h.status &
3183                                      CAM_DEV_QFRZN) != 0) {
3184                                         /* Don't wedge this device's queue */
3185                                         cam_release_devq(done_ccb->ccb_h.path,
3186                                                          /*relsim_flags*/0,
3187                                                          /*reduction*/0,
3188                                                          /*timeout*/0,
3189                                                          /*getcount_only*/0);
3190                                 }
3191                         }
3192                 }
3193
3194                 free(ataio->data_ptr, M_ATADA);
3195
3196                 if ((error == 0)
3197                  && (softc->flags & ADA_FLAG_CAN_ZONE)) {
3198                         softc->state = ADA_STATE_ZONE;
3199                         xpt_release_ccb(done_ccb);
3200                         xpt_schedule(periph, priority);
3201                 } else
3202                         adaprobedone(periph, done_ccb);
3203                 return;
3204         }
3205         case ADA_CCB_ZONE: {
3206                 int error;
3207
3208                 if ((ataio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
3209                         struct ata_zoned_info_log *zi_log;
3210                         uint32_t valid_len;
3211                         size_t needed_size;
3212
3213                         zi_log = (struct ata_zoned_info_log *)ataio->data_ptr;
3214
3215                         valid_len = ataio->dxfer_len - ataio->resid;
3216                         needed_size = __offsetof(struct ata_zoned_info_log,
3217                             version_info) + 1 + sizeof(zi_log->version_info);
3218                         if (valid_len >= needed_size) {
3219                                 uint64_t tmpvar;
3220
3221                                 tmpvar = le64dec(zi_log->zoned_cap);
3222                                 if (tmpvar & ATA_ZDI_CAP_VALID) {
3223                                         if (tmpvar & ATA_ZDI_CAP_URSWRZ)
3224                                                 softc->zone_flags |=
3225                                                     ADA_ZONE_FLAG_URSWRZ;
3226                                         else
3227                                                 softc->zone_flags &=
3228                                                     ~ADA_ZONE_FLAG_URSWRZ;
3229                                 }
3230                                 tmpvar = le64dec(zi_log->optimal_seq_zones);
3231                                 if (tmpvar & ATA_ZDI_OPT_SEQ_VALID) {
3232                                         softc->zone_flags |=
3233                                             ADA_ZONE_FLAG_OPT_SEQ_SET;
3234                                         softc->optimal_seq_zones = (tmpvar &
3235                                             ATA_ZDI_OPT_SEQ_MASK);
3236                                 } else {
3237                                         softc->zone_flags &=
3238                                             ~ADA_ZONE_FLAG_OPT_SEQ_SET;
3239                                         softc->optimal_seq_zones = 0;
3240                                 }
3241
3242                                 tmpvar =le64dec(zi_log->optimal_nonseq_zones);
3243                                 if (tmpvar & ATA_ZDI_OPT_NS_VALID) {
3244                                         softc->zone_flags |=
3245                                             ADA_ZONE_FLAG_OPT_NONSEQ_SET;
3246                                         softc->optimal_nonseq_zones =
3247                                             (tmpvar & ATA_ZDI_OPT_NS_MASK);
3248                                 } else {
3249                                         softc->zone_flags &=
3250                                             ~ADA_ZONE_FLAG_OPT_NONSEQ_SET;
3251                                         softc->optimal_nonseq_zones = 0;
3252                                 }
3253
3254                                 tmpvar = le64dec(zi_log->max_seq_req_zones);
3255                                 if (tmpvar & ATA_ZDI_MAX_SEQ_VALID) {
3256                                         softc->zone_flags |=
3257                                             ADA_ZONE_FLAG_MAX_SEQ_SET;
3258                                         softc->max_seq_zones =
3259                                             (tmpvar & ATA_ZDI_MAX_SEQ_MASK);
3260                                 } else {
3261                                         softc->zone_flags &=
3262                                             ~ADA_ZONE_FLAG_MAX_SEQ_SET;
3263                                         softc->max_seq_zones = 0;
3264                                 }
3265                         }
3266                 } else {
3267                         error = adaerror(done_ccb, CAM_RETRY_SELTO,
3268                                          SF_RETRY_UA|SF_NO_PRINT);
3269                         if (error == ERESTART)
3270                                 return;
3271                         else if (error != 0) {
3272                                 softc->flags &= ~ADA_FLAG_CAN_ZONE;
3273                                 softc->flags &= ~ADA_ZONE_FLAG_SET_MASK;
3274
3275                                 if ((done_ccb->ccb_h.status &
3276                                      CAM_DEV_QFRZN) != 0) {
3277                                         /* Don't wedge this device's queue */
3278                                         cam_release_devq(done_ccb->ccb_h.path,
3279                                                          /*relsim_flags*/0,
3280                                                          /*reduction*/0,
3281                                                          /*timeout*/0,
3282                                                          /*getcount_only*/0);
3283                                 }
3284                         }
3285                 }
3286                 free(ataio->data_ptr, M_ATADA);
3287
3288                 adaprobedone(periph, done_ccb);
3289                 return;
3290         }
3291         case ADA_CCB_DUMP:
3292                 /* No-op.  We're polling */
3293                 return;
3294         default:
3295                 break;
3296         }
3297         xpt_release_ccb(done_ccb);
3298 }
3299
3300 static int
3301 adaerror(union ccb *ccb, u_int32_t cam_flags, u_int32_t sense_flags)
3302 {
3303 #ifdef CAM_IO_STATS
3304         struct ada_softc *softc;
3305         struct cam_periph *periph;
3306
3307         periph = xpt_path_periph(ccb->ccb_h.path);
3308         softc = (struct ada_softc *)periph->softc;
3309
3310         switch (ccb->ccb_h.status & CAM_STATUS_MASK) {
3311         case CAM_CMD_TIMEOUT:
3312                 softc->timeouts++;
3313                 break;
3314         case CAM_REQ_ABORTED:
3315         case CAM_REQ_CMP_ERR:
3316         case CAM_REQ_TERMIO:
3317         case CAM_UNREC_HBA_ERROR:
3318         case CAM_DATA_RUN_ERR:
3319         case CAM_ATA_STATUS_ERROR:
3320                 softc->errors++;
3321                 break;
3322         default:
3323                 break;
3324         }
3325 #endif
3326
3327         return(cam_periph_error(ccb, cam_flags, sense_flags));
3328 }
3329
3330 static void
3331 adasetgeom(struct ada_softc *softc, struct ccb_getdev *cgd)
3332 {
3333         struct disk_params *dp = &softc->params;
3334         u_int64_t lbasize48;
3335         u_int32_t lbasize;
3336         u_int maxio, d_flags;
3337
3338         dp->secsize = ata_logical_sector_size(&cgd->ident_data);
3339         if ((cgd->ident_data.atavalid & ATA_FLAG_54_58) &&
3340             cgd->ident_data.current_heads != 0 &&
3341             cgd->ident_data.current_sectors != 0) {
3342                 dp->heads = cgd->ident_data.current_heads;
3343                 dp->secs_per_track = cgd->ident_data.current_sectors;
3344                 dp->cylinders = cgd->ident_data.cylinders;
3345                 dp->sectors = (u_int32_t)cgd->ident_data.current_size_1 |
3346                           ((u_int32_t)cgd->ident_data.current_size_2 << 16);
3347         } else {
3348                 dp->heads = cgd->ident_data.heads;
3349                 dp->secs_per_track = cgd->ident_data.sectors;
3350                 dp->cylinders = cgd->ident_data.cylinders;
3351                 dp->sectors = cgd->ident_data.cylinders *
3352                               (u_int32_t)(dp->heads * dp->secs_per_track);
3353         }
3354         lbasize = (u_int32_t)cgd->ident_data.lba_size_1 |
3355                   ((u_int32_t)cgd->ident_data.lba_size_2 << 16);
3356
3357         /* use the 28bit LBA size if valid or bigger than the CHS mapping */
3358         if (cgd->ident_data.cylinders == 16383 || dp->sectors < lbasize)
3359                 dp->sectors = lbasize;
3360
3361         /* use the 48bit LBA size if valid */
3362         lbasize48 = ((u_int64_t)cgd->ident_data.lba_size48_1) |
3363                     ((u_int64_t)cgd->ident_data.lba_size48_2 << 16) |
3364                     ((u_int64_t)cgd->ident_data.lba_size48_3 << 32) |
3365                     ((u_int64_t)cgd->ident_data.lba_size48_4 << 48);
3366         if ((cgd->ident_data.support.command2 & ATA_SUPPORT_ADDRESS48) &&
3367             lbasize48 > ATA_MAX_28BIT_LBA)
3368                 dp->sectors = lbasize48;
3369
3370         maxio = softc->cpi.maxio;               /* Honor max I/O size of SIM */
3371         if (maxio == 0)
3372                 maxio = DFLTPHYS;       /* traditional default */
3373         else if (maxio > MAXPHYS)
3374                 maxio = MAXPHYS;        /* for safety */
3375         if (softc->flags & ADA_FLAG_CAN_48BIT)
3376                 maxio = min(maxio, 65536 * softc->params.secsize);
3377         else                                    /* 28bit ATA command limit */
3378                 maxio = min(maxio, 256 * softc->params.secsize);
3379         if (softc->quirks & ADA_Q_128KB)
3380                 maxio = min(maxio, 128 * 1024);
3381         softc->disk->d_maxsize = maxio;
3382         d_flags = DISKFLAG_DIRECT_COMPLETION | DISKFLAG_CANZONE;
3383         if (softc->flags & ADA_FLAG_CAN_FLUSHCACHE)
3384                 d_flags |= DISKFLAG_CANFLUSHCACHE;
3385         if (softc->flags & ADA_FLAG_CAN_TRIM) {
3386                 d_flags |= DISKFLAG_CANDELETE;
3387                 softc->disk->d_delmaxsize = softc->params.secsize *
3388                     ATA_DSM_RANGE_MAX * softc->trim_max_ranges;
3389         } else if ((softc->flags & ADA_FLAG_CAN_CFA) &&
3390             !(softc->flags & ADA_FLAG_CAN_48BIT)) {
3391                 d_flags |= DISKFLAG_CANDELETE;
3392                 softc->disk->d_delmaxsize = 256 * softc->params.secsize;
3393         } else
3394                 softc->disk->d_delmaxsize = maxio;
3395         if ((softc->cpi.hba_misc & PIM_UNMAPPED) != 0) {
3396                 d_flags |= DISKFLAG_UNMAPPED_BIO;
3397                 softc->unmappedio = 1;
3398         }
3399         softc->disk->d_flags = d_flags;
3400         strlcpy(softc->disk->d_descr, cgd->ident_data.model,
3401             MIN(sizeof(softc->disk->d_descr), sizeof(cgd->ident_data.model)));
3402         strlcpy(softc->disk->d_ident, cgd->ident_data.serial,
3403             MIN(sizeof(softc->disk->d_ident), sizeof(cgd->ident_data.serial)));
3404
3405         softc->disk->d_sectorsize = softc->params.secsize;
3406         softc->disk->d_mediasize = (off_t)softc->params.sectors *
3407             softc->params.secsize;
3408         if (ata_physical_sector_size(&cgd->ident_data) !=
3409             softc->params.secsize) {
3410                 softc->disk->d_stripesize =
3411                     ata_physical_sector_size(&cgd->ident_data);
3412                 softc->disk->d_stripeoffset = (softc->disk->d_stripesize -
3413                     ata_logical_sector_offset(&cgd->ident_data)) %
3414                     softc->disk->d_stripesize;
3415         } else if (softc->quirks & ADA_Q_4K) {
3416                 softc->disk->d_stripesize = 4096;
3417                 softc->disk->d_stripeoffset = 0;
3418         }
3419         softc->disk->d_fwsectors = softc->params.secs_per_track;
3420         softc->disk->d_fwheads = softc->params.heads;
3421         ata_disk_firmware_geom_adjust(softc->disk);
3422         softc->disk->d_rotation_rate = cgd->ident_data.media_rotation_rate;
3423 }
3424
3425 static void
3426 adasendorderedtag(void *arg)
3427 {
3428         struct ada_softc *softc = arg;
3429
3430         if (ada_send_ordered) {
3431                 if (softc->outstanding_cmds > 0) {
3432                         if ((softc->flags & ADA_FLAG_WAS_OTAG) == 0)
3433                                 softc->flags |= ADA_FLAG_NEED_OTAG;
3434                         softc->flags &= ~ADA_FLAG_WAS_OTAG;
3435                 }
3436         }
3437         /* Queue us up again */
3438         callout_reset(&softc->sendordered_c,
3439             (ada_default_timeout * hz) / ADA_ORDEREDTAG_INTERVAL,
3440             adasendorderedtag, softc);
3441 }
3442
3443 /*
3444  * Step through all ADA peripheral drivers, and if the device is still open,
3445  * sync the disk cache to physical media.
3446  */
3447 static void
3448 adaflush(void)
3449 {
3450         struct cam_periph *periph;
3451         struct ada_softc *softc;
3452         union ccb *ccb;
3453         int error;
3454
3455         CAM_PERIPH_FOREACH(periph, &adadriver) {
3456                 softc = (struct ada_softc *)periph->softc;
3457                 if (SCHEDULER_STOPPED()) {
3458                         /* If we paniced with the lock held, do not recurse. */
3459                         if (!cam_periph_owned(periph) &&
3460                             (softc->flags & ADA_FLAG_OPEN)) {
3461                                 adadump(softc->disk, NULL, 0, 0, 0);
3462                         }
3463                         continue;
3464                 }
3465                 cam_periph_lock(periph);
3466                 /*
3467                  * We only sync the cache if the drive is still open, and
3468                  * if the drive is capable of it..
3469                  */
3470                 if (((softc->flags & ADA_FLAG_OPEN) == 0) ||
3471                     (softc->flags & ADA_FLAG_CAN_FLUSHCACHE) == 0) {
3472                         cam_periph_unlock(periph);
3473                         continue;
3474                 }
3475
3476                 ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL);
3477                 cam_fill_ataio(&ccb->ataio,
3478                                     0,
3479                                     NULL,
3480                                     CAM_DIR_NONE,
3481                                     0,
3482                                     NULL,
3483                                     0,
3484                                     ada_default_timeout*1000);
3485                 if (softc->flags & ADA_FLAG_CAN_48BIT)
3486                         ata_48bit_cmd(&ccb->ataio, ATA_FLUSHCACHE48, 0, 0, 0);
3487                 else
3488                         ata_28bit_cmd(&ccb->ataio, ATA_FLUSHCACHE, 0, 0, 0);
3489
3490                 error = cam_periph_runccb(ccb, adaerror, /*cam_flags*/0,
3491                     /*sense_flags*/ SF_NO_RECOVERY | SF_NO_RETRY,
3492                     softc->disk->d_devstat);
3493                 if (error != 0)
3494                         xpt_print(periph->path, "Synchronize cache failed\n");
3495                 xpt_release_ccb(ccb);
3496                 cam_periph_unlock(periph);
3497         }
3498 }
3499
3500 static void
3501 adaspindown(uint8_t cmd, int flags)
3502 {
3503         struct cam_periph *periph;
3504         struct ada_softc *softc;
3505         struct ccb_ataio local_ccb;
3506         int error;
3507
3508         CAM_PERIPH_FOREACH(periph, &adadriver) {
3509                 /* If we paniced with lock held - not recurse here. */
3510                 if (cam_periph_owned(periph))
3511                         continue;
3512                 cam_periph_lock(periph);
3513                 softc = (struct ada_softc *)periph->softc;
3514                 /*
3515                  * We only spin-down the drive if it is capable of it..
3516                  */
3517                 if ((softc->flags & ADA_FLAG_CAN_POWERMGT) == 0) {
3518                         cam_periph_unlock(periph);
3519                         continue;
3520                 }
3521
3522                 if (bootverbose)
3523                         xpt_print(periph->path, "spin-down\n");
3524
3525                 memset(&local_ccb, 0, sizeof(local_ccb));
3526                 xpt_setup_ccb(&local_ccb.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
3527                 local_ccb.ccb_h.ccb_state = ADA_CCB_DUMP;
3528
3529                 cam_fill_ataio(&local_ccb,
3530                                     0,
3531                                     NULL,
3532                                     CAM_DIR_NONE | flags,
3533                                     0,
3534                                     NULL,
3535                                     0,
3536                                     ada_default_timeout*1000);
3537                 ata_28bit_cmd(&local_ccb, cmd, 0, 0, 0);
3538                 error = cam_periph_runccb((union ccb *)&local_ccb, adaerror,
3539                     /*cam_flags*/0, /*sense_flags*/ SF_NO_RECOVERY | SF_NO_RETRY,
3540                     softc->disk->d_devstat);
3541                 if (error != 0)
3542                         xpt_print(periph->path, "Spin-down disk failed\n");
3543                 cam_periph_unlock(periph);
3544         }
3545 }
3546
3547 static void
3548 adashutdown(void *arg, int howto)
3549 {
3550         int how;
3551
3552         adaflush();
3553
3554         /*
3555          * STANDBY IMMEDIATE saves any volatile data to the drive. It also spins
3556          * down hard drives. IDLE IMMEDIATE also saves the volatile data without
3557          * a spindown. We send the former when we expect to lose power soon. For
3558          * a warm boot, we send the latter to avoid a thundering herd of spinups
3559          * just after the kernel loads while probing. We have to do something to
3560          * flush the data because the BIOS in many systems resets the HBA
3561          * causing a COMINIT/COMRESET negotiation, which some drives interpret
3562          * as license to toss the volatile data, and others count as unclean
3563          * shutdown when in the Active PM state in SMART attributes.
3564          *
3565          * adaspindown will ensure that we don't send this to a drive that
3566          * doesn't support it.
3567          */
3568         if (ada_spindown_shutdown != 0) {
3569                 how = (howto & (RB_HALT | RB_POWEROFF | RB_POWERCYCLE)) ?
3570                     ATA_STANDBY_IMMEDIATE : ATA_IDLE_IMMEDIATE;
3571                 adaspindown(how, 0);
3572         }
3573 }
3574
3575 static void
3576 adasuspend(void *arg)
3577 {
3578
3579         adaflush();
3580         /*
3581          * SLEEP also fushes any volatile data, like STANDBY IMEDIATE,
3582          * so we don't need to send it as well.
3583          */
3584         if (ada_spindown_suspend != 0)
3585                 adaspindown(ATA_SLEEP, CAM_DEV_QFREEZE);
3586 }
3587
3588 static void
3589 adaresume(void *arg)
3590 {
3591         struct cam_periph *periph;
3592         struct ada_softc *softc;
3593
3594         if (ada_spindown_suspend == 0)
3595                 return;
3596
3597         CAM_PERIPH_FOREACH(periph, &adadriver) {
3598                 cam_periph_lock(periph);
3599                 softc = (struct ada_softc *)periph->softc;
3600                 /*
3601                  * We only spin-down the drive if it is capable of it..
3602                  */
3603                 if ((softc->flags & ADA_FLAG_CAN_POWERMGT) == 0) {
3604                         cam_periph_unlock(periph);
3605                         continue;
3606                 }
3607
3608                 if (bootverbose)
3609                         xpt_print(periph->path, "resume\n");
3610
3611                 /*
3612                  * Drop freeze taken due to CAM_DEV_QFREEZE flag set on
3613                  * sleep request.
3614                  */
3615                 cam_release_devq(periph->path,
3616                          /*relsim_flags*/0,
3617                          /*openings*/0,
3618                          /*timeout*/0,
3619                          /*getcount_only*/0);
3620
3621                 cam_periph_unlock(periph);
3622         }
3623 }
3624
3625 #endif /* _KERNEL */